Add category converter in /dev called by arxiv_fetch.py to generate user-friendly names

Author: Goziee-git

dev/arxiv_category_converter.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""
+ArXiv category code to user-friendly name converter.
+Called by arxiv_fetch.py to convert category codes to readable names.
+"""
+import csv
+import os
+import yaml
+
+def load_category_mapping(data_dir):
+    """Load category code to label mapping from YAML file."""
+    mapping_file = os.path.join(data_dir, "arxiv_category_map.yaml")
+    
+    if not os.path.exists(mapping_file):
+        return {}
+    
+    try:
+        with open(mapping_file, 'r') as f:
+            return yaml.safe_load(f) or {}
+    except Exception:
+        return {}
+
+def convert_categories_to_friendly_names(input_file, output_file, data_dir):
+    """
+    Convert category codes in CSV to user-friendly names.
+    
+    Args:
+        input_file: Path to input CSV with category codes
+        output_file: Path to output CSV with friendly names
+        data_dir: Directory containing arxiv_category_map.yaml
+    """
+    if not os.path.exists(input_file):
+        return
+    
+    # Load category mapping
+    category_mapping = load_category_mapping(data_dir)
+    
+    with open(input_file, 'r') as infile, open(output_file, 'w', newline='') as outfile:
+        reader = csv.DictReader(infile)
+        
+        # Create new fieldnames with both code and label
+        fieldnames = []
+        for field in reader.fieldnames:
+            fieldnames.append(field)
+            if field == 'CATEGORY':
+                fieldnames.append('CATEGORY_LABEL')
+        
+        writer = csv.DictWriter(outfile, fieldnames=fieldnames, dialect='unix')
+        writer.writeheader()
+        
+        for row in reader:
+            if 'CATEGORY' in row:
+                category_code = row['CATEGORY']
+                # Convert code to label, fallback to uppercase first part if not found
+                category_label = category_mapping.get(
+                    category_code,
+                    category_code.split('.')[0].upper() if category_code and '.' in category_code else category_code
+                )
+                row['CATEGORY_LABEL'] = category_label
+            
+            writer.writerow(row)

scripts/1-fetch/arxiv_fetch.py

@@ -28,9 +28,12 @@
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+# Add dev directory for category converter
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "dev"))
 
 # First-party/Local
 import shared  # noqa: E402
+import arxiv_category_converter  # noqa: E402
 
 # Setup
 LOGGER, PATHS = shared.setup(__file__)
@@ -532,6 +535,17 @@ def query_arxiv(args):
         save_count_data(
             license_counts, category_counts, year_counts, author_counts
         )
+        
+        # Convert category codes to user-friendly names
+        try:
+            input_file = FILE_ARXIV_CATEGORY
+            output_file = shared.path_join(PATHS["data"], "arxiv_2_count_by_category_converted.csv")
+            arxiv_category_converter.convert_categories_to_friendly_names(
+                input_file, output_file, PATHS["data"]
+            )
+            LOGGER.info(f"Category conversion completed: {output_file}")
+        except Exception as e:
+            LOGGER.warning(f"Category conversion failed: {e}")
 
     # save provenance
     provenance_data = {