feat: centralize ArXiv category management in shared.py

Author: Goziee-git

scripts/shared.py

@@ -4,6 +4,7 @@
 from datetime import datetime, timezone
 
 # Third-party
+import yaml
 from git import InvalidGitRepositoryError, NoSuchPathError, Repo
 from pandas import PeriodIndex
 
@@ -161,6 +162,223 @@ def setup(current_file):
     return logger, paths
 
 
+def get_arxiv_categories():
+    """Get comprehensive ArXiv category taxonomy mapping."""
+    # ArXiv's comprehensive category mapping
+    categories = {
+        # Computer Science
+        "cs.AI": "Artificial Intelligence",
+        "cs.AR": "Hardware Architecture",
+        "cs.CC": "Computational Complexity",
+        "cs.CE": "Computational Engineering, Finance, and Science",
+        "cs.CG": "Computational Geometry",
+        "cs.CL": "Computation and Language",
+        "cs.CR": "Cryptography and Security",
+        "cs.CV": "Computer Vision and Pattern Recognition",
+        "cs.CY": "Computers and Society",
+        "cs.DB": "Databases",
+        "cs.DC": "Distributed, Parallel, and Cluster Computing",
+        "cs.DL": "Digital Libraries",
+        "cs.DM": "Discrete Mathematics",
+        "cs.DS": "Data Structures and Algorithms",
+        "cs.ET": "Emerging Technologies",
+        "cs.FL": "Formal Languages and Automata Theory",
+        "cs.GL": "General Literature",
+        "cs.GR": "Graphics",
+        "cs.GT": "Computer Science and Game Theory",
+        "cs.HC": "Human-Computer Interaction",
+        "cs.IR": "Information Retrieval",
+        "cs.IT": "Information Theory",
+        "cs.LG": "Machine Learning",
+        "cs.LO": "Logic in Computer Science",
+        "cs.MA": "Multiagent Systems",
+        "cs.MM": "Multimedia",
+        "cs.MS": "Mathematical Software",
+        "cs.NA": "Numerical Analysis",
+        "cs.NE": "Neural and Evolutionary Computing",
+        "cs.NI": "Networking and Internet Architecture",
+        "cs.OH": "Other Computer Science",
+        "cs.OS": "Operating Systems",
+        "cs.PF": "Performance",
+        "cs.PL": "Programming Languages",
+        "cs.RO": "Robotics",
+        "cs.SC": "Symbolic Computation",
+        "cs.SD": "Sound",
+        "cs.SE": "Software Engineering",
+        "cs.SI": "Social and Information Networks",
+        "cs.SY": "Systems and Control",
+        # Mathematics
+        "math.AC": "Commutative Algebra",
+        "math.AG": "Algebraic Geometry",
+        "math.AP": "Analysis of PDEs",
+        "math.AT": "Algebraic Topology",
+        "math.CA": "Classical Analysis and ODEs",
+        "math.CO": "Combinatorics",
+        "math.CT": "Category Theory",
+        "math.CV": "Complex Variables",
+        "math.DG": "Differential Geometry",
+        "math.DS": "Dynamical Systems",
+        "math.FA": "Functional Analysis",
+        "math.GM": "General Mathematics",
+        "math.GN": "General Topology",
+        "math.GR": "Group Theory",
+        "math.GT": "Geometric Topology",
+        "math.HO": "History and Overview",
+        "math.IT": "Information Theory",
+        "math.KT": "K-Theory and Homology",
+        "math.LO": "Logic",
+        "math.MG": "Metric Geometry",
+        "math.MP": "Mathematical Physics",
+        "math.NA": "Numerical Analysis",
+        "math.NT": "Number Theory",
+        "math.OA": "Operator Algebras",
+        "math.OC": "Optimization and Control",
+        "math.PR": "Probability",
+        "math.QA": "Quantum Algebra",
+        "math.RA": "Rings and Algebras",
+        "math.RT": "Representation Theory",
+        "math.SG": "Symplectic Geometry",
+        "math.SP": "Spectral Theory",
+        "math.ST": "Statistics Theory",
+        # Physics
+        "physics.acc-ph": "Accelerator Physics",
+        "physics.ao-ph": "Atmospheric and Oceanic Physics",
+        "physics.app-ph": "Applied Physics",
+        "physics.atm-clus": "Atomic and Molecular Clusters",
+        "physics.atom-ph": "Atomic Physics",
+        "physics.bio-ph": "Biological Physics",
+        "physics.chem-ph": "Chemical Physics",
+        "physics.class-ph": "Classical Physics",
+        "physics.comp-ph": "Computational Physics",
+        "physics.data-an": "Data Analysis, Statistics and Probability",
+        "physics.ed-ph": "Physics Education",
+        "physics.flu-dyn": "Fluid Dynamics",
+        "physics.gen-ph": "General Physics",
+        "physics.geo-ph": "Geophysics",
+        "physics.hist-ph": "History and Philosophy of Physics",
+        "physics.ins-det": "Instrumentation and Detectors",
+        "physics.med-ph": "Medical Physics",
+        "physics.optics": "Optics",
+        "physics.plasm-ph": "Plasma Physics",
+        "physics.pop-ph": "Popular Physics",
+        "physics.soc-ph": "Physics and Society",
+        "physics.space-ph": "Space Physics",
+        # Statistics
+        "stat.AP": "Applications",
+        "stat.CO": "Computation",
+        "stat.ME": "Methodology",
+        "stat.ML": "Machine Learning",
+        "stat.OT": "Other Statistics",
+        "stat.TH": "Statistics Theory",
+        # Quantitative Biology
+        "q-bio.BM": "Biomolecules",
+        "q-bio.CB": "Cell Behavior",
+        "q-bio.GN": "Genomics",
+        "q-bio.MN": "Molecular Networks",
+        "q-bio.NC": "Neurons and Cognition",
+        "q-bio.OT": "Other Quantitative Biology",
+        "q-bio.PE": "Populations and Evolution",
+        "q-bio.QM": "Quantitative Methods",
+        "q-bio.SC": "Subcellular Processes",
+        "q-bio.TO": "Tissues and Organs",
+        # Economics
+        "econ.EM": "Econometrics",
+        "econ.GN": "General Economics",
+        "econ.TH": "Theoretical Economics",
+        # Electrical Engineering
+        "eess.AS": "Audio and Speech Processing",
+        "eess.IV": "Image and Video Processing",
+        "eess.SP": "Signal Processing",
+        "eess.SY": "Systems and Control",
+        # High Energy Physics
+        "hep-ex": "High Energy Physics - Experiment",
+        "hep-lat": "High Energy Physics - Lattice",
+        "hep-ph": "High Energy Physics - Phenomenology",
+        "hep-th": "High Energy Physics - Theory",
+        # Other Physics
+        "astro-ph": "Astrophysics",
+        "astro-ph.CO": "Cosmology and Nongalactic Astrophysics",
+        "astro-ph.EP": "Earth and Planetary Astrophysics",
+        "astro-ph.GA": "Astrophysics of Galaxies",
+        "astro-ph.HE": "High Energy Astrophysical Phenomena",
+        "astro-ph.IM": "Instrumentation and Methods for Astrophysics",
+        "astro-ph.SR": "Solar and Stellar Astrophysics",
+        "cond-mat.dis-nn": "Disordered Systems and Neural Networks",
+        "cond-mat.mes-hall": "Mesoscale and Nanoscale Physics",
+        "cond-mat.mtrl-sci": "Materials Science",
+        "cond-mat.other": "Other Condensed Matter",
+        "cond-mat.quant-gas": "Quantum Gases",
+        "cond-mat.soft": "Soft Condensed Matter",
+        "cond-mat.stat-mech": "Statistical Mechanics",
+        "cond-mat.str-el": "Strongly Correlated Electrons",
+        "cond-mat.supr-con": "Superconductivity",
+        "gr-qc": "General Relativity and Quantum Cosmology",
+        "nlin.AO": "Adaptation and Self-Organizing Systems",
+        "nlin.CD": "Chaotic Dynamics",
+        "nlin.CG": "Cellular Automata and Lattice Gases",
+        "nlin.PS": "Pattern Formation and Solitons",
+        "nlin.SI": "Exactly Solvable and Integrable Systems",
+        "nucl-ex": "Nuclear Experiment",
+        "nucl-th": "Nuclear Theory",
+        "quant-ph": "Quantum Physics",
+    }
+
+    return categories
+
+
+def load_arxiv_categories(data_dir=None):
+    """Load ArXiv category mappings with fallback to comprehensive mapping."""
+    categories = {}
+
+    # Try loading from YAML file first
+    if data_dir:
+        yaml_path = os.path.join(data_dir, "arxiv_category_map.yaml")
+        if os.path.exists(yaml_path):
+            try:
+                with open(yaml_path, "r", encoding="utf-8") as f:
+                    categories = yaml.safe_load(f) or {}
+                logging.info(
+                    f"Loaded {len(categories)} categories from {yaml_path}"
+                )
+            except (yaml.YAMLError, IOError, OSError) as e:
+                logging.warning(f"Failed to load category YAML: {e}")
+
+    # Fallback to comprehensive mapping if no local categories
+    if not categories:
+        categories = get_arxiv_categories()
+        if categories and data_dir:
+            # Save fetched categories for future use
+            try:
+                os.makedirs(data_dir, exist_ok=True)
+                yaml_path = os.path.join(data_dir, "arxiv_category_map.yaml")
+                with open(yaml_path, "w", encoding="utf-8") as f:
+                    yaml.dump(
+                        categories, f, default_flow_style=False, sort_keys=True
+                    )
+                logging.info(
+                    f"Saved {len(categories)} categories to {yaml_path}"
+                )
+            except (yaml.YAMLError, IOError, OSError) as e:
+                logging.warning(f"Failed to save categories: {e}")
+
+    return categories
+
+
+def normalize_arxiv_category(code, categories=None):
+    """Convert category code to human-readable label."""
+    if not code or code == "Unknown":
+        return code
+
+    if categories and code in categories:
+        return categories[code]
+
+    # Fallback: use uppercase first part of code
+    if "." in code:
+        return code.split(".")[0].upper()
+
+    return code
+
+
 def update_readme(
     args,
     section_title,