Revert shared.py to pre-category state, make arxiv_fetch.py fully self-contained

Goziee-git · Goziee-git · commit 9993111da2c1 · 2025-10-27T17:22:39.000+01:00
diff --git a/scripts/1-fetch/arxiv_fetch.py b/scripts/1-fetch/arxiv_fetch.py
@@ -472,7 +472,7 @@ def save_count_data(
         for lic, cats in category_counts.items():
             total_for_license = sum(cats.values()) or 1
             for code, c in cats.items():
-                label = shared.normalize_arxiv_category(code, CATEGORIES)
+                label = CATEGORIES.get(code, code)
                 pct = round((c / total_for_license) * 100, 2)
                 writer.writerow(
                     {
@@ -510,7 +510,7 @@ def save_count_data(
             others = sorted_cats[TOP_N:]
             other_count = sum(c for _, c in others)
             for code, c in top:
-                label = shared.normalize_arxiv_category(code, CATEGORIES)
+                label = CATEGORIES.get(code, code)
                 writer.writerow(
                     {
                         "TOOL_IDENTIFIER": lic,
diff --git a/scripts/shared.py b/scripts/shared.py
@@ -4,7 +4,6 @@
 from datetime import datetime, timezone
 
 # Third-party
-import yaml
 from git import InvalidGitRepositoryError, NoSuchPathError, Repo
 from pandas import PeriodIndex
 
@@ -162,223 +161,6 @@ def setup(current_file):
     return logger, paths
 
 
-def get_arxiv_categories():
-    """Get comprehensive ArXiv category taxonomy mapping."""
-    # ArXiv's comprehensive category mapping
-    categories = {
-        # Computer Science
-        "cs.AI": "Artificial Intelligence",
-        "cs.AR": "Hardware Architecture",
-        "cs.CC": "Computational Complexity",
-        "cs.CE": "Computational Engineering, Finance, and Science",
-        "cs.CG": "Computational Geometry",
-        "cs.CL": "Computation and Language",
-        "cs.CR": "Cryptography and Security",
-        "cs.CV": "Computer Vision and Pattern Recognition",
-        "cs.CY": "Computers and Society",
-        "cs.DB": "Databases",
-        "cs.DC": "Distributed, Parallel, and Cluster Computing",
-        "cs.DL": "Digital Libraries",
-        "cs.DM": "Discrete Mathematics",
-        "cs.DS": "Data Structures and Algorithms",
-        "cs.ET": "Emerging Technologies",
-        "cs.FL": "Formal Languages and Automata Theory",
-        "cs.GL": "General Literature",
-        "cs.GR": "Graphics",
-        "cs.GT": "Computer Science and Game Theory",
-        "cs.HC": "Human-Computer Interaction",
-        "cs.IR": "Information Retrieval",
-        "cs.IT": "Information Theory",
-        "cs.LG": "Machine Learning",
-        "cs.LO": "Logic in Computer Science",
-        "cs.MA": "Multiagent Systems",
-        "cs.MM": "Multimedia",
-        "cs.MS": "Mathematical Software",
-        "cs.NA": "Numerical Analysis",
-        "cs.NE": "Neural and Evolutionary Computing",
-        "cs.NI": "Networking and Internet Architecture",
-        "cs.OH": "Other Computer Science",
-        "cs.OS": "Operating Systems",
-        "cs.PF": "Performance",
-        "cs.PL": "Programming Languages",
-        "cs.RO": "Robotics",
-        "cs.SC": "Symbolic Computation",
-        "cs.SD": "Sound",
-        "cs.SE": "Software Engineering",
-        "cs.SI": "Social and Information Networks",
-        "cs.SY": "Systems and Control",
-        # Mathematics
-        "math.AC": "Commutative Algebra",
-        "math.AG": "Algebraic Geometry",
-        "math.AP": "Analysis of PDEs",
-        "math.AT": "Algebraic Topology",
-        "math.CA": "Classical Analysis and ODEs",
-        "math.CO": "Combinatorics",
-        "math.CT": "Category Theory",
-        "math.CV": "Complex Variables",
-        "math.DG": "Differential Geometry",
-        "math.DS": "Dynamical Systems",
-        "math.FA": "Functional Analysis",
-        "math.GM": "General Mathematics",
-        "math.GN": "General Topology",
-        "math.GR": "Group Theory",
-        "math.GT": "Geometric Topology",
-        "math.HO": "History and Overview",
-        "math.IT": "Information Theory",
-        "math.KT": "K-Theory and Homology",
-        "math.LO": "Logic",
-        "math.MG": "Metric Geometry",
-        "math.MP": "Mathematical Physics",
-        "math.NA": "Numerical Analysis",
-        "math.NT": "Number Theory",
-        "math.OA": "Operator Algebras",
-        "math.OC": "Optimization and Control",
-        "math.PR": "Probability",
-        "math.QA": "Quantum Algebra",
-        "math.RA": "Rings and Algebras",
-        "math.RT": "Representation Theory",
-        "math.SG": "Symplectic Geometry",
-        "math.SP": "Spectral Theory",
-        "math.ST": "Statistics Theory",
-        # Physics
-        "physics.acc-ph": "Accelerator Physics",
-        "physics.ao-ph": "Atmospheric and Oceanic Physics",
-        "physics.app-ph": "Applied Physics",
-        "physics.atm-clus": "Atomic and Molecular Clusters",
-        "physics.atom-ph": "Atomic Physics",
-        "physics.bio-ph": "Biological Physics",
-        "physics.chem-ph": "Chemical Physics",
-        "physics.class-ph": "Classical Physics",
-        "physics.comp-ph": "Computational Physics",
-        "physics.data-an": "Data Analysis, Statistics and Probability",
-        "physics.ed-ph": "Physics Education",
-        "physics.flu-dyn": "Fluid Dynamics",
-        "physics.gen-ph": "General Physics",
-        "physics.geo-ph": "Geophysics",
-        "physics.hist-ph": "History and Philosophy of Physics",
-        "physics.ins-det": "Instrumentation and Detectors",
-        "physics.med-ph": "Medical Physics",
-        "physics.optics": "Optics",
-        "physics.plasm-ph": "Plasma Physics",
-        "physics.pop-ph": "Popular Physics",
-        "physics.soc-ph": "Physics and Society",
-        "physics.space-ph": "Space Physics",
-        # Statistics
-        "stat.AP": "Applications",
-        "stat.CO": "Computation",
-        "stat.ME": "Methodology",
-        "stat.ML": "Machine Learning",
-        "stat.OT": "Other Statistics",
-        "stat.TH": "Statistics Theory",
-        # Quantitative Biology
-        "q-bio.BM": "Biomolecules",
-        "q-bio.CB": "Cell Behavior",
-        "q-bio.GN": "Genomics",
-        "q-bio.MN": "Molecular Networks",
-        "q-bio.NC": "Neurons and Cognition",
-        "q-bio.OT": "Other Quantitative Biology",
-        "q-bio.PE": "Populations and Evolution",
-        "q-bio.QM": "Quantitative Methods",
-        "q-bio.SC": "Subcellular Processes",
-        "q-bio.TO": "Tissues and Organs",
-        # Economics
-        "econ.EM": "Econometrics",
-        "econ.GN": "General Economics",
-        "econ.TH": "Theoretical Economics",
-        # Electrical Engineering
-        "eess.AS": "Audio and Speech Processing",
-        "eess.IV": "Image and Video Processing",
-        "eess.SP": "Signal Processing",
-        "eess.SY": "Systems and Control",
-        # High Energy Physics
-        "hep-ex": "High Energy Physics - Experiment",
-        "hep-lat": "High Energy Physics - Lattice",
-        "hep-ph": "High Energy Physics - Phenomenology",
-        "hep-th": "High Energy Physics - Theory",
-        # Other Physics
-        "astro-ph": "Astrophysics",
-        "astro-ph.CO": "Cosmology and Nongalactic Astrophysics",
-        "astro-ph.EP": "Earth and Planetary Astrophysics",
-        "astro-ph.GA": "Astrophysics of Galaxies",
-        "astro-ph.HE": "High Energy Astrophysical Phenomena",
-        "astro-ph.IM": "Instrumentation and Methods for Astrophysics",
-        "astro-ph.SR": "Solar and Stellar Astrophysics",
-        "cond-mat.dis-nn": "Disordered Systems and Neural Networks",
-        "cond-mat.mes-hall": "Mesoscale and Nanoscale Physics",
-        "cond-mat.mtrl-sci": "Materials Science",
-        "cond-mat.other": "Other Condensed Matter",
-        "cond-mat.quant-gas": "Quantum Gases",
-        "cond-mat.soft": "Soft Condensed Matter",
-        "cond-mat.stat-mech": "Statistical Mechanics",
-        "cond-mat.str-el": "Strongly Correlated Electrons",
-        "cond-mat.supr-con": "Superconductivity",
-        "gr-qc": "General Relativity and Quantum Cosmology",
-        "nlin.AO": "Adaptation and Self-Organizing Systems",
-        "nlin.CD": "Chaotic Dynamics",
-        "nlin.CG": "Cellular Automata and Lattice Gases",
-        "nlin.PS": "Pattern Formation and Solitons",
-        "nlin.SI": "Exactly Solvable and Integrable Systems",
-        "nucl-ex": "Nuclear Experiment",
-        "nucl-th": "Nuclear Theory",
-        "quant-ph": "Quantum Physics",
-    }
-
-    return categories
-
-
-def load_arxiv_categories(data_dir=None):
-    """Load ArXiv category mappings with fallback to comprehensive mapping."""
-    categories = {}
-
-    # Try loading from YAML file first
-    if data_dir:
-        yaml_path = os.path.join(data_dir, "arxiv_category_map.yaml")
-        if os.path.exists(yaml_path):
-            try:
-                with open(yaml_path, "r", encoding="utf-8") as f:
-                    categories = yaml.safe_load(f) or {}
-                logging.info(
-                    f"Loaded {len(categories)} categories from {yaml_path}"
-                )
-            except (yaml.YAMLError, IOError, OSError) as e:
-                logging.warning(f"Failed to load category YAML: {e}")
-
-    # Fallback to comprehensive mapping if no local categories
-    if not categories:
-        categories = get_arxiv_categories()
-        if categories and data_dir:
-            # Save fetched categories for future use
-            try:
-                os.makedirs(data_dir, exist_ok=True)
-                yaml_path = os.path.join(data_dir, "arxiv_category_map.yaml")
-                with open(yaml_path, "w", encoding="utf-8") as f:
-                    yaml.dump(
-                        categories, f, default_flow_style=False, sort_keys=True
-                    )
-                logging.info(
-                    f"Saved {len(categories)} categories to {yaml_path}"
-                )
-            except (yaml.YAMLError, IOError, OSError) as e:
-                logging.warning(f"Failed to save categories: {e}")
-
-    return categories
-
-
-def normalize_arxiv_category(code, categories=None):
-    """Convert category code to human-readable label."""
-    if not code or code == "Unknown":
-        return code
-
-    if categories and code in categories:
-        return categories[code]
-
-    # Fallback: use uppercase first part of code
-    if "." in code:
-        return code.split(".")[0].upper()
-
-    return code
-
-
 def update_readme(
     args,
     section_title,
