Refactor arxiv_fetch.py: move CATEGORIES constant local, reorganize constants, remove redundancies

Author: Goziee-git

scripts/1-fetch/arxiv_fetch.py

@@ -41,11 +41,10 @@
 MAX_RESULTS_PER_QUERY = 500  # Maximum results to fetch per search query
 DEFAULT_FETCH_LIMIT = 800  # Default total papers to fetch
 
-# HTTP Retry Configuration (using shared constants where available)
+# HTTP Retry Configuration
 RETRY_TOTAL = 5
 RETRY_BACKOFF_FACTOR = 1
 
-
 # Search Queries
 SEARCH_QUERIES = [
     'all:"creative commons"',
@@ -66,23 +65,7 @@
     'all:"CC-0"',
 ]
 
-# File Paths
-FILE_ARXIV_COUNT = shared.path_join(PATHS["data_1-fetch"], "arxiv_1_count.csv")
-FILE_ARXIV_CATEGORY_REPORT = shared.path_join(
-    PATHS["data_1-fetch"], "arxiv_2_count_by_category_report.csv"
-)
-FILE_ARXIV_CATEGORY_REPORT_AGGREGATE = shared.path_join(
-    PATHS["data_1-fetch"], "arxiv_2_count_by_category_report_agg.csv"
-)
-FILE_ARXIV_YEAR = shared.path_join(
-    PATHS["data_1-fetch"], "arxiv_3_count_by_year.csv"
-)
-FILE_ARXIV_AUTHOR_BUCKET = shared.path_join(
-    PATHS["data_1-fetch"], "arxiv_4_count_by_author_bucket.csv"
-)
-# records metadata for each run for audit, reproducibility, and provenance
-FILE_PROVENANCE = shared.path_join(PATHS["data"], "arxiv_provenance.yaml")
-
+# CSV Headers
 HEADER_COUNT = ["TOOL_IDENTIFIER", "COUNT"]
 HEADER_CATEGORY_REPORT = [
     "TOOL_IDENTIFIER",
@@ -94,10 +77,6 @@
 HEADER_YEAR = ["TOOL_IDENTIFIER", "YEAR", "COUNT"]
 HEADER_AUTHOR_BUCKET = ["TOOL_IDENTIFIER", "AUTHOR_BUCKET", "COUNT"]
 
-QUARTER = os.path.basename(PATHS["data_quarter"])
-
-CATEGORY_LABELS = {}
-
 # Compiled regex patterns for CC license detection
 CC_PATTERNS = [
     (re.compile(r"\bCC[-\s]?0\b", re.IGNORECASE), "CC0"),
@@ -119,6 +98,186 @@
     ),
 ]
 
+# ArXiv Categories - manually curated from ArXiv official taxonomy
+# Source: https://arxiv.org/category_taxonomy
+CATEGORIES = {
+    # Computer Science
+    "cs.AI": "Artificial Intelligence",
+    "cs.AR": "Hardware Architecture",
+    "cs.CC": "Computational Complexity",
+    "cs.CE": "Computational Engineering, Finance, and Science",
+    "cs.CG": "Computational Geometry",
+    "cs.CL": "Computation and Language",
+    "cs.CR": "Cryptography and Security",
+    "cs.CV": "Computer Vision and Pattern Recognition",
+    "cs.CY": "Computers and Society",
+    "cs.DB": "Databases",
+    "cs.DC": "Distributed, Parallel, and Cluster Computing",
+    "cs.DL": "Digital Libraries",
+    "cs.DM": "Discrete Mathematics",
+    "cs.DS": "Data Structures and Algorithms",
+    "cs.ET": "Emerging Technologies",
+    "cs.FL": "Formal Languages and Automata Theory",
+    "cs.GL": "General Literature",
+    "cs.GR": "Graphics",
+    "cs.GT": "Computer Science and Game Theory",
+    "cs.HC": "Human-Computer Interaction",
+    "cs.IR": "Information Retrieval",
+    "cs.IT": "Information Theory",
+    "cs.LG": "Machine Learning",
+    "cs.LO": "Logic in Computer Science",
+    "cs.MA": "Multiagent Systems",
+    "cs.MM": "Multimedia",
+    "cs.MS": "Mathematical Software",
+    "cs.NA": "Numerical Analysis",
+    "cs.NE": "Neural and Evolutionary Computing",
+    "cs.NI": "Networking and Internet Architecture",
+    "cs.OH": "Other Computer Science",
+    "cs.OS": "Operating Systems",
+    "cs.PF": "Performance",
+    "cs.PL": "Programming Languages",
+    "cs.RO": "Robotics",
+    "cs.SC": "Symbolic Computation",
+    "cs.SD": "Sound",
+    "cs.SE": "Software Engineering",
+    "cs.SI": "Social and Information Networks",
+    "cs.SY": "Systems and Control",
+    # Mathematics
+    "math.AC": "Commutative Algebra",
+    "math.AG": "Algebraic Geometry",
+    "math.AP": "Analysis of PDEs",
+    "math.AT": "Algebraic Topology",
+    "math.CA": "Classical Analysis and ODEs",
+    "math.CO": "Combinatorics",
+    "math.CT": "Category Theory",
+    "math.CV": "Complex Variables",
+    "math.DG": "Differential Geometry",
+    "math.DS": "Dynamical Systems",
+    "math.FA": "Functional Analysis",
+    "math.GM": "General Mathematics",
+    "math.GN": "General Topology",
+    "math.GR": "Group Theory",
+    "math.GT": "Geometric Topology",
+    "math.HO": "History and Overview",
+    "math.IT": "Information Theory",
+    "math.KT": "K-Theory and Homology",
+    "math.LO": "Logic",
+    "math.MG": "Metric Geometry",
+    "math.MP": "Mathematical Physics",
+    "math.NA": "Numerical Analysis",
+    "math.NT": "Number Theory",
+    "math.OA": "Operator Algebras",
+    "math.OC": "Optimization and Control",
+    "math.PR": "Probability",
+    "math.QA": "Quantum Algebra",
+    "math.RA": "Rings and Algebras",
+    "math.RT": "Representation Theory",
+    "math.SG": "Symplectic Geometry",
+    "math.SP": "Spectral Theory",
+    "math.ST": "Statistics Theory",
+    # Physics
+    "physics.acc-ph": "Accelerator Physics",
+    "physics.ao-ph": "Atmospheric and Oceanic Physics",
+    "physics.app-ph": "Applied Physics",
+    "physics.atm-clus": "Atomic and Molecular Clusters",
+    "physics.atom-ph": "Atomic Physics",
+    "physics.bio-ph": "Biological Physics",
+    "physics.chem-ph": "Chemical Physics",
+    "physics.class-ph": "Classical Physics",
+    "physics.comp-ph": "Computational Physics",
+    "physics.data-an": "Data Analysis, Statistics and Probability",
+    "physics.ed-ph": "Physics Education",
+    "physics.flu-dyn": "Fluid Dynamics",
+    "physics.gen-ph": "General Physics",
+    "physics.geo-ph": "Geophysics",
+    "physics.hist-ph": "History and Philosophy of Physics",
+    "physics.ins-det": "Instrumentation and Detectors",
+    "physics.med-ph": "Medical Physics",
+    "physics.optics": "Optics",
+    "physics.plasm-ph": "Plasma Physics",
+    "physics.pop-ph": "Popular Physics",
+    "physics.soc-ph": "Physics and Society",
+    "physics.space-ph": "Space Physics",
+    # Statistics
+    "stat.AP": "Applications",
+    "stat.CO": "Computation",
+    "stat.ME": "Methodology",
+    "stat.ML": "Machine Learning",
+    "stat.OT": "Other Statistics",
+    "stat.TH": "Statistics Theory",
+    # Quantitative Biology
+    "q-bio.BM": "Biomolecules",
+    "q-bio.CB": "Cell Behavior",
+    "q-bio.GN": "Genomics",
+    "q-bio.MN": "Molecular Networks",
+    "q-bio.NC": "Neurons and Cognition",
+    "q-bio.OT": "Other Quantitative Biology",
+    "q-bio.PE": "Populations and Evolution",
+    "q-bio.QM": "Quantitative Methods",
+    "q-bio.SC": "Subcellular Processes",
+    "q-bio.TO": "Tissues and Organs",
+    # Economics
+    "econ.EM": "Econometrics",
+    "econ.GN": "General Economics",
+    "econ.TH": "Theoretical Economics",
+    # Electrical Engineering
+    "eess.AS": "Audio and Speech Processing",
+    "eess.IV": "Image and Video Processing",
+    "eess.SP": "Signal Processing",
+    "eess.SY": "Systems and Control",
+    # High Energy Physics
+    "hep-ex": "High Energy Physics - Experiment",
+    "hep-lat": "High Energy Physics - Lattice",
+    "hep-ph": "High Energy Physics - Phenomenology",
+    "hep-th": "High Energy Physics - Theory",
+    # Other Physics
+    "astro-ph": "Astrophysics",
+    "astro-ph.CO": "Cosmology and Nongalactic Astrophysics",
+    "astro-ph.EP": "Earth and Planetary Astrophysics",
+    "astro-ph.GA": "Astrophysics of Galaxies",
+    "astro-ph.HE": "High Energy Astrophysical Phenomena",
+    "astro-ph.IM": "Instrumentation and Methods for Astrophysics",
+    "astro-ph.SR": "Solar and Stellar Astrophysics",
+    "cond-mat.dis-nn": "Disordered Systems and Neural Networks",
+    "cond-mat.mes-hall": "Mesoscale and Nanoscale Physics",
+    "cond-mat.mtrl-sci": "Materials Science",
+    "cond-mat.other": "Other Condensed Matter",
+    "cond-mat.quant-gas": "Quantum Gases",
+    "cond-mat.soft": "Soft Condensed Matter",
+    "cond-mat.stat-mech": "Statistical Mechanics",
+    "cond-mat.str-el": "Strongly Correlated Electrons",
+    "cond-mat.supr-con": "Superconductivity",
+    "gr-qc": "General Relativity and Quantum Cosmology",
+    "nlin.AO": "Adaptation and Self-Organizing Systems",
+    "nlin.CD": "Chaotic Dynamics",
+    "nlin.CG": "Cellular Automata and Lattice Gases",
+    "nlin.PS": "Pattern Formation and Solitons",
+    "nlin.SI": "Exactly Solvable and Integrable Systems",
+    "nucl-ex": "Nuclear Experiment",
+    "nucl-th": "Nuclear Theory",
+    "quant-ph": "Quantum Physics",
+}
+
+# File Paths
+FILE_ARXIV_COUNT = shared.path_join(PATHS["data_1-fetch"], "arxiv_1_count.csv")
+FILE_ARXIV_CATEGORY_REPORT = shared.path_join(
+    PATHS["data_1-fetch"], "arxiv_2_count_by_category_report.csv"
+)
+FILE_ARXIV_CATEGORY_REPORT_AGGREGATE = shared.path_join(
+    PATHS["data_1-fetch"], "arxiv_2_count_by_category_report_agg.csv"
+)
+FILE_ARXIV_YEAR = shared.path_join(
+    PATHS["data_1-fetch"], "arxiv_3_count_by_year.csv"
+)
+FILE_ARXIV_AUTHOR_BUCKET = shared.path_join(
+    PATHS["data_1-fetch"], "arxiv_4_count_by_author_bucket.csv"
+)
+# records metadata for each run for audit, reproducibility, and provenance
+FILE_PROVENANCE = shared.path_join(PATHS["data"], "arxiv_provenance.yaml")
+
+# Runtime variables
+QUARTER = os.path.basename(PATHS["data_quarter"])
+
 # Log the start of the script execution
 LOGGER.info("Script execution started.")
 
@@ -313,7 +472,7 @@ def save_count_data(
         for lic, cats in category_counts.items():
             total_for_license = sum(cats.values()) or 1
             for code, c in cats.items():
-                label = shared.normalize_arxiv_category(code, CATEGORY_LABELS)
+                label = shared.normalize_arxiv_category(code, CATEGORIES)
                 pct = round((c / total_for_license) * 100, 2)
                 writer.writerow(
                     {
@@ -351,7 +510,7 @@ def save_count_data(
             others = sorted_cats[TOP_N:]
             other_count = sum(c for _, c in others)
             for code, c in top:
-                label = shared.normalize_arxiv_category(code, CATEGORY_LABELS)
+                label = shared.normalize_arxiv_category(code, CATEGORIES)
                 writer.writerow(
                     {
                         "TOOL_IDENTIFIER": lic,
@@ -413,9 +572,6 @@ def query_arxiv(args):
     LOGGER.info("Beginning to fetch results from ArXiv API")
     session = get_requests_session()
 
-    # Load category mappings using shared function
-    CATEGORY_LABELS.update(shared.load_arxiv_categories(PATHS.get("data")))
-
     results_per_iteration = RESULTS_PER_REQUEST
 
     search_queries = SEARCH_QUERIES