query category names instead of using manual list

Author: TimidRobot

scripts/1-fetch/arxiv_fetch.py

@@ -36,166 +36,7 @@
 # Constants
 BASE_URL = "https://oaipmh.arxiv.org/oai"
 # Defaults should result in quick operation (not complete operation)
-# ArXiv Categories - manually curated from ArXiv official taxonomy
-# Source: https://arxiv.org/category_taxonomy
-CATEGORIES = {
-    # Computer Science
-    "cs.AI": "Artificial Intelligence",
-    "cs.AR": "Hardware Architecture",
-    "cs.CC": "Computational Complexity",
-    "cs.CE": "Computational Engineering, Finance, and Science",
-    "cs.CG": "Computational Geometry",
-    "cs.CL": "Computation and Language",
-    "cs.CR": "Cryptography and Security",
-    "cs.CV": "Computer Vision and Pattern Recognition",
-    "cs.CY": "Computers and Society",
-    "cs.DB": "Databases",
-    "cs.DC": "Distributed, Parallel, and Cluster Computing",
-    "cs.DL": "Digital Libraries",
-    "cs.DM": "Discrete Mathematics",
-    "cs.DS": "Data Structures and Algorithms",
-    "cs.ET": "Emerging Technologies",
-    "cs.FL": "Formal Languages and Automata Theory",
-    "cs.GL": "General Literature",
-    "cs.GR": "Graphics",
-    "cs.GT": "Computer Science and Game Theory",
-    "cs.HC": "Human-Computer Interaction",
-    "cs.IR": "Information Retrieval",
-    "cs.IT": "Information Theory",
-    "cs.LG": "Machine Learning",
-    "cs.LO": "Logic in Computer Science",
-    "cs.MA": "Multiagent Systems",
-    "cs.MM": "Multimedia",
-    "cs.MS": "Mathematical Software",
-    "cs.NA": "Numerical Analysis",
-    "cs.NE": "Neural and Evolutionary Computing",
-    "cs.NI": "Networking and Internet Architecture",
-    "cs.OH": "Other Computer Science",
-    "cs.OS": "Operating Systems",
-    "cs.PF": "Performance",
-    "cs.PL": "Programming Languages",
-    "cs.RO": "Robotics",
-    "cs.SC": "Symbolic Computation",
-    "cs.SD": "Sound",
-    "cs.SE": "Software Engineering",
-    "cs.SI": "Social and Information Networks",
-    "cs.SY": "Systems and Control",
-    # Mathematics
-    "math.AC": "Commutative Algebra",
-    "math.AG": "Algebraic Geometry",
-    "math.AP": "Analysis of PDEs",
-    "math.AT": "Algebraic Topology",
-    "math.CA": "Classical Analysis and ODEs",
-    "math.CO": "Combinatorics",
-    "math.CT": "Category Theory",
-    "math.CV": "Complex Variables",
-    "math.DG": "Differential Geometry",
-    "math.DS": "Dynamical Systems",
-    "math.FA": "Functional Analysis",
-    "math.GM": "General Mathematics",
-    "math.GN": "General Topology",
-    "math.GR": "Group Theory",
-    "math.GT": "Geometric Topology",
-    "math.HO": "History and Overview",
-    "math.IT": "Information Theory",
-    "math.KT": "K-Theory and Homology",
-    "math.LO": "Logic",
-    "math.MG": "Metric Geometry",
-    "math.MP": "Mathematical Physics",
-    "math.NA": "Numerical Analysis",
-    "math.NT": "Number Theory",
-    "math.OA": "Operator Algebras",
-    "math.OC": "Optimization and Control",
-    "math.PR": "Probability",
-    "math.QA": "Quantum Algebra",
-    "math.RA": "Rings and Algebras",
-    "math.RT": "Representation Theory",
-    "math.SG": "Symplectic Geometry",
-    "math.SP": "Spectral Theory",
-    "math.ST": "Statistics Theory",
-    # Physics
-    "physics.acc-ph": "Accelerator Physics",
-    "physics.ao-ph": "Atmospheric and Oceanic Physics",
-    "physics.app-ph": "Applied Physics",
-    "physics.atm-clus": "Atomic and Molecular Clusters",
-    "physics.atom-ph": "Atomic Physics",
-    "physics.bio-ph": "Biological Physics",
-    "physics.chem-ph": "Chemical Physics",
-    "physics.class-ph": "Classical Physics",
-    "physics.comp-ph": "Computational Physics",
-    "physics.data-an": "Data Analysis, Statistics and Probability",
-    "physics.ed-ph": "Physics Education",
-    "physics.flu-dyn": "Fluid Dynamics",
-    "physics.gen-ph": "General Physics",
-    "physics.geo-ph": "Geophysics",
-    "physics.hist-ph": "History and Philosophy of Physics",
-    "physics.ins-det": "Instrumentation and Detectors",
-    "physics.med-ph": "Medical Physics",
-    "physics.optics": "Optics",
-    "physics.plasm-ph": "Plasma Physics",
-    "physics.pop-ph": "Popular Physics",
-    "physics.soc-ph": "Physics and Society",
-    "physics.space-ph": "Space Physics",
-    # Statistics
-    "stat.AP": "Applications",
-    "stat.CO": "Computation",
-    "stat.ME": "Methodology",
-    "stat.ML": "Machine Learning",
-    "stat.OT": "Other Statistics",
-    "stat.TH": "Statistics Theory",
-    # Quantitative Biology
-    "q-bio.BM": "Biomolecules",
-    "q-bio.CB": "Cell Behavior",
-    "q-bio.GN": "Genomics",
-    "q-bio.MN": "Molecular Networks",
-    "q-bio.NC": "Neurons and Cognition",
-    "q-bio.OT": "Other Quantitative Biology",
-    "q-bio.PE": "Populations and Evolution",
-    "q-bio.QM": "Quantitative Methods",
-    "q-bio.SC": "Subcellular Processes",
-    "q-bio.TO": "Tissues and Organs",
-    # Economics
-    "econ.EM": "Econometrics",
-    "econ.GN": "General Economics",
-    "econ.TH": "Theoretical Economics",
-    # Electrical Engineering
-    "eess.AS": "Audio and Speech Processing",
-    "eess.IV": "Image and Video Processing",
-    "eess.SP": "Signal Processing",
-    "eess.SY": "Systems and Control",
-    # High Energy Physics
-    "hep-ex": "High Energy Physics - Experiment",
-    "hep-lat": "High Energy Physics - Lattice",
-    "hep-ph": "High Energy Physics - Phenomenology",
-    "hep-th": "High Energy Physics - Theory",
-    # Other Physics
-    "astro-ph": "Astrophysics",
-    "astro-ph.CO": "Cosmology and Nongalactic Astrophysics",
-    "astro-ph.EP": "Earth and Planetary Astrophysics",
-    "astro-ph.GA": "Astrophysics of Galaxies",
-    "astro-ph.HE": "High Energy Astrophysical Phenomena",
-    "astro-ph.IM": "Instrumentation and Methods for Astrophysics",
-    "astro-ph.SR": "Solar and Stellar Astrophysics",
-    "cond-mat.dis-nn": "Disordered Systems and Neural Networks",
-    "cond-mat.mes-hall": "Mesoscale and Nanoscale Physics",
-    "cond-mat.mtrl-sci": "Materials Science",
-    "cond-mat.other": "Other Condensed Matter",
-    "cond-mat.quant-gas": "Quantum Gases",
-    "cond-mat.soft": "Soft Condensed Matter",
-    "cond-mat.stat-mech": "Statistical Mechanics",
-    "cond-mat.str-el": "Strongly Correlated Electrons",
-    "cond-mat.supr-con": "Superconductivity",
-    "gr-qc": "General Relativity and Quantum Cosmology",
-    "nlin.AO": "Adaptation and Self-Organizing Systems",
-    "nlin.CD": "Chaotic Dynamics",
-    "nlin.CG": "Cellular Automata and Lattice Gases",
-    "nlin.PS": "Pattern Formation and Solitons",
-    "nlin.SI": "Exactly Solvable and Integrable Systems",
-    "nucl-ex": "Nuclear Experiment",
-    "nucl-th": "Nuclear Theory",
-    "quant-ph": "Quantum Physics",
-}
-DEFAULT_FETCH_LIMIT = 1000
+DEFAULT_FETCH_LIMIT = 4500  # Fetch 3 batches of 1,500 articles each
 DEFAULT_YEARS_BACK = 5
 # CSV file paths
 FILE_ARXIV_AUTHOR_BUCKET = shared.path_join(
@@ -334,6 +175,45 @@ def get_license_mapping():
     )
 
 
+def query_category_mapping(args, session):
+    """
+    Query to establish mapping of category codes and names.
+
+    Also see https://arxiv.org/category_taxonomy
+    """
+    global CATEGORY_MAPPING
+
+    params = {"verb": "ListSets"}
+    try:
+        response = session.get(BASE_URL, params=params, timeout=60)
+        response.raise_for_status()
+    except requests.HTTPError as e:
+        raise shared.QuantifyingException(f"HTTP Error: {e}", 1)
+    except requests.RequestException as e:
+        raise shared.QuantifyingException(f"Request Exception: {e}", 1)
+
+    root = etree.fromstring(response.content)
+    CATEGORY_MAPPING = {}
+    sets = root.findall(".//{http://www.openarchives.org/OAI/2.0/}set")
+    for set_ in sets:
+        spec, name = set_.getchildren()
+        # Ensure category code (key) matches code used in articles
+        spec_list = spec.text.split(":")
+        if len(spec_list) > 1:
+            # Remove parent category and replace colon with period
+            # 3 part examples:
+            #     match:math:AC       => math.AC
+            #     physics:astro-ph:CO => astro-ph.CO
+            # 2 part examples
+            #     physics:astro-ph    => astro-ph
+            #     physics:quant-ph    => quant-ph
+            spec_text = ".".join(spec_list[1:])
+        else:
+            spec_text = spec.text
+        CATEGORY_MAPPING[spec_text] = name.text
+    CATEGORY_MAPPING = dict(sorted(CATEGORY_MAPPING.items()))
+
+
 def extract_record_license(record):
     """
     Extract CC license information from OAI-PMH XML record.
@@ -457,16 +337,20 @@ def query_arxiv(args, session):
     # resumption token)
     proceed = True
     while proceed:
+        if args.limit > 0 and args.limit <= total_fetched:
+            proceed = False
+            break
+
         if resumption_token:
             # Continue with resumption token
-            query_params = {
+            params = {
                 "verb": "ListRecords",
                 "resumptionToken": resumption_token,
             }
             verb = "resuming"
         else:
             # Initial request with date range
-            query_params = {
+            params = {
                 "verb": "ListRecords",
                 "metadataPrefix": "arXiv",
                 "from": args.from_date,
@@ -481,7 +365,7 @@ def query_arxiv(args, session):
 
         try:
             # Build OAI-PMH request URL
-            response = session.get(BASE_URL, params=query_params, timeout=60)
+            response = session.get(BASE_URL, params=params, timeout=60)
             response.raise_for_status()
         except requests.HTTPError as e:
             raise shared.QuantifyingException(f"HTTP Error: {e}", 1)
@@ -614,7 +498,7 @@ def write_data(args, data):
     rows = []
     for license_name, categories in data["category_counts"].items():
         for code, count in categories.items():
-            label = CATEGORIES.get(code, code)
+            label = CATEGORY_MAPPING.get(code, code)
             rows.append(
                 {
                     "TOOL_IDENTIFIER": license_name,
@@ -687,6 +571,7 @@ def main():
     initialize_all_data_files(args)
     get_license_mapping()
     session = shared.get_session()
+    query_category_mapping(args, session)
     data, cc_articles_found = query_arxiv(args, session)
     write_data(args, data)
     write_provence(args, cc_articles_found)