Merge branch 'main' into process-report-2025Q4

Author: TimidRobot

README.md

@@ -227,6 +227,13 @@ When run this way, the shared library (`scripts/shared.py`) provides easy access
 to all of the necessary paths and all of the modules managed by pipenv are
 available.
 
+In order for scripts to be run directly (as shown above), the script must be
+executable. For more information on making files executable, please see:
+[File Permissions - Foundational technologies — Creative Commons Open
+Source][file-perms].
+
+[file-perms]: https://opensource.creativecommons.org/contributing-code/foundational-tech/#file-permissions
+
 
 ### Static analysis
 

scripts/2-process/gcs_process.py

@@ -4,7 +4,6 @@
 """
 # Standard library
 import argparse
-import csv
 import os
 import sys
 import textwrap
@@ -82,16 +81,6 @@ def parse_arguments():
     return args
 
 
-def data_to_csv(args, data, file_path):
-    if not args.enable_save:
-        return
-    os.makedirs(PATHS["data_phase"], exist_ok=True)
-    # emulate csv.unix_dialect
-    data.to_csv(
-        file_path, index=False, quoting=csv.QUOTE_ALL, lineterminator="\n"
-    )
-
-
 def process_product_totals(args, count_data):
     """
     Processing count data: totals by product
@@ -131,7 +120,7 @@ def process_product_totals(args, count_data):
         data.items(), columns=["CC legal tool product", "Count"]
     )
     file_path = shared.path_join(PATHS["data_phase"], "gcs_product_totals.csv")
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def process_latest_prior_retired_totals(args, count_data):
@@ -212,7 +201,7 @@ def process_latest_prior_retired_totals(args, count_data):
         file_path = shared.path_join(
             PATHS["data_phase"], f"gcs_status_{key}_totals.csv"
         )
-        data_to_csv(args, dataframe, file_path)
+        shared.data_to_csv(args, dataframe, file_path)
 
 
 def process_totals_by_free_cultural(args, count_data):
@@ -245,7 +234,7 @@ def process_totals_by_free_cultural(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "gcs_totals_by_free_cultural.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def process_totals_by_restrictions(args, count_data):
@@ -279,7 +268,7 @@ def process_totals_by_restrictions(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "gcs_totals_by_restrictions.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def process_totals_by_language(args, data):
@@ -300,7 +289,7 @@ def process_totals_by_language(args, data):
     file_path = shared.path_join(
         PATHS["data_phase"], "gcs_totals_by_language.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def process_totals_by_country(args, data):
@@ -321,14 +310,14 @@ def process_totals_by_country(args, data):
     file_path = shared.path_join(
         PATHS["data_phase"], "gcs_totals_by_country.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
+    shared.check_completion_file_exists(args, FILE_PATHS)
 
     # Count data
     file1_count = shared.path_join(PATHS["data_1-fetch"], "gcs_1_count.csv")

scripts/2-process/github_process.py

@@ -5,7 +5,6 @@
 """
 # Standard library
 import argparse
-import csv
 import os
 import sys
 import traceback
@@ -74,16 +73,6 @@ def parse_arguments():
     return args
 
 
-def data_to_csv(args, data, file_path):
-    if not args.enable_save:
-        return
-    os.makedirs(PATHS["data_phase"], exist_ok=True)
-    # emulate csv.unix_dialect
-    data.to_csv(
-        file_path, index=False, quoting=csv.QUOTE_ALL, lineterminator="\n"
-    )
-
-
 def process_totals_by_license(args, count_data):
     """
     Processing count data: totals by License
@@ -106,7 +95,7 @@ def process_totals_by_license(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "github_totals_by_license.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def process_totals_by_restriction(args, count_data):
@@ -140,14 +129,14 @@ def process_totals_by_restriction(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "github_totals_by_restriction.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
+    shared.check_completion_file_exists(args, FILE_PATHS)
     file_count = shared.path_join(PATHS["data_1-fetch"], "github_1_count.csv")
     count_data = shared.open_data_file(
         LOGGER, file_count, usecols=["TOOL_IDENTIFIER", "COUNT"]

scripts/2-process/wikipedia_process.py

@@ -5,7 +5,6 @@
 """
 # Standard library
 import argparse
-import csv
 import os
 import sys
 import textwrap
@@ -85,16 +84,6 @@ def parse_arguments():
     return args
 
 
-def data_to_csv(args, data, file_path):
-    if not args.enable_save:
-        return
-    os.makedirs(PATHS["data_phase"], exist_ok=True)
-    # emulate csv.unix_dialect
-    data.to_csv(
-        file_path, index=False, quoting=csv.QUOTE_ALL, lineterminator="\n"
-    )
-
-
 def process_highest_language_usage(args, count_data):
     """
     Processing count data: Most represented languages
@@ -113,7 +102,7 @@ def process_highest_language_usage(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "wikipedia_highest_language_usage.csv"
     )
-    data_to_csv(args, top_10, file_path)
+    shared.data_to_csv(args, top_10, file_path)
 
 
 def process_least_language_usage(args, count_data):
@@ -136,7 +125,7 @@ def process_least_language_usage(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "wikipedia_least_language_usage.csv"
     )
-    data_to_csv(args, bottom_10, file_path)
+    shared.data_to_csv(args, bottom_10, file_path)
 
 
 def process_language_representation(args, count_data):
@@ -162,14 +151,14 @@ def process_language_representation(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "wikipedia_language_representation.csv"
     )
-    data_to_csv(args, language_counts, file_path)
+    shared.data_to_csv(args, language_counts, file_path)
 
 
 def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
+    shared.check_completion_file_exists(args, FILE_PATHS)
     file_count = shared.path_join(
         PATHS["data_1-fetch"], "wikipedia_count_by_languages.csv"
     )

scripts/3-report/gcs_report.py

@@ -36,6 +36,7 @@ def parse_arguments():
     """
     Parses command-line arguments, returns parsed arguments.
     """
+    global QUARTER
     LOGGER.info("Parsing command-line arguments")
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -59,12 +60,18 @@ def parse_arguments():
         help="Enable git actions such as fetch, merge, add, commit, and push"
         " (default: False)",
     )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Regenerate data even if report files exist",
+    )
     args = parser.parse_args()
     if not args.enable_save and args.enable_git:
         parser.error("--enable-git requires --enable-save")
     if args.quarter != QUARTER:
         global PATHS
         PATHS = shared.paths_update(LOGGER, PATHS, QUARTER, args.quarter)
+        QUARTER = args.quarter
     args.logger = LOGGER
     args.paths = PATHS
     return args
@@ -491,7 +498,8 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-
+    last_entry = shared.path_join(PATHS["data_phase"], "gcs_free_culture.png")
+    shared.check_completion_file_exists(args, last_entry)
     gcs_intro(args)
     plot_products(args)
     plot_tool_status(args)

scripts/3-report/github_report.py

@@ -34,6 +34,7 @@ def parse_arguments():
     """
     Parses command-line arguments, returns parsed arguments.
     """
+    global QUARTER
     LOGGER.info("Parsing command-line arguments")
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -60,14 +61,15 @@ def parse_arguments():
     parser.add_argument(
         "--force",
         action="store_true",
-        help="Regenerate data even if images files already exist",
+        help="Regenerate data even if report files exist",
     )
     args = parser.parse_args()
     if not args.enable_save and args.enable_git:
         parser.error("--enable-git requires --enable-save")
     if args.quarter != QUARTER:
         global PATHS
         PATHS = shared.paths_update(LOGGER, PATHS, QUARTER, args.quarter)
+        QUARTER = args.quarter
     args.logger = LOGGER
     args.paths = PATHS
     return args
@@ -243,6 +245,10 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
+    last_entry = shared.path_join(
+        PATHS["data_phase"], "github_restriction.png"
+    )
+    shared.check_completion_file_exists(args, last_entry)
     github_intro(args)
     plot_totals_by_license_type(args)
     plot_totals_by_restriction(args)

scripts/3-report/wikipedia_report.py

@@ -34,6 +34,7 @@ def parse_arguments():
     """
     Parses command-line arguments, returns parsed arguments.
     """
+    global QUARTER
     LOGGER.info("Parsing command-line arguments")
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -57,12 +58,18 @@ def parse_arguments():
         help="Enable git actions such as fetch, merge, add, commit, and push"
         " (default: False)",
     )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Regenerate data even if report files exist",
+    )
     args = parser.parse_args()
     if not args.enable_save and args.enable_git:
         parser.error("--enable-git requires --enable-save")
     if args.quarter != QUARTER:
         global PATHS
         PATHS = shared.paths_update(LOGGER, PATHS, QUARTER, args.quarter)
+        QUARTER = args.quarter
     args.logger = LOGGER
     args.paths = PATHS
     return args
@@ -261,6 +268,10 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
+    last_entry = shared.path_join(
+        PATHS["data_phase"], "wikipedia_least_language_usage.png"
+    )
+    shared.check_completion_file_exists(args, last_entry)
     wikipedia_intro(args)
     plot_language_representation(args)
     plot_highest_language_usage(args)

scripts/shared.py

@@ -1,4 +1,5 @@
 # Standard library
+import csv
 import logging
 import os
 import sys
@@ -36,13 +37,31 @@ def __init__(self, message, exit_code=None):
         super().__init__(self.message)
 
 
-def check_for_data_files(args, file_paths, QUARTER):
+def data_to_csv(args, data, file_path):
+    if not args.enable_save:
+        return
+    os.makedirs(args.paths["data_phase"], exist_ok=True)
+    # emulate csv.unix_dialect
+    data.to_csv(
+        file_path, index=False, quoting=csv.QUOTE_ALL, lineterminator="\n"
+    )
+
+
+def check_completion_file_exists(args, file_paths):
+    """ "
+    This function checks if expected output files
+    exists. If any exist and --force is not provided,
+    the script exits early by raising a QuantifyingException.
+    In the case of a report file, we check if last output exists.
+    """
     if args.force:
         return
+    if isinstance(file_paths, str):
+        file_paths = [file_paths]
     for path in file_paths:
         if os.path.exists(path):
             raise QuantifyingException(
-                f"Processed data already exists for {QUARTER}", 0
+                f"Output files already exists for {args.quarter}", 0
             )
 
 
@@ -326,12 +345,12 @@ def update_readme(
     readme_path = path_join(paths["data"], args.quarter, "README.md")
 
     # Define section markers for each data source
-    section_start_line = f"<!-- section start {section_file} -->\n"
-    section_end_line = f"<!-- section end {section_file} -->\n"
+    section_start_line = f"<!-- SECTION start {section_file} -->\n"
+    section_end_line = f"<!-- SECTION end {section_file} -->\n"
 
     # Define entry markers for each plot (optional) and description
-    entry_start_line = f"<!-- entry start {entry_title} -->\n"
-    entry_end_line = f"<!-- entry end {entry_title} -->\n"
+    entry_start_line = f"<!-- {section_file} entry start {entry_title} -->\n"
+    entry_end_line = f"<!-- {section_file} entry end {entry_title} -->\n"
 
     if os.path.exists(readme_path):
         with open(readme_path, "r", encoding="utf-8") as f:
@@ -356,7 +375,7 @@ def update_readme(
         sections_before = ordered_sections[:current_postion]
         # we find the last existing section that comes before this section
         for prev_section_title in reversed(sections_before):
-            prev_end_line = f"<!-- section end {prev_section_title} -->\n"
+            prev_end_line = f"<!-- SECTION end {prev_section_title} -->\n"
             if prev_end_line in lines:
                 insert_index = lines.index(prev_end_line) + 1
                 break