Made review changes

Author: oree-xx

scripts/2-process/gcs_process.py

@@ -77,14 +77,6 @@ def parse_arguments():
     return args
 
 
-def check_for_data_files(args, file_paths):
-    for path in file_paths:
-        if os.path.exists(path) and not args.force:
-            raise shared.QuantifyingException(
-                f"Processed data already exists for {QUARTER}", 0
-            )
-
-
 def data_to_csv(args, data, file_path):
     if not args.enable_save:
         return
@@ -331,7 +323,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    check_for_data_files(args, FILE_PATHS)
+    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
 
     # Count data
     file1_count = shared.path_join(PATHS["data_1-fetch"], "gcs_1_count.csv")

scripts/2-process/github_process.py

@@ -69,14 +69,6 @@ def parse_arguments():
     return args
 
 
-def check_for_data_files(args, file_paths):
-    for path in file_paths:
-        if os.path.exists(path) and not args.force:
-            raise shared.QuantifyingException(
-                f"Processed data already exists for {QUARTER}", 0
-            )
-
-
 def data_to_csv(args, data, file_path):
     if not args.enable_save:
         return
@@ -150,7 +142,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    check_for_data_files(args, FILE_PATHS)
+    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
     file_count = shared.path_join(PATHS["data_1-fetch"], "github_1_count.csv")
     count_data = shared.open_data_file(
         LOGGER, file_count, usecols=["TOOL_IDENTIFIER", "COUNT"]
@@ -176,7 +168,7 @@ def main():
             LOGGER.info(e.message)
         else:
             LOGGER.error(e.message)
-        sys.exit(e.code)
+        sys.exit(e.exit_code)
     except SystemExit as e:
         LOGGER.error(f"System exit with code: {e.code}")
         sys.exit(e.code)

scripts/2-process/wikipedia_process.py

@@ -80,14 +80,6 @@ def parse_arguments():
     return args
 
 
-def check_for_data_files(args, file_paths):
-    for path in file_paths:
-        if os.path.exists(path) and not args.force:
-            raise shared.QuantifyingException(
-                f"Processed data already exists for {QUARTER}", 0
-            )
-
-
 def data_to_csv(args, data, file_path):
     if not args.enable_save:
         return
@@ -172,7 +164,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    check_for_data_files(args, FILE_PATHS)
+    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
     file_count = shared.path_join(
         PATHS["data_1-fetch"], "wikipedia_count_by_languages.csv"
     )

scripts/3-report/gcs_report.py

@@ -28,7 +28,8 @@
 
 # Constants
 QUARTER = os.path.basename(PATHS["data_quarter"])
-SECTION = Path(__file__).name
+SECTION_FILE = Path(__file__).name
+SECTION_TITLE = "Google Custom Search (GCS)"
 
 
 def parse_arguments():
@@ -84,7 +85,8 @@ def gcs_intro(args):
     total_count = f"{data['Count'].sum():,d}"
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         "Overview",
         None,
         None,
@@ -138,7 +140,8 @@ def plot_products(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing Creative Commons (CC) legal tool product totals and"
@@ -181,7 +184,8 @@ def plot_tool_status(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing Creative Commons (CC) legal tool status totals and"
@@ -224,7 +228,8 @@ def plot_latest_tools(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing latest Creative Commons (CC) legal tool totals and"
@@ -266,7 +271,8 @@ def plot_prior_tools(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing prior Creative Commons (CC) legal tool totals and"
@@ -312,7 +318,8 @@ def plot_retired_tools(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing retired Creative Commons (CC) legal tools total and"
@@ -361,7 +368,8 @@ def plot_countries_highest_usage(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing countries with the highest useage of the latest"
@@ -414,7 +422,8 @@ def plot_languages_highest_usage(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing languages with the highest useage of the latest"
@@ -461,7 +470,8 @@ def plot_free_culture(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing Approved for Free Cultural Works legal tool usage.",

scripts/3-report/github_report.py

@@ -26,7 +26,13 @@
 # Setup
 LOGGER, PATHS = shared.setup(__file__)
 QUARTER = os.path.basename(PATHS["data_quarter"])
-SECTION = Path(__file__).name
+SECTION_FILE = Path(__file__).name
+SECTION_TITLE = "Github"
+
+IMAGE_PATHS = [
+    shared.path_join(PATHS["data_phase"], "github_totals_by_license_type.png"),
+    shared.path_join(PATHS["data_phase"], "github_restriction.png"),
+]
 
 
 def parse_arguments():
@@ -56,6 +62,11 @@ def parse_arguments():
         help="Enable git actions such as fetch, merge, add, commit, and push"
         " (default: False)",
     )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Regenerate data even if images files already exist",
+    )
     args = parser.parse_args()
     if not args.enable_save and args.enable_git:
         parser.error("--enable-git requires --enable-save")
@@ -67,6 +78,14 @@ def parse_arguments():
     return args
 
 
+def check_image_files(args, image_paths):
+    for path in image_paths:
+        if os.path.exists(path) and not args.force:
+            raise shared.QuantifyingException(
+                f"image file already exists for {path}", 0
+            )
+
+
 def load_data(args):
     """
     Load the collected data from the CSV file.
@@ -100,7 +119,8 @@ def github_intro(args):
     cc_percentage = f"{(cc_total / total_repositories) * 100:.2f}%"
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         "Overview",
         None,
         None,
@@ -111,7 +131,7 @@ def github_intro(args):
         f"** of the {total_repositories} total public repositories"
         " on GitHub that use a CC legal tool. Additionally,"
         " many more use a non-CC use a Public domain"
-        " equivalent legal tools.**\n"
+        " equivalent legal tools.\n"
         "\n"
         " The Github data showcases the different level of"
         " rights reserved on repositories We have Public"
@@ -121,7 +141,7 @@ def github_intro(args):
         " without restriction."
         " See more at"
         " [Public-domain-equivalent license]"
-        "(https://en.wikipedia.org/wiki/Public-domain-equivalent_license)"
+        "(https://en.wikipedia.org/wiki/Public-domain-equivalent_license).\n"
         " While a Permissive category of license contains works"
         " under MIT-0 and CC BY 4.0 allows users to"
         " reuse the code with some conditions and attribution"
@@ -130,7 +150,7 @@ def github_intro(args):
         " and Copyleft contains works under CC BY-SA 4.0."
         " which requires any derivative works to be licensed"
         " under the same terms."
-        " [Copyleft](https://en.wikipedia.org/wiki/Copyleft)"
+        " [Copyleft](https://en.wikipedia.org/wiki/Copyleft).\n"
         "\n"
         "Thank you GitHub for providing public API"
         " access to repository metadata!",
@@ -172,7 +192,8 @@ def plot_totals_by_license_type(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing totals by license type."
@@ -220,7 +241,8 @@ def plot_totals_by_restriction(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing totals by different levels of restrictions."
@@ -234,6 +256,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
+    check_image_files(args, IMAGE_PATHS)
     github_intro(args)
     plot_totals_by_license_type(args)
     plot_totals_by_restriction(args)

scripts/3-report/wikipedia_report.py

@@ -26,7 +26,8 @@
 # Setup
 LOGGER, PATHS = shared.setup(__file__)
 QUARTER = os.path.basename(PATHS["data_quarter"])
-SECTION = Path(__file__).name
+SECTION_FILE = Path(__file__).name
+SECTION_TITLE = "Wikipedia"
 
 
 def parse_arguments():
@@ -98,7 +99,8 @@ def wikipedia_intro(args):
     language_count = len(data)
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         "Overview",
         None,
         None,
@@ -156,7 +158,8 @@ def plot_language_representation(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing the language representation across different language"
@@ -201,7 +204,8 @@ def plot_highest_language_usage(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing the most represented languages across the different"
@@ -244,7 +248,8 @@ def plot_least_language_usage(args):
 
     shared.update_readme(
         args,
-        SECTION,
+        SECTION_FILE,
+        SECTION_TITLE,
         title,
         image_path,
         "Plots showing the least represented languages across the different"

scripts/shared.py

@@ -36,6 +36,16 @@ def __init__(self, message, exit_code=None):
         super().__init__(self.message)
 
 
+def check_for_data_files(args, file_paths, QUARTER):
+    if args.force:
+        return
+    for path in file_paths:
+        if os.path.exists(path):
+            raise QuantifyingException(
+                f"Processed data already exists for {QUARTER}", 0
+            )
+
+
 def get_session(accept_header=None, session=None):
     """
     Create or configure a reusable HTTPS session with retry logic and
@@ -272,11 +282,13 @@ def setup(current_file):
 def section_order():
     report_dir = os.path.join(os.path.dirname(__file__), "3-report")
     report_files = os.listdir(report_dir)
+    report_files.sort()
     return report_files
 
 
 def update_readme(
     args,
+    section_file,
     section_title,
     entry_title,
     image_path,
@@ -289,8 +301,8 @@ def update_readme(
     logger = args.logger
     paths = args.paths
     ordered_sections = section_order()
-    logger.info("ordered_sections:", ordered_sections)
-    logger.info("section_title:", repr(section_title))
+    logger.info(f"ordered_sections:, {ordered_sections}")
+    logger.info(f"section_title:, {section_title}")
 
     if not args.enable_save:
         return
@@ -308,8 +320,8 @@ def update_readme(
     readme_path = path_join(paths["data"], args.quarter, "README.md")
 
     # Define section markers for each data source
-    section_start_line = f"<!-- section start {section_title} -->\n"
-    section_end_line = f"<!-- section end {section_title} -->\n"
+    section_start_line = f"<!-- section start {section_file} -->\n"
+    section_end_line = f"<!-- section end {section_file} -->\n"
 
     # Define entry markers for each plot (optional) and description
     entry_start_line = f"<!-- entry start {entry_title} -->\n"
@@ -333,7 +345,7 @@ def update_readme(
     else:
         insert_index = None
         # If not present, we find the position to insert the section
-        current_postion = ordered_sections.index(section_title)
+        current_postion = ordered_sections.index(section_file)
         # Sections that should come before this section
         sections_before = ordered_sections[:current_postion]
         # we find the last existing section that comes before this section