Consolidated checks function

Author: oree-xx

scripts/2-process/gcs_process.py

@@ -317,7 +317,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
+    shared.check_completion_file_exists(args, FILE_PATHS)
 
     # Count data
     file1_count = shared.path_join(PATHS["data_1-fetch"], "gcs_1_count.csv")

scripts/2-process/github_process.py

@@ -136,7 +136,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
+    shared.check_completion_file_exists(args, FILE_PATHS)
     file_count = shared.path_join(PATHS["data_1-fetch"], "github_1_count.csv")
     count_data = shared.open_data_file(
         LOGGER, file_count, usecols=["TOOL_IDENTIFIER", "COUNT"]

scripts/2-process/wikipedia_process.py

@@ -158,7 +158,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
+    shared.check_completion_file_exists(args, FILE_PATHS)
     file_count = shared.path_join(
         PATHS["data_1-fetch"], "wikipedia_count_by_languages.csv"
     )

scripts/3-report/gcs_report.py

@@ -30,6 +30,7 @@
 QUARTER = os.path.basename(PATHS["data_quarter"])
 SECTION_FILE = Path(__file__).name
 SECTION_TITLE = "Google Custom Search (GCS)"
+LAST_ENTRY = shared.path_join(PATHS["data_phase"], "gcs_free_culture.png")
 
 
 def parse_arguments():
@@ -75,23 +76,6 @@ def parse_arguments():
     return args
 
 
-def check_report_completion(args):
-    """ "
-    The function checks for the last plot and image
-    caption created in this script. This helps to
-    immediately know if all plots in the script have
-    been created and should not be regenerated.
-
-    """
-    if args.force:
-        return
-    last_entry = shared.path_join(PATHS["data_phase"], "gcs_free_culture.png")
-    if os.path.exists(last_entry):
-        raise shared.QuantifyingException(
-            f"{last_entry} already exists. Report script completed", 0
-        )
-
-
 def gcs_intro(args):
     """
     Write Google Custom Search (GCS) introduction.
@@ -513,7 +497,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    check_report_completion(args)
+    shared.check_completion_file_exists(args, LAST_ENTRY)
     gcs_intro(args)
     plot_products(args)
     plot_tool_status(args)

scripts/3-report/github_report.py

@@ -28,6 +28,7 @@
 QUARTER = os.path.basename(PATHS["data_quarter"])
 SECTION_FILE = Path(__file__).name
 SECTION_TITLE = "Github"
+LAST_ENTRY = shared.path_join(PATHS["data_phase"], "github_restriction.png")
 
 
 def parse_arguments():
@@ -78,25 +79,6 @@ def parse_arguments():
     return args
 
 
-def check_report_completion(args):
-    """ "
-    The function checks for the last plot and image
-    caption created in this script. This helps to
-    immediately know if all plots in the script have
-    been created and should not be regenerated.
-
-    """
-    if args.force:
-        return
-    last_entry = shared.path_join(
-        PATHS["data_phase"], "github_restriction.png"
-    )
-    if os.path.exists(last_entry):
-        raise shared.QuantifyingException(
-            f"{last_entry} already exists. Report script completed", 0
-        )
-
-
 def load_data(args):
     """
     Load the collected data from the CSV file.
@@ -267,7 +249,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    check_report_completion(args)
+    shared.check_completion_file_exists(args, LAST_ENTRY)
     github_intro(args)
     plot_totals_by_license_type(args)
     plot_totals_by_restriction(args)

scripts/3-report/wikipedia_report.py

@@ -28,6 +28,9 @@
 QUARTER = os.path.basename(PATHS["data_quarter"])
 SECTION_FILE = Path(__file__).name
 SECTION_TITLE = "Wikipedia"
+LAST_ENTRY = shared.path_join(
+    PATHS["data_phase"], "wikipedia_least_language_usage.png"
+)
 
 
 def parse_arguments():
@@ -73,25 +76,6 @@ def parse_arguments():
     return args
 
 
-def check_report_completion(args):
-    """ "
-    The function checks for the last plot and image
-    caption created in this script. This helps to
-    immediately know if all plots in the script have
-    been created and should not be regenerated.
-
-    """
-    if args.force:
-        return
-    last_entry = shared.path_join(
-        PATHS["data_phase"], "wikipedia_least_language_usage.png"
-    )
-    if os.path.exists(last_entry):
-        raise shared.QuantifyingException(
-            f"{last_entry} already exists. Report script completed", 0
-        )
-
-
 def wikipedia_intro(args):
     """
     Write Wikipedia introduction.
@@ -285,7 +269,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    check_report_completion(args)
+    shared.check_completion_file_exists(args, LAST_ENTRY)
     wikipedia_intro(args)
     plot_language_representation(args)
     plot_highest_language_usage(args)

scripts/shared.py

@@ -47,13 +47,21 @@ def data_to_csv(args, data, file_path):
     )
 
 
-def check_for_data_files(args, file_paths, QUARTER):
+def check_completion_file_exists(args, file_paths):
+    """ "
+    This function checks if expected output files
+    exists. If any exist and --force is not provided,
+    the script exits early by raising a QuantifyingException.
+    In the case of a report file, we check if last output exists.
+    """
     if args.force:
         return
+    if isinstance(file_paths, str):
+        file_paths = [file_paths]
     for path in file_paths:
         if os.path.exists(path):
             raise QuantifyingException(
-                f"Processed data already exists for {QUARTER}", 0
+                f"Output files already exists for {args.quarter}", 0
             )