added check function for process script

Author: oree-xx

scripts/1-fetch/wikipedia_fetch.py

@@ -67,7 +67,7 @@ def check_for_completion():
     try:
         with open(FILE_LANGUAGES, "r", newline="") as file_obj:
             reader = csv.DictReader(file_obj, dialect="unix")
-            if len(list(reader)) > 0:
+            if len(list(reader)) > 300:
                 raise shared.QuantifyingException(
                     f"Data fetch completed for {QUARTER}", 0
                 )

scripts/2-process/github_process.py

@@ -59,6 +59,13 @@ def parse_arguments():
     return args
 
 
+def check_for_data_file(file_path):
+    if os.path.exists(file_path):
+        raise shared.QuantifyingException(
+            f"Processed data already exists for {QUARTER}", 0
+        )
+
+
 def data_to_csv(args, data, file_path):
     if not args.enable_save:
         return
@@ -91,6 +98,7 @@ def process_totals_by_license(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "github_totals_by_license.csv"
     )
+    check_for_data_file(file_path)
     data_to_csv(args, data, file_path)
 
 
@@ -125,6 +133,7 @@ def process_totals_by_restriction(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "github_totals_by_restriction.csv"
     )
+    check_for_data_file(file_path)
     data_to_csv(args, data, file_path)
 
 

scripts/2-process/wikipedia_process.py

@@ -63,6 +63,13 @@ def parse_arguments():
     return args
 
 
+def check_for_data_file(file_path):
+    if os.path.exists(file_path):
+        raise shared.QuantifyingException(
+            f"Processed data already exists for {QUARTER}", 0
+        )
+
+
 def data_to_csv(args, data, file_path):
     if not args.enable_save:
         return
@@ -91,6 +98,7 @@ def process_highest_language_usage(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "wikipedia_highest_language_usage.csv"
     )
+    check_for_data_file(file_path)
     data_to_csv(args, top_10, file_path)
 
 
@@ -114,6 +122,7 @@ def process_least_language_usage(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "wikipedia_least_language_usage.csv"
     )
+    check_for_data_file(file_path)
     data_to_csv(args, bottom_10, file_path)
 
 
@@ -140,6 +149,7 @@ def process_language_representation(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "wikipedia_language_representation.csv"
     )
+    check_for_data_file(file_path)
     data_to_csv(args, language_counts, file_path)