@@ -79,34 +79,6 @@ def backfill_dataset(location: str, *, dry_run: bool = False) -> None:
7979 )
8080
8181
82- def get_s3_object_creation_date (file_path : str , filesystem : fs .FileSystem ) -> datetime :
83- """Get the creation date of an S3 object.
84-
85- Args:
86- file_path: Path to the S3 object
87- filesystem: PyArrow S3 filesystem instance
88-
89- Returns:
90- datetime: Creation date of the S3 object in UTC
91- """
92- try :
93- # Get creation date of S3 object
94- file_info = filesystem .get_file_info (file_path )
95- creation_date : datetime = file_info .mtime # type: ignore[assignment]
96-
97- # Ensure it's timezone-aware and in UTC
98- if creation_date .tzinfo is None :
99- creation_date = creation_date .replace (tzinfo = UTC )
100- elif creation_date .tzinfo != UTC :
101- creation_date = creation_date .astimezone (UTC )
102-
103- return creation_date
104-
105- except Exception as e :
106- logger .error (f"Error getting S3 object creation date for { file_path } : { e } " )
107- raise
108-
109-
11082def backfill_parquet_file (
11183 parquet_filepath : str ,
11284 dataset : ds .Dataset ,
@@ -183,6 +155,34 @@ def backfill_parquet_file(
183155 }
184156
185157
158+ def get_s3_object_creation_date (file_path : str , filesystem : fs .FileSystem ) -> datetime :
159+ """Get the creation date of an S3 object.
160+
161+ Args:
162+ file_path: Path to the S3 object
163+ filesystem: PyArrow S3 filesystem instance
164+
165+ Returns:
166+ datetime: Creation date of the S3 object in UTC
167+ """
168+ try :
169+ # Get creation date of S3 object
170+ file_info = filesystem .get_file_info (file_path )
171+ creation_date : datetime = file_info .mtime # type: ignore[assignment]
172+
173+ # Ensure it's timezone-aware and in UTC
174+ if creation_date .tzinfo is None :
175+ creation_date = creation_date .replace (tzinfo = UTC )
176+ elif creation_date .tzinfo != UTC :
177+ creation_date = creation_date .astimezone (UTC )
178+
179+ return creation_date
180+
181+ except Exception as e :
182+ logger .error (f"Error getting S3 object creation date for { file_path } : { e } " )
183+ raise
184+
185+
186186if __name__ == "__main__" :
187187 parser = argparse .ArgumentParser (
188188 description = (
0 commit comments