Download model file to current work dir by default and fix chinese file name issue (#103)

- Introduced logging in download_model.py
- Added logging in upload_repo.py
- Updated CLI command help text
- Replaced print statements with logger in file_upload.py
- Enhanced logging in repository.py
- Improved logging in snapshot_download.py
- Added cache path constants in local_folder.py
- Cleaned up cache after upload in main.py
- Added error logging for dataset and model info retrieval in utils.py
- Migrated setup configuration to pyproject.toml

Author: HaiHui886

examples/download_model.py

@@ -1,3 +1,4 @@
+import logging
 from pycsghub.snapshot_download import snapshot_download
 # token = "your access token"
 token = None
@@ -9,6 +10,14 @@
 allow_patterns = ["*.json"]
 ignore_patterns = ["tokenizer.json"]
 
+# set log level
+logging.basicConfig(
+    level=getattr(logging, "INFO"),
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    handlers=[logging.StreamHandler()]
+)
+
 result = snapshot_download(
     repo_id, 
     repo_type=repo_type, 

examples/upload_repo.py

@@ -1,12 +1,21 @@
+import logging
 from pycsghub.repository import Repository
 
 # token = "your access token"
 token = None
 
+# set log level
+logging.basicConfig(
+    level=getattr(logging, "INFO"),
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    handlers=[logging.StreamHandler()]
+)
+
 r = Repository(
-    repo_id="wanghh2003/ds15",
+    repo_id="wanghh2000/ds16",
     upload_path="/Users/hhwang/temp/bbb/jsonl",
-    user_name="wanghh2003",
+    user_name="wanghh2000",
     token=token,
     repo_type="dataset",
 )

pycsghub/cli.py

@@ -49,7 +49,7 @@ def version_callback(value: bool):
     "source": typer.Option("--source", help="Specify the source of the repository (e.g. 'csg', 'hf', 'ms')."),
 }
 
-@app.command(name="download", help="Download model/dataset from OpenCSG Hub", no_args_is_help=True)
+@app.command(name="download", help="Download model/dataset/space from OpenCSG Hub", no_args_is_help=True)
 def download(
         repo_id: Annotated[str, OPTIONS["repoID"]],
         repo_type: Annotated[RepoType, OPTIONS["repoType"]] = RepoType.MODEL, 
@@ -245,7 +245,6 @@ def stop_finetune(
     }
 )
 def main(
-    version: bool = OPTIONS["version"],
     log_level: str = OPTIONS["log_level"]
 ):
     # for example: format='%(asctime)s - %(name)s:%(funcName)s:%(lineno)d - %(levelname)s - %(message)s',

pycsghub/file_upload.py

@@ -3,6 +3,9 @@
 from typing import Optional
 from pycsghub.constants import (DEFAULT_REVISION)
 from pycsghub.utils import (build_csg_headers, get_endpoint)
+import logging
+
+logger = logging.getLogger(__name__)
 
 def http_upload_file(
         repo_id: str,
@@ -25,7 +28,7 @@ def http_upload_file(
     form_data = {'file_path': destination_path, 'branch': revision, 'message': 'upload' + file_path}
     response = requests.post(http_url, headers=post_headers, data=form_data, files=file_data)
     if response.status_code == 200:
-        print(f"file '{file_path}' upload successfully.")
+        logger.info(f"file '{file_path}' upload successfully.")
     else:
-        print(f"fail to upload {file_path} with response code: {response.status_code}, error: {response.content.decode()}")
+        logger.error(f"fail to upload {file_path} with response code: {response.status_code}, error: {response.content.decode()}")
 

pycsghub/repository.py

@@ -17,6 +17,9 @@
 from pycsghub.utils import (build_csg_headers,
                             model_id_to_group_owner_name,
                             get_endpoint)
+import logging
+
+logger = logging.getLogger(__name__)
 
 def ignore_folders(folder, contents):
     ignored = []
@@ -185,7 +188,7 @@ def create_new_branch(self):
         })
         response = requests.post(url, json=data, headers=headers)
         if response.status_code != 200:
-            print(f"create branch on {url} response: {response.text}")
+            logger.info(f"create branch on {url} response: {response.text}")
         response.raise_for_status()
         return response
 
@@ -208,7 +211,7 @@ def create_new_repo(self):
         })
         response = requests.post(url, json=data, headers=headers)
         if response.status_code != 200:
-            print(f"create repo on {url} response: {response.text}")
+            logger.info(f"create repo on {url} response: {response.text}")
         response.raise_for_status()
         return response
 
@@ -291,6 +294,7 @@ def track_large_files(self, work_dir: str, pattern: str = ".") -> List[str]:
             if filename in deleted_files:
                 continue
 
+            logger.debug(f"Checking file {filename} for LFS tracking.")
             path_to_file = os.path.join(os.getcwd(), work_dir, filename)
             size_in_mb = os.path.getsize(path_to_file) / (1024 * 1024)
 
@@ -302,11 +306,23 @@ def track_large_files(self, work_dir: str, pattern: str = ".") -> List[str]:
 
         return files_to_be_tracked_with_lfs
 
+    def set_git_config_quotepath(self, work_dir: str) -> None:
+        try:
+            self.run_subprocess("git config --global core.quotepath false".split(), work_dir)
+        except subprocess.CalledProcessError:
+            try:
+                self.run_subprocess("git config core.quotepath false".split(), work_dir)
+            except subprocess.CalledProcessError:
+                pass
+
     def list_files_to_be_staged(self, work_dir: str, pattern: str = ".") -> List[str]:
         try:
+            self.set_git_config_quotepath(work_dir)
             p = self.run_subprocess("git ls-files --exclude-standard -mo".split() + [pattern], work_dir)
-            if len(p.stdout.strip()):
-                files = p.stdout.strip().split("\n")
+            output = p.stdout.strip()
+            if len(output):
+                logger.debug(f"Files to be staged: {output}")
+                files = output.split("\n")
             else:
                 files = []
         except subprocess.CalledProcessError as exc:

pycsghub/snapshot_download.py

@@ -14,6 +14,9 @@
 from pycsghub.constants import DEFAULT_REVISION, REPO_TYPES
 from pycsghub import utils
 from pycsghub.constants import REPO_TYPE_MODEL
+import logging
+
+logger = logging.getLogger(__name__)
 
 def snapshot_download(
         repo_id: str,
@@ -35,14 +38,18 @@ def snapshot_download(
         repo_type = REPO_TYPE_MODEL
     if repo_type not in REPO_TYPES:
         raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(REPO_TYPES)}")
+    
     if cache_dir is None:
         cache_dir = get_cache_dir(repo_type=repo_type)
     if isinstance(cache_dir, Path):
         cache_dir = str(cache_dir)
+
     temporary_cache_dir = os.path.join(cache_dir, 'temp')
     os.makedirs(temporary_cache_dir, exist_ok=True)
 
-    if local_dir is not None and isinstance(local_dir, Path):
+    if local_dir is None:
+        local_dir = os.getcwd()
+    if isinstance(local_dir, Path):
         local_dir = str(local_dir)
 
     group_or_owner, name = model_id_to_group_owner_name(repo_id)
@@ -83,7 +90,7 @@ def snapshot_download(
                 repo_file_info = pack_repo_file_info(repo_file, revision)
                 if cache.exists(repo_file_info):
                     file_name = os.path.basename(repo_file_info['Path'])
-                    print(f"File {file_name} already in '{cache.get_root_location()}', skip downloading!")
+                    logger.info(f"File {file_name} already in '{cache.get_root_location()}', skip downloading!")
                     continue
 
                 # get download url
@@ -95,6 +102,7 @@ def snapshot_download(
                     endpoint=download_endpoint,
                     source=source)
                 # todo support parallel download api
+                logger.debug(f"Downloading {repo_file} from {url}")
                 http_get(
                     url=url,
                     local_dir=temp_cache_dir,
@@ -106,7 +114,7 @@ def snapshot_download(
                 # todo using hash to check file integrity
                 temp_file = os.path.join(temp_cache_dir, repo_file)
                 savedFile = cache.put_file(repo_file_info, temp_file)
-                print(f"Saved file to '{savedFile}'")
+                logger.info(f"Saved file to '{savedFile}'")
 
         cache.save_model_version(revision_info={'Revision': revision})
         return os.path.join(cache.get_root_location())

pycsghub/upload_large_folder/local_folder.py

@@ -22,6 +22,9 @@
 key_lfs_uploaded_ids = "lfs_uploaded_ids"
 key_remote_oid = "remote_oid"
 
+cache_path = ".cache"
+cache_csghub = "csghub"
+
 @dataclass(frozen=True)
 class LocalUploadFilePaths:
     path_in_repo: str
@@ -184,7 +187,7 @@ def get_local_upload_paths(local_dir: Path, filename: str) -> LocalUploadFilePat
 
 
 def csghub_dir(local_dir: Path) -> Path:
-    path = local_dir / ".cache" / "csghub"
+    path = local_dir / cache_path / cache_csghub
     path.mkdir(exist_ok=True, parents=True)
 
     gitignore = path / ".gitignore"

pycsghub/upload_large_folder/main.py

@@ -17,6 +17,8 @@
 from pycsghub.constants import DEFAULT_REVISION
 import os
 import signal
+import shutil
+from .local_folder import cache_path, cache_csghub
 
 logger = logging.getLogger(__name__)
 
@@ -67,7 +69,7 @@ def upload_large_folder_internal(
 
         items = [
             (paths, read_upload_metadata(folder_path, paths.path_in_repo))
-            for paths in tqdm(paths_list, desc=f"recovering from cache metadata from {folder_path}/.cache")
+            for paths in tqdm(paths_list, desc=f"recovering from cache metadata from {folder_path}/f{cache_path}")
         ]
 
         logger.info(f"starting {num_workers} worker threads for upload tasks")
@@ -109,6 +111,13 @@ def upload_large_folder_internal(
 
         print(status.current_report())
         logging.info("large folder upload process is complete!")
+
+        clean_path = os.path.join(folder_path, cache_path, cache_csghub)
+        if os.path.exists(clean_path):
+            try:
+                shutil.rmtree(clean_path)
+            except Exception as e:
+                logging.error(f"failed to remove cache path: {e}")
     except KeyboardInterrupt:
         print("Terminated by Ctrl+C")
         os.kill(os.getpid(), signal.SIGTERM)

pycsghub/utils.py

@@ -14,7 +14,9 @@
 from pycsghub._token import _get_token_from_file, _get_token_from_environment
 from urllib.parse import quote, urlparse
 from pycsghub.constants import S3_INTERNAL
+import logging
 
+logger = logging.getLogger(__name__)
 
 def get_session() -> requests.Session:
     session = requests.Session()
@@ -221,6 +223,8 @@ def dataset_info(
     if files_metadata:
         params["blobs"] = True
     r = requests.get(path, headers=headers, timeout=timeout, params=params)
+    if r.status_code != 200:
+        logger.error(f"get dataset meta info from {path} response: {r.text}")
     r.raise_for_status()
     data = r.json()
     return DatasetInfo(**data)
@@ -282,6 +286,8 @@ def space_info(
     if files_metadata:
         params["blobs"] = True
     r = requests.get(path, headers=headers, timeout=timeout, params=params)
+    if r.status_code != 200:
+        logger.error(f"get space meta info from {path} response: {r.text}")
     r.raise_for_status()
     data = r.json()
     return SpaceInfo(**data)
@@ -349,6 +355,8 @@ def model_info(
     if files_metadata:
         params["blobs"] = True
     r = requests.get(path, headers=headers, timeout=timeout, params=params)
+    if r.status_code != 200:
+        logger.error(f"get model meta info from {path} response: {r.text}")
     r.raise_for_status()
     data = r.json()
     return ModelInfo(**data)

pyproject.toml

@@ -0,0 +1,59 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "csghub-sdk"
+version = "0.7.5"
+description = "CSGHub SDK for downloading and uploading models, datasets, and spaces"
+readme = "README.md"
+license = { text = "Apache-2.0" }
+authors = [
+    { name = "opencsg", email = "contact@opencsg.com" }
+]
+keywords = ["ai", "machine-learning", "models", "datasets", "huggingface"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+requires-python = ">=3.8,<3.14"
+dependencies = [
+    "typer",
+    "typing_extensions",
+    "huggingface_hub>=0.22.2",
+]
+
+[project.optional-dependencies]
+train = [
+    "torch",
+    "transformers>=4.33.3",
+    "datasets>=2.20.0"
+]
+
+[project.scripts]
+csghub-cli = "pycsghub.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/opencsg/csghub-sdk"
+Documentation = "https://github.com/opencsg/csghub-sdk"
+Repository = "https://github.com/opencsg/csghub-sdk"
+Issues = "https://github.com/opencsg/csghub-sdk/issues"
+
+[tool.setuptools.packages.find]
+include = ["pycsghub*"]
+
+[tool.setuptools.package-data]
+pycsghub = ["*"]
+
+[tool.setuptools.dynamic]
+version = { attr = "pycsghub.__version__" }