Support python 3.8~3.13 (#87)

* fix: update model repo name in README

Updated model repo name from 'wanghh2003/model05' to 'wanghh2000/model05' in README files.

- Changed repo name in README.md
- Changed repo name in README_cn.md
- Updated repo_type usage in cli.py
- Renamed upload command in cli.py
- Adjusted token handling in repo.py
- Bumped version to 0.6.2 in setup.py

* feat: update README and CLI commands

- Added installation instructions for train dependencies
- Updated example commands to use new model names
- Adjusted roadmap items for clarity
- Refactored type hints in CLI and upload modules
- Updated package version to 0.7.0
- Modified install_requires for cleaner dependencies
- Changed python_requires to support 3.8 to 3.14

Author: HaiHui886

README.md

@@ -38,6 +38,9 @@ To get started with the CSGHub SDK, ensure you have Python installed on your sys
 
 ```python
 pip install csghub-sdk
+
+# install with train dependencies
+pip install "csghub-sdk[train]"
 ```
 
 After installation, you can begin using the SDK to connect to your CSGHub server by importing it into your Python script:
@@ -86,16 +89,16 @@ pip install '.[train]'
 export CSGHUB_TOKEN=your_access_token
 
 # download model
-csghub-cli download wanghh2000/myprivate1
+csghub-cli download OpenCSG/csg-wukong-1B
 
 # download model with allow patterns '*.json' and ignore '*_config.json' pattern of files
-csghub-cli download wanghh2000/myprivate1 --allow-patterns "*.json" --ignore-patterns "*_config.json"
+csghub-cli download OpenCSG/csg-wukong-1B --allow-patterns "*.json" --ignore-patterns "*_config.json"
 
 # download dataset
-csghub-cli download wanghh2000/myds1 -t dataset
+csghub-cli download OpenCSG/GitLab-DataSets-V1 -t dataset
 
 # download space
-csghub-cli download wanghh2000/space1 -t space
+csghub-cli download OpenCSG/csg-wukong-1B -t space
 
 # upload a single file to folder1
 csghub-cli upload wanghh2000/myprivate1 abc/3.txt folder1
@@ -130,8 +133,8 @@ csghub-cli finetune start OpenCSG/csg-wukong-1B 326
 # stop fine-tuning instance for model repo 'OpenCSG/csg-wukong-1B' with ID '326'
 csghub-cli finetune stop OpenCSG/csg-wukong-1B 326
 
-# upload local large folder '/Users/hhwang/temp/abc' to model repo 'wanghh2003/model05'
-csghub-cli upload-large-folder wanghh2003/model05 /Users/hhwang/temp/abc
+# upload local large folder '/Users/hhwang/temp/abc' to model repo 'wanghh2000/model05'
+csghub-cli upload-large-folder wanghh2000/model05 /Users/hhwang/temp/abc
 ```
 
 Notes: `csghub-cli upload` will create repo and its branch if they do not exist. The default branch is `main`. If you want to upload to a specific branch, you can use the `--revision` option. If the branch does not exist, it will be created. If the branch already exists, the files will be uploaded to that branch. 
@@ -310,9 +313,7 @@ This code:
 
 ## Roadmap
 
-1. Download repo file by CLI
-2. Interacting with CSGHub via command-line tools
-3. Management operations such as creation and modification of CSGHub repositories
-4. Model deployment locally or online
-5. Model fine-tuning locally or online
-6. Quick upload large folder to CSGHub
+1. Interacting with CSGHub via command-line tools
+2. Management operations such as creation and modification of CSGHub repositories
+3. Model deployment locally or online
+4. Model fine-tuning locally or online

README_cn.md

@@ -38,6 +38,9 @@ CSGHub SDK 是一个强大的 Python 客户端，专门设计用于与 CSGHub 
 
 ```python
 pip install csghub-sdk
+
+# 使用pip安装训练相关的依赖
+pip install "csghub-sdk[train]"
 ```
 
 安装后，您可以开始将 SDK 导入到您的 Python 脚本中，以连接到您的 CSGHub 服务器：
@@ -85,16 +88,16 @@ pip install '.[train]'
 export CSGHUB_TOKEN=your_access_token
 
 # 模型下载
-csghub-cli download wanghh2000/myprivate1 
+csghub-cli download OpenCSG/csg-wukong-1B
 
 # 模型下载时允许'*.json'模式的文件并忽略'*_config.json'模式的文件
-csghub-cli download wanghh2000/myprivate1 --allow-patterns "*.json" --ignore-patterns "*_config.json"
+csghub-cli download OpenCSG/csg-wukong-1B --allow-patterns "*.json" --ignore-patterns "*_config.json"
 
 # 数据集下载
-csghub-cli download wanghh2000/myds1 -t dataset
+csghub-cli download OpenCSG/chinese-fineweb-edu-v2 -t dataset
 
 # 应用下载
-csghub-cli download wanghh2000/space1 -t space
+csghub-cli download OpenCSG/csg-wukong-1B -t space
 
 # 上传单个文件到仓库目录folder1
 csghub-cli upload wanghh2000/myprivate1 abc/3.txt folder1
@@ -129,8 +132,8 @@ csghub-cli finetune start OpenCSG/csg-wukong-1B 326
 # 停止ID为326使用模型OpenCSG/csg-wukong-1B的微调实例
 csghub-cli finetune stop OpenCSG/csg-wukong-1B 326
 
-# 上传本地目录/Users/hhwang/temp/abc中的所有文件到远程仓库wanghh2003/model05
-csghub-cli upload-large-folder wanghh2003/model05 /Users/hhwang/temp/abc
+# 上传本地目录/Users/hhwang/temp/abc中的所有文件到远程仓库wanghh2000/model05
+csghub-cli upload-large-folder wanghh2000/model05 /Users/hhwang/temp/abc
 ```
 
 注意：csghub-cli upload 将在仓库和分支不存在时创建它们。默认分支为main。如果您想上传到特定分支，可以使用 --revision 选项。如果该分支不存在，将会被创建。如果分支已存在，文件将上传到该分支。
@@ -312,9 +315,7 @@ model = AutoModelForCausalLM.from_pretrained('model/repoid')
 
 ## 功能计划
 
-1. 使用命令行工具下载仓库文件
-2. 使用命令行工具的方式与CSGHub交互
-3. CSGHub仓库的创建、修改等管理操作
-4. 模型本地或在线部署
-5. 模型本地或在线微调
-6. 快速上传大文件夹到CSGHub仓库
+1. 使用命令行工具的方式与CSGHub交互
+2. CSGHub仓库的创建、修改等管理操作
+3. 模型本地或在线部署
+4. 模型本地或在线微调

pycsghub/cli.py

@@ -1,7 +1,8 @@
 import typer
 import os
 import logging
-from typing import Annotated, List, Optional
+from typing import List, Optional
+from typing_extensions import Annotated
 from pycsghub.cmd import repo, inference, finetune
 from pycsghub.cmd.repo_types import RepoType
 from importlib.metadata import version
@@ -58,7 +59,7 @@ def download(
     ):
     repo.download(
         repo_id=repo_id,
-        repo_type=repo_type, 
+        repo_type=repo_type.value, 
         revision=revision, 
         cache_dir=cache_dir,
         endpoint=endpoint,
@@ -67,7 +68,7 @@ def download(
         ignore_patterns=ignore_patterns,
     )
 
-@app.command(name="upload-files", help="Upload repository files to OpenCSG Hub", no_args_is_help=True)
+@app.command(name="upload", help="Upload repository files to OpenCSG Hub", no_args_is_help=True)
 def upload(
         repo_id: Annotated[str, OPTIONS["repoID"]],
         local_path: Annotated[str, OPTIONS["localPath"]],
@@ -82,7 +83,7 @@ def upload(
     if os.path.isfile(local_path):
         repo.upload_files(
             repo_id=repo_id, 
-            repo_type=repo_type, 
+            repo_type=repo_type.value, 
             repo_file=local_path,
             path_in_repo=path_in_repo,
             revision=revision, 
@@ -93,7 +94,7 @@ def upload(
     else:
         repo.upload_folder(
             repo_id=repo_id, 
-            repo_type=repo_type,
+            repo_type=repo_type.value,
             local_path=local_path,
             path_in_repo=path_in_repo,
             revision=revision,
@@ -119,7 +120,7 @@ def upload_large_folder(
     upload_large_folder_internal(
         repo_id=repo_id,
         local_path=local_path,
-        repo_type=repo_type,
+        repo_type=repo_type.value,
         revision=revision,
         endpoint=endpoint,
         token=token,

pycsghub/cmd/repo.py

@@ -3,8 +3,8 @@
 from pathlib import Path
 from typing import Optional, Union, List
 from pycsghub.constants import DEFAULT_REVISION
-import requests
 from pycsghub.repository import Repository
+from pycsghub.utils import get_token_to_send
 
 def download(
         repo_id: str,
@@ -73,7 +73,7 @@ def upload_folder(
         branch_name=revision,
         endpoint=endpoint,
         user_name=user_name,
-        token=token,
+        token=get_token_to_send(token),
         auto_create=auto_create,
     )
     r.upload()

pycsghub/repo_reader/model/huggingface/model_auto.py

@@ -11,8 +11,7 @@ def from_pretrained(cls, pretrained_model_name_or_path,
     try:
         token = get_token_to_send(None)
     except Exception:
-        token = ("f3a7b9c1d6e5f8e2a1b5d4f9e6a2b8d7c3a4e2b1d9f6e7a8d2c5a7b4c1e3f5b8a1d4"
-                 "f9b7d6e2f8a5d3b1e7f9c6a8b2d1e4f7d5b6e9f2a4b3c8e1d7f995hd82hf")
+        token = None
     if os.path.isdir(pretrained_model_name_or_path):
         path = Path(pretrained_model_name_or_path)
     else:

pycsghub/upload_large_folder/status.py

@@ -164,7 +164,7 @@ def is_done(self) -> bool:
         with self.lock:
             return all(metadata.is_committed or metadata.should_ignore for _, metadata in self.items)
 
-    def get_lfs_uploaded_slice_ids(self, file_path: str) -> str|None:
+    def get_lfs_uploaded_slice_ids(self, file_path: str) -> str:
         with self.lock:
             return self._lfs_uploaded_ids.get(file_path)
 
@@ -182,7 +182,7 @@ def append_lfs_uploaded_slice_id(self, file_path: str, id: int, etag: str):
                     new_ids = old_ids
             self._lfs_uploaded_ids[file_path] = new_ids
 
-    def convert_uploaded_ids_to_map(self, ids: str | None):
+    def convert_uploaded_ids_to_map(self, ids: str):
         id_map = {}
         if ids is None or ids == "":
             return id_map

pycsghub/upload_large_folder/workers.py

@@ -422,7 +422,7 @@ def _perform_lfs_slice_upload(item: JOB_ITEM_T):
     etag = resp_header.get("etag")
     if etag is None or etag == "":
         raise ValueError(f"invalid slice upload response header: {resp_header}, etag: {etag}")
-    return etag.removeprefix('"').removesuffix('"')
+    return etag.strip('"')
 
 def _commit(
     items: List[JOB_ITEM_T],

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name='csghub-sdk',
-    version='0.6.1',
+    version='0.7.0',
     author="opencsg",
     author_email="contact@opencsg.com",
     long_description=long_description,
@@ -18,29 +18,9 @@
         ]
     },
     install_requires=[
-        "typer>=0.12.3",
-        "attr>=0.3.2",
-        "ConfigParser>=7.0.0",
-        "contextlib2>=21.6.0",
-        "cryptography>=43.0.1",
-        "Cython>=3.0.10",
-        "dl>=0.1.0",
+        "typer",
+        "typing_extensions",
         "huggingface_hub>=0.22.2",
-        "ipython>=8.12.3",
-        "ipywidgets>=8.1.2",
-        "keyring>=25.2.1",
-        "lockfile>=0.12.2",
-        "mock>=5.1.0",
-        "Pillow>=10.3.0",
-        "protobuf>=4.25.1",
-        "pyOpenSSL>=24.1.0",
-        "railroad>=0.5.0",
-        "thread>=2.0.3",
-        "tornado>=6.4.1",
-        "tqdm>=4.66.3",
-        "trove_classifiers>=2024.5.22",
-        "truststore>=0.9.1",
-        "urllib3_secure_extra>=0.1.0",
     ],
     extras_require={
         "train": [
@@ -49,5 +29,5 @@
             "datasets>=2.20.0"
         ],
     },
-    python_requires=">=3.10",
+    python_requires=">=3.8,<3.14",
 )