From c6c7b40b7794c6e554d9dcae7bf4930bba67e403 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 29 Apr 2026 16:09:20 -0400 Subject: [PATCH 1/3] Revert "DEP: de-duplicate and sort optional dependencies (#2021)" This reverts commit 3bc67f85c3b67c5adef11af35761fd839de306fb. --- pyproject.toml | 86 ++++++++++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1ce46e3fc..79fe1428d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,13 +24,34 @@ classifiers = [ ] [project.optional-dependencies] -# user-facing abfs = ["adlfs"] adl = ["adlfs"] arrow = ["pyarrow >= 1"] dask = ["dask", "distributed"] +dev = ["ruff >= 0.5", "pre-commit"] +doc = ["sphinx", "numpydoc", "sphinx-design", "sphinx-rtd-theme", "yarl"] dropbox = ["dropbox", "dropboxdrivefs", "requests"] entrypoints = [] +full = [ + 'adlfs', + 'aiohttp !=4.0.0a0, !=4.0.0a1', + 'dask', + 'distributed', + 'dropbox', + 'dropboxdrivefs', + 'fusepy', + 'gcsfs >2024.2.0', + 'libarchive-c', + 'ocifs', + 'panel', + 'paramiko', + 'pyarrow >= 1', + 'pygit2', + 'requests', + 's3fs >2024.2.0', + 'smbprotocol', + 'tqdm', +] fuse = ["fusepy"] gcs = ["gcsfs >2024.2.0"] git = ["pygit2"] @@ -45,42 +66,8 @@ s3 = ["s3fs >2024.2.0"] sftp = ["paramiko"] smb = ["smbprotocol"] ssh = ["paramiko"] -tqdm = ["tqdm"] -full = [ - 'fsspec[abfs]', - 'fsspec[adl]', - 'fsspec[arrow]', - 'fsspec[dask]', - 'fsspec[dropbox]', - 'fsspec[entrypoints]', - 'fsspec[fuse]', - 'fsspec[gcs]', - 'fsspec[git]', - 'fsspec[github]', - 'fsspec[gs]', - 'fsspec[gui]', - 'fsspec[hdfs]', - 'fsspec[http]', - 'fsspec[libarchive]', - 'fsspec[oci]', - 'fsspec[s3]', - 'fsspec[sftp]', - 'fsspec[smb]', - 'fsspec[ssh]', - 'fsspec[tqdm]', -] - -# dev-only -dev = ["ruff >= 0.5", "pre-commit"] -doc = [ - 'sphinx', - 'numpydoc', - 'sphinx-design', - 'sphinx-rtd-theme', - 'yarl', -] test = [ - "fsspec[http]", + "aiohttp!=4.0.0a0, !=4.0.0a1", "numpy", "pytest", "requests", @@ -92,20 +79,42 @@ test = [ "pytest-asyncio !=0.22.0", ] test_full = [ - 'fsspec[test]', - 'fsspec[full]', 'Jinja2', + 'adlfs', + 'aiohttp !=4.0.0a0, !=4.0.0a1', + 'aiohttp!=4.0.0a0, !=4.0.0a1', 'cloudpickle', + 'dask', + 'distributed', + 'dropbox', + 'dropboxdrivefs', 'fastparquet', + 'fusepy', + 'gcsfs', "kerchunk", + 'libarchive-c', 'lz4', 'notebook', 'numpy', + 'ocifs', 'pandas <3.0.0', 'panel', + 'paramiko', + 'pyarrow', + 'pyarrow >= 1', 'pyftpdlib', + 'pygit2', 'pytest', + 'pytest-asyncio !=0.22.0', + 'pytest-benchmark', + 'pytest-cov', + 'pytest-mock', + 'pytest-rerunfailures', + 'pytest-recording', + 'requests', + 'smbprotocol', 'python-snappy', + 'tqdm', 'urllib3', 'zarr', 'backports.zstd; python_version < "3.14"', @@ -118,6 +127,7 @@ test_downstream = [ "xarray", "aiobotocore>=2.5.4,<3.0.0", ] +tqdm = ["tqdm"] [project.urls] Changelog = "https://filesystem-spec.readthedocs.io/en/latest/changelog.html" From a9584a1f6cc4ee72dd0e32a59fb062fae6987dbf Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 1 May 2026 16:09:59 -0400 Subject: [PATCH 2/3] Honout simple_templates --- fsspec/implementations/reference.py | 5 +++++ fsspec/implementations/tests/test_reference.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/fsspec/implementations/reference.py b/fsspec/implementations/reference.py index 562e76160..3148b1f8a 100644 --- a/fsspec/implementations/reference.py +++ b/fsspec/implementations/reference.py @@ -615,6 +615,9 @@ class ReferenceFileSystem(AsyncFileSystem): Reference dict format: {path0: bytes_data, path1: (target_url, offset, size)} https://github.com/fsspec/kerchunk/blob/main/README.md + + simple_references: if True (default), no jinja interpreting is done, + which is the safe option. """ protocol = "reference" @@ -1056,6 +1059,8 @@ def _process_templates(self, tmp): def _process_gen(self, gens): out = {} + if self.simple_templates: + return out for gen in gens: dimension = { k: ( diff --git a/fsspec/implementations/tests/test_reference.py b/fsspec/implementations/tests/test_reference.py index 52d39fb3a..7d55eb518 100644 --- a/fsspec/implementations/tests/test_reference.py +++ b/fsspec/implementations/tests/test_reference.py @@ -961,3 +961,20 @@ def test_parquet_no_references(m): lz.flush() assert arr[...].tolist() == 1 # scalar, equal to fill value + + +def test_no_default_jinja_execution(): + manifest = { + "version": 1, + "templates": {}, + "refs": {}, + "gen": [ + {"key": "{{ 1 / 0 }}", "url": "file:///dev/null", "dimensions": {"i": [0]}} + ], + } + + # no error - not evaluated + fsspec.filesystem("reference", fo=manifest) + + with pytest.raises(ZeroDivisionError): + fsspec.filesystem("reference", fo=manifest, simple_templates=False) From 84c72d81b0dea31c2c60dda9672d2dc53d33603a Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 1 May 2026 16:24:18 -0400 Subject: [PATCH 3/3] more --- .../implementations/tests/test_reference.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/fsspec/implementations/tests/test_reference.py b/fsspec/implementations/tests/test_reference.py index 7d55eb518..fd020a1c7 100644 --- a/fsspec/implementations/tests/test_reference.py +++ b/fsspec/implementations/tests/test_reference.py @@ -365,7 +365,12 @@ def test_spec1_gen_variants(): }, ], } - fsspec.filesystem("reference", fo=missing_length_spec, target_protocol="http") + fsspec.filesystem( + "reference", + fo=missing_length_spec, + simple_templates=False, + target_protocol="http", + ) with pytest.raises(ValueError): missing_offset_spec = { @@ -380,7 +385,12 @@ def test_spec1_gen_variants(): }, ], } - fsspec.filesystem("reference", fo=missing_offset_spec, target_protocol="http") + fsspec.filesystem( + "reference", + simple_templates=False, + fo=missing_offset_spec, + target_protocol="http", + ) url_only_gen_spec = { "version": 1, @@ -394,7 +404,12 @@ def test_spec1_gen_variants(): ], } - fs = fsspec.filesystem("reference", fo=url_only_gen_spec, target_protocol="http") + fs = fsspec.filesystem( + "reference", + simple_templates=False, + fo=url_only_gen_spec, + target_protocol="http", + ) assert fs.references == { "gen_key0": ["http://server.domain/path_0"], "gen_key1": ["http://server.domain/path_1"], @@ -964,6 +979,7 @@ def test_parquet_no_references(m): def test_no_default_jinja_execution(): + pytest.importorskip("jinja2") manifest = { "version": 1, "templates": {},