diff --git a/fsspec/implementations/reference.py b/fsspec/implementations/reference.py index 562e76160..3148b1f8a 100644 --- a/fsspec/implementations/reference.py +++ b/fsspec/implementations/reference.py @@ -615,6 +615,9 @@ class ReferenceFileSystem(AsyncFileSystem): Reference dict format: {path0: bytes_data, path1: (target_url, offset, size)} https://github.com/fsspec/kerchunk/blob/main/README.md + + simple_references: if True (default), no jinja interpreting is done, + which is the safe option. """ protocol = "reference" @@ -1056,6 +1059,8 @@ def _process_templates(self, tmp): def _process_gen(self, gens): out = {} + if self.simple_templates: + return out for gen in gens: dimension = { k: ( diff --git a/fsspec/implementations/tests/test_reference.py b/fsspec/implementations/tests/test_reference.py index 52d39fb3a..fd020a1c7 100644 --- a/fsspec/implementations/tests/test_reference.py +++ b/fsspec/implementations/tests/test_reference.py @@ -365,7 +365,12 @@ def test_spec1_gen_variants(): }, ], } - fsspec.filesystem("reference", fo=missing_length_spec, target_protocol="http") + fsspec.filesystem( + "reference", + fo=missing_length_spec, + simple_templates=False, + target_protocol="http", + ) with pytest.raises(ValueError): missing_offset_spec = { @@ -380,7 +385,12 @@ def test_spec1_gen_variants(): }, ], } - fsspec.filesystem("reference", fo=missing_offset_spec, target_protocol="http") + fsspec.filesystem( + "reference", + simple_templates=False, + fo=missing_offset_spec, + target_protocol="http", + ) url_only_gen_spec = { "version": 1, @@ -394,7 +404,12 @@ def test_spec1_gen_variants(): ], } - fs = fsspec.filesystem("reference", fo=url_only_gen_spec, target_protocol="http") + fs = fsspec.filesystem( + "reference", + simple_templates=False, + fo=url_only_gen_spec, + target_protocol="http", + ) assert fs.references == { "gen_key0": ["http://server.domain/path_0"], "gen_key1": ["http://server.domain/path_1"], @@ -961,3 +976,21 @@ def test_parquet_no_references(m): lz.flush() assert arr[...].tolist() == 1 # scalar, equal to fill value + + +def test_no_default_jinja_execution(): + pytest.importorskip("jinja2") + manifest = { + "version": 1, + "templates": {}, + "refs": {}, + "gen": [ + {"key": "{{ 1 / 0 }}", "url": "file:///dev/null", "dimensions": {"i": [0]}} + ], + } + + # no error - not evaluated + fsspec.filesystem("reference", fo=manifest) + + with pytest.raises(ZeroDivisionError): + fsspec.filesystem("reference", fo=manifest, simple_templates=False)