Merge branch 'main' into feat/add-missing-registration-methods

Author: timsaucer

.ai/skills/check-upstream/SKILL.md

@@ -0,0 +1 @@
+../.ai/skills

.claude/skills

@@ -159,6 +159,19 @@ jobs:
         with:
           enable-cache: true
 
+      - name: Add extra swap for release build
+        if: inputs.build_mode == 'release'
+        run: |
+          set -euxo pipefail
+          sudo swapoff -a || true
+          sudo rm -f /swapfile
+          sudo fallocate -l 8G /swapfile || sudo dd if=/dev/zero of=/swapfile bs=1M count=8192
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          free -h
+          swapon --show
+
       - name: Build (release mode)
         uses: PyO3/maturin-action@v1
         if: inputs.build_mode == 'release'
@@ -233,7 +246,7 @@ jobs:
           set -euxo pipefail
           sudo swapoff -a || true
           sudo rm -f /swapfile
-          sudo fallocate -l 16G /swapfile || sudo dd if=/dev/zero of=/swapfile bs=1M count=16384
+          sudo fallocate -l 8G /swapfile || sudo dd if=/dev/zero of=/swapfile bs=1M count=8192
           sudo chmod 600 /swapfile
           sudo mkswap /swapfile
           sudo swapon /swapfile

.github/workflows/build.yml

@@ -0,0 +1,56 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Agent Instructions
+
+This project uses AI agent skills stored in `.ai/skills/`. Each skill is a directory containing a `SKILL.md` file with instructions for performing a specific task.
+
+Skills follow the [Agent Skills](https://agentskills.io) open standard. Each skill directory contains:
+
+- `SKILL.md` — The skill definition with YAML frontmatter (name, description, argument-hint) and detailed instructions.
+- Additional supporting files as needed.
+
+## Python Function Docstrings
+
+Every Python function must include a docstring with usage examples.
+
+- **Examples are required**: Each function needs at least one doctest-style example
+  demonstrating basic usage.
+- **Optional parameters**: If a function has optional parameters, include separate
+  examples that show usage both without and with the optional arguments. Pass
+  optional arguments using their keyword name (e.g., `step=dfn.lit(3)`) so readers
+  can immediately see which parameter is being demonstrated.
+- **Reuse input data**: Use the same input data across examples wherever possible.
+  The examples should demonstrate how different optional arguments change the output
+  for the same input, making the effect of each option easy to understand.
+- **Alias functions**: Functions that are simple aliases (e.g., `list_sort` aliasing
+  `array_sort`) only need a one-line description and a `See Also` reference to the
+  primary function. They do not need their own examples.
+
+## Aggregate and Window Function Documentation
+
+When adding or updating an aggregate or window function, ensure the corresponding
+site documentation is kept in sync:
+
+- **Aggregations**: `docs/source/user-guide/common-operations/aggregations.rst` —
+  add new aggregate functions to the "Aggregate Functions" list and include usage
+  examples if appropriate.
+- **Window functions**: `docs/source/user-guide/common-operations/windows.rst` —
+  add new window functions to the "Available Functions" list and include usage
+  examples if appropriate.

AGENTS.md

@@ -0,0 +1 @@
+AGENTS.md

CLAUDE.md

@@ -64,8 +64,8 @@ pyo3-build-config = "0.28"
 datafusion-python-util = { path = "crates/util" }
 
 [profile.release]
-lto = true
-codegen-units = 1
+lto = "thin"
+codegen-units = 2
 
 # We cannot publish to crates.io with any patches in the below section. Developers
 # must remove any entries in this section before creating a release candidate.

Cargo.toml

@@ -275,7 +275,7 @@ needing to activate the virtual environment:
 
 ```bash
 uv run --no-project maturin develop --uv
-uv run --no-project pytest .
+uv run --no-project pytest
 ```
 
 To run the FFI tests within the examples folder, after you have built
@@ -312,6 +312,33 @@ There are scripts in `ci/scripts` for running Rust and Python linters.
 ./ci/scripts/rust_toml_fmt.sh
 ```
 
+## Checking Upstream DataFusion Coverage
+
+This project includes an [AI agent skill](.ai/skills/check-upstream/SKILL.md) for auditing which
+features from the upstream Apache DataFusion Rust library are not yet exposed in these Python
+bindings. This is useful when adding missing functions, auditing API coverage, or ensuring parity
+with upstream.
+
+The skill accepts an optional area argument:
+
+```
+scalar functions
+aggregate functions
+window functions
+dataframe
+session context
+ffi types
+all
+```
+
+If no argument is provided, it defaults to checking all areas. The skill will fetch the upstream
+DataFusion documentation, compare it against the functions and methods exposed in this project, and
+produce a coverage report listing what is currently exposed and what is missing.
+
+The skill definition lives in `.ai/skills/check-upstream/SKILL.md` and follows the
+[Agent Skills](https://agentskills.io) open standard. It can be used by any AI coding agent that
+supports skill discovery, or followed manually.
+
 ## How to update dependencies
 
 To change test dependencies, change the `pyproject.toml` and run

README.md

@@ -434,11 +434,25 @@ impl PySessionContext {
             &upstream_host
         };
         let url_string = format!("{scheme}{derived_host}");
-        let url = Url::parse(&url_string).unwrap();
+        let url = Url::parse(&url_string).map_err(|e| PyValueError::new_err(e.to_string()))?;
         self.ctx.runtime_env().register_object_store(&url, store);
         Ok(())
     }
 
+    /// Deregister an object store with the given url
+    #[pyo3(signature = (scheme, host=None))]
+    pub fn deregister_object_store(
+        &self,
+        scheme: &str,
+        host: Option<&str>,
+    ) -> PyDataFusionResult<()> {
+        let host = host.unwrap_or("");
+        let url_string = format!("{scheme}{host}");
+        let url = Url::parse(&url_string).map_err(|e| PyDataFusionError::Common(e.to_string()))?;
+        self.ctx.runtime_env().deregister_object_store(&url)?;
+        Ok(())
+    }
+
     #[allow(clippy::too_many_arguments)]
     #[pyo3(signature = (name, path, table_partition_cols=vec![],
     file_extension=".parquet",
@@ -492,6 +506,10 @@ impl PySessionContext {
         self.ctx.register_udtf(&name, func);
     }
 
+    pub fn deregister_udtf(&self, name: &str) {
+        self.ctx.deregister_udtf(name);
+    }
+
     #[pyo3(signature = (query, options=None, param_values=HashMap::default(), param_strings=HashMap::default()))]
     pub fn sql_with_options(
         &self,
@@ -1008,16 +1026,28 @@ impl PySessionContext {
         Ok(())
     }
 
+    pub fn deregister_udf(&self, name: &str) {
+        self.ctx.deregister_udf(name);
+    }
+
     pub fn register_udaf(&self, udaf: PyAggregateUDF) -> PyResult<()> {
         self.ctx.register_udaf(udaf.function);
         Ok(())
     }
 
+    pub fn deregister_udaf(&self, name: &str) {
+        self.ctx.deregister_udaf(name);
+    }
+
     pub fn register_udwf(&self, udwf: PyWindowUDF) -> PyResult<()> {
         self.ctx.register_udwf(udwf.function);
         Ok(())
     }
 
+    pub fn deregister_udwf(&self, name: &str) {
+        self.ctx.deregister_udwf(name);
+    }
+
     #[pyo3(signature = (name="datafusion"))]
     pub fn catalog(&self, py: Python, name: &str) -> PyResult<Py<PyAny>> {
         let catalog = self.ctx.catalog(name).ok_or(PyKeyError::new_err(format!(