subcommand: add new subsystem/subcommand - storage

Add new subcommand to use KernelCI storage.
Useful to store logs, artifacts and etc.

Signed-off-by: Denys Fedoryshchenko <denys.f@collabora.com>

Author: nuclearcat

kcidev/_data/kci-dev.toml.example

@@ -11,10 +11,14 @@ api="https://staging.kernelci.org:9000/"
 token="example"
 kcidb_rest_url="https://staging.kcidb.kernelci.org/submit"
 kcidb_token="your-kcidb-token-here"
+storage_url="https://files-staging.kernelci.org"
+storage_token="your-storage-jwt-token"
 
 [production]
 pipeline="https://kernelci-pipeline.westus3.cloudapp.azure.com/"
 api="https://kernelci-api.westus3.cloudapp.azure.com/"
 token="example"
 kcidb_rest_url="https://db.kernelci.org/submit"
-kcidb_token="your-kcidb-token-here"
+kcidb_token="your-kcidb-token-here"
+storage_url="https://files.kernelci.org"
+storage_token="your-storage-jwt-token"

kcidev/libs/common.py

@@ -78,7 +78,7 @@ def load_toml(settings, subcommand):
         return config
 
     # config and results subcommand work without a config file
-    if subcommand not in ("config", "results", "submit"):
+    if subcommand not in ("config", "results", "submit", "storage"):
         if not config:
             logging.warning(f"No config file found for subcommand {subcommand}")
             kci_err(

kcidev/libs/storage.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import logging
+import os
+
+import click
+import requests
+
+from kcidev.libs.common import kci_err, kci_msg, kcidev_session
+
+
+def resolve_storage_config(cfg, instance, cli_url, cli_token):
+    """
+    Resolve storage URL and token. Priority:
+    1. CLI flags (--storage-url, --storage-token)
+    2. Environment variables (KCI_STORAGE_URL, KCI_STORAGE_TOKEN)
+    3. Instance config (storage_url, storage_token in TOML)
+
+    Returns (storage_url, token) tuple.
+    Raises click.Abort if no valid credentials found.
+    """
+    # 1. CLI flags
+    if cli_url or cli_token:
+        if not cli_url or not cli_token:
+            kci_err("Both --storage-url and --storage-token must be provided together")
+            raise click.Abort()
+        logging.debug(f"Using storage config from CLI flags: {cli_url}")
+        return cli_url, cli_token
+
+    # 2. Environment variables
+    env_url = os.environ.get("KCI_STORAGE_URL")
+    env_token = os.environ.get("KCI_STORAGE_TOKEN")
+    if env_url or env_token:
+        if env_url and env_token:
+            logging.debug(f"Using storage config from env vars: {env_url}")
+            return env_url, env_token
+        kci_err(
+            "Both KCI_STORAGE_URL and KCI_STORAGE_TOKEN env vars must be set together"
+        )
+        raise click.Abort()
+
+    # 3. Instance config
+    if cfg and instance and instance in cfg:
+        inst_cfg = cfg[instance]
+        storage_url = inst_cfg.get("storage_url")
+        storage_token = inst_cfg.get("storage_token")
+        if storage_url and storage_token:
+            logging.debug(
+                f"Using storage config from instance '{instance}': {storage_url}"
+            )
+            return storage_url, storage_token
+
+    kci_err(
+        "No storage credentials found. Provide --storage-url and --storage-token, "
+        "set KCI_STORAGE_URL/KCI_STORAGE_TOKEN env vars, "
+        "or configure storage_url/storage_token in config file"
+    )
+    raise click.Abort()
+
+
+def upload_file(storage_url, token, remote_path, local_file_path, timeout=120):
+    """
+    Upload a file to kernelci-storage.
+
+    POST /v1/file with multipart form:
+      - path: remote directory path
+      - file0: file content
+    """
+    url = storage_url.rstrip("/") + "/v1/file"
+    headers = {
+        "Authorization": f"Bearer {token}",
+    }
+
+    logging.info(f"Uploading {local_file_path} to {remote_path}/")
+    logging.debug(f"POST request to: {url}")
+
+    try:
+        with open(local_file_path, "rb") as f:
+            files = {"file0": (os.path.basename(local_file_path), f)}
+            data = {"path": remote_path}
+            response = kcidev_session.post(
+                url, headers=headers, files=files, data=data, timeout=timeout
+            )
+        logging.debug(f"Upload response status: {response.status_code}")
+    except requests.exceptions.RequestException as e:
+        logging.error(f"Upload request failed: {e}")
+        kci_err(f"Storage connection error: {e}")
+        raise click.Abort()
+
+    if response.status_code == 200:
+        logging.info(f"Upload successful: {os.path.basename(local_file_path)}")
+        return response.text
+    elif response.status_code == 401:
+        kci_err("Authentication failed: invalid or expired token")
+        raise click.Abort()
+    else:
+        kci_err(f"Upload failed (HTTP {response.status_code}): {response.text}")
+        raise click.Abort()
+
+
+def check_auth(storage_url, token, timeout=30):
+    """
+    Validate a JWT token against the storage server.
+
+    GET /v1/checkauth
+    Returns the response text on success.
+    """
+    url = storage_url.rstrip("/") + "/v1/checkauth"
+    headers = {
+        "Authorization": f"Bearer {token}",
+    }
+
+    logging.info("Checking storage authentication")
+    logging.debug(f"GET request to: {url}")
+
+    try:
+        response = kcidev_session.get(url, headers=headers, timeout=timeout)
+        logging.debug(f"Auth check response status: {response.status_code}")
+    except requests.exceptions.RequestException as e:
+        logging.error(f"Auth check request failed: {e}")
+        kci_err(f"Storage connection error: {e}")
+        raise click.Abort()
+
+    if response.status_code == 200:
+        logging.info(f"Authentication valid: {response.text}")
+        return response.text
+    elif response.status_code == 401:
+        kci_err("Authentication failed: invalid or expired token")
+        raise click.Abort()
+    else:
+        kci_err(f"Auth check failed (HTTP {response.status_code}): {response.text}")
+        raise click.Abort()

kcidev/main.py

@@ -13,6 +13,7 @@
     config,
     maestro,
     results,
+    storage,
     submit,
     testretry,
     watch,
@@ -44,7 +45,7 @@ def cli(ctx, settings, instance, debug):
     if subcommand not in ("results", "config"):
         if instance:
             ctx.obj["INSTANCE"] = instance
-        elif subcommand != "submit":
+        elif subcommand not in ("submit", "storage"):
             ctx.obj["INSTANCE"] = ctx.obj["CFG"].get("default_instance")
             fconfig = config_path(settings)
             if not ctx.obj["INSTANCE"]:
@@ -63,6 +64,7 @@ def run():
     cli.add_command(maestro.maestro)
     cli.add_command(testretry.testretry)
     cli.add_command(results.results)
+    cli.add_command(storage.storage)
     cli.add_command(submit.submit)
     cli.add_command(watch.watch)
     cli()

kcidev/subcommands/storage.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+
+import click
+
+from kcidev.libs.common import kci_err, kci_msg
+from kcidev.libs.storage import check_auth, resolve_storage_config, upload_file
+
+
+@click.group(
+    help="""Interact with KernelCI storage.
+
+This command group provides access to KernelCI storage operations including
+uploading files and validating authentication tokens.
+
+\b
+Examples:
+  # Upload a file (path recommended to match your origin)
+  kci-dev storage upload --path myci/build-123 ./some-files.tar.xz
+  # Upload multiple files
+  kci-dev storage upload --path myci/build-123 log.txt config.gz
+  # Validate authentication token
+  kci-dev storage checkauth
+""",
+    invoke_without_command=True,
+)
+@click.pass_context
+def storage(ctx):
+    """Commands related to KernelCI storage."""
+    cfg = ctx.obj.get("CFG")
+    if cfg:
+        instance = ctx.obj.get("INSTANCE")
+        if not instance:
+            instance = cfg.get("default_instance")
+        ctx.obj["INSTANCE"] = instance
+
+    if ctx.invoked_subcommand is None:
+        click.echo(ctx.get_help())
+        sys.exit(0)
+
+
+@storage.command()
+@click.option(
+    "--storage-url",
+    help="Storage server URL (overrides config and env)",
+)
+@click.option(
+    "--storage-token",
+    help="Storage JWT token (overrides config and env)",
+)
+@click.option(
+    "--path",
+    "remote_path",
+    required=True,
+    help="Remote directory path, recommended to match your origin (e.g. origin/build-123)",
+)
+@click.argument(
+    "files",
+    nargs=-1,
+    required=True,
+    type=click.Path(exists=True),
+)
+@click.pass_context
+def upload(ctx, storage_url, storage_token, remote_path, files):
+    """Upload files to KernelCI storage."""
+    cfg = ctx.obj.get("CFG")
+    instance = ctx.obj.get("INSTANCE")
+
+    url, token = resolve_storage_config(cfg, instance, storage_url, storage_token)
+
+    for file_path in files:
+        if os.path.isdir(file_path):
+            kci_err(f"Skipping directory: {file_path}")
+            continue
+        filename = os.path.basename(file_path)
+        kci_msg(f"Uploading {filename} to {remote_path}/...")
+        result = upload_file(url, token, remote_path, file_path)
+        kci_msg(f"Uploaded: {filename} - {result}")
+
+
+@storage.command()
+@click.option(
+    "--storage-url",
+    help="Storage server URL (overrides config and env)",
+)
+@click.option(
+    "--storage-token",
+    help="Storage JWT token (overrides config and env)",
+)
+@click.pass_context
+def checkauth(ctx, storage_url, storage_token):
+    """Validate storage authentication token."""
+    cfg = ctx.obj.get("CFG")
+    instance = ctx.obj.get("INSTANCE")
+
+    url, token = resolve_storage_config(cfg, instance, storage_url, storage_token)
+
+    result = check_auth(url, token)
+    kci_msg(result)