diff --git a/README.md b/README.md index 03ca7af..6fecc88 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,11 @@ src/artifactapi/ | Method | Path | Description | |---|---|---| | `GET` | `/api/v1/remote/{remote}/{path}` | Fetch artifact (auto-cache on miss) | -| `PUT` | `/api/v1/remote/{remote}/{path}` | Upload to local remote | -| `HEAD` | `/api/v1/remote/{remote}/{path}` | Check existence (local remotes) | -| `DELETE` | `/api/v1/remote/{remote}/{path}` | Delete from local remote | | `GET` | `/api/v1/virtual/{virtual}/{path}` | Fetch from virtual (merged) repository | +| `GET` | `/api/v1/local/{local}/{path}` | Download from local repository | +| `PUT` | `/api/v1/local/{local}/{path}` | Upload to local repository | +| `HEAD` | `/api/v1/local/{local}/{path}` | Check existence (local) | +| `DELETE` | `/api/v1/local/{local}/{path}` | Delete from local repository | | `GET` | `/v2/{remote}/{path}` | Docker Registry v2 proxy | | `PUT` | `/cache/flush` | Flush cache entries | | `GET` | `/health` | Health check | @@ -120,13 +121,14 @@ config_dir: conf.d # or an absolute path remotes: {} # optional base remotes ``` -### remotes.yaml Structure +### Configuration structure + +Repositories are declared under three top-level keys matching their type: ```yaml -remotes: +remotes: # proxy (caching) remotes remote-name: base_url: "https://example.com" - type: "remote" # "remote", "local", or "virtual" package: "generic" # generic, alpine, rpm, docker, pypi, npm, helm description: "..." immutable_patterns: # regex — cached forever @@ -137,6 +139,20 @@ remotes: cache: immutable_ttl: 0 # 0 = indefinitely mutable_ttl: 3600 + +virtuals: # virtual (merged-index) repositories + virtual-name: + package: "helm" + members: + - remote-a + - remote-b + +locals: # local upload repositories (no base_url) + local-name: + package: "generic" + cache: + immutable_ttl: 0 + mutable_ttl: 0 ``` ## Remote Types @@ -149,7 +165,6 @@ Arbitrary HTTP file servers — GitHub releases, HashiCorp, custom servers. remotes: github: base_url: "https://github.com" - type: "remote" package: "generic" immutable_patterns: - "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*" @@ -158,7 +173,6 @@ remotes: github-archive: base_url: "https://github.com" - type: "remote" package: "generic" immutable_patterns: - ".*/archive/refs/tags/.*\\.tar\\.gz$" # tag archives never change @@ -178,7 +192,6 @@ Access: `GET /api/v1/remote/github/owner/repo/releases/download/v1.0/binary.tar. remotes: alpine: base_url: "https://dl-cdn.alpinelinux.org" - type: "remote" package: "alpine" immutable_patterns: - ".*/x86_64/.*\\.apk$" @@ -195,7 +208,6 @@ remotes: remotes: almalinux: base_url: "https://mirror.example.com/almalinux" - type: "remote" package: "rpm" immutable_patterns: - ".*/x86_64/.*\\.rpm$" @@ -213,7 +225,6 @@ remotes: remotes: dockerhub: base_url: "https://registry-1.docker.io" - type: "remote" package: "docker" # username / password optional for public images cache: @@ -222,7 +233,6 @@ remotes: ghcr: base_url: "https://ghcr.io" - type: "remote" package: "docker" username: "your-github-username" password: "ghp_your_pat" # read:packages scope @@ -255,7 +265,6 @@ mirrors: remotes: pypi: base_url: "https://files.pythonhosted.org" - type: "remote" package: "pypi" check_mutable_updates: true immutable_patterns: @@ -287,7 +296,6 @@ default = true remotes: npm: base_url: "https://registry.npmjs.org" - type: "remote" package: "npm" check_mutable_updates: true immutable_patterns: @@ -314,7 +322,6 @@ registry=https://artifacts.example.com/api/v1/remote/npm/ remotes: hashicorp-helm: base_url: "https://helm.releases.hashicorp.com" - type: "remote" package: "helm" check_mutable_updates: true immutable_patterns: @@ -343,7 +350,6 @@ All members must share the same `package` type as the virtual repo. Currently su remotes: helm-hashicorp: base_url: "https://helm.releases.hashicorp.com" - type: "remote" package: "helm" immutable_patterns: - "\\.tgz$" @@ -353,7 +359,6 @@ remotes: helm-bitnami: base_url: "https://charts.bitnami.com/bitnami" - type: "remote" package: "helm" immutable_patterns: - "\\.tgz$" @@ -361,8 +366,8 @@ remotes: immutable_ttl: 0 mutable_ttl: 3600 +virtuals: helm-all: - type: "virtual" package: "helm" members: - helm-hashicorp # listed first = highest priority @@ -399,9 +404,8 @@ Chart tarball URLs in the merged `index.yaml` are rewritten to point at the indi ### local ```yaml -remotes: +locals: local-generic: - type: "local" package: "generic" description: "Local file repository" cache: @@ -409,7 +413,7 @@ remotes: mutable_ttl: 0 ``` -No `base_url`. Files are uploaded via `PUT` and served via `GET`. +No `base_url`. Files are uploaded via `PUT /api/v1/local/{name}/{path}` and downloaded via `GET /api/v1/local/{name}/{path}`. ## Caching Model @@ -451,7 +455,6 @@ Set `quarantine_new: true` and `quarantine_days: N` on a remote to block immutab remotes: pypi: base_url: "https://files.pythonhosted.org" - type: "remote" package: "pypi" quarantine_new: true quarantine_days: 3 # block packages published in the last 3 days diff --git a/conf.d/alpine.yaml b/conf.d/alpine.yaml deleted file mode 100644 index a8e12f7..0000000 --- a/conf.d/alpine.yaml +++ /dev/null @@ -1,11 +0,0 @@ -remotes: - alpine: - base_url: "https://dl-cdn.alpinelinux.org" - type: "remote" - package: "alpine" - description: "Alpine Linux APK package repository" - immutable_patterns: - - ".*/x86_64/.*\\.apk$" - cache: - immutable_ttl: 0 - mutable_ttl: 7200 diff --git a/conf.d/github.yaml b/conf.d/github.yaml deleted file mode 100644 index 51aa20a..0000000 --- a/conf.d/github.yaml +++ /dev/null @@ -1,12 +0,0 @@ -remotes: - github: - base_url: "https://github.com" - type: "remote" - package: "generic" - description: "GitHub releases and files" - immutable_patterns: - - "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*" - - "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$" - cache: - immutable_ttl: 0 - mutable_ttl: 0 diff --git a/conf.d/pypi.yaml b/conf.d/pypi.yaml deleted file mode 100644 index 4de0058..0000000 --- a/conf.d/pypi.yaml +++ /dev/null @@ -1,17 +0,0 @@ -remotes: - pypi: - base_url: "https://files.pythonhosted.org" - type: "remote" - package: "pypi" - description: "Python Package Index" - check_mutable_updates: true - quarantine_new: true - quarantine_days: 3 - immutable_patterns: - - "packages/.*\\.whl$" - - "packages/.*\\.whl\\.metadata$" - - "packages/.*\\.tar\\.gz$" - - "packages/.*\\.zip$" - cache: - immutable_ttl: 0 - mutable_ttl: 600 diff --git a/examples/conf.d-method/alpine.yaml b/examples/conf.d-method/alpine.yaml index a8e12f7..55f8c65 100644 --- a/examples/conf.d-method/alpine.yaml +++ b/examples/conf.d-method/alpine.yaml @@ -1,7 +1,6 @@ remotes: alpine: base_url: "https://dl-cdn.alpinelinux.org" - type: "remote" package: "alpine" description: "Alpine Linux APK package repository" immutable_patterns: diff --git a/examples/conf.d-method/github.yaml b/examples/conf.d-method/github.yaml index 51aa20a..81ec2e2 100644 --- a/examples/conf.d-method/github.yaml +++ b/examples/conf.d-method/github.yaml @@ -1,7 +1,6 @@ remotes: github: base_url: "https://github.com" - type: "remote" package: "generic" description: "GitHub releases and files" immutable_patterns: diff --git a/examples/conf.d-method/pypi.yaml b/examples/conf.d-method/pypi.yaml index 4de0058..0950dc2 100644 --- a/examples/conf.d-method/pypi.yaml +++ b/examples/conf.d-method/pypi.yaml @@ -1,7 +1,6 @@ remotes: pypi: base_url: "https://files.pythonhosted.org" - type: "remote" package: "pypi" description: "Python Package Index" check_mutable_updates: true diff --git a/examples/single-file/remotes.yaml b/examples/single-file/remotes.yaml index 6e36b5b..07f0c8b 100644 --- a/examples/single-file/remotes.yaml +++ b/examples/single-file/remotes.yaml @@ -34,9 +34,7 @@ # remotes: github: - base_url: "https://github.com" - type: "remote" - package: "generic" + base_url: "https://github.com" package: "generic" description: "GitHub releases and files" immutable_patterns: - "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*" @@ -66,9 +64,7 @@ remotes: mutable_ttl: 0 github-archive: - base_url: "https://github.com" - type: "remote" - package: "generic" + base_url: "https://github.com" package: "generic" description: "GitHub repository archive tarballs" immutable_patterns: # Tag archives are immutable — a tag never changes @@ -86,9 +82,7 @@ remotes: mutable_ttl: 86400 # Branch archives refreshed after 1 day gitea-dl: - base_url: "https://dl.gitea.com" - type: "remote" - package: "generic" + base_url: "https://dl.gitea.com" package: "generic" description: "Gitea download site" immutable_patterns: - "act_runner/.*/act_runner-.*-linux-amd64$" @@ -97,9 +91,7 @@ remotes: mutable_ttl: 0 hashicorp-releases: - base_url: "https://releases.hashicorp.com" - type: "remote" - package: "generic" + base_url: "https://releases.hashicorp.com" package: "generic" description: "HashiCorp product releases" immutable_patterns: - "terraform/.*terraform_.*_linux_amd64\\.zip$" @@ -118,9 +110,7 @@ remotes: mutable_ttl: 0 alpine: - base_url: "https://dl-cdn.alpinelinux.org" - type: "remote" - package: "alpine" + base_url: "https://dl-cdn.alpinelinux.org" package: "alpine" description: "Alpine Linux APK package repository" immutable_patterns: - ".*/x86_64/.*\\.apk$" @@ -132,9 +122,7 @@ remotes: mutable_ttl: 7200 # Index files (APKINDEX.tar.gz) cached for 2 hours almalinux: - base_url: "https://gsl-syd.mm.fcix.net/almalinux" - type: "remote" - package: "rpm" + base_url: "https://gsl-syd.mm.fcix.net/almalinux" package: "rpm" description: "AlmaLinux RPM package repository" immutable_patterns: - ".*/x86_64/.*\\.rpm$" @@ -149,9 +137,7 @@ remotes: mutable_ttl: 7200 # Metadata files cached for 2 hours epel: - base_url: "http://mirror.aarnet.edu.au/pub/epel" - type: "remote" - package: "rpm" + base_url: "http://mirror.aarnet.edu.au/pub/epel" package: "rpm" description: "EPEL (Extra Packages for Enterprise Linux)" immutable_patterns: - "8/Everything/x86_64/.*\\.rpm$" @@ -164,9 +150,7 @@ remotes: mutable_ttl: 7200 # Metadata files cached for 2 hours fedora: - base_url: "https://gsl-syd.mm.fcix.net/fedora/linux" - type: "remote" - package: "rpm" + base_url: "https://gsl-syd.mm.fcix.net/fedora/linux" package: "rpm" description: "Fedora Linux RPM package repository" immutable_patterns: - "releases/.*/Everything/x86_64/.*\\.rpm$" @@ -179,9 +163,7 @@ remotes: mutable_ttl: 300 # Metadata files cached for 5 minutes ghcr: - base_url: "https://ghcr.io" - type: "remote" - package: "docker" + base_url: "https://ghcr.io" package: "docker" description: "GitHub Container Registry" # username: "your-github-username" # password: "your-github-pat" # needs read:packages scope @@ -193,18 +175,14 @@ remotes: mutable_ttl: 300 dockerhub: - base_url: "https://registry-1.docker.io" - type: "remote" - package: "docker" + base_url: "https://registry-1.docker.io" package: "docker" description: "Docker Hub registry" cache: immutable_ttl: 0 mutable_ttl: 300 pypi: - base_url: "https://files.pythonhosted.org" - type: "remote" - package: "pypi" + base_url: "https://files.pythonhosted.org" package: "pypi" description: "Python Package Index — simple index and package files via a single remote" # simple/ requests are transparently fetched from pypi.org; package files come from # files.pythonhosted.org (base_url). URLs in the simple index are rewritten to this remote. @@ -225,9 +203,7 @@ remotes: mutable_ttl: 600 # Simple index pages refreshed after 10 minutes pypi-gitea: - base_url: "https://gitea.example.com/api/packages/myorg/pypi" - type: "remote" - package: "pypi" + base_url: "https://gitea.example.com/api/packages/myorg/pypi" package: "pypi" description: "Private Gitea PyPI registry — simple index and files at the same host" # username: "your-gitea-username" # password: "your-personal-access-token" # needs package:read scope @@ -243,9 +219,7 @@ remotes: mutable_ttl: 600 npm: - base_url: "https://registry.npmjs.org" - type: "remote" - package: "npm" + base_url: "https://registry.npmjs.org" package: "npm" description: "npm registry — package metadata with tarball URL rewriting" check_mutable_updates: true immutable_patterns: @@ -257,9 +231,7 @@ remotes: mutable_ttl: 600 # Package metadata refreshed after 10 minutes hashicorp-helm: - base_url: "https://helm.releases.hashicorp.com" - type: "remote" - package: "helm" + base_url: "https://helm.releases.hashicorp.com" package: "helm" description: "HashiCorp Helm chart repository (Vault, Consul, Nomad, etc.)" check_mutable_updates: true immutable_patterns: @@ -269,9 +241,7 @@ remotes: mutable_ttl: 3600 # index.yaml refreshed after 1 hour metallb: - base_url: "https://metallb.github.io/metallb" - type: "remote" - package: "helm" + base_url: "https://metallb.github.io/metallb" package: "helm" description: "MetalLB load balancer Helm charts" check_mutable_updates: true immutable_patterns: @@ -281,9 +251,7 @@ remotes: mutable_ttl: 3600 jetstack: - base_url: "https://charts.jetstack.io" - type: "remote" - package: "helm" + base_url: "https://charts.jetstack.io" package: "helm" description: "Jetstack Helm charts (cert-manager)" check_mutable_updates: true immutable_patterns: @@ -293,9 +261,7 @@ remotes: mutable_ttl: 3600 rancher-stable: - base_url: "https://releases.rancher.com/server-charts/stable" - type: "remote" - package: "helm" + base_url: "https://releases.rancher.com/server-charts/stable" package: "helm" description: "Rancher stable Helm charts" check_mutable_updates: true immutable_patterns: @@ -305,9 +271,7 @@ remotes: mutable_ttl: 3600 purelb: - base_url: "https://gitlab.com/api/v4/projects/20400619/packages/helm/stable" - type: "remote" - package: "helm" + base_url: "https://gitlab.com/api/v4/projects/20400619/packages/helm/stable" package: "helm" description: "PureLB load balancer Helm charts" check_mutable_updates: true immutable_patterns: @@ -317,9 +281,7 @@ remotes: mutable_ttl: 3600 istio: - base_url: "https://istio-release.storage.googleapis.com/charts" - type: "remote" - package: "helm" + base_url: "https://istio-release.storage.googleapis.com/charts" package: "helm" description: "Istio service mesh Helm charts" check_mutable_updates: true immutable_patterns: @@ -329,9 +291,7 @@ remotes: mutable_ttl: 3600 cnpg: - base_url: "https://cloudnative-pg.github.io/charts" - type: "remote" - package: "helm" + base_url: "https://cloudnative-pg.github.io/charts" package: "helm" description: "CloudNativePG operator Helm charts" check_mutable_updates: true immutable_patterns: @@ -341,9 +301,7 @@ remotes: mutable_ttl: 3600 ceph-csi: - base_url: "https://ceph.github.io/csi-charts" - type: "remote" - package: "helm" + base_url: "https://ceph.github.io/csi-charts" package: "helm" description: "Ceph CSI driver Helm charts" check_mutable_updates: true immutable_patterns: @@ -353,9 +311,7 @@ remotes: mutable_ttl: 3600 external-dns: - base_url: "https://kubernetes-sigs.github.io/external-dns/" - type: "remote" - package: "helm" + base_url: "https://kubernetes-sigs.github.io/external-dns/" package: "helm" description: "ExternalDNS Helm charts" check_mutable_updates: true immutable_patterns: @@ -365,9 +321,7 @@ remotes: mutable_ttl: 3600 intel-helm: - base_url: "https://intel.github.io/helm-charts/" - type: "remote" - package: "helm" + base_url: "https://intel.github.io/helm-charts/" package: "helm" description: "Intel Helm charts" check_mutable_updates: true immutable_patterns: @@ -377,9 +331,7 @@ remotes: mutable_ttl: 3600 elastic: - base_url: "https://helm.elastic.co" - type: "remote" - package: "helm" + base_url: "https://helm.elastic.co" package: "helm" description: "Elastic stack Helm charts" check_mutable_updates: true immutable_patterns: @@ -389,9 +341,7 @@ remotes: mutable_ttl: 3600 k8up-io: - base_url: "https://k8up-io.github.io/k8up" - type: "remote" - package: "helm" + base_url: "https://k8up-io.github.io/k8up" package: "helm" description: "K8up backup operator Helm charts" check_mutable_updates: true immutable_patterns: @@ -401,9 +351,7 @@ remotes: mutable_ttl: 3600 victoriametrics: - base_url: "https://victoriametrics.github.io/helm-charts/" - type: "remote" - package: "helm" + base_url: "https://victoriametrics.github.io/helm-charts/" package: "helm" description: "VictoriaMetrics observability Helm charts" check_mutable_updates: true immutable_patterns: @@ -413,9 +361,7 @@ remotes: mutable_ttl: 3600 grafana: - base_url: "https://grafana.github.io/helm-charts" - type: "remote" - package: "helm" + base_url: "https://grafana.github.io/helm-charts" package: "helm" description: "Grafana observability Helm charts" check_mutable_updates: true immutable_patterns: @@ -425,9 +371,7 @@ remotes: mutable_ttl: 3600 helm-openldap: - base_url: "https://jp-gouin.github.io/helm-openldap/" - type: "remote" - package: "helm" + base_url: "https://jp-gouin.github.io/helm-openldap/" package: "helm" description: "OpenLDAP Helm charts" check_mutable_updates: true immutable_patterns: @@ -437,9 +381,7 @@ remotes: mutable_ttl: 3600 woodpecker: - base_url: "https://woodpecker-ci.org/" - type: "remote" - package: "helm" + base_url: "https://woodpecker-ci.org/" package: "helm" description: "Woodpecker CI Helm charts" check_mutable_updates: true immutable_patterns: @@ -449,9 +391,7 @@ remotes: mutable_ttl: 3600 stakater: - base_url: "https://stakater.github.io/stakater-charts" - type: "remote" - package: "helm" + base_url: "https://stakater.github.io/stakater-charts" package: "helm" description: "Stakater Helm charts" check_mutable_updates: true immutable_patterns: @@ -461,9 +401,7 @@ remotes: mutable_ttl: 3600 jfrog: - base_url: "https://charts.jfrog.io/" - type: "remote" - package: "helm" + base_url: "https://charts.jfrog.io/" package: "helm" description: "JFrog Helm charts" check_mutable_updates: true immutable_patterns: @@ -473,9 +411,7 @@ remotes: mutable_ttl: 3600 openvox: - base_url: "https://openvoxproject.github.io/openvox-helm-chart" - type: "remote" - package: "helm" + base_url: "https://openvoxproject.github.io/openvox-helm-chart" package: "helm" description: "OpenVox Helm charts" check_mutable_updates: true immutable_patterns: @@ -484,8 +420,9 @@ remotes: immutable_ttl: 0 mutable_ttl: 3600 + +virtuals: helm-all: - type: "virtual" package: "helm" description: "Virtual repository merging all helm remotes — member order is priority order for duplicate chart+version" members: @@ -509,8 +446,8 @@ remotes: - jfrog - openvox +locals: local-generic: - type: "local" package: "generic" description: "Local generic file repository" cache: diff --git a/src/artifactapi/artifact/local.py b/src/artifactapi/artifact/local.py index 5b76da1..ab978e8 100644 --- a/src/artifactapi/artifact/local.py +++ b/src/artifactapi/artifact/local.py @@ -1,5 +1,6 @@ import hashlib import logging +import os from fastapi import HTTPException, Response, UploadFile from fastapi.responses import JSONResponse @@ -7,12 +8,23 @@ from fastapi.responses import JSONResponse logger = logging.getLogger(__name__) +def download(remote_name: str, path: str, storage, database, config) -> Response: + if not config.get_local_config(remote_name): + raise HTTPException(status_code=404, detail=f"Local repository '{remote_name}' not configured") + metadata = database.get_local_file_metadata(remote_name, path) + if not metadata: + raise HTTPException(status_code=404, detail="File not found") + content = storage.download_object(metadata["s3_key"]) + return Response( + content=content, + media_type=metadata.get("content_type", "application/octet-stream"), + headers={"Content-Disposition": f"attachment; filename={os.path.basename(path)}"}, + ) + + async def upload(remote_name: str, path: str, file: UploadFile, storage, database, config) -> JSONResponse: - remote_config = config.get_remote_config(remote_name) - if not remote_config: - raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured") - if remote_config.get("type") != "local": - raise HTTPException(status_code=400, detail="Upload only supported for local repositories") + if not config.get_local_config(remote_name): + raise HTTPException(status_code=404, detail=f"Local repository '{remote_name}' not configured") try: content = await file.read() @@ -59,12 +71,8 @@ async def upload(remote_name: str, path: str, file: UploadFile, storage, databas def check_exists(remote_name: str, path: str, database, config) -> Response: - remote_config = config.get_remote_config(remote_name) - if not remote_config: - raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured") - - if remote_config.get("type") != "local": - raise HTTPException(status_code=405, detail="HEAD method only supported for local repositories") + if not config.get_local_config(remote_name): + raise HTTPException(status_code=404, detail=f"Local repository '{remote_name}' not configured") try: metadata = database.get_local_file_metadata(remote_name, path) @@ -87,11 +95,8 @@ def check_exists(remote_name: str, path: str, database, config) -> Response: def delete(remote_name: str, path: str, storage, database, config) -> JSONResponse: - remote_config = config.get_remote_config(remote_name) - if not remote_config: - raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured") - if remote_config.get("type") != "local": - raise HTTPException(status_code=400, detail="Delete only supported for local repositories") + if not config.get_local_config(remote_name): + raise HTTPException(status_code=404, detail=f"Local repository '{remote_name}' not configured") try: s3_key = database.delete_local_file(remote_name, path) diff --git a/src/artifactapi/artifact/proxy.py b/src/artifactapi/artifact/proxy.py index c3506a6..79f0e7e 100644 --- a/src/artifactapi/artifact/proxy.py +++ b/src/artifactapi/artifact/proxy.py @@ -218,19 +218,6 @@ async def handle(request: Request, remote_name: str, path: str, storage, cache, if not remote_config: raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured") - if remote_config.get("type") == "local": - metadata = database.get_local_file_metadata(remote_name, path) - if not metadata: - raise HTTPException(status_code=404, detail="File not found") - content = storage.download_object(metadata["s3_key"]) - if content is None: - raise HTTPException(status_code=500, detail="File not accessible") - return Response( - content=content, - media_type=metadata.get("content_type", "application/octet-stream"), - headers={"Content-Disposition": f"attachment; filename={os.path.basename(path)}"}, - ) - path_parts = path.split("/") if len(path_parts) >= 2: repo_path = f"{path_parts[0]}/{path_parts[1]}" diff --git a/src/artifactapi/artifact/virtual.py b/src/artifactapi/artifact/virtual.py index 0bae5e6..4703aba 100644 --- a/src/artifactapi/artifact/virtual.py +++ b/src/artifactapi/artifact/virtual.py @@ -147,11 +147,9 @@ _HANDLERS: dict[str, _VirtualHandler] = { async def handle(request: Request, virtual_name: str, path: str, storage, cache, config) -> Response: - virtual_cfg = config.get_remote_config(virtual_name) + virtual_cfg = config.get_virtual_config(virtual_name) if not virtual_cfg: raise HTTPException(status_code=404, detail=f"Virtual repository '{virtual_name}' not configured") - if virtual_cfg.get("type") != "virtual": - raise HTTPException(status_code=400, detail=f"'{virtual_name}' is not a virtual repository") package = virtual_cfg.get("package") handler = _HANDLERS.get(package) diff --git a/src/artifactapi/config.py b/src/artifactapi/config.py index 6de80e4..290ba2e 100644 --- a/src/artifactapi/config.py +++ b/src/artifactapi/config.py @@ -50,8 +50,8 @@ class ConfigManager: def _merge(base: dict, overlay: dict) -> dict: result = {**base} for key, value in overlay.items(): - if key == "remotes" and isinstance(base.get("remotes"), dict) and isinstance(value, dict): - result["remotes"] = {**base.get("remotes", {}), **value} + if key in ("remotes", "virtuals", "locals") and isinstance(base.get(key), dict) and isinstance(value, dict): + result[key] = {**base.get(key, {}), **value} else: result[key] = value return result @@ -67,11 +67,11 @@ class ConfigManager: self._config_dir = None if os.path.isdir(self.config_path): - return self._load_from_dir(self.config_path) or {"remotes": {}} + return self._load_from_dir(self.config_path) or {"remotes": {}, "virtuals": {}, "locals": {}} config = self._load_single_file(self.config_path) if not config: - return {"remotes": {}} + return {"remotes": {}, "virtuals": {}, "locals": {}} config_dir = config.pop("config_dir", None) if config_dir: @@ -119,6 +119,14 @@ class ConfigManager: self._check_reload() return self.config.get("remotes", {}).get(remote_name) + def get_virtual_config(self, virtual_name: str) -> dict | None: + self._check_reload() + return self.config.get("virtuals", {}).get(virtual_name) + + def get_local_config(self, local_name: str) -> dict | None: + self._check_reload() + return self.config.get("locals", {}).get(local_name) + def get_immutable_patterns(self, remote_name: str, repo_path: str = "") -> list[str]: remote_config = self.get_remote_config(remote_name) if not remote_config: diff --git a/src/artifactapi/main.py b/src/artifactapi/main.py index e9e42ee..ae9d566 100644 --- a/src/artifactapi/main.py +++ b/src/artifactapi/main.py @@ -49,7 +49,13 @@ class ArtifactRequest(BaseModel): @app.get("/") def read_root(): config._check_reload() - return {"message": "Artifact Storage API", "version": app.version, "remotes": list(config.config.get("remotes", {}).keys())} + return { + "message": "Artifact Storage API", + "version": app.version, + "remotes": list(config.config.get("remotes", {}).keys()), + "virtuals": list(config.config.get("virtuals", {}).keys()), + "locals": list(config.config.get("locals", {}).keys()), + } @app.get("/health") @@ -99,19 +105,24 @@ async def get_artifact(request: Request, remote_name: str, path: str): return await proxy.handle(request, remote_name, path, storage, cache, config, database, metrics) -@app.put("/api/v1/remote/{remote_name}/{path:path}") -async def upload_file(remote_name: str, path: str, file: UploadFile = File(...)): - return await local.upload(remote_name, path, file, storage, database, config) +@app.get("/api/v1/local/{local_name}/{path:path}") +def get_local_artifact(local_name: str, path: str): + return local.download(local_name, path, storage, database, config) -@app.head("/api/v1/remote/{remote_name}/{path:path}") -def check_file_exists(remote_name: str, path: str): - return local.check_exists(remote_name, path, database, config) +@app.put("/api/v1/local/{local_name}/{path:path}") +async def upload_local_file(local_name: str, path: str, file: UploadFile = File(...)): + return await local.upload(local_name, path, file, storage, database, config) -@app.delete("/api/v1/remote/{remote_name}/{path:path}") -def delete_file(remote_name: str, path: str): - return local.delete(remote_name, path, storage, database, config) +@app.head("/api/v1/local/{local_name}/{path:path}") +def check_local_file_exists(local_name: str, path: str): + return local.check_exists(local_name, path, database, config) + + +@app.delete("/api/v1/local/{local_name}/{path:path}") +def delete_local_file(local_name: str, path: str): + return local.delete(local_name, path, storage, database, config) @app.post("/api/v1/artifacts/cache") diff --git a/src/artifactapi/metrics.py b/src/artifactapi/metrics.py index 10d6a23..04473fb 100644 --- a/src/artifactapi/metrics.py +++ b/src/artifactapi/metrics.py @@ -87,9 +87,10 @@ class MetricsManager: # Get from database if available db_sizes = self.database_manager.get_storage_by_remote() if db_sizes: - # Initialize all configured remotes to 0 + # Initialize all configured remotes and locals to 0 remote_sizes = {} - for remote in config_manager.config.get("remotes", {}).keys(): + all_names = list(config_manager.config.get("remotes", {}).keys()) + list(config_manager.config.get("locals", {}).keys()) + for remote in all_names: remote_sizes[remote] = db_sizes.get(remote, 0) # Update Prometheus gauges @@ -101,10 +102,10 @@ class MetricsManager: # Fallback to S3 scanning if database not available try: remote_sizes = {} - remotes = config_manager.config.get("remotes", {}).keys() + all_names = list(config_manager.config.get("remotes", {}).keys()) + list(config_manager.config.get("locals", {}).keys()) - # Initialize all remotes to 0 - for remote in remotes: + # Initialize all remotes and locals to 0 + for remote in all_names: remote_sizes[remote] = 0 paginator = storage.client.get_paginator("list_objects_v2") @@ -174,8 +175,13 @@ class MetricsManager: metrics["requests"]["cache_hit_ratio"] = cache_hits / total_requests if total_requests > 0 else 0.0 metrics["bandwidth"]["saved_bytes"] = bandwidth_saved - # Get per-remote metrics - for remote in config_manager.config.get("remotes", {}).keys(): + # Get per-repo metrics + all_repos = { + **config_manager.config.get("remotes", {}), + **config_manager.config.get("virtuals", {}), + **config_manager.config.get("locals", {}), + } + for remote in all_repos.keys(): remote_cache_hits = int(self.redis_client.client.get(f"metrics:cache_hits:{remote}") or 0) remote_cache_misses = int(self.redis_client.client.get(f"metrics:cache_misses:{remote}") or 0) remote_total = remote_cache_hits + remote_cache_misses diff --git a/tests/conftest.py b/tests/conftest.py index 2e23c3a..4c931c2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,61 +20,48 @@ TEST_REMOTES = { "remotes": { "alpine-test": { "base_url": "https://dl-cdn.alpinelinux.org", - "type": "remote", "package": "alpine", "immutable_patterns": [".*/x86_64/.*\\.apk$"], "cache": {"immutable_ttl": 0, "mutable_ttl": 3600}, }, "rpm-test": { "base_url": "https://example.com/rpm", - "type": "remote", "package": "rpm", "immutable_patterns": [".*/x86_64/.*\\.rpm$", ".*/repodata/.*$"], "cache": {"immutable_ttl": 0, "mutable_ttl": 3600}, }, "docker-test": { "base_url": "https://registry.example.com", - "type": "remote", "package": "docker", "cache": {"immutable_ttl": 0, "mutable_ttl": 300}, }, "docker-restricted": { "base_url": "https://registry.example.com", - "type": "remote", "package": "docker", "immutable_patterns": ["^library/nginx"], "cache": {"immutable_ttl": 0, "mutable_ttl": 300}, }, "generic-test": { "base_url": "https://releases.example.com", - "type": "remote", "package": "generic", "immutable_patterns": [".*\\.tar\\.gz$"], "cache": {"immutable_ttl": 0, "mutable_ttl": 0}, }, "custom-index-test": { "base_url": "https://example.com", - "type": "remote", "package": "generic", "mutable_patterns": ["metadata\\.json$"], "cache": {"immutable_ttl": 0, "mutable_ttl": 600}, }, "check-mutable-test": { "base_url": "https://example.com", - "type": "remote", "package": "generic", "mutable_patterns": ["metadata\\.json$"], "check_mutable_updates": True, "cache": {"immutable_ttl": 0, "mutable_ttl": 600}, }, - "local-test": { - "type": "local", - "package": "generic", - "cache": {"immutable_ttl": 0, "mutable_ttl": 0}, - }, "pypi-test": { "base_url": "https://files.pythonhosted.org", - "type": "remote", "package": "pypi", "immutable_patterns": [ r"packages/.*\.whl$", @@ -85,7 +72,6 @@ TEST_REMOTES = { }, "npm-test": { "base_url": "https://registry.npmjs.org", - "type": "remote", "package": "npm", "immutable_patterns": [r"\.tgz$"], "mutable_patterns": [r"^(?!.*\.tgz$).*"], @@ -93,14 +79,12 @@ TEST_REMOTES = { }, "helm-test": { "base_url": "https://helm.releases.hashicorp.com", - "type": "remote", "package": "helm", "immutable_patterns": [r"\.tgz$"], "cache": {"immutable_ttl": 0, "mutable_ttl": 3600}, }, "quarantine-test": { "base_url": "https://releases.example.com", - "type": "remote", "package": "generic", "immutable_patterns": [r".*\.tar\.gz$"], "quarantine_new": True, @@ -109,7 +93,6 @@ TEST_REMOTES = { }, "quarantine-disabled": { "base_url": "https://releases.example.com", - "type": "remote", "package": "generic", "immutable_patterns": [r".*\.tar\.gz$"], "quarantine_new": False, @@ -118,27 +101,31 @@ TEST_REMOTES = { }, "helm-member-2": { "base_url": "https://charts.example.com", - "type": "remote", "package": "helm", "immutable_patterns": [r"\.tgz$"], "cache": {"immutable_ttl": 0, "mutable_ttl": 1800}, }, + }, + "locals": { + "local-test": { + "package": "generic", + "cache": {"immutable_ttl": 0, "mutable_ttl": 0}, + }, + }, + "virtuals": { "helm-virtual-test": { - "type": "virtual", "package": "helm", "members": ["helm-test", "helm-member-2"], }, "unsupported-virtual-test": { - "type": "virtual", "package": "rpm", "members": ["rpm-test"], }, "empty-virtual-test": { - "type": "virtual", "package": "helm", "members": [], }, - } + }, } # --------------------------------------------------------------------------- diff --git a/tests/test_config.py b/tests/test_config.py index a042086..bb719b7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -27,24 +27,24 @@ def make_config(tmp_path): class TestGetMutablePatterns: def test_alpine_returns_package_defaults(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "alpine", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "alpine", "base_url": "https://x.com"}}) patterns = cfg.get_mutable_patterns("r") assert r"APKINDEX\.tar\.gz$" in patterns def test_rpm_returns_package_defaults(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "rpm", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "rpm", "base_url": "https://x.com"}}) patterns = cfg.get_mutable_patterns("r") assert r"repomd\.xml$" in patterns assert any("repodata" in p for p in patterns) def test_docker_returns_package_defaults(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "docker", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "docker", "base_url": "https://x.com"}}) patterns = cfg.get_mutable_patterns("r") assert any("manifests" in p for p in patterns) assert any("tags/list" in p for p in patterns) def test_generic_returns_empty_list(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "generic", "base_url": "https://x.com"}}) assert cfg.get_mutable_patterns("r") == [] def test_unknown_remote_returns_empty_list(self, make_config): @@ -52,12 +52,12 @@ class TestGetMutablePatterns: assert cfg.get_mutable_patterns("nonexistent") == [] def test_missing_package_field_defaults_to_generic(self, make_config): - cfg = make_config({"r": {"type": "remote", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"base_url": "https://x.com"}}) assert cfg.get_mutable_patterns("r") == [] def test_unknown_package_type_returns_empty_list(self, make_config): # A mis-spelled package type silently returns [] — this is a known footgun - cfg = make_config({"r": {"type": "remote", "package": "deb", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "deb", "base_url": "https://x.com"}}) assert cfg.get_mutable_patterns("r") == [] def test_extra_patterns_appended_after_defaults(self, make_config): @@ -134,7 +134,7 @@ class TestGetMutablePatterns: assert r"custom-meta\.xml$" in patterns def test_npm_has_no_package_defaults(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "npm", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "npm", "base_url": "https://x.com"}}) assert cfg.get_mutable_patterns("r") == [] def test_npm_explicit_mutable_pattern_matches_metadata(self, make_config): @@ -155,14 +155,14 @@ class TestGetMutablePatterns: assert any(re.search(p, "@babel/core") for p in patterns) def test_helm_returns_index_yaml_as_mutable(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "helm", "base_url": "https://helm.example.com"}}) + cfg = make_config({"r": {"package": "helm", "base_url": "https://helm.example.com"}}) patterns = cfg.get_mutable_patterns("r") assert r"index\.yaml$" in patterns def test_helm_chart_tarballs_not_mutable_by_default(self, make_config): import re - cfg = make_config({"r": {"type": "remote", "package": "helm", "base_url": "https://helm.example.com"}}) + cfg = make_config({"r": {"package": "helm", "base_url": "https://helm.example.com"}}) patterns = cfg.get_mutable_patterns("r") # Only index.yaml is mutable; .tgz chart tarballs are not assert not any(re.search(p, "vault-0.29.1.tgz") for p in patterns) @@ -210,7 +210,7 @@ class TestGetImmutablePatterns: assert cfg.get_immutable_patterns("nonexistent") == [] def test_returns_empty_when_no_patterns_configured(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "generic", "base_url": "https://x.com"}}) assert cfg.get_immutable_patterns("r") == [] def test_multiple_patterns_returned(self, make_config): @@ -281,7 +281,7 @@ class TestGetUserMutablePatterns: def test_excludes_package_defaults(self, make_config): # Package defaults (APKINDEX etc.) must NOT appear here - cfg = make_config({"r": {"type": "remote", "package": "alpine", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "alpine", "base_url": "https://x.com"}}) assert cfg.get_user_mutable_patterns("r") == [] def test_returns_empty_for_missing_remote(self, make_config): @@ -289,7 +289,7 @@ class TestGetUserMutablePatterns: assert cfg.get_user_mutable_patterns("nonexistent") == [] def test_returns_empty_when_key_absent(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "generic", "base_url": "https://x.com"}}) assert cfg.get_user_mutable_patterns("r") == [] @@ -317,7 +317,7 @@ class TestGetCacheConfig: assert cfg.get_cache_config("nonexistent") == {} def test_returns_empty_dict_when_no_cache_key(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "generic", "base_url": "https://x.com"}}) assert cfg.get_cache_config("r") == {} @@ -329,11 +329,11 @@ class TestGetCacheConfig: class TestConfigReload: def test_reloads_when_file_mtime_advances(self, tmp_path): cfg_file = tmp_path / "remotes.yaml" - cfg_file.write_text(yaml.dump({"remotes": {"repo-a": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}})) + cfg_file.write_text(yaml.dump({"remotes": {"repo-a": {"package": "generic", "base_url": "https://x.com"}}})) cfg = ConfigManager(str(cfg_file)) assert "repo-a" in cfg.config["remotes"] - cfg_file.write_text(yaml.dump({"remotes": {"repo-b": {"type": "remote", "package": "generic", "base_url": "https://y.com"}}})) + cfg_file.write_text(yaml.dump({"remotes": {"repo-b": {"package": "generic", "base_url": "https://y.com"}}})) future_mtime = cfg._last_modified + 1 os.utime(str(cfg_file), (future_mtime, future_mtime)) @@ -344,7 +344,7 @@ class TestConfigReload: def test_no_reload_when_file_unchanged(self, tmp_path): cfg_file = tmp_path / "remotes.yaml" - cfg_file.write_text(yaml.dump({"remotes": {"repo-a": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}})) + cfg_file.write_text(yaml.dump({"remotes": {"repo-a": {"package": "generic", "base_url": "https://x.com"}}})) cfg = ConfigManager(str(cfg_file)) # Call check_reload without touching the file — should not reload @@ -360,7 +360,7 @@ class TestConfigReload: class TestGetQuarantineConfig: def test_returns_false_zero_when_not_configured(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "generic", "base_url": "https://x.com"}}) enabled, days = cfg.get_quarantine_config("r") assert enabled is False assert days == 0 @@ -426,7 +426,7 @@ class TestGetQuarantineConfig: def _remote(base_url: str = "https://x.com") -> dict: - return {"type": "remote", "package": "generic", "base_url": base_url} + return {"package": "generic", "base_url": base_url} class TestConfigDirMode: @@ -445,7 +445,7 @@ class TestConfigDirMode: def test_empty_directory_returns_empty_remotes(self, tmp_path): cfg = ConfigManager(str(tmp_path)) - assert cfg.config == {"remotes": {}} + assert cfg.config == {"remotes": {}, "virtuals": {}, "locals": {}} def test_ignores_non_yaml_files(self, tmp_path): (tmp_path / "notes.txt").write_text("not yaml") diff --git a/tests/test_routes.py b/tests/test_routes.py index bf9a1db..1bea394 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -523,68 +523,53 @@ class TestGenericArtifactRoute: deps["database"].get_local_file_metadata.return_value = None deps["database"].available = True - response = client.get("/api/v1/remote/local-test/path/to/nonexistent.bin") + response = client.get("/api/v1/local/local-test/path/to/nonexistent.bin") assert response.status_code == 404 # --------------------------------------------------------------------------- -# Upload route PUT /api/v1/remote/{remote}/{path} +# Upload route PUT /api/v1/local/{local}/{path} # --------------------------------------------------------------------------- class TestUploadRoute: - def test_unknown_remote_returns_404(self, client, patched_deps): + def test_unknown_local_returns_404(self, client, patched_deps): response = client.put( - "/api/v1/remote/nonexistent/path/to/file.tar.gz", + "/api/v1/local/nonexistent/path/to/file.tar.gz", files={"file": ("file.tar.gz", b"content", "application/octet-stream")}, ) assert response.status_code == 404 - def test_non_local_remote_returns_400(self, client, patched_deps): - response = client.put( - "/api/v1/remote/generic-test/path/to/file.tar.gz", - files={"file": ("file.tar.gz", b"content", "application/octet-stream")}, - ) - assert response.status_code == 400 - # --------------------------------------------------------------------------- -# HEAD route HEAD /api/v1/remote/{remote}/{path} +# HEAD route HEAD /api/v1/local/{local}/{path} # --------------------------------------------------------------------------- class TestHeadRoute: - def test_non_local_remote_returns_405(self, client, patched_deps): - response = client.head("/api/v1/remote/generic-test/path/to/file.tar.gz") - assert response.status_code == 405 - def test_local_repo_file_not_found_returns_404(self, client, patched_deps): deps = patched_deps deps["database"].get_local_file_metadata.return_value = None deps["database"].available = True - response = client.head("/api/v1/remote/local-test/path/to/nonexistent.bin") + response = client.head("/api/v1/local/local-test/path/to/nonexistent.bin") assert response.status_code == 404 - def test_unknown_remote_returns_404(self, client, patched_deps): - response = client.head("/api/v1/remote/nonexistent/path/to/file.bin") + def test_unknown_local_returns_404(self, client, patched_deps): + response = client.head("/api/v1/local/nonexistent/path/to/file.bin") assert response.status_code == 404 # --------------------------------------------------------------------------- -# DELETE route DELETE /api/v1/remote/{remote}/{path} +# DELETE route DELETE /api/v1/local/{local}/{path} # --------------------------------------------------------------------------- class TestDeleteRoute: - def test_unknown_remote_returns_404(self, client, patched_deps): - response = client.delete("/api/v1/remote/nonexistent/path/to/file.tar.gz") + def test_unknown_local_returns_404(self, client, patched_deps): + response = client.delete("/api/v1/local/nonexistent/path/to/file.tar.gz") assert response.status_code == 404 - def test_non_local_remote_returns_400(self, client, patched_deps): - response = client.delete("/api/v1/remote/generic-test/path/to/file.tar.gz") - assert response.status_code == 400 - # --------------------------------------------------------------------------- # Cache flush PUT /cache/flush diff --git a/tests/test_virtual.py b/tests/test_virtual.py index 5094e2d..0a2dcaa 100644 --- a/tests/test_virtual.py +++ b/tests/test_virtual.py @@ -430,10 +430,10 @@ class TestVirtualRoute: response = client.get("/api/v1/virtual/no-such-virtual/index.yaml") assert response.status_code == 404 - def test_non_virtual_type_returns_400(self, client, patched_virtual_deps): - # helm-test is type "remote", not "virtual" + def test_non_virtual_name_returns_404(self, client, patched_virtual_deps): + # helm-test is in remotes, not virtuals response = client.get("/api/v1/virtual/helm-test/index.yaml") - assert response.status_code == 400 + assert response.status_code == 404 def test_unsupported_package_returns_400(self, client, patched_virtual_deps): # unsupported-virtual-test has package "rpm"