Compare commits
1 Commits
v2.5.0
..
7124fb5f53
| Author | SHA1 | Date | |
|---|---|---|---|
 unkinben
|
7124fb5f53 |
@@ -35,6 +35,7 @@ env/
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
remotes.yaml
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
@@ -5,3 +5,13 @@ repos:
|
||||
- id: ruff
|
||||
args: [--fix, --exit-non-zero-on-fix]
|
||||
- id: ruff-format
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: docker-build
|
||||
name: docker build
|
||||
language: system
|
||||
entry: buildah --storage-driver vfs bud -t artifactapi:pre-commit-test .
|
||||
pass_filenames: false
|
||||
files: ^(Dockerfile|\.dockerignore|pyproject\.toml|src/)
|
||||
stages: [pre-commit]
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: docker-build
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
repo: git.unkin.net/unkin/artifactapi
|
||||
dry_run: true
|
||||
@@ -5,5 +5,5 @@ steps:
|
||||
- name: pre-commit
|
||||
image: git.unkin.net/unkin/almalinux9-base:20260308
|
||||
commands:
|
||||
- dnf install -y buildah
|
||||
- uvx pre-commit run --all-files
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
.PHONY: build install dev clean test lint format pre-commit tox docker-build docker-up docker-down docker-logs docker-rebuild docker-clean docker-restart
|
||||
|
||||
build:
|
||||
docker build -t artifactapi:dev .
|
||||
docker build --no-cache -t artifactapi:latest .
|
||||
|
||||
install: build
|
||||
|
||||
@@ -74,3 +74,4 @@ major:
|
||||
|
||||
_tag:
|
||||
git push origin $(TAG)
|
||||
docker-compose build --no-cache --build-arg VERSION=$(TAG:v%=%)
|
||||
|
||||
@@ -6,15 +6,10 @@ A generic FastAPI-based artifact caching system that downloads and stores files
|
||||
|
||||
- **Generic Remote Support**: Works with any HTTP-based file server (GitHub, Gitea, HashiCorp, custom servers)
|
||||
- **Configuration-Based**: YAML configuration for remotes, patterns, and access control
|
||||
- **Direct URL API**: Access cached files via clean URLs like `/api/v1/remote/github/owner/repo/path/file.tar.gz`
|
||||
- **Immutable/Mutable Pattern Model**: Per-remote regex patterns distinguish forever-cached artifacts from TTL-expiring metadata
|
||||
- **Direct URL API**: Access cached files via clean URLs like `/api/github/owner/repo/path/file.tar.gz`
|
||||
- **Pattern Filtering**: Regex-based inclusion patterns for security and organization
|
||||
- **Smart Caching**: Automatic download and cache on first access, serve from cache afterward
|
||||
- **Conditional Revalidation**: Optional `check_mutable_updates` flag — sends `If-None-Match`/`If-Modified-Since` on expiry; skips re-download on 304
|
||||
- **Stale-on-Upstream-Error**: Expired mutable files are kept and their TTL refreshed when the backend cannot be reached, so cached data remains available during upstream outages
|
||||
- **S3 Storage**: MinIO/S3 backend with predictable paths
|
||||
- **Docker Registry Proxy**: Full Docker Registry HTTP API v2 for transparent container image caching
|
||||
- **npm Package Proxy**: Caching proxy for the npm registry with metadata URL rewriting so tarballs also pass through cache
|
||||
- **Helm Chart Repository Proxy**: Caching proxy for Helm chart repositories with `index.yaml` URL rewriting so chart tarballs also pass through cache
|
||||
- **Content-Type Detection**: Automatic MIME type detection for downloads
|
||||
|
||||
## Architecture
|
||||
@@ -76,18 +71,15 @@ The system uses `remotes.yaml` to define remote repositories and access patterns
|
||||
remotes:
|
||||
remote-name:
|
||||
base_url: "https://example.com" # Base URL for the remote
|
||||
type: "remote" # "remote" or "local"
|
||||
package: "generic" # "generic", "alpine", "rpm", or "docker"
|
||||
type: "remote" # Type: "remote" or "local"
|
||||
package: "generic" # Package type: "generic", "alpine", "rpm"
|
||||
description: "Human readable description"
|
||||
immutable_patterns: # Files cached forever (release binaries, versioned tags)
|
||||
include_patterns: # Regex patterns for allowed files
|
||||
- "pattern1"
|
||||
- "pattern2"
|
||||
mutable_patterns: # Files that expire after mutable_ttl (optional)
|
||||
- "pattern3"
|
||||
check_mutable_updates: false # Enable conditional HEAD before re-fetching (optional)
|
||||
cache:
|
||||
immutable_ttl: 0 # TTL for immutable files (0 = indefinitely)
|
||||
mutable_ttl: 3600 # TTL in seconds for mutable files
|
||||
cache: # Cache configuration (optional)
|
||||
file_ttl: 0 # File cache TTL (0 = indefinite)
|
||||
index_ttl: 300 # Index file TTL in seconds
|
||||
```
|
||||
|
||||
### Remote Types
|
||||
@@ -102,30 +94,30 @@ remotes:
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "GitHub releases and files"
|
||||
immutable_patterns:
|
||||
include_patterns:
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
- "lxc/incus/.*\\.tar\\.gz$"
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Cache files indefinitely
|
||||
file_ttl: 0 # Cache files indefinitely
|
||||
index_ttl: 0 # No index files for generic remotes
|
||||
|
||||
github-archive:
|
||||
base_url: "https://github.com"
|
||||
hashicorp-releases:
|
||||
base_url: "https://releases.hashicorp.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "GitHub repository archive tarballs"
|
||||
immutable_patterns:
|
||||
- ".*/archive/refs/tags/.*\\.tar\\.gz$" # tag archives never change
|
||||
mutable_patterns:
|
||||
- ".*/archive/refs/heads/main\\.tar\\.gz$" # branch archives can change
|
||||
check_mutable_updates: true # send If-None-Match on expiry; skip re-download on 304
|
||||
description: "HashiCorp product releases"
|
||||
include_patterns:
|
||||
- "terraform/.*terraform_.*_linux_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_linux_amd64\\.zip$"
|
||||
- "consul/.*/consul_.*_linux_amd64\\.zip$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 86400 # re-check branch archives after 1 day
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
```
|
||||
|
||||
#### Package Repository Remotes
|
||||
For Linux package repositories:
|
||||
For Linux package repositories with index files:
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
@@ -134,25 +126,23 @@ remotes:
|
||||
type: "remote"
|
||||
package: "alpine"
|
||||
description: "Alpine Linux APK package repository"
|
||||
immutable_patterns:
|
||||
- ".*/x86_64/.*\\.apk$" # packages are immutable by content-hash
|
||||
# APKINDEX.tar.gz is a package-type default mutable file — no mutable_patterns needed
|
||||
include_patterns:
|
||||
- ".*/x86_64/.*\\.apk$" # Only x86_64 packages
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 7200 # re-fetch APKINDEX.tar.gz after 2 hours
|
||||
file_ttl: 0 # Cache packages indefinitely
|
||||
index_ttl: 7200 # Cache APKINDEX.tar.gz for 2 hours
|
||||
|
||||
almalinux:
|
||||
base_url: "https://mirror.example.com/almalinux"
|
||||
base_url: "http://mirror.aarnet.edu.au/pub/almalinux"
|
||||
type: "remote"
|
||||
package: "rpm"
|
||||
description: "AlmaLinux RPM package repository"
|
||||
immutable_patterns:
|
||||
include_patterns:
|
||||
- ".*/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
# repomd.xml and repodata/* are package-type defaults
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 7200
|
||||
file_ttl: 0
|
||||
index_ttl: 7200 # Cache metadata files for 2 hours
|
||||
```
|
||||
|
||||
#### Local Repositories
|
||||
@@ -165,45 +155,62 @@ remotes:
|
||||
package: "generic"
|
||||
description: "Local generic file repository"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 0
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
```
|
||||
|
||||
### Immutable Patterns
|
||||
### Include Patterns
|
||||
|
||||
`immutable_patterns` are regular expressions that control which files can be accessed. Patterns use Python `re.search`, so they match anywhere in the path unless anchored with `^` or `$`. Only files matching at least one pattern are served; all others return HTTP 403.
|
||||
|
||||
Matched files are cached with `immutable_ttl` (default 0 = forever). Use these for versioned release artifacts that never change once published.
|
||||
Include patterns are regular expressions that control which files can be accessed. Patterns use Python `re.search`, so they match anywhere in the path unless anchored with `^` or `$`. Only files matching at least one pattern are served; all others return HTTP 403.
|
||||
|
||||
```yaml
|
||||
immutable_patterns:
|
||||
include_patterns:
|
||||
# Exact project + architecture — most restrictive
|
||||
- "^gruntwork-io/terragrunt/releases/download/.*/terragrunt_linux_amd64$"
|
||||
|
||||
# Any release asset for a project, any version
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
|
||||
# File extension only — allow all files of a given type from any path
|
||||
- ".*\\.tar\\.gz$"
|
||||
- ".*\\.rpm$"
|
||||
- ".*\\.zip$"
|
||||
|
||||
# Architecture subtree — allow everything under x86_64/
|
||||
- ".*/x86_64/.*"
|
||||
|
||||
# Combined: architecture + extension
|
||||
- ".*/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
|
||||
# Docker image names (used with package: docker remotes)
|
||||
- "^library/nginx" # nginx official images only
|
||||
- "^rancher/" # all rancher/* images
|
||||
- "^rancher/rke2-runtime" # specific image
|
||||
|
||||
# Repodata directories — allow all metadata for an RPM repo
|
||||
- ".*/repodata/.*$"
|
||||
```
|
||||
|
||||
**Security note**: Omitting `immutable_patterns` entirely allows all files from that remote.
|
||||
**Security note**: Omitting `include_patterns` entirely allows all files from that remote. Index files (e.g. `APKINDEX.tar.gz`, `repomd.xml`, tag manifests) always bypass pattern enforcement — they are served unconditionally so clients can discover available packages.
|
||||
|
||||
### Mutable Patterns
|
||||
### Index Patterns
|
||||
|
||||
`mutable_patterns` identify files that change over time (index files, branch archives, metadata). Mutable files:
|
||||
- **Always served** regardless of `immutable_patterns`
|
||||
- **Cached with `mutable_ttl`** and re-fetched from upstream when the TTL expires
|
||||
- **Kept stale** when the upstream backend is unreachable — TTL is refreshed automatically so the cached copy remains available until the backend recovers (see below)
|
||||
Index patterns identify repository metadata files. Index files get special treatment:
|
||||
- **Always served** regardless of `include_patterns`
|
||||
- **Cached with `index_ttl`** instead of `file_ttl`
|
||||
- **Automatically refreshed** when the TTL expires — the cached copy is evicted and re-fetched on next request
|
||||
|
||||
Built-in defaults per package type (no configuration needed):
|
||||
Built-in defaults per package type:
|
||||
|
||||
| Package type | Built-in mutable patterns |
|
||||
| Package type | Built-in index patterns |
|
||||
|---|---|
|
||||
| `alpine` | `APKINDEX\.tar\.gz$` |
|
||||
| `rpm` | `repomd\.xml$`, `repodata/` metadata (xml, sqlite, yaml, asc, txt variants), `Packages\.gz$` |
|
||||
| `docker` | Tag manifests (non-digest refs), `/tags/list` |
|
||||
| `generic` | *(none)* |
|
||||
|
||||
Use `mutable_patterns` to add extra patterns on top of the defaults. Duplicates are ignored automatically.
|
||||
Use `index_patterns` to add extra patterns on top of the defaults. Duplicates are ignored automatically.
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
@@ -211,74 +218,60 @@ remotes:
|
||||
base_url: "https://charts.example.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
immutable_patterns:
|
||||
- ".*\\.tgz$"
|
||||
mutable_patterns:
|
||||
- "index\\.yaml$" # Helm repo index
|
||||
include_patterns:
|
||||
- ".*\\.tgz$" # chart archives
|
||||
index_patterns:
|
||||
- "index\\.yaml$" # Helm repo index — re-fetched on every TTL expiry
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600 # re-check the index every 10 minutes
|
||||
file_ttl: 0
|
||||
index_ttl: 600 # re-check the index every 10 minutes
|
||||
|
||||
apt-mirror:
|
||||
base_url: "https://apt.example.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
immutable_patterns:
|
||||
include_patterns:
|
||||
- ".*\\.deb$"
|
||||
mutable_patterns:
|
||||
- "InRelease$"
|
||||
- "Release$"
|
||||
- "Packages\\.gz$"
|
||||
index_patterns:
|
||||
- "InRelease$" # signed APT release file
|
||||
- "Release$" # unsigned APT release file
|
||||
- "Packages\\.gz$" # compressed package list
|
||||
- "Packages\\.xz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
```
|
||||
file_ttl: 0
|
||||
index_ttl: 3600 # hourly index refresh
|
||||
|
||||
### Conditional Revalidation (`check_mutable_updates`)
|
||||
|
||||
By default, when a mutable file's TTL expires the cached copy is evicted and the full file is re-downloaded on the next request. Setting `check_mutable_updates: true` on a remote enables a cheaper conditional check first:
|
||||
|
||||
1. On TTL expiry, a `HEAD` request is sent to the upstream with `If-None-Match` / `If-Modified-Since` headers (populated from the original download).
|
||||
2. If the upstream replies **304 Not Modified**, the TTL is refreshed in place — no re-download, no S3 traffic.
|
||||
3. If the upstream replies **200**, the cached copy is evicted and re-downloaded normally.
|
||||
|
||||
This only applies to user-defined `mutable_patterns`. Package-type built-in patterns (APKINDEX, repomd.xml, Docker manifests) are always re-fetched unconditionally.
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
github-archive:
|
||||
base_url: "https://github.com"
|
||||
almalinux-with-extras:
|
||||
base_url: "https://mirror.example.com/almalinux"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
immutable_patterns:
|
||||
- ".*/archive/refs/tags/.*\\.tar\\.gz$"
|
||||
mutable_patterns:
|
||||
- ".*/archive/refs/heads/main\\.tar\\.gz$"
|
||||
check_mutable_updates: true
|
||||
package: "rpm" # inherits repomd.xml + repodata/* defaults
|
||||
include_patterns:
|
||||
- ".*/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
index_patterns:
|
||||
- "comps\\.xml$" # optional group metadata (adds to rpm defaults)
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 86400
|
||||
file_ttl: 0
|
||||
index_ttl: 7200
|
||||
```
|
||||
|
||||
### Stale-on-Upstream-Error
|
||||
|
||||
When a mutable file's TTL expires and the upstream backend **cannot be reached** (connection refused, DNS failure, timeout), the cached copy is **kept and its TTL refreshed** rather than evicted. This means:
|
||||
|
||||
- RPM repodata, Alpine indexes, branch archives, and other mutable files remain available during upstream outages.
|
||||
- Clients continue to receive the last-known-good copy without errors.
|
||||
- Once the backend recovers and the refreshed TTL next expires, normal eviction resumes.
|
||||
|
||||
This behaviour is automatic and requires no configuration. Only network-level failures trigger it — HTTP error responses (404, 503, etc.) are treated as the backend being reachable and proceed with normal expiry.
|
||||
Pattern matching uses `re.search`, so `"index\\.yaml$"` matches `/stable/index.yaml` and `/index.yaml`. Anchor with `^` to restrict to the path root.
|
||||
|
||||
### Cache Configuration
|
||||
|
||||
Control how long different file types are cached:
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
immutable_ttl: 0 # Immutable files (0 = cache indefinitely, rarely changed)
|
||||
mutable_ttl: 3600 # Mutable files — TTL in seconds before re-fetch is attempted
|
||||
file_ttl: 0 # Regular files (0 = cache indefinitely)
|
||||
index_ttl: 300 # Index files like APKINDEX.tar.gz (seconds)
|
||||
```
|
||||
|
||||
**Index Files**: Repository metadata files that change frequently:
|
||||
- Alpine: `APKINDEX.tar.gz`
|
||||
- RPM: `repomd.xml`, `*-primary.xml.gz`, etc.
|
||||
- These are automatically detected and use `index_ttl`
|
||||
|
||||
### Environment Variables
|
||||
|
||||
All runtime configuration comes from environment variables:
|
||||
@@ -358,26 +351,26 @@ data:
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "GitHub releases and files"
|
||||
immutable_patterns:
|
||||
include_patterns:
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
- "lxc/incus/.*\\.tar\\.gz$"
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 0
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
|
||||
hashicorp-releases:
|
||||
base_url: "https://releases.hashicorp.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "HashiCorp product releases"
|
||||
immutable_patterns:
|
||||
include_patterns:
|
||||
- "terraform/.*terraform_.*_linux_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_linux_amd64\\.zip$"
|
||||
- "consul/.*/consul_.*_linux_amd64\\.zip$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 0
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
```
|
||||
|
||||
### 3. Secret for Environment Variables
|
||||
@@ -785,8 +778,8 @@ remotes:
|
||||
username: "your-dockerhub-username"
|
||||
password: "your-dockerhub-token" # PAT with read scope
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 300
|
||||
file_ttl: 0
|
||||
index_ttl: 300
|
||||
```
|
||||
|
||||
A pull of `nginx:latest` becomes `/v2/dockerhub/library/nginx/manifests/latest` on the artifact API.
|
||||
@@ -811,8 +804,8 @@ remotes:
|
||||
username: "your-github-username"
|
||||
password: "ghp_your_github_pat" # read:packages scope required
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 300
|
||||
file_ttl: 0
|
||||
index_ttl: 300
|
||||
```
|
||||
|
||||
A pull of `ghcr.io/rancher/rke2-runtime:v1.30.0-rke2r1` becomes `/v2/ghcr/rancher/rke2-runtime/manifests/v1.30.0-rke2r1`.
|
||||
@@ -851,7 +844,7 @@ Each entry needs a matching remote in `remotes.yaml` using the name from the rew
|
||||
|
||||
#### Restricting which images are cached
|
||||
|
||||
Use `immutable_patterns` on the remote to allow only specific images through the proxy. Requests for images not matching any pattern return HTTP 403 to the node.
|
||||
Use `include_patterns` on the remote to allow only specific images through the proxy. Requests for images not matching any pattern return HTTP 403 to the node.
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
@@ -859,17 +852,17 @@ remotes:
|
||||
base_url: "https://registry-1.docker.io"
|
||||
type: "remote"
|
||||
package: "docker"
|
||||
immutable_patterns:
|
||||
include_patterns:
|
||||
- "^library/nginx" # official nginx only
|
||||
- "^library/redis" # official redis only
|
||||
- "^rancher/" # all rancher images
|
||||
- "^grafana/grafana" # specific image
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 300
|
||||
file_ttl: 0
|
||||
index_ttl: 300
|
||||
```
|
||||
|
||||
Omit `immutable_patterns` to allow all images from that registry.
|
||||
Omit `include_patterns` to allow all images from that registry.
|
||||
|
||||
#### TLS configuration
|
||||
|
||||
@@ -933,216 +926,4 @@ curl -I https://artifacts.example.com/v2/dockerhub/library/nginx/manifests/lates
|
||||
|
||||
# Check what's stored in the cache
|
||||
curl https://artifacts.example.com/ | jq '.remotes'
|
||||
```
|
||||
|
||||
## Python Package Proxy with uv
|
||||
|
||||
The `pypi` package type turns the artifact API into a caching PyPI proxy. Simple index pages (`/simple/{package}/`) are mutable and expire after `mutable_ttl`; package files (wheels, sdists, metadata) are immutable and cached forever. URLs in the simple index HTML are rewritten on the fly to point back through the proxy, so both the index lookup and the file download are served from cache.
|
||||
|
||||
### remotes.yaml
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
pypi:
|
||||
base_url: "https://pypi.org"
|
||||
type: "remote"
|
||||
package: "pypi"
|
||||
pypi_files_url: "https://files.pythonhosted.org" # host to rewrite in index HTML
|
||||
pypi_files_remote: "pypi-files" # our proxy remote to replace it with
|
||||
check_mutable_updates: true
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600 # re-check simple indexes after 10 minutes
|
||||
|
||||
pypi-files:
|
||||
base_url: "https://files.pythonhosted.org"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
immutable_patterns:
|
||||
- "packages/.*\\.whl$"
|
||||
- "packages/.*\\.whl\\.metadata$"
|
||||
- "packages/.*\\.tar\\.gz$"
|
||||
- "packages/.*\\.zip$"
|
||||
- "packages/.*\\.egg$"
|
||||
cache:
|
||||
immutable_ttl: 0 # package files are content-addressed — cache forever
|
||||
|
||||
# Self-hosted Gitea PyPI registry (index and files share the same base URL)
|
||||
pypi-gitea:
|
||||
base_url: "https://gitea.example.com/api/packages/myorg/pypi"
|
||||
type: "remote"
|
||||
package: "pypi"
|
||||
# username: "your-gitea-username"
|
||||
# password: "your-personal-access-token" # needs package:read scope
|
||||
pypi_files_url: "https://gitea.example.com/api/packages/myorg/pypi"
|
||||
pypi_files_remote: "pypi-gitea" # point back to itself — Gitea serves both index and files
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "files/.*\\.whl$"
|
||||
- "files/.*\\.whl\\.metadata$"
|
||||
- "files/.*\\.tar\\.gz$"
|
||||
- "files/.*\\.zip$"
|
||||
- "files/.*\\.egg$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600
|
||||
```
|
||||
|
||||
### Configuring uv system- or user-wide
|
||||
|
||||
uv reads `uv.toml` from two locations outside any project, applied in order from broadest to narrowest scope:
|
||||
|
||||
| Scope | Path (Linux/macOS) |
|
||||
|---|---|
|
||||
| System | `/etc/uv/uv.toml` |
|
||||
| User | `~/.config/uv/uv.toml` |
|
||||
|
||||
Use these files to route **all** package installs on a machine through the proxy without touching individual projects or their `pyproject.toml`.
|
||||
|
||||
**`/etc/uv/uv.toml`** — applies to every user on the host:
|
||||
|
||||
```toml
|
||||
# Replace the default PyPI index with the caching proxy
|
||||
[[index]]
|
||||
url = "https://artifacts.example.com/api/v1/remote/pypi/simple"
|
||||
default = true
|
||||
|
||||
# Optionally add a private index (searched alongside the default)
|
||||
[[index]]
|
||||
url = "https://artifacts.example.com/api/v1/remote/pypi-gitea/simple"
|
||||
name = "gitea"
|
||||
```
|
||||
|
||||
**`~/.config/uv/uv.toml`** — same syntax, single-user scope:
|
||||
|
||||
```toml
|
||||
[[index]]
|
||||
url = "https://artifacts.example.com/api/v1/remote/pypi/simple"
|
||||
default = true
|
||||
```
|
||||
|
||||
Setting `default = true` replaces uv's built-in PyPI index. The first install of a package fetches it from upstream and populates the cache; every subsequent install — from any machine or fresh environment pointing at the same proxy — is served directly from S3.
|
||||
|
||||
### How the rewriting works
|
||||
|
||||
When uv requests the simple index for a package, the proxy:
|
||||
|
||||
1. Fetches `https://pypi.org/simple/{package}/` (or returns a valid cached copy within `mutable_ttl`)
|
||||
2. Rewrites every `https://files.pythonhosted.org/...` href to `https://artifacts.example.com/api/v1/remote/pypi-files/...`
|
||||
3. Returns the rewritten HTML to uv
|
||||
|
||||
uv then downloads wheels and `.whl.metadata` files via the rewritten URLs, which also pass through the proxy and are cached as immutable artifacts.
|
||||
|
||||
For self-hosted registries like Gitea, both the index and file downloads share the same base URL. Setting `pypi_files_url` and `pypi_files_remote` to the same remote causes file links to be rewritten back through the same proxy entry.
|
||||
|
||||
## npm Package Proxy
|
||||
|
||||
The `npm` package type turns the artifact API into a caching npm registry proxy. Since the npm registry serves both metadata and tarballs from the same host, a single remote handles everything. Package metadata (e.g. `GET /express`) is mutable and expires after `mutable_ttl`; tarballs (`.tgz`) are immutable and cached forever. `dist.tarball` URLs in metadata JSON are rewritten on the fly to point back through the same remote, so both the metadata lookup and the tarball download are served from cache.
|
||||
|
||||
### remotes.yaml
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
npm:
|
||||
base_url: "https://registry.npmjs.org"
|
||||
type: "remote"
|
||||
package: "npm"
|
||||
npm_files_url: "https://registry.npmjs.org" # URL prefix to rewrite in metadata JSON
|
||||
npm_files_remote: "npm" # rewrite back to this same remote
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\.tgz$" # versioned tarballs are content-addressed — cache forever
|
||||
mutable_patterns:
|
||||
- "^(?!.*\.tgz$).*" # everything else (package metadata) expires after mutable_ttl
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600 # re-check package metadata after 10 minutes
|
||||
```
|
||||
|
||||
### Configuring npm / yarn / pnpm
|
||||
|
||||
**npm** — per-project `.npmrc` or `~/.npmrc`:
|
||||
|
||||
```ini
|
||||
registry=https://artifacts.example.com/api/v1/remote/npm/
|
||||
```
|
||||
|
||||
**yarn** — `~/.yarnrc.yml`:
|
||||
|
||||
```yaml
|
||||
npmRegistryServer: "https://artifacts.example.com/api/v1/remote/npm/"
|
||||
```
|
||||
|
||||
**pnpm** — `.npmrc`:
|
||||
|
||||
```ini
|
||||
registry=https://artifacts.example.com/api/v1/remote/npm/
|
||||
```
|
||||
|
||||
### How the rewriting works
|
||||
|
||||
When a client requests package metadata, the proxy:
|
||||
|
||||
1. Fetches `https://registry.npmjs.org/{package}` (or returns a cached copy within `mutable_ttl`)
|
||||
2. Rewrites every `https://registry.npmjs.org/...` tarball URL to `https://artifacts.example.com/api/v1/remote/npm/...`
|
||||
3. Returns the rewritten JSON to the client
|
||||
|
||||
The client then downloads the tarball via the rewritten URL, which hits the same `npm` remote and is cached as an immutable artifact. Subsequent installs of the same package version are served entirely from S3.
|
||||
|
||||
### Mutable vs immutable paths
|
||||
|
||||
| Path pattern | Type | Example |
|
||||
|---|---|---|
|
||||
| `/{package}` | Mutable (TTL) | `/express` |
|
||||
| `/@{scope}/{package}` | Mutable (TTL) | `/@babel/core` |
|
||||
| `/-/all` | Mutable (TTL) | `/-/all` |
|
||||
| `/{package}/-/{package}-{version}.tgz` | Immutable (forever) | `/express/-/express-4.18.2.tgz` |
|
||||
| `/@{scope}/{pkg}/-/{pkg}-{ver}.tgz` | Immutable (forever) | `/@babel/core/-/core-7.21.0.tgz` |
|
||||
|
||||
## Helm Chart Repository Proxy
|
||||
|
||||
The `helm` package type turns the artifact API into a caching Helm chart repository proxy. A single remote handles both the mutable `index.yaml` and the immutable versioned chart tarballs, since they are served from the same upstream host. Chart URLs inside `index.yaml` are rewritten on the fly to point back through the same remote, so both the index lookup and the chart download are served from cache.
|
||||
|
||||
### remotes.yaml
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
hashicorp-helm:
|
||||
base_url: "https://helm.releases.hashicorp.com"
|
||||
type: "remote"
|
||||
package: "helm"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$" # chart tarballs — cache forever
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600 # index.yaml refreshed after 1 hour
|
||||
```
|
||||
|
||||
### Configuring Helm
|
||||
|
||||
Point Helm at the proxy with `helm repo add`:
|
||||
|
||||
```bash
|
||||
helm repo add hashicorp https://artifacts.example.com/api/v1/remote/hashicorp-helm
|
||||
helm repo update
|
||||
helm search repo hashicorp/vault
|
||||
helm install vault hashicorp/vault
|
||||
```
|
||||
|
||||
### How the rewriting works
|
||||
|
||||
When a client requests `index.yaml`, the proxy:
|
||||
|
||||
1. Fetches `https://helm.releases.hashicorp.com/index.yaml` (or returns a cached copy within `mutable_ttl`)
|
||||
2. Rewrites every `https://helm.releases.hashicorp.com/...` chart URL to `https://artifacts.example.com/api/v1/remote/hashicorp-helm/...`
|
||||
3. Returns the rewritten YAML to the client
|
||||
|
||||
The client then downloads chart tarballs via the rewritten URLs, which hit the same `hashicorp-helm` remote and are cached as immutable artifacts. Subsequent installs of the same chart version are served entirely from S3.
|
||||
|
||||
### Mutable vs immutable paths
|
||||
|
||||
| Path | Type | Example |
|
||||
|---|---|---|
|
||||
| `index.yaml` | Mutable (TTL) | `index.yaml` |
|
||||
| `{chart}-{version}.tgz` | Immutable (forever) | `vault-0.29.1.tgz` |
|
||||
```
|
||||
+3
-3
@@ -6,12 +6,12 @@ services:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
- VERSION=2.2.2.dev0
|
||||
- VERSION=dev
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- ./remotes.yaml:/app/remotes.yaml:ro,z
|
||||
- ./ca-bundle.pem:/app/ca-bundle.pem:ro,z
|
||||
- ./remotes.yaml:/app/remotes.yaml:ro
|
||||
- ./ca-bundle.pem:/app/ca-bundle.pem:ro
|
||||
environment:
|
||||
- CONFIG_PATH=/app/remotes.yaml
|
||||
- DBHOST=postgres
|
||||
|
||||
-282
@@ -1,282 +0,0 @@
|
||||
# Example remotes configuration — copy and adapt for your environment.
|
||||
#
|
||||
# immutable_patterns: artifacts cached forever (e.g. release binaries, versioned tags).
|
||||
# mutable_patterns: artifacts that expire after cache.mutable_ttl seconds and are
|
||||
# re-fetched from upstream on next request (e.g. index files,
|
||||
# branch archives). Defaults to the package-type built-ins when
|
||||
# not set (APKINDEX, repomd.xml, Docker manifests, etc.).
|
||||
# cache:
|
||||
# immutable_ttl: TTL for immutable files (0 = forever, rarely needed to change).
|
||||
# mutable_ttl: TTL in seconds for mutable files. Omit to use the default (3600).
|
||||
#
|
||||
# WARNING: this file may contain credentials — do not commit real values.
|
||||
#
|
||||
# Global configuration
|
||||
#s3:
|
||||
# endpoint: "localhost:9000"
|
||||
# access_key: "minioadmin"
|
||||
# secret_key: "minioadmin"
|
||||
# bucket: "artifacts"
|
||||
# secure: false
|
||||
#
|
||||
#redis:
|
||||
# url: "redis://localhost:6379/0"
|
||||
#
|
||||
#database:
|
||||
# url: "postgresql://artifacts:artifacts123@localhost:5432/artifacts"
|
||||
#
|
||||
remotes:
|
||||
github:
|
||||
base_url: "https://github.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "GitHub releases and files"
|
||||
immutable_patterns:
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
- "lxc/incus/.*\\.tar\\.gz$"
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "VictoriaMetrics/VictoriaMetrics/.*/vmutils-linux-amd64-.*\\.tar\\.gz$"
|
||||
- "VictoriaMetrics/VictoriaMetrics/.*/victoria-metrics-linux-amd64-.*-cluster\\.tar\\.gz$"
|
||||
- "VictoriaMetrics/VictoriaMetrics/.*/victoria-logs-linux-amd64-.*\\.tar\\.gz$"
|
||||
- "VictoriaMetrics/VictoriaMetrics/.*/vlutils-linux-amd64-.*\\.tar\\.gz$"
|
||||
- "prometheus-community/bind_exporter/.*/bind_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "prometheus-community/pgbouncer_exporter/.*/pgbouncer_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "prometheus-community/postgres_exporter/.*/postgres_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "onedr0p/exportarr/.*/exportarr_.*_linux_amd64\\.tar\\.gz$"
|
||||
- "tynany/frr_exporter/.*/frr_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "camptocamp/prometheus-puppetdb-exporter/.*/prometheus-puppetdb-exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "grafana/jsonnet-language-server/.*/jsonnet-language-server_.*_linux_amd64$"
|
||||
- "helmfile/helmfile/.*/helmfile_.*_linux_amd64\\.tar\\.gz$"
|
||||
- "helmfile/vals/.*/vals_.*_linux_amd64\\.tar\\.gz$"
|
||||
- "openbao/openbao-plugins/.*/openbao-plugin-secrets-consul_linux_amd64_.*\\.tar\\.gz$"
|
||||
- "openbao/openbao-plugins/.*/openbao-plugin-secrets-nomad_linux_amd64_.*\\.tar\\.gz$"
|
||||
- "apple/foundationdb/.*/libfdb_c\\.x86_64\\.so$"
|
||||
- "stalwartlabs/stalwart/.*/stalwart-cli-x86_64-unknown-linux-gnu\\.tar\\.gz$"
|
||||
- "stalwartlabs/stalwart/.*/stalwart-foundationdb-x86_64-unknown-linux-gnu\\.tar\\.gz$"
|
||||
- "stalwartlabs/stalwart/.*/stalwart-x86_64-unknown-linux-gnu\\.tar\\.gz$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 0
|
||||
|
||||
github-archive:
|
||||
base_url: "https://github.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "GitHub repository archive tarballs"
|
||||
immutable_patterns:
|
||||
# Tag archives are immutable — a tag never changes
|
||||
- ".*/archive/refs/tags/.*\\.tar\\.gz$"
|
||||
mutable_patterns:
|
||||
# Branch archives can change on every push
|
||||
- ".*/archive/refs/heads/main\\.tar\\.gz$"
|
||||
- ".*/archive/refs/heads/master\\.tar\\.gz$"
|
||||
# Before re-downloading an expired branch archive, check whether it has
|
||||
# actually changed (304 Not Modified → just refresh the TTL, no transfer).
|
||||
# Only applies to user-defined mutable_patterns, not package-type defaults.
|
||||
check_mutable_updates: true
|
||||
cache:
|
||||
immutable_ttl: 0 # Tag archives cached indefinitely
|
||||
mutable_ttl: 86400 # Branch archives refreshed after 1 day
|
||||
|
||||
gitea-dl:
|
||||
base_url: "https://dl.gitea.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "Gitea download site"
|
||||
immutable_patterns:
|
||||
- "act_runner/.*/act_runner-.*-linux-amd64$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 0
|
||||
|
||||
hashicorp-releases:
|
||||
base_url: "https://releases.hashicorp.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "HashiCorp product releases"
|
||||
immutable_patterns:
|
||||
- "terraform/.*terraform_.*_linux_amd64\\.zip$"
|
||||
- "terraform/.*terraform_.*_windows_amd64\\.zip$"
|
||||
- "terraform/.*terraform_.*_darwin_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_linux_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_windows_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_darwin_amd64\\.zip$"
|
||||
- "consul-cni/.*/consul-cni_.*_linux_amd64\\.zip$"
|
||||
- "consul/.*/consul_.*_linux_amd64\\.zip$"
|
||||
- "nomad-autoscaler/.*/nomad-autoscaler_.*_linux_amd64\\.zip$"
|
||||
- "nomad/.*/nomad_.*_linux_amd64\\.zip$"
|
||||
- "packer/.*/packer_.*_linux_amd64\\.zip$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 0
|
||||
|
||||
alpine:
|
||||
base_url: "https://dl-cdn.alpinelinux.org"
|
||||
type: "remote"
|
||||
package: "alpine"
|
||||
description: "Alpine Linux APK package repository"
|
||||
immutable_patterns:
|
||||
- ".*/x86_64/.*\\.apk$"
|
||||
# check_mutable_updates not set: APKINDEX.tar.gz is a package-type default
|
||||
# and is always re-fetched on expiry — conditional checks are skipped for
|
||||
# built-in mutable patterns regardless of this flag.
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 7200 # Index files (APKINDEX.tar.gz) cached for 2 hours
|
||||
|
||||
almalinux:
|
||||
base_url: "https://gsl-syd.mm.fcix.net/almalinux"
|
||||
type: "remote"
|
||||
package: "rpm"
|
||||
description: "AlmaLinux RPM package repository"
|
||||
immutable_patterns:
|
||||
- ".*/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
- ".*/repodata/.*$"
|
||||
- ".*\\.rpm$" # Allow all RPM files
|
||||
# repomd.xml / repodata are package-type defaults — always re-fetched on
|
||||
# expiry. check_mutable_updates would only apply to any custom
|
||||
# mutable_patterns added here.
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 7200 # Metadata files cached for 2 hours
|
||||
|
||||
epel:
|
||||
base_url: "http://mirror.aarnet.edu.au/pub/epel"
|
||||
type: "remote"
|
||||
package: "rpm"
|
||||
description: "EPEL (Extra Packages for Enterprise Linux)"
|
||||
immutable_patterns:
|
||||
- "8/Everything/x86_64/.*\\.rpm$"
|
||||
- "9/Everything/x86_64/.*\\.rpm$"
|
||||
- "10/Everything/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
- ".*/repodata/.*$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 7200 # Metadata files cached for 2 hours
|
||||
|
||||
fedora:
|
||||
base_url: "https://gsl-syd.mm.fcix.net/fedora/linux"
|
||||
type: "remote"
|
||||
package: "rpm"
|
||||
description: "Fedora Linux RPM package repository"
|
||||
immutable_patterns:
|
||||
- "releases/.*/Everything/x86_64/.*\\.rpm$"
|
||||
- "updates/.*/Everything/x86_64/.*\\.rpm$"
|
||||
- "development/.*/Everything/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
- "updates/.*/Everything/x86_64/repodata/.*$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 300 # Metadata files cached for 5 minutes
|
||||
|
||||
ghcr:
|
||||
base_url: "https://ghcr.io"
|
||||
type: "remote"
|
||||
package: "docker"
|
||||
description: "GitHub Container Registry"
|
||||
# username: "your-github-username"
|
||||
# password: "your-github-pat" # needs read:packages scope
|
||||
# Docker manifest/tag-list patterns are package-type defaults — always
|
||||
# re-fetched on expiry. check_mutable_updates only applies to any custom
|
||||
# mutable_patterns you add (e.g. a metadata endpoint).
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 300
|
||||
|
||||
dockerhub:
|
||||
base_url: "https://registry-1.docker.io"
|
||||
type: "remote"
|
||||
package: "docker"
|
||||
description: "Docker Hub registry"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 300
|
||||
|
||||
pypi:
|
||||
base_url: "https://pypi.org"
|
||||
type: "remote"
|
||||
package: "pypi"
|
||||
description: "Python Package Index — simple repository API"
|
||||
# pypi_files_url: the upstream host used in simple-index hrefs (default: files.pythonhosted.org)
|
||||
# pypi_files_remote: our proxy remote that will serve those files (default: pypi-files)
|
||||
pypi_files_url: "https://files.pythonhosted.org"
|
||||
pypi_files_remote: "pypi-files"
|
||||
check_mutable_updates: true
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600 # Simple index pages refreshed after 10 minutes
|
||||
|
||||
pypi-gitea:
|
||||
base_url: "https://gitea.example.com/api/packages/myorg/pypi"
|
||||
type: "remote"
|
||||
package: "pypi"
|
||||
description: "Private Gitea PyPI registry"
|
||||
# username: "your-gitea-username"
|
||||
# password: "your-personal-access-token" # needs package:read scope
|
||||
# Files are served from the same Gitea instance — rewrite back to this same remote
|
||||
pypi_files_url: "https://gitea.example.com/api/packages/myorg/pypi"
|
||||
pypi_files_remote: "pypi-gitea"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "files/.*\\.whl$"
|
||||
- "files/.*\\.whl\\.metadata$"
|
||||
- "files/.*\\.tar\\.gz$"
|
||||
- "files/.*\\.zip$"
|
||||
- "files/.*\\.egg$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600
|
||||
|
||||
pypi-files:
|
||||
base_url: "https://files.pythonhosted.org"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "Python Package Index — file storage (wheels, sdists)"
|
||||
immutable_patterns:
|
||||
- "packages/.*\\.whl$"
|
||||
- "packages/.*\\.whl\\.metadata$"
|
||||
- "packages/.*\\.tar\\.gz$"
|
||||
- "packages/.*\\.zip$"
|
||||
- "packages/.*\\.egg$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Package files are content-addressed — cache forever
|
||||
|
||||
npm:
|
||||
base_url: "https://registry.npmjs.org"
|
||||
type: "remote"
|
||||
package: "npm"
|
||||
description: "npm registry — package metadata with tarball URL rewriting"
|
||||
# npm_files_url: the upstream host used in metadata tarball hrefs (default: https://registry.npmjs.org)
|
||||
# npm_files_remote: our proxy remote that will serve those tarballs (default: npm-files)
|
||||
npm_files_url: "https://registry.npmjs.org"
|
||||
npm_files_remote: "npm"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- \.tgz$
|
||||
mutable_patterns:
|
||||
- ^(?!.*\.tgz$).*
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600 # Package metadata refreshed after 10 minutes
|
||||
|
||||
hashicorp-helm:
|
||||
base_url: "https://helm.releases.hashicorp.com"
|
||||
type: "remote"
|
||||
package: "helm"
|
||||
description: "HashiCorp Helm chart repository (Vault, Consul, Nomad, etc.)"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Chart tarballs are versioned — cache forever
|
||||
mutable_ttl: 3600 # index.yaml refreshed after 1 hour
|
||||
|
||||
local-generic:
|
||||
type: "local"
|
||||
package: "generic"
|
||||
description: "Local generic file repository"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 0
|
||||
@@ -19,20 +19,18 @@ class RedisCache:
|
||||
self.client = None
|
||||
self.available = False
|
||||
|
||||
def is_mutable_file(self, file_path: str, patterns: list[str] | None = None) -> bool:
|
||||
"""Return True if file_path matches any of the mutable patterns."""
|
||||
def is_index_file(self, file_path: str, patterns: list[str] | None = None) -> bool:
|
||||
"""Return True if file_path matches any of the index patterns."""
|
||||
if patterns is None:
|
||||
patterns = []
|
||||
return any(re.search(p, file_path) for p in patterns)
|
||||
|
||||
def get_index_cache_key(self, remote_name: str, path: str) -> str:
|
||||
"""Generate cache key for index files"""
|
||||
return f"index:{remote_name}:{hashlib.sha256(path.encode()).hexdigest()[:16]}"
|
||||
|
||||
def get_mutable_meta_key(self, remote_name: str, path: str) -> str:
|
||||
return f"mutable:meta:{remote_name}:{hashlib.sha256(path.encode()).hexdigest()[:16]}"
|
||||
|
||||
def is_index_valid(self, remote_name: str, path: str) -> bool:
|
||||
"""Check if mutable file is still within its TTL window."""
|
||||
def is_index_valid(self, remote_name: str, path: str, ttl_override: int = None) -> bool:
|
||||
"""Check if index file is still valid (not expired)"""
|
||||
if not self.available:
|
||||
return False
|
||||
|
||||
@@ -43,7 +41,7 @@ class RedisCache:
|
||||
return False
|
||||
|
||||
def mark_index_cached(self, remote_name: str, path: str, ttl: int = 300) -> None:
|
||||
"""Set or refresh the TTL key for a mutable file."""
|
||||
"""Mark index file as cached with TTL"""
|
||||
if not self.available:
|
||||
return
|
||||
|
||||
@@ -53,45 +51,13 @@ class RedisCache:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def store_mutable_meta(self, remote_name: str, path: str, etag: str | None, last_modified: str | None) -> None:
|
||||
"""Persist ETag and Last-Modified for future conditional requests."""
|
||||
if not self.available:
|
||||
return
|
||||
data = {}
|
||||
if etag:
|
||||
data["etag"] = etag
|
||||
if last_modified:
|
||||
data["last_modified"] = last_modified
|
||||
if not data:
|
||||
return
|
||||
try:
|
||||
self.client.hset(self.get_mutable_meta_key(remote_name, path), mapping=data)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_mutable_meta(self, remote_name: str, path: str) -> dict:
|
||||
"""Return stored ETag/Last-Modified for a mutable file, or {}."""
|
||||
if not self.available:
|
||||
return {}
|
||||
try:
|
||||
return self.client.hgetall(self.get_mutable_meta_key(remote_name, path)) or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def delete_mutable_meta(self, remote_name: str, path: str) -> None:
|
||||
if not self.available:
|
||||
return
|
||||
try:
|
||||
self.client.delete(self.get_mutable_meta_key(remote_name, path))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def cleanup_expired_index(self, storage, remote_name: str, path: str) -> None:
|
||||
"""Remove an expired mutable file from S3 and clear its Redis meta."""
|
||||
"""Remove expired index from S3 storage"""
|
||||
if not self.available:
|
||||
return
|
||||
|
||||
try:
|
||||
# Construct the URL the same way as in the main flow
|
||||
import os
|
||||
|
||||
from .config import ConfigManager
|
||||
@@ -103,10 +69,9 @@ class RedisCache:
|
||||
if remote_config:
|
||||
base_url = remote_config.get("base_url")
|
||||
if base_url:
|
||||
# Use hierarchical path-based key (same as cache_single_artifact)
|
||||
s3_key = storage.get_object_key(remote_name, path)
|
||||
if storage.exists(s3_key):
|
||||
storage.client.delete_object(Bucket=storage.bucket, Key=s3_key)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.delete_mutable_meta(remote_name, path)
|
||||
|
||||
+15
-24
@@ -3,7 +3,7 @@ import os
|
||||
|
||||
import yaml
|
||||
|
||||
_PACKAGE_MUTABLE_PATTERNS: dict[str, list[str]] = {
|
||||
_PACKAGE_INDEX_PATTERNS: dict[str, list[str]] = {
|
||||
"alpine": [
|
||||
r"APKINDEX\.tar\.gz$",
|
||||
],
|
||||
@@ -18,13 +18,6 @@ _PACKAGE_MUTABLE_PATTERNS: dict[str, list[str]] = {
|
||||
r"/manifests/(?!sha256:)[^/]+$",
|
||||
r"/tags/list$",
|
||||
],
|
||||
"pypi": [
|
||||
r"simple/", # Per-package and top-level simple index pages
|
||||
],
|
||||
"npm": [],
|
||||
"helm": [
|
||||
r"index\.yaml$",
|
||||
],
|
||||
"generic": [],
|
||||
}
|
||||
|
||||
@@ -62,21 +55,26 @@ class ConfigManager:
|
||||
self._check_reload()
|
||||
return self.config.get("remotes", {}).get(remote_name)
|
||||
|
||||
def get_immutable_patterns(self, remote_name: str, repo_path: str = "") -> list[str]:
|
||||
def get_repository_patterns(self, remote_name: str, repo_path: str) -> list:
|
||||
remote_config = self.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
return []
|
||||
|
||||
repositories = remote_config.get("repositories", {})
|
||||
|
||||
# Handle both dict (GitHub style) and list (Alpine style) repositories
|
||||
if isinstance(repositories, dict):
|
||||
repo_config = repositories.get(repo_path)
|
||||
if repo_config:
|
||||
patterns = repo_config.get("immutable_patterns", [])
|
||||
patterns = repo_config.get("include_patterns", [])
|
||||
else:
|
||||
patterns = remote_config.get("immutable_patterns", [])
|
||||
patterns = remote_config.get("include_patterns", [])
|
||||
elif isinstance(repositories, list):
|
||||
# For Alpine, repositories is just a list of allowed repo names
|
||||
# Pattern matching is handled by the main include_patterns
|
||||
patterns = remote_config.get("include_patterns", [])
|
||||
else:
|
||||
patterns = remote_config.get("immutable_patterns", [])
|
||||
patterns = remote_config.get("include_patterns", [])
|
||||
|
||||
return patterns
|
||||
|
||||
@@ -131,25 +129,18 @@ class ConfigManager:
|
||||
db_url = f"postgresql://{db_user}:{db_pass}@{db_host}:{db_port}/{db_name}"
|
||||
return {"url": db_url}
|
||||
|
||||
def get_user_mutable_patterns(self, remote_name: str) -> list[str]:
|
||||
"""Return only user-configured mutable_patterns, excluding package-type defaults."""
|
||||
remote_config = self.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
return []
|
||||
return remote_config.get("mutable_patterns", [])
|
||||
|
||||
def get_mutable_patterns(self, remote_name: str) -> list[str]:
|
||||
"""Return mutable-file patterns for a remote (TTL is configured per-remote in cache.index_ttl).
|
||||
def get_index_patterns(self, remote_name: str) -> list[str]:
|
||||
"""Return index-file patterns for a remote.
|
||||
|
||||
Merges the package-level defaults with any extra patterns listed under
|
||||
``mutable_patterns`` in the remote's config.
|
||||
``index_patterns`` in the remote's config.
|
||||
"""
|
||||
remote_config = self.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
return []
|
||||
package = remote_config.get("package", "generic")
|
||||
defaults = _PACKAGE_MUTABLE_PATTERNS.get(package, [])
|
||||
extra = remote_config.get("mutable_patterns", [])
|
||||
defaults = _PACKAGE_INDEX_PATTERNS.get(package, [])
|
||||
extra = remote_config.get("index_patterns", [])
|
||||
return defaults + [p for p in extra if p not in defaults]
|
||||
|
||||
def get_cache_config(self, remote_name: str) -> dict:
|
||||
|
||||
+69
-179
@@ -1,4 +1,3 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
@@ -33,10 +32,6 @@ class ArtifactRequest(BaseModel):
|
||||
include_pattern: str
|
||||
|
||||
|
||||
class UpstreamUnreachable(Exception):
|
||||
"""Raised when the upstream backend cannot be contacted (network or timeout error)."""
|
||||
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -92,10 +87,8 @@ def flush_cache(
|
||||
if cache_type in ["all", "index"]:
|
||||
if remote:
|
||||
patterns.append(f"index:{remote}:*")
|
||||
patterns.append(f"mutable:meta:{remote}:*")
|
||||
else:
|
||||
patterns.append("index:*")
|
||||
patterns.append("mutable:meta:*")
|
||||
|
||||
if cache_type in ["all", "metrics"]:
|
||||
if remote:
|
||||
@@ -170,13 +163,13 @@ async def construct_remote_url(remote_name: str, path: str) -> str:
|
||||
|
||||
|
||||
async def check_artifact_patterns(remote_name: str, repo_path: str, file_path: str, full_path: str) -> bool:
|
||||
# Mutable files (index files) are always allowed through
|
||||
mutable_patterns = config.get_mutable_patterns(remote_name)
|
||||
if cache.is_mutable_file(file_path, mutable_patterns) or cache.is_mutable_file(full_path, mutable_patterns):
|
||||
# First check if this is an index file - always allow index files
|
||||
index_patterns = config.get_index_patterns(remote_name)
|
||||
if cache.is_index_file(file_path, index_patterns) or cache.is_index_file(full_path, index_patterns):
|
||||
return True
|
||||
|
||||
# Check immutable include patterns
|
||||
patterns = config.get_immutable_patterns(remote_name, repo_path)
|
||||
# Then check basic include patterns
|
||||
patterns = config.get_repository_patterns(remote_name, repo_path)
|
||||
if not patterns:
|
||||
return True # Allow all if no patterns configured
|
||||
|
||||
@@ -190,6 +183,7 @@ async def check_artifact_patterns(remote_name: str, repo_path: str, file_path: s
|
||||
if not pattern_matched:
|
||||
return False
|
||||
|
||||
# All remotes now use pattern-based filtering only - no additional checks needed
|
||||
return True
|
||||
|
||||
|
||||
@@ -209,11 +203,8 @@ async def cache_single_artifact(url: str, remote_name: str, path: str) -> dict:
|
||||
remote_config = config.get_remote_config(remote_name) or {}
|
||||
is_docker = remote_config.get("package") == "docker" or "/v2/" in url
|
||||
|
||||
# Prepare headers
|
||||
# Prepare headers for Docker registry requests
|
||||
headers = {}
|
||||
username = remote_config.get("username")
|
||||
password = remote_config.get("password")
|
||||
|
||||
if is_docker:
|
||||
if "/manifests/" in url:
|
||||
headers["Accept"] = (
|
||||
@@ -224,8 +215,6 @@ async def cache_single_artifact(url: str, remote_name: str, path: str) -> dict:
|
||||
)
|
||||
elif "/blobs/" in url:
|
||||
headers["Accept"] = "application/octet-stream"
|
||||
elif username and password:
|
||||
headers["Authorization"] = "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
@@ -252,148 +241,14 @@ async def cache_single_artifact(url: str, remote_name: str, path: str) -> dict:
|
||||
"storage_path": storage_path,
|
||||
"size": len(response.content),
|
||||
"status": "cached",
|
||||
"etag": response.headers.get("ETag"),
|
||||
"last_modified": response.headers.get("Last-Modified"),
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {"url": url, "status": "error", "error": str(e)}
|
||||
|
||||
|
||||
def _basic_auth_header(remote_cfg: dict) -> dict[str, str]:
|
||||
username = remote_cfg.get("username")
|
||||
password = remote_cfg.get("password")
|
||||
if username and password:
|
||||
token = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
return {"Authorization": f"Basic {token}"}
|
||||
return {}
|
||||
|
||||
|
||||
async def _upstream_reachable(url: str, auth_headers: dict | None = None) -> bool:
|
||||
"""HEAD with a short timeout. Returns False only on network/timeout errors."""
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
await client.head(url, headers=auth_headers or {}, timeout=10.0)
|
||||
return True
|
||||
except (httpx.NetworkError, httpx.TimeoutException):
|
||||
return False
|
||||
except Exception:
|
||||
return True # 4xx/5xx means backend is up
|
||||
|
||||
|
||||
async def check_upstream_changed(remote_url: str, remote_name: str, path: str, auth_headers: dict | None = None) -> bool:
|
||||
"""Conditional HEAD against upstream. Returns False only on a definitive 304.
|
||||
Raises UpstreamUnreachable if the backend cannot be contacted."""
|
||||
meta = cache.get_mutable_meta(remote_name, path)
|
||||
if not meta:
|
||||
return True
|
||||
|
||||
headers = dict(auth_headers or {})
|
||||
if meta.get("etag"):
|
||||
headers["If-None-Match"] = meta["etag"]
|
||||
if meta.get("last_modified"):
|
||||
headers["If-Modified-Since"] = meta["last_modified"]
|
||||
if not (meta.get("etag") or meta.get("last_modified")):
|
||||
return True
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
response = await client.head(remote_url, headers=headers)
|
||||
return response.status_code != 304
|
||||
except (httpx.NetworkError, httpx.TimeoutException) as exc:
|
||||
raise UpstreamUnreachable(str(exc)) from exc
|
||||
|
||||
|
||||
async def handle_expired_mutable(remote_name: str, path: str, remote_url: str) -> bool:
|
||||
"""Handle an expired mutable file. Returns True if the cached copy is still valid."""
|
||||
mutable_ttl = config.get_cache_config(remote_name).get("mutable_ttl", 3600)
|
||||
|
||||
remote_cfg = config.get_remote_config(remote_name) or {}
|
||||
auth = _basic_auth_header(remote_cfg)
|
||||
check_updates = remote_cfg.get("check_mutable_updates", False)
|
||||
user_mutable = check_updates and cache.is_mutable_file(path, config.get_user_mutable_patterns(remote_name))
|
||||
|
||||
if user_mutable:
|
||||
try:
|
||||
changed = await check_upstream_changed(remote_url, remote_name, path, auth)
|
||||
except UpstreamUnreachable:
|
||||
cache.mark_index_cached(remote_name, path, mutable_ttl)
|
||||
logger.warning(f"Mutable STALE (backend unreachable): {remote_name}/{path} - TTL extended ({mutable_ttl}s)")
|
||||
return True
|
||||
if not changed:
|
||||
cache.mark_index_cached(remote_name, path, mutable_ttl)
|
||||
logger.info(f"Mutable file UNCHANGED: {remote_name}/{path} - TTL refreshed ({mutable_ttl}s)")
|
||||
return True
|
||||
logger.info(f"Mutable file CHANGED: {remote_name}/{path} - re-downloading")
|
||||
else:
|
||||
if not await _upstream_reachable(remote_url, auth):
|
||||
cache.mark_index_cached(remote_name, path, mutable_ttl)
|
||||
logger.warning(f"Mutable STALE (backend unreachable): {remote_name}/{path} - TTL extended ({mutable_ttl}s)")
|
||||
return True
|
||||
logger.info(f"Mutable file EXPIRED: {remote_name}/{path} - removing from cache")
|
||||
|
||||
cache.cleanup_expired_index(storage, remote_name, path)
|
||||
return False
|
||||
|
||||
|
||||
def _get_content_type(filename: str) -> str:
|
||||
if filename.endswith((".tar.gz", ".tgz")):
|
||||
return "application/gzip"
|
||||
if filename.endswith(".zip") or filename.endswith(".whl"):
|
||||
return "application/zip"
|
||||
if filename.endswith(".exe"):
|
||||
return "application/x-msdownload"
|
||||
if filename.endswith(".rpm"):
|
||||
return "application/x-rpm"
|
||||
if filename.endswith(".xml"):
|
||||
return "application/xml"
|
||||
if filename.endswith((".xml.gz", ".xml.bz2", ".xml.xz")):
|
||||
return "application/gzip"
|
||||
if filename.endswith((".yaml", ".yml")):
|
||||
return "text/yaml"
|
||||
return "application/octet-stream"
|
||||
|
||||
|
||||
def _resolve_content(
|
||||
data: bytes,
|
||||
path: str,
|
||||
filename: str,
|
||||
remote_config: dict,
|
||||
request: Request,
|
||||
remote_name: str = "",
|
||||
) -> tuple[bytes, str]:
|
||||
"""Return (possibly-rewritten data, content_type) for a cached artifact."""
|
||||
if remote_config.get("package") == "pypi" and "simple/" in path:
|
||||
files_url = remote_config.get("pypi_files_url", "https://files.pythonhosted.org")
|
||||
files_remote = remote_config.get("pypi_files_remote", "pypi-files")
|
||||
proxy_base = str(request.base_url).rstrip("/")
|
||||
data = data.replace(
|
||||
files_url.rstrip("/").encode(),
|
||||
f"{proxy_base}/api/v1/remote/{files_remote}".encode(),
|
||||
)
|
||||
return data, "text/html; charset=utf-8"
|
||||
if remote_config.get("package") == "npm" and not path.endswith(".tgz"):
|
||||
files_url = remote_config.get("npm_files_url", "https://registry.npmjs.org")
|
||||
files_remote = remote_config.get("npm_files_remote", "npm-files")
|
||||
proxy_base = str(request.base_url).rstrip("/")
|
||||
data = data.replace(
|
||||
files_url.rstrip("/").encode(),
|
||||
f"{proxy_base}/api/v1/remote/{files_remote}".encode(),
|
||||
)
|
||||
return data, "application/json"
|
||||
if remote_config.get("package") == "helm" and filename == "index.yaml":
|
||||
proxy_base = str(request.base_url).rstrip("/")
|
||||
base_url = remote_config.get("base_url", "").rstrip("/")
|
||||
data = data.replace(
|
||||
base_url.encode(),
|
||||
f"{proxy_base}/api/v1/remote/{remote_name}".encode(),
|
||||
)
|
||||
return data, "text/yaml"
|
||||
return data, _get_content_type(filename)
|
||||
|
||||
|
||||
@app.get("/api/v1/remote/{remote_name}/{path:path}")
|
||||
async def get_artifact(request: Request, remote_name: str, path: str):
|
||||
async def get_artifact(remote_name: str, path: str):
|
||||
# Check if remote is configured
|
||||
remote_config = config.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
@@ -442,25 +297,46 @@ async def get_artifact(request: Request, remote_name: str, path: str):
|
||||
if not storage.exists(cached_key):
|
||||
cached_key = None
|
||||
|
||||
# For mutable files, check Redis TTL validity
|
||||
# For index files, check Redis TTL validity
|
||||
filename = os.path.basename(path)
|
||||
is_mutable = cache.is_mutable_file(path, config.get_mutable_patterns(remote_name))
|
||||
is_index = cache.is_index_file(path, config.get_index_patterns(remote_name))
|
||||
|
||||
if cached_key and is_mutable:
|
||||
if cached_key and is_index:
|
||||
# Index file exists, but check if it's still valid
|
||||
if not cache.is_index_valid(remote_name, path):
|
||||
if not await handle_expired_mutable(remote_name, path, remote_url):
|
||||
cached_key = None
|
||||
# Index has expired, remove it from S3
|
||||
logger.info(f"Index EXPIRED: {remote_name}/{path} - removing from cache")
|
||||
cache.cleanup_expired_index(storage, remote_name, path)
|
||||
cached_key = None # Force re-download
|
||||
|
||||
if cached_key:
|
||||
# Return cached artifact
|
||||
try:
|
||||
artifact_data = storage.download_object(cached_key)
|
||||
filename = os.path.basename(path)
|
||||
artifact_data, content_type = _resolve_content(artifact_data, path, filename, remote_config, request, remote_name)
|
||||
|
||||
# Log cache hit
|
||||
logger.info(f"Cache HIT: {remote_name}/{path} (size: {len(artifact_data)} bytes, key: {cached_key})")
|
||||
|
||||
# Determine content type based on file extension
|
||||
content_type = "application/octet-stream"
|
||||
if filename.endswith(".tar.gz"):
|
||||
content_type = "application/gzip"
|
||||
elif filename.endswith(".zip"):
|
||||
content_type = "application/zip"
|
||||
elif filename.endswith(".exe"):
|
||||
content_type = "application/x-msdownload"
|
||||
elif filename.endswith(".rpm"):
|
||||
content_type = "application/x-rpm"
|
||||
elif filename.endswith(".xml"):
|
||||
content_type = "application/xml"
|
||||
elif filename.endswith((".xml.gz", ".xml.bz2", ".xml.xz")):
|
||||
content_type = "application/gzip"
|
||||
|
||||
# Record cache hit metrics
|
||||
metrics.record_cache_hit(remote_name, len(artifact_data))
|
||||
|
||||
# Record artifact mapping in database if not already recorded
|
||||
database.record_artifact_mapping(cached_key, remote_name, path, len(artifact_data))
|
||||
|
||||
return Response(
|
||||
@@ -483,23 +359,38 @@ async def get_artifact(request: Request, remote_name: str, path: str):
|
||||
logger.error(f"Cache ADD FAILED: {remote_name}/{path} - {result['error']}")
|
||||
raise HTTPException(status_code=502, detail=f"Failed to fetch artifact: {result['error']}")
|
||||
|
||||
# Mark mutable files as cached in Redis with TTL
|
||||
if result["status"] == "cached" and is_mutable:
|
||||
# Mark index files as cached in Redis if this was a new download
|
||||
if result["status"] == "cached" and is_index:
|
||||
# Get TTL from remote config
|
||||
cache_config = config.get_cache_config(remote_name)
|
||||
mutable_ttl = cache_config.get("mutable_ttl", 3600)
|
||||
cache.mark_index_cached(remote_name, path, mutable_ttl)
|
||||
logger.info(f"Mutable file cached with TTL: {remote_name}/{path} (ttl: {mutable_ttl}s)")
|
||||
if result.get("etag") or result.get("last_modified"):
|
||||
cache.store_mutable_meta(remote_name, path, result.get("etag"), result.get("last_modified"))
|
||||
index_ttl = cache_config.get("index_ttl", 300) # Default 5 minutes
|
||||
cache.mark_index_cached(remote_name, path, index_ttl)
|
||||
logger.info(f"Index file cached with TTL: {remote_name}/{path} (ttl: {index_ttl}s)")
|
||||
|
||||
# Now return the cached artifact
|
||||
try:
|
||||
cache_key = storage.get_object_key(remote_name, path)
|
||||
artifact_data = storage.download_object(cache_key)
|
||||
filename = os.path.basename(path)
|
||||
artifact_data, content_type = _resolve_content(artifact_data, path, filename, remote_config, request, remote_name)
|
||||
|
||||
content_type = "application/octet-stream"
|
||||
if filename.endswith(".tar.gz"):
|
||||
content_type = "application/gzip"
|
||||
elif filename.endswith(".zip"):
|
||||
content_type = "application/zip"
|
||||
elif filename.endswith(".exe"):
|
||||
content_type = "application/x-msdownload"
|
||||
elif filename.endswith(".rpm"):
|
||||
content_type = "application/x-rpm"
|
||||
elif filename.endswith(".xml"):
|
||||
content_type = "application/xml"
|
||||
elif filename.endswith((".xml.gz", ".xml.bz2", ".xml.xz")):
|
||||
content_type = "application/gzip"
|
||||
|
||||
# Record cache miss metrics
|
||||
metrics.record_cache_miss(remote_name, len(artifact_data))
|
||||
|
||||
# Record artifact mapping in database
|
||||
cache_key = storage.get_object_key(remote_name, path)
|
||||
database.record_artifact_mapping(cache_key, remote_name, path, len(artifact_data))
|
||||
|
||||
@@ -533,8 +424,8 @@ async def docker_v2_proxy(request: Request, remote_name: str, path: str):
|
||||
if remote_config.get("package") != "docker":
|
||||
raise HTTPException(status_code=400, detail=f"Remote '{remote_name}' is not a docker remote")
|
||||
|
||||
# Check immutable_patterns against the image name (e.g. "library/nginx")
|
||||
patterns = config.get_immutable_patterns(remote_name, "")
|
||||
# Check include_patterns against the image name (e.g. "library/nginx")
|
||||
patterns = config.get_repository_patterns(remote_name, "")
|
||||
if patterns:
|
||||
path_parts = path.split("/")
|
||||
image_name = "/".join(path_parts[:2]) if len(path_parts) >= 2 else path
|
||||
@@ -548,25 +439,24 @@ async def docker_v2_proxy(request: Request, remote_name: str, path: str):
|
||||
if not storage.exists(cached_key):
|
||||
cached_key = None
|
||||
|
||||
is_mutable = cache.is_mutable_file(path, config.get_mutable_patterns(remote_name))
|
||||
is_index = cache.is_index_file(path, config.get_index_patterns(remote_name))
|
||||
|
||||
if cached_key and is_mutable:
|
||||
if cached_key and is_index:
|
||||
if not cache.is_index_valid(remote_name, path):
|
||||
if not await handle_expired_mutable(remote_name, path, remote_url):
|
||||
cached_key = None
|
||||
logger.info(f"Index EXPIRED: {remote_name}/{path} - removing from cache")
|
||||
cache.cleanup_expired_index(storage, remote_name, path)
|
||||
cached_key = None
|
||||
|
||||
if not cached_key:
|
||||
logger.info(f"Cache MISS: {remote_name}/{path} - fetching from remote: {remote_url}")
|
||||
result = await cache_single_artifact(remote_url, remote_name, path)
|
||||
if result["status"] == "error":
|
||||
raise HTTPException(status_code=502, detail=f"Failed to fetch: {result['error']}")
|
||||
if result["status"] == "cached" and is_mutable:
|
||||
if result["status"] == "cached" and is_index:
|
||||
cache_config = config.get_cache_config(remote_name)
|
||||
mutable_ttl = cache_config.get("mutable_ttl", 3600)
|
||||
cache.mark_index_cached(remote_name, path, mutable_ttl)
|
||||
logger.info(f"Mutable file cached with TTL: {remote_name}/{path} (ttl: {mutable_ttl}s)")
|
||||
if result.get("etag") or result.get("last_modified"):
|
||||
cache.store_mutable_meta(remote_name, path, result.get("etag"), result.get("last_modified"))
|
||||
index_ttl = cache_config.get("index_ttl", 300)
|
||||
cache.mark_index_cached(remote_name, path, index_ttl)
|
||||
logger.info(f"Index file cached with TTL: {remote_name}/{path} (ttl: {index_ttl}s)")
|
||||
|
||||
artifact_data = storage.download_object(storage.get_object_key(remote_name, path))
|
||||
|
||||
|
||||
+12
-56
@@ -22,91 +22,47 @@ TEST_REMOTES = {
|
||||
"base_url": "https://dl-cdn.alpinelinux.org",
|
||||
"type": "remote",
|
||||
"package": "alpine",
|
||||
"immutable_patterns": [".*/x86_64/.*\\.apk$"],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 3600},
|
||||
"include_patterns": [".*/x86_64/.*\\.apk$"],
|
||||
"cache": {"file_ttl": 0, "index_ttl": 3600},
|
||||
},
|
||||
"rpm-test": {
|
||||
"base_url": "https://example.com/rpm",
|
||||
"type": "remote",
|
||||
"package": "rpm",
|
||||
"immutable_patterns": [".*/x86_64/.*\\.rpm$", ".*/repodata/.*$"],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 3600},
|
||||
"include_patterns": [".*/x86_64/.*\\.rpm$", ".*/repodata/.*$"],
|
||||
"cache": {"file_ttl": 0, "index_ttl": 3600},
|
||||
},
|
||||
"docker-test": {
|
||||
"base_url": "https://registry.example.com",
|
||||
"type": "remote",
|
||||
"package": "docker",
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 300},
|
||||
"cache": {"file_ttl": 0, "index_ttl": 300},
|
||||
},
|
||||
"docker-restricted": {
|
||||
"base_url": "https://registry.example.com",
|
||||
"type": "remote",
|
||||
"package": "docker",
|
||||
"immutable_patterns": ["^library/nginx"],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 300},
|
||||
"include_patterns": ["^library/nginx"],
|
||||
"cache": {"file_ttl": 0, "index_ttl": 300},
|
||||
},
|
||||
"generic-test": {
|
||||
"base_url": "https://releases.example.com",
|
||||
"type": "remote",
|
||||
"package": "generic",
|
||||
"immutable_patterns": [".*\\.tar\\.gz$"],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 0},
|
||||
"include_patterns": [".*\\.tar\\.gz$"],
|
||||
"cache": {"file_ttl": 0, "index_ttl": 0},
|
||||
},
|
||||
"custom-index-test": {
|
||||
"base_url": "https://example.com",
|
||||
"type": "remote",
|
||||
"package": "generic",
|
||||
"mutable_patterns": ["metadata\\.json$"],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 600},
|
||||
},
|
||||
"check-mutable-test": {
|
||||
"base_url": "https://example.com",
|
||||
"type": "remote",
|
||||
"package": "generic",
|
||||
"mutable_patterns": ["metadata\\.json$"],
|
||||
"check_mutable_updates": True,
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 600},
|
||||
"index_patterns": ["metadata\\.json$"],
|
||||
"cache": {"file_ttl": 0, "index_ttl": 600},
|
||||
},
|
||||
"local-test": {
|
||||
"type": "local",
|
||||
"package": "generic",
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 0},
|
||||
},
|
||||
"pypi-test": {
|
||||
"base_url": "https://pypi.org",
|
||||
"type": "remote",
|
||||
"package": "pypi",
|
||||
"pypi_files_url": "https://files.pythonhosted.org",
|
||||
"pypi_files_remote": "pypi-files-test",
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 600},
|
||||
},
|
||||
"pypi-files-test": {
|
||||
"base_url": "https://files.pythonhosted.org",
|
||||
"type": "remote",
|
||||
"package": "generic",
|
||||
"immutable_patterns": [
|
||||
"packages/.*\\.whl$",
|
||||
"packages/.*\\.whl\\.metadata$",
|
||||
"packages/.*\\.tar\\.gz$",
|
||||
],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 0},
|
||||
},
|
||||
"npm-test": {
|
||||
"base_url": "https://registry.npmjs.org",
|
||||
"type": "remote",
|
||||
"package": "npm",
|
||||
"npm_files_url": "https://registry.npmjs.org",
|
||||
"npm_files_remote": "npm-test",
|
||||
"immutable_patterns": [r"\.tgz$"],
|
||||
"mutable_patterns": [r"^(?!.*\.tgz$).*"],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 600},
|
||||
},
|
||||
"helm-test": {
|
||||
"base_url": "https://helm.releases.hashicorp.com",
|
||||
"type": "remote",
|
||||
"package": "helm",
|
||||
"immutable_patterns": [r"\.tgz$"],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 3600},
|
||||
"cache": {"file_ttl": 0, "index_ttl": 0},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
+61
-109
@@ -1,4 +1,4 @@
|
||||
"""Tests for RedisCache, focusing on is_mutable_file with configurable patterns."""
|
||||
"""Tests for RedisCache, focusing on is_index_file with configurable patterns."""
|
||||
|
||||
import hashlib
|
||||
from unittest.mock import ANY, MagicMock, patch
|
||||
@@ -6,7 +6,7 @@ from unittest.mock import ANY, MagicMock, patch
|
||||
import pytest
|
||||
|
||||
from artifactapi.cache import RedisCache
|
||||
from artifactapi.config import _PACKAGE_MUTABLE_PATTERNS
|
||||
from artifactapi.config import _PACKAGE_INDEX_PATTERNS
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -38,139 +38,139 @@ def cache_with_redis(mock_redis_client):
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_mutable_file — alpine patterns
|
||||
# is_index_file — alpine patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsMutableFileAlpine:
|
||||
class TestIsIndexFileAlpine:
|
||||
def test_apkindex_tarball_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["alpine"]
|
||||
assert bare_cache.is_mutable_file("alpine/v3.18/x86_64/APKINDEX.tar.gz", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
|
||||
assert bare_cache.is_index_file("alpine/v3.18/x86_64/APKINDEX.tar.gz", patterns)
|
||||
|
||||
def test_nested_apkindex_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["alpine"]
|
||||
assert bare_cache.is_mutable_file("mirrors/dl-cdn/alpine/v3.19/community/x86_64/APKINDEX.tar.gz", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
|
||||
assert bare_cache.is_index_file("mirrors/dl-cdn/alpine/v3.19/community/x86_64/APKINDEX.tar.gz", patterns)
|
||||
|
||||
def test_apk_package_is_not_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["alpine"]
|
||||
assert not bare_cache.is_mutable_file("alpine/v3.18/x86_64/musl-1.2.4-r2.apk", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
|
||||
assert not bare_cache.is_index_file("alpine/v3.18/x86_64/musl-1.2.4-r2.apk", patterns)
|
||||
|
||||
def test_random_tarball_is_not_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["alpine"]
|
||||
assert not bare_cache.is_mutable_file("some/path/archive.tar.gz", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
|
||||
assert not bare_cache.is_index_file("some/path/archive.tar.gz", patterns)
|
||||
|
||||
def test_apkindex_signature_file_is_not_index(self, bare_cache):
|
||||
# Signature file adjacent to the index should not be treated as an index
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["alpine"]
|
||||
assert not bare_cache.is_mutable_file("alpine/v3.18/x86_64/APKINDEX.tar.gz.sig", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
|
||||
assert not bare_cache.is_index_file("alpine/v3.18/x86_64/APKINDEX.tar.gz.sig", patterns)
|
||||
|
||||
def test_apkindex_tmp_file_is_not_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["alpine"]
|
||||
assert not bare_cache.is_mutable_file("alpine/v3.18/x86_64/APKINDEX.tar.gz.tmp", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
|
||||
assert not bare_cache.is_index_file("alpine/v3.18/x86_64/APKINDEX.tar.gz.tmp", patterns)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_mutable_file — rpm patterns
|
||||
# is_index_file — rpm patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsMutableFileRpm:
|
||||
class TestIsIndexFileRpm:
|
||||
def test_repomd_xml_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["rpm"]
|
||||
assert bare_cache.is_mutable_file("almalinux/9/x86_64/repomd.xml", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
|
||||
assert bare_cache.is_index_file("almalinux/9/x86_64/repomd.xml", patterns)
|
||||
|
||||
def test_repodata_primary_xml_gz_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["rpm"]
|
||||
assert bare_cache.is_mutable_file("repo/repodata/primary.xml.gz", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
|
||||
assert bare_cache.is_index_file("repo/repodata/primary.xml.gz", patterns)
|
||||
|
||||
def test_repodata_sqlite_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["rpm"]
|
||||
assert bare_cache.is_mutable_file("repo/repodata/primary.sqlite", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
|
||||
assert bare_cache.is_index_file("repo/repodata/primary.sqlite", patterns)
|
||||
|
||||
def test_repodata_sqlite_bz2_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["rpm"]
|
||||
assert bare_cache.is_mutable_file("repo/repodata/other.sqlite.bz2", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
|
||||
assert bare_cache.is_index_file("repo/repodata/other.sqlite.bz2", patterns)
|
||||
|
||||
def test_repodata_yaml_xz_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["rpm"]
|
||||
assert bare_cache.is_mutable_file("repo/repodata/comps.yaml.xz", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
|
||||
assert bare_cache.is_index_file("repo/repodata/comps.yaml.xz", patterns)
|
||||
|
||||
def test_packages_gz_pattern_matches_any_path(self, bare_cache):
|
||||
# The Packages.gz$ regex is a carryover from the original hardcoded logic and
|
||||
# deliberately matches any path ending in Packages.gz — including Debian-style paths.
|
||||
# This test documents that intentional behaviour.
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["rpm"]
|
||||
assert bare_cache.is_mutable_file("debian/dists/stable/main/binary-amd64/Packages.gz", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
|
||||
assert bare_cache.is_index_file("debian/dists/stable/main/binary-amd64/Packages.gz", patterns)
|
||||
|
||||
def test_rpm_package_is_not_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["rpm"]
|
||||
assert not bare_cache.is_mutable_file("almalinux/9/x86_64/Packages/bash-5.1.8.x86_64.rpm", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
|
||||
assert not bare_cache.is_index_file("almalinux/9/x86_64/Packages/bash-5.1.8.x86_64.rpm", patterns)
|
||||
|
||||
def test_arbitrary_xml_outside_repodata_is_not_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["rpm"]
|
||||
assert not bare_cache.is_mutable_file("some/path/config.xml", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
|
||||
assert not bare_cache.is_index_file("some/path/config.xml", patterns)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_mutable_file — docker patterns
|
||||
# is_index_file — docker patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsMutableFileDocker:
|
||||
class TestIsIndexFileDocker:
|
||||
def test_tag_manifest_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["docker"]
|
||||
assert bare_cache.is_mutable_file("library/nginx/manifests/latest", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
|
||||
assert bare_cache.is_index_file("library/nginx/manifests/latest", patterns)
|
||||
|
||||
def test_version_tag_manifest_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["docker"]
|
||||
assert bare_cache.is_mutable_file("library/nginx/manifests/1.25.3", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
|
||||
assert bare_cache.is_index_file("library/nginx/manifests/1.25.3", patterns)
|
||||
|
||||
def test_hyphenated_tag_manifest_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["docker"]
|
||||
assert bare_cache.is_mutable_file("library/nginx/manifests/latest-rc", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
|
||||
assert bare_cache.is_index_file("library/nginx/manifests/latest-rc", patterns)
|
||||
|
||||
def test_numeric_date_tag_manifest_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["docker"]
|
||||
assert bare_cache.is_mutable_file("library/nginx/manifests/20240101", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
|
||||
assert bare_cache.is_index_file("library/nginx/manifests/20240101", patterns)
|
||||
|
||||
def test_digest_manifest_is_not_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["docker"]
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
|
||||
digest = "sha256:" + "a" * 64
|
||||
assert not bare_cache.is_mutable_file(f"library/nginx/manifests/{digest}", patterns)
|
||||
assert not bare_cache.is_index_file(f"library/nginx/manifests/{digest}", patterns)
|
||||
|
||||
def test_tags_list_is_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["docker"]
|
||||
assert bare_cache.is_mutable_file("library/nginx/tags/list", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
|
||||
assert bare_cache.is_index_file("library/nginx/tags/list", patterns)
|
||||
|
||||
def test_blob_is_not_index(self, bare_cache):
|
||||
patterns = _PACKAGE_MUTABLE_PATTERNS["docker"]
|
||||
assert not bare_cache.is_mutable_file("library/nginx/blobs/sha256:abc123", patterns)
|
||||
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
|
||||
assert not bare_cache.is_index_file("library/nginx/blobs/sha256:abc123", patterns)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_mutable_file — edge cases
|
||||
# is_index_file — edge cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsMutableFileEdgeCases:
|
||||
class TestIsIndexFileEdgeCases:
|
||||
def test_empty_patterns_nothing_is_index(self, bare_cache):
|
||||
assert not bare_cache.is_mutable_file("APKINDEX.tar.gz", [])
|
||||
assert not bare_cache.is_mutable_file("repomd.xml", [])
|
||||
assert not bare_cache.is_mutable_file("library/nginx/manifests/latest", [])
|
||||
assert not bare_cache.is_index_file("APKINDEX.tar.gz", [])
|
||||
assert not bare_cache.is_index_file("repomd.xml", [])
|
||||
assert not bare_cache.is_index_file("library/nginx/manifests/latest", [])
|
||||
|
||||
def test_none_patterns_nothing_is_index(self, bare_cache):
|
||||
assert not bare_cache.is_mutable_file("APKINDEX.tar.gz", None)
|
||||
assert not bare_cache.is_mutable_file("repomd.xml", None)
|
||||
assert not bare_cache.is_index_file("APKINDEX.tar.gz", None)
|
||||
assert not bare_cache.is_index_file("repomd.xml", None)
|
||||
|
||||
def test_custom_patterns_match(self, bare_cache):
|
||||
patterns = [r"metadata\.json$", r"index\.yaml$"]
|
||||
assert bare_cache.is_mutable_file("repo/metadata.json", patterns)
|
||||
assert bare_cache.is_mutable_file("repo/subdir/index.yaml", patterns)
|
||||
assert not bare_cache.is_mutable_file("repo/data.tar.gz", patterns)
|
||||
assert bare_cache.is_index_file("repo/metadata.json", patterns)
|
||||
assert bare_cache.is_index_file("repo/subdir/index.yaml", patterns)
|
||||
assert not bare_cache.is_index_file("repo/data.tar.gz", patterns)
|
||||
|
||||
def test_custom_pattern_does_not_match_standard_index(self, bare_cache):
|
||||
patterns = [r"metadata\.json$"]
|
||||
assert not bare_cache.is_mutable_file("APKINDEX.tar.gz", patterns)
|
||||
assert not bare_cache.is_index_file("APKINDEX.tar.gz", patterns)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -235,51 +235,3 @@ class TestIndexValidity:
|
||||
# client is None when Redis is unavailable — setex cannot be called
|
||||
assert unavailable_cache.client is None
|
||||
unavailable_cache.mark_index_cached("remote", "some/path", 300) # must not raise
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# mutable meta (ETag / Last-Modified storage)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMutableMeta:
|
||||
def test_meta_key_format(self, bare_cache):
|
||||
path = "repo/metadata.json"
|
||||
expected_hash = hashlib.sha256(path.encode()).hexdigest()[:16]
|
||||
assert bare_cache.get_mutable_meta_key("myremote", path) == f"mutable:meta:myremote:{expected_hash}"
|
||||
|
||||
def test_meta_key_hash_is_16_chars(self, bare_cache):
|
||||
key = bare_cache.get_mutable_meta_key("remote", "some/path/file.json")
|
||||
assert len(key.split(":")[-1]) == 16
|
||||
|
||||
def test_store_and_retrieve_etag(self, cache_with_redis, mock_redis_client):
|
||||
mock_redis_client.hgetall.return_value = {"etag": '"abc123"'}
|
||||
cache_with_redis.store_mutable_meta("remote", "path/meta.json", '"abc123"', None)
|
||||
mock_redis_client.hset.assert_called_once()
|
||||
meta = cache_with_redis.get_mutable_meta("remote", "path/meta.json")
|
||||
assert meta["etag"] == '"abc123"'
|
||||
|
||||
def test_store_and_retrieve_last_modified(self, cache_with_redis, mock_redis_client):
|
||||
lm = "Mon, 01 Jan 2024 00:00:00 GMT"
|
||||
mock_redis_client.hgetall.return_value = {"last_modified": lm}
|
||||
cache_with_redis.store_mutable_meta("remote", "path/meta.json", None, lm)
|
||||
meta = cache_with_redis.get_mutable_meta("remote", "path/meta.json")
|
||||
assert meta["last_modified"] == lm
|
||||
|
||||
def test_store_no_op_when_both_none(self, cache_with_redis, mock_redis_client):
|
||||
cache_with_redis.store_mutable_meta("remote", "path/meta.json", None, None)
|
||||
mock_redis_client.hset.assert_not_called()
|
||||
|
||||
def test_store_no_op_when_unavailable(self, unavailable_cache):
|
||||
unavailable_cache.store_mutable_meta("remote", "path", "etag", None) # must not raise
|
||||
|
||||
def test_get_returns_empty_when_unavailable(self, unavailable_cache):
|
||||
assert unavailable_cache.get_mutable_meta("remote", "path") == {}
|
||||
|
||||
def test_delete_removes_meta_key(self, cache_with_redis, mock_redis_client):
|
||||
expected_key = cache_with_redis.get_mutable_meta_key("remote", "path/meta.json")
|
||||
cache_with_redis.delete_mutable_meta("remote", "path/meta.json")
|
||||
mock_redis_client.delete.assert_called_once_with(expected_key)
|
||||
|
||||
def test_delete_no_op_when_unavailable(self, unavailable_cache):
|
||||
unavailable_cache.delete_mutable_meta("remote", "path") # must not raise
|
||||
|
||||
+38
-122
@@ -1,4 +1,4 @@
|
||||
"""Tests for ConfigManager, focusing on get_mutable_patterns and get_immutable_patterns."""
|
||||
"""Tests for ConfigManager, focusing on get_index_patterns (new logic)."""
|
||||
|
||||
import os
|
||||
|
||||
@@ -21,44 +21,45 @@ def make_config(tmp_path):
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_mutable_patterns
|
||||
# get_index_patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetMutablePatterns:
|
||||
class TestGetIndexPatterns:
|
||||
def test_alpine_returns_package_defaults(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "package": "alpine", "base_url": "https://x.com"}})
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
patterns = cfg.get_index_patterns("r")
|
||||
# Assert against literal strings, not the live constant, so a rename doesn't mask a regression
|
||||
assert r"APKINDEX\.tar\.gz$" in patterns
|
||||
|
||||
def test_rpm_returns_package_defaults(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "package": "rpm", "base_url": "https://x.com"}})
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
patterns = cfg.get_index_patterns("r")
|
||||
assert r"repomd\.xml$" in patterns
|
||||
assert any("repodata" in p for p in patterns)
|
||||
|
||||
def test_docker_returns_package_defaults(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "package": "docker", "base_url": "https://x.com"}})
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
patterns = cfg.get_index_patterns("r")
|
||||
assert any("manifests" in p for p in patterns)
|
||||
assert any("tags/list" in p for p in patterns)
|
||||
|
||||
def test_generic_returns_empty_list(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}})
|
||||
assert cfg.get_mutable_patterns("r") == []
|
||||
assert cfg.get_index_patterns("r") == []
|
||||
|
||||
def test_unknown_remote_returns_empty_list(self, make_config):
|
||||
cfg = make_config({})
|
||||
assert cfg.get_mutable_patterns("nonexistent") == []
|
||||
assert cfg.get_index_patterns("nonexistent") == []
|
||||
|
||||
def test_missing_package_field_defaults_to_generic(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "base_url": "https://x.com"}})
|
||||
assert cfg.get_mutable_patterns("r") == []
|
||||
assert cfg.get_index_patterns("r") == []
|
||||
|
||||
def test_unknown_package_type_returns_empty_list(self, make_config):
|
||||
# A mis-spelled package type silently returns [] — this is a known footgun
|
||||
cfg = make_config({"r": {"type": "remote", "package": "deb", "base_url": "https://x.com"}})
|
||||
assert cfg.get_mutable_patterns("r") == []
|
||||
assert cfg.get_index_patterns("r") == []
|
||||
|
||||
def test_extra_patterns_appended_after_defaults(self, make_config):
|
||||
cfg = make_config(
|
||||
@@ -67,11 +68,11 @@ class TestGetMutablePatterns:
|
||||
"type": "remote",
|
||||
"package": "alpine",
|
||||
"base_url": "https://x.com",
|
||||
"mutable_patterns": [r"custom\.json$"],
|
||||
"index_patterns": [r"custom\.json$"],
|
||||
}
|
||||
}
|
||||
)
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
patterns = cfg.get_index_patterns("r")
|
||||
assert r"APKINDEX\.tar\.gz$" in patterns
|
||||
assert r"custom\.json$" in patterns
|
||||
# Defaults come first
|
||||
@@ -84,11 +85,11 @@ class TestGetMutablePatterns:
|
||||
"type": "remote",
|
||||
"package": "alpine",
|
||||
"base_url": "https://x.com",
|
||||
"mutable_patterns": [],
|
||||
"index_patterns": [],
|
||||
}
|
||||
}
|
||||
)
|
||||
assert r"APKINDEX\.tar\.gz$" in cfg.get_mutable_patterns("r")
|
||||
assert r"APKINDEX\.tar\.gz$" in cfg.get_index_patterns("r")
|
||||
|
||||
def test_duplicate_extra_pattern_not_added_twice(self, make_config):
|
||||
existing = r"APKINDEX\.tar\.gz$"
|
||||
@@ -98,11 +99,11 @@ class TestGetMutablePatterns:
|
||||
"type": "remote",
|
||||
"package": "alpine",
|
||||
"base_url": "https://x.com",
|
||||
"mutable_patterns": [existing],
|
||||
"index_patterns": [existing],
|
||||
}
|
||||
}
|
||||
)
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
patterns = cfg.get_index_patterns("r")
|
||||
assert patterns.count(existing) == 1
|
||||
|
||||
def test_generic_with_only_extra_patterns(self, make_config):
|
||||
@@ -112,11 +113,11 @@ class TestGetMutablePatterns:
|
||||
"type": "remote",
|
||||
"package": "generic",
|
||||
"base_url": "https://x.com",
|
||||
"mutable_patterns": [r"meta\.json$", r"index\.yaml$"],
|
||||
"index_patterns": [r"meta\.json$", r"index\.yaml$"],
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cfg.get_mutable_patterns("r") == [r"meta\.json$", r"index\.yaml$"]
|
||||
assert cfg.get_index_patterns("r") == [r"meta\.json$", r"index\.yaml$"]
|
||||
|
||||
def test_rpm_extra_patterns_merged(self, make_config):
|
||||
cfg = make_config(
|
||||
@@ -125,93 +126,41 @@ class TestGetMutablePatterns:
|
||||
"type": "remote",
|
||||
"package": "rpm",
|
||||
"base_url": "https://x.com",
|
||||
"mutable_patterns": [r"custom-meta\.xml$"],
|
||||
"index_patterns": [r"custom-meta\.xml$"],
|
||||
}
|
||||
}
|
||||
)
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
patterns = cfg.get_index_patterns("r")
|
||||
assert r"repomd\.xml$" in patterns
|
||||
assert r"custom-meta\.xml$" in patterns
|
||||
|
||||
def test_npm_has_no_package_defaults(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "package": "npm", "base_url": "https://x.com"}})
|
||||
assert cfg.get_mutable_patterns("r") == []
|
||||
|
||||
def test_npm_explicit_mutable_pattern_matches_metadata(self, make_config):
|
||||
import re
|
||||
|
||||
cfg = make_config(
|
||||
{
|
||||
"r": {
|
||||
"type": "remote",
|
||||
"package": "npm",
|
||||
"base_url": "https://x.com",
|
||||
"mutable_patterns": [r"^(?!.*\.tgz$).*"],
|
||||
}
|
||||
}
|
||||
)
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
assert any(re.search(p, "express") for p in patterns)
|
||||
assert any(re.search(p, "@babel/core") for p in patterns)
|
||||
|
||||
def test_helm_returns_index_yaml_as_mutable(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "package": "helm", "base_url": "https://helm.example.com"}})
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
assert r"index\.yaml$" in patterns
|
||||
|
||||
def test_helm_chart_tarballs_not_mutable_by_default(self, make_config):
|
||||
import re
|
||||
|
||||
cfg = make_config({"r": {"type": "remote", "package": "helm", "base_url": "https://helm.example.com"}})
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
# Only index.yaml is mutable; .tgz chart tarballs are not
|
||||
assert not any(re.search(p, "vault-0.29.1.tgz") for p in patterns)
|
||||
assert not any(re.search(p, "consul-1.5.0.tgz") for p in patterns)
|
||||
|
||||
def test_npm_explicit_mutable_pattern_excludes_tarballs(self, make_config):
|
||||
import re
|
||||
|
||||
cfg = make_config(
|
||||
{
|
||||
"r": {
|
||||
"type": "remote",
|
||||
"package": "npm",
|
||||
"base_url": "https://x.com",
|
||||
"mutable_patterns": [r"^(?!.*\.tgz$).*"],
|
||||
}
|
||||
}
|
||||
)
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
assert not any(re.search(p, "express-4.18.2.tgz") for p in patterns)
|
||||
assert not any(re.search(p, "express/-/express-4.18.2.tgz") for p in patterns)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_immutable_patterns
|
||||
# get_repository_patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetImmutablePatterns:
|
||||
def test_returns_immutable_patterns(self, make_config):
|
||||
class TestGetRepositoryPatterns:
|
||||
def test_returns_include_patterns(self, make_config):
|
||||
cfg = make_config(
|
||||
{
|
||||
"r": {
|
||||
"type": "remote",
|
||||
"package": "generic",
|
||||
"base_url": "https://x.com",
|
||||
"immutable_patterns": [r".*\.tar\.gz$"],
|
||||
"include_patterns": [r".*\.tar\.gz$"],
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cfg.get_immutable_patterns("r") == [r".*\.tar\.gz$"]
|
||||
assert cfg.get_repository_patterns("r", "") == [r".*\.tar\.gz$"]
|
||||
|
||||
def test_returns_empty_for_missing_remote(self, make_config):
|
||||
cfg = make_config({})
|
||||
assert cfg.get_immutable_patterns("nonexistent") == []
|
||||
assert cfg.get_repository_patterns("nonexistent", "") == []
|
||||
|
||||
def test_returns_empty_when_no_patterns_configured(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}})
|
||||
assert cfg.get_immutable_patterns("r") == []
|
||||
assert cfg.get_repository_patterns("r", "") == []
|
||||
|
||||
def test_multiple_patterns_returned(self, make_config):
|
||||
patterns = [r".*\.rpm$", r".*/repodata/.*$"]
|
||||
@@ -221,11 +170,11 @@ class TestGetImmutablePatterns:
|
||||
"type": "remote",
|
||||
"package": "rpm",
|
||||
"base_url": "https://x.com",
|
||||
"immutable_patterns": patterns,
|
||||
"include_patterns": patterns,
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cfg.get_immutable_patterns("r") == patterns
|
||||
assert cfg.get_repository_patterns("r", "") == patterns
|
||||
|
||||
def test_dict_keyed_repositories_returns_per_repo_patterns(self, make_config):
|
||||
cfg = make_config(
|
||||
@@ -234,14 +183,14 @@ class TestGetImmutablePatterns:
|
||||
"type": "remote",
|
||||
"package": "generic",
|
||||
"base_url": "https://x.com",
|
||||
"immutable_patterns": [r".*\.tar\.gz$"],
|
||||
"include_patterns": [r".*\.tar\.gz$"],
|
||||
"repositories": {
|
||||
"/path/to/repo": {"immutable_patterns": [r".*\.rpm$"]},
|
||||
"/path/to/repo": {"include_patterns": [r".*\.rpm$"]},
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cfg.get_immutable_patterns("r", "/path/to/repo") == [r".*\.rpm$"]
|
||||
assert cfg.get_repository_patterns("r", "/path/to/repo") == [r".*\.rpm$"]
|
||||
|
||||
def test_dict_keyed_repositories_falls_back_to_remote_patterns(self, make_config):
|
||||
cfg = make_config(
|
||||
@@ -250,47 +199,14 @@ class TestGetImmutablePatterns:
|
||||
"type": "remote",
|
||||
"package": "generic",
|
||||
"base_url": "https://x.com",
|
||||
"immutable_patterns": [r".*\.tar\.gz$"],
|
||||
"include_patterns": [r".*\.tar\.gz$"],
|
||||
"repositories": {
|
||||
"/path/to/repo": {"immutable_patterns": [r".*\.rpm$"]},
|
||||
"/path/to/repo": {"include_patterns": [r".*\.rpm$"]},
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cfg.get_immutable_patterns("r", "/unknown/path") == [r".*\.tar\.gz$"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_user_mutable_patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetUserMutablePatterns:
|
||||
def test_returns_only_user_patterns(self, make_config):
|
||||
cfg = make_config(
|
||||
{
|
||||
"r": {
|
||||
"type": "remote",
|
||||
"package": "alpine",
|
||||
"base_url": "https://x.com",
|
||||
"mutable_patterns": [r"custom\.json$"],
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cfg.get_user_mutable_patterns("r") == [r"custom\.json$"]
|
||||
|
||||
def test_excludes_package_defaults(self, make_config):
|
||||
# Package defaults (APKINDEX etc.) must NOT appear here
|
||||
cfg = make_config({"r": {"type": "remote", "package": "alpine", "base_url": "https://x.com"}})
|
||||
assert cfg.get_user_mutable_patterns("r") == []
|
||||
|
||||
def test_returns_empty_for_missing_remote(self, make_config):
|
||||
cfg = make_config({})
|
||||
assert cfg.get_user_mutable_patterns("nonexistent") == []
|
||||
|
||||
def test_returns_empty_when_key_absent(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}})
|
||||
assert cfg.get_user_mutable_patterns("r") == []
|
||||
assert cfg.get_repository_patterns("r", "/unknown/path") == [r".*\.tar\.gz$"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -306,11 +222,11 @@ class TestGetCacheConfig:
|
||||
"type": "remote",
|
||||
"package": "generic",
|
||||
"base_url": "https://x.com",
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 7200},
|
||||
"cache": {"file_ttl": 0, "index_ttl": 7200},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cfg.get_cache_config("r") == {"immutable_ttl": 0, "mutable_ttl": 7200}
|
||||
assert cfg.get_cache_config("r") == {"file_ttl": 0, "index_ttl": 7200}
|
||||
|
||||
def test_returns_empty_dict_for_missing_remote(self, make_config):
|
||||
cfg = make_config({})
|
||||
|
||||
+33
-403
@@ -25,7 +25,7 @@ def mock_storage():
|
||||
@pytest.fixture
|
||||
def mock_cache():
|
||||
m = MagicMock()
|
||||
m.is_mutable_file.return_value = False
|
||||
m.is_index_file.return_value = False
|
||||
m.is_index_valid.return_value = True
|
||||
m.available = False
|
||||
m.client = None
|
||||
@@ -123,7 +123,7 @@ class TestDockerProxy:
|
||||
).encode()
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = manifest
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/v2/docker-restricted/library/nginx/manifests/latest")
|
||||
@@ -140,7 +140,7 @@ class TestDockerProxy:
|
||||
).encode()
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = manifest
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
|
||||
@@ -158,7 +158,7 @@ class TestDockerProxy:
|
||||
).encode()
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = manifest
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
|
||||
@@ -170,7 +170,7 @@ class TestDockerProxy:
|
||||
manifest = json.dumps({"mediaType": "application/vnd.oci.image.manifest.v1+json", "layers": []}).encode()
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = manifest
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
client.get("/v2/docker-test/library/nginx/manifests/latest")
|
||||
deps["metrics"].record_cache_hit.assert_called_once_with("docker-test", ANY)
|
||||
@@ -185,7 +185,7 @@ class TestDockerProxy:
|
||||
).encode()
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = manifest
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
response = client.head("/v2/docker-test/library/nginx/manifests/latest")
|
||||
assert response.status_code == 200
|
||||
@@ -201,7 +201,7 @@ class TestDockerProxy:
|
||||
).encode()
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = manifest
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_file.return_value = True
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
@@ -223,7 +223,7 @@ class TestDockerProxy:
|
||||
).encode()
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = manifest
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_file.return_value = True
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
@@ -244,17 +244,16 @@ class TestDockerProxy:
|
||||
}
|
||||
).encode()
|
||||
deps["storage"].exists.return_value = True # cached in S3
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = False # but TTL expired
|
||||
deps["storage"].download_object.return_value = manifest
|
||||
|
||||
with patch("artifactapi.main._upstream_reachable", new_callable=AsyncMock, return_value=True):
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
new_callable=AsyncMock,
|
||||
return_value={"status": "cached"},
|
||||
) as mock_fetch:
|
||||
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
new_callable=AsyncMock,
|
||||
return_value={"status": "cached"},
|
||||
) as mock_fetch:
|
||||
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
|
||||
|
||||
mock_fetch.assert_called_once()
|
||||
assert response.status_code == 200
|
||||
@@ -279,7 +278,7 @@ class TestGenericArtifactRoute:
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"tar content"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
|
||||
assert response.status_code == 200
|
||||
@@ -290,7 +289,7 @@ class TestGenericArtifactRoute:
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"content"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
|
||||
disposition = response.headers["content-disposition"]
|
||||
@@ -302,7 +301,7 @@ class TestGenericArtifactRoute:
|
||||
content = b"some artifact content bytes"
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = content
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
|
||||
assert response.headers["X-Artifact-Size"] == str(len(content))
|
||||
@@ -311,7 +310,7 @@ class TestGenericArtifactRoute:
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"content"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
|
||||
deps["metrics"].record_cache_hit.assert_called_once_with("generic-test", ANY)
|
||||
@@ -320,7 +319,7 @@ class TestGenericArtifactRoute:
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"content"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
|
||||
deps["database"].record_artifact_mapping.assert_called_once()
|
||||
@@ -329,7 +328,7 @@ class TestGenericArtifactRoute:
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"rpm bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/rpm-test/almalinux/9/x86_64/bash-5.1.8.x86_64.rpm")
|
||||
assert response.status_code == 200
|
||||
@@ -339,7 +338,7 @@ class TestGenericArtifactRoute:
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"<?xml version='1.0'?>"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/rpm-test/repo/repodata/primary.xml")
|
||||
assert response.status_code == 200
|
||||
@@ -349,7 +348,7 @@ class TestGenericArtifactRoute:
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = b"fresh content"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
@@ -366,7 +365,7 @@ class TestGenericArtifactRoute:
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = b"fresh content"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
@@ -381,7 +380,7 @@ class TestGenericArtifactRoute:
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = b"APKINDEX content"
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_file.return_value = True
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
@@ -396,7 +395,7 @@ class TestGenericArtifactRoute:
|
||||
def test_upstream_error_returns_502(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
deps["cache"].is_index_file.return_value = False
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
@@ -407,116 +406,19 @@ class TestGenericArtifactRoute:
|
||||
|
||||
assert response.status_code == 502
|
||||
|
||||
def test_mutable_file_bypasses_immutable_patterns(self, client, patched_deps):
|
||||
"""Mutable files must be served even when they don't match immutable_patterns."""
|
||||
def test_index_file_bypasses_include_patterns(self, client, patched_deps):
|
||||
"""Index files must be served even when they don't match include_patterns."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"APKINDEX content"
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
# APKINDEX.tar.gz does not match alpine-test's immutable_patterns (.*.apk$),
|
||||
# but since is_mutable_file returns True it must be allowed through.
|
||||
# APKINDEX.tar.gz does not match alpine-test's include_patterns (.*.apk$),
|
||||
# but since is_index_file returns True it must be allowed through.
|
||||
response = client.get("/api/v1/remote/alpine-test/alpine/v3.18/x86_64/APKINDEX.tar.gz")
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_mutable_unchanged_refreshes_ttl_without_redownload(self, client, patched_deps):
|
||||
"""When check_mutable_updates=True and upstream says 304, TTL is refreshed in place."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"metadata content"
|
||||
# File is mutable and its TTL has expired
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = False
|
||||
deps["cache"].get_mutable_meta.return_value = {"etag": '"abc"'}
|
||||
|
||||
with patch("artifactapi.main.check_upstream_changed", new_callable=AsyncMock, return_value=False):
|
||||
response = client.get("/api/v1/remote/check-mutable-test/metadata.json")
|
||||
|
||||
assert response.status_code == 200
|
||||
deps["cache"].mark_index_cached.assert_called()
|
||||
# S3 object must NOT have been deleted (no re-download)
|
||||
deps["storage"].client.delete_object.assert_not_called()
|
||||
|
||||
def test_mutable_changed_triggers_redownload(self, client, patched_deps):
|
||||
"""When check_mutable_updates=True and upstream says 200, cache is invalidated."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = False
|
||||
deps["cache"].get_mutable_meta.return_value = {"etag": '"abc"'}
|
||||
|
||||
with patch("artifactapi.main.check_upstream_changed", new_callable=AsyncMock, return_value=True):
|
||||
with patch("artifactapi.main.cache_single_artifact", new_callable=AsyncMock) as mock_cache:
|
||||
mock_cache.return_value = {"status": "error", "error": "upstream gone"}
|
||||
response = client.get("/api/v1/remote/check-mutable-test/metadata.json")
|
||||
|
||||
assert response.status_code == 502
|
||||
|
||||
def test_mutable_changed_redownloads_successfully(self, client, patched_deps):
|
||||
"""When check_mutable_updates=True and upstream says 200, fresh copy is fetched and served."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"fresh metadata"
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = False
|
||||
deps["cache"].get_mutable_meta.return_value = {"etag": '"abc"'}
|
||||
|
||||
with patch("artifactapi.main.check_upstream_changed", new_callable=AsyncMock, return_value=True):
|
||||
with patch("artifactapi.main.cache_single_artifact", new_callable=AsyncMock) as mock_cache:
|
||||
mock_cache.return_value = {"status": "cached", "etag": '"def"', "last_modified": None}
|
||||
response = client.get("/api/v1/remote/check-mutable-test/metadata.json")
|
||||
|
||||
assert response.status_code == 200
|
||||
mock_cache.assert_called_once()
|
||||
|
||||
def test_mutable_backend_unreachable_on_check_updates_keeps_stale(self, client, patched_deps):
|
||||
"""When check_mutable_updates=True and backend is unreachable, stale copy is kept and TTL refreshed."""
|
||||
from artifactapi.main import UpstreamUnreachable
|
||||
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"stale metadata"
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = False
|
||||
deps["cache"].get_mutable_meta.return_value = {"etag": '"abc"'}
|
||||
|
||||
with patch("artifactapi.main.check_upstream_changed", side_effect=UpstreamUnreachable("connection refused")):
|
||||
response = client.get("/api/v1/remote/check-mutable-test/metadata.json")
|
||||
|
||||
assert response.status_code == 200
|
||||
deps["cache"].mark_index_cached.assert_called()
|
||||
deps["storage"].client.delete_object.assert_not_called()
|
||||
|
||||
def test_mutable_backend_unreachable_on_expiry_keeps_stale(self, client, patched_deps):
|
||||
"""When a regular mutable file expires and backend is unreachable, stale copy is kept and TTL refreshed."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"stale APKINDEX"
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = False
|
||||
|
||||
with patch("artifactapi.main._upstream_reachable", new_callable=AsyncMock, return_value=False):
|
||||
response = client.get("/api/v1/remote/alpine-test/alpine/v3.18/x86_64/APKINDEX.tar.gz")
|
||||
|
||||
assert response.status_code == 200
|
||||
deps["cache"].mark_index_cached.assert_called()
|
||||
deps["storage"].client.delete_object.assert_not_called()
|
||||
|
||||
def test_mutable_flag_off_skips_conditional_check(self, client, patched_deps):
|
||||
"""When check_mutable_updates is not set, expired mutable files are always re-fetched."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = False
|
||||
|
||||
with patch("artifactapi.main.check_upstream_changed", new_callable=AsyncMock) as mock_check:
|
||||
with patch("artifactapi.main.cache_single_artifact", new_callable=AsyncMock) as mock_cache:
|
||||
mock_cache.return_value = {"status": "error", "error": "upstream gone"}
|
||||
client.get("/api/v1/remote/custom-index-test/metadata.json")
|
||||
|
||||
mock_check.assert_not_called()
|
||||
|
||||
def test_local_repo_file_not_found_returns_404(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["database"].get_local_file_metadata.return_value = None
|
||||
@@ -617,8 +519,8 @@ class TestCacheFlushEndpoint:
|
||||
deps["cache"].available = True
|
||||
redis_mock = MagicMock()
|
||||
deps["cache"].client = redis_mock
|
||||
# index:* returns keys; mutable:meta:* and metrics:* return nothing
|
||||
redis_mock.keys.side_effect = [["index:test:abc", "index:test:def"], [], []]
|
||||
# First pattern (index:*) returns keys; subsequent pattern returns nothing
|
||||
redis_mock.keys.side_effect = [["index:test:abc", "index:test:def"], []]
|
||||
deps["storage"].client.list_objects_v2.return_value = {}
|
||||
|
||||
response = client.put("/cache/flush")
|
||||
@@ -652,275 +554,3 @@ class TestConfigEndpoint:
|
||||
data = response.json()
|
||||
assert "remotes" in data
|
||||
assert "alpine-test" in data["remotes"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PyPI remote /api/v1/remote/pypi-test/...
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPyPIRemote:
|
||||
def test_simple_index_is_mutable(self, client, patched_deps):
|
||||
"""simple/ paths are detected as mutable (package-type default)."""
|
||||
deps = patched_deps
|
||||
html = b"<html><body><a href='https://files.pythonhosted.org/packages/requests-2.31.0.tar.gz'>...</a></body></html>"
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = html
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/pypi-test/simple/requests/")
|
||||
assert response.status_code == 200
|
||||
deps["cache"].mark_index_cached.assert_not_called()
|
||||
|
||||
def test_simple_index_urls_rewritten_to_proxy(self, client, patched_deps):
|
||||
"""files.pythonhosted.org URLs in a cached simple index are rewritten to our proxy."""
|
||||
deps = patched_deps
|
||||
html = b"<html><body><a href='https://files.pythonhosted.org/packages/requests-2.31.0.tar.gz'>...</a></body></html>"
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = html
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/pypi-test/simple/requests/")
|
||||
assert response.status_code == 200
|
||||
assert b"files.pythonhosted.org" not in response.content
|
||||
assert b"/api/v1/remote/pypi-files-test/packages/requests-2.31.0.tar.gz" in response.content
|
||||
|
||||
def test_simple_index_content_type_is_html(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"<html></html>"
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/pypi-test/simple/requests/")
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers["content-type"]
|
||||
|
||||
def test_simple_index_cache_miss_fetches_upstream(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
html = b"<html><body><a href='https://files.pythonhosted.org/packages/p-1.0.whl'>...</a></body></html>"
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = html
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
new_callable=AsyncMock,
|
||||
return_value={"status": "cached"},
|
||||
) as mock_fetch:
|
||||
response = client.get("/api/v1/remote/pypi-test/simple/requests/")
|
||||
|
||||
mock_fetch.assert_called_once()
|
||||
assert response.status_code == 200
|
||||
assert b"files.pythonhosted.org" not in response.content
|
||||
|
||||
def test_wheel_file_immutable_returns_correct_content_type(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"PK wheel bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/pypi-files-test/packages/requests-2.31.0-py3-none-any.whl")
|
||||
assert response.status_code == 200
|
||||
assert "application/zip" in response.headers["content-type"]
|
||||
assert response.headers["X-Artifact-Source"] == "cache"
|
||||
|
||||
def test_sdist_immutable_returns_correct_content_type(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"tar bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/pypi-files-test/packages/requests-2.31.0.tar.gz")
|
||||
assert response.status_code == 200
|
||||
assert "application/gzip" in response.headers["content-type"]
|
||||
|
||||
def test_blocked_path_on_files_remote_returns_403(self, client, patched_deps):
|
||||
"""Paths that don't match immutable_patterns on pypi-files-test are blocked."""
|
||||
response = client.get("/api/v1/remote/pypi-files-test/packages/requests.unknown")
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# npm remote /api/v1/remote/npm-test/...
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNpmRemote:
|
||||
def test_package_metadata_is_mutable(self, client, patched_deps):
|
||||
"""Top-level package metadata paths are detected as mutable."""
|
||||
deps = patched_deps
|
||||
meta = b'{"name":"express","versions":{}}'
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = meta
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/express")
|
||||
assert response.status_code == 200
|
||||
deps["cache"].mark_index_cached.assert_not_called()
|
||||
|
||||
def test_metadata_tarball_urls_rewritten_to_proxy(self, client, patched_deps):
|
||||
"""registry.npmjs.org tarball URLs in metadata JSON are rewritten to our proxy."""
|
||||
deps = patched_deps
|
||||
meta = b'{"dist":{"tarball":"https://registry.npmjs.org/express/-/express-4.18.2.tgz"}}'
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = meta
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/express")
|
||||
assert response.status_code == 200
|
||||
assert b"registry.npmjs.org" not in response.content
|
||||
assert b"/api/v1/remote/npm-test/express/-/express-4.18.2.tgz" in response.content
|
||||
|
||||
def test_metadata_content_type_is_json(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b'{"name":"express"}'
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/express")
|
||||
assert response.status_code == 200
|
||||
assert "application/json" in response.headers["content-type"]
|
||||
|
||||
def test_scoped_package_metadata_rewritten(self, client, patched_deps):
|
||||
"""@scope/package metadata URLs are also rewritten back to the same npm-test remote."""
|
||||
deps = patched_deps
|
||||
meta = b'{"dist":{"tarball":"https://registry.npmjs.org/@babel/core/-/core-7.21.0.tgz"}}'
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = meta
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/@babel/core")
|
||||
assert response.status_code == 200
|
||||
assert b"registry.npmjs.org" not in response.content
|
||||
assert b"/api/v1/remote/npm-test/@babel/core/-/core-7.21.0.tgz" in response.content
|
||||
|
||||
def test_tarball_not_rewritten(self, client, patched_deps):
|
||||
"""Tarball requests (.tgz) bypass URL rewriting and return binary."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"\x1f\x8b tgz bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/express/-/express-4.18.2.tgz")
|
||||
assert response.status_code == 200
|
||||
assert "application/gzip" in response.headers["content-type"]
|
||||
assert response.headers["X-Artifact-Source"] == "cache"
|
||||
|
||||
def test_metadata_cache_miss_fetches_upstream(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
meta = b'{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"}}'
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = meta
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
new_callable=AsyncMock,
|
||||
return_value={"status": "cached"},
|
||||
) as mock_fetch:
|
||||
response = client.get("/api/v1/remote/npm-test/lodash")
|
||||
|
||||
mock_fetch.assert_called_once()
|
||||
assert response.status_code == 200
|
||||
assert b"registry.npmjs.org" not in response.content
|
||||
|
||||
def test_tarball_immutable_allowed_on_npm_remote(self, client, patched_deps):
|
||||
"""Tarballs (.tgz) match immutable_patterns and are served without rewriting."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"tgz bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/express/-/express-4.18.2.tgz")
|
||||
assert response.status_code == 200
|
||||
assert "application/gzip" in response.headers["content-type"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helm remote /api/v1/remote/helm-test/...
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHelmRemote:
|
||||
def test_index_yaml_is_mutable(self, client, patched_deps):
|
||||
"""index.yaml is detected as mutable (package-type default)."""
|
||||
deps = patched_deps
|
||||
index = b"apiVersion: v1\nentries:\n vault:\n - urls:\n - https://helm.releases.hashicorp.com/vault-0.29.1.tgz\n"
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = index
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/helm-test/index.yaml")
|
||||
assert response.status_code == 200
|
||||
deps["cache"].mark_index_cached.assert_not_called()
|
||||
|
||||
def test_index_yaml_urls_rewritten_to_proxy(self, client, patched_deps):
|
||||
"""base_url chart URLs in a cached index.yaml are rewritten to our proxy."""
|
||||
deps = patched_deps
|
||||
index = b"apiVersion: v1\nentries:\n vault:\n - urls:\n - https://helm.releases.hashicorp.com/vault-0.29.1.tgz\n"
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = index
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/helm-test/index.yaml")
|
||||
assert response.status_code == 200
|
||||
assert b"helm.releases.hashicorp.com" not in response.content
|
||||
assert b"/api/v1/remote/helm-test/vault-0.29.1.tgz" in response.content
|
||||
|
||||
def test_index_yaml_content_type_is_yaml(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"apiVersion: v1\nentries: {}\n"
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/helm-test/index.yaml")
|
||||
assert response.status_code == 200
|
||||
assert "text/yaml" in response.headers["content-type"]
|
||||
|
||||
def test_chart_tarball_immutable_returns_gzip_content_type(self, client, patched_deps):
|
||||
"""Versioned chart tarballs match immutable_patterns and are served as binary."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"\x1f\x8b chart bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/helm-test/vault-0.29.1.tgz")
|
||||
assert response.status_code == 200
|
||||
assert "application/gzip" in response.headers["content-type"]
|
||||
assert response.headers["X-Artifact-Source"] == "cache"
|
||||
|
||||
def test_index_yaml_cache_miss_fetches_upstream(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
index = b"apiVersion: v1\nentries:\n vault:\n - urls:\n - https://helm.releases.hashicorp.com/vault-0.29.1.tgz\n"
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = index
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
new_callable=AsyncMock,
|
||||
return_value={"status": "cached"},
|
||||
) as mock_fetch:
|
||||
response = client.get("/api/v1/remote/helm-test/index.yaml")
|
||||
|
||||
mock_fetch.assert_called_once()
|
||||
assert response.status_code == 200
|
||||
assert b"helm.releases.hashicorp.com" not in response.content
|
||||
|
||||
def test_non_tgz_non_yaml_path_blocked_by_pattern(self, client, patched_deps):
|
||||
"""Paths that don't match immutable_patterns and aren't mutable are blocked."""
|
||||
deps = patched_deps
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/helm-test/vault.zip")
|
||||
assert response.status_code == 403
|
||||
|
||||
Reference in New Issue
Block a user