Compare commits
7 Commits
4c1f77e679
...
v2.1.3
| Author | SHA1 | Date | |
|---|---|---|---|
| b3d12f4962 | |||
| 92b9f9a03e | |||
| 7930023de8 | |||
| 869a1f8c02 | |||
| 1b2ee0d37f | |||
| 33e7365a88 | |||
| cf854a2ace |
@@ -32,6 +32,9 @@ COPY --chown=appuser:appuser pyproject.toml uv.lock README.md ./
|
|||||||
|
|
||||||
# Switch to appuser and install Python dependencies
|
# Switch to appuser and install Python dependencies
|
||||||
USER appuser
|
USER appuser
|
||||||
|
ARG VERSION=dev
|
||||||
|
ENV HATCH_VCS_PRETEND_VERSION=${VERSION} \
|
||||||
|
SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}
|
||||||
RUN uv sync --frozen
|
RUN uv sync --frozen
|
||||||
|
|
||||||
# Copy application source
|
# Copy application source
|
||||||
|
|||||||
@@ -45,3 +45,27 @@ docker-clean:
|
|||||||
docker system prune -f
|
docker system prune -f
|
||||||
|
|
||||||
docker-restart: docker-down docker-up
|
docker-restart: docker-down docker-up
|
||||||
|
|
||||||
|
# Bump helpers — reads the latest semver tag and creates the next one.
|
||||||
|
# If no tag exists yet, starts from v0.0.0.
|
||||||
|
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
||||||
|
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
||||||
|
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
||||||
|
_MIN := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f2)
|
||||||
|
_PAT := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f3)
|
||||||
|
|
||||||
|
patch:
|
||||||
|
@NEW=v$(_MAJ).$(_MIN).$(shell expr $(_PAT) + 1); \
|
||||||
|
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||||
|
|
||||||
|
minor:
|
||||||
|
@NEW=v$(_MAJ).$(shell expr $(_MIN) + 1).0; \
|
||||||
|
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||||
|
|
||||||
|
major:
|
||||||
|
@NEW=v$(shell expr $(_MAJ) + 1).0.0; \
|
||||||
|
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||||
|
|
||||||
|
_tag:
|
||||||
|
git push origin $(TAG)
|
||||||
|
docker-compose build --no-cache --build-arg VERSION=$(TAG:v%=%)
|
||||||
|
|||||||
@@ -0,0 +1,137 @@
|
|||||||
|
# ArtifactAPI Specification
|
||||||
|
|
||||||
|
## Repository model
|
||||||
|
|
||||||
|
Every repository entry in `remotes.yaml` has two orthogonal fields:
|
||||||
|
|
||||||
|
| field | values | meaning |
|
||||||
|
|---|---|---|
|
||||||
|
| `type` | `local`, `remote`, `virtual` | repository kind — how the repo is served |
|
||||||
|
| `package` | `docker`, `rpm`, `alpine`, `generic` | package format — what protocol and caching rules to apply |
|
||||||
|
|
||||||
|
**type**
|
||||||
|
|
||||||
|
- `local` — files are uploaded directly to the API and stored in S3; no upstream.
|
||||||
|
- `remote` — proxies and caches content from an upstream URL (`base_url`).
|
||||||
|
- `virtual` — aggregates multiple repositories (not yet implemented).
|
||||||
|
|
||||||
|
**package**
|
||||||
|
|
||||||
|
- `docker` — upstream speaks the OCI Distribution API (Bearer auth, manifest/blob paths).
|
||||||
|
- `rpm` — upstream is an RPM repository; repodata files are index files.
|
||||||
|
- `alpine` — upstream is an Alpine APK repository; `APKINDEX.tar.gz` is an index file.
|
||||||
|
- `generic` — plain HTTP file download; no format-specific logic.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Caching
|
||||||
|
|
||||||
|
Two cache classes determine retention:
|
||||||
|
|
||||||
|
| class | stored | TTL |
|
||||||
|
|---|---|---|
|
||||||
|
| **file** | S3 object, no Redis entry | `file_ttl` — `0` means indefinite |
|
||||||
|
| **index** | S3 object + Redis TTL key | `index_ttl` — when the Redis key expires the S3 object is deleted and re-fetched |
|
||||||
|
|
||||||
|
Index files are mutable metadata that must expire. File-class objects are treated as immutable and cached indefinitely (unless `file_ttl` is set).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Docker package rules
|
||||||
|
|
||||||
|
### URL construction
|
||||||
|
|
||||||
|
Remote URLs are prefixed with `/v2/` for `package: docker` remotes:
|
||||||
|
|
||||||
|
```
|
||||||
|
{base_url}/v2/{path}
|
||||||
|
```
|
||||||
|
|
||||||
|
e.g. `library/nginx/manifests/latest` → `https://registry-1.docker.io/v2/library/nginx/manifests/latest`
|
||||||
|
|
||||||
|
### Authentication
|
||||||
|
|
||||||
|
Docker registries use Bearer token challenges. On a `401 Unauthorized` response, the API:
|
||||||
|
|
||||||
|
1. Parses the `WWW-Authenticate: Bearer` header for `realm`, `service`, and `scope`.
|
||||||
|
2. Fetches a token from the auth realm, supplying `username`/`password` from the remote config if present.
|
||||||
|
3. Retries the request with `Authorization: Bearer <token>`.
|
||||||
|
|
||||||
|
Tokens are cached in-memory keyed by `(realm, service, scope, username)` and expire 30 seconds before their stated `expires_in`.
|
||||||
|
|
||||||
|
### Cache classification
|
||||||
|
|
||||||
|
| path pattern | mutable | class | TTL source |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `/manifests/<tag>` | yes | index | `index_ttl` |
|
||||||
|
| `/tags/list` | yes | index | `index_ttl` |
|
||||||
|
| `/manifests/sha256:<digest>` | no | file | `file_ttl` |
|
||||||
|
| `/blobs/sha256:<digest>` | no | file | `file_ttl` |
|
||||||
|
|
||||||
|
Tag-based manifests and tag lists are mutable and cached as index. Digest-pinned manifests and blobs are content-addressed and cached indefinitely as files.
|
||||||
|
|
||||||
|
### Blob deduplication
|
||||||
|
|
||||||
|
Blobs are stored under a digest-keyed path shared across all images on the same remote:
|
||||||
|
|
||||||
|
```
|
||||||
|
{remote_name}/blobs/sha256/{digest}
|
||||||
|
```
|
||||||
|
|
||||||
|
The same layer pulled by different images is stored once.
|
||||||
|
|
||||||
|
### Accept headers
|
||||||
|
|
||||||
|
| path | `Accept` header sent upstream |
|
||||||
|
|---|---|
|
||||||
|
| `/manifests/…` | `application/vnd.docker.distribution.manifest.v2+json`, `application/vnd.oci.image.manifest.v1+json`, `application/vnd.oci.image.index.v1+json`, `application/vnd.docker.distribution.manifest.list.v2+json` |
|
||||||
|
| `/blobs/…` | `application/octet-stream` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## OCI Distribution API endpoint
|
||||||
|
|
||||||
|
The API exposes a native Docker registry interface so clients can use `docker pull` directly:
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /v2/ — version ping
|
||||||
|
GET /v2/{remote}/{image}/manifests/{ref} — fetch manifest
|
||||||
|
HEAD /v2/{remote}/{image}/manifests/{ref} — manifest metadata
|
||||||
|
GET /v2/{remote}/{image}/blobs/{digest} — fetch blob
|
||||||
|
HEAD /v2/{remote}/{image}/blobs/{digest} — blob metadata
|
||||||
|
```
|
||||||
|
|
||||||
|
Responses include `Docker-Distribution-Api-Version`, `Docker-Content-Digest`, and the correct OCI `Content-Type` (detected from the manifest `mediaType` field).
|
||||||
|
|
||||||
|
Only remotes with `package: docker` are accessible via this endpoint. All other remotes return `400`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## include_patterns
|
||||||
|
|
||||||
|
`include_patterns` is a list of Python regexes applied to every request before any upstream fetch or cache lookup.
|
||||||
|
|
||||||
|
**Generic remotes (`/api/v1/remote/…`):**
|
||||||
|
- Patterns match against the file path and the full path.
|
||||||
|
- Index files (mutable metadata) bypass pattern checks and are always allowed.
|
||||||
|
|
||||||
|
**Docker remotes (`/v2/…`):**
|
||||||
|
- Patterns match against the image name (first two path segments, e.g. `library/nginx`) and the full path.
|
||||||
|
- The index-file exemption does **not** apply — patterns restrict whole images, including their manifests and tag lists.
|
||||||
|
- No patterns configured → all images allowed.
|
||||||
|
|
||||||
|
Returns `403` when a request is blocked.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Versioning
|
||||||
|
|
||||||
|
The package version is derived from git tags via `hatch-vcs`. Tags follow the format `v{MAJOR}.{MINOR}.{PATCH}`.
|
||||||
|
|
||||||
|
Docker images are built with the version injected at build time:
|
||||||
|
|
||||||
|
```
|
||||||
|
SETUPTOOLS_SCM_PRETEND_VERSION=<version> uv sync --frozen
|
||||||
|
```
|
||||||
|
|
||||||
|
The `Makefile` provides `patch`, `minor`, and `major` targets that tag the current commit and rebuild the container image.
|
||||||
+5
-9
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "artifactapi"
|
name = "artifactapi"
|
||||||
version = "2.0.5"
|
dynamic = ["version"]
|
||||||
description = "Generic artifact caching system with support for various package managers"
|
description = "Generic artifact caching system with support for various package managers"
|
||||||
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
@@ -23,9 +23,12 @@ license = {text = "MIT"}
|
|||||||
artifactapi = "artifactapi.main:main"
|
artifactapi = "artifactapi.main:main"
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["hatchling"]
|
requires = ["hatchling", "hatch-vcs"]
|
||||||
build-backend = "hatchling.build"
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[tool.hatch.version]
|
||||||
|
source = "vcs"
|
||||||
|
|
||||||
[tool.hatch.metadata]
|
[tool.hatch.metadata]
|
||||||
allow-direct-references = true
|
allow-direct-references = true
|
||||||
|
|
||||||
@@ -40,11 +43,4 @@ dev = [
|
|||||||
"isort>=5.12.0",
|
"isort>=5.12.0",
|
||||||
"mypy>=1.6.0",
|
"mypy>=1.6.0",
|
||||||
"ruff>=0.1.0",
|
"ruff>=0.1.0",
|
||||||
"bump-my-version>=1.2.0",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[tool.bumpversion]
|
|
||||||
current_version = "2.0.5"
|
|
||||||
commit = true
|
|
||||||
tag = true
|
|
||||||
message = "Bump version: {current_version} → {new_version}"
|
|
||||||
|
|||||||
+12
-3
@@ -167,7 +167,7 @@ async def construct_remote_url(remote_name: str, path: str) -> str:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Handle Docker registry URLs
|
# Handle Docker registry URLs
|
||||||
if remote_config.get("type") == "docker":
|
if remote_config.get("package") == "docker":
|
||||||
# Convert Docker paths to v2 API format
|
# Convert Docker paths to v2 API format
|
||||||
# e.g., library/nginx/manifests/latest -> v2/library/nginx/manifests/latest
|
# e.g., library/nginx/manifests/latest -> v2/library/nginx/manifests/latest
|
||||||
return f"{base_url}/v2/{path}"
|
return f"{base_url}/v2/{path}"
|
||||||
@@ -215,7 +215,7 @@ async def cache_single_artifact(url: str, remote_name: str, path: str) -> dict:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
remote_config = config.get_remote_config(remote_name) or {}
|
remote_config = config.get_remote_config(remote_name) or {}
|
||||||
is_docker = remote_config.get("type") == "docker" or "/v2/" in url
|
is_docker = remote_config.get("package") == "docker" or "/v2/" in url
|
||||||
|
|
||||||
# Prepare headers for Docker registry requests
|
# Prepare headers for Docker registry requests
|
||||||
headers = {}
|
headers = {}
|
||||||
@@ -449,9 +449,18 @@ async def docker_v2_proxy(request: Request, remote_name: str, path: str):
|
|||||||
remote_config = config.get_remote_config(remote_name)
|
remote_config = config.get_remote_config(remote_name)
|
||||||
if not remote_config:
|
if not remote_config:
|
||||||
raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
|
raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
|
||||||
if remote_config.get("type") != "docker":
|
if remote_config.get("package") != "docker":
|
||||||
raise HTTPException(status_code=400, detail=f"Remote '{remote_name}' is not a docker remote")
|
raise HTTPException(status_code=400, detail=f"Remote '{remote_name}' is not a docker remote")
|
||||||
|
|
||||||
|
# Check include_patterns against the image name (e.g. "library/nginx")
|
||||||
|
patterns = config.get_repository_patterns(remote_name, "")
|
||||||
|
if patterns:
|
||||||
|
path_parts = path.split("/")
|
||||||
|
image_name = "/".join(path_parts[:2]) if len(path_parts) >= 2 else path
|
||||||
|
if not any(re.search(p, path) or re.search(p, image_name) for p in patterns):
|
||||||
|
logger.info(f"PATTERN BLOCKED: {remote_name}/{path}")
|
||||||
|
raise HTTPException(status_code=403, detail="Image not allowed by configuration patterns")
|
||||||
|
|
||||||
remote_url = await construct_remote_url(remote_name, path)
|
remote_url = await construct_remote_url(remote_name, path)
|
||||||
|
|
||||||
cached_key = storage.get_object_key(remote_name, path)
|
cached_key = storage.get_object_key(remote_name, path)
|
||||||
|
|||||||
Reference in New Issue
Block a user