Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 unkinben
|
34160032fc |
@@ -0,0 +1,15 @@
|
|||||||
|
.git/
|
||||||
|
.venv/
|
||||||
|
dist/
|
||||||
|
tests/
|
||||||
|
remotes.yaml
|
||||||
|
ca-bundle.pem
|
||||||
|
.env
|
||||||
|
*.log
|
||||||
|
docker-compose.yml
|
||||||
|
.woodpecker/
|
||||||
|
.tox/
|
||||||
|
.ruff_cache/
|
||||||
|
.pytest_cache/
|
||||||
|
.pre-commit-cache/
|
||||||
|
minio_data/
|
||||||
+60
-5
@@ -1,6 +1,61 @@
|
|||||||
bin/
|
# Python
|
||||||
/terraform/
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
*.so
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
# e2e-docker fixtures are real package files (.rpm, .tgz, .whl, .zip, ...) that
|
# Virtual environment
|
||||||
# are intentionally tracked, overriding any global ignore of those extensions.
|
.venv/
|
||||||
!e2e-docker/fixtures/**
|
venv/
|
||||||
|
ENV/
|
||||||
|
env/
|
||||||
|
|
||||||
|
# IDE
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
|
||||||
|
# Environment variables
|
||||||
|
.env
|
||||||
|
|
||||||
|
# Logs
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# uv
|
||||||
|
uv.lock
|
||||||
|
|
||||||
|
# tox
|
||||||
|
.tox/
|
||||||
|
|
||||||
|
# pytest
|
||||||
|
.pytest_cache/
|
||||||
|
|
||||||
|
# pre-commit
|
||||||
|
.pre-commit-cache/
|
||||||
|
|
||||||
|
# ruff
|
||||||
|
.ruff_cache/
|
||||||
|
|
||||||
|
# Docker volumes
|
||||||
|
minio_data/
|
||||||
|
|
||||||
|
# Local configuration overrides
|
||||||
|
ca-bundle.pem
|
||||||
|
|||||||
+5
-22
@@ -1,24 +1,7 @@
|
|||||||
repos:
|
repos:
|
||||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||||
rev: v5.0.0
|
rev: v0.15.12
|
||||||
hooks:
|
hooks:
|
||||||
- id: trailing-whitespace
|
- id: ruff
|
||||||
- id: end-of-file-fixer
|
args: [--fix, --exit-non-zero-on-fix]
|
||||||
- id: check-yaml
|
- id: ruff-format
|
||||||
- id: check-added-large-files
|
|
||||||
- id: check-merge-conflict
|
|
||||||
|
|
||||||
- repo: https://github.com/dnephin/pre-commit-golang
|
|
||||||
rev: v0.5.1
|
|
||||||
hooks:
|
|
||||||
- id: go-fmt
|
|
||||||
- id: go-mod-tidy
|
|
||||||
|
|
||||||
- repo: local
|
|
||||||
hooks:
|
|
||||||
- id: go-vet
|
|
||||||
name: go vet
|
|
||||||
entry: go vet ./...
|
|
||||||
language: system
|
|
||||||
types: [go]
|
|
||||||
pass_filenames: false
|
|
||||||
|
|||||||
+8
-38
@@ -3,46 +3,16 @@ when:
|
|||||||
ref: refs/tags/v*
|
ref: refs/tags/v*
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: docker-api
|
- name: docker
|
||||||
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
|
image: woodpeckerci/plugin-docker-buildx
|
||||||
settings:
|
settings:
|
||||||
registry: artifactapi.k8s.syd1.au.unkin.net
|
registry: git.unkin.net
|
||||||
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi
|
repo: git.unkin.net/unkin/artifactapi
|
||||||
build_args:
|
username: droneci
|
||||||
VERSION: ${CI_COMMIT_TAG}
|
password:
|
||||||
|
from_secret: DRONECI_PASSWORD
|
||||||
tags:
|
tags:
|
||||||
- ${CI_COMMIT_TAG}
|
- ${CI_COMMIT_TAG}
|
||||||
- latest
|
- latest
|
||||||
backend_options:
|
|
||||||
kubernetes:
|
|
||||||
serviceAccountName: default
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: 1Gi
|
|
||||||
cpu: 1
|
|
||||||
limits:
|
|
||||||
memory: 4Gi
|
|
||||||
cpu: 2
|
|
||||||
|
|
||||||
- name: docker-web
|
|
||||||
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
|
|
||||||
settings:
|
|
||||||
registry: artifactapi.k8s.syd1.au.unkin.net
|
|
||||||
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui
|
|
||||||
dockerfile: ui/Dockerfile.ui
|
|
||||||
context: ui
|
|
||||||
build_args:
|
build_args:
|
||||||
BASE_PATH: /ui
|
- VERSION=${CI_COMMIT_TAG##v}
|
||||||
tags:
|
|
||||||
- ${CI_COMMIT_TAG}
|
|
||||||
- latest
|
|
||||||
backend_options:
|
|
||||||
kubernetes:
|
|
||||||
serviceAccountName: default
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: 1Gi
|
|
||||||
cpu: 1
|
|
||||||
limits:
|
|
||||||
memory: 4Gi
|
|
||||||
cpu: 2
|
|
||||||
|
|||||||
@@ -3,32 +3,7 @@ when:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: pre-commit
|
- name: pre-commit
|
||||||
# gobuilder trusts the internal CA, which the S3 build cache endpoint needs.
|
image: git.unkin.net/unkin/almalinux9-base:20260308
|
||||||
# go-cache-plugin is baked into the image; S3 errors degrade to cache misses.
|
|
||||||
image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.2-alma9"
|
|
||||||
commands:
|
commands:
|
||||||
- uvx pre-commit run --all-files
|
- uvx pre-commit run --all-files
|
||||||
environment:
|
|
||||||
# golib lives on Gitea; skip the public proxy/sum db.
|
|
||||||
GOPRIVATE: git.unkin.net
|
|
||||||
GOCACHEPROG: "go-cache-plugin --cache-dir=/tmp/gocache"
|
|
||||||
GOCACHE_S3_BUCKET: gocache
|
|
||||||
# Explicit region skips a GetBucketLocation probe RGW handles poorly.
|
|
||||||
GOCACHE_S3_REGION: us-east-1
|
|
||||||
GOCACHE_S3_ENDPOINT_URL: "https://s3.ceph.unkin.net"
|
|
||||||
GOCACHE_S3_PATH_STYLE: "true"
|
|
||||||
GOCACHE_KEY_PREFIX: ci-artifactapi
|
|
||||||
AWS_ACCESS_KEY_ID:
|
|
||||||
from_secret: GOCACHE_AWS_ACCESS_KEY_ID
|
|
||||||
AWS_SECRET_ACCESS_KEY:
|
|
||||||
from_secret: GOCACHE_AWS_SECRET_ACCESS_KEY
|
|
||||||
backend_options:
|
|
||||||
kubernetes:
|
|
||||||
serviceAccountName: default
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: 512Mi
|
|
||||||
cpu: 1
|
|
||||||
limits:
|
|
||||||
memory: 2Gi
|
|
||||||
cpu: 2
|
|
||||||
|
|||||||
+2
-28
@@ -3,32 +3,6 @@ when:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: test
|
- name: test
|
||||||
# gobuilder trusts the internal CA, which the S3 build cache endpoint needs.
|
image: git.unkin.net/unkin/almalinux9-base:20260308
|
||||||
# go-cache-plugin is baked into the image; S3 errors degrade to cache misses.
|
|
||||||
image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.2-alma9"
|
|
||||||
commands:
|
commands:
|
||||||
- go test -race -count=1 ./pkg/... ./internal/...
|
- uvx --python 3.11 --with tox-uv tox
|
||||||
environment:
|
|
||||||
# golib lives on Gitea; skip the public proxy/sum db.
|
|
||||||
GOPRIVATE: git.unkin.net
|
|
||||||
GOCACHEPROG: "go-cache-plugin --cache-dir=/tmp/gocache"
|
|
||||||
GOCACHE_S3_BUCKET: gocache
|
|
||||||
# Explicit region skips a GetBucketLocation probe RGW handles poorly.
|
|
||||||
GOCACHE_S3_REGION: us-east-1
|
|
||||||
GOCACHE_S3_ENDPOINT_URL: "https://s3.ceph.unkin.net"
|
|
||||||
GOCACHE_S3_PATH_STYLE: "true"
|
|
||||||
GOCACHE_KEY_PREFIX: ci-artifactapi
|
|
||||||
AWS_ACCESS_KEY_ID:
|
|
||||||
from_secret: GOCACHE_AWS_ACCESS_KEY_ID
|
|
||||||
AWS_SECRET_ACCESS_KEY:
|
|
||||||
from_secret: GOCACHE_AWS_SECRET_ACCESS_KEY
|
|
||||||
backend_options:
|
|
||||||
kubernetes:
|
|
||||||
serviceAccountName: default
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: 1Gi
|
|
||||||
cpu: 1
|
|
||||||
limits:
|
|
||||||
memory: 4Gi
|
|
||||||
cpu: 2
|
|
||||||
|
|||||||
+16
-18
@@ -1,25 +1,23 @@
|
|||||||
FROM golang:1.25-alpine AS builder
|
FROM git.unkin.net/unkin/almalinux9-base:latest
|
||||||
|
|
||||||
RUN apk add --no-cache git
|
ARG VERSION=0.0.0.dev0
|
||||||
|
|
||||||
WORKDIR /build
|
COPY . /build
|
||||||
|
|
||||||
# golib is fetched straight from Gitea; the public proxy and sum db have no
|
RUN HATCH_VCS_PRETEND_VERSION=${VERSION} \
|
||||||
# view of git.unkin.net modules.
|
SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} \
|
||||||
ENV GOPRIVATE=git.unkin.net
|
uv build --wheel --directory /build && \
|
||||||
|
useradd -m -r -s /bin/sh appuser
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
USER appuser
|
||||||
RUN go mod download
|
RUN uv tool install --from /build/dist/*.whl artifactapi
|
||||||
|
|
||||||
COPY . .
|
USER root
|
||||||
|
RUN rm -rf /build
|
||||||
ARG VERSION=dev
|
|
||||||
RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=${VERSION}" -o artifactapi ./cmd/artifactapi
|
|
||||||
|
|
||||||
FROM gcr.io/distroless/static-debian12:nonroot
|
|
||||||
|
|
||||||
COPY --from=builder /build/artifactapi /usr/local/bin/artifactapi
|
|
||||||
|
|
||||||
EXPOSE 8000
|
EXPOSE 8000
|
||||||
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 CMD curl -f http://localhost:8000/health || exit 1
|
||||||
ENTRYPOINT ["artifactapi"]
|
USER appuser
|
||||||
|
ENV PATH="/home/appuser/.local/bin:$PATH"
|
||||||
|
WORKDIR /app
|
||||||
|
CMD ["artifactapi"]
|
||||||
|
|||||||
@@ -1,59 +1,59 @@
|
|||||||
.PHONY: build test lint fmt e2e docker-e2e docker docker-ui compose clean tidy check-go
|
.PHONY: build install dev clean test lint format pre-commit tox docker-build docker-up docker-down docker-logs docker-rebuild docker-clean docker-restart
|
||||||
|
|
||||||
BINARY := bin/artifactapi
|
build:
|
||||||
MODULE := git.unkin.net/unkin/artifactapi
|
docker build -t artifactapi:dev .
|
||||||
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0.0-dev")
|
|
||||||
GO_VERSION_REQUIRED := 1.23
|
|
||||||
GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/')
|
|
||||||
|
|
||||||
# git.unkin.net modules (golib) are fetched straight from Gitea, never via the
|
install: build
|
||||||
# public proxy or sum db, which have no view of them. Exported here rather than
|
|
||||||
# written with `go env -w`, so a fresh checkout needs no machine-local setup.
|
|
||||||
export GOPRIVATE := git.unkin.net
|
|
||||||
|
|
||||||
check-go:
|
docker-build: build
|
||||||
@if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \
|
|
||||||
echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \
|
|
||||||
fi
|
|
||||||
|
|
||||||
build: check-go tidy
|
dev: build
|
||||||
go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(BINARY) ./cmd/artifactapi
|
uv sync --dev
|
||||||
|
|
||||||
test: check-go
|
|
||||||
go test -race -count=1 ./pkg/... ./internal/...
|
|
||||||
|
|
||||||
lint: check-go
|
|
||||||
golangci-lint run ./...
|
|
||||||
go vet ./...
|
|
||||||
|
|
||||||
fmt: check-go
|
|
||||||
gofmt -w .
|
|
||||||
goimports -w .
|
|
||||||
|
|
||||||
e2e: check-go
|
|
||||||
TESTCONTAINERS_RYUK_DISABLED=true go test -tags=e2e -race -count=1 -timeout=5m ./e2e/...
|
|
||||||
|
|
||||||
# Build the container, bring up the full docker-compose stack + a mock upstream,
|
|
||||||
# and run the black-box suite against the running product.
|
|
||||||
docker-e2e: check-go
|
|
||||||
./scripts/docker-e2e.sh
|
|
||||||
|
|
||||||
docker:
|
|
||||||
docker build -t artifactapi:$(VERSION) .
|
|
||||||
|
|
||||||
docker-ui:
|
|
||||||
docker build -t artifactapi-ui:$(VERSION) -f ui/Dockerfile.ui ui/
|
|
||||||
|
|
||||||
compose:
|
|
||||||
docker compose up -d
|
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf bin/
|
rm -rf .venv
|
||||||
|
rm -rf build/
|
||||||
|
rm -rf dist/
|
||||||
|
rm -rf *.egg-info/
|
||||||
|
|
||||||
tidy:
|
test:
|
||||||
go mod tidy
|
uvx --python 3.11 --with tox-uv tox
|
||||||
|
|
||||||
|
tox:
|
||||||
|
uvx --python 3.11 --with tox-uv tox
|
||||||
|
|
||||||
|
pre-commit:
|
||||||
|
uvx --python 3.11 pre-commit run --all-files
|
||||||
|
|
||||||
|
lint:
|
||||||
|
uv run ruff check --fix .
|
||||||
|
|
||||||
|
format:
|
||||||
|
uv run ruff format .
|
||||||
|
|
||||||
|
run:
|
||||||
|
uv run python -m src.artifactapi.main
|
||||||
|
|
||||||
|
docker-up:
|
||||||
|
docker-compose up --build --force-recreate -d
|
||||||
|
|
||||||
|
docker-down:
|
||||||
|
docker-compose down
|
||||||
|
|
||||||
|
docker-logs:
|
||||||
|
docker-compose logs -f
|
||||||
|
|
||||||
|
docker-rebuild:
|
||||||
|
docker-compose build --no-cache
|
||||||
|
|
||||||
|
docker-clean:
|
||||||
|
docker-compose down -v --remove-orphans
|
||||||
|
docker system prune -f
|
||||||
|
|
||||||
|
docker-restart: docker-down docker-up
|
||||||
|
|
||||||
# Bump helpers — reads the latest semver tag and creates the next one.
|
# Bump helpers — reads the latest semver tag and creates the next one.
|
||||||
|
# If no tag exists yet, starts from v0.0.0.
|
||||||
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
||||||
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
||||||
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
||||||
|
|||||||
@@ -1,880 +0,0 @@
|
|||||||
# ArtifactAPI v3 — Go Rewrite Plan
|
|
||||||
|
|
||||||
## Context
|
|
||||||
|
|
||||||
ArtifactAPI is a production artifact proxy/cache serving ~42 remotes (Docker registries, Helm repos, RPM/Alpine repos, GitHub releases, PyPI, npm, Puppet Forge, Terraform registries, Go module proxies) across a Kubernetes cluster. The current Python (FastAPI) implementation works but has architectural debt: opaque hashed S3 paths, no UI for visibility, YAML config files that drift, no garbage collection, no access logging, and virtual repos limited to Helm only.
|
|
||||||
|
|
||||||
The v3 rewrite targets: a single Go binary (API + TUI), a separate React frontend (own Dockerfile), a Terraform provider (separate repo), content-addressable storage, and a cleaner data model that makes the cache inspectable and manageable.
|
|
||||||
|
|
||||||
**Repo**: Same repo (`git.unkin.net/unkin/artifactapi`), new branch.
|
|
||||||
**Module**: `git.unkin.net/unkin/artifactapi`
|
|
||||||
**Frontend**: React + Vite, separate Dockerfile, talks to API
|
|
||||||
**Terraform provider**: Separate repo (`terraform-provider-artifactapi`)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Architecture Overview
|
|
||||||
|
|
||||||
```
|
|
||||||
┌───────────────────────────────────┐ ┌──────────────────────┐
|
|
||||||
│ Go Binary (API + TUI) │ │ Frontend Container │
|
|
||||||
│ │ │ │
|
|
||||||
│ ┌──────────┐ ┌───────────────┐ │ │ React + Vite SPA │
|
|
||||||
│ │ REST API │ │ Proxy Engine │ │◄───│ nginx / node serve │
|
|
||||||
│ │ /api/v2 │ │ /api/v1/... │ │ │ Dockerfile.ui │
|
|
||||||
│ │ │ │ /v2/... (OCI) │ │ └──────────────────────┘
|
|
||||||
│ └────┬─────┘ └──────┬────────┘ │
|
|
||||||
│ │ │ │ ┌──────────────────────┐
|
|
||||||
│ ┌────┴───────────────┴────────┐ │ │ Terraform Provider │
|
|
||||||
│ │ Data Layer │ │◄───│ (separate repo) │
|
|
||||||
│ │ PostgreSQL · Redis · S3 │ │ └──────────────────────┘
|
|
||||||
│ └─────────────────────────────┘ │
|
|
||||||
│ │ ┌──────────────────────┐
|
|
||||||
│ ┌─────────────────────────────┐ │ │ TUI (subcommand) │
|
|
||||||
│ │ artifactapi tui │──│───►│ artifactapi tui │
|
|
||||||
│ └─────────────────────────────┘ │ │ --endpoint <url> │
|
|
||||||
└───────────────────────────────────┘ └──────────────────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
Three independent deployment units:
|
|
||||||
1. **Go binary** — API server + TUI subcommand (single `Dockerfile`)
|
|
||||||
2. **React frontend** — SPA served by nginx (`Dockerfile.ui`), talks to `/api/v2`
|
|
||||||
3. **Terraform provider** — separate repo, calls `/api/v2` CRUD
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Project Structure (Modular)
|
|
||||||
|
|
||||||
```
|
|
||||||
artifactapi/
|
|
||||||
├── cmd/
|
|
||||||
│ └── artifactapi/
|
|
||||||
│ └── main.go # entrypoint: serve / tui subcommands
|
|
||||||
│
|
|
||||||
├── pkg/ # PUBLIC — importable by terraform provider, CLI tools
|
|
||||||
│ ├── models/ # shared domain types
|
|
||||||
│ │ ├── remote.go # Remote, RemoteConfig, PackageType enum
|
|
||||||
│ │ ├── virtual.go # Virtual, VirtualConfig
|
|
||||||
│ │ ├── artifact.go # Artifact, Blob, AccessLogEntry
|
|
||||||
│ │ ├── local.go # LocalFile, LocalRepo
|
|
||||||
│ │ └── stats.go # RemoteStats, OverviewStats
|
|
||||||
│ └── client/ # typed Go API client (used by TUI + Terraform provider)
|
|
||||||
│ ├── client.go # Client struct, base HTTP
|
|
||||||
│ ├── remotes.go # remote CRUD methods
|
|
||||||
│ ├── virtuals.go # virtual CRUD methods
|
|
||||||
│ ├── objects.go # object browse/evict methods
|
|
||||||
│ └── stats.go # stats methods
|
|
||||||
│
|
|
||||||
├── internal/ # PRIVATE — server internals
|
|
||||||
│ ├── server/
|
|
||||||
│ │ ├── server.go # HTTP server setup, router
|
|
||||||
│ │ └── middleware.go # logging, recovery, request-id, access logging
|
|
||||||
│ │
|
|
||||||
│ ├── api/
|
|
||||||
│ │ ├── v1/ # proxy endpoints (v1 compat)
|
|
||||||
│ │ │ ├── proxy.go # GET /api/v1/remote/{name}/{path}
|
|
||||||
│ │ │ ├── docker.go # /v2/{name}/{path}
|
|
||||||
│ │ │ ├── virtual.go # GET /api/v1/virtual/{name}/{path}
|
|
||||||
│ │ │ └── local.go # CRUD /api/v1/local/{name}/{path}
|
|
||||||
│ │ └── v2/ # management API
|
|
||||||
│ │ ├── remotes.go # CRUD + stats
|
|
||||||
│ │ ├── virtuals.go # CRUD
|
|
||||||
│ │ ├── objects.go # browse/evict cached objects
|
|
||||||
│ │ ├── stats.go # overview, top-remotes
|
|
||||||
│ │ ├── events.go # SSE stream
|
|
||||||
│ │ └── health.go # health, metrics
|
|
||||||
│ │
|
|
||||||
│ ├── provider/ # package-type providers (registry protocol handlers)
|
|
||||||
│ │ ├── provider.go # Provider interface + registry
|
|
||||||
│ │ ├── generic/
|
|
||||||
│ │ │ ├── generic.go
|
|
||||||
│ │ │ └── generic_test.go
|
|
||||||
│ │ ├── docker/
|
|
||||||
│ │ │ ├── docker.go # OCI Distribution v2 via go-containerregistry
|
|
||||||
│ │ │ ├── auth.go # Bearer token fetch + cache
|
|
||||||
│ │ │ └── docker_test.go
|
|
||||||
│ │ ├── helm/
|
|
||||||
│ │ │ ├── helm.go # index rewriting via helm.sh/helm/v3/pkg/repo
|
|
||||||
│ │ │ ├── merger.go # virtual index merge
|
|
||||||
│ │ │ └── helm_test.go
|
|
||||||
│ │ ├── pypi/
|
|
||||||
│ │ │ ├── pypi.go # simple index HTML rewriting
|
|
||||||
│ │ │ ├── merger.go # virtual simple index merge
|
|
||||||
│ │ │ └── pypi_test.go
|
|
||||||
│ │ ├── npm/
|
|
||||||
│ │ │ ├── npm.go # metadata JSON rewriting
|
|
||||||
│ │ │ └── npm_test.go
|
|
||||||
│ │ ├── rpm/
|
|
||||||
│ │ │ ├── rpm.go # repodata patterns
|
|
||||||
│ │ │ └── rpm_test.go
|
|
||||||
│ │ ├── alpine/
|
|
||||||
│ │ │ ├── alpine.go # APKINDEX patterns
|
|
||||||
│ │ │ └── alpine_test.go
|
|
||||||
│ │ ├── puppet/
|
|
||||||
│ │ │ ├── puppet.go # file_uri JSON rewriting
|
|
||||||
│ │ │ └── puppet_test.go
|
|
||||||
│ │ ├── terraform/
|
|
||||||
│ │ │ ├── terraform.go # registry protocol, download URL rewriting
|
|
||||||
│ │ │ └── terraform_test.go
|
|
||||||
│ │ └── goproxy/
|
|
||||||
│ │ ├── goproxy.go # Go module proxy protocol (GOPROXY)
|
|
||||||
│ │ └── goproxy_test.go
|
|
||||||
│ │
|
|
||||||
│ ├── proxy/
|
|
||||||
│ │ ├── engine.go # core fetch-or-cache logic
|
|
||||||
│ │ ├── engine_test.go
|
|
||||||
│ │ ├── classifier.go # immutable vs mutable classification
|
|
||||||
│ │ ├── classifier_test.go
|
|
||||||
│ │ ├── revalidator.go # conditional HEAD requests (ETag/Last-Modified)
|
|
||||||
│ │ └── circuit.go # per-remote circuit breaker
|
|
||||||
│ │
|
|
||||||
│ ├── storage/
|
|
||||||
│ │ ├── s3.go # S3 client (minio-go — works with MinIO, Ceph, AWS)
|
|
||||||
│ │ ├── s3_test.go
|
|
||||||
│ │ ├── cas.go # content-addressable store logic
|
|
||||||
│ │ └── cas_test.go
|
|
||||||
│ │
|
|
||||||
│ ├── cache/
|
|
||||||
│ │ ├── redis.go # TTL management, fetch locks
|
|
||||||
│ │ ├── redis_test.go
|
|
||||||
│ │ └── lock.go # distributed lock abstraction
|
|
||||||
│ │
|
|
||||||
│ ├── database/
|
|
||||||
│ │ ├── postgres.go # connection pool, migration runner
|
|
||||||
│ │ ├── queries/ # SQL query files or sqlc-generated code
|
|
||||||
│ │ │ ├── remotes.sql.go
|
|
||||||
│ │ │ ├── virtuals.sql.go
|
|
||||||
│ │ │ ├── artifacts.sql.go
|
|
||||||
│ │ │ └── access_log.sql.go
|
|
||||||
│ │ └── migrations/ # golang-migrate SQL files
|
|
||||||
│ │ ├── 001_initial.up.sql
|
|
||||||
│ │ └── 001_initial.down.sql
|
|
||||||
│ │
|
|
||||||
│ ├── metrics/
|
|
||||||
│ │ └── prometheus.go # counters, gauges, histograms
|
|
||||||
│ │
|
|
||||||
│ ├── gc/
|
|
||||||
│ │ ├── gc.go # background garbage collection goroutine
|
|
||||||
│ │ └── gc_test.go
|
|
||||||
│ │
|
|
||||||
│ ├── tui/
|
|
||||||
│ │ ├── app.go # Bubble Tea main model
|
|
||||||
│ │ ├── views/
|
|
||||||
│ │ │ ├── dashboard.go
|
|
||||||
│ │ │ ├── remotes.go
|
|
||||||
│ │ │ ├── objects.go
|
|
||||||
│ │ │ └── virtuals.go
|
|
||||||
│ │ └── components/
|
|
||||||
│ │ ├── table.go
|
|
||||||
│ │ └── statusbar.go
|
|
||||||
│ │
|
|
||||||
│ └── config/
|
|
||||||
│ └── env.go # environment variable parsing + validation
|
|
||||||
│
|
|
||||||
├── ui/ # React frontend — SEPARATE DOCKERFILE
|
|
||||||
│ ├── src/
|
|
||||||
│ │ ├── App.tsx
|
|
||||||
│ │ ├── pages/
|
|
||||||
│ │ │ ├── Dashboard.tsx
|
|
||||||
│ │ │ ├── Remotes.tsx
|
|
||||||
│ │ │ ├── RemoteDetail.tsx
|
|
||||||
│ │ │ ├── Virtuals.tsx
|
|
||||||
│ │ │ └── Objects.tsx
|
|
||||||
│ │ ├── components/
|
|
||||||
│ │ │ ├── RemoteTable.tsx
|
|
||||||
│ │ │ ├── ObjectBrowser.tsx
|
|
||||||
│ │ │ ├── StatsCard.tsx
|
|
||||||
│ │ │ └── EventFeed.tsx
|
|
||||||
│ │ └── api/
|
|
||||||
│ │ └── client.ts # typed API client
|
|
||||||
│ ├── package.json
|
|
||||||
│ ├── vite.config.ts
|
|
||||||
│ ├── tsconfig.json
|
|
||||||
│ ├── Dockerfile.ui # multi-stage: node build → nginx
|
|
||||||
│ └── nginx.conf # proxy /api/* to backend, serve SPA
|
|
||||||
│
|
|
||||||
├── e2e/ # end-to-end integration tests
|
|
||||||
│ ├── e2e_test.go # TestMain spins up docker-compose stack
|
|
||||||
│ ├── proxy_test.go # proxy through real remotes
|
|
||||||
│ ├── docker_test.go # Docker v2 protocol e2e
|
|
||||||
│ ├── management_test.go # v2 API CRUD
|
|
||||||
│ ├── virtual_test.go # virtual repo merge e2e
|
|
||||||
│ └── docker-compose.e2e.yml # postgres + redis + minio for tests
|
|
||||||
│
|
|
||||||
├── go.mod
|
|
||||||
├── go.sum
|
|
||||||
├── Makefile
|
|
||||||
├── Dockerfile # Go binary (API server + TUI)
|
|
||||||
├── Dockerfile.ui # symlink or copy → ui/Dockerfile.ui
|
|
||||||
└── docker-compose.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
### Key Modularisation Decisions
|
|
||||||
|
|
||||||
- **`pkg/models/`** — Shared domain types importable by the Terraform provider and any external tooling. No dependencies on internal packages
|
|
||||||
- **`pkg/client/`** — Typed Go API client used by both the TUI and the Terraform provider. Depends only on `pkg/models/` and stdlib
|
|
||||||
- **`internal/provider/`** — Each package type is its own subpackage with isolated tests. A provider registry maps `PackageType → Provider`
|
|
||||||
- **`internal/database/queries/`** — Use [sqlc](https://sqlc.dev/) to generate type-safe query functions from SQL, or hand-written query files
|
|
||||||
- **`e2e/`** — Separate test binary that spins up a real docker-compose stack
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Go Ecosystem Libraries
|
|
||||||
|
|
||||||
Prefer existing, maintained Go modules over writing protocol handlers from scratch.
|
|
||||||
|
|
||||||
### Package-Type Libraries
|
|
||||||
|
|
||||||
| Package Type | Go Module | What It Gives Us |
|
|
||||||
|---|---|---|
|
|
||||||
| **Docker/OCI** | `github.com/google/go-containerregistry` | Full Registry v2/OCI client: manifest parsing, auth challenges, blob operations. `pkg/registry` can implement a v2 server. Reference: `github.com/regclient/regclient` |
|
|
||||||
| **Helm** | `helm.sh/helm/v3/pkg/repo` | Parse/generate `index.yaml`, `IndexFile`/`ChartVersion` types, URL entries. Used directly for merge |
|
|
||||||
| **Terraform** | `github.com/hashicorp/terraform-registry-address` | Provider/module address parsing, `ForRegistryProtocol()` URL generation. Protocol spec: provider registry protocol v1 |
|
|
||||||
| **Go Modules** | `github.com/goproxy/goproxy` | Minimalist GOPROXY protocol handler, implements full spec as `http.Handler`. Handles `/@v/list`, `/@v/{v}.info`, `/@v/{v}.mod`, `/@v/{v}.zip`, `/@latest` |
|
|
||||||
| **RPM** | `rs3.io/go/rpm/repomd` | Parse `repomd.xml`, `primary.xml` with proper XML namespace handling |
|
|
||||||
| **Alpine** | `gitlab.alpinelinux.org/alpine/go` | Official Alpine library: parse APKINDEX, `.apk` files |
|
|
||||||
| **PyPI** | stdlib `golang.org/x/net/html` | No dedicated Go PyPI library exists. Parse simple index HTML with `x/net/html`, extract `<a>` tags. Minimal — the rewriting is just href replacement |
|
|
||||||
| **npm** | stdlib `encoding/json` | npm metadata is JSON — parse with stdlib, rewrite `dist.tarball` URLs. No special library needed |
|
|
||||||
| **Puppet Forge** | stdlib `encoding/json` | Forge API is JSON — parse and rewrite `file_uri` fields. Community lib `github.com/johnmccabe/go-puppetforge` exists but is thin; stdlib suffices |
|
|
||||||
|
|
||||||
### Infrastructure Libraries
|
|
||||||
|
|
||||||
| Purpose | Go Module | Why This One |
|
|
||||||
|---|---|---|
|
|
||||||
| **HTTP router** | `github.com/go-chi/chi/v5` | Lightweight, stdlib `http.Handler` compatible, middleware chain |
|
|
||||||
| **PostgreSQL** | `github.com/jackc/pgx/v5` | Pure Go, connection pooling, COPY support, prepared statements |
|
|
||||||
| **SQL generation** | `github.com/sqlc-dev/sqlc` | Generate type-safe Go from SQL queries — no ORM, no reflection |
|
|
||||||
| **Redis** | `github.com/redis/go-redis/v9` | Full Redis client, pipelining, pub/sub |
|
|
||||||
| **S3 (MinIO/Ceph/AWS)** | `github.com/minio/minio-go/v7` | Native S3-compatible client. Works with MinIO, Ceph RGW, AWS S3, any S3-compatible backend out of the box. Lighter than aws-sdk-go-v2, purpose-built for S3 compat |
|
|
||||||
| **DB migrations** | `github.com/golang-migrate/migrate/v4` | SQL file-based migrations, CLI + library |
|
|
||||||
| **Prometheus** | `github.com/prometheus/client_golang` | Counters, gauges, histograms |
|
|
||||||
| **TUI** | `github.com/charmbracelet/bubbletea` | Elm-architecture TUI framework |
|
|
||||||
| **TUI styling** | `github.com/charmbracelet/lipgloss` | Terminal styling |
|
|
||||||
| **TUI components** | `github.com/charmbracelet/bubbles` | Table, text input, spinner, etc. |
|
|
||||||
| **Structured logging** | `log/slog` (stdlib) | Go 1.21+ structured logging, zero dependencies |
|
|
||||||
| **Testing** | `github.com/stretchr/testify` | Assertions + require for unit tests |
|
|
||||||
| **Test containers** | `github.com/testcontainers/testcontainers-go` | Spin up Postgres/Redis/MinIO in e2e tests |
|
|
||||||
|
|
||||||
### S3 Client: Multi-Backend Support
|
|
||||||
|
|
||||||
Using `minio-go/v7` as the S3 client because it natively supports:
|
|
||||||
- **MinIO** — primary development/production target
|
|
||||||
- **Ceph RGW** — S3-compatible via endpoint config
|
|
||||||
- **AWS S3** — via region + credential config
|
|
||||||
- **Any S3-compatible** — GCS (interop mode), Wasabi, DigitalOcean Spaces, etc.
|
|
||||||
|
|
||||||
No abstraction layer needed — `minio-go` handles endpoint differences internally. Config:
|
|
||||||
```go
|
|
||||||
client, _ := minio.New(endpoint, &minio.Options{
|
|
||||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
|
||||||
Secure: useTLS,
|
|
||||||
Region: region, // optional, for AWS
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Data Layer
|
|
||||||
|
|
||||||
### PostgreSQL Schema
|
|
||||||
|
|
||||||
```sql
|
|
||||||
-- Remotes: managed exclusively by Terraform
|
|
||||||
CREATE TABLE remotes (
|
|
||||||
name TEXT PRIMARY KEY,
|
|
||||||
package_type TEXT NOT NULL, -- generic, docker, helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy
|
|
||||||
base_url TEXT NOT NULL,
|
|
||||||
description TEXT DEFAULT '',
|
|
||||||
username TEXT DEFAULT '',
|
|
||||||
password TEXT DEFAULT '',
|
|
||||||
immutable_ttl INTEGER DEFAULT 0,
|
|
||||||
mutable_ttl INTEGER DEFAULT 3600,
|
|
||||||
check_mutable BOOLEAN DEFAULT TRUE,
|
|
||||||
immutable_patterns TEXT[] DEFAULT '{}', -- user-defined immutable patterns
|
|
||||||
mutable_patterns TEXT[] DEFAULT '{}', -- user-defined mutable patterns (merged with provider built-ins)
|
|
||||||
allowlist TEXT[] DEFAULT '{}', -- if empty, allow all paths; if non-empty, only matching paths proxied
|
|
||||||
blocklist TEXT[] DEFAULT '{}', -- always denied, checked before allowlist
|
|
||||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
|
||||||
ban_tags TEXT[] DEFAULT '{}',
|
|
||||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
|
||||||
quarantine_days INTEGER DEFAULT 3,
|
|
||||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
|
||||||
releases_remote TEXT DEFAULT '', -- terraform type: name of CDN remote for download URL rewriting
|
|
||||||
managed_by TEXT DEFAULT '', -- 'terraform' or empty
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
-- Virtual repositories
|
|
||||||
CREATE TABLE virtuals (
|
|
||||||
name TEXT PRIMARY KEY,
|
|
||||||
package_type TEXT NOT NULL,
|
|
||||||
description TEXT DEFAULT '',
|
|
||||||
members TEXT[] NOT NULL,
|
|
||||||
managed_by TEXT DEFAULT '',
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
-- Content-addressable blob storage tracking
|
|
||||||
CREATE TABLE blobs (
|
|
||||||
content_hash TEXT PRIMARY KEY,
|
|
||||||
s3_key TEXT NOT NULL,
|
|
||||||
size_bytes BIGINT NOT NULL,
|
|
||||||
content_type TEXT DEFAULT 'application/octet-stream',
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
-- Artifact metadata: maps (remote, path) → content blob
|
|
||||||
CREATE TABLE artifacts (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
|
||||||
path TEXT NOT NULL,
|
|
||||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
|
||||||
upstream_etag TEXT DEFAULT '',
|
|
||||||
upstream_last_modified TIMESTAMPTZ,
|
|
||||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
fetch_count BIGINT DEFAULT 1,
|
|
||||||
access_count BIGINT DEFAULT 1,
|
|
||||||
UNIQUE(remote_name, path)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_artifacts_remote ON artifacts(remote_name);
|
|
||||||
CREATE INDEX idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
|
||||||
|
|
||||||
-- Local file uploads
|
|
||||||
CREATE TABLE local_files (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
repo_name TEXT NOT NULL,
|
|
||||||
file_path TEXT NOT NULL,
|
|
||||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
UNIQUE(repo_name, file_path)
|
|
||||||
);
|
|
||||||
|
|
||||||
-- Access log (append-only, powers dashboards)
|
|
||||||
CREATE TABLE access_log (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
remote_name TEXT NOT NULL,
|
|
||||||
path TEXT NOT NULL,
|
|
||||||
cache_hit BOOLEAN NOT NULL,
|
|
||||||
size_bytes BIGINT DEFAULT 0,
|
|
||||||
upstream_ms INTEGER DEFAULT 0,
|
|
||||||
client_ip TEXT DEFAULT '',
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_access_log_remote_time ON access_log(remote_name, created_at);
|
|
||||||
```
|
|
||||||
|
|
||||||
### Redis Usage (Ephemeral Only)
|
|
||||||
|
|
||||||
| Key pattern | Type | TTL | Purpose |
|
|
||||||
|---|---|---|---|
|
|
||||||
| `ttl:{remote}:{path}` | STRING | remote's immutable/mutable TTL | Artifact freshness — existence = still fresh |
|
|
||||||
| `lock:{remote}:{path}` | STRING (NX) | 30s | Fetch lock — prevents thundering herd |
|
|
||||||
| `etag:{remote}:{path}` | STRING | same as TTL key | Cached ETag for conditional revalidation |
|
|
||||||
| `circuit:{remote}` | STRING | configurable | Circuit breaker — consecutive failure count |
|
|
||||||
|
|
||||||
Losing Redis = all TTLs expire = next request re-validates upstream. No data loss.
|
|
||||||
|
|
||||||
### S3 Layout (Content-Addressable)
|
|
||||||
|
|
||||||
```
|
|
||||||
artifacts-bucket/
|
|
||||||
├── blobs/sha256/{content_hash} # immutable CAS blobs
|
|
||||||
├── indexes/{remote}/{path} # mutable index files (helm, pypi, rpm, etc.)
|
|
||||||
├── indexes/{virtual}/{path} # merged virtual indexes
|
|
||||||
└── local/{repo}/{path} # user uploads (CAS-backed via blobs table)
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Terraform Remote Type (New in v2)
|
|
||||||
|
|
||||||
The `terraform` package type proxies the Terraform Provider Registry Protocol:
|
|
||||||
|
|
||||||
- **URL construction**: prepends `/v1/providers/` to request paths
|
|
||||||
- **Built-in mutable pattern**: `[^/]+/[^/]+/versions$` (version listings change over time)
|
|
||||||
- **Built-in immutable pattern**: `[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$` (per-version download info is fixed)
|
|
||||||
- **Response rewriting**: download info JSON — rewrites `download_url`, `shasums_url`, `shasums_signature_url` to route through a companion `releases_remote` (e.g., `hashicorp-releases` generic remote)
|
|
||||||
- **Config**: requires `releases_remote` field pointing to the CDN remote that serves the actual binaries
|
|
||||||
|
|
||||||
Uses `github.com/hashicorp/terraform-registry-address` for address parsing and protocol-compliant URL generation.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Go Module Proxy Remote Type (New)
|
|
||||||
|
|
||||||
The `goproxy` package type implements the GOPROXY protocol (Go module proxy):
|
|
||||||
|
|
||||||
| Endpoint | Mutability | Description |
|
|
||||||
|---|---|---|
|
|
||||||
| `{module}/@v/list` | Mutable | Plain text list of known versions |
|
|
||||||
| `{module}/@latest` | Mutable | JSON metadata for latest version |
|
|
||||||
| `{module}/@v/{version}.info` | Immutable | JSON version metadata (`Version`, `Time`) |
|
|
||||||
| `{module}/@v/{version}.mod` | Immutable | `go.mod` file for that version |
|
|
||||||
| `{module}/@v/{version}.zip` | Immutable | Source archive for that version |
|
|
||||||
|
|
||||||
- **No URL rewriting needed** — responses are self-contained (no embedded URLs)
|
|
||||||
- **Config**: `base_url` points to upstream proxy (e.g., `https://proxy.golang.org`)
|
|
||||||
- **Client usage**: set `GOPROXY=https://artifactapi.example.com/api/v1/remote/goproxy`
|
|
||||||
- Uses `github.com/goproxy/goproxy` for protocol handling
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Allowlist / Blocklist / Automatic Mutable Patterns
|
|
||||||
|
|
||||||
### Access Control (Per-Remote)
|
|
||||||
|
|
||||||
| Field | Default | Behavior |
|
|
||||||
|---|---|---|
|
|
||||||
| `blocklist` | `[]` (empty) | If a path matches any blocklist pattern → **403 Forbidden**. Checked first |
|
|
||||||
| `allowlist` | `[]` (empty) | If empty → **allow everything**. If non-empty → only matching paths are proxied; everything else → **403** |
|
|
||||||
|
|
||||||
Evaluation order: blocklist → allowlist → proxy. No allowlist + no blocklist = open proxy (default).
|
|
||||||
|
|
||||||
### Automatic Mutable Patterns (Per-Provider Built-ins)
|
|
||||||
|
|
||||||
Each provider declares built-in mutable patterns that are **always merged** with user-defined `mutable_patterns`. Users never need to configure these — the provider knows which paths change over time.
|
|
||||||
|
|
||||||
| Provider | Built-in Mutable Patterns | Rationale |
|
|
||||||
|---|---|---|
|
|
||||||
| **generic** | *(none)* | No convention for what's mutable |
|
|
||||||
| **docker** | `/manifests/(?!sha256:)[^/]+$`, `/tags/list$` | Tag manifests change; digest manifests don't |
|
|
||||||
| **helm** | `index\.yaml$` | Chart index changes when new charts are published |
|
|
||||||
| **pypi** | `simple/` | Package index pages change with new releases |
|
|
||||||
| **npm** | `^[^/]+$` (package metadata, not `.tgz`) | Package metadata changes; tarballs are immutable |
|
|
||||||
| **rpm** | `repomd\.xml$`, `repodata/.*`, `Packages\.gz$` | Repo metadata rebuilt on every publish |
|
|
||||||
| **alpine** | `APKINDEX\.tar\.gz$` | Package index rebuilt on every publish |
|
|
||||||
| **puppet** | `^v3/modules/`, `^v3/releases` | Module metadata changes with new releases |
|
|
||||||
| **terraform** | `[^/]+/[^/]+/versions$` | Provider version listings grow over time |
|
|
||||||
| **goproxy** | `@v/list$`, `@latest$` | Version list and latest pointer change |
|
|
||||||
|
|
||||||
These are returned by `Provider.BuiltinMutablePatterns()` and merged at classification time:
|
|
||||||
```
|
|
||||||
effective_mutable = provider.BuiltinMutablePatterns() ∪ remote.mutable_patterns
|
|
||||||
```
|
|
||||||
|
|
||||||
If a path matches `effective_mutable` → use `mutable_ttl`. If it matches `remote.immutable_patterns` → use `immutable_ttl`. Immutable patterns take precedence over mutable when both match.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## API Design
|
|
||||||
|
|
||||||
### v1 Proxy Endpoints (Backwards Compatible)
|
|
||||||
|
|
||||||
| Method | Path | Description |
|
|
||||||
|---|---|---|
|
|
||||||
| `GET` | `/api/v1/remote/{name}/{path}` | Proxy/cache artifact |
|
|
||||||
| `GET` | `/api/v1/virtual/{name}/{path}` | Virtual repo proxy |
|
|
||||||
| `GET/HEAD` | `/v2/{name}/{path}` | Docker Registry v2 |
|
|
||||||
| `GET` | `/v2/` | Docker v2 ping |
|
|
||||||
| `GET/PUT/HEAD/DELETE` | `/api/v1/local/{name}/{path}` | Local repo CRUD |
|
|
||||||
|
|
||||||
### v2 Management API (New)
|
|
||||||
|
|
||||||
```
|
|
||||||
GET /api/v2/remotes → [{name, package_type, base_url, description, stats}]
|
|
||||||
GET /api/v2/remotes/{name} → {full config + stats + health}
|
|
||||||
POST /api/v2/remotes → create remote (Terraform provider)
|
|
||||||
PUT /api/v2/remotes/{name} → update remote (Terraform provider)
|
|
||||||
DELETE /api/v2/remotes/{name} → delete remote — cascades artifacts, GC cleans S3
|
|
||||||
|
|
||||||
GET /api/v2/virtuals → [{name, package_type, members, stats}]
|
|
||||||
GET /api/v2/virtuals/{name} → {full config + member details}
|
|
||||||
POST /api/v2/virtuals → create virtual
|
|
||||||
PUT /api/v2/virtuals/{name} → update virtual
|
|
||||||
DELETE /api/v2/virtuals/{name} → delete virtual
|
|
||||||
|
|
||||||
GET /api/v2/remotes/{name}/objects → paginated objects
|
|
||||||
?q=pattern&sort=size|accessed|age&page=1&per_page=50
|
|
||||||
DELETE /api/v2/remotes/{name}/objects/{path} → evict specific cached object
|
|
||||||
DELETE /api/v2/remotes/{name}/cache → flush cache
|
|
||||||
?type=all|indexes|blobs
|
|
||||||
|
|
||||||
GET /api/v2/stats → overview stats
|
|
||||||
GET /api/v2/stats/top-remotes → top remotes by size/requests/hit-rate
|
|
||||||
|
|
||||||
GET /api/v2/health → {status, postgres, redis, s3, uptime}
|
|
||||||
GET /metrics → Prometheus format
|
|
||||||
GET /api/v2/events → SSE stream
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Proxy Engine
|
|
||||||
|
|
||||||
### Request Flow
|
|
||||||
|
|
||||||
```
|
|
||||||
Client Request
|
|
||||||
│
|
|
||||||
▼
|
|
||||||
Classify (immutable/mutable/denied)
|
|
||||||
│
|
|
||||||
├── blocklist match → 403
|
|
||||||
├── allowlist non-empty + no match → 403
|
|
||||||
│
|
|
||||||
▼
|
|
||||||
Check Redis TTL key
|
|
||||||
│
|
|
||||||
├── exists (fresh) → serve from S3, log access
|
|
||||||
│
|
|
||||||
├── missing (expired or uncached)
|
|
||||||
│ │
|
|
||||||
│ ▼
|
|
||||||
│ Acquire fetch lock (Redis SETNX, 30s TTL)
|
|
||||||
│ │
|
|
||||||
│ ├── lock acquired
|
|
||||||
│ │ ├── mutable + check_mutable + have ETag → HEAD upstream
|
|
||||||
│ │ │ ├── 304 → refresh TTL, serve from S3
|
|
||||||
│ │ │ └── changed → full fetch
|
|
||||||
│ │ └── full fetch from upstream
|
|
||||||
│ │ → provider.RewriteResponse() if needed
|
|
||||||
│ │ → CAS store (hash → check blobs → upload if new)
|
|
||||||
│ │ → upsert artifact in Postgres
|
|
||||||
│ │ → set Redis TTL + release lock
|
|
||||||
│ │ → on upstream error + stale_on_error → refresh TTL, serve stale
|
|
||||||
│ │
|
|
||||||
│ └── lock not acquired → poll S3 briefly, serve if another pod fetched it
|
|
||||||
│
|
|
||||||
▼
|
|
||||||
Stream response from S3, log access
|
|
||||||
```
|
|
||||||
|
|
||||||
### Circuit Breaker
|
|
||||||
|
|
||||||
Per-remote, tracked in Redis. Closed → Open (after N failures) → Half-open (after cooldown). Exposed via `GET /api/v2/remotes/{name}` health field.
|
|
||||||
|
|
||||||
### Content-Addressable Storage
|
|
||||||
|
|
||||||
1. Stream upstream → temp file, compute SHA256 inline
|
|
||||||
2. Check `blobs` table for hash
|
|
||||||
3. Exists → skip S3 upload, upsert `artifacts` row only
|
|
||||||
4. New → upload to `blobs/sha256/{hash}`, insert both rows
|
|
||||||
|
|
||||||
### Garbage Collection
|
|
||||||
|
|
||||||
Background goroutine (configurable interval, default 1h):
|
|
||||||
1. Orphaned blobs: delete S3 objects whose `content_hash` has no referencing `artifacts` or `local_files` rows
|
|
||||||
2. Cold artifacts: optional per-remote, delete artifacts not accessed in N days
|
|
||||||
3. Remote deletion: `ON DELETE CASCADE` handles Postgres; GC sweeps orphaned blobs
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Package Providers
|
|
||||||
|
|
||||||
### Provider Interface
|
|
||||||
|
|
||||||
```go
|
|
||||||
type Provider interface {
|
|
||||||
Type() models.PackageType
|
|
||||||
BuiltinMutablePatterns() []*regexp.Regexp
|
|
||||||
BuiltinImmutablePatterns() []*regexp.Regexp
|
|
||||||
ContentType(path string) string
|
|
||||||
UpstreamURL(remote models.Remote, path string) string
|
|
||||||
RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error)
|
|
||||||
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type IndexMerger interface {
|
|
||||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Provider Registry
|
|
||||||
|
|
||||||
```go
|
|
||||||
var registry = map[models.PackageType]Provider{
|
|
||||||
models.PackageGeneric: &generic.Provider{},
|
|
||||||
models.PackageDocker: &docker.Provider{},
|
|
||||||
models.PackageHelm: &helm.Provider{},
|
|
||||||
models.PackagePyPI: &pypi.Provider{},
|
|
||||||
models.PackageNPM: &npm.Provider{},
|
|
||||||
models.PackageRPM: &rpm.Provider{},
|
|
||||||
models.PackageAlpine: &alpine.Provider{},
|
|
||||||
models.PackagePuppet: &puppet.Provider{},
|
|
||||||
models.PackageTerraform: &terraform.Provider{},
|
|
||||||
models.PackageGoProxy: &goproxy.Provider{},
|
|
||||||
}
|
|
||||||
|
|
||||||
func Get(t models.PackageType) (Provider, error) { ... }
|
|
||||||
```
|
|
||||||
|
|
||||||
Each provider lives in its own subpackage under `internal/provider/` with its own `_test.go`.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Testing Strategy
|
|
||||||
|
|
||||||
### Unit Tests
|
|
||||||
|
|
||||||
Every package gets `_test.go` files alongside the source. Run with `go test ./...`.
|
|
||||||
|
|
||||||
| Package | What's Tested |
|
|
||||||
|---|---|
|
|
||||||
| `internal/provider/docker/` | Auth token parsing/caching, manifest classification, tag banning, URL construction, blob key generation |
|
|
||||||
| `internal/provider/helm/` | `index.yaml` parsing (using `helm.sh/helm/v3/pkg/repo`), URL rewriting, index merging |
|
|
||||||
| `internal/provider/pypi/` | Simple index HTML parsing, URL rewriting, index merging |
|
|
||||||
| `internal/provider/npm/` | Metadata JSON rewriting (`dist.tarball` URLs) |
|
|
||||||
| `internal/provider/terraform/` | Registry URL construction, download info JSON rewriting, `releases_remote` URL extraction |
|
|
||||||
| `internal/provider/rpm/` | Mutable pattern matching (repodata) |
|
|
||||||
| `internal/provider/alpine/` | Mutable pattern matching (APKINDEX) |
|
|
||||||
| `internal/provider/puppet/` | `file_uri` JSON rewriting |
|
|
||||||
| `internal/proxy/` | Classifier (immutable vs mutable vs denied), circuit breaker state transitions, revalidator logic |
|
|
||||||
| `internal/storage/` | CAS key generation, dedup detection, S3 operation mocking |
|
|
||||||
| `internal/cache/` | Redis TTL set/check, fetch lock acquire/release/contention |
|
|
||||||
| `internal/gc/` | Orphan detection queries, cold artifact selection |
|
|
||||||
| `pkg/models/` | Model validation, PackageType enum |
|
|
||||||
| `pkg/client/` | API client request/response serialization |
|
|
||||||
|
|
||||||
### End-to-End Tests
|
|
||||||
|
|
||||||
Located in `e2e/`. Use `testcontainers-go` to spin up real Postgres, Redis, and MinIO containers. The test binary starts the actual `artifactapi` server against these backends.
|
|
||||||
|
|
||||||
```go
|
|
||||||
// e2e/e2e_test.go
|
|
||||||
func TestMain(m *testing.M) {
|
|
||||||
// Start postgres, redis, minio via testcontainers-go
|
|
||||||
// Run migrations
|
|
||||||
// Start artifactapi server on random port
|
|
||||||
// Run tests
|
|
||||||
// Tear down
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
| Test File | What's Tested |
|
|
||||||
|---|---|
|
|
||||||
| `e2e/proxy_test.go` | Proxy a real GitHub release through generic remote, verify S3 storage, verify Redis TTL, verify Postgres artifact row, verify cache hit on second request |
|
|
||||||
| `e2e/docker_test.go` | Pull a real image manifest + blob through Docker v2 proxy, verify blob deduplication, tag banning |
|
|
||||||
| `e2e/management_test.go` | Full CRUD lifecycle: create remote via v2 API, proxy through it, list objects, evict object, flush cache, delete remote |
|
|
||||||
| `e2e/virtual_test.go` | Create two helm remotes + virtual, fetch merged index, verify priority ordering |
|
|
||||||
| `e2e/terraform_test.go` | Proxy terraform provider version listing + download info, verify URL rewriting to releases_remote |
|
|
||||||
| `e2e/goproxy_test.go` | Proxy Go module `@v/list`, `.info`, `.mod`, `.zip` through GOPROXY remote, verify mutable vs immutable classification |
|
|
||||||
| `e2e/gc_test.go` | Create artifact, delete remote, trigger GC, verify S3 blob cleaned up |
|
|
||||||
|
|
||||||
### Code Quality
|
|
||||||
|
|
||||||
- `gofmt` / `goimports` — enforced in CI, run on save
|
|
||||||
- `golangci-lint` — comprehensive linter suite (staticcheck, errcheck, govet, etc.)
|
|
||||||
- `go vet ./...` — run in CI
|
|
||||||
- Makefile targets: `make test`, `make lint`, `make e2e`, `make fmt`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Terraform Provider (Separate Repo)
|
|
||||||
|
|
||||||
**Repo**: `terraform-provider-artifactapi`
|
|
||||||
**Uses**: `pkg/client/` and `pkg/models/` from the main module
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
provider "artifactapi" {
|
|
||||||
endpoint = "https://artifactapi.example.com"
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "artifactapi_remote" "terraform_registry" {
|
|
||||||
name = "terraform-registry"
|
|
||||||
package_type = "terraform"
|
|
||||||
base_url = "https://registry.terraform.io"
|
|
||||||
description = "Terraform provider registry"
|
|
||||||
releases_remote = artifactapi_remote.hashicorp_releases.name
|
|
||||||
|
|
||||||
immutable_patterns = [
|
|
||||||
"[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$",
|
|
||||||
]
|
|
||||||
|
|
||||||
cache {
|
|
||||||
immutable_ttl = 0
|
|
||||||
mutable_ttl = 300
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "artifactapi_remote" "hashicorp_releases" {
|
|
||||||
name = "hashicorp-releases"
|
|
||||||
package_type = "generic"
|
|
||||||
base_url = "https://releases.hashicorp.com"
|
|
||||||
|
|
||||||
immutable_patterns = [
|
|
||||||
".*\\.zip$",
|
|
||||||
".*SHA256SUMS(\\.sig)?$",
|
|
||||||
]
|
|
||||||
|
|
||||||
cache {
|
|
||||||
immutable_ttl = 0
|
|
||||||
mutable_ttl = 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "artifactapi_virtual" "helm" {
|
|
||||||
name = "helm"
|
|
||||||
package_type = "helm"
|
|
||||||
description = "All helm repos merged"
|
|
||||||
members = [
|
|
||||||
artifactapi_remote.jetstack.name,
|
|
||||||
artifactapi_remote.hashicorp_helm.name,
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Web UI (React + Vite — Separate Container)
|
|
||||||
|
|
||||||
### Deployment
|
|
||||||
|
|
||||||
Separate `Dockerfile.ui`: multi-stage build (node → nginx). Served as its own container/pod. nginx proxies `/api/*` to the Go backend.
|
|
||||||
|
|
||||||
### Pages
|
|
||||||
|
|
||||||
| Route | Content |
|
|
||||||
|---|---|
|
|
||||||
| `/` | Dashboard: total objects, storage used, dedup savings, bandwidth saved, top remotes chart, live SSE event feed, health indicators |
|
|
||||||
| `/remotes` | Remote table: name, type, description, object count, size, hit rate, health. Filter by type, sort any column |
|
|
||||||
| `/remotes/:name` | Config (read-only, "Managed by Terraform" badge), stats, object browser with search/sort/evict, flush actions |
|
|
||||||
| `/virtuals` | Virtual table: name, type, members, merged object count |
|
|
||||||
| `/virtuals/:name` | Member list with individual stats |
|
|
||||||
|
|
||||||
All config is read-only — managed by Terraform.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## TUI (Bubble Tea — Subcommand)
|
|
||||||
|
|
||||||
`artifactapi tui --endpoint http://localhost:8000` or via `ARTIFACTAPI_ENDPOINT` env.
|
|
||||||
|
|
||||||
Uses `pkg/client/` for all API calls (same client as Terraform provider).
|
|
||||||
|
|
||||||
| View | Key bindings |
|
|
||||||
|---|---|
|
|
||||||
| Dashboard | summary stats, top remotes |
|
|
||||||
| Remotes list | `j`/`k` navigate, `/` filter, `Enter` detail |
|
|
||||||
| Remote detail | config + stats, `Enter` → object browser |
|
|
||||||
| Object browser | `/` search, `d` evict, `f` flush |
|
|
||||||
| Virtuals | `j`/`k`, `Enter` detail |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Improvements Over v2
|
|
||||||
|
|
||||||
| Area | v2 (Python) | v3 (Go) |
|
|
||||||
|---|---|---|
|
|
||||||
| S3 paths | Hashed, opaque | Content-addressed CAS |
|
|
||||||
| Config | YAML files, mtime reload | Terraform via API |
|
|
||||||
| Package types | 8 types | 10 types (+ terraform, goproxy) |
|
|
||||||
| Virtual repos | Helm only | Helm + PyPI, extensible |
|
|
||||||
| Deduplication | Docker blobs only | All types via CAS |
|
|
||||||
| Revalidation | Opt-in flag | Default for all mutable |
|
|
||||||
| Access logging | None | Per-artifact in Postgres |
|
|
||||||
| GC | None | Background goroutine |
|
|
||||||
| Upstream health | Per-request | Circuit breaker |
|
|
||||||
| S3 backends | MinIO only | MinIO, Ceph, AWS (minio-go) |
|
|
||||||
| UI | None | Web dashboard + TUI |
|
|
||||||
| Binary | Python + venv | Static Go binary |
|
|
||||||
| Frontend | N/A | Separate container (React) |
|
|
||||||
| Testing | Mocked unit tests | Unit + e2e with real backends |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Implementation Phases
|
|
||||||
|
|
||||||
### Phase 1: Core Engine + Models
|
|
||||||
- Go module, Makefile (`make build test lint fmt e2e`), Dockerfile, docker-compose
|
|
||||||
- `pkg/models/` — all domain types
|
|
||||||
- PostgreSQL schema + migrations
|
|
||||||
- S3 storage layer with CAS (`minio-go/v7`)
|
|
||||||
- Redis cache layer (TTL, locks)
|
|
||||||
- Proxy engine: fetch-or-cache, classifier, revalidator
|
|
||||||
- Generic + Docker providers (most complexity: OCI auth, CAS, tag banning)
|
|
||||||
- Health + metrics endpoints
|
|
||||||
- Unit tests for all packages
|
|
||||||
- **Milestone**: proxy Docker + generic, cache in S3, track in Postgres
|
|
||||||
|
|
||||||
### Phase 2: All Providers
|
|
||||||
- Helm (using `helm.sh/helm/v3/pkg/repo`)
|
|
||||||
- PyPI (stdlib `x/net/html`)
|
|
||||||
- npm (stdlib `encoding/json`)
|
|
||||||
- RPM (using `rs3.io/go/rpm/repomd`)
|
|
||||||
- Alpine (using `gitlab.alpinelinux.org/alpine/go`)
|
|
||||||
- Puppet Forge (stdlib `encoding/json`)
|
|
||||||
- Terraform (using `hashicorp/terraform-registry-address`)
|
|
||||||
- Go Modules / GOPROXY (using `github.com/goproxy/goproxy`)
|
|
||||||
- Unit tests per provider
|
|
||||||
- **Milestone**: feature parity with v2 + goproxy
|
|
||||||
|
|
||||||
### Phase 3: Management API + Virtual Repos + GC
|
|
||||||
- `pkg/client/` — shared Go API client
|
|
||||||
- v2 CRUD endpoints
|
|
||||||
- Virtual repo engine: `IndexMerger` for Helm + PyPI
|
|
||||||
- Circuit breaker
|
|
||||||
- Access logging middleware
|
|
||||||
- GC goroutine
|
|
||||||
- **Milestone**: full API, virtuals, GC
|
|
||||||
|
|
||||||
### Phase 4: End-to-End Tests
|
|
||||||
- `e2e/` test suite with `testcontainers-go`
|
|
||||||
- Proxy, Docker, management, virtual, terraform, GC tests
|
|
||||||
- CI pipeline: `make e2e`
|
|
||||||
- **Milestone**: comprehensive e2e coverage
|
|
||||||
|
|
||||||
### Phase 5: Terraform Provider
|
|
||||||
- Separate repo: `terraform-provider-artifactapi`
|
|
||||||
- Imports `pkg/client/` and `pkg/models/`
|
|
||||||
- `artifactapi_remote` + `artifactapi_virtual` resources + data sources
|
|
||||||
- Import support
|
|
||||||
- **Milestone**: manage all config via Terraform
|
|
||||||
|
|
||||||
### Phase 6: Web UI
|
|
||||||
- React + Vite in `ui/`
|
|
||||||
- `Dockerfile.ui` (multi-stage → nginx)
|
|
||||||
- Dashboard, remotes, objects, virtuals pages
|
|
||||||
- SSE event feed
|
|
||||||
- **Milestone**: full web UI in separate container
|
|
||||||
|
|
||||||
### Phase 7: TUI
|
|
||||||
- Bubble Tea in `internal/tui/`
|
|
||||||
- Uses `pkg/client/`
|
|
||||||
- Dashboard, remotes, objects, virtuals views
|
|
||||||
- **Milestone**: TUI feature parity with web UI
|
|
||||||
|
|
||||||
### Phase 8: Migration + Cutover
|
|
||||||
- Migration tool: v2 YAML → Terraform HCL + `terraform import` commands
|
|
||||||
- S3 rehash script: `{remote}/{hash16}/{file}` → `blobs/sha256/{content_hash}`
|
|
||||||
- Parallel run, response comparison
|
|
||||||
- Cutover
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Makefile Targets
|
|
||||||
|
|
||||||
```makefile
|
|
||||||
.PHONY: build test lint fmt e2e docker docker-ui
|
|
||||||
|
|
||||||
build: ## Build Go binary
|
|
||||||
go build -o bin/artifactapi ./cmd/artifactapi
|
|
||||||
|
|
||||||
test: ## Run unit tests
|
|
||||||
go test ./...
|
|
||||||
|
|
||||||
lint: ## Run golangci-lint + go vet
|
|
||||||
golangci-lint run ./...
|
|
||||||
go vet ./...
|
|
||||||
|
|
||||||
fmt: ## Format code (gofmt + goimports)
|
|
||||||
gofmt -w .
|
|
||||||
goimports -w .
|
|
||||||
|
|
||||||
e2e: ## Run end-to-end tests (requires Docker)
|
|
||||||
go test -tags=e2e -count=1 -timeout=5m ./e2e/...
|
|
||||||
|
|
||||||
docker: ## Build API server Docker image
|
|
||||||
docker build -t artifactapi .
|
|
||||||
|
|
||||||
docker-ui: ## Build frontend Docker image
|
|
||||||
docker build -t artifactapi-ui -f ui/Dockerfile.ui ui/
|
|
||||||
|
|
||||||
compose: ## Start full stack (API + UI + Postgres + Redis + MinIO)
|
|
||||||
docker compose up -d
|
|
||||||
```
|
|
||||||
@@ -1,241 +1,557 @@
|
|||||||
# ArtifactAPI
|
# Artifact Storage System
|
||||||
|
|
||||||
Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
|
FastAPI caching proxy that downloads and stores files from remote sources in S3-compatible storage.
|
||||||
|
|
||||||
## Quick Start
|
## Features
|
||||||
|
|
||||||
```bash
|
- Remote definitions via `remotes.yaml` — generic HTTP, Alpine APK, RPM, Docker, PyPI, npm, Helm
|
||||||
# Start backing services
|
- Virtual repositories — merge multiple remotes of the same package type into a single unified index
|
||||||
docker compose up -d postgres redis minio
|
- Immutable/mutable caching model with per-remote TTLs
|
||||||
|
- Conditional revalidation (`If-None-Match` / `If-Modified-Since`) on TTL expiry
|
||||||
# Build and run
|
- Stale-on-upstream-error: refreshes TTL when backend is unreachable rather than evicting
|
||||||
make build
|
- URL rewriting for PyPI simple index, npm metadata, and Helm `index.yaml`
|
||||||
./bin/artifactapi
|
- Access control via regex patterns — unmatched paths return 403
|
||||||
|
|
||||||
# Frontend (separate container or dev server)
|
|
||||||
cd ui && npm install && npm run dev
|
|
||||||
```
|
|
||||||
|
|
||||||
API: `http://localhost:8000` | Frontend: `http://localhost:5173`
|
|
||||||
|
|
||||||
## Package Types
|
|
||||||
|
|
||||||
Providers classify paths automatically. Users only configure what to proxy and TTLs.
|
|
||||||
|
|
||||||
Classification precedence, TTLs and eviction for every remote: [docs/common/caching.md](docs/common/caching.md).
|
|
||||||
|
|
||||||
Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<type>/`:
|
|
||||||
|
|
||||||
- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md)
|
|
||||||
- `cargo`: [remote](docs/cargo/remote.md)
|
|
||||||
- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md)
|
|
||||||
- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)
|
|
||||||
- `generic`: [remote](docs/generic/remote.md)
|
|
||||||
- `gems`: [remote](docs/gems/remote.md)
|
|
||||||
- `goproxy`: [remote](docs/goproxy/remote.md)
|
|
||||||
- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md)
|
|
||||||
- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md)
|
|
||||||
- `nuget`: [remote](docs/nuget/remote.md)
|
|
||||||
- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md)
|
|
||||||
- `pypi`: [remote](docs/pypi/remote.md)
|
|
||||||
- `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md)
|
|
||||||
- `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md)
|
|
||||||
|
|
||||||
### GitHub authentication
|
|
||||||
|
|
||||||
Anonymous GitHub is capped at **60 requests/hour** and cannot read private
|
|
||||||
repositories. Configure a **server-level GitHub credential** to raise the ceiling
|
|
||||||
to roughly **5000 requests/hour** and to read private-repo release assets. The
|
|
||||||
credential is a process-wide machine identity applied by default to *every*
|
|
||||||
outbound GitHub request — the releases scan, the ranged asset-header fetches, and
|
|
||||||
the generic-github byte proxy that streams private release assets.
|
|
||||||
|
|
||||||
The credential is read from the environment (deliver it from a Vault or
|
|
||||||
Kubernetes secret). It is **never** stored per-remote in the database, **never**
|
|
||||||
returned by any API, and **never** logged. Configure **exactly one** mode.
|
|
||||||
|
|
||||||
**Precedence.** A remote's own `username`/`password` credential still wins for
|
|
||||||
that remote's requests; the server credential is the default for everything else.
|
|
||||||
With no credential configured at all, requests stay anonymous (current behavior).
|
|
||||||
Partial configuration (e.g. an App id with no private key) is a **startup error**
|
|
||||||
— artifactapi fails closed rather than silently falling back to anonymous.
|
|
||||||
|
|
||||||
Both modes share the syncer's single global rate limiter, so a token simply
|
|
||||||
raises the effective GitHub ceiling; the default limiter settings stay safe.
|
|
||||||
|
|
||||||
#### Mode 1 — Personal Access Token (minimum viable, recommended for free accounts)
|
|
||||||
|
|
||||||
Set `GITHUB_TOKEN`. It is sent as `Authorization: Bearer <token>`.
|
|
||||||
|
|
||||||
Recommended free-account setup — a **fine-grained PAT** scoped to just the target
|
|
||||||
repositories:
|
|
||||||
|
|
||||||
1. GitHub → *Settings → Developer settings → Personal access tokens →
|
|
||||||
Fine-grained tokens → Generate new token*.
|
|
||||||
2. Limit *Repository access* to the specific repo(s) serving releases.
|
|
||||||
3. Grant repository permissions **Contents: Read-only** and **Metadata:
|
|
||||||
Read-only** (Metadata is mandatory and auto-selected).
|
|
||||||
|
|
||||||
A classic PAT with the `repo` scope also works but is broader than necessary.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GITHUB_TOKEN=github_pat_xxxxxxxx
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Mode 2 — GitHub App installation token (proper machine identity)
|
|
||||||
|
|
||||||
A GitHub App is not tied to a personal account and can be created and installed on
|
|
||||||
free personal repos. artifactapi mints a short-lived RS256 **JWT** from the app
|
|
||||||
private key, exchanges it at `POST /app/installations/{id}/access_tokens` for a
|
|
||||||
~1-hour **installation access token**, caches that token, and refreshes it a few
|
|
||||||
minutes before expiry (thread-safe, single-flighted).
|
|
||||||
|
|
||||||
1. GitHub → *Settings → Developer settings → GitHub Apps → New GitHub App*.
|
|
||||||
2. Under *Permissions → Repository permissions* grant **Contents: Read-only**
|
|
||||||
(Metadata: Read-only is implied).
|
|
||||||
3. Generate a **private key** (downloads a PEM) and note the **App ID**.
|
|
||||||
4. *Install* the App on the account and select the target repositories, then read
|
|
||||||
the **Installation ID** from the installation URL
|
|
||||||
(`.../settings/installations/<installation-id>`).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GITHUB_APP_ID=123456
|
|
||||||
GITHUB_APP_INSTALLATION_ID=7654321
|
|
||||||
GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem
|
|
||||||
# or inline PEM (e.g. mounted from a secret):
|
|
||||||
# GITHUB_APP_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Terraform
|
|
||||||
|
|
||||||
Remotes and virtuals are managed by Terraform. Each package type has its own resource; see `docs/<type>/` for examples.
|
|
||||||
|
|
||||||
Provider: [terraform-provider-artifactapi](https://git.unkin.net/unkin/terraform-provider-artifactapi)
|
|
||||||
|
|
||||||
## Access Control
|
|
||||||
|
|
||||||
| Field | Default | Behaviour |
|
|
||||||
|---|---|---|
|
|
||||||
| `patterns` | empty (proxy all) | If set, only matching paths are proxied. Acts as allowlist. |
|
|
||||||
| `blocklist` | empty | Matching paths always denied. Checked first. |
|
|
||||||
| `mutable_patterns` | empty | Override: force paths to mutable TTL. |
|
|
||||||
| `immutable_patterns` | empty | Override: force paths to immutable TTL. |
|
|
||||||
|
|
||||||
No patterns + no blocklist = open proxy. Provider handles mutability classification automatically.
|
|
||||||
|
|
||||||
## API
|
|
||||||
|
|
||||||
### Proxy (v1)
|
|
||||||
|
|
||||||
```
|
|
||||||
GET /api/v1/remote/{name}/{path} Proxy/cache artifact
|
|
||||||
GET /api/v1/virtual/{name}/{path} Virtual repo (merged index)
|
|
||||||
PUT /api/v1/local/{name}/{package} npm publish (local npm repos)
|
|
||||||
GET /v2/{name}/{path} Docker Registry v2
|
|
||||||
```
|
|
||||||
|
|
||||||
### Management (v2)
|
|
||||||
|
|
||||||
```
|
|
||||||
GET/POST /api/v2/remotes List / create remotes
|
|
||||||
GET/PUT/DELETE /api/v2/remotes/{name} Read / update / delete remote
|
|
||||||
GET/DELETE /api/v2/remotes/{name}/objects Browse / evict cached objects
|
|
||||||
GET /api/v2/stats Overview stats
|
|
||||||
GET /api/v2/health Service health
|
|
||||||
POST /api/v2/probe Test a remote (fetch without streaming to client)
|
|
||||||
GET /api/v2/events SSE event stream
|
|
||||||
```
|
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
PostgreSQL ─── config (remotes, virtuals), artifact metadata, access log
|
client → /api/v1/remote/{remote}/{path}
|
||||||
Redis ─── TTL keys, fetch locks, circuit breaker state
|
↓
|
||||||
S3/MinIO ─── content-addressable blob storage (blobs/sha256/{hash})
|
Redis: mutable TTL check
|
||||||
|
↓ miss / expired
|
||||||
|
S3: object exists?
|
||||||
|
↓ no
|
||||||
|
upstream remote → S3 + PostgreSQL metadata
|
||||||
|
↓
|
||||||
|
response (X-Artifact-Source: cache|remote)
|
||||||
```
|
```
|
||||||
|
|
||||||
S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go).
|
Docker Registry traffic uses the `/v2/{remote}/{path}` endpoint implementing the Docker Registry HTTP API v2.
|
||||||
|
|
||||||
### Schema migrations
|
### Code layout
|
||||||
|
|
||||||
The SQL schema lives in `migrations/` as numbered `.sql` files, embedded into
|
```
|
||||||
the binary and applied at startup before the server listens, so there is no
|
src/artifactapi/
|
||||||
mirrored copy of the schema in the deployment to drift out of sync.
|
├── main.py — FastAPI app + thin route declarations only
|
||||||
|
├── config.py — ConfigManager (loads remotes.yaml)
|
||||||
|
├── metrics.py — Prometheus + Redis metrics
|
||||||
|
├── docker_auth.py — backwards-compat shim → auth/docker.py
|
||||||
|
├── artifact/ — route handler implementations
|
||||||
|
│ ├── proxy.py — GET /api/v1/remote (remote proxy, cache, revalidation)
|
||||||
|
│ ├── virtual.py — GET /api/v1/virtual (virtual repo index merging)
|
||||||
|
│ ├── local.py — PUT/HEAD/DELETE /api/v1/remote (local repos)
|
||||||
|
│ ├── docker.py — /v2/ Docker Registry v2 proxy
|
||||||
|
│ ├── discovery.py — /api/v1/artifacts discovery + bulk cache
|
||||||
|
│ └── flush.py — PUT /cache/flush
|
||||||
|
├── auth/
|
||||||
|
│ ├── __init__.py — re-exports Docker auth helpers
|
||||||
|
│ └── docker.py — Bearer token fetching + in-memory cache
|
||||||
|
├── cache/
|
||||||
|
│ ├── __init__.py — re-exports RedisCache
|
||||||
|
│ └── redis.py — RedisCache (TTL keys, ETag metadata)
|
||||||
|
├── database/
|
||||||
|
│ ├── __init__.py — re-exports DatabaseManager
|
||||||
|
│ └── postgres.py — DatabaseManager (artifact + local-file tables)
|
||||||
|
├── storage/
|
||||||
|
│ ├── __init__.py — re-exports S3Storage
|
||||||
|
│ └── s3.py — S3Storage (MinIO/S3 abstraction)
|
||||||
|
└── remote/
|
||||||
|
├── __init__.py
|
||||||
|
├── base.py — content-type detection
|
||||||
|
├── generic.py — generic HTTP remotes
|
||||||
|
├── helm.py — Helm index.yaml URL rewriting
|
||||||
|
├── npm.py — npm metadata URL rewriting
|
||||||
|
├── python.py — PyPI URL construction + HTML rewriting
|
||||||
|
└── rpm.py — RPM remotes
|
||||||
|
```
|
||||||
|
|
||||||
The runner is [`golib/pg`](https://git.unkin.net/unkin/golib)'s `pg.NewMigrated`
|
## API Endpoints
|
||||||
— artifactapi owns the SQL, the shared library owns the mechanics.
|
|
||||||
|
|
||||||
Each start takes `pg_advisory_lock` on a fixed key (FNV-1a/64 of the lock name
|
| Method | Path | Description |
|
||||||
`artifactapi-migrations`), creates `schema_migrations` (`version`, `applied_at`)
|
|
||||||
if missing, and applies every embedded file whose filename is not yet recorded —
|
|
||||||
in lexical (version) order, each file's SQL and its tracking row in one
|
|
||||||
transaction — then unlocks. Replicas starting together queue on the lock and
|
|
||||||
then find nothing to do.
|
|
||||||
|
|
||||||
A file absent from `schema_migrations` is re-run even when the database already
|
|
||||||
has the schema, which is how a database migrated by the old untracked inline DDL
|
|
||||||
picks `0001_init.sql` up: the statements are `IF NOT EXISTS`-guarded, so the
|
|
||||||
re-run is a no-op that only lands the tracking row. New migrations must stay
|
|
||||||
additive and idempotent for the same reason; a test enforces it.
|
|
||||||
|
|
||||||
## Environment Variables
|
|
||||||
|
|
||||||
| Variable | Default | Description |
|
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `LISTEN_ADDR` | `:8000` | Server listen address |
|
| `GET` | `/api/v1/remote/{remote}/{path}` | Fetch artifact (auto-cache on miss) |
|
||||||
| `DBHOST` | `localhost` | PostgreSQL host |
|
| `PUT` | `/api/v1/remote/{remote}/{path}` | Upload to local remote |
|
||||||
| `DBPORT` | `5432` | PostgreSQL port |
|
| `HEAD` | `/api/v1/remote/{remote}/{path}` | Check existence (local remotes) |
|
||||||
| `DBUSER` | `artifacts` | PostgreSQL user |
|
| `DELETE` | `/api/v1/remote/{remote}/{path}` | Delete from local remote |
|
||||||
| `DBPASS` | | PostgreSQL password |
|
| `GET` | `/api/v1/virtual/{virtual}/{path}` | Fetch from virtual (merged) repository |
|
||||||
| `DBNAME` | `artifacts` | PostgreSQL database |
|
| `GET` | `/v2/{remote}/{path}` | Docker Registry v2 proxy |
|
||||||
| `DBSSL` | `disable` | PostgreSQL `sslmode` |
|
| `PUT` | `/cache/flush` | Flush cache entries |
|
||||||
| `REDIS_URL` | `redis://localhost:6379` | Redis URL |
|
| `GET` | `/health` | Health check |
|
||||||
| `MINIO_ENDPOINT` | `localhost:9000` | S3 endpoint |
|
| `GET` | `/config` | View loaded configuration |
|
||||||
| `MINIO_ACCESS_KEY` | | S3 access key |
|
| `GET` | `/` | API info and available remotes |
|
||||||
| `MINIO_SECRET_KEY` | | S3 secret key |
|
|
||||||
| `MINIO_BUCKET` | `artifacts` | S3 bucket |
|
|
||||||
| `MINIO_SECURE` | `false` | Use HTTPS for S3 |
|
|
||||||
| `MINIO_REGION` | | S3 region (AWS) |
|
|
||||||
| `GITHUB_SYNC_RATE` | `1` | `github_rpm` syncer global GitHub request rate (req/s), shared across all remotes. `1`/s = 3600/hr, under an authenticated token's ~5000/hr; unauthenticated (~60/hr) relies on ETag/304 |
|
|
||||||
| `GITHUB_SYNC_BURST` | `5` | Token-bucket burst for the shared limiter |
|
|
||||||
| `GITHUB_SYNC_WORKERS` | `3` | Concurrent `github_rpm` scan workers |
|
|
||||||
| `GITHUB_SYNC_POLL_INTERVAL` | `60` | Base scheduler tick in seconds; per-remote cadence is its `mutable_ttl`, enforced by the DB lease |
|
|
||||||
| `GITHUB_TOKEN` | | Server-level GitHub PAT (fine-grained or classic), sent as `Authorization: Bearer`. Applies to every GitHub request; per-remote creds override it. See [GitHub authentication](#github-authentication) |
|
|
||||||
| `GITHUB_APP_ID` | | GitHub App id (App auth mode; mutually exclusive with `GITHUB_TOKEN`) |
|
|
||||||
| `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id |
|
|
||||||
| `GITHUB_APP_PRIVATE_KEY` | | GitHub App private key, inline PEM |
|
|
||||||
| `GITHUB_APP_PRIVATE_KEY_PATH` | | GitHub App private key, file path (alternative to inline PEM) |
|
|
||||||
| `ARTIFACTAPI_OIDC_ISSUER` | | OIDC issuer URL (e.g. Vault `https://vault.service.consul:8200/v1/identity/oidc`). When set, `POST /api/v2/remotes`, `POST /api/v2/virtuals` and `PUT`/`DELETE` on `/api/v2/{remotes,virtuals}/{name}` require `Authorization: Bearer <jwt>` verified against the issuer's JWKS. Uploads, object/file deletes, probe, reads and `/v2` stay open. Unset = no auth |
|
|
||||||
| `ARTIFACTAPI_OIDC_AUDIENCE` | | Comma-separated allowed `aud` values (e.g. the Vault OIDC role's client_id). Required when `ARTIFACTAPI_OIDC_ISSUER` is set |
|
|
||||||
|
|
||||||
## Development
|
## Configuration
|
||||||
|
|
||||||
|
Runtime settings come from environment variables; remote definitions live in one or more YAML files pointed to by `CONFIG_PATH`.
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
|
||||||
|
| Variable | Description |
|
||||||
|
|---|---|
|
||||||
|
| `CONFIG_PATH` | Path to a config YAML file **or** a directory of YAML files |
|
||||||
|
| `DBHOST`, `DBPORT`, `DBUSER`, `DBPASS`, `DBNAME` | PostgreSQL connection |
|
||||||
|
| `REDIS_URL` | Redis URL (e.g. `redis://localhost:6379`) |
|
||||||
|
| `MINIO_ENDPOINT` | MinIO/S3 endpoint |
|
||||||
|
| `MINIO_ACCESS_KEY` | S3 access key |
|
||||||
|
| `MINIO_SECRET_KEY` | S3 secret key |
|
||||||
|
| `MINIO_BUCKET` | S3 bucket name |
|
||||||
|
| `MINIO_SECURE` | Use HTTPS (`true`/`false`) |
|
||||||
|
|
||||||
|
### Split configuration
|
||||||
|
|
||||||
|
`CONFIG_PATH` accepts three forms:
|
||||||
|
|
||||||
|
**Single file** (original behaviour):
|
||||||
|
```
|
||||||
|
CONFIG_PATH=/etc/artifactapi/remotes.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
**Directory** — all `*.yaml` / `*.yml` files in the directory are loaded and merged alphabetically. `remotes` keys are merged across files; later files win on conflict:
|
||||||
|
```
|
||||||
|
CONFIG_PATH=/etc/artifactapi/conf.d/
|
||||||
|
```
|
||||||
|
|
||||||
|
**Main file + `config_dir`** — the main file holds global settings and a `config_dir` pointer; each file in that directory contributes its own `remotes`. Relative `config_dir` paths are resolved relative to the main file:
|
||||||
|
```yaml
|
||||||
|
# /etc/artifactapi/config.yaml
|
||||||
|
config_dir: conf.d # or an absolute path
|
||||||
|
|
||||||
|
# s3/redis/database settings go here (or in env vars)
|
||||||
|
remotes: {} # optional base remotes
|
||||||
|
```
|
||||||
|
|
||||||
|
### remotes.yaml Structure
|
||||||
|
|
||||||
|
The top-level key declares the repository type — no `type:` field needed:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
remote-name:
|
||||||
|
base_url: "https://example.com"
|
||||||
|
package: "generic" # generic, alpine, rpm, docker, pypi, npm, helm
|
||||||
|
description: "..."
|
||||||
|
immutable_patterns: # regex — cached forever
|
||||||
|
- ".*\\.tar\\.gz$"
|
||||||
|
mutable_patterns: # regex — expire after mutable_ttl
|
||||||
|
- "index\\.yaml$"
|
||||||
|
check_mutable_updates: false # send HEAD (If-None-Match) on TTL expiry
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0 # 0 = indefinitely
|
||||||
|
mutable_ttl: 3600
|
||||||
|
|
||||||
|
virtual:
|
||||||
|
virtual-name:
|
||||||
|
package: "helm"
|
||||||
|
members:
|
||||||
|
- remote-name-1
|
||||||
|
- remote-name-2
|
||||||
|
|
||||||
|
local:
|
||||||
|
local-name:
|
||||||
|
package: "generic"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Remote Types
|
||||||
|
|
||||||
|
### generic
|
||||||
|
|
||||||
|
Arbitrary HTTP file servers — GitHub releases, HashiCorp, custom servers.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
github:
|
||||||
|
base_url: "https://github.com"
|
||||||
|
package: "generic"
|
||||||
|
immutable_patterns:
|
||||||
|
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
|
||||||
|
github-archive:
|
||||||
|
base_url: "https://github.com"
|
||||||
|
package: "generic"
|
||||||
|
immutable_patterns:
|
||||||
|
- ".*/archive/refs/tags/.*\\.tar\\.gz$" # tag archives never change
|
||||||
|
mutable_patterns:
|
||||||
|
- ".*/archive/refs/heads/main\\.tar\\.gz$" # branch archives can change
|
||||||
|
check_mutable_updates: true
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 86400
|
||||||
|
```
|
||||||
|
|
||||||
|
Access: `GET /api/v1/remote/github/owner/repo/releases/download/v1.0/binary.tar.gz`
|
||||||
|
|
||||||
|
### alpine
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
alpine:
|
||||||
|
base_url: "https://dl-cdn.alpinelinux.org"
|
||||||
|
package: "alpine"
|
||||||
|
immutable_patterns:
|
||||||
|
- ".*/x86_64/.*\\.apk$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 7200
|
||||||
|
```
|
||||||
|
|
||||||
|
`APKINDEX.tar.gz` is a built-in mutable pattern — no `mutable_patterns` entry needed.
|
||||||
|
|
||||||
|
### rpm
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
almalinux:
|
||||||
|
base_url: "https://mirror.example.com/almalinux"
|
||||||
|
package: "rpm"
|
||||||
|
immutable_patterns:
|
||||||
|
- ".*/x86_64/.*\\.rpm$"
|
||||||
|
- ".*/noarch/.*\\.rpm$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 7200
|
||||||
|
```
|
||||||
|
|
||||||
|
`repomd.xml` and `repodata/` metadata files are built-in mutable patterns.
|
||||||
|
|
||||||
|
### docker
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
dockerhub:
|
||||||
|
base_url: "https://registry-1.docker.io"
|
||||||
|
package: "docker"
|
||||||
|
# username / password optional for public images
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 300
|
||||||
|
|
||||||
|
ghcr:
|
||||||
|
base_url: "https://ghcr.io"
|
||||||
|
package: "docker"
|
||||||
|
username: "your-github-username"
|
||||||
|
password: "ghp_your_pat" # read:packages scope
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 300
|
||||||
|
```
|
||||||
|
|
||||||
|
Tag manifests and `/tags/list` are built-in mutable patterns. Digest-addressed blobs are immutable.
|
||||||
|
|
||||||
|
For RKE2/containerd, configure `/etc/rancher/rke2/registries.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mirrors:
|
||||||
|
docker.io:
|
||||||
|
endpoint:
|
||||||
|
- "https://artifacts.example.com"
|
||||||
|
rewrite:
|
||||||
|
"^(.*)$": "dockerhub/$1"
|
||||||
|
ghcr.io:
|
||||||
|
endpoint:
|
||||||
|
- "https://artifacts.example.com"
|
||||||
|
rewrite:
|
||||||
|
"^(.*)$": "ghcr/$1"
|
||||||
|
```
|
||||||
|
|
||||||
|
### pypi
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
pypi:
|
||||||
|
base_url: "https://files.pythonhosted.org"
|
||||||
|
package: "pypi"
|
||||||
|
check_mutable_updates: true
|
||||||
|
immutable_patterns:
|
||||||
|
- "packages/.*\\.whl$"
|
||||||
|
- "packages/.*\\.whl\\.metadata$"
|
||||||
|
- "packages/.*\\.tar\\.gz$"
|
||||||
|
- "packages/.*\\.zip$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 600
|
||||||
|
```
|
||||||
|
|
||||||
|
> **Note**: Simple index requests (`/simple/{package}/`) are always fetched from `https://pypi.org`, regardless of `base_url`. This is hardcoded — `base_url` only controls where package files are downloaded from. For self-hosted registries (Gitea, Nexus) where both index and files share the same host, set `base_url` to that host and the override does not apply.
|
||||||
|
|
||||||
|
URLs in simple index HTML are rewritten to route package file downloads back through the same remote.
|
||||||
|
|
||||||
|
Configure uv:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
# /etc/uv/uv.toml or ~/.config/uv/uv.toml
|
||||||
|
[[index]]
|
||||||
|
url = "https://artifacts.example.com/api/v1/remote/pypi/simple"
|
||||||
|
default = true
|
||||||
|
```
|
||||||
|
|
||||||
|
### npm
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
npm:
|
||||||
|
base_url: "https://registry.npmjs.org"
|
||||||
|
package: "npm"
|
||||||
|
check_mutable_updates: true
|
||||||
|
immutable_patterns:
|
||||||
|
- "\.tgz$"
|
||||||
|
mutable_patterns:
|
||||||
|
- "^(?!.*\.tgz$).*"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 600
|
||||||
|
```
|
||||||
|
|
||||||
|
`dist.tarball` URLs in package metadata JSON are rewritten to route tarball downloads back through the same remote.
|
||||||
|
|
||||||
|
Configure npm / yarn / pnpm:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# .npmrc or ~/.npmrc
|
||||||
|
registry=https://artifacts.example.com/api/v1/remote/npm/
|
||||||
|
```
|
||||||
|
|
||||||
|
### helm
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
hashicorp-helm:
|
||||||
|
base_url: "https://helm.releases.hashicorp.com"
|
||||||
|
package: "helm"
|
||||||
|
check_mutable_updates: true
|
||||||
|
immutable_patterns:
|
||||||
|
- "\\.tgz$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 3600
|
||||||
|
```
|
||||||
|
|
||||||
|
`index.yaml` is a built-in mutable pattern. Chart URLs inside `index.yaml` are rewritten to route tarball downloads back through the same remote.
|
||||||
|
|
||||||
|
Configure Helm:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make build # Build binary
|
helm repo add hashicorp https://artifacts.example.com/api/v1/remote/hashicorp-helm
|
||||||
make test # Unit tests
|
helm repo update
|
||||||
make e2e # E2E tests (needs Docker)
|
|
||||||
make lint # golangci-lint + go vet
|
|
||||||
make fmt # gofmt + goimports
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### `GOPRIVATE`
|
### virtual
|
||||||
|
|
||||||
artifactapi depends on `git.unkin.net/unkin/golib`, which is served by Gitea and
|
A virtual repository presents a single unified index built from multiple member remotes of the same package type. Clients configure one endpoint and get access to all member remotes transparently.
|
||||||
is unknown to `proxy.golang.org` / `sum.golang.org`. Module resolution therefore
|
|
||||||
needs:
|
|
||||||
|
|
||||||
```
|
All members must share the same `package` type as the virtual repo. Currently supported package types: `helm`.
|
||||||
export GOPRIVATE=git.unkin.net
|
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
helm-hashicorp:
|
||||||
|
base_url: "https://helm.releases.hashicorp.com"
|
||||||
|
package: "helm"
|
||||||
|
immutable_patterns:
|
||||||
|
- "\\.tgz$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 3600
|
||||||
|
|
||||||
|
helm-bitnami:
|
||||||
|
base_url: "https://charts.bitnami.com/bitnami"
|
||||||
|
package: "helm"
|
||||||
|
immutable_patterns:
|
||||||
|
- "\\.tgz$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 3600
|
||||||
|
|
||||||
|
virtual:
|
||||||
|
helm-all:
|
||||||
|
package: "helm"
|
||||||
|
members:
|
||||||
|
- helm-hashicorp # listed first = highest priority
|
||||||
|
- helm-bitnami
|
||||||
```
|
```
|
||||||
|
|
||||||
The `Makefile` exports it for every target, and the `Dockerfile` and the
|
**How it works:**
|
||||||
woodpecker Go steps set it themselves, so `make build|test|lint` and CI work on
|
|
||||||
a clean checkout. Only bare `go` commands run outside `make` need it in your
|
|
||||||
shell — set it there (or in your shell profile) rather than with `go env -w`,
|
|
||||||
which is machine state this repo cannot carry.
|
|
||||||
|
|
||||||
### TUI
|
1. A request for the package index triggers a parallel fetch of each member's index from S3 cache, falling back to upstream if not yet cached.
|
||||||
|
2. Member indexes are merged into a single index with URL rewriting so artifact download URLs continue to resolve through the individual member remote.
|
||||||
|
3. The merged index is cached in Redis with a TTL equal to the minimum `mutable_ttl` across all members.
|
||||||
|
|
||||||
|
**Priority / conflict resolution:**
|
||||||
|
|
||||||
|
When the same artifact name and version appears in more than one member, the member listed **first** in `members` wins. Subsequent members contribute only artifacts not already present.
|
||||||
|
|
||||||
|
**Partial failures:**
|
||||||
|
|
||||||
|
If a member is unreachable and has no cached index, it is skipped and a warning is logged. The merged index is still served from available members. If *no* members can be reached, the request returns `502`.
|
||||||
|
|
||||||
|
**Caching:**
|
||||||
|
|
||||||
|
The merged index is cached using `min(mutable_ttl)` across all members. Each member's raw index is cached in S3 under its own remote key by the normal proxy rules; the virtual handler reuses those copies when available.
|
||||||
|
|
||||||
|
**Helm example:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./bin/artifactapi tui --endpoint http://localhost:8000
|
helm repo add all https://artifacts.example.com/api/v1/virtual/helm-all
|
||||||
|
helm repo update
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Chart tarball URLs in the merged `index.yaml` are rewritten to point at the individual member remote (e.g. `…/api/v1/remote/helm-hashicorp/vault-0.27.0.tgz`), so downloads bypass the virtual endpoint entirely.
|
||||||
|
|
||||||
|
### local
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
local:
|
||||||
|
local-generic:
|
||||||
|
package: "generic"
|
||||||
|
description: "Local file repository"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 0
|
||||||
|
```
|
||||||
|
|
||||||
|
No `base_url`. Files are uploaded via `PUT` and served via `GET`.
|
||||||
|
|
||||||
|
## Migration
|
||||||
|
|
||||||
|
### Splitting a single remotes file into per-type files
|
||||||
|
|
||||||
|
The old format used a single `remotes:` map with an explicit `type:` field on each entry. The new format uses top-level type keys (`remote:`, `virtual:`, `local:`) and supports splitting across multiple files via `config_dir`.
|
||||||
|
|
||||||
|
**Before** (`remotes.yaml`):
|
||||||
|
```yaml
|
||||||
|
remotes:
|
||||||
|
dockerhub:
|
||||||
|
base_url: "https://registry-1.docker.io"
|
||||||
|
type: "remote"
|
||||||
|
package: "docker"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 300
|
||||||
|
|
||||||
|
hashicorp-helm:
|
||||||
|
base_url: "https://helm.releases.hashicorp.com"
|
||||||
|
type: "remote"
|
||||||
|
package: "helm"
|
||||||
|
immutable_patterns:
|
||||||
|
- "\\.tgz$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 3600
|
||||||
|
|
||||||
|
helm-all:
|
||||||
|
type: "virtual"
|
||||||
|
package: "helm"
|
||||||
|
members:
|
||||||
|
- hashicorp-helm
|
||||||
|
|
||||||
|
local-files:
|
||||||
|
type: "local"
|
||||||
|
package: "generic"
|
||||||
|
```
|
||||||
|
|
||||||
|
**After** — one file per type + package type, with a main config pointing at the directory:
|
||||||
|
|
||||||
|
`config.yaml`:
|
||||||
|
```yaml
|
||||||
|
config_dir: conf.d
|
||||||
|
```
|
||||||
|
|
||||||
|
`conf.d/remote-docker.yaml`:
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
dockerhub:
|
||||||
|
base_url: "https://registry-1.docker.io"
|
||||||
|
package: "docker"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 300
|
||||||
|
```
|
||||||
|
|
||||||
|
`conf.d/remote-helm.yaml`:
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
hashicorp-helm:
|
||||||
|
base_url: "https://helm.releases.hashicorp.com"
|
||||||
|
package: "helm"
|
||||||
|
immutable_patterns:
|
||||||
|
- "\\.tgz$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 3600
|
||||||
|
```
|
||||||
|
|
||||||
|
`conf.d/virtual-helm.yaml`:
|
||||||
|
```yaml
|
||||||
|
virtual:
|
||||||
|
helm-all:
|
||||||
|
package: "helm"
|
||||||
|
members:
|
||||||
|
- hashicorp-helm
|
||||||
|
```
|
||||||
|
|
||||||
|
`conf.d/local-generic.yaml`:
|
||||||
|
```yaml
|
||||||
|
local:
|
||||||
|
local-files:
|
||||||
|
package: "generic"
|
||||||
|
```
|
||||||
|
|
||||||
|
Set `CONFIG_PATH` to the main file:
|
||||||
|
```
|
||||||
|
CONFIG_PATH=/etc/artifactapi/config.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
Files in `conf.d/` are merged alphabetically; later files win on conflicts within the same remote name.
|
||||||
|
|
||||||
|
## Caching Model
|
||||||
|
|
||||||
|
### Immutable patterns
|
||||||
|
|
||||||
|
Files matching `immutable_patterns` are cached for `immutable_ttl` seconds (0 = indefinitely). Use for versioned release artifacts that never change once published.
|
||||||
|
|
||||||
|
**Access control**: only paths matching an immutable or mutable pattern are served; all others return 403. Omitting `immutable_patterns` entirely allows all paths from that remote.
|
||||||
|
|
||||||
|
### Mutable patterns
|
||||||
|
|
||||||
|
Files matching `mutable_patterns` expire after `mutable_ttl` seconds and are re-fetched on the next request. Mutable files are always served regardless of `immutable_patterns`.
|
||||||
|
|
||||||
|
Each package type has built-in defaults that are merged with any user-defined `mutable_patterns`:
|
||||||
|
|
||||||
|
| Package type | Built-in mutable patterns |
|
||||||
|
|---|---|
|
||||||
|
| `alpine` | `APKINDEX\.tar\.gz$` |
|
||||||
|
| `rpm` | `repomd\.xml$`, `repodata/` metadata variants, `Packages\.gz$` |
|
||||||
|
| `docker` | Tag manifests (non-digest refs), `/tags/list` |
|
||||||
|
| `pypi` | `simple/` (per-package and top-level index pages) |
|
||||||
|
| `helm` | `index\.yaml$` |
|
||||||
|
| `npm` | *(none built-in — define via `mutable_patterns`)* |
|
||||||
|
| `generic` | *(none)* |
|
||||||
|
|
||||||
|
### Conditional revalidation
|
||||||
|
|
||||||
|
Set `check_mutable_updates: true` to send `HEAD` with `If-None-Match` / `If-Modified-Since` on TTL expiry. A 304 response refreshes the TTL without re-downloading. Only applies to user-defined `mutable_patterns` — built-in patterns are always re-fetched unconditionally.
|
||||||
|
|
||||||
|
### Stale-on-upstream-error
|
||||||
|
|
||||||
|
When a mutable file expires and the upstream is unreachable (connection refused, DNS failure, timeout), the cached copy is kept and its TTL refreshed. HTTP error responses (4xx, 5xx) are not treated as network failures and proceed with normal expiry.
|
||||||
|
|
||||||
|
### Quarantine (supply-chain protection)
|
||||||
|
|
||||||
|
Set `quarantine_new: true` and `quarantine_days: N` on a remote to block immutable artifacts published within the last N days. Requests return `404` until the quarantine period expires, giving time to detect malicious packages before they are consumed.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
remote:
|
||||||
|
pypi:
|
||||||
|
base_url: "https://files.pythonhosted.org"
|
||||||
|
package: "pypi"
|
||||||
|
quarantine_new: true
|
||||||
|
quarantine_days: 3 # block packages published in the last 3 days
|
||||||
|
immutable_patterns:
|
||||||
|
- "packages/.*\\.whl$"
|
||||||
|
- "packages/.*\\.tar\\.gz$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 600
|
||||||
|
```
|
||||||
|
|
||||||
|
The upstream `Last-Modified` response header is used as the publish date proxy. Artifacts that have no `Last-Modified` header are allowed through (fail-open). Mutable files (index pages, tag manifests) are never quarantined.
|
||||||
|
|||||||
@@ -0,0 +1,137 @@
|
|||||||
|
# ArtifactAPI Specification
|
||||||
|
|
||||||
|
## Repository model
|
||||||
|
|
||||||
|
Every repository entry in `remotes.yaml` has two orthogonal fields:
|
||||||
|
|
||||||
|
| field | values | meaning |
|
||||||
|
|---|---|---|
|
||||||
|
| `type` | `local`, `remote`, `virtual` | repository kind — how the repo is served |
|
||||||
|
| `package` | `docker`, `rpm`, `alpine`, `generic` | package format — what protocol and caching rules to apply |
|
||||||
|
|
||||||
|
**type**
|
||||||
|
|
||||||
|
- `local` — files are uploaded directly to the API and stored in S3; no upstream.
|
||||||
|
- `remote` — proxies and caches content from an upstream URL (`base_url`).
|
||||||
|
- `virtual` — aggregates multiple repositories (not yet implemented).
|
||||||
|
|
||||||
|
**package**
|
||||||
|
|
||||||
|
- `docker` — upstream speaks the OCI Distribution API (Bearer auth, manifest/blob paths).
|
||||||
|
- `rpm` — upstream is an RPM repository; repodata files are index files.
|
||||||
|
- `alpine` — upstream is an Alpine APK repository; `APKINDEX.tar.gz` is an index file.
|
||||||
|
- `generic` — plain HTTP file download; no format-specific logic.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Caching
|
||||||
|
|
||||||
|
Two cache classes determine retention:
|
||||||
|
|
||||||
|
| class | stored | TTL |
|
||||||
|
|---|---|---|
|
||||||
|
| **file** | S3 object, no Redis entry | `file_ttl` — `0` means indefinite |
|
||||||
|
| **index** | S3 object + Redis TTL key | `index_ttl` — when the Redis key expires the S3 object is deleted and re-fetched |
|
||||||
|
|
||||||
|
Index files are mutable metadata that must expire. File-class objects are treated as immutable and cached indefinitely (unless `file_ttl` is set).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Docker package rules
|
||||||
|
|
||||||
|
### URL construction
|
||||||
|
|
||||||
|
Remote URLs are prefixed with `/v2/` for `package: docker` remotes:
|
||||||
|
|
||||||
|
```
|
||||||
|
{base_url}/v2/{path}
|
||||||
|
```
|
||||||
|
|
||||||
|
e.g. `library/nginx/manifests/latest` → `https://registry-1.docker.io/v2/library/nginx/manifests/latest`
|
||||||
|
|
||||||
|
### Authentication
|
||||||
|
|
||||||
|
Docker registries use Bearer token challenges. On a `401 Unauthorized` response, the API:
|
||||||
|
|
||||||
|
1. Parses the `WWW-Authenticate: Bearer` header for `realm`, `service`, and `scope`.
|
||||||
|
2. Fetches a token from the auth realm, supplying `username`/`password` from the remote config if present.
|
||||||
|
3. Retries the request with `Authorization: Bearer <token>`.
|
||||||
|
|
||||||
|
Tokens are cached in-memory keyed by `(realm, service, scope, username)` and expire 30 seconds before their stated `expires_in`.
|
||||||
|
|
||||||
|
### Cache classification
|
||||||
|
|
||||||
|
| path pattern | mutable | class | TTL source |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `/manifests/<tag>` | yes | index | `index_ttl` |
|
||||||
|
| `/tags/list` | yes | index | `index_ttl` |
|
||||||
|
| `/manifests/sha256:<digest>` | no | file | `file_ttl` |
|
||||||
|
| `/blobs/sha256:<digest>` | no | file | `file_ttl` |
|
||||||
|
|
||||||
|
Tag-based manifests and tag lists are mutable and cached as index. Digest-pinned manifests and blobs are content-addressed and cached indefinitely as files.
|
||||||
|
|
||||||
|
### Blob deduplication
|
||||||
|
|
||||||
|
Blobs are stored under a digest-keyed path shared across all images on the same remote:
|
||||||
|
|
||||||
|
```
|
||||||
|
{remote_name}/blobs/sha256/{digest}
|
||||||
|
```
|
||||||
|
|
||||||
|
The same layer pulled by different images is stored once.
|
||||||
|
|
||||||
|
### Accept headers
|
||||||
|
|
||||||
|
| path | `Accept` header sent upstream |
|
||||||
|
|---|---|
|
||||||
|
| `/manifests/…` | `application/vnd.docker.distribution.manifest.v2+json`, `application/vnd.oci.image.manifest.v1+json`, `application/vnd.oci.image.index.v1+json`, `application/vnd.docker.distribution.manifest.list.v2+json` |
|
||||||
|
| `/blobs/…` | `application/octet-stream` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## OCI Distribution API endpoint
|
||||||
|
|
||||||
|
The API exposes a native Docker registry interface so clients can use `docker pull` directly:
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /v2/ — version ping
|
||||||
|
GET /v2/{remote}/{image}/manifests/{ref} — fetch manifest
|
||||||
|
HEAD /v2/{remote}/{image}/manifests/{ref} — manifest metadata
|
||||||
|
GET /v2/{remote}/{image}/blobs/{digest} — fetch blob
|
||||||
|
HEAD /v2/{remote}/{image}/blobs/{digest} — blob metadata
|
||||||
|
```
|
||||||
|
|
||||||
|
Responses include `Docker-Distribution-Api-Version`, `Docker-Content-Digest`, and the correct OCI `Content-Type` (detected from the manifest `mediaType` field).
|
||||||
|
|
||||||
|
Only remotes with `package: docker` are accessible via this endpoint. All other remotes return `400`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## include_patterns
|
||||||
|
|
||||||
|
`include_patterns` is a list of Python regexes applied to every request before any upstream fetch or cache lookup.
|
||||||
|
|
||||||
|
**Generic remotes (`/api/v1/remote/…`):**
|
||||||
|
- Patterns match against the file path and the full path.
|
||||||
|
- Index files (mutable metadata) bypass pattern checks and are always allowed.
|
||||||
|
|
||||||
|
**Docker remotes (`/v2/…`):**
|
||||||
|
- Patterns match against the image name (first two path segments, e.g. `library/nginx`) and the full path.
|
||||||
|
- The index-file exemption does **not** apply — patterns restrict whole images, including their manifests and tag lists.
|
||||||
|
- No patterns configured → all images allowed.
|
||||||
|
|
||||||
|
Returns `403` when a request is blocked.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Versioning
|
||||||
|
|
||||||
|
The package version is derived from git tags via `hatch-vcs`. Tags follow the format `v{MAJOR}.{MINOR}.{PATCH}`.
|
||||||
|
|
||||||
|
Docker images are built with the version injected at build time:
|
||||||
|
|
||||||
|
```
|
||||||
|
SETUPTOOLS_SCM_PRETEND_VERSION=<version> uv sync --frozen
|
||||||
|
```
|
||||||
|
|
||||||
|
The `Makefile` provides `patch`, `minor`, and `major` targets that tag the current commit and rebuild the container image.
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"os"
|
|
||||||
"os/signal"
|
|
||||||
"syscall"
|
|
||||||
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/server"
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/tui"
|
|
||||||
)
|
|
||||||
|
|
||||||
var version = "dev"
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
if len(os.Args) > 1 && os.Args[1] == "tui" {
|
|
||||||
endpoint := os.Getenv("ARTIFACTAPI_ENDPOINT")
|
|
||||||
if endpoint == "" {
|
|
||||||
endpoint = "http://localhost:8000"
|
|
||||||
}
|
|
||||||
for i, arg := range os.Args {
|
|
||||||
if arg == "--endpoint" && i+1 < len(os.Args) {
|
|
||||||
endpoint = os.Args[i+1]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
app := tui.New(endpoint)
|
|
||||||
if err := app.Run(); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "tui error: %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.Load()
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("failed to load config", "error", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
srv, err := server.New(cfg, version)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("failed to create server", "error", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := srv.Run(ctx); err != nil {
|
|
||||||
slog.Error("server exited with error", "error", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
remotes:
|
||||||
|
alpine:
|
||||||
|
base_url: "https://dl-cdn.alpinelinux.org"
|
||||||
|
type: "remote"
|
||||||
|
package: "alpine"
|
||||||
|
description: "Alpine Linux APK package repository"
|
||||||
|
immutable_patterns:
|
||||||
|
- ".*/x86_64/.*\\.apk$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 7200
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
remotes:
|
||||||
|
github:
|
||||||
|
base_url: "https://github.com"
|
||||||
|
type: "remote"
|
||||||
|
package: "generic"
|
||||||
|
description: "GitHub releases and files"
|
||||||
|
immutable_patterns:
|
||||||
|
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||||
|
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 0
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
remotes:
|
||||||
|
pypi:
|
||||||
|
base_url: "https://files.pythonhosted.org"
|
||||||
|
type: "remote"
|
||||||
|
package: "pypi"
|
||||||
|
description: "Python Package Index"
|
||||||
|
check_mutable_updates: true
|
||||||
|
quarantine_new: true
|
||||||
|
quarantine_days: 3
|
||||||
|
immutable_patterns:
|
||||||
|
- "packages/.*\\.whl$"
|
||||||
|
- "packages/.*\\.whl\\.metadata$"
|
||||||
|
- "packages/.*\\.tar\\.gz$"
|
||||||
|
- "packages/.*\\.zip$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 600
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
# Overlay for the dockerised end-to-end suite (scripts/docker-e2e.sh).
|
|
||||||
# Adds a static mock upstream that the artifactapi container proxies, so the
|
|
||||||
# caching tests are hermetic and need no internet access.
|
|
||||||
services:
|
|
||||||
mockupstream:
|
|
||||||
image: nginx:alpine
|
|
||||||
volumes:
|
|
||||||
- ./e2e-docker/fixtures:/usr/share/nginx/html:ro,z
|
|
||||||
# No host port needed: only the artifactapi container talks to it, and the
|
|
||||||
# tests compare served bytes against the on-disk fixtures.
|
|
||||||
|
|
||||||
# Two constant-body upstreams for the multi-base_url suite: each returns a
|
|
||||||
# distinct, upstream-identifying body for any path, so round-robin
|
|
||||||
# distribution across a two-mirror remote is directly observable.
|
|
||||||
mockupstreama:
|
|
||||||
image: nginx:alpine
|
|
||||||
volumes:
|
|
||||||
- ./e2e-docker/mirror-conf/a.conf:/etc/nginx/conf.d/default.conf:ro,z
|
|
||||||
mockupstreamb:
|
|
||||||
image: nginx:alpine
|
|
||||||
volumes:
|
|
||||||
- ./e2e-docker/mirror-conf/b.conf:/etc/nginx/conf.d/default.conf:ro,z
|
|
||||||
|
|
||||||
artifactapi:
|
|
||||||
# The host port is set via ARTIFACTAPI_PORT (see scripts/docker-e2e.sh),
|
|
||||||
# defaulting to 8000; the e2e run uses 8001 to avoid colliding with a
|
|
||||||
# locally-running instance.
|
|
||||||
depends_on:
|
|
||||||
mockupstream:
|
|
||||||
condition: service_started
|
|
||||||
mockupstreama:
|
|
||||||
condition: service_started
|
|
||||||
mockupstreamb:
|
|
||||||
condition: service_started
|
|
||||||
+56
-56
@@ -1,22 +1,31 @@
|
|||||||
|
version: '3.8'
|
||||||
|
|
||||||
services:
|
services:
|
||||||
artifactapi:
|
artifactapi:
|
||||||
build: .
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
args:
|
||||||
|
- VERSION=2.2.2.dev0
|
||||||
ports:
|
ports:
|
||||||
- "${ARTIFACTAPI_PORT:-8000}:8000"
|
- "8000:8000"
|
||||||
|
volumes:
|
||||||
|
- ./examples/single-file/remotes.yaml:/app/remotes.yaml:ro,z
|
||||||
|
- ./ca-bundle.pem:/app/ca-bundle.pem:ro,z
|
||||||
environment:
|
environment:
|
||||||
LISTEN_ADDR: ":8000"
|
- CONFIG_PATH=/app/remotes.yaml
|
||||||
DBHOST: postgres
|
- DBHOST=postgres
|
||||||
DBPORT: "5432"
|
- DBPORT=5432
|
||||||
DBUSER: artifacts
|
- DBUSER=artifacts
|
||||||
DBPASS: artifacts123
|
- DBPASS=artifacts123
|
||||||
DBNAME: artifacts
|
- DBNAME=artifacts
|
||||||
DBSSL: disable
|
- REDIS_URL=redis://redis:6379
|
||||||
REDIS_URL: redis://redis:6379
|
- MINIO_ENDPOINT=minio:9000
|
||||||
MINIO_ENDPOINT: minio:9000
|
- MINIO_ACCESS_KEY=minioadmin
|
||||||
MINIO_ACCESS_KEY: minioadmin
|
- MINIO_SECRET_KEY=minioadmin
|
||||||
MINIO_SECRET_KEY: minioadmin
|
- MINIO_BUCKET=artifacts
|
||||||
MINIO_BUCKET: artifacts
|
- MINIO_SECURE=false
|
||||||
MINIO_SECURE: "false"
|
- REQUESTS_CA_BUNDLE=/app/ca-bundle.pem
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
@@ -25,35 +34,27 @@ services:
|
|||||||
minio:
|
minio:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8000/health"]
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||||
interval: 10s
|
interval: 30s
|
||||||
timeout: 5s
|
timeout: 10s
|
||||||
retries: 3
|
retries: 3
|
||||||
|
|
||||||
ui:
|
minio:
|
||||||
build:
|
image: minio/minio:latest
|
||||||
context: ui
|
|
||||||
dockerfile: Dockerfile.ui
|
|
||||||
ports:
|
ports:
|
||||||
- "8080:80"
|
- "9000:9000"
|
||||||
depends_on:
|
- "9001:9001"
|
||||||
- artifactapi
|
|
||||||
|
|
||||||
postgres:
|
|
||||||
image: postgres:17-alpine
|
|
||||||
ports:
|
|
||||||
- "5432:5432"
|
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_USER: artifacts
|
MINIO_ROOT_USER: minioadmin
|
||||||
POSTGRES_PASSWORD: artifacts123
|
MINIO_ROOT_PASSWORD: minioadmin
|
||||||
POSTGRES_DB: artifacts
|
command: server /data --console-address ":9001"
|
||||||
volumes:
|
volumes:
|
||||||
- postgres_data:/var/lib/postgresql/data
|
- minio_data:/data
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD-SHELL", "pg_isready -U artifacts -d artifacts"]
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||||
interval: 5s
|
interval: 30s
|
||||||
timeout: 5s
|
timeout: 20s
|
||||||
retries: 5
|
retries: 3
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:7-alpine
|
||||||
@@ -64,28 +65,27 @@ services:
|
|||||||
command: redis-server --save 20 1
|
command: redis-server --save 20 1
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "redis-cli", "ping"]
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
interval: 5s
|
interval: 30s
|
||||||
timeout: 5s
|
timeout: 10s
|
||||||
retries: 5
|
retries: 3
|
||||||
|
|
||||||
minio:
|
postgres:
|
||||||
image: minio/minio:latest
|
image: postgres:15-alpine
|
||||||
command: server /data --console-address ":9001"
|
|
||||||
environment:
|
|
||||||
MINIO_ROOT_USER: minioadmin
|
|
||||||
MINIO_ROOT_PASSWORD: minioadmin
|
|
||||||
ports:
|
ports:
|
||||||
- "9000:9000"
|
- "5432:5432"
|
||||||
- "9001:9001"
|
environment:
|
||||||
|
POSTGRES_DB: artifacts
|
||||||
|
POSTGRES_USER: artifacts
|
||||||
|
POSTGRES_PASSWORD: artifacts123
|
||||||
volumes:
|
volumes:
|
||||||
- minio_data:/data
|
- postgres_data:/var/lib/postgresql/data
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "mc", "ready", "local"]
|
test: ["CMD-SHELL", "pg_isready -U artifacts -d artifacts"]
|
||||||
interval: 5s
|
interval: 30s
|
||||||
timeout: 5s
|
timeout: 10s
|
||||||
retries: 5
|
retries: 3
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
postgres_data:
|
|
||||||
redis_data:
|
|
||||||
minio_data:
|
minio_data:
|
||||||
|
redis_data:
|
||||||
|
postgres_data:
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
# `github_alpine` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `github_alpine` | `APKINDEX.tar.gz` (synthesized) | `.apk` (redirected) |
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `alpine` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `cargo` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
|
|
||||||
@@ -1,101 +0,0 @@
|
|||||||
# Caching and classification (all remotes)
|
|
||||||
|
|
||||||
Every proxied request on a remote is classified as **denied**, **immutable** or
|
|
||||||
**mutable**. The class picks the storage path and the freshness TTL.
|
|
||||||
|
|
||||||
## Fields
|
|
||||||
|
|
||||||
| Field | Meaning |
|
|
||||||
|---|---|
|
|
||||||
| `patterns` | Allowlist of regexes. Empty allows every path; otherwise a non-matching path is denied (403). |
|
|
||||||
| `blocklist` | Regexes; a match is denied (403). |
|
|
||||||
| `immutable_patterns` | Regexes forcing a path immutable. |
|
|
||||||
| `mutable_patterns` | Regexes forcing a path mutable. |
|
|
||||||
| `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. |
|
|
||||||
| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires. Omitted on create, it defaults to `3600`; omitted on update, the stored value is kept. |
|
|
||||||
| `check_mutable` | When a mutable path's `mutable_ttl` expires, sends a conditional `HEAD` (`If-None-Match`) with the stored upstream ETag. A `304` re-arms the TTL and serves the stored copy; anything else refetches in full. The ETag is kept for `max(10 × mutable_ttl, 7 days)`, re-armed on each `304`, so idle paths age out. It is only stored when `check_mutable` is set and `mutable_ttl` is non-zero, and never for immutable paths. |
|
|
||||||
| `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. |
|
|
||||||
| `quarantine_enabled` / `quarantine_days` | Stored on the remote but not enforced by the proxy yet. |
|
|
||||||
| `mirrorlist` / `mirror_strategy` | rpm/deb/alpine only. Upstream pool is `[base_url] + mirrorlist`; `round_robin` (default) rotates the starting mirror, `least_conn` starts at the one with the fewest in-flight requests. Network errors and 5xx fail over to the next mirror; other statuses return as-is. |
|
|
||||||
|
|
||||||
## Precedence
|
|
||||||
|
|
||||||
The first rule that matches wins:
|
|
||||||
|
|
||||||
1. `blocklist` matches: **denied**
|
|
||||||
2. `patterns` is non-empty and nothing matches: **denied**
|
|
||||||
3. `immutable_patterns` matches: **immutable**
|
|
||||||
4. `mutable_patterns` matches: **mutable**
|
|
||||||
5. the provider's built-in `Classify` says mutable: **mutable**
|
|
||||||
6. anything else: **immutable**
|
|
||||||
|
|
||||||
So `blocklist` beats the allowlist, and `immutable_patterns` beats
|
|
||||||
`mutable_patterns`. Either user list overrides the built-in rules. `GET` and
|
|
||||||
`HEAD` both apply this order.
|
|
||||||
|
|
||||||
## Matching rules
|
|
||||||
|
|
||||||
- Regexes are Go `regexp`, matched **unanchored** (`MatchString`). Add `^` or `$` yourself when you need them.
|
|
||||||
- They match the path after `/api/v1/remote/<name>/`, without a leading slash, e.g. `9/BaseOS/x86_64/os/repodata/repomd.xml`.
|
|
||||||
- The query string is not part of the path. It is dropped and not forwarded upstream.
|
|
||||||
- `create` and `update` reject an invalid regex in `patterns`, `blocklist`, `mutable_patterns`, `immutable_patterns` or `ban_tags` with a 400. Only invalid regexes already stored before that check are skipped at match time.
|
|
||||||
|
|
||||||
## Lifecycles
|
|
||||||
|
|
||||||
| | Immutable | Mutable |
|
|
||||||
|---|---|---|
|
|
||||||
| Stored as | content-addressed blob (sha256), streamed | per-path index object, buffered and rewritten by the provider |
|
|
||||||
| Fresh for | `immutable_ttl` (`0` = forever) | `mutable_ttl` |
|
|
||||||
| After expiry | refetched from upstream | revalidated with `If-None-Match` when `check_mutable` is set and an ETag is stored, else refetched |
|
|
||||||
| Upstream down | `stale_on_error` serves the stored copy and re-arms the TTL; an open circuit breaker always serves stale if a copy exists, else 503 | same |
|
|
||||||
|
|
||||||
Concurrent misses on one path share a fetch lock. Waiters poll the store for
|
|
||||||
up to 5s instead of all going upstream.
|
|
||||||
|
|
||||||
## Eviction
|
|
||||||
|
|
||||||
```sh
|
|
||||||
curl -X DELETE https://artifactapi.example.com/api/v2/remotes/<name>/objects/<path>
|
|
||||||
curl -X DELETE https://artifactapi.example.com/api/v2/remotes/<name>/objects/<dir>/*
|
|
||||||
```
|
|
||||||
|
|
||||||
- Evict drops the artifact row, the index object and the Redis TTL and ETag keys, so the next request goes upstream whatever the class or TTL.
|
|
||||||
- Deleting a remote drops its Redis TTL and ETag keys, so a re-created remote of the same name starts cold.
|
|
||||||
- A single-path evict takes the fetch lock first, so an in-flight fetch can't re-cache the path. It returns 503 if the lock stays busy.
|
|
||||||
- A wildcard must end in `<dir>/*` and takes no lock.
|
|
||||||
- Returns 204, or 404 for an unknown remote.
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
Bodies for `POST https://artifactapi.example.com/api/v2/remotes`.
|
|
||||||
|
|
||||||
GitHub release assets through a `generic` remote, allowlisted:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"name": "github", "package_type": "generic", "base_url": "https://github.com",
|
|
||||||
"immutable_ttl": 0, "mutable_ttl": 7200,
|
|
||||||
"patterns": ["^mikefarah/yq/releases/download/.*/yq_linux_amd64$"]}
|
|
||||||
```
|
|
||||||
|
|
||||||
rpm remote: `repodata/*` and `repomd.xml` are mutable by default, so only the TTLs are needed:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"name": "almalinux", "package_type": "rpm", "base_url": "https://repo.almalinux.org/almalinux",
|
|
||||||
"immutable_ttl": 0, "mutable_ttl": 3600, "check_mutable": true, "stale_on_error": true}
|
|
||||||
```
|
|
||||||
|
|
||||||
Force a moving path mutable, e.g. branch tarballs on `generic`:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"mutable_patterns": ["/archive/refs/heads/.*\\.tar\\.gz$"], "mutable_ttl": 600}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Built-in classification per type
|
|
||||||
|
|
||||||
Each type's page lists its auto-detected mutable and immutable paths:
|
|
||||||
[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md),
|
|
||||||
[cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md),
|
|
||||||
[docker](../docker/remote.md), [generic](../generic/remote.md), [gems](../gems/remote.md), [goproxy](../goproxy/remote.md),
|
|
||||||
[helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md),
|
|
||||||
[puppet](../puppet/remote.md), [pypi](../pypi/remote.md), [rpm](../rpm/remote.md),
|
|
||||||
[rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md).
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `github_deb` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `github_deb` | `Packages*`, `Release*` (synthesized) | `.deb` (redirected) |
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `deb` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `deb` | `Packages*`, `Sources*`, `Release`, `InRelease`, `Release.gpg`, `Contents-*` | `.deb` |
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
# Local docker registry
|
|
||||||
|
|
||||||
A local `docker` repo is a real container registry, not a mirror: it serves the
|
|
||||||
Docker Registry HTTP API V2 for both push and pull, so any client (`docker`,
|
|
||||||
`podman`, `skopeo`, `buildah`) can use it directly.
|
|
||||||
|
|
||||||
```sh
|
|
||||||
docker tag myapp:latest artifactapi.example.com/docker-internal/myapp:latest
|
|
||||||
docker push artifactapi.example.com/docker-internal/myapp:latest
|
|
||||||
docker pull artifactapi.example.com/docker-internal/myapp:latest
|
|
||||||
```
|
|
||||||
|
|
||||||
The first path segment after `/v2/` is the artifactapi repo name; the remainder
|
|
||||||
is the image name. Blobs and manifests are stored through the shared
|
|
||||||
content-addressable store (deduplicated by digest, reaped by GC once
|
|
||||||
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
|
|
||||||
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
# `docker` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
|
|
||||||
|
|
||||||
## Terraform
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
resource "artifactapi_remote_docker" "dockerhub" {
|
|
||||||
name = "dockerhub"
|
|
||||||
base_url = "https://registry-1.docker.io"
|
|
||||||
|
|
||||||
immutable_ttl = 0
|
|
||||||
mutable_ttl = 300
|
|
||||||
ban_tags_enabled = true
|
|
||||||
ban_tags = ["latest"]
|
|
||||||
|
|
||||||
patterns = [
|
|
||||||
"^library/postgres",
|
|
||||||
"^library/redis",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `gems` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `gems` | compact index (`versions`, `info/*`, `names`), `*specs.4.8.gz` | `gems/*.gem`, `quick/Marshal.4.8/*.gemspec.rz` |
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
# `generic` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `generic` | nothing | everything |
|
|
||||||
|
|
||||||
## Terraform
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
resource "artifactapi_remote_generic" "github" {
|
|
||||||
name = "github"
|
|
||||||
base_url = "https://github.com"
|
|
||||||
|
|
||||||
immutable_ttl = 0
|
|
||||||
mutable_ttl = 7200
|
|
||||||
|
|
||||||
patterns = [
|
|
||||||
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
|
|
||||||
"mikefarah/yq/.*/yq_linux_amd64$",
|
|
||||||
]
|
|
||||||
|
|
||||||
mutable_patterns = [
|
|
||||||
".*/archive/refs/heads/.*\\.tar\\.gz$",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `goproxy` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
# Local helm repository
|
|
||||||
|
|
||||||
A local `helm` repo is a real chart repository: `PUT` a packaged chart to
|
|
||||||
`/api/v2/remotes/<repo>/files/<chart>-<version>.tgz` and `index.yaml` is
|
|
||||||
generated from each chart's `Chart.yaml` (with sha256 digests) at
|
|
||||||
`/api/v1/local/<repo>/index.yaml`; a file not named after its `Chart.yaml`
|
|
||||||
name and version is stored but left out of the index. Add the local as a
|
|
||||||
member of a helm virtual to serve it alongside remotes.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
helm package ./mychart
|
|
||||||
curl -fsSL --upload-file mychart-0.1.0.tgz \
|
|
||||||
https://artifactapi.example.com/api/v2/remotes/<repo>/files/mychart-0.1.0.tgz
|
|
||||||
helm repo add <repo> https://artifactapi.example.com/api/v1/local/<repo>
|
|
||||||
```
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
# `helm` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `helm` | `index.yaml` | `.tgz` charts |
|
|
||||||
|
|
||||||
## Terraform
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
resource "artifactapi_remote_helm" "jetstack" {
|
|
||||||
name = "jetstack"
|
|
||||||
base_url = "https://charts.jetstack.io"
|
|
||||||
|
|
||||||
immutable_ttl = 0
|
|
||||||
mutable_ttl = 3600
|
|
||||||
}
|
|
||||||
```
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
# `helm` virtual
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
resource "artifactapi_virtual" "helm" {
|
|
||||||
name = "helm"
|
|
||||||
package_type = "helm"
|
|
||||||
members = [artifactapi_remote_helm.jetstack.name]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
# Local npm registry
|
|
||||||
|
|
||||||
A local `npm` repo is a real npm registry: `npm publish` (scoped and unscoped)
|
|
||||||
and `npm install` work against `/api/v1/local/<repo>/`. npm requires an
|
|
||||||
`_authToken` for the registry before it will publish; any value works.
|
|
||||||
|
|
||||||
```ini
|
|
||||||
# .npmrc
|
|
||||||
registry=https://artifactapi.example.com/api/v1/local/<repo>/
|
|
||||||
//artifactapi.example.com/api/v1/local/<repo>/:_authToken=unused
|
|
||||||
```
|
|
||||||
|
|
||||||
Versions are immutable: republishing an existing version is rejected. The
|
|
||||||
dist-tags sent with a publish (`latest`, or `--tag`) move to that version.
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `npm` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `npm` | package metadata | `.tgz` tarballs |
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
# `npm` virtual
|
|
||||||
|
|
||||||
An npm virtual serves one registry over local and remote npm members:
|
|
||||||
`npm install` and `npm view` work against `/api/v1/virtual/<name>/`.
|
|
||||||
|
|
||||||
- Versions are the union across members; on a duplicate version the earlier
|
|
||||||
member wins.
|
|
||||||
- Each dist-tag comes from the first member defining it, and is dropped if it
|
|
||||||
names a version missing from the merged set.
|
|
||||||
- Tarball URLs point at the virtual (`<member>/<path>`), which 302s to the
|
|
||||||
owning member.
|
|
||||||
- Full and abbreviated (`application/vnd.npm.install-v1+json`) packuments are
|
|
||||||
both served.
|
|
||||||
- A member error fails the request with 502; a member without the package is
|
|
||||||
skipped. Merges are cached for 60s per package.
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
resource "artifactapi_virtual" "npm" {
|
|
||||||
name = "npm"
|
|
||||||
package_type = "npm"
|
|
||||||
members = [artifactapi_local_npm.internal.name, artifactapi_remote_npm.npmjs.name]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
```ini
|
|
||||||
# .npmrc
|
|
||||||
registry=https://artifactapi.example.com/api/v1/virtual/npm/
|
|
||||||
```
|
|
||||||
|
|
||||||
Publish to the local member directly; the virtual is read-only.
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `nuget` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.snupkg`, `.nuspec`, `readme`, `icon` |
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
# Local puppet forge
|
|
||||||
|
|
||||||
A local `puppet` repo is a Forge: `PUT` a module tarball to
|
|
||||||
`/api/v2/remotes/<repo>/files/<owner>-<module>-<version>.tar.gz` and the Forge
|
|
||||||
v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each
|
|
||||||
release's `metadata.json` at `/api/v1/local/<repo>`. Versions are immutable.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
puppet module build ./hello
|
|
||||||
curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \
|
|
||||||
https://artifactapi.example.com/api/v2/remotes/<repo>/files/unkin-hello-0.1.0.tar.gz
|
|
||||||
puppet module install unkin-hello \
|
|
||||||
--module_repository https://artifactapi.example.com/api/v1/local/<repo>
|
|
||||||
```
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `puppet` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `pypi` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
|
|
||||||
@@ -1,82 +0,0 @@
|
|||||||
# `github_rpm` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
|
|
||||||
|
|
||||||
## GitHub releases as a yum repo (metadata-only, no precache)
|
|
||||||
|
|
||||||
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
|
|
||||||
repository without ever caching the packages. It scans releases for `.rpm`
|
|
||||||
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
|
|
||||||
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
|
|
||||||
metadata comes from a **ranged GET of just the RPM header** (the header sits at
|
|
||||||
the front of the file, so the whole package is never downloaded); the sha256
|
|
||||||
checksum comes from the GitHub asset `digest` when present, else a one-time
|
|
||||||
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
|
|
||||||
is served from primed DB rows, never a cold on-demand derive.
|
|
||||||
|
|
||||||
Each package's `<location>` points back at the remote, which **302-redirects**
|
|
||||||
the download to the `releases_remote` — an existing generic `github.com` remote
|
|
||||||
that streams the actual bytes. `dnf` follows the redirect transparently.
|
|
||||||
|
|
||||||
### Background syncer
|
|
||||||
|
|
||||||
A single process-wide **background syncer** keeps every `github_rpm` remote's
|
|
||||||
derived metadata current off the client request path:
|
|
||||||
|
|
||||||
- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime
|
|
||||||
scan, so its metadata is derived right away without blocking the create call.
|
|
||||||
The first `dnf` request is served from cache. If a request arrives before the
|
|
||||||
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
|
|
||||||
serving an empty repo or blocking on a multi-minute derive.
|
|
||||||
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
|
|
||||||
new or changed releases no more often than its `mutable_ttl`. New/changed
|
|
||||||
assets are derived incrementally; assets already cached are never re-fetched,
|
|
||||||
and assets that disappear upstream are pruned.
|
|
||||||
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
|
|
||||||
remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
|
|
||||||
and the syncer derives nothing. GitHub does not count `304` conditional
|
|
||||||
responses against the rate limit, so an unchanged repo is nearly free — this is
|
|
||||||
the main lever keeping GitHub traffic low.
|
|
||||||
- **Global rate limit.** Every GitHub call (releases list + each ranged asset
|
|
||||||
header GET) passes through a single token-bucket limiter **shared across all
|
|
||||||
remotes**, so GitHub is never hammered. Configure a token (`password`) on the
|
|
||||||
remote for the higher authenticated rate limit (~5000/hr vs ~60/hr
|
|
||||||
unauthenticated).
|
|
||||||
- **Multi-replica coordination.** State is shared through the database. Before a
|
|
||||||
periodic scan a replica must atomically claim a per-remote lease
|
|
||||||
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
|
|
||||||
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
|
|
||||||
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
|
|
||||||
lets any replica issue the conditional request.
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
# Backend that serves the actual .rpm bytes from github.com.
|
|
||||||
resource "artifactapi_remote_generic" "github" {
|
|
||||||
name = "github"
|
|
||||||
base_url = "https://github.com"
|
|
||||||
patterns = [
|
|
||||||
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "artifactapi_remote_github_rpm" "acme-tools" {
|
|
||||||
name = "acme-tools"
|
|
||||||
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
|
|
||||||
releases_remote = "github" # backend for downloads
|
|
||||||
mutable_ttl = 3600 # release re-scan interval
|
|
||||||
|
|
||||||
# Optional: restrict which release assets become packages (regex on filename).
|
|
||||||
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
|
|
||||||
|
|
||||||
# Optional: a token for private repos / higher API rate limits.
|
|
||||||
# password = "ghp_..."
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
|
|
||||||
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
|
|
||||||
from the synthesized metadata.
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `rpm` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
# Serving providers as a registry
|
|
||||||
|
|
||||||
A local `terraform` repo is a real provider registry: upload
|
|
||||||
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
|
|
||||||
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
|
|
||||||
no `.terraformrc` mirror config:
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
terraform {
|
|
||||||
required_providers {
|
|
||||||
artifactapi = {
|
|
||||||
source = "artifactapi.example.com/<repo>/<type>"
|
|
||||||
version = "0.1.2"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The Terraform *namespace* segment is the artifactapi repo name; the provider is
|
|
||||||
matched by *type*. The registry serves service discovery
|
|
||||||
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
|
|
||||||
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
|
|
||||||
|
|
||||||
Signing needs a GPG key. By default artifactapi generates one on first start and
|
|
||||||
stores it in the database (`signing_keys` table), so every replica shares it and
|
|
||||||
there's nothing to provision. To bring your own key instead, point
|
|
||||||
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
|
|
||||||
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
|
|
||||||
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# `terraform` remote
|
|
||||||
|
|
||||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
|
||||||
|
|
||||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
|
||||||
|---|---|---|
|
|
||||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
|
||||||
@@ -1,54 +0,0 @@
|
|||||||
# Dockerised end-to-end suite
|
|
||||||
|
|
||||||
Black-box tests that run against a fully **containerised** artifactapi stack
|
|
||||||
(built image + Postgres + Redis + MinIO) plus a static mock upstream. Unlike the
|
|
||||||
in-process `e2e/` suite (testcontainers, server run in-process), these only speak
|
|
||||||
HTTP to the running product, so they exercise the shipped container image.
|
|
||||||
|
|
||||||
## Run
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make docker-e2e # build image, compose up, run suite, compose down
|
|
||||||
```
|
|
||||||
|
|
||||||
`scripts/docker-e2e.sh` builds and starts `docker-compose.yml` +
|
|
||||||
`docker-compose.e2e.yml`, waits for `/health`, then runs
|
|
||||||
`go test -tags=dockere2e ./e2e-docker/...` and tears everything down.
|
|
||||||
|
|
||||||
The stack publishes artifactapi on host port **8001** (to avoid colliding with a
|
|
||||||
local instance on 8000). Override with `ARTIFACTAPI_URL` to point the tests at an
|
|
||||||
already-running stack.
|
|
||||||
|
|
||||||
## Coverage
|
|
||||||
|
|
||||||
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
|
|
||||||
- **Caching** — one immutable artifact per remote package type (one
|
|
||||||
`caching_<type>_test.go` case each) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
|
||||||
verified against the origin fixture.
|
|
||||||
- **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`),
|
|
||||||
generic (upload/download), pypi (wheel + generated `simple/`
|
|
||||||
index), rpm (real package + **automatic repodata** generation), npm (stock
|
|
||||||
`node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`,
|
|
||||||
second version, republish rejected).
|
|
||||||
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
|
|
||||||
across two members; npm packument merge over two locals via a stock `node:22`
|
|
||||||
client (merged versions, install through the member redirect).
|
|
||||||
- **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors
|
|
||||||
(pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors
|
|
||||||
(constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary,
|
|
||||||
no-mirrorlist regression, and a real `dnf` (stock `rockylinux:9` container)
|
|
||||||
`makecache` + `install` through a two-mirror rpm remote whose `base_url` is dead
|
|
||||||
— a dead mirror must not break the client.
|
|
||||||
- **Mirror strategy (`least_conn`)** — a `mirror_strategy: least_conn` rpm remote
|
|
||||||
over the two constant-body mirrors exercises the least-connections selection
|
|
||||||
path end-to-end (both mirrors serve, all requests succeed), plus a real `dnf`
|
|
||||||
install through a `least_conn` remote with a dead primary (failover unchanged).
|
|
||||||
The precise least-loaded pick is asserted deterministically in the proxy unit
|
|
||||||
test, since an in-flight-skew assertion over HTTP is timing-sensitive.
|
|
||||||
|
|
||||||
## Fixtures
|
|
||||||
|
|
||||||
`fixtures/` is served by the mock upstream at its web root. Paths mirror each
|
|
||||||
provider's upstream URL layout (e.g. `v2/...` for docker, `v1/providers/...` for
|
|
||||||
terraform). The RPM under `fixtures/rpmrepo/Packages/` is a real package so the
|
|
||||||
rpm provider can parse its metadata for repodata generation.
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"gems", "gems/mygem-1.0.0.gem", "gems/mygem-1.0.0.gem"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"generic", "blobs/hello.bin", "blobs/hello.bin"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"})
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cachingCases = append(cachingCases, cachingCase{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"})
|
|
||||||
}
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
type cachingCase struct {
|
|
||||||
pkgType string
|
|
||||||
// path is the request path under /api/v1/remote/<name>/. The provider
|
|
||||||
// derives the upstream URL from it (docker prepends /v2/, terraform
|
|
||||||
// prepends /v1/providers/), and the fixture lives at that resolved path.
|
|
||||||
path string
|
|
||||||
fixture string
|
|
||||||
}
|
|
||||||
|
|
||||||
// cachingCases is filled by one caching_<type>_test.go file per package type.
|
|
||||||
var cachingCases []cachingCase
|
|
||||||
|
|
||||||
// TestCachingPerProvider proxies one immutable artifact for every remote
|
|
||||||
// package type through the mock upstream and asserts: first fetch is served
|
|
||||||
// from the remote, the second from cache, and the bytes match the origin.
|
|
||||||
func TestCachingPerProvider(t *testing.T) {
|
|
||||||
for _, tc := range cachingCases {
|
|
||||||
t.Run(tc.pkgType, func(t *testing.T) {
|
|
||||||
name := "cache-" + tc.pkgType
|
|
||||||
createRepo(t, fmt.Sprintf(`{
|
|
||||||
"name": %q,
|
|
||||||
"package_type": %q,
|
|
||||||
"repo_type": "remote",
|
|
||||||
"base_url": %q,
|
|
||||||
"stale_on_error": true
|
|
||||||
}`, name, tc.pkgType, mockUpstream()))
|
|
||||||
defer deleteRepo(t, name)
|
|
||||||
|
|
||||||
want := fixtureBytes(t, tc.fixture)
|
|
||||||
url := api("/api/v1/remote/" + name + "/" + tc.path)
|
|
||||||
|
|
||||||
// First fetch: from remote.
|
|
||||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("first fetch: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if src := resp.Header.Get("X-Artifact-Source"); src != "remote" {
|
|
||||||
t.Fatalf("first fetch source = %q, want remote", src)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(body, want) {
|
|
||||||
t.Fatalf("first fetch body mismatch: got %d bytes, want %d", len(body), len(want))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Second fetch: from cache, identical bytes.
|
|
||||||
resp, body = doRequest(t, http.MethodGet, url, nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("second fetch: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if src := resp.Header.Get("X-Artifact-Source"); src != "cache" {
|
|
||||||
t.Fatalf("second fetch source = %q, want cache", src)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(body, want) {
|
|
||||||
t.Fatalf("cached body mismatch: got %d bytes, want %d", len(body), len(want))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,177 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func digestOf(b []byte) string {
|
|
||||||
sum := sha256.Sum256(b)
|
|
||||||
return "sha256:" + hex.EncodeToString(sum[:])
|
|
||||||
}
|
|
||||||
|
|
||||||
// pushBlobMonolithic uploads a blob with POST (open session) then PUT?digest
|
|
||||||
// (whole body) — the monolithic-after-POST flow.
|
|
||||||
func pushBlobMonolithic(t *testing.T, repo, image string, blob []byte) {
|
|
||||||
t.Helper()
|
|
||||||
dgst := digestOf(blob)
|
|
||||||
|
|
||||||
resp, body := doRequest(t, http.MethodPost, api("/v2/"+repo+"/"+image+"/blobs/uploads/"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusAccepted {
|
|
||||||
t.Fatalf("start upload: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
loc := resp.Header.Get("Location")
|
|
||||||
if loc == "" {
|
|
||||||
t.Fatalf("start upload: no Location header")
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, body = doRequest(t, http.MethodPut, baseURL()+loc+"?digest="+dgst, blob, "application/octet-stream")
|
|
||||||
if resp.StatusCode != http.StatusCreated {
|
|
||||||
t.Fatalf("finish upload: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Docker-Content-Digest"); got != dgst {
|
|
||||||
t.Fatalf("finish upload: digest mismatch: got %q want %q", got, dgst)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// pushBlobChunked uploads a blob with POST then PATCH (body) then PUT?digest
|
|
||||||
// (empty) — the chunked flow a real docker daemon uses.
|
|
||||||
func pushBlobChunked(t *testing.T, repo, image string, blob []byte) {
|
|
||||||
t.Helper()
|
|
||||||
dgst := digestOf(blob)
|
|
||||||
|
|
||||||
resp, body := doRequest(t, http.MethodPost, api("/v2/"+repo+"/"+image+"/blobs/uploads/"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusAccepted {
|
|
||||||
t.Fatalf("start upload: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
loc := resp.Header.Get("Location")
|
|
||||||
|
|
||||||
resp, body = doRequest(t, http.MethodPatch, baseURL()+loc, blob, "application/octet-stream")
|
|
||||||
if resp.StatusCode != http.StatusAccepted {
|
|
||||||
t.Fatalf("patch upload: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Range"); got != fmt.Sprintf("0-%d", len(blob)-1) {
|
|
||||||
t.Fatalf("patch upload: unexpected Range %q", got)
|
|
||||||
}
|
|
||||||
loc = resp.Header.Get("Location")
|
|
||||||
|
|
||||||
resp, body = doRequest(t, http.MethodPut, baseURL()+loc+"?digest="+dgst, nil, "")
|
|
||||||
if resp.StatusCode != http.StatusCreated {
|
|
||||||
t.Fatalf("finish upload: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLocalDockerPushPull exercises a full container push and pull against a
|
|
||||||
// local docker repo using the Docker Registry HTTP API V2, the way a docker
|
|
||||||
// client would: upload the config and layer blobs, push the manifest under a
|
|
||||||
// tag, then pull the manifest and blobs back byte-identically.
|
|
||||||
func TestLocalDockerPushPull(t *testing.T) {
|
|
||||||
createRepo(t, `{"name":"docker-internal","package_type":"docker","repo_type":"local"}`)
|
|
||||||
defer deleteRepo(t, "docker-internal")
|
|
||||||
|
|
||||||
const image = "team/app"
|
|
||||||
const tag = "v1.0.0"
|
|
||||||
|
|
||||||
// /v2/ version check.
|
|
||||||
resp, _ := doRequest(t, http.MethodGet, api("/v2/"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("/v2/ ping: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
config := []byte(`{"architecture":"amd64","os":"linux","config":{},"rootfs":{"type":"layers","diff_ids":["sha256:0000000000000000000000000000000000000000000000000000000000000000"]}}`)
|
|
||||||
layer := bytes.Repeat([]byte("artifactapi-layer-data-"), 4096) // ~90 KB opaque layer
|
|
||||||
|
|
||||||
configDigest := digestOf(config)
|
|
||||||
layerDigest := digestOf(layer)
|
|
||||||
|
|
||||||
// A brand-new blob should be absent (this is the client's mount check).
|
|
||||||
resp, _ = doRequest(t, http.MethodHead, api("/v2/"+"docker-internal/"+image+"/blobs/"+configDigest), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusNotFound {
|
|
||||||
t.Fatalf("pre-push blob HEAD: expected 404, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
pushBlobMonolithic(t, "docker-internal", image, config)
|
|
||||||
pushBlobChunked(t, "docker-internal", image, layer)
|
|
||||||
|
|
||||||
manifest := []byte(fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":%d,"digest":%q},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":%d,"digest":%q}]}`,
|
|
||||||
len(config), configDigest, len(layer), layerDigest))
|
|
||||||
manifestDigest := digestOf(manifest)
|
|
||||||
manifestType := "application/vnd.docker.distribution.manifest.v2+json"
|
|
||||||
|
|
||||||
resp, body := doRequest(t, http.MethodPut, api("/v2/docker-internal/"+image+"/manifests/"+tag), manifest, manifestType)
|
|
||||||
if resp.StatusCode != http.StatusCreated {
|
|
||||||
t.Fatalf("push manifest: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Docker-Content-Digest"); got != manifestDigest {
|
|
||||||
t.Fatalf("push manifest: digest %q want %q", got, manifestDigest)
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- pull back ---
|
|
||||||
|
|
||||||
// Manifest by tag.
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/v2/docker-internal/"+image+"/manifests/"+tag), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("pull manifest by tag: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(body, manifest) {
|
|
||||||
t.Fatalf("pulled manifest bytes differ from pushed")
|
|
||||||
}
|
|
||||||
if ct := resp.Header.Get("Content-Type"); ct != manifestType {
|
|
||||||
t.Fatalf("pulled manifest content-type %q want %q", ct, manifestType)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Docker-Content-Digest"); got != manifestDigest {
|
|
||||||
t.Fatalf("pulled manifest digest %q want %q", got, manifestDigest)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Manifest by digest.
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/v2/docker-internal/"+image+"/manifests/"+manifestDigest), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK || !bytes.Equal(body, manifest) {
|
|
||||||
t.Fatalf("pull manifest by digest: status %d, equal=%v", resp.StatusCode, bytes.Equal(body, manifest))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Blobs by digest.
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
digest string
|
|
||||||
want []byte
|
|
||||||
}{
|
|
||||||
{"config", configDigest, config},
|
|
||||||
{"layer", layerDigest, layer},
|
|
||||||
} {
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/v2/docker-internal/"+image+"/blobs/"+tc.digest), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("pull %s blob: status %d", tc.name, resp.StatusCode)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(body, tc.want) {
|
|
||||||
t.Fatalf("pulled %s blob bytes differ", tc.name)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Docker-Content-Digest"); got != tc.digest {
|
|
||||||
t.Fatalf("pulled %s blob digest %q want %q", tc.name, got, tc.digest)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tags/list reflects the pushed tag.
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/v2/docker-internal/"+image+"/tags/list"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("tags/list: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(body), `"`+tag+`"`) {
|
|
||||||
t.Fatalf("tags/list missing tag %q: %s", tag, body)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(body), `"docker-internal/`+image+`"`) {
|
|
||||||
t.Fatalf("tags/list wrong repository name: %s", body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A now-present blob HEAD should succeed (client would skip re-upload).
|
|
||||||
resp, _ = doRequest(t, http.MethodHead, api("/v2/docker-internal/"+image+"/blobs/"+layerDigest), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("post-push blob HEAD: expected 200, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@@ -1 +0,0 @@
|
|||||||
hello artifactapi generic blob
|
|
||||||
Binary file not shown.
Binary file not shown.
@@ -1 +0,0 @@
|
|||||||
mygem-1.0.0 e2e fixture
|
|
||||||
Binary file not shown.
@@ -1,8 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
entries:
|
|
||||||
alpha:
|
|
||||||
- name: alpha
|
|
||||||
version: 1.0.0
|
|
||||||
urls:
|
|
||||||
- charts/alpha-1.0.0.tgz
|
|
||||||
generated: "2026-01-01T00:00:00Z"
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
entries:
|
|
||||||
beta:
|
|
||||||
- name: beta
|
|
||||||
version: 2.0.0
|
|
||||||
urls:
|
|
||||||
- charts/beta-2.0.0.tgz
|
|
||||||
generated: "2026-01-01T00:00:00Z"
|
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -1,55 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
|
|
||||||
<revision>1786573032</revision>
|
|
||||||
<data type="primary">
|
|
||||||
<checksum type="sha256">d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08</checksum>
|
|
||||||
<open-checksum type="sha256">3345bb631380ae6c0620fe2a29cf7dff2ed4e28c5cb06c91e8a1628ba1979bcb</open-checksum>
|
|
||||||
<location href="repodata/d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08-primary.xml.gz"/>
|
|
||||||
<timestamp>1786573032</timestamp>
|
|
||||||
<size>631</size>
|
|
||||||
<open-size>1192</open-size>
|
|
||||||
</data>
|
|
||||||
<data type="filelists">
|
|
||||||
<checksum type="sha256">daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be</checksum>
|
|
||||||
<open-checksum type="sha256">648bd0ce00fda09abbc6e9c3ff3278518a76f258576ac24cd10c12e41e0e5bd7</open-checksum>
|
|
||||||
<location href="repodata/daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be-filelists.xml.gz"/>
|
|
||||||
<timestamp>1786573032</timestamp>
|
|
||||||
<size>256</size>
|
|
||||||
<open-size>338</open-size>
|
|
||||||
</data>
|
|
||||||
<data type="other">
|
|
||||||
<checksum type="sha256">8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85</checksum>
|
|
||||||
<open-checksum type="sha256">c42cfd3843e9c53a60ad84bded44aa46c65faae7b3da99a099a9ca0b018872a9</open-checksum>
|
|
||||||
<location href="repodata/8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85-other.xml.gz"/>
|
|
||||||
<timestamp>1786573032</timestamp>
|
|
||||||
<size>296</size>
|
|
||||||
<open-size>399</open-size>
|
|
||||||
</data>
|
|
||||||
<data type="primary_db">
|
|
||||||
<checksum type="sha256">f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71</checksum>
|
|
||||||
<open-checksum type="sha256">c45c85d12ccb0f8172b7bfae466362c08ac1867559574a9fb9cb2118c04daddb</open-checksum>
|
|
||||||
<location href="repodata/f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71-primary.sqlite.bz2"/>
|
|
||||||
<timestamp>1786573032</timestamp>
|
|
||||||
<size>1740</size>
|
|
||||||
<open-size>106496</open-size>
|
|
||||||
<database_version>10</database_version>
|
|
||||||
</data>
|
|
||||||
<data type="filelists_db">
|
|
||||||
<checksum type="sha256">be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57</checksum>
|
|
||||||
<open-checksum type="sha256">1ccfa3dff532d782ce3225aae807506a4ce4534291386f1c47455dcc6b70cfd6</open-checksum>
|
|
||||||
<location href="repodata/be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57-filelists.sqlite.bz2"/>
|
|
||||||
<timestamp>1786573032</timestamp>
|
|
||||||
<size>764</size>
|
|
||||||
<open-size>28672</open-size>
|
|
||||||
<database_version>10</database_version>
|
|
||||||
</data>
|
|
||||||
<data type="other_db">
|
|
||||||
<checksum type="sha256">ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4</checksum>
|
|
||||||
<open-checksum type="sha256">5d4d38380f0e359bfc0a50033d4faa84c75c5ae8fe2ef11c1c82d64748e9b8e2</open-checksum>
|
|
||||||
<location href="repodata/ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4-other.sqlite.bz2"/>
|
|
||||||
<timestamp>1786573032</timestamp>
|
|
||||||
<size>738</size>
|
|
||||||
<open-size>24576</open-size>
|
|
||||||
<database_version>10</database_version>
|
|
||||||
</data>
|
|
||||||
</repomd>
|
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,71 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestLocalHelmRealClient packages a chart with the real helm CLI, publishes it
|
|
||||||
// to a helm local, then drives helm repo add/search/pull/template against the
|
|
||||||
// local and against a helm virtual that has the local as a member.
|
|
||||||
func TestLocalHelmRealClient(t *testing.T) {
|
|
||||||
network := os.Getenv("COMPOSE_NETWORK")
|
|
||||||
internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
|
|
||||||
if network == "" || internal == "" {
|
|
||||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
|
||||||
}
|
|
||||||
if _, err := exec.LookPath("docker"); err != nil {
|
|
||||||
t.Skip("docker not available on the test host")
|
|
||||||
}
|
|
||||||
|
|
||||||
createRepo(t, `{"name":"local-helm","package_type":"helm","repo_type":"local"}`)
|
|
||||||
defer deleteRepo(t, "local-helm")
|
|
||||||
createVirtual(t, `{"name":"vhelm","package_type":"helm","members":["local-helm"]}`)
|
|
||||||
defer deleteVirtual(t, "vhelm")
|
|
||||||
|
|
||||||
helm := func(script string) string {
|
|
||||||
t.Helper()
|
|
||||||
out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh",
|
|
||||||
"alpine/helm:3.16.4", "-c", "set -eu; "+script).Output()
|
|
||||||
if err != nil {
|
|
||||||
stderr := ""
|
|
||||||
if ee, ok := err.(*exec.ExitError); ok {
|
|
||||||
stderr = string(ee.Stderr)
|
|
||||||
}
|
|
||||||
t.Fatalf("helm script failed: %v\n%s%s", err, out, stderr)
|
|
||||||
}
|
|
||||||
return string(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
chart := helm("cd /tmp && helm create e2echart >/dev/null && helm package e2echart >/dev/null && cat e2echart-0.1.0.tgz")
|
|
||||||
uploadFile(t, "local-helm", "e2echart-0.1.0.tgz", []byte(chart), "application/gzip")
|
|
||||||
|
|
||||||
// Chart metadata is recorded asynchronously after upload; poll the index.
|
|
||||||
var body []byte
|
|
||||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) {
|
|
||||||
var resp *http.Response
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-helm/index.yaml"), nil, "")
|
|
||||||
if resp.StatusCode == http.StatusOK && strings.Contains(string(body), "e2echart-0.1.0.tgz") {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(body), "e2echart-0.1.0.tgz") || !strings.Contains(string(body), "digest: ") {
|
|
||||||
t.Fatalf("index.yaml never listed the chart:\n%s", body)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, repoURL := range []string{internal + "/api/v1/local/local-helm", internal + "/api/v1/virtual/vhelm"} {
|
|
||||||
out := helm("helm repo add e2e " + repoURL + " >/dev/null && " +
|
|
||||||
"helm search repo e2e/e2echart && " +
|
|
||||||
"cd /tmp && helm pull e2e/e2echart --version 0.1.0 && " +
|
|
||||||
"helm template rel e2echart-0.1.0.tgz | grep -c 'kind: Deployment'")
|
|
||||||
if !strings.Contains(out, "e2e/e2echart") || !strings.Contains(out, "0.1.0") || !strings.HasSuffix(strings.TrimSpace(out), "1") {
|
|
||||||
t.Fatalf("helm via %s:\n%s", repoURL, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,108 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
// Package e2edocker holds the black-box end-to-end suite that runs against a
|
|
||||||
// fully dockerised artifactapi stack (see scripts/docker-e2e.sh). Unlike the
|
|
||||||
// in-process e2e suite, these tests only speak HTTP to the running container.
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func baseURL() string {
|
|
||||||
if v := os.Getenv("ARTIFACTAPI_URL"); v != "" {
|
|
||||||
return strings.TrimRight(v, "/")
|
|
||||||
}
|
|
||||||
return "http://localhost:8000"
|
|
||||||
}
|
|
||||||
|
|
||||||
// mockUpstream is the base URL the artifactapi *container* uses to reach the
|
|
||||||
// static mock upstream. It is resolved on the compose network, not the host.
|
|
||||||
func mockUpstream() string {
|
|
||||||
if v := os.Getenv("MOCK_UPSTREAM_INTERNAL"); v != "" {
|
|
||||||
return strings.TrimRight(v, "/")
|
|
||||||
}
|
|
||||||
return "http://mockupstream"
|
|
||||||
}
|
|
||||||
|
|
||||||
func api(path string) string { return baseURL() + path }
|
|
||||||
|
|
||||||
func fixtureBytes(t *testing.T, rel string) []byte {
|
|
||||||
t.Helper()
|
|
||||||
b, err := os.ReadFile(filepath.Join("fixtures", rel))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read fixture %s: %v", rel, err)
|
|
||||||
}
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
func doRequest(t *testing.T, method, url string, body []byte, contentType string) (*http.Response, []byte) {
|
|
||||||
t.Helper()
|
|
||||||
var r io.Reader
|
|
||||||
if body != nil {
|
|
||||||
r = bytes.NewReader(body)
|
|
||||||
}
|
|
||||||
req, err := http.NewRequest(method, url, r)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s %s: %v", method, url, err)
|
|
||||||
}
|
|
||||||
if contentType != "" {
|
|
||||||
req.Header.Set("Content-Type", contentType)
|
|
||||||
}
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s %s: %v", method, url, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
respBody, _ := io.ReadAll(resp.Body)
|
|
||||||
return resp, respBody
|
|
||||||
}
|
|
||||||
|
|
||||||
func createRepo(t *testing.T, jsonBody string) {
|
|
||||||
t.Helper()
|
|
||||||
resp, body := doRequest(t, http.MethodPost, api("/api/v2/remotes"), []byte(jsonBody), "application/json")
|
|
||||||
if resp.StatusCode != http.StatusCreated {
|
|
||||||
t.Fatalf("create repo: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteRepo(t *testing.T, name string) {
|
|
||||||
t.Helper()
|
|
||||||
doRequest(t, http.MethodDelete, api("/api/v2/remotes/"+name), nil, "")
|
|
||||||
}
|
|
||||||
|
|
||||||
func createVirtual(t *testing.T, jsonBody string) {
|
|
||||||
t.Helper()
|
|
||||||
resp, body := doRequest(t, http.MethodPost, api("/api/v2/virtuals"), []byte(jsonBody), "application/json")
|
|
||||||
if resp.StatusCode != http.StatusCreated {
|
|
||||||
t.Fatalf("create virtual: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteVirtual(t *testing.T, name string) {
|
|
||||||
t.Helper()
|
|
||||||
doRequest(t, http.MethodDelete, api("/api/v2/virtuals/"+name), nil, "")
|
|
||||||
}
|
|
||||||
|
|
||||||
// getEventually retries a GET until it returns 200 or the deadline passes. Used
|
|
||||||
// for asynchronously-generated artifacts (e.g. rpm repodata after upload).
|
|
||||||
func getEventually(t *testing.T, url string, timeout time.Duration) (*http.Response, []byte) {
|
|
||||||
t.Helper()
|
|
||||||
deadline := time.Now().Add(timeout)
|
|
||||||
var resp *http.Response
|
|
||||||
var body []byte
|
|
||||||
for {
|
|
||||||
resp, body = doRequest(t, http.MethodGet, url, nil, "")
|
|
||||||
if resp.StatusCode == http.StatusOK || time.Now().After(deadline) {
|
|
||||||
return resp, body
|
|
||||||
}
|
|
||||||
time.Sleep(250 * time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,105 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestLeastConnMultiBaseURL configures an rpm remote with mirror_strategy =
|
|
||||||
// least_conn over a two-mirror pool and drives distinct cache-miss paths through
|
|
||||||
// it, asserting every request succeeds and both mirrors serve traffic. This
|
|
||||||
// exercises the least-connections selection path (leastConnOrder + the in-flight
|
|
||||||
// gauge inc/dec around each upstream call) end-to-end through a real HTTP client.
|
|
||||||
// A precise least-loaded assertion is timing-sensitive over HTTP and is covered
|
|
||||||
// deterministically by the proxy unit test (TestLeastConnPicksLeastLoaded);
|
|
||||||
// here, with requests issued serially, in-flight counts return to zero between
|
|
||||||
// them so equal-load mirrors are spread by the round-robin tie-break.
|
|
||||||
func TestLeastConnMultiBaseURL(t *testing.T) {
|
|
||||||
name := "e2e-leastconn"
|
|
||||||
createRepo(t, fmt.Sprintf(`{
|
|
||||||
"name": %q,
|
|
||||||
"package_type": "rpm",
|
|
||||||
"repo_type": "remote",
|
|
||||||
"base_url": %q,
|
|
||||||
"mirrorlist": [%q],
|
|
||||||
"mirror_strategy": "least_conn",
|
|
||||||
"stale_on_error": false
|
|
||||||
}`, name, mockUpstreamA(), mockUpstreamB()))
|
|
||||||
defer deleteRepo(t, name)
|
|
||||||
|
|
||||||
seenA, seenB := false, false
|
|
||||||
const n = 12
|
|
||||||
for i := 0; i < n; i++ {
|
|
||||||
url := api(fmt.Sprintf("/api/v1/remote/%s/lc/%d", name, i))
|
|
||||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("request %d: status %d: %s", i, resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
switch strings.TrimSpace(string(body)) {
|
|
||||||
case "UPSTREAM-A":
|
|
||||||
seenA = true
|
|
||||||
case "UPSTREAM-B":
|
|
||||||
seenB = true
|
|
||||||
default:
|
|
||||||
t.Fatalf("request %d: unexpected body %q", i, body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !seenA || !seenB {
|
|
||||||
t.Fatalf("least_conn remote did not reach both upstreams: A=%v B=%v", seenA, seenB)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLeastConnDnfInstall drives a real dnf (stock rockylinux container) at a
|
|
||||||
// two-mirror rpm remote configured with mirror_strategy = least_conn whose
|
|
||||||
// primary base_url is dead: makecache + install must succeed via the live mirror.
|
|
||||||
// This proves a real package-manager client installs correctly through a
|
|
||||||
// least_conn remote and that failover semantics are unchanged under the new
|
|
||||||
// strategy. Requires the compose network exported by scripts/docker-e2e.sh.
|
|
||||||
func TestLeastConnDnfInstall(t *testing.T) {
|
|
||||||
network := os.Getenv("COMPOSE_NETWORK")
|
|
||||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
|
||||||
if network == "" || internal == "" {
|
|
||||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
|
||||||
}
|
|
||||||
if _, err := exec.LookPath("docker"); err != nil {
|
|
||||||
t.Skip("docker not available on the test host")
|
|
||||||
}
|
|
||||||
|
|
||||||
name := "e2e-leastconn-dnf"
|
|
||||||
createRepo(t, fmt.Sprintf(`{
|
|
||||||
"name": %q,
|
|
||||||
"package_type": "rpm",
|
|
||||||
"repo_type": "remote",
|
|
||||||
"base_url": "http://mockupstream-dead:80",
|
|
||||||
"mirrorlist": [%q],
|
|
||||||
"mirror_strategy": "least_conn",
|
|
||||||
"stale_on_error": false
|
|
||||||
}`, name, mockUpstream()))
|
|
||||||
defer deleteRepo(t, name)
|
|
||||||
|
|
||||||
repoURL := strings.TrimRight(internal, "/") + "/api/v1/remote/" + name + "/rpm-mirror"
|
|
||||||
repoConf := fmt.Sprintf("[dnflc]\nname=dnflc\nbaseurl=%s\nenabled=1\ngpgcheck=0\nsslverify=0\nmetadata_expire=0\n", repoURL)
|
|
||||||
script := "set -euo pipefail; " +
|
|
||||||
"printf '%s' \"$REPO\" > /etc/yum.repos.d/dnflc.repo; " +
|
|
||||||
"dnf -y --disablerepo='*' --enablerepo=dnflc makecache; " +
|
|
||||||
"dnf -y --disablerepo='*' --enablerepo=dnflc install e2e-testpkg; " +
|
|
||||||
"rpm -q e2e-testpkg"
|
|
||||||
|
|
||||||
cmd := exec.Command("docker", "run", "--rm",
|
|
||||||
"--network", network,
|
|
||||||
"-e", "REPO="+repoConf,
|
|
||||||
"rockylinux:9", "bash", "-c", script)
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("real dnf install through a least_conn remote failed: %v\n%s", err, out)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(out), "e2e-testpkg-1.0-1") {
|
|
||||||
t.Fatalf("dnf did not install the expected package via least_conn remote; output:\n%s", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,193 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"archive/tar"
|
|
||||||
"bytes"
|
|
||||||
"compress/gzip"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
|
||||||
)
|
|
||||||
|
|
||||||
func uploadFile(t *testing.T, repo, filePath string, body []byte, contentType string) {
|
|
||||||
t.Helper()
|
|
||||||
url := api("/api/v2/remotes/" + repo + "/files/" + filePath)
|
|
||||||
resp, respBody := doRequest(t, http.MethodPut, url, body, contentType)
|
|
||||||
if resp.StatusCode != http.StatusCreated {
|
|
||||||
t.Fatalf("upload %s: status %d: %s", filePath, resp.StatusCode, respBody)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLocalGenericUpload uploads a generic file and downloads it back.
|
|
||||||
func TestLocalGenericUpload(t *testing.T) {
|
|
||||||
createRepo(t, `{"name":"local-generic","package_type":"generic","repo_type":"local"}`)
|
|
||||||
defer deleteRepo(t, "local-generic")
|
|
||||||
|
|
||||||
content := []byte("artifactapi local generic upload payload")
|
|
||||||
uploadFile(t, "local-generic", "data/hello.bin", content, "application/octet-stream")
|
|
||||||
|
|
||||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-generic/data/hello.bin"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("download: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(body, content) {
|
|
||||||
t.Fatalf("downloaded content mismatch")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLocalPyPIUpload uploads a wheel and validates the generated simple index.
|
|
||||||
func TestLocalPyPIUpload(t *testing.T) {
|
|
||||||
createRepo(t, `{"name":"local-pypi","package_type":"pypi","repo_type":"local"}`)
|
|
||||||
defer deleteRepo(t, "local-pypi")
|
|
||||||
|
|
||||||
wheel := fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl")
|
|
||||||
uploadFile(t, "local-pypi", "foo-1.0-py3-none-any.whl", wheel, "application/zip")
|
|
||||||
|
|
||||||
// Root index lists the package.
|
|
||||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("simple index: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(body), "foo") {
|
|
||||||
t.Fatalf("simple index missing package 'foo': %s", body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Per-package index lists the wheel file.
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/foo/"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("package index: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(body), "foo-1.0-py3-none-any.whl") {
|
|
||||||
t.Fatalf("package index missing wheel: %s", body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The wheel downloads back byte-identical.
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/foo/foo-1.0-py3-none-any.whl"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("download wheel: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(body, wheel) {
|
|
||||||
t.Fatalf("wheel content mismatch")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLocalRPMRepodata uploads a real RPM and validates that repodata is
|
|
||||||
// generated automatically (the special rpm-local feature).
|
|
||||||
func TestLocalRPMRepodata(t *testing.T) {
|
|
||||||
createRepo(t, `{"name":"local-rpm","package_type":"rpm","repo_type":"local"}`)
|
|
||||||
defer deleteRepo(t, "local-rpm")
|
|
||||||
|
|
||||||
rpm := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
|
|
||||||
uploadFile(t, "local-rpm", "e2e-testpkg-1.0-1.noarch.rpm", rpm, "application/x-rpm")
|
|
||||||
|
|
||||||
// repodata is generated asynchronously after upload; poll for it.
|
|
||||||
resp, body := getEventually(t, api("/api/v1/local/local-rpm/repodata/repomd.xml"), 15*time.Second)
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("repomd.xml: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
s := string(body)
|
|
||||||
if !strings.Contains(s, "<repomd") || !strings.Contains(s, "primary") {
|
|
||||||
t.Fatalf("repomd.xml not a valid repodata document: %s", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLocalDebRepo uploads a .deb and validates that the flat apt index
|
|
||||||
// (Packages / Release) is generated automatically from the parsed control
|
|
||||||
// stanza (the deb-local analog of rpm repodata generation).
|
|
||||||
func TestLocalDebRepo(t *testing.T) {
|
|
||||||
createRepo(t, `{"name":"local-deb","package_type":"deb","repo_type":"local"}`)
|
|
||||||
defer deleteRepo(t, "local-deb")
|
|
||||||
|
|
||||||
deb := testsupport.MinimalDeb("e2e-testpkg", "1.0.0", "amd64")
|
|
||||||
uploadFile(t, "local-deb", "e2e-testpkg_1.0.0_amd64.deb", deb, "application/vnd.debian.binary-package")
|
|
||||||
|
|
||||||
// The index is generated asynchronously after upload; poll for it.
|
|
||||||
resp, body := getEventually(t, api("/api/v1/local/local-deb/Packages"), 15*time.Second)
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("Packages: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
pkgs := string(body)
|
|
||||||
for _, want := range []string{"Package: e2e-testpkg", "Version: 1.0.0", "Architecture: amd64", "Filename: pool/e2e-testpkg_1.0.0_amd64.deb", "SHA256:"} {
|
|
||||||
if !strings.Contains(pkgs, want) {
|
|
||||||
t.Fatalf("Packages missing %q:\n%s", want, pkgs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-deb/Release"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("Release: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
rel := string(body)
|
|
||||||
for _, want := range []string{"Architectures: amd64", "SHA256:", "Packages"} {
|
|
||||||
if !strings.Contains(rel, want) {
|
|
||||||
t.Fatalf("Release missing %q:\n%s", want, rel)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The .deb downloads back byte-identical from its pool path.
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-deb/pool/e2e-testpkg_1.0.0_amd64.deb"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("download deb: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(body, deb) {
|
|
||||||
t.Fatalf("deb content mismatch")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLocalAlpineIndex uploads an .apk to an alpine local repo and validates
|
|
||||||
// that a per-arch APKINDEX.tar.gz is generated automatically from the parsed
|
|
||||||
// .PKGINFO (the apk-local analog of rpm repodata / deb Packages generation).
|
|
||||||
func TestLocalAlpineIndex(t *testing.T) {
|
|
||||||
createRepo(t, `{"name":"local-alpine","package_type":"alpine","repo_type":"local"}`)
|
|
||||||
defer deleteRepo(t, "local-alpine")
|
|
||||||
|
|
||||||
apk := testsupport.MinimalApk("e2e-testpkg", "1.0-r0", "x86_64")
|
|
||||||
uploadFile(t, "local-alpine", "x86_64/e2e-testpkg-1.0-r0.apk", apk, "application/vnd.android.package-archive")
|
|
||||||
|
|
||||||
// The index is generated asynchronously after upload; poll for it.
|
|
||||||
resp, body := getEventually(t, api("/api/v1/local/local-alpine/x86_64/APKINDEX.tar.gz"), 15*time.Second)
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("APKINDEX: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
|
|
||||||
zr, err := gzip.NewReader(bytes.NewReader(body))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("APKINDEX not gzip: %v", err)
|
|
||||||
}
|
|
||||||
tarBytes, _ := io.ReadAll(zr)
|
|
||||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
|
||||||
var index string
|
|
||||||
for {
|
|
||||||
hdr, err := tr.Next()
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("APKINDEX not tar: %v", err)
|
|
||||||
}
|
|
||||||
if hdr.Name == "APKINDEX" {
|
|
||||||
b, _ := io.ReadAll(tr)
|
|
||||||
index = string(b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, want := range []string{"P:e2e-testpkg", "V:1.0-r0", "A:x86_64", "C:Q1", "S:", "I:"} {
|
|
||||||
if !strings.Contains(index, want) {
|
|
||||||
t.Fatalf("APKINDEX missing %q:\n%s", want, index)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The .apk downloads back byte-identical from its arch path.
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-alpine/x86_64/e2e-testpkg-1.0-r0.apk"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("download apk: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(body, apk) {
|
|
||||||
t.Fatalf("apk content mismatch")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
# Mock upstream A for the multi-base_url e2e: any path returns a constant,
|
|
||||||
# upstream-identifying body so round-robin distribution is observable.
|
|
||||||
server {
|
|
||||||
listen 80;
|
|
||||||
location / {
|
|
||||||
default_type text/plain;
|
|
||||||
return 200 "UPSTREAM-A";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
# Mock upstream B for the multi-base_url e2e (see a.conf).
|
|
||||||
server {
|
|
||||||
listen 80;
|
|
||||||
location / {
|
|
||||||
default_type text/plain;
|
|
||||||
return 200 "UPSTREAM-B";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,163 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// mockUpstreamA/B are the constant-body upstreams (see docker-compose.e2e.yml)
|
|
||||||
// that let the round-robin test observe which mirror served each request.
|
|
||||||
func mockUpstreamA() string {
|
|
||||||
if v := os.Getenv("MOCK_UPSTREAM_A_INTERNAL"); v != "" {
|
|
||||||
return strings.TrimRight(v, "/")
|
|
||||||
}
|
|
||||||
return "http://mockupstreama"
|
|
||||||
}
|
|
||||||
|
|
||||||
func mockUpstreamB() string {
|
|
||||||
if v := os.Getenv("MOCK_UPSTREAM_B_INTERNAL"); v != "" {
|
|
||||||
return strings.TrimRight(v, "/")
|
|
||||||
}
|
|
||||||
return "http://mockupstreamb"
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestMultiBaseURLRoundRobin configures an rpm remote with base_url = mirror A
|
|
||||||
// and mirrorlist = [mirror B] and drives distinct paths through it, asserting
|
|
||||||
// both mirrors serve traffic. Each path is a cache miss, so every request reaches
|
|
||||||
// upstream and the round-robin cursor alternates mirrors.
|
|
||||||
func TestMultiBaseURLRoundRobin(t *testing.T) {
|
|
||||||
name := "e2e-rr"
|
|
||||||
createRepo(t, fmt.Sprintf(`{
|
|
||||||
"name": %q,
|
|
||||||
"package_type": "rpm",
|
|
||||||
"repo_type": "remote",
|
|
||||||
"base_url": %q,
|
|
||||||
"mirrorlist": [%q],
|
|
||||||
"stale_on_error": false
|
|
||||||
}`, name, mockUpstreamA(), mockUpstreamB()))
|
|
||||||
defer deleteRepo(t, name)
|
|
||||||
|
|
||||||
seenA, seenB := false, false
|
|
||||||
const n = 12
|
|
||||||
for i := 0; i < n; i++ {
|
|
||||||
url := api(fmt.Sprintf("/api/v1/remote/%s/rr/%d", name, i))
|
|
||||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("request %d: status %d: %s", i, resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
switch strings.TrimSpace(string(body)) {
|
|
||||||
case "UPSTREAM-A":
|
|
||||||
seenA = true
|
|
||||||
case "UPSTREAM-B":
|
|
||||||
seenB = true
|
|
||||||
default:
|
|
||||||
t.Fatalf("request %d: unexpected body %q", i, body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !seenA || !seenB {
|
|
||||||
t.Fatalf("round-robin did not reach both upstreams: A=%v B=%v", seenA, seenB)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestMultiBaseURLFailover points a two-mirror remote at a dead primary and a
|
|
||||||
// healthy secondary and asserts every request still succeeds via the secondary.
|
|
||||||
func TestMultiBaseURLFailover(t *testing.T) {
|
|
||||||
name := "e2e-failover"
|
|
||||||
createRepo(t, fmt.Sprintf(`{
|
|
||||||
"name": %q,
|
|
||||||
"package_type": "rpm",
|
|
||||||
"repo_type": "remote",
|
|
||||||
"base_url": "http://mockupstream-dead:80",
|
|
||||||
"mirrorlist": [%q],
|
|
||||||
"stale_on_error": false
|
|
||||||
}`, name, mockUpstreamB()))
|
|
||||||
defer deleteRepo(t, name)
|
|
||||||
|
|
||||||
for i := 0; i < 6; i++ {
|
|
||||||
url := api(fmt.Sprintf("/api/v1/remote/%s/fo/%d", name, i))
|
|
||||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("request %d: dead primary broke fetch: status %d: %s", i, resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if got := strings.TrimSpace(string(body)); got != "UPSTREAM-B" {
|
|
||||||
t.Fatalf("request %d: body %q, want UPSTREAM-B (served via failover)", i, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSingleBaseURLRegression asserts a remote with no mirrorlist works exactly
|
|
||||||
// as before the mirrorlist change.
|
|
||||||
func TestSingleBaseURLRegression(t *testing.T) {
|
|
||||||
name := "e2e-single"
|
|
||||||
createRepo(t, fmt.Sprintf(`{
|
|
||||||
"name": %q,
|
|
||||||
"package_type": "rpm",
|
|
||||||
"repo_type": "remote",
|
|
||||||
"base_url": %q,
|
|
||||||
"stale_on_error": false
|
|
||||||
}`, name, mockUpstreamA()))
|
|
||||||
defer deleteRepo(t, name)
|
|
||||||
|
|
||||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/remote/"+name+"/solo/0"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("single-url fetch: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if got := strings.TrimSpace(string(body)); got != "UPSTREAM-A" {
|
|
||||||
t.Fatalf("single-url body %q, want UPSTREAM-A", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestMultiBaseURLDnfFailover drives a real dnf (stock rockylinux container) at
|
|
||||||
// a two-mirror rpm remote whose primary is dead: makecache + install must
|
|
||||||
// succeed via the live secondary mirror, proving a dead mirror does not break a
|
|
||||||
// real package-manager client. Requires the compose network and internal API
|
|
||||||
// URL exported by scripts/docker-e2e.sh; skipped when run standalone.
|
|
||||||
func TestMultiBaseURLDnfFailover(t *testing.T) {
|
|
||||||
network := os.Getenv("COMPOSE_NETWORK")
|
|
||||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
|
||||||
if network == "" || internal == "" {
|
|
||||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
|
||||||
}
|
|
||||||
if _, err := exec.LookPath("docker"); err != nil {
|
|
||||||
t.Skip("docker not available on the test host")
|
|
||||||
}
|
|
||||||
|
|
||||||
name := "e2e-dnf-failover"
|
|
||||||
// Primary base_url is dead; the live mirror serves the real yum repo under
|
|
||||||
// fixtures/rpm-mirror via the shared mock upstream.
|
|
||||||
createRepo(t, fmt.Sprintf(`{
|
|
||||||
"name": %q,
|
|
||||||
"package_type": "rpm",
|
|
||||||
"repo_type": "remote",
|
|
||||||
"base_url": "http://mockupstream-dead:80",
|
|
||||||
"mirrorlist": [%q],
|
|
||||||
"stale_on_error": false
|
|
||||||
}`, name, mockUpstream()))
|
|
||||||
defer deleteRepo(t, name)
|
|
||||||
|
|
||||||
repoURL := strings.TrimRight(internal, "/") + "/api/v1/remote/" + name + "/rpm-mirror"
|
|
||||||
repoConf := fmt.Sprintf("[dnffo]\nname=dnffo\nbaseurl=%s\nenabled=1\ngpgcheck=0\nsslverify=0\nmetadata_expire=0\n", repoURL)
|
|
||||||
script := "set -euo pipefail; " +
|
|
||||||
"printf '%s' \"$REPO\" > /etc/yum.repos.d/dnffo.repo; " +
|
|
||||||
"dnf -y --disablerepo='*' --enablerepo=dnffo makecache; " +
|
|
||||||
"dnf -y --disablerepo='*' --enablerepo=dnffo install e2e-testpkg; " +
|
|
||||||
"rpm -q e2e-testpkg"
|
|
||||||
|
|
||||||
cmd := exec.Command("docker", "run", "--rm",
|
|
||||||
"--network", network,
|
|
||||||
"-e", "REPO="+repoConf,
|
|
||||||
"rockylinux:9", "bash", "-c", script)
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("real dnf install through a dead primary mirror failed: %v\n%s", err, out)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(out), "e2e-testpkg-1.0-1") {
|
|
||||||
t.Fatalf("dnf did not install the expected package via failover; output:\n%s", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestLocalNPMPublishInstall drives a stock npm client against a local npm
|
|
||||||
// repo: publish unscoped and scoped packages, install and require() them in a
|
|
||||||
// fresh project, publish a second version, and reject a republish.
|
|
||||||
func TestLocalNPMPublishInstall(t *testing.T) {
|
|
||||||
network := os.Getenv("COMPOSE_NETWORK")
|
|
||||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
|
||||||
if network == "" || internal == "" {
|
|
||||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
|
||||||
}
|
|
||||||
if _, err := exec.LookPath("docker"); err != nil {
|
|
||||||
t.Skip("docker not available on the test host")
|
|
||||||
}
|
|
||||||
|
|
||||||
name := fmt.Sprintf("e2e-npm-local-%d", time.Now().UnixNano())
|
|
||||||
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name))
|
|
||||||
defer deleteRepo(t, name)
|
|
||||||
|
|
||||||
reg := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + "/"
|
|
||||||
script := `set -euo pipefail
|
|
||||||
printf 'registry=%s\n%s:_authToken=unused\n' "$REG" "${REG#http:}" > ~/.npmrc
|
|
||||||
mk() { d="/w/src/$1-$2"; mkdir -p "$d"; cd "$d"
|
|
||||||
printf '{"name":"%s","version":"%s","main":"index.js"}' "$1" "$2" > package.json
|
|
||||||
echo "module.exports = '$1@$2';" > index.js; }
|
|
||||||
mk e2e-npm-plain 1.0.0 && npm publish
|
|
||||||
mk @e2e/npm-scoped 1.0.0 && npm publish
|
|
||||||
mk e2e-npm-plain 1.0.0
|
|
||||||
if npm publish 2>/dev/null; then echo "republish accepted"; exit 1; fi
|
|
||||||
mkdir -p /w/app && cd /w/app && npm init -y >/dev/null
|
|
||||||
npm install --no-audit --cache /tmp/fresh-cache e2e-npm-plain @e2e/npm-scoped
|
|
||||||
node -e "console.log('REQ', require('e2e-npm-plain'), require('@e2e/npm-scoped'))"
|
|
||||||
mk e2e-npm-plain 1.1.0 && npm publish
|
|
||||||
echo "VERSIONS $(npm view e2e-npm-plain versions --json | tr -d ' \n')"
|
|
||||||
echo "LATEST $(npm view e2e-npm-plain dist-tags.latest)"`
|
|
||||||
|
|
||||||
out, err := exec.Command("docker", "run", "--rm", "--network", network,
|
|
||||||
"-e", "REG="+reg, "node:22", "bash", "-c", script).CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("npm client run failed: %v\n%s", err, out)
|
|
||||||
}
|
|
||||||
for _, want := range []string{
|
|
||||||
"REQ e2e-npm-plain@1.0.0 @e2e/npm-scoped@1.0.0",
|
|
||||||
`VERSIONS ["1.0.0","1.1.0"]`,
|
|
||||||
"LATEST 1.1.0",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(string(out), want) {
|
|
||||||
t.Fatalf("missing %q in npm output:\n%s", want, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestVirtualNPMInstall publishes to two local npm repos and drives a stock
|
|
||||||
// npm client through a virtual over both: merged versions and dist-tags, and
|
|
||||||
// an install whose tarballs resolve via the virtual's member redirect.
|
|
||||||
func TestVirtualNPMInstall(t *testing.T) {
|
|
||||||
network := os.Getenv("COMPOSE_NETWORK")
|
|
||||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
|
||||||
if network == "" || internal == "" {
|
|
||||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
|
||||||
}
|
|
||||||
if _, err := exec.LookPath("docker"); err != nil {
|
|
||||||
t.Skip("docker not available on the test host")
|
|
||||||
}
|
|
||||||
|
|
||||||
id := time.Now().UnixNano()
|
|
||||||
a, b, virt := fmt.Sprintf("e2e-npm-a-%d", id), fmt.Sprintf("e2e-npm-b-%d", id), fmt.Sprintf("e2e-npm-v-%d", id)
|
|
||||||
for _, name := range []string{a, b} {
|
|
||||||
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name))
|
|
||||||
defer deleteRepo(t, name)
|
|
||||||
}
|
|
||||||
createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","members":[%q,%q]}`, virt, a, b))
|
|
||||||
defer deleteVirtual(t, virt)
|
|
||||||
|
|
||||||
base := strings.TrimRight(internal, "/")
|
|
||||||
script := `set -euo pipefail
|
|
||||||
pub() { d="/w/src/$2-$3-$1"; mkdir -p "$d"; cd "$d"
|
|
||||||
printf '{"name":"%s","version":"%s","main":"index.js"}' "$2" "$3" > package.json
|
|
||||||
echo "module.exports = '$1:$2@$3';" > index.js
|
|
||||||
printf 'registry=%s\n%s:_authToken=unused\n' "$BASE/api/v1/local/$1/" "${BASE#http:}/api/v1/local/$1/" > .npmrc
|
|
||||||
npm publish; }
|
|
||||||
pub "$A" @e2e/virt 1.0.0
|
|
||||||
pub "$B" @e2e/virt 1.0.0
|
|
||||||
pub "$B" @e2e/virt 1.1.0
|
|
||||||
mkdir -p /w/app && cd /w/app && npm init -y >/dev/null
|
|
||||||
echo "registry=$BASE/api/v1/virtual/$V/" > .npmrc
|
|
||||||
echo "VERSIONS $(npm view @e2e/virt versions --json | tr -d ' \n')"
|
|
||||||
npm install --no-audit --cache /tmp/fresh-cache @e2e/virt@1.0.0
|
|
||||||
node -e "console.log('REQ', require('@e2e/virt'))"
|
|
||||||
echo "RESOLVED $(node -p "require('./package-lock.json').packages['node_modules/@e2e/virt'].resolved")"`
|
|
||||||
|
|
||||||
out, err := exec.Command("docker", "run", "--rm", "--network", network,
|
|
||||||
"-e", "BASE="+base, "-e", "A="+a, "-e", "B="+b, "-e", "V="+virt,
|
|
||||||
"node:22", "bash", "-c", script).CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("npm client run failed: %v\n%s", err, out)
|
|
||||||
}
|
|
||||||
for _, want := range []string{
|
|
||||||
`VERSIONS ["1.0.0","1.1.0"]`,
|
|
||||||
"REQ " + a + ":@e2e/virt@1.0.0",
|
|
||||||
"RESOLVED " + base + "/api/v1/virtual/" + virt + "/" + a + "/@e2e/virt/-/virt-1.0.0.tgz",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(string(out), want) {
|
|
||||||
t.Fatalf("missing %q in npm output:\n%s", want, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"archive/tar"
|
|
||||||
"bytes"
|
|
||||||
"compress/gzip"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const puppetHelloMetadata = `{"name":"unkin-hello","version":"0.1.0","author":"unkin","summary":"e2e","license":"Apache-2.0","source":"https://example.invalid","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0 < 10.0.0"}]}`
|
|
||||||
|
|
||||||
func puppetModuleTarball(t *testing.T) []byte {
|
|
||||||
t.Helper()
|
|
||||||
var buf bytes.Buffer
|
|
||||||
zw := gzip.NewWriter(&buf)
|
|
||||||
tw := tar.NewWriter(zw)
|
|
||||||
for name, body := range map[string]string{
|
|
||||||
"unkin-hello-0.1.0/metadata.json": puppetHelloMetadata,
|
|
||||||
"unkin-hello-0.1.0/manifests/init.pp": "class hello {}\n",
|
|
||||||
} {
|
|
||||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: time.Now()}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, _ = tw.Write([]byte(body))
|
|
||||||
}
|
|
||||||
_ = tw.Close()
|
|
||||||
_ = zw.Close()
|
|
||||||
return buf.Bytes()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLocalPuppetRealClient publishes a module to a puppet local and installs
|
|
||||||
// it with the real r10k and puppet module tool. The stdlib dependency is not on
|
|
||||||
// a local-only forge, so puppet runs with --ignore-dependencies.
|
|
||||||
func TestLocalPuppetRealClient(t *testing.T) {
|
|
||||||
network := os.Getenv("COMPOSE_NETWORK")
|
|
||||||
internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
|
|
||||||
if network == "" || internal == "" {
|
|
||||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
|
||||||
}
|
|
||||||
if _, err := exec.LookPath("docker"); err != nil {
|
|
||||||
t.Skip("docker not available on the test host")
|
|
||||||
}
|
|
||||||
|
|
||||||
createRepo(t, `{"name":"local-puppet","package_type":"puppet","repo_type":"local"}`)
|
|
||||||
defer deleteRepo(t, "local-puppet")
|
|
||||||
|
|
||||||
upload := api("/api/v2/remotes/local-puppet/files/unkin-hello-0.1.0.tar.gz")
|
|
||||||
defer doRequest(t, http.MethodDelete, api("/api/v2/remotes/local-puppet/files/v3/files/unkin-hello-0.1.0.tar.gz"), nil, "")
|
|
||||||
tarball := puppetModuleTarball(t)
|
|
||||||
if resp, body := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusCreated {
|
|
||||||
t.Fatalf("upload: %d %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
if resp, _ := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusConflict {
|
|
||||||
t.Fatalf("re-upload of an existing version: %d, want 409", resp.StatusCode)
|
|
||||||
}
|
|
||||||
if resp, body := getEventually(t, api("/api/v1/local/local-puppet/v3/releases/unkin-hello-0.1.0"), 15*time.Second); resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"file_md5"`) {
|
|
||||||
t.Fatalf("release never indexed: %d %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
|
|
||||||
forge := internal + "/api/v1/local/local-puppet"
|
|
||||||
run := func(image, script string) string {
|
|
||||||
t.Helper()
|
|
||||||
out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", image, "-c", "set -eu; "+script).CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s failed: %v\n%s", image, err, out)
|
|
||||||
}
|
|
||||||
return string(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
out := run("ghcr.io/voxpupuli/r10k:latest", "cd /tmp && printf 'forge:\\n baseurl: "+forge+"\\n' > r10k.yaml && "+
|
|
||||||
"printf \"forge '"+forge+"'\\nmod 'unkin-hello', '0.1.0'\\n\" > Puppetfile && "+
|
|
||||||
"r10k puppetfile install -c r10k.yaml --verbose && cat modules/hello/metadata.json")
|
|
||||||
if !strings.Contains(out, `"unkin-hello"`) {
|
|
||||||
t.Fatalf("r10k did not install unkin-hello:\n%s", out)
|
|
||||||
}
|
|
||||||
|
|
||||||
out = run("ghcr.io/openvoxproject/openvoxagent:latest", "export PATH=/opt/puppetlabs/bin:$PATH; puppet module install unkin-hello --module_repository "+forge+" --ignore-dependencies --target-dir /tmp/mods && puppet module list --color=false --modulepath /tmp/mods")
|
|
||||||
if !strings.Contains(out, "unkin-hello (v0.1.0)") {
|
|
||||||
t.Fatalf("puppet module install did not install unkin-hello:\n%s", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,134 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestHealth(t *testing.T) {
|
|
||||||
resp, body := doRequest(t, http.MethodGet, api("/health"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("health: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRemoteLifecycle covers add/change/delete for a remote repository.
|
|
||||||
func TestRemoteLifecycle(t *testing.T) {
|
|
||||||
createRepo(t, `{
|
|
||||||
"name": "crud-remote",
|
|
||||||
"package_type": "generic",
|
|
||||||
"repo_type": "remote",
|
|
||||||
"base_url": "https://example.com",
|
|
||||||
"mutable_ttl": 600,
|
|
||||||
"stale_on_error": true
|
|
||||||
}`)
|
|
||||||
defer deleteRepo(t, "crud-remote")
|
|
||||||
|
|
||||||
got := getRepo(t, "crud-remote")
|
|
||||||
if got["base_url"] != "https://example.com" || got["mutable_ttl"].(float64) != 600 {
|
|
||||||
t.Fatalf("unexpected created remote: %v", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// change
|
|
||||||
resp, body := doRequest(t, http.MethodPut, api("/api/v2/remotes/crud-remote"), []byte(`{
|
|
||||||
"package_type": "generic",
|
|
||||||
"base_url": "https://updated.example.com",
|
|
||||||
"mutable_ttl": 120,
|
|
||||||
"stale_on_error": true
|
|
||||||
}`), "application/json")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("update remote: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
got = getRepo(t, "crud-remote")
|
|
||||||
if got["base_url"] != "https://updated.example.com" || got["mutable_ttl"].(float64) != 120 {
|
|
||||||
t.Fatalf("update not applied: %v", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// delete
|
|
||||||
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-remote"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusNoContent {
|
|
||||||
t.Fatalf("delete remote: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
resp, _ = doRequest(t, http.MethodGet, api("/api/v2/remotes/crud-remote"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusNotFound {
|
|
||||||
t.Fatalf("expected 404 after delete, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLocalLifecycle covers add/delete for a local repository.
|
|
||||||
func TestLocalLifecycle(t *testing.T) {
|
|
||||||
createRepo(t, `{
|
|
||||||
"name": "crud-local",
|
|
||||||
"package_type": "generic",
|
|
||||||
"repo_type": "local"
|
|
||||||
}`)
|
|
||||||
defer deleteRepo(t, "crud-local")
|
|
||||||
|
|
||||||
got := getRepo(t, "crud-local")
|
|
||||||
if got["repo_type"] != "local" {
|
|
||||||
t.Fatalf("expected repo_type local, got %v", got["repo_type"])
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, _ := doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-local"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusNoContent {
|
|
||||||
t.Fatalf("delete local: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestVirtualLifecycle covers add/change/delete for a virtual repository.
|
|
||||||
func TestVirtualLifecycle(t *testing.T) {
|
|
||||||
createRepo(t, `{"name":"vmem-a","package_type":"helm","repo_type":"remote","base_url":"https://a.example.com","stale_on_error":true}`)
|
|
||||||
createRepo(t, `{"name":"vmem-b","package_type":"helm","repo_type":"remote","base_url":"https://b.example.com","stale_on_error":true}`)
|
|
||||||
defer deleteRepo(t, "vmem-a")
|
|
||||||
defer deleteRepo(t, "vmem-b")
|
|
||||||
|
|
||||||
createVirtual(t, `{
|
|
||||||
"name": "crud-virtual",
|
|
||||||
"package_type": "helm",
|
|
||||||
"members": ["vmem-a"]
|
|
||||||
}`)
|
|
||||||
defer deleteVirtual(t, "crud-virtual")
|
|
||||||
|
|
||||||
// change members
|
|
||||||
resp, body := doRequest(t, http.MethodPut, api("/api/v2/virtuals/crud-virtual"), []byte(`{
|
|
||||||
"package_type": "helm",
|
|
||||||
"members": ["vmem-a", "vmem-b"]
|
|
||||||
}`), "application/json")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("update virtual: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, body = doRequest(t, http.MethodGet, api("/api/v2/virtuals/crud-virtual"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("get virtual: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
var v map[string]any
|
|
||||||
if err := json.Unmarshal(body, &v); err != nil {
|
|
||||||
t.Fatalf("decode virtual: %v", err)
|
|
||||||
}
|
|
||||||
members, _ := v["members"].([]any)
|
|
||||||
if len(members) != 2 {
|
|
||||||
t.Fatalf("expected 2 members after update, got %v", v["members"])
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/virtuals/crud-virtual"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusNoContent {
|
|
||||||
t.Fatalf("delete virtual: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func getRepo(t *testing.T, name string) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
resp, body := doRequest(t, http.MethodGet, api("/api/v2/remotes/"+name), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("get remote %s: status %d: %s", name, resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
var m map[string]any
|
|
||||||
if err := json.Unmarshal(body, &m); err != nil {
|
|
||||||
t.Fatalf("decode remote %s: %v", name, err)
|
|
||||||
}
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
@@ -1,138 +0,0 @@
|
|||||||
//go:build dockere2e
|
|
||||||
|
|
||||||
package e2edocker
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"compress/gzip"
|
|
||||||
"encoding/xml"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
|
|
||||||
// checks that a virtual over them serves a merged simple index.
|
|
||||||
func TestVirtualPyPIMerge(t *testing.T) {
|
|
||||||
createRepo(t, `{"name":"pmerge-a","package_type":"pypi","repo_type":"local"}`)
|
|
||||||
createRepo(t, `{"name":"pmerge-b","package_type":"pypi","repo_type":"local"}`)
|
|
||||||
defer deleteRepo(t, "pmerge-a")
|
|
||||||
defer deleteRepo(t, "pmerge-b")
|
|
||||||
|
|
||||||
uploadFile(t, "pmerge-a", "foo-1.0-py3-none-any.whl", fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl"), "application/zip")
|
|
||||||
uploadFile(t, "pmerge-b", "bar-2.0-py3-none-any.whl", []byte("bar wheel payload"), "application/zip")
|
|
||||||
|
|
||||||
createVirtual(t, `{"name":"pmerge-v","package_type":"pypi","members":["pmerge-a","pmerge-b"]}`)
|
|
||||||
defer deleteVirtual(t, "pmerge-v")
|
|
||||||
|
|
||||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/pmerge-v/simple/"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("virtual simple index: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
s := string(body)
|
|
||||||
if !strings.Contains(s, "foo") || !strings.Contains(s, "bar") {
|
|
||||||
t.Fatalf("merged index missing a member package (want foo and bar): %s", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestVirtualHelmMerge points two helm remotes at mock index.yaml documents
|
|
||||||
// with distinct charts and checks the virtual merges both into one index.
|
|
||||||
func TestVirtualHelmMerge(t *testing.T) {
|
|
||||||
createRepo(t, `{"name":"hmerge-a","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-a","stale_on_error":true}`)
|
|
||||||
createRepo(t, `{"name":"hmerge-b","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-b","stale_on_error":true}`)
|
|
||||||
defer deleteRepo(t, "hmerge-a")
|
|
||||||
defer deleteRepo(t, "hmerge-b")
|
|
||||||
|
|
||||||
createVirtual(t, `{"name":"hmerge-v","package_type":"helm","members":["hmerge-a","hmerge-b"]}`)
|
|
||||||
defer deleteVirtual(t, "hmerge-v")
|
|
||||||
|
|
||||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/hmerge-v/index.yaml"), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("virtual index.yaml: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
s := string(body)
|
|
||||||
if !strings.Contains(s, "alpha") || !strings.Contains(s, "beta") {
|
|
||||||
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestVirtualRPMMerge merges a remote rpm repo and a local rpm repo carrying the
|
|
||||||
// same package: the first member wins the duplicate, its package download
|
|
||||||
// routes through the virtual, and a real dnf can consume the merged repo.
|
|
||||||
func TestVirtualRPMMerge(t *testing.T) {
|
|
||||||
createRepo(t, `{"name":"vrpm-remote","package_type":"rpm","repo_type":"remote","base_url":"`+mockUpstream()+`/rpm-mirror","stale_on_error":true}`)
|
|
||||||
createRepo(t, `{"name":"vrpm-local","package_type":"rpm","repo_type":"local"}`)
|
|
||||||
defer deleteRepo(t, "vrpm-remote")
|
|
||||||
defer deleteRepo(t, "vrpm-local")
|
|
||||||
|
|
||||||
pkg := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
|
|
||||||
uploadFile(t, "vrpm-local", "e2e-testpkg-1.0-1.noarch.rpm", pkg, "application/x-rpm")
|
|
||||||
|
|
||||||
createVirtual(t, `{"name":"vrpm","package_type":"rpm","members":["vrpm-remote","vrpm-local"]}`)
|
|
||||||
defer deleteVirtual(t, "vrpm")
|
|
||||||
|
|
||||||
resp, body := getEventually(t, api("/api/v1/virtual/vrpm/repodata/repomd.xml"), 15*time.Second)
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("virtual repomd.xml: status %d: %s", resp.StatusCode, body)
|
|
||||||
}
|
|
||||||
var md struct {
|
|
||||||
Data []struct {
|
|
||||||
Type string `xml:"type,attr"`
|
|
||||||
Location struct {
|
|
||||||
Href string `xml:"href,attr"`
|
|
||||||
} `xml:"location"`
|
|
||||||
} `xml:"data"`
|
|
||||||
}
|
|
||||||
if err := xml.Unmarshal(body, &md); err != nil {
|
|
||||||
t.Fatalf("parse repomd.xml: %v\n%s", err, body)
|
|
||||||
}
|
|
||||||
var primary []byte
|
|
||||||
for _, d := range md.Data {
|
|
||||||
if d.Type == "primary" {
|
|
||||||
resp, gz := doRequest(t, http.MethodGet, api("/api/v1/virtual/vrpm/"+d.Location.Href), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("primary: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
r, err := gzip.NewReader(bytes.NewReader(gz))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
primary, _ = io.ReadAll(r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
href := "vrpm-remote/Packages/e2e-testpkg-1.0-1.noarch.rpm"
|
|
||||||
if strings.Count(string(primary), "<name>e2e-testpkg</name>") != 1 || !strings.Contains(string(primary), `href="`+href+`"`) {
|
|
||||||
t.Fatalf("merged primary should hold one e2e-testpkg owned by the first member:\n%s", primary)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, got := doRequest(t, http.MethodGet, api("/api/v1/virtual/vrpm/"+href), nil, "")
|
|
||||||
if resp.StatusCode != http.StatusOK || !bytes.Equal(got, pkg) {
|
|
||||||
t.Fatalf("package via virtual: status %d, %d bytes (want %d)", resp.StatusCode, len(got), len(pkg))
|
|
||||||
}
|
|
||||||
|
|
||||||
network := os.Getenv("COMPOSE_NETWORK")
|
|
||||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
|
||||||
if network == "" || internal == "" {
|
|
||||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; skipping real dnf")
|
|
||||||
}
|
|
||||||
if _, err := exec.LookPath("docker"); err != nil {
|
|
||||||
t.Skip("docker not available on the test host")
|
|
||||||
}
|
|
||||||
repoConf := fmt.Sprintf("[vrpm]\nname=vrpm\nbaseurl=%s/api/v1/virtual/vrpm/\nenabled=1\ngpgcheck=0\nmetadata_expire=0\n", strings.TrimRight(internal, "/"))
|
|
||||||
script := "set -euo pipefail; " +
|
|
||||||
"printf '%s' \"$REPO\" > /etc/yum.repos.d/vrpm.repo; " +
|
|
||||||
"dnf -y --disablerepo='*' --enablerepo=vrpm makecache; " +
|
|
||||||
"dnf -y --disablerepo='*' --enablerepo=vrpm list --available; " +
|
|
||||||
"dnf -y --disablerepo='*' --enablerepo=vrpm install e2e-testpkg; " +
|
|
||||||
"rpm -q e2e-testpkg"
|
|
||||||
out, err := exec.Command("docker", "run", "--rm", "--network", network, "-e", "REPO="+repoConf,
|
|
||||||
"rockylinux:9", "bash", "-c", script).CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("real dnf against rpm virtual failed: %v\n%s", err, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-137
@@ -1,137 +0,0 @@
|
|||||||
//go:build e2e
|
|
||||||
|
|
||||||
package e2e
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/testcontainers/testcontainers-go"
|
|
||||||
tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres"
|
|
||||||
tcredis "github.com/testcontainers/testcontainers-go/modules/redis"
|
|
||||||
"github.com/testcontainers/testcontainers-go/wait"
|
|
||||||
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/server"
|
|
||||||
)
|
|
||||||
|
|
||||||
var baseURL string
|
|
||||||
|
|
||||||
func TestMain(m *testing.M) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
pgContainer, err := tcpostgres.Run(ctx,
|
|
||||||
"postgres:17-alpine",
|
|
||||||
tcpostgres.WithDatabase("artifacts"),
|
|
||||||
tcpostgres.WithUsername("artifacts"),
|
|
||||||
tcpostgres.WithPassword("artifacts123"),
|
|
||||||
testcontainers.WithWaitStrategy(
|
|
||||||
wait.ForListeningPort("5432/tcp").WithStartupTimeout(30*time.Second),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("postgres: %v", err)
|
|
||||||
}
|
|
||||||
defer pgContainer.Terminate(ctx)
|
|
||||||
|
|
||||||
redisContainer, err := tcredis.Run(ctx,
|
|
||||||
"redis:7-alpine",
|
|
||||||
testcontainers.WithWaitStrategy(
|
|
||||||
wait.ForListeningPort("6379/tcp").WithStartupTimeout(30*time.Second),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("redis: %v", err)
|
|
||||||
}
|
|
||||||
defer redisContainer.Terminate(ctx)
|
|
||||||
|
|
||||||
minioContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
|
||||||
ContainerRequest: testcontainers.ContainerRequest{
|
|
||||||
Image: "minio/minio:latest",
|
|
||||||
ExposedPorts: []string{"9000/tcp"},
|
|
||||||
Cmd: []string{"server", "/data"},
|
|
||||||
Env: map[string]string{
|
|
||||||
"MINIO_ROOT_USER": "minioadmin",
|
|
||||||
"MINIO_ROOT_PASSWORD": "minioadmin",
|
|
||||||
},
|
|
||||||
WaitingFor: wait.ForHTTP("/minio/health/live").WithPort("9000/tcp").WithStartupTimeout(30 * time.Second),
|
|
||||||
},
|
|
||||||
Started: true,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("minio: %v", err)
|
|
||||||
}
|
|
||||||
defer minioContainer.Terminate(ctx)
|
|
||||||
|
|
||||||
pgHost, _ := pgContainer.Host(ctx)
|
|
||||||
pgPort, _ := pgContainer.MappedPort(ctx, "5432/tcp")
|
|
||||||
redisHost, _ := redisContainer.Host(ctx)
|
|
||||||
redisPort, _ := redisContainer.MappedPort(ctx, "6379/tcp")
|
|
||||||
minioHost, _ := minioContainer.Host(ctx)
|
|
||||||
minioPort, _ := minioContainer.MappedPort(ctx, "9000/tcp")
|
|
||||||
|
|
||||||
os.Setenv("DBHOST", pgHost)
|
|
||||||
os.Setenv("DBPORT", pgPort.Port())
|
|
||||||
os.Setenv("DBUSER", "artifacts")
|
|
||||||
os.Setenv("DBPASS", "artifacts123")
|
|
||||||
os.Setenv("DBNAME", "artifacts")
|
|
||||||
os.Setenv("DBSSL", "disable")
|
|
||||||
os.Setenv("REDIS_URL", fmt.Sprintf("redis://%s:%s", redisHost, redisPort.Port()))
|
|
||||||
os.Setenv("MINIO_ENDPOINT", fmt.Sprintf("%s:%s", minioHost, minioPort.Port()))
|
|
||||||
os.Setenv("MINIO_ACCESS_KEY", "minioadmin")
|
|
||||||
os.Setenv("MINIO_SECRET_KEY", "minioadmin")
|
|
||||||
os.Setenv("MINIO_BUCKET", "artifacts-test")
|
|
||||||
os.Setenv("MINIO_SECURE", "false")
|
|
||||||
os.Setenv("LISTEN_ADDR", ":0")
|
|
||||||
|
|
||||||
cfg, err := config.Load()
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("config: %v", err)
|
|
||||||
}
|
|
||||||
cfg.ListenAddr = "127.0.0.1:0"
|
|
||||||
|
|
||||||
srv, err := server.New(cfg, "e2e-test")
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("server: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
srvCtx, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
addr := startServer(srvCtx, srv)
|
|
||||||
baseURL = "http://" + addr
|
|
||||||
|
|
||||||
code := m.Run()
|
|
||||||
cancel()
|
|
||||||
os.Exit(code)
|
|
||||||
}
|
|
||||||
|
|
||||||
func startServer(ctx context.Context, srv *server.Server) string {
|
|
||||||
ln, err := findListener()
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("listener: %v", err)
|
|
||||||
}
|
|
||||||
addr := ln.Addr().String()
|
|
||||||
|
|
||||||
go srv.RunOnListener(ctx, ln)
|
|
||||||
|
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
resp, err := http.Get("http://" + addr + "/health")
|
|
||||||
if err == nil && resp.StatusCode == 200 {
|
|
||||||
resp.Body.Close()
|
|
||||||
return addr
|
|
||||||
}
|
|
||||||
if resp != nil {
|
|
||||||
resp.Body.Close()
|
|
||||||
}
|
|
||||||
time.Sleep(50 * time.Millisecond)
|
|
||||||
}
|
|
||||||
log.Fatalf("server did not start in time at %s", addr)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
@@ -1,182 +0,0 @@
|
|||||||
//go:build e2e
|
|
||||||
|
|
||||||
package e2e
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
|
||||||
)
|
|
||||||
|
|
||||||
// assertHeadMatchesGet checks HEAD returns GET's status, type and length with no body.
|
|
||||||
func assertHeadMatchesGet(t *testing.T, url string) {
|
|
||||||
t.Helper()
|
|
||||||
get, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GET: %v", err)
|
|
||||||
}
|
|
||||||
body, _ := io.ReadAll(get.Body)
|
|
||||||
get.Body.Close()
|
|
||||||
if get.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("GET %s: %d %s", url, get.StatusCode, body)
|
|
||||||
}
|
|
||||||
|
|
||||||
head, err := http.Head(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("HEAD: %v", err)
|
|
||||||
}
|
|
||||||
hb, _ := io.ReadAll(head.Body)
|
|
||||||
head.Body.Close()
|
|
||||||
if head.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("HEAD %s: got %d, want 200", url, head.StatusCode)
|
|
||||||
}
|
|
||||||
if len(hb) != 0 {
|
|
||||||
t.Errorf("HEAD returned a %d-byte body", len(hb))
|
|
||||||
}
|
|
||||||
for _, h := range []string{"Content-Type", "Cache-Control", "Vary"} {
|
|
||||||
if head.Header.Get(h) != get.Header.Get(h) {
|
|
||||||
t.Errorf("HEAD %s %q, GET %q", h, head.Header.Get(h), get.Header.Get(h))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if head.ContentLength != int64(len(body)) {
|
|
||||||
t.Errorf("HEAD Content-Length %d, GET body %d bytes", head.ContentLength, len(body))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProxyHeadNuGetRewritten(t *testing.T) {
|
|
||||||
var upstream *httptest.Server
|
|
||||||
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
json.NewEncoder(w).Encode(map[string]any{"version": "3.0.0", "resources": []map[string]string{
|
|
||||||
{"@id": upstream.URL + "/v3-flatcontainer/", "@type": "PackageBaseAddress/3.0.0"},
|
|
||||||
}})
|
|
||||||
}))
|
|
||||||
defer upstream.Close()
|
|
||||||
|
|
||||||
createRemote(t, mustFormat(`{"name": "head-nuget", "package_type": "nuget", "base_url": %q}`, upstream.URL+"/v3/index.json"))
|
|
||||||
defer deleteRemote(t, "head-nuget")
|
|
||||||
|
|
||||||
url := apiURL("/api/v1/remote/head-nuget/index.json")
|
|
||||||
assertHeadMatchesGet(t, url)
|
|
||||||
|
|
||||||
req, _ := http.NewRequest(http.MethodHead, url, nil)
|
|
||||||
req.Host = "evil.example:"
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusBadRequest {
|
|
||||||
t.Errorf("HEAD with invalid Host: got %d, want 400", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProxyHeadGitHubRPMRepodata(t *testing.T) {
|
|
||||||
rpm := testsupport.MinimalRPM("demo", "1.2", "3", "x86_64")
|
|
||||||
var gh *httptest.Server
|
|
||||||
mux := http.NewServeMux()
|
|
||||||
mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if p := r.URL.Query().Get("page"); p != "" && p != "1" {
|
|
||||||
w.Write([]byte("[]"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
json.NewEncoder(w).Encode([]map[string]any{{"tag_name": "v1.2-3", "assets": []map[string]any{{
|
|
||||||
"name": "demo-1.2-3.x86_64.rpm",
|
|
||||||
"size": len(rpm),
|
|
||||||
"browser_download_url": gh.URL + "/acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm",
|
|
||||||
}}}})
|
|
||||||
})
|
|
||||||
mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("Content-Length", strconv.Itoa(len(rpm)))
|
|
||||||
w.Write(rpm)
|
|
||||||
})
|
|
||||||
gh = httptest.NewServer(mux)
|
|
||||||
defer gh.Close()
|
|
||||||
|
|
||||||
createRemote(t, mustFormat(`{"name": "head-ghrpm", "package_type": "github_rpm", "base_url": %q, "releases_remote": "github"}`, gh.URL+"/repos/acme/tools"))
|
|
||||||
defer deleteRemote(t, "head-ghrpm")
|
|
||||||
|
|
||||||
url := apiURL("/api/v1/remote/head-ghrpm/repodata/repomd.xml")
|
|
||||||
deadline := time.Now().Add(30 * time.Second)
|
|
||||||
for {
|
|
||||||
if _, status := getBody(t, url); status == http.StatusOK || time.Now().After(deadline) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
time.Sleep(500 * time.Millisecond)
|
|
||||||
}
|
|
||||||
assertHeadMatchesGet(t, url)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProxyHeadNpmUncachedPackument(t *testing.T) {
|
|
||||||
const abbreviated = "application/vnd.npm.install-v1+json"
|
|
||||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
doc := map[string]any{"name": "left-pad", "versions": map[string]any{"1.3.0": map[string]any{}}}
|
|
||||||
if r.Header.Get("Accept") != abbreviated {
|
|
||||||
doc["readme"] = strings.Repeat("full packument ", 200)
|
|
||||||
}
|
|
||||||
json.NewEncoder(w).Encode(doc)
|
|
||||||
}))
|
|
||||||
defer upstream.Close()
|
|
||||||
|
|
||||||
createRemote(t, mustFormat(`{"name": "head-npm", "package_type": "npm", "base_url": %q}`, upstream.URL))
|
|
||||||
defer deleteRemote(t, "head-npm")
|
|
||||||
|
|
||||||
url := apiURL("/api/v1/remote/head-npm/left-pad")
|
|
||||||
do := func(method string) (*http.Response, []byte) {
|
|
||||||
req, _ := http.NewRequest(method, url, nil)
|
|
||||||
req.Header.Set("Accept", abbreviated)
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s: %v", method, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
b, _ := io.ReadAll(resp.Body)
|
|
||||||
return resp, b
|
|
||||||
}
|
|
||||||
head, _ := do(http.MethodHead)
|
|
||||||
get, body := do(http.MethodGet)
|
|
||||||
if head.StatusCode != http.StatusOK || get.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("HEAD %d, GET %d", head.StatusCode, get.StatusCode)
|
|
||||||
}
|
|
||||||
if head.ContentLength != int64(len(body)) {
|
|
||||||
t.Errorf("uncached HEAD Content-Length %d, GET body %d bytes", head.ContentLength, len(body))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProxyHeadNonRewritingUsesUpstreamHead(t *testing.T) {
|
|
||||||
methods := make(chan string, 4)
|
|
||||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
methods <- r.Method
|
|
||||||
w.Header().Set("Content-Type", "text/xml")
|
|
||||||
w.Write([]byte("<repomd/>"))
|
|
||||||
}))
|
|
||||||
defer upstream.Close()
|
|
||||||
|
|
||||||
createRemote(t, mustFormat(`{"name": "head-rpm", "package_type": "rpm", "base_url": %q}`, upstream.URL))
|
|
||||||
defer deleteRemote(t, "head-rpm")
|
|
||||||
|
|
||||||
resp, err := http.Head(apiURL("/api/v1/remote/head-rpm/repodata/repomd.xml"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("HEAD: %v", err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK || resp.Header.Get("X-Artifact-Source") != "remote" {
|
|
||||||
t.Fatalf("HEAD %d source %q", resp.StatusCode, resp.Header.Get("X-Artifact-Source"))
|
|
||||||
}
|
|
||||||
close(methods)
|
|
||||||
var seen []string
|
|
||||||
for m := range methods {
|
|
||||||
seen = append(seen, m)
|
|
||||||
}
|
|
||||||
if len(seen) != 1 || seen[0] != http.MethodHead {
|
|
||||||
t.Errorf("upstream saw %v, want a single HEAD", seen)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,109 +0,0 @@
|
|||||||
//go:build e2e
|
|
||||||
|
|
||||||
package e2e
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func findListener() (net.Listener, error) {
|
|
||||||
return net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
}
|
|
||||||
|
|
||||||
func apiURL(path string) string {
|
|
||||||
return baseURL + path
|
|
||||||
}
|
|
||||||
|
|
||||||
func createRemote(t *testing.T, body string) {
|
|
||||||
t.Helper()
|
|
||||||
resp, err := http.Post(apiURL("/api/v2/remotes"), "application/json", strings.NewReader(body))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("create remote: %v", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusCreated {
|
|
||||||
b, _ := io.ReadAll(resp.Body)
|
|
||||||
t.Fatalf("create remote: status %d: %s", resp.StatusCode, b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteRemote(t *testing.T, name string) {
|
|
||||||
t.Helper()
|
|
||||||
req, _ := http.NewRequest(http.MethodDelete, apiURL("/api/v2/remotes/"+name), nil)
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("delete remote: %v", err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
func getJSON(t *testing.T, url string) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GET %s: %v", url, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
b, _ := io.ReadAll(resp.Body)
|
|
||||||
t.Fatalf("GET %s: status %d: %s", url, resp.StatusCode, b)
|
|
||||||
}
|
|
||||||
var result map[string]any
|
|
||||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
|
||||||
t.Fatalf("decode: %v", err)
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func getBody(t *testing.T, url string) ([]byte, int) {
|
|
||||||
t.Helper()
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GET %s: %v", url, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
b, _ := io.ReadAll(resp.Body)
|
|
||||||
return b, resp.StatusCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func getString(t *testing.T, url string) string {
|
|
||||||
t.Helper()
|
|
||||||
b, status := getBody(t, url)
|
|
||||||
if status != http.StatusOK {
|
|
||||||
t.Fatalf("GET %s: status %d: %s", url, status, b)
|
|
||||||
}
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertStatus(t *testing.T, url string, wantStatus int) {
|
|
||||||
t.Helper()
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GET %s: %v", url, err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != wantStatus {
|
|
||||||
t.Errorf("GET %s: got %d, want %d", url, resp.StatusCode, wantStatus)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteRequest(t *testing.T, url string) int {
|
|
||||||
t.Helper()
|
|
||||||
req, _ := http.NewRequest(http.MethodDelete, url, nil)
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("DELETE %s: %v", url, err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
return resp.StatusCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustFormat(format string, args ...any) string {
|
|
||||||
return fmt.Sprintf(format, args...)
|
|
||||||
}
|
|
||||||
@@ -1,183 +0,0 @@
|
|||||||
//go:build e2e
|
|
||||||
|
|
||||||
package e2e
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestHealth(t *testing.T) {
|
|
||||||
result := getJSON(t, apiURL("/health"))
|
|
||||||
if result["status"] != "ok" {
|
|
||||||
t.Errorf("expected ok, got %v", result["status"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRoot(t *testing.T) {
|
|
||||||
result := getJSON(t, apiURL("/"))
|
|
||||||
if result["name"] != "artifactapi" {
|
|
||||||
t.Errorf("expected artifactapi, got %v", result["name"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoteUpstreamTimeouts(t *testing.T) {
|
|
||||||
createRemote(t, `{
|
|
||||||
"name": "timeout-test",
|
|
||||||
"package_type": "generic",
|
|
||||||
"base_url": "https://example.com",
|
|
||||||
"stale_on_error": true,
|
|
||||||
"upstream_dial_timeout": 3,
|
|
||||||
"upstream_tls_timeout": 4,
|
|
||||||
"upstream_response_header_timeout": 5
|
|
||||||
}`)
|
|
||||||
defer deleteRemote(t, "timeout-test")
|
|
||||||
|
|
||||||
remote := getJSON(t, apiURL("/api/v2/remotes/timeout-test"))
|
|
||||||
for field, want := range map[string]float64{
|
|
||||||
"upstream_dial_timeout": 3,
|
|
||||||
"upstream_tls_timeout": 4,
|
|
||||||
"upstream_response_header_timeout": 5,
|
|
||||||
} {
|
|
||||||
if got, _ := remote[field].(float64); got != want {
|
|
||||||
t.Errorf("%s: got %v, want %v", field, remote[field], want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoteCRUD(t *testing.T) {
|
|
||||||
createRemote(t, `{
|
|
||||||
"name": "test-generic",
|
|
||||||
"package_type": "generic",
|
|
||||||
"base_url": "https://example.com",
|
|
||||||
"description": "test remote",
|
|
||||||
"mutable_ttl": 600,
|
|
||||||
"check_mutable": true,
|
|
||||||
"stale_on_error": true
|
|
||||||
}`)
|
|
||||||
defer deleteRemote(t, "test-generic")
|
|
||||||
|
|
||||||
remote := getJSON(t, apiURL("/api/v2/remotes/test-generic"))
|
|
||||||
if remote["name"] != "test-generic" {
|
|
||||||
t.Errorf("expected test-generic, got %v", remote["name"])
|
|
||||||
}
|
|
||||||
if remote["package_type"] != "generic" {
|
|
||||||
t.Errorf("expected generic, got %v", remote["package_type"])
|
|
||||||
}
|
|
||||||
|
|
||||||
req, _ := http.NewRequest(http.MethodPut, apiURL("/api/v2/remotes/test-generic"),
|
|
||||||
strings.NewReader(`{
|
|
||||||
"package_type": "generic",
|
|
||||||
"base_url": "https://updated.example.com",
|
|
||||||
"description": "updated",
|
|
||||||
"mutable_ttl": 300,
|
|
||||||
"check_mutable": true,
|
|
||||||
"stale_on_error": true
|
|
||||||
}`))
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("update: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
updated := getJSON(t, apiURL("/api/v2/remotes/test-generic"))
|
|
||||||
if updated["base_url"] != "https://updated.example.com" {
|
|
||||||
t.Errorf("expected updated URL, got %v", updated["base_url"])
|
|
||||||
}
|
|
||||||
|
|
||||||
status := deleteRequest(t, apiURL("/api/v2/remotes/test-generic"))
|
|
||||||
if status != http.StatusNoContent {
|
|
||||||
t.Errorf("delete: got %d, want 204", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertStatus(t, apiURL("/api/v2/remotes/test-generic"), http.StatusNotFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoteList(t *testing.T) {
|
|
||||||
createRemote(t, `{"name":"list-a","package_type":"generic","base_url":"https://a.example.com","stale_on_error":true}`)
|
|
||||||
createRemote(t, `{"name":"list-b","package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`)
|
|
||||||
defer deleteRemote(t, "list-a")
|
|
||||||
defer deleteRemote(t, "list-b")
|
|
||||||
|
|
||||||
resp, err := http.Get(apiURL("/api/v2/remotes"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
body, _ := io.ReadAll(resp.Body)
|
|
||||||
|
|
||||||
var remotes []map[string]any
|
|
||||||
json.Unmarshal(body, &remotes)
|
|
||||||
|
|
||||||
if len(remotes) < 2 {
|
|
||||||
t.Fatalf("expected at least 2 remotes, got %d", len(remotes))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestVirtualCRUD(t *testing.T) {
|
|
||||||
createRemote(t, `{"name":"virt-member-a","package_type":"helm","base_url":"https://a.example.com","stale_on_error":true}`)
|
|
||||||
createRemote(t, `{"name":"virt-member-b","package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`)
|
|
||||||
defer deleteRemote(t, "virt-member-a")
|
|
||||||
defer deleteRemote(t, "virt-member-b")
|
|
||||||
|
|
||||||
resp, err := http.Post(apiURL("/api/v2/virtuals"), "application/json",
|
|
||||||
strings.NewReader(`{
|
|
||||||
"name": "test-virtual",
|
|
||||||
"package_type": "helm",
|
|
||||||
"description": "test virtual",
|
|
||||||
"members": ["virt-member-a", "virt-member-b"]
|
|
||||||
}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusCreated {
|
|
||||||
t.Fatalf("create virtual: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
virt := getJSON(t, apiURL("/api/v2/virtuals/test-virtual"))
|
|
||||||
if virt["name"] != "test-virtual" {
|
|
||||||
t.Errorf("expected test-virtual, got %v", virt["name"])
|
|
||||||
}
|
|
||||||
|
|
||||||
status := deleteRequest(t, apiURL("/api/v2/virtuals/test-virtual"))
|
|
||||||
if status != http.StatusNoContent {
|
|
||||||
t.Errorf("delete virtual: got %d, want 204", status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStatsEndpoint(t *testing.T) {
|
|
||||||
result := getJSON(t, apiURL("/api/v2/stats"))
|
|
||||||
if _, ok := result["total_remotes"]; !ok {
|
|
||||||
t.Error("expected total_remotes in stats")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHealthV2(t *testing.T) {
|
|
||||||
result := getJSON(t, apiURL("/api/v2/health"))
|
|
||||||
if result["status"] != "ok" {
|
|
||||||
t.Errorf("expected ok, got %v", result["status"])
|
|
||||||
}
|
|
||||||
if result["postgres"] != "ok" {
|
|
||||||
t.Errorf("expected postgres ok, got %v", result["postgres"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInvalidPackageType(t *testing.T) {
|
|
||||||
resp, err := http.Post(apiURL("/api/v2/remotes"), "application/json",
|
|
||||||
strings.NewReader(`{"name":"bad","package_type":"bogus","base_url":"https://x.com"}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusBadRequest {
|
|
||||||
t.Errorf("expected 400 for invalid package type, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
//go:build e2e
|
|
||||||
|
|
||||||
package e2e
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestProxyUnknownRemote(t *testing.T) {
|
|
||||||
assertStatus(t, apiURL("/api/v1/remote/nonexistent/some/path"), http.StatusNotFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProxyBlocklist(t *testing.T) {
|
|
||||||
createRemote(t, `{
|
|
||||||
"name": "blocklist-test",
|
|
||||||
"package_type": "generic",
|
|
||||||
"base_url": "https://example.com",
|
|
||||||
"blocklist": ["\\.exe$"],
|
|
||||||
"stale_on_error": true
|
|
||||||
}`)
|
|
||||||
defer deleteRemote(t, "blocklist-test")
|
|
||||||
|
|
||||||
assertStatus(t, apiURL("/api/v1/remote/blocklist-test/malware.exe"), http.StatusForbidden)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProxyHeadBlocklist(t *testing.T) {
|
|
||||||
createRemote(t, `{
|
|
||||||
"name": "head-block-test",
|
|
||||||
"package_type": "generic",
|
|
||||||
"base_url": "https://example.com",
|
|
||||||
"blocklist": ["\\.exe$"],
|
|
||||||
"stale_on_error": true
|
|
||||||
}`)
|
|
||||||
defer deleteRemote(t, "head-block-test")
|
|
||||||
|
|
||||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/head-block-test/malware.exe"), nil)
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("HEAD: %v", err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusForbidden {
|
|
||||||
t.Fatalf("HEAD blocklisted path: got %d, want 403", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProxyHeadUnknownRemote(t *testing.T) {
|
|
||||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/nonexistent/some/path"), nil)
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("HEAD: %v", err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusNotFound {
|
|
||||||
t.Fatalf("HEAD unknown remote: got %d, want 404", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProxyPatterns(t *testing.T) {
|
|
||||||
createRemote(t, `{
|
|
||||||
"name": "patterns-test",
|
|
||||||
"package_type": "generic",
|
|
||||||
"base_url": "https://example.com",
|
|
||||||
"patterns": ["^releases/"],
|
|
||||||
"stale_on_error": true
|
|
||||||
}`)
|
|
||||||
defer deleteRemote(t, "patterns-test")
|
|
||||||
|
|
||||||
assertStatus(t, apiURL("/api/v1/remote/patterns-test/uploads/file.tar.gz"), http.StatusForbidden)
|
|
||||||
}
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
//go:build e2e
|
|
||||||
|
|
||||||
package e2e
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func getForwardedHTTPS(t *testing.T, url string) *http.Response {
|
|
||||||
t.Helper()
|
|
||||||
req, _ := http.NewRequest(http.MethodGet, url, nil)
|
|
||||||
req.Header.Set("X-Forwarded-Proto", "https")
|
|
||||||
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GET %s: %v", url, err)
|
|
||||||
}
|
|
||||||
return resp
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestForwardedProtoCargoConfig(t *testing.T) {
|
|
||||||
createRemote(t, `{"name": "scheme-cargo", "package_type": "cargo", "base_url": "https://index.crates.io"}`)
|
|
||||||
defer deleteRemote(t, "scheme-cargo")
|
|
||||||
|
|
||||||
resp := getForwardedHTTPS(t, apiURL("/api/v1/remote/scheme-cargo/config.json"))
|
|
||||||
defer resp.Body.Close()
|
|
||||||
var cfg map[string]string
|
|
||||||
if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if want := "https://" + strings.TrimPrefix(baseURL, "http://") + "/api/v1/remote/scheme-cargo/"; !strings.HasPrefix(cfg["dl"], want) {
|
|
||||||
t.Errorf("dl = %q, want prefix %q", cfg["dl"], want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestForwardedProtoVirtualRedirect(t *testing.T) {
|
|
||||||
createRemote(t, `{"name": "scheme-rpm", "package_type": "rpm", "base_url": "https://rpm.example.com"}`)
|
|
||||||
defer deleteRemote(t, "scheme-rpm")
|
|
||||||
resp, err := http.Post(apiURL("/api/v2/virtuals"), "application/json",
|
|
||||||
strings.NewReader(`{"name": "scheme-virt", "package_type": "rpm", "members": ["scheme-rpm"]}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusCreated {
|
|
||||||
t.Fatalf("create virtual: status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
defer deleteRequest(t, apiURL("/api/v2/virtuals/scheme-virt"))
|
|
||||||
|
|
||||||
resp = getForwardedHTTPS(t, apiURL("/api/v1/virtual/scheme-virt/scheme-rpm/Packages/a.rpm"))
|
|
||||||
resp.Body.Close()
|
|
||||||
loc := resp.Header.Get("Location")
|
|
||||||
if want := "https://" + strings.TrimPrefix(baseURL, "http://") + "/api/v1/remote/scheme-rpm/Packages/a.rpm"; resp.StatusCode != http.StatusFound || loc != want {
|
|
||||||
t.Errorf("redirect = %d %q, want 302 %q", resp.StatusCode, loc, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
remote:
|
||||||
|
alpine:
|
||||||
|
base_url: "https://dl-cdn.alpinelinux.org"
|
||||||
|
package: "alpine"
|
||||||
|
description: "Alpine Linux APK package repository"
|
||||||
|
immutable_patterns:
|
||||||
|
- ".*/x86_64/.*\\.apk$"
|
||||||
|
cache:
|
||||||
|
immutable_ttl: 0
|
||||||
|
mutable_ttl: 7200
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user