Compare commits
1 Commits
v3.7.2
..
9e7944835e
| Author | SHA1 | Date | |
|---|---|---|---|
 unkinben
|
9e7944835e |
@@ -0,0 +1,15 @@
|
||||
.git/
|
||||
.venv/
|
||||
dist/
|
||||
tests/
|
||||
remotes.yaml
|
||||
ca-bundle.pem
|
||||
.env
|
||||
*.log
|
||||
docker-compose.yml
|
||||
.woodpecker/
|
||||
.tox/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
.pre-commit-cache/
|
||||
minio_data/
|
||||
+60
-5
@@ -1,6 +1,61 @@
|
||||
bin/
|
||||
terraform/
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# e2e-docker fixtures are real package files (.rpm, .tgz, .whl, .zip, ...) that
|
||||
# are intentionally tracked, overriding any global ignore of those extensions.
|
||||
!e2e-docker/fixtures/**
|
||||
# Virtual environment
|
||||
.venv/
|
||||
venv/
|
||||
ENV/
|
||||
env/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# uv
|
||||
uv.lock
|
||||
|
||||
# tox
|
||||
.tox/
|
||||
|
||||
# pytest
|
||||
.pytest_cache/
|
||||
|
||||
# pre-commit
|
||||
.pre-commit-cache/
|
||||
|
||||
# ruff
|
||||
.ruff_cache/
|
||||
|
||||
# Docker volumes
|
||||
minio_data/
|
||||
|
||||
# Local configuration overrides
|
||||
ca-bundle.pem
|
||||
|
||||
+5
-22
@@ -1,24 +1,7 @@
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v5.0.0
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.12
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
- id: check-added-large-files
|
||||
- id: check-merge-conflict
|
||||
|
||||
- repo: https://github.com/dnephin/pre-commit-golang
|
||||
rev: v0.5.1
|
||||
hooks:
|
||||
- id: go-fmt
|
||||
- id: go-mod-tidy
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: go-vet
|
||||
name: go vet
|
||||
entry: go vet ./...
|
||||
language: system
|
||||
types: [go]
|
||||
pass_filenames: false
|
||||
- id: ruff
|
||||
args: [--fix, --exit-non-zero-on-fix]
|
||||
- id: ruff-format
|
||||
|
||||
+2
-18
@@ -3,32 +3,16 @@ when:
|
||||
ref: refs/tags/v*
|
||||
|
||||
steps:
|
||||
- name: docker-api
|
||||
- name: docker
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi
|
||||
build_args:
|
||||
VERSION: ${CI_COMMIT_TAG}
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
|
||||
- name: docker-web
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi-ui
|
||||
dockerfile: ui/Dockerfile.ui
|
||||
context: ui
|
||||
build_args:
|
||||
BASE_PATH: /ui
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
- VERSION=${CI_COMMIT_TAG##v}
|
||||
|
||||
@@ -3,15 +3,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: pre-commit
|
||||
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
||||
image: git.unkin.net/unkin/almalinux9-base:20260308
|
||||
commands:
|
||||
- uvx pre-commit run --all-files
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
|
||||
|
||||
@@ -3,6 +3,6 @@ when:
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
image: golang:1.25
|
||||
image: git.unkin.net/unkin/almalinux9-base:20260308
|
||||
commands:
|
||||
- go test -race -count=1 ./pkg/... ./internal/...
|
||||
- uvx --python 3.11 --with tox-uv tox
|
||||
|
||||
+16
-14
@@ -1,21 +1,23 @@
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM git.unkin.net/unkin/almalinux9-base:latest
|
||||
|
||||
RUN apk add --no-cache git
|
||||
ARG VERSION=0.0.0.dev0
|
||||
|
||||
WORKDIR /build
|
||||
COPY . /build
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
RUN HATCH_VCS_PRETEND_VERSION=${VERSION} \
|
||||
SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} \
|
||||
uv build --wheel --directory /build && \
|
||||
useradd -m -r -s /bin/sh appuser
|
||||
|
||||
COPY . .
|
||||
USER appuser
|
||||
RUN uv tool install --from /build/dist/*.whl artifactapi
|
||||
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=${VERSION}" -o artifactapi ./cmd/artifactapi
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:nonroot
|
||||
|
||||
COPY --from=builder /build/artifactapi /usr/local/bin/artifactapi
|
||||
USER root
|
||||
RUN rm -rf /build
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
ENTRYPOINT ["artifactapi"]
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 CMD curl -f http://localhost:8000/health || exit 1
|
||||
USER appuser
|
||||
ENV PATH="/home/appuser/.local/bin:$PATH"
|
||||
WORKDIR /app
|
||||
CMD ["artifactapi"]
|
||||
|
||||
@@ -1,54 +1,59 @@
|
||||
.PHONY: build test lint fmt e2e docker-e2e docker docker-ui compose clean tidy check-go
|
||||
.PHONY: build install dev clean test lint format pre-commit tox docker-build docker-up docker-down docker-logs docker-rebuild docker-clean docker-restart
|
||||
|
||||
BINARY := bin/artifactapi
|
||||
MODULE := git.unkin.net/unkin/artifactapi
|
||||
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0.0-dev")
|
||||
GO_VERSION_REQUIRED := 1.23
|
||||
GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/')
|
||||
build:
|
||||
docker build -t artifactapi:dev .
|
||||
|
||||
check-go:
|
||||
@if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \
|
||||
echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \
|
||||
fi
|
||||
install: build
|
||||
|
||||
build: check-go tidy
|
||||
go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(BINARY) ./cmd/artifactapi
|
||||
docker-build: build
|
||||
|
||||
test: check-go
|
||||
go test -race -count=1 ./pkg/... ./internal/...
|
||||
|
||||
lint: check-go
|
||||
golangci-lint run ./...
|
||||
go vet ./...
|
||||
|
||||
fmt: check-go
|
||||
gofmt -w .
|
||||
goimports -w .
|
||||
|
||||
e2e: check-go
|
||||
TESTCONTAINERS_RYUK_DISABLED=true go test -tags=e2e -race -count=1 -timeout=5m ./e2e/...
|
||||
|
||||
# Build the container, bring up the full docker-compose stack + a mock upstream,
|
||||
# and run the black-box suite against the running product.
|
||||
docker-e2e: check-go
|
||||
./scripts/docker-e2e.sh
|
||||
|
||||
docker:
|
||||
docker build -t artifactapi:$(VERSION) .
|
||||
|
||||
docker-ui:
|
||||
docker build -t artifactapi-ui:$(VERSION) -f ui/Dockerfile.ui ui/
|
||||
|
||||
compose:
|
||||
docker compose up -d
|
||||
dev: build
|
||||
uv sync --dev
|
||||
|
||||
clean:
|
||||
rm -rf bin/
|
||||
rm -rf .venv
|
||||
rm -rf build/
|
||||
rm -rf dist/
|
||||
rm -rf *.egg-info/
|
||||
|
||||
tidy:
|
||||
go mod tidy
|
||||
test:
|
||||
uvx --python 3.11 --with tox-uv tox
|
||||
|
||||
tox:
|
||||
uvx --python 3.11 --with tox-uv tox
|
||||
|
||||
pre-commit:
|
||||
uvx --python 3.11 pre-commit run --all-files
|
||||
|
||||
lint:
|
||||
uv run ruff check --fix .
|
||||
|
||||
format:
|
||||
uv run ruff format .
|
||||
|
||||
run:
|
||||
uv run python -m src.artifactapi.main
|
||||
|
||||
docker-up:
|
||||
docker-compose up --build --force-recreate -d
|
||||
|
||||
docker-down:
|
||||
docker-compose down
|
||||
|
||||
docker-logs:
|
||||
docker-compose logs -f
|
||||
|
||||
docker-rebuild:
|
||||
docker-compose build --no-cache
|
||||
|
||||
docker-clean:
|
||||
docker-compose down -v --remove-orphans
|
||||
docker system prune -f
|
||||
|
||||
docker-restart: docker-down docker-up
|
||||
|
||||
# Bump helpers — reads the latest semver tag and creates the next one.
|
||||
# If no tag exists yet, starts from v0.0.0.
|
||||
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
||||
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
||||
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
||||
|
||||
@@ -1,880 +0,0 @@
|
||||
# ArtifactAPI v3 — Go Rewrite Plan
|
||||
|
||||
## Context
|
||||
|
||||
ArtifactAPI is a production artifact proxy/cache serving ~42 remotes (Docker registries, Helm repos, RPM/Alpine repos, GitHub releases, PyPI, npm, Puppet Forge, Terraform registries, Go module proxies) across a Kubernetes cluster. The current Python (FastAPI) implementation works but has architectural debt: opaque hashed S3 paths, no UI for visibility, YAML config files that drift, no garbage collection, no access logging, and virtual repos limited to Helm only.
|
||||
|
||||
The v3 rewrite targets: a single Go binary (API + TUI), a separate React frontend (own Dockerfile), a Terraform provider (separate repo), content-addressable storage, and a cleaner data model that makes the cache inspectable and manageable.
|
||||
|
||||
**Repo**: Same repo (`git.unkin.net/unkin/artifactapi`), new branch.
|
||||
**Module**: `git.unkin.net/unkin/artifactapi`
|
||||
**Frontend**: React + Vite, separate Dockerfile, talks to API
|
||||
**Terraform provider**: Separate repo (`terraform-provider-artifactapi`)
|
||||
|
||||
---
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
┌───────────────────────────────────┐ ┌──────────────────────┐
|
||||
│ Go Binary (API + TUI) │ │ Frontend Container │
|
||||
│ │ │ │
|
||||
│ ┌──────────┐ ┌───────────────┐ │ │ React + Vite SPA │
|
||||
│ │ REST API │ │ Proxy Engine │ │◄───│ nginx / node serve │
|
||||
│ │ /api/v2 │ │ /api/v1/... │ │ │ Dockerfile.ui │
|
||||
│ │ │ │ /v2/... (OCI) │ │ └──────────────────────┘
|
||||
│ └────┬─────┘ └──────┬────────┘ │
|
||||
│ │ │ │ ┌──────────────────────┐
|
||||
│ ┌────┴───────────────┴────────┐ │ │ Terraform Provider │
|
||||
│ │ Data Layer │ │◄───│ (separate repo) │
|
||||
│ │ PostgreSQL · Redis · S3 │ │ └──────────────────────┘
|
||||
│ └─────────────────────────────┘ │
|
||||
│ │ ┌──────────────────────┐
|
||||
│ ┌─────────────────────────────┐ │ │ TUI (subcommand) │
|
||||
│ │ artifactapi tui │──│───►│ artifactapi tui │
|
||||
│ └─────────────────────────────┘ │ │ --endpoint <url> │
|
||||
└───────────────────────────────────┘ └──────────────────────┘
|
||||
```
|
||||
|
||||
Three independent deployment units:
|
||||
1. **Go binary** — API server + TUI subcommand (single `Dockerfile`)
|
||||
2. **React frontend** — SPA served by nginx (`Dockerfile.ui`), talks to `/api/v2`
|
||||
3. **Terraform provider** — separate repo, calls `/api/v2` CRUD
|
||||
|
||||
---
|
||||
|
||||
## Project Structure (Modular)
|
||||
|
||||
```
|
||||
artifactapi/
|
||||
├── cmd/
|
||||
│ └── artifactapi/
|
||||
│ └── main.go # entrypoint: serve / tui subcommands
|
||||
│
|
||||
├── pkg/ # PUBLIC — importable by terraform provider, CLI tools
|
||||
│ ├── models/ # shared domain types
|
||||
│ │ ├── remote.go # Remote, RemoteConfig, PackageType enum
|
||||
│ │ ├── virtual.go # Virtual, VirtualConfig
|
||||
│ │ ├── artifact.go # Artifact, Blob, AccessLogEntry
|
||||
│ │ ├── local.go # LocalFile, LocalRepo
|
||||
│ │ └── stats.go # RemoteStats, OverviewStats
|
||||
│ └── client/ # typed Go API client (used by TUI + Terraform provider)
|
||||
│ ├── client.go # Client struct, base HTTP
|
||||
│ ├── remotes.go # remote CRUD methods
|
||||
│ ├── virtuals.go # virtual CRUD methods
|
||||
│ ├── objects.go # object browse/evict methods
|
||||
│ └── stats.go # stats methods
|
||||
│
|
||||
├── internal/ # PRIVATE — server internals
|
||||
│ ├── server/
|
||||
│ │ ├── server.go # HTTP server setup, router
|
||||
│ │ └── middleware.go # logging, recovery, request-id, access logging
|
||||
│ │
|
||||
│ ├── api/
|
||||
│ │ ├── v1/ # proxy endpoints (v1 compat)
|
||||
│ │ │ ├── proxy.go # GET /api/v1/remote/{name}/{path}
|
||||
│ │ │ ├── docker.go # /v2/{name}/{path}
|
||||
│ │ │ ├── virtual.go # GET /api/v1/virtual/{name}/{path}
|
||||
│ │ │ └── local.go # CRUD /api/v1/local/{name}/{path}
|
||||
│ │ └── v2/ # management API
|
||||
│ │ ├── remotes.go # CRUD + stats
|
||||
│ │ ├── virtuals.go # CRUD
|
||||
│ │ ├── objects.go # browse/evict cached objects
|
||||
│ │ ├── stats.go # overview, top-remotes
|
||||
│ │ ├── events.go # SSE stream
|
||||
│ │ └── health.go # health, metrics
|
||||
│ │
|
||||
│ ├── provider/ # package-type providers (registry protocol handlers)
|
||||
│ │ ├── provider.go # Provider interface + registry
|
||||
│ │ ├── generic/
|
||||
│ │ │ ├── generic.go
|
||||
│ │ │ └── generic_test.go
|
||||
│ │ ├── docker/
|
||||
│ │ │ ├── docker.go # OCI Distribution v2 via go-containerregistry
|
||||
│ │ │ ├── auth.go # Bearer token fetch + cache
|
||||
│ │ │ └── docker_test.go
|
||||
│ │ ├── helm/
|
||||
│ │ │ ├── helm.go # index rewriting via helm.sh/helm/v3/pkg/repo
|
||||
│ │ │ ├── merger.go # virtual index merge
|
||||
│ │ │ └── helm_test.go
|
||||
│ │ ├── pypi/
|
||||
│ │ │ ├── pypi.go # simple index HTML rewriting
|
||||
│ │ │ ├── merger.go # virtual simple index merge
|
||||
│ │ │ └── pypi_test.go
|
||||
│ │ ├── npm/
|
||||
│ │ │ ├── npm.go # metadata JSON rewriting
|
||||
│ │ │ └── npm_test.go
|
||||
│ │ ├── rpm/
|
||||
│ │ │ ├── rpm.go # repodata patterns
|
||||
│ │ │ └── rpm_test.go
|
||||
│ │ ├── alpine/
|
||||
│ │ │ ├── alpine.go # APKINDEX patterns
|
||||
│ │ │ └── alpine_test.go
|
||||
│ │ ├── puppet/
|
||||
│ │ │ ├── puppet.go # file_uri JSON rewriting
|
||||
│ │ │ └── puppet_test.go
|
||||
│ │ ├── terraform/
|
||||
│ │ │ ├── terraform.go # registry protocol, download URL rewriting
|
||||
│ │ │ └── terraform_test.go
|
||||
│ │ └── goproxy/
|
||||
│ │ ├── goproxy.go # Go module proxy protocol (GOPROXY)
|
||||
│ │ └── goproxy_test.go
|
||||
│ │
|
||||
│ ├── proxy/
|
||||
│ │ ├── engine.go # core fetch-or-cache logic
|
||||
│ │ ├── engine_test.go
|
||||
│ │ ├── classifier.go # immutable vs mutable classification
|
||||
│ │ ├── classifier_test.go
|
||||
│ │ ├── revalidator.go # conditional HEAD requests (ETag/Last-Modified)
|
||||
│ │ └── circuit.go # per-remote circuit breaker
|
||||
│ │
|
||||
│ ├── storage/
|
||||
│ │ ├── s3.go # S3 client (minio-go — works with MinIO, Ceph, AWS)
|
||||
│ │ ├── s3_test.go
|
||||
│ │ ├── cas.go # content-addressable store logic
|
||||
│ │ └── cas_test.go
|
||||
│ │
|
||||
│ ├── cache/
|
||||
│ │ ├── redis.go # TTL management, fetch locks
|
||||
│ │ ├── redis_test.go
|
||||
│ │ └── lock.go # distributed lock abstraction
|
||||
│ │
|
||||
│ ├── database/
|
||||
│ │ ├── postgres.go # connection pool, migration runner
|
||||
│ │ ├── queries/ # SQL query files or sqlc-generated code
|
||||
│ │ │ ├── remotes.sql.go
|
||||
│ │ │ ├── virtuals.sql.go
|
||||
│ │ │ ├── artifacts.sql.go
|
||||
│ │ │ └── access_log.sql.go
|
||||
│ │ └── migrations/ # golang-migrate SQL files
|
||||
│ │ ├── 001_initial.up.sql
|
||||
│ │ └── 001_initial.down.sql
|
||||
│ │
|
||||
│ ├── metrics/
|
||||
│ │ └── prometheus.go # counters, gauges, histograms
|
||||
│ │
|
||||
│ ├── gc/
|
||||
│ │ ├── gc.go # background garbage collection goroutine
|
||||
│ │ └── gc_test.go
|
||||
│ │
|
||||
│ ├── tui/
|
||||
│ │ ├── app.go # Bubble Tea main model
|
||||
│ │ ├── views/
|
||||
│ │ │ ├── dashboard.go
|
||||
│ │ │ ├── remotes.go
|
||||
│ │ │ ├── objects.go
|
||||
│ │ │ └── virtuals.go
|
||||
│ │ └── components/
|
||||
│ │ ├── table.go
|
||||
│ │ └── statusbar.go
|
||||
│ │
|
||||
│ └── config/
|
||||
│ └── env.go # environment variable parsing + validation
|
||||
│
|
||||
├── ui/ # React frontend — SEPARATE DOCKERFILE
|
||||
│ ├── src/
|
||||
│ │ ├── App.tsx
|
||||
│ │ ├── pages/
|
||||
│ │ │ ├── Dashboard.tsx
|
||||
│ │ │ ├── Remotes.tsx
|
||||
│ │ │ ├── RemoteDetail.tsx
|
||||
│ │ │ ├── Virtuals.tsx
|
||||
│ │ │ └── Objects.tsx
|
||||
│ │ ├── components/
|
||||
│ │ │ ├── RemoteTable.tsx
|
||||
│ │ │ ├── ObjectBrowser.tsx
|
||||
│ │ │ ├── StatsCard.tsx
|
||||
│ │ │ └── EventFeed.tsx
|
||||
│ │ └── api/
|
||||
│ │ └── client.ts # typed API client
|
||||
│ ├── package.json
|
||||
│ ├── vite.config.ts
|
||||
│ ├── tsconfig.json
|
||||
│ ├── Dockerfile.ui # multi-stage: node build → nginx
|
||||
│ └── nginx.conf # proxy /api/* to backend, serve SPA
|
||||
│
|
||||
├── e2e/ # end-to-end integration tests
|
||||
│ ├── e2e_test.go # TestMain spins up docker-compose stack
|
||||
│ ├── proxy_test.go # proxy through real remotes
|
||||
│ ├── docker_test.go # Docker v2 protocol e2e
|
||||
│ ├── management_test.go # v2 API CRUD
|
||||
│ ├── virtual_test.go # virtual repo merge e2e
|
||||
│ └── docker-compose.e2e.yml # postgres + redis + minio for tests
|
||||
│
|
||||
├── go.mod
|
||||
├── go.sum
|
||||
├── Makefile
|
||||
├── Dockerfile # Go binary (API server + TUI)
|
||||
├── Dockerfile.ui # symlink or copy → ui/Dockerfile.ui
|
||||
└── docker-compose.yml
|
||||
```
|
||||
|
||||
### Key Modularisation Decisions
|
||||
|
||||
- **`pkg/models/`** — Shared domain types importable by the Terraform provider and any external tooling. No dependencies on internal packages
|
||||
- **`pkg/client/`** — Typed Go API client used by both the TUI and the Terraform provider. Depends only on `pkg/models/` and stdlib
|
||||
- **`internal/provider/`** — Each package type is its own subpackage with isolated tests. A provider registry maps `PackageType → Provider`
|
||||
- **`internal/database/queries/`** — Use [sqlc](https://sqlc.dev/) to generate type-safe query functions from SQL, or hand-written query files
|
||||
- **`e2e/`** — Separate test binary that spins up a real docker-compose stack
|
||||
|
||||
---
|
||||
|
||||
## Go Ecosystem Libraries
|
||||
|
||||
Prefer existing, maintained Go modules over writing protocol handlers from scratch.
|
||||
|
||||
### Package-Type Libraries
|
||||
|
||||
| Package Type | Go Module | What It Gives Us |
|
||||
|---|---|---|
|
||||
| **Docker/OCI** | `github.com/google/go-containerregistry` | Full Registry v2/OCI client: manifest parsing, auth challenges, blob operations. `pkg/registry` can implement a v2 server. Reference: `github.com/regclient/regclient` |
|
||||
| **Helm** | `helm.sh/helm/v3/pkg/repo` | Parse/generate `index.yaml`, `IndexFile`/`ChartVersion` types, URL entries. Used directly for merge |
|
||||
| **Terraform** | `github.com/hashicorp/terraform-registry-address` | Provider/module address parsing, `ForRegistryProtocol()` URL generation. Protocol spec: provider registry protocol v1 |
|
||||
| **Go Modules** | `github.com/goproxy/goproxy` | Minimalist GOPROXY protocol handler, implements full spec as `http.Handler`. Handles `/@v/list`, `/@v/{v}.info`, `/@v/{v}.mod`, `/@v/{v}.zip`, `/@latest` |
|
||||
| **RPM** | `rs3.io/go/rpm/repomd` | Parse `repomd.xml`, `primary.xml` with proper XML namespace handling |
|
||||
| **Alpine** | `gitlab.alpinelinux.org/alpine/go` | Official Alpine library: parse APKINDEX, `.apk` files |
|
||||
| **PyPI** | stdlib `golang.org/x/net/html` | No dedicated Go PyPI library exists. Parse simple index HTML with `x/net/html`, extract `<a>` tags. Minimal — the rewriting is just href replacement |
|
||||
| **npm** | stdlib `encoding/json` | npm metadata is JSON — parse with stdlib, rewrite `dist.tarball` URLs. No special library needed |
|
||||
| **Puppet Forge** | stdlib `encoding/json` | Forge API is JSON — parse and rewrite `file_uri` fields. Community lib `github.com/johnmccabe/go-puppetforge` exists but is thin; stdlib suffices |
|
||||
|
||||
### Infrastructure Libraries
|
||||
|
||||
| Purpose | Go Module | Why This One |
|
||||
|---|---|---|
|
||||
| **HTTP router** | `github.com/go-chi/chi/v5` | Lightweight, stdlib `http.Handler` compatible, middleware chain |
|
||||
| **PostgreSQL** | `github.com/jackc/pgx/v5` | Pure Go, connection pooling, COPY support, prepared statements |
|
||||
| **SQL generation** | `github.com/sqlc-dev/sqlc` | Generate type-safe Go from SQL queries — no ORM, no reflection |
|
||||
| **Redis** | `github.com/redis/go-redis/v9` | Full Redis client, pipelining, pub/sub |
|
||||
| **S3 (MinIO/Ceph/AWS)** | `github.com/minio/minio-go/v7` | Native S3-compatible client. Works with MinIO, Ceph RGW, AWS S3, any S3-compatible backend out of the box. Lighter than aws-sdk-go-v2, purpose-built for S3 compat |
|
||||
| **DB migrations** | `github.com/golang-migrate/migrate/v4` | SQL file-based migrations, CLI + library |
|
||||
| **Prometheus** | `github.com/prometheus/client_golang` | Counters, gauges, histograms |
|
||||
| **TUI** | `github.com/charmbracelet/bubbletea` | Elm-architecture TUI framework |
|
||||
| **TUI styling** | `github.com/charmbracelet/lipgloss` | Terminal styling |
|
||||
| **TUI components** | `github.com/charmbracelet/bubbles` | Table, text input, spinner, etc. |
|
||||
| **Structured logging** | `log/slog` (stdlib) | Go 1.21+ structured logging, zero dependencies |
|
||||
| **Testing** | `github.com/stretchr/testify` | Assertions + require for unit tests |
|
||||
| **Test containers** | `github.com/testcontainers/testcontainers-go` | Spin up Postgres/Redis/MinIO in e2e tests |
|
||||
|
||||
### S3 Client: Multi-Backend Support
|
||||
|
||||
Using `minio-go/v7` as the S3 client because it natively supports:
|
||||
- **MinIO** — primary development/production target
|
||||
- **Ceph RGW** — S3-compatible via endpoint config
|
||||
- **AWS S3** — via region + credential config
|
||||
- **Any S3-compatible** — GCS (interop mode), Wasabi, DigitalOcean Spaces, etc.
|
||||
|
||||
No abstraction layer needed — `minio-go` handles endpoint differences internally. Config:
|
||||
```go
|
||||
client, _ := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: useTLS,
|
||||
Region: region, // optional, for AWS
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Data Layer
|
||||
|
||||
### PostgreSQL Schema
|
||||
|
||||
```sql
|
||||
-- Remotes: managed exclusively by Terraform
|
||||
CREATE TABLE remotes (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL, -- generic, docker, helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy
|
||||
base_url TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
immutable_ttl INTEGER DEFAULT 0,
|
||||
mutable_ttl INTEGER DEFAULT 3600,
|
||||
check_mutable BOOLEAN DEFAULT TRUE,
|
||||
immutable_patterns TEXT[] DEFAULT '{}', -- user-defined immutable patterns
|
||||
mutable_patterns TEXT[] DEFAULT '{}', -- user-defined mutable patterns (merged with provider built-ins)
|
||||
allowlist TEXT[] DEFAULT '{}', -- if empty, allow all paths; if non-empty, only matching paths proxied
|
||||
blocklist TEXT[] DEFAULT '{}', -- always denied, checked before allowlist
|
||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
||||
ban_tags TEXT[] DEFAULT '{}',
|
||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
||||
quarantine_days INTEGER DEFAULT 3,
|
||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
||||
releases_remote TEXT DEFAULT '', -- terraform type: name of CDN remote for download URL rewriting
|
||||
managed_by TEXT DEFAULT '', -- 'terraform' or empty
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Virtual repositories
|
||||
CREATE TABLE virtuals (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
members TEXT[] NOT NULL,
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Content-addressable blob storage tracking
|
||||
CREATE TABLE blobs (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
s3_key TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
content_type TEXT DEFAULT 'application/octet-stream',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Artifact metadata: maps (remote, path) → content blob
|
||||
CREATE TABLE artifacts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
upstream_etag TEXT DEFAULT '',
|
||||
upstream_last_modified TIMESTAMPTZ,
|
||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
fetch_count BIGINT DEFAULT 1,
|
||||
access_count BIGINT DEFAULT 1,
|
||||
UNIQUE(remote_name, path)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_artifacts_remote ON artifacts(remote_name);
|
||||
CREATE INDEX idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
||||
|
||||
-- Local file uploads
|
||||
CREATE TABLE local_files (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
-- Access log (append-only, powers dashboards)
|
||||
CREATE TABLE access_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
cache_hit BOOLEAN NOT NULL,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
upstream_ms INTEGER DEFAULT 0,
|
||||
client_ip TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
```
|
||||
|
||||
### Redis Usage (Ephemeral Only)
|
||||
|
||||
| Key pattern | Type | TTL | Purpose |
|
||||
|---|---|---|---|
|
||||
| `ttl:{remote}:{path}` | STRING | remote's immutable/mutable TTL | Artifact freshness — existence = still fresh |
|
||||
| `lock:{remote}:{path}` | STRING (NX) | 30s | Fetch lock — prevents thundering herd |
|
||||
| `etag:{remote}:{path}` | STRING | same as TTL key | Cached ETag for conditional revalidation |
|
||||
| `circuit:{remote}` | STRING | configurable | Circuit breaker — consecutive failure count |
|
||||
|
||||
Losing Redis = all TTLs expire = next request re-validates upstream. No data loss.
|
||||
|
||||
### S3 Layout (Content-Addressable)
|
||||
|
||||
```
|
||||
artifacts-bucket/
|
||||
├── blobs/sha256/{content_hash} # immutable CAS blobs
|
||||
├── indexes/{remote}/{path} # mutable index files (helm, pypi, rpm, etc.)
|
||||
├── indexes/{virtual}/{path} # merged virtual indexes
|
||||
└── local/{repo}/{path} # user uploads (CAS-backed via blobs table)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Terraform Remote Type (New in v2)
|
||||
|
||||
The `terraform` package type proxies the Terraform Provider Registry Protocol:
|
||||
|
||||
- **URL construction**: prepends `/v1/providers/` to request paths
|
||||
- **Built-in mutable pattern**: `[^/]+/[^/]+/versions$` (version listings change over time)
|
||||
- **Built-in immutable pattern**: `[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$` (per-version download info is fixed)
|
||||
- **Response rewriting**: download info JSON — rewrites `download_url`, `shasums_url`, `shasums_signature_url` to route through a companion `releases_remote` (e.g., `hashicorp-releases` generic remote)
|
||||
- **Config**: requires `releases_remote` field pointing to the CDN remote that serves the actual binaries
|
||||
|
||||
Uses `github.com/hashicorp/terraform-registry-address` for address parsing and protocol-compliant URL generation.
|
||||
|
||||
---
|
||||
|
||||
## Go Module Proxy Remote Type (New)
|
||||
|
||||
The `goproxy` package type implements the GOPROXY protocol (Go module proxy):
|
||||
|
||||
| Endpoint | Mutability | Description |
|
||||
|---|---|---|
|
||||
| `{module}/@v/list` | Mutable | Plain text list of known versions |
|
||||
| `{module}/@latest` | Mutable | JSON metadata for latest version |
|
||||
| `{module}/@v/{version}.info` | Immutable | JSON version metadata (`Version`, `Time`) |
|
||||
| `{module}/@v/{version}.mod` | Immutable | `go.mod` file for that version |
|
||||
| `{module}/@v/{version}.zip` | Immutable | Source archive for that version |
|
||||
|
||||
- **No URL rewriting needed** — responses are self-contained (no embedded URLs)
|
||||
- **Config**: `base_url` points to upstream proxy (e.g., `https://proxy.golang.org`)
|
||||
- **Client usage**: set `GOPROXY=https://artifactapi.example.com/api/v1/remote/goproxy`
|
||||
- Uses `github.com/goproxy/goproxy` for protocol handling
|
||||
|
||||
---
|
||||
|
||||
## Allowlist / Blocklist / Automatic Mutable Patterns
|
||||
|
||||
### Access Control (Per-Remote)
|
||||
|
||||
| Field | Default | Behavior |
|
||||
|---|---|---|
|
||||
| `blocklist` | `[]` (empty) | If a path matches any blocklist pattern → **403 Forbidden**. Checked first |
|
||||
| `allowlist` | `[]` (empty) | If empty → **allow everything**. If non-empty → only matching paths are proxied; everything else → **403** |
|
||||
|
||||
Evaluation order: blocklist → allowlist → proxy. No allowlist + no blocklist = open proxy (default).
|
||||
|
||||
### Automatic Mutable Patterns (Per-Provider Built-ins)
|
||||
|
||||
Each provider declares built-in mutable patterns that are **always merged** with user-defined `mutable_patterns`. Users never need to configure these — the provider knows which paths change over time.
|
||||
|
||||
| Provider | Built-in Mutable Patterns | Rationale |
|
||||
|---|---|---|
|
||||
| **generic** | *(none)* | No convention for what's mutable |
|
||||
| **docker** | `/manifests/(?!sha256:)[^/]+$`, `/tags/list$` | Tag manifests change; digest manifests don't |
|
||||
| **helm** | `index\.yaml$` | Chart index changes when new charts are published |
|
||||
| **pypi** | `simple/` | Package index pages change with new releases |
|
||||
| **npm** | `^[^/]+$` (package metadata, not `.tgz`) | Package metadata changes; tarballs are immutable |
|
||||
| **rpm** | `repomd\.xml$`, `repodata/.*`, `Packages\.gz$` | Repo metadata rebuilt on every publish |
|
||||
| **alpine** | `APKINDEX\.tar\.gz$` | Package index rebuilt on every publish |
|
||||
| **puppet** | `^v3/modules/`, `^v3/releases` | Module metadata changes with new releases |
|
||||
| **terraform** | `[^/]+/[^/]+/versions$` | Provider version listings grow over time |
|
||||
| **goproxy** | `@v/list$`, `@latest$` | Version list and latest pointer change |
|
||||
|
||||
These are returned by `Provider.BuiltinMutablePatterns()` and merged at classification time:
|
||||
```
|
||||
effective_mutable = provider.BuiltinMutablePatterns() ∪ remote.mutable_patterns
|
||||
```
|
||||
|
||||
If a path matches `effective_mutable` → use `mutable_ttl`. If it matches `remote.immutable_patterns` → use `immutable_ttl`. Immutable patterns take precedence over mutable when both match.
|
||||
|
||||
---
|
||||
|
||||
## API Design
|
||||
|
||||
### v1 Proxy Endpoints (Backwards Compatible)
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| `GET` | `/api/v1/remote/{name}/{path}` | Proxy/cache artifact |
|
||||
| `GET` | `/api/v1/virtual/{name}/{path}` | Virtual repo proxy |
|
||||
| `GET/HEAD` | `/v2/{name}/{path}` | Docker Registry v2 |
|
||||
| `GET` | `/v2/` | Docker v2 ping |
|
||||
| `GET/PUT/HEAD/DELETE` | `/api/v1/local/{name}/{path}` | Local repo CRUD |
|
||||
|
||||
### v2 Management API (New)
|
||||
|
||||
```
|
||||
GET /api/v2/remotes → [{name, package_type, base_url, description, stats}]
|
||||
GET /api/v2/remotes/{name} → {full config + stats + health}
|
||||
POST /api/v2/remotes → create remote (Terraform provider)
|
||||
PUT /api/v2/remotes/{name} → update remote (Terraform provider)
|
||||
DELETE /api/v2/remotes/{name} → delete remote — cascades artifacts, GC cleans S3
|
||||
|
||||
GET /api/v2/virtuals → [{name, package_type, members, stats}]
|
||||
GET /api/v2/virtuals/{name} → {full config + member details}
|
||||
POST /api/v2/virtuals → create virtual
|
||||
PUT /api/v2/virtuals/{name} → update virtual
|
||||
DELETE /api/v2/virtuals/{name} → delete virtual
|
||||
|
||||
GET /api/v2/remotes/{name}/objects → paginated objects
|
||||
?q=pattern&sort=size|accessed|age&page=1&per_page=50
|
||||
DELETE /api/v2/remotes/{name}/objects/{path} → evict specific cached object
|
||||
DELETE /api/v2/remotes/{name}/cache → flush cache
|
||||
?type=all|indexes|blobs
|
||||
|
||||
GET /api/v2/stats → overview stats
|
||||
GET /api/v2/stats/top-remotes → top remotes by size/requests/hit-rate
|
||||
|
||||
GET /api/v2/health → {status, postgres, redis, s3, uptime}
|
||||
GET /metrics → Prometheus format
|
||||
GET /api/v2/events → SSE stream
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Proxy Engine
|
||||
|
||||
### Request Flow
|
||||
|
||||
```
|
||||
Client Request
|
||||
│
|
||||
▼
|
||||
Classify (immutable/mutable/denied)
|
||||
│
|
||||
├── blocklist match → 403
|
||||
├── allowlist non-empty + no match → 403
|
||||
│
|
||||
▼
|
||||
Check Redis TTL key
|
||||
│
|
||||
├── exists (fresh) → serve from S3, log access
|
||||
│
|
||||
├── missing (expired or uncached)
|
||||
│ │
|
||||
│ ▼
|
||||
│ Acquire fetch lock (Redis SETNX, 30s TTL)
|
||||
│ │
|
||||
│ ├── lock acquired
|
||||
│ │ ├── mutable + check_mutable + have ETag → HEAD upstream
|
||||
│ │ │ ├── 304 → refresh TTL, serve from S3
|
||||
│ │ │ └── changed → full fetch
|
||||
│ │ └── full fetch from upstream
|
||||
│ │ → provider.RewriteResponse() if needed
|
||||
│ │ → CAS store (hash → check blobs → upload if new)
|
||||
│ │ → upsert artifact in Postgres
|
||||
│ │ → set Redis TTL + release lock
|
||||
│ │ → on upstream error + stale_on_error → refresh TTL, serve stale
|
||||
│ │
|
||||
│ └── lock not acquired → poll S3 briefly, serve if another pod fetched it
|
||||
│
|
||||
▼
|
||||
Stream response from S3, log access
|
||||
```
|
||||
|
||||
### Circuit Breaker
|
||||
|
||||
Per-remote, tracked in Redis. Closed → Open (after N failures) → Half-open (after cooldown). Exposed via `GET /api/v2/remotes/{name}` health field.
|
||||
|
||||
### Content-Addressable Storage
|
||||
|
||||
1. Stream upstream → temp file, compute SHA256 inline
|
||||
2. Check `blobs` table for hash
|
||||
3. Exists → skip S3 upload, upsert `artifacts` row only
|
||||
4. New → upload to `blobs/sha256/{hash}`, insert both rows
|
||||
|
||||
### Garbage Collection
|
||||
|
||||
Background goroutine (configurable interval, default 1h):
|
||||
1. Orphaned blobs: delete S3 objects whose `content_hash` has no referencing `artifacts` or `local_files` rows
|
||||
2. Cold artifacts: optional per-remote, delete artifacts not accessed in N days
|
||||
3. Remote deletion: `ON DELETE CASCADE` handles Postgres; GC sweeps orphaned blobs
|
||||
|
||||
---
|
||||
|
||||
## Package Providers
|
||||
|
||||
### Provider Interface
|
||||
|
||||
```go
|
||||
type Provider interface {
|
||||
Type() models.PackageType
|
||||
BuiltinMutablePatterns() []*regexp.Regexp
|
||||
BuiltinImmutablePatterns() []*regexp.Regexp
|
||||
ContentType(path string) string
|
||||
UpstreamURL(remote models.Remote, path string) string
|
||||
RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error)
|
||||
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
```
|
||||
|
||||
### Provider Registry
|
||||
|
||||
```go
|
||||
var registry = map[models.PackageType]Provider{
|
||||
models.PackageGeneric: &generic.Provider{},
|
||||
models.PackageDocker: &docker.Provider{},
|
||||
models.PackageHelm: &helm.Provider{},
|
||||
models.PackagePyPI: &pypi.Provider{},
|
||||
models.PackageNPM: &npm.Provider{},
|
||||
models.PackageRPM: &rpm.Provider{},
|
||||
models.PackageAlpine: &alpine.Provider{},
|
||||
models.PackagePuppet: &puppet.Provider{},
|
||||
models.PackageTerraform: &terraform.Provider{},
|
||||
models.PackageGoProxy: &goproxy.Provider{},
|
||||
}
|
||||
|
||||
func Get(t models.PackageType) (Provider, error) { ... }
|
||||
```
|
||||
|
||||
Each provider lives in its own subpackage under `internal/provider/` with its own `_test.go`.
|
||||
|
||||
---
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
Every package gets `_test.go` files alongside the source. Run with `go test ./...`.
|
||||
|
||||
| Package | What's Tested |
|
||||
|---|---|
|
||||
| `internal/provider/docker/` | Auth token parsing/caching, manifest classification, tag banning, URL construction, blob key generation |
|
||||
| `internal/provider/helm/` | `index.yaml` parsing (using `helm.sh/helm/v3/pkg/repo`), URL rewriting, index merging |
|
||||
| `internal/provider/pypi/` | Simple index HTML parsing, URL rewriting, index merging |
|
||||
| `internal/provider/npm/` | Metadata JSON rewriting (`dist.tarball` URLs) |
|
||||
| `internal/provider/terraform/` | Registry URL construction, download info JSON rewriting, `releases_remote` URL extraction |
|
||||
| `internal/provider/rpm/` | Mutable pattern matching (repodata) |
|
||||
| `internal/provider/alpine/` | Mutable pattern matching (APKINDEX) |
|
||||
| `internal/provider/puppet/` | `file_uri` JSON rewriting |
|
||||
| `internal/proxy/` | Classifier (immutable vs mutable vs denied), circuit breaker state transitions, revalidator logic |
|
||||
| `internal/storage/` | CAS key generation, dedup detection, S3 operation mocking |
|
||||
| `internal/cache/` | Redis TTL set/check, fetch lock acquire/release/contention |
|
||||
| `internal/gc/` | Orphan detection queries, cold artifact selection |
|
||||
| `pkg/models/` | Model validation, PackageType enum |
|
||||
| `pkg/client/` | API client request/response serialization |
|
||||
|
||||
### End-to-End Tests
|
||||
|
||||
Located in `e2e/`. Use `testcontainers-go` to spin up real Postgres, Redis, and MinIO containers. The test binary starts the actual `artifactapi` server against these backends.
|
||||
|
||||
```go
|
||||
// e2e/e2e_test.go
|
||||
func TestMain(m *testing.M) {
|
||||
// Start postgres, redis, minio via testcontainers-go
|
||||
// Run migrations
|
||||
// Start artifactapi server on random port
|
||||
// Run tests
|
||||
// Tear down
|
||||
}
|
||||
```
|
||||
|
||||
| Test File | What's Tested |
|
||||
|---|---|
|
||||
| `e2e/proxy_test.go` | Proxy a real GitHub release through generic remote, verify S3 storage, verify Redis TTL, verify Postgres artifact row, verify cache hit on second request |
|
||||
| `e2e/docker_test.go` | Pull a real image manifest + blob through Docker v2 proxy, verify blob deduplication, tag banning |
|
||||
| `e2e/management_test.go` | Full CRUD lifecycle: create remote via v2 API, proxy through it, list objects, evict object, flush cache, delete remote |
|
||||
| `e2e/virtual_test.go` | Create two helm remotes + virtual, fetch merged index, verify priority ordering |
|
||||
| `e2e/terraform_test.go` | Proxy terraform provider version listing + download info, verify URL rewriting to releases_remote |
|
||||
| `e2e/goproxy_test.go` | Proxy Go module `@v/list`, `.info`, `.mod`, `.zip` through GOPROXY remote, verify mutable vs immutable classification |
|
||||
| `e2e/gc_test.go` | Create artifact, delete remote, trigger GC, verify S3 blob cleaned up |
|
||||
|
||||
### Code Quality
|
||||
|
||||
- `gofmt` / `goimports` — enforced in CI, run on save
|
||||
- `golangci-lint` — comprehensive linter suite (staticcheck, errcheck, govet, etc.)
|
||||
- `go vet ./...` — run in CI
|
||||
- Makefile targets: `make test`, `make lint`, `make e2e`, `make fmt`
|
||||
|
||||
---
|
||||
|
||||
## Terraform Provider (Separate Repo)
|
||||
|
||||
**Repo**: `terraform-provider-artifactapi`
|
||||
**Uses**: `pkg/client/` and `pkg/models/` from the main module
|
||||
|
||||
```hcl
|
||||
provider "artifactapi" {
|
||||
endpoint = "https://artifactapi.k8s.syd1.au.unkin.net"
|
||||
}
|
||||
|
||||
resource "artifactapi_remote" "terraform_registry" {
|
||||
name = "terraform-registry"
|
||||
package_type = "terraform"
|
||||
base_url = "https://registry.terraform.io"
|
||||
description = "Terraform provider registry"
|
||||
releases_remote = artifactapi_remote.hashicorp_releases.name
|
||||
|
||||
immutable_patterns = [
|
||||
"[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$",
|
||||
]
|
||||
|
||||
cache {
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
}
|
||||
}
|
||||
|
||||
resource "artifactapi_remote" "hashicorp_releases" {
|
||||
name = "hashicorp-releases"
|
||||
package_type = "generic"
|
||||
base_url = "https://releases.hashicorp.com"
|
||||
|
||||
immutable_patterns = [
|
||||
".*\\.zip$",
|
||||
".*SHA256SUMS(\\.sig)?$",
|
||||
]
|
||||
|
||||
cache {
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 0
|
||||
}
|
||||
}
|
||||
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
description = "All helm repos merged"
|
||||
members = [
|
||||
artifactapi_remote.jetstack.name,
|
||||
artifactapi_remote.hashicorp_helm.name,
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Web UI (React + Vite — Separate Container)
|
||||
|
||||
### Deployment
|
||||
|
||||
Separate `Dockerfile.ui`: multi-stage build (node → nginx). Served as its own container/pod. nginx proxies `/api/*` to the Go backend.
|
||||
|
||||
### Pages
|
||||
|
||||
| Route | Content |
|
||||
|---|---|
|
||||
| `/` | Dashboard: total objects, storage used, dedup savings, bandwidth saved, top remotes chart, live SSE event feed, health indicators |
|
||||
| `/remotes` | Remote table: name, type, description, object count, size, hit rate, health. Filter by type, sort any column |
|
||||
| `/remotes/:name` | Config (read-only, "Managed by Terraform" badge), stats, object browser with search/sort/evict, flush actions |
|
||||
| `/virtuals` | Virtual table: name, type, members, merged object count |
|
||||
| `/virtuals/:name` | Member list with individual stats |
|
||||
|
||||
All config is read-only — managed by Terraform.
|
||||
|
||||
---
|
||||
|
||||
## TUI (Bubble Tea — Subcommand)
|
||||
|
||||
`artifactapi tui --endpoint http://localhost:8000` or via `ARTIFACTAPI_ENDPOINT` env.
|
||||
|
||||
Uses `pkg/client/` for all API calls (same client as Terraform provider).
|
||||
|
||||
| View | Key bindings |
|
||||
|---|---|
|
||||
| Dashboard | summary stats, top remotes |
|
||||
| Remotes list | `j`/`k` navigate, `/` filter, `Enter` detail |
|
||||
| Remote detail | config + stats, `Enter` → object browser |
|
||||
| Object browser | `/` search, `d` evict, `f` flush |
|
||||
| Virtuals | `j`/`k`, `Enter` detail |
|
||||
|
||||
---
|
||||
|
||||
## Improvements Over v2
|
||||
|
||||
| Area | v2 (Python) | v3 (Go) |
|
||||
|---|---|---|
|
||||
| S3 paths | Hashed, opaque | Content-addressed CAS |
|
||||
| Config | YAML files, mtime reload | Terraform via API |
|
||||
| Package types | 8 types | 10 types (+ terraform, goproxy) |
|
||||
| Virtual repos | Helm only | Helm + PyPI, extensible |
|
||||
| Deduplication | Docker blobs only | All types via CAS |
|
||||
| Revalidation | Opt-in flag | Default for all mutable |
|
||||
| Access logging | None | Per-artifact in Postgres |
|
||||
| GC | None | Background goroutine |
|
||||
| Upstream health | Per-request | Circuit breaker |
|
||||
| S3 backends | MinIO only | MinIO, Ceph, AWS (minio-go) |
|
||||
| UI | None | Web dashboard + TUI |
|
||||
| Binary | Python + venv | Static Go binary |
|
||||
| Frontend | N/A | Separate container (React) |
|
||||
| Testing | Mocked unit tests | Unit + e2e with real backends |
|
||||
|
||||
---
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Core Engine + Models
|
||||
- Go module, Makefile (`make build test lint fmt e2e`), Dockerfile, docker-compose
|
||||
- `pkg/models/` — all domain types
|
||||
- PostgreSQL schema + migrations
|
||||
- S3 storage layer with CAS (`minio-go/v7`)
|
||||
- Redis cache layer (TTL, locks)
|
||||
- Proxy engine: fetch-or-cache, classifier, revalidator
|
||||
- Generic + Docker providers (most complexity: OCI auth, CAS, tag banning)
|
||||
- Health + metrics endpoints
|
||||
- Unit tests for all packages
|
||||
- **Milestone**: proxy Docker + generic, cache in S3, track in Postgres
|
||||
|
||||
### Phase 2: All Providers
|
||||
- Helm (using `helm.sh/helm/v3/pkg/repo`)
|
||||
- PyPI (stdlib `x/net/html`)
|
||||
- npm (stdlib `encoding/json`)
|
||||
- RPM (using `rs3.io/go/rpm/repomd`)
|
||||
- Alpine (using `gitlab.alpinelinux.org/alpine/go`)
|
||||
- Puppet Forge (stdlib `encoding/json`)
|
||||
- Terraform (using `hashicorp/terraform-registry-address`)
|
||||
- Go Modules / GOPROXY (using `github.com/goproxy/goproxy`)
|
||||
- Unit tests per provider
|
||||
- **Milestone**: feature parity with v2 + goproxy
|
||||
|
||||
### Phase 3: Management API + Virtual Repos + GC
|
||||
- `pkg/client/` — shared Go API client
|
||||
- v2 CRUD endpoints
|
||||
- Virtual repo engine: `IndexMerger` for Helm + PyPI
|
||||
- Circuit breaker
|
||||
- Access logging middleware
|
||||
- GC goroutine
|
||||
- **Milestone**: full API, virtuals, GC
|
||||
|
||||
### Phase 4: End-to-End Tests
|
||||
- `e2e/` test suite with `testcontainers-go`
|
||||
- Proxy, Docker, management, virtual, terraform, GC tests
|
||||
- CI pipeline: `make e2e`
|
||||
- **Milestone**: comprehensive e2e coverage
|
||||
|
||||
### Phase 5: Terraform Provider
|
||||
- Separate repo: `terraform-provider-artifactapi`
|
||||
- Imports `pkg/client/` and `pkg/models/`
|
||||
- `artifactapi_remote` + `artifactapi_virtual` resources + data sources
|
||||
- Import support
|
||||
- **Milestone**: manage all config via Terraform
|
||||
|
||||
### Phase 6: Web UI
|
||||
- React + Vite in `ui/`
|
||||
- `Dockerfile.ui` (multi-stage → nginx)
|
||||
- Dashboard, remotes, objects, virtuals pages
|
||||
- SSE event feed
|
||||
- **Milestone**: full web UI in separate container
|
||||
|
||||
### Phase 7: TUI
|
||||
- Bubble Tea in `internal/tui/`
|
||||
- Uses `pkg/client/`
|
||||
- Dashboard, remotes, objects, virtuals views
|
||||
- **Milestone**: TUI feature parity with web UI
|
||||
|
||||
### Phase 8: Migration + Cutover
|
||||
- Migration tool: v2 YAML → Terraform HCL + `terraform import` commands
|
||||
- S3 rehash script: `{remote}/{hash16}/{file}` → `blobs/sha256/{content_hash}`
|
||||
- Parallel run, response comparison
|
||||
- Cutover
|
||||
|
||||
---
|
||||
|
||||
## Makefile Targets
|
||||
|
||||
```makefile
|
||||
.PHONY: build test lint fmt e2e docker docker-ui
|
||||
|
||||
build: ## Build Go binary
|
||||
go build -o bin/artifactapi ./cmd/artifactapi
|
||||
|
||||
test: ## Run unit tests
|
||||
go test ./...
|
||||
|
||||
lint: ## Run golangci-lint + go vet
|
||||
golangci-lint run ./...
|
||||
go vet ./...
|
||||
|
||||
fmt: ## Format code (gofmt + goimports)
|
||||
gofmt -w .
|
||||
goimports -w .
|
||||
|
||||
e2e: ## Run end-to-end tests (requires Docker)
|
||||
go test -tags=e2e -count=1 -timeout=5m ./e2e/...
|
||||
|
||||
docker: ## Build API server Docker image
|
||||
docker build -t artifactapi .
|
||||
|
||||
docker-ui: ## Build frontend Docker image
|
||||
docker build -t artifactapi-ui -f ui/Dockerfile.ui ui/
|
||||
|
||||
compose: ## Start full stack (API + UI + Postgres + Redis + MinIO)
|
||||
docker compose up -d
|
||||
```
|
||||
@@ -1,167 +1,469 @@
|
||||
# ArtifactAPI
|
||||
# Artifact Storage System
|
||||
|
||||
Caching proxy for package repositories. Single Go binary, 10 package types, content-addressable storage, managed by Terraform.
|
||||
FastAPI caching proxy that downloads and stores files from remote sources in S3-compatible storage.
|
||||
|
||||
## Quick Start
|
||||
## Features
|
||||
|
||||
```bash
|
||||
# Start backing services
|
||||
docker compose up -d postgres redis minio
|
||||
|
||||
# Build and run
|
||||
make build
|
||||
./bin/artifactapi
|
||||
|
||||
# Frontend (separate container or dev server)
|
||||
cd ui && npm install && npm run dev
|
||||
```
|
||||
|
||||
API: `http://localhost:8000` | Frontend: `http://localhost:5173`
|
||||
|
||||
## Package Types
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `generic` | nothing | everything |
|
||||
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
|
||||
| `helm` | `index.yaml` | `.tgz` charts |
|
||||
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
|
||||
| `npm` | package metadata | `.tgz` tarballs |
|
||||
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
|
||||
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
|
||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||
|
||||
Providers classify paths automatically. Users only configure what to proxy and TTLs.
|
||||
|
||||
## Terraform
|
||||
|
||||
Remotes and virtuals are managed by Terraform. Each package type has its own resource:
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 7200
|
||||
|
||||
patterns = [
|
||||
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
|
||||
"mikefarah/yq/.*/yq_linux_amd64$",
|
||||
]
|
||||
|
||||
mutable_patterns = [
|
||||
".*/archive/refs/heads/.*\\.tar\\.gz$",
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_docker" "dockerhub" {
|
||||
name = "dockerhub"
|
||||
base_url = "https://registry-1.docker.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
ban_tags_enabled = true
|
||||
ban_tags = ["latest"]
|
||||
|
||||
patterns = [
|
||||
"^library/postgres",
|
||||
"^library/redis",
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_helm" "jetstack" {
|
||||
name = "jetstack"
|
||||
base_url = "https://charts.jetstack.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 3600
|
||||
}
|
||||
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
members = [artifactapi_remote_helm.jetstack.name]
|
||||
}
|
||||
```
|
||||
|
||||
Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi)
|
||||
|
||||
## Access Control
|
||||
|
||||
| Field | Default | Behaviour |
|
||||
|---|---|---|
|
||||
| `patterns` | empty (proxy all) | If set, only matching paths are proxied. Acts as allowlist. |
|
||||
| `blocklist` | empty | Matching paths always denied. Checked first. |
|
||||
| `mutable_patterns` | empty | Override: force paths to mutable TTL. |
|
||||
| `immutable_patterns` | empty | Override: force paths to immutable TTL. |
|
||||
|
||||
No patterns + no blocklist = open proxy. Provider handles mutability classification automatically.
|
||||
|
||||
## API
|
||||
|
||||
### Proxy (v1)
|
||||
|
||||
```
|
||||
GET /api/v1/remote/{name}/{path} Proxy/cache artifact
|
||||
GET /api/v1/virtual/{name}/{path} Virtual repo (merged index)
|
||||
GET /v2/{name}/{path} Docker Registry v2
|
||||
```
|
||||
|
||||
### Management (v2)
|
||||
|
||||
```
|
||||
GET/POST /api/v2/remotes List / create remotes
|
||||
GET/PUT/DELETE /api/v2/remotes/{name} Read / update / delete remote
|
||||
GET/DELETE /api/v2/remotes/{name}/objects Browse / evict cached objects
|
||||
GET /api/v2/stats Overview stats
|
||||
GET /api/v2/health Service health
|
||||
POST /api/v2/probe Test a remote (fetch without streaming to client)
|
||||
GET /api/v2/events SSE event stream
|
||||
```
|
||||
- Remote definitions via `remotes.yaml` — generic HTTP, Alpine APK, RPM, Docker, PyPI, npm, Helm
|
||||
- Virtual repositories — merge multiple remotes of the same package type into a single unified index
|
||||
- Immutable/mutable caching model with per-remote TTLs
|
||||
- Conditional revalidation (`If-None-Match` / `If-Modified-Since`) on TTL expiry
|
||||
- Stale-on-upstream-error: refreshes TTL when backend is unreachable rather than evicting
|
||||
- URL rewriting for PyPI simple index, npm metadata, and Helm `index.yaml`
|
||||
- Access control via regex patterns — unmatched paths return 403
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
PostgreSQL ─── config (remotes, virtuals), artifact metadata, access log
|
||||
Redis ─── TTL keys, fetch locks, circuit breaker state
|
||||
S3/MinIO ─── content-addressable blob storage (blobs/sha256/{hash})
|
||||
client → /api/v1/remote/{remote}/{path}
|
||||
↓
|
||||
Redis: mutable TTL check
|
||||
↓ miss / expired
|
||||
S3: object exists?
|
||||
↓ no
|
||||
upstream remote → S3 + PostgreSQL metadata
|
||||
↓
|
||||
response (X-Artifact-Source: cache|remote)
|
||||
```
|
||||
|
||||
S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go).
|
||||
Docker Registry traffic uses the `/v2/{remote}/{path}` endpoint implementing the Docker Registry HTTP API v2.
|
||||
|
||||
## Environment Variables
|
||||
### Code layout
|
||||
|
||||
| Variable | Default | Description |
|
||||
```
|
||||
src/artifactapi/
|
||||
├── main.py — FastAPI app + thin route declarations only
|
||||
├── config.py — ConfigManager (loads remotes.yaml)
|
||||
├── metrics.py — Prometheus + Redis metrics
|
||||
├── docker_auth.py — backwards-compat shim → auth/docker.py
|
||||
├── artifact/ — route handler implementations
|
||||
│ ├── proxy.py — GET /api/v1/remote (remote proxy, cache, revalidation)
|
||||
│ ├── virtual.py — GET /api/v1/virtual (virtual repo index merging)
|
||||
│ ├── local.py — PUT/HEAD/DELETE /api/v1/remote (local repos)
|
||||
│ ├── docker.py — /v2/ Docker Registry v2 proxy
|
||||
│ ├── discovery.py — /api/v1/artifacts discovery + bulk cache
|
||||
│ └── flush.py — PUT /cache/flush
|
||||
├── auth/
|
||||
│ ├── __init__.py — re-exports Docker auth helpers
|
||||
│ └── docker.py — Bearer token fetching + in-memory cache
|
||||
├── cache/
|
||||
│ ├── __init__.py — re-exports RedisCache
|
||||
│ └── redis.py — RedisCache (TTL keys, ETag metadata)
|
||||
├── database/
|
||||
│ ├── __init__.py — re-exports DatabaseManager
|
||||
│ └── postgres.py — DatabaseManager (artifact + local-file tables)
|
||||
├── storage/
|
||||
│ ├── __init__.py — re-exports S3Storage
|
||||
│ └── s3.py — S3Storage (MinIO/S3 abstraction)
|
||||
└── remote/
|
||||
├── __init__.py
|
||||
├── base.py — content-type detection
|
||||
├── generic.py — generic HTTP remotes
|
||||
├── helm.py — Helm index.yaml URL rewriting
|
||||
├── npm.py — npm metadata URL rewriting
|
||||
├── python.py — PyPI URL construction + HTML rewriting
|
||||
└── rpm.py — RPM remotes
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| `LISTEN_ADDR` | `:8000` | Server listen address |
|
||||
| `DBHOST` | `localhost` | PostgreSQL host |
|
||||
| `DBPORT` | `5432` | PostgreSQL port |
|
||||
| `DBUSER` | `artifacts` | PostgreSQL user |
|
||||
| `DBPASS` | | PostgreSQL password |
|
||||
| `DBNAME` | `artifacts` | PostgreSQL database |
|
||||
| `REDIS_URL` | `redis://localhost:6379` | Redis URL |
|
||||
| `MINIO_ENDPOINT` | `localhost:9000` | S3 endpoint |
|
||||
| `MINIO_ACCESS_KEY` | | S3 access key |
|
||||
| `MINIO_SECRET_KEY` | | S3 secret key |
|
||||
| `MINIO_BUCKET` | `artifacts` | S3 bucket |
|
||||
| `MINIO_SECURE` | `false` | Use HTTPS for S3 |
|
||||
| `MINIO_REGION` | | S3 region (AWS) |
|
||||
| `GET` | `/api/v1/remote/{remote}/{path}` | Fetch artifact (auto-cache on miss) |
|
||||
| `GET` | `/api/v1/virtual/{virtual}/{path}` | Fetch from virtual (merged) repository |
|
||||
| `GET` | `/api/v1/local/{local}/{path}` | Download from local repository |
|
||||
| `PUT` | `/api/v1/local/{local}/{path}` | Upload to local repository |
|
||||
| `HEAD` | `/api/v1/local/{local}/{path}` | Check existence (local) |
|
||||
| `DELETE` | `/api/v1/local/{local}/{path}` | Delete from local repository |
|
||||
| `GET` | `/v2/{remote}/{path}` | Docker Registry v2 proxy |
|
||||
| `PUT` | `/cache/flush` | Flush cache entries |
|
||||
| `GET` | `/health` | Health check |
|
||||
| `GET` | `/config` | View loaded configuration |
|
||||
| `GET` | `/` | API info and available remotes |
|
||||
|
||||
## Development
|
||||
## Configuration
|
||||
|
||||
```bash
|
||||
make build # Build binary
|
||||
make test # Unit tests
|
||||
make e2e # E2E tests (needs Docker)
|
||||
make lint # golangci-lint + go vet
|
||||
make fmt # gofmt + goimports
|
||||
Runtime settings come from environment variables; remote definitions live in one or more YAML files pointed to by `CONFIG_PATH`.
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description |
|
||||
|---|---|
|
||||
| `CONFIG_PATH` | Path to a config YAML file **or** a directory of YAML files |
|
||||
| `DBHOST`, `DBPORT`, `DBUSER`, `DBPASS`, `DBNAME` | PostgreSQL connection |
|
||||
| `REDIS_URL` | Redis URL (e.g. `redis://localhost:6379`) |
|
||||
| `MINIO_ENDPOINT` | MinIO/S3 endpoint |
|
||||
| `MINIO_ACCESS_KEY` | S3 access key |
|
||||
| `MINIO_SECRET_KEY` | S3 secret key |
|
||||
| `MINIO_BUCKET` | S3 bucket name |
|
||||
| `MINIO_SECURE` | Use HTTPS (`true`/`false`) |
|
||||
|
||||
### Split configuration
|
||||
|
||||
`CONFIG_PATH` accepts three forms:
|
||||
|
||||
**Single file** (original behaviour):
|
||||
```
|
||||
CONFIG_PATH=/etc/artifactapi/remotes.yaml
|
||||
```
|
||||
|
||||
### TUI
|
||||
**Directory** — all `*.yaml` / `*.yml` files in the directory are loaded and merged alphabetically. `remotes` keys are merged across files; later files win on conflict:
|
||||
```
|
||||
CONFIG_PATH=/etc/artifactapi/conf.d/
|
||||
```
|
||||
|
||||
**Main file + `config_dir`** — the main file holds global settings and a `config_dir` pointer; each file in that directory contributes its own `remotes`. Relative `config_dir` paths are resolved relative to the main file:
|
||||
```yaml
|
||||
# /etc/artifactapi/config.yaml
|
||||
config_dir: conf.d # or an absolute path
|
||||
|
||||
# s3/redis/database settings go here (or in env vars)
|
||||
remotes: {} # optional base remotes
|
||||
```
|
||||
|
||||
### Configuration structure
|
||||
|
||||
Repositories are declared under three top-level keys matching their type:
|
||||
|
||||
```yaml
|
||||
remotes: # proxy (caching) remotes
|
||||
remote-name:
|
||||
base_url: "https://example.com"
|
||||
package: "generic" # generic, alpine, rpm, docker, pypi, npm, helm
|
||||
description: "..."
|
||||
immutable_patterns: # regex — cached forever
|
||||
- ".*\\.tar\\.gz$"
|
||||
mutable_patterns: # regex — expire after mutable_ttl
|
||||
- "index\\.yaml$"
|
||||
check_mutable_updates: false # send HEAD (If-None-Match) on TTL expiry
|
||||
cache:
|
||||
immutable_ttl: 0 # 0 = indefinitely
|
||||
mutable_ttl: 3600
|
||||
|
||||
virtuals: # virtual (merged-index) repositories
|
||||
virtual-name:
|
||||
package: "helm"
|
||||
members:
|
||||
- remote-a
|
||||
- remote-b
|
||||
|
||||
locals: # local upload repositories (no base_url)
|
||||
local-name:
|
||||
package: "generic"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 0
|
||||
```
|
||||
|
||||
## Remote Types
|
||||
|
||||
### generic
|
||||
|
||||
Arbitrary HTTP file servers — GitHub releases, HashiCorp, custom servers.
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
github:
|
||||
base_url: "https://github.com"
|
||||
package: "generic"
|
||||
immutable_patterns:
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
|
||||
github-archive:
|
||||
base_url: "https://github.com"
|
||||
package: "generic"
|
||||
immutable_patterns:
|
||||
- ".*/archive/refs/tags/.*\\.tar\\.gz$" # tag archives never change
|
||||
mutable_patterns:
|
||||
- ".*/archive/refs/heads/main\\.tar\\.gz$" # branch archives can change
|
||||
check_mutable_updates: true
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 86400
|
||||
```
|
||||
|
||||
Access: `GET /api/v1/remote/github/owner/repo/releases/download/v1.0/binary.tar.gz`
|
||||
|
||||
### alpine
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
alpine:
|
||||
base_url: "https://dl-cdn.alpinelinux.org"
|
||||
package: "alpine"
|
||||
immutable_patterns:
|
||||
- ".*/x86_64/.*\\.apk$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 7200
|
||||
```
|
||||
|
||||
`APKINDEX.tar.gz` is a built-in mutable pattern — no `mutable_patterns` entry needed.
|
||||
|
||||
### rpm
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
almalinux:
|
||||
base_url: "https://mirror.example.com/almalinux"
|
||||
package: "rpm"
|
||||
immutable_patterns:
|
||||
- ".*/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 7200
|
||||
```
|
||||
|
||||
`repomd.xml` and `repodata/` metadata files are built-in mutable patterns.
|
||||
|
||||
### docker
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
dockerhub:
|
||||
base_url: "https://registry-1.docker.io"
|
||||
package: "docker"
|
||||
# username / password optional for public images
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 300
|
||||
|
||||
ghcr:
|
||||
base_url: "https://ghcr.io"
|
||||
package: "docker"
|
||||
username: "your-github-username"
|
||||
password: "ghp_your_pat" # read:packages scope
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 300
|
||||
```
|
||||
|
||||
Tag manifests and `/tags/list` are built-in mutable patterns. Digest-addressed blobs are immutable.
|
||||
|
||||
For RKE2/containerd, configure `/etc/rancher/rke2/registries.yaml`:
|
||||
|
||||
```yaml
|
||||
mirrors:
|
||||
docker.io:
|
||||
endpoint:
|
||||
- "https://artifacts.example.com"
|
||||
rewrite:
|
||||
"^(.*)$": "dockerhub/$1"
|
||||
ghcr.io:
|
||||
endpoint:
|
||||
- "https://artifacts.example.com"
|
||||
rewrite:
|
||||
"^(.*)$": "ghcr/$1"
|
||||
```
|
||||
|
||||
### pypi
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
pypi:
|
||||
base_url: "https://files.pythonhosted.org"
|
||||
package: "pypi"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "packages/.*\\.whl$"
|
||||
- "packages/.*\\.whl\\.metadata$"
|
||||
- "packages/.*\\.tar\\.gz$"
|
||||
- "packages/.*\\.zip$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600
|
||||
```
|
||||
|
||||
> **Note**: Simple index requests (`/simple/{package}/`) are always fetched from `https://pypi.org`, regardless of `base_url`. This is hardcoded — `base_url` only controls where package files are downloaded from. For self-hosted registries (Gitea, Nexus) where both index and files share the same host, set `base_url` to that host and the override does not apply.
|
||||
|
||||
URLs in simple index HTML are rewritten to route package file downloads back through the same remote.
|
||||
|
||||
Configure uv:
|
||||
|
||||
```toml
|
||||
# /etc/uv/uv.toml or ~/.config/uv/uv.toml
|
||||
[[index]]
|
||||
url = "https://artifacts.example.com/api/v1/remote/pypi/simple"
|
||||
default = true
|
||||
```
|
||||
|
||||
### npm
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
npm:
|
||||
base_url: "https://registry.npmjs.org"
|
||||
package: "npm"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\.tgz$"
|
||||
mutable_patterns:
|
||||
- "^(?!.*\.tgz$).*"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600
|
||||
```
|
||||
|
||||
`dist.tarball` URLs in package metadata JSON are rewritten to route tarball downloads back through the same remote.
|
||||
|
||||
Configure npm / yarn / pnpm:
|
||||
|
||||
```ini
|
||||
# .npmrc or ~/.npmrc
|
||||
registry=https://artifacts.example.com/api/v1/remote/npm/
|
||||
```
|
||||
|
||||
### helm
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
hashicorp-helm:
|
||||
base_url: "https://helm.releases.hashicorp.com"
|
||||
package: "helm"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
```
|
||||
|
||||
`index.yaml` is a built-in mutable pattern. Chart URLs inside `index.yaml` are rewritten to route tarball downloads back through the same remote.
|
||||
|
||||
Configure Helm:
|
||||
|
||||
```bash
|
||||
./bin/artifactapi tui --endpoint http://localhost:8000
|
||||
helm repo add hashicorp https://artifacts.example.com/api/v1/remote/hashicorp-helm
|
||||
helm repo update
|
||||
```
|
||||
|
||||
### virtual
|
||||
|
||||
A virtual repository presents a single unified index built from multiple member remotes of the same package type. Clients configure one endpoint and get access to all member remotes transparently.
|
||||
|
||||
All members must share the same `package` type as the virtual repo. Currently supported package types: `helm`.
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
helm-hashicorp:
|
||||
base_url: "https://helm.releases.hashicorp.com"
|
||||
package: "helm"
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
helm-bitnami:
|
||||
base_url: "https://charts.bitnami.com/bitnami"
|
||||
package: "helm"
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
virtuals:
|
||||
helm-all:
|
||||
package: "helm"
|
||||
members:
|
||||
- helm-hashicorp # listed first = highest priority
|
||||
- helm-bitnami
|
||||
```
|
||||
|
||||
**How it works:**
|
||||
|
||||
1. A request for the package index triggers a parallel fetch of each member's index from S3 cache, falling back to upstream if not yet cached.
|
||||
2. Member indexes are merged into a single index with URL rewriting so artifact download URLs continue to resolve through the individual member remote.
|
||||
3. The merged index is cached in Redis with a TTL equal to the minimum `mutable_ttl` across all members.
|
||||
|
||||
**Priority / conflict resolution:**
|
||||
|
||||
When the same artifact name and version appears in more than one member, the member listed **first** in `members` wins. Subsequent members contribute only artifacts not already present.
|
||||
|
||||
**Partial failures:**
|
||||
|
||||
If a member is unreachable and has no cached index, it is skipped and a warning is logged. The merged index is still served from available members. If *no* members can be reached, the request returns `502`.
|
||||
|
||||
**Caching:**
|
||||
|
||||
The merged index is cached using `min(mutable_ttl)` across all members. Each member's raw index is cached in S3 under its own remote key; the virtual handler reuses those copies when available. On rebuild, each member's parsed index is also stored as a compact msgpack file (`index.msgpack`) alongside the raw YAML, eliminating the YAML parse cost on subsequent rebuilds.
|
||||
|
||||
**Helm example:**
|
||||
|
||||
```bash
|
||||
helm repo add all https://artifacts.example.com/api/v1/virtual/helm-all
|
||||
helm repo update
|
||||
```
|
||||
|
||||
Chart tarball URLs in the merged `index.yaml` are rewritten to point at the individual member remote (e.g. `…/api/v1/remote/helm-hashicorp/vault-0.27.0.tgz`), so downloads bypass the virtual endpoint entirely.
|
||||
|
||||
### local
|
||||
|
||||
```yaml
|
||||
locals:
|
||||
local-generic:
|
||||
package: "generic"
|
||||
description: "Local file repository"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 0
|
||||
```
|
||||
|
||||
No `base_url`. Files are uploaded via `PUT /api/v1/local/{name}/{path}` and downloaded via `GET /api/v1/local/{name}/{path}`.
|
||||
|
||||
## Caching Model
|
||||
|
||||
### Immutable patterns
|
||||
|
||||
Files matching `immutable_patterns` are cached for `immutable_ttl` seconds (0 = indefinitely). Use for versioned release artifacts that never change once published.
|
||||
|
||||
**Access control**: only paths matching an immutable or mutable pattern are served; all others return 403. Omitting `immutable_patterns` entirely allows all paths from that remote.
|
||||
|
||||
### Mutable patterns
|
||||
|
||||
Files matching `mutable_patterns` expire after `mutable_ttl` seconds and are re-fetched on the next request. Mutable files are always served regardless of `immutable_patterns`.
|
||||
|
||||
Each package type has built-in defaults that are merged with any user-defined `mutable_patterns`:
|
||||
|
||||
| Package type | Built-in mutable patterns |
|
||||
|---|---|
|
||||
| `alpine` | `APKINDEX\.tar\.gz$` |
|
||||
| `rpm` | `repomd\.xml$`, `repodata/` metadata variants, `Packages\.gz$` |
|
||||
| `docker` | Tag manifests (non-digest refs), `/tags/list` |
|
||||
| `pypi` | `simple/` (per-package and top-level index pages) |
|
||||
| `helm` | `index\.yaml$` |
|
||||
| `npm` | *(none built-in — define via `mutable_patterns`)* |
|
||||
| `generic` | *(none)* |
|
||||
|
||||
### Conditional revalidation
|
||||
|
||||
Set `check_mutable_updates: true` to send `HEAD` with `If-None-Match` / `If-Modified-Since` on TTL expiry. A 304 response refreshes the TTL without re-downloading. Only applies to user-defined `mutable_patterns` — built-in patterns are always re-fetched unconditionally.
|
||||
|
||||
### Stale-on-upstream-error
|
||||
|
||||
When a mutable file expires and the upstream is unreachable (connection refused, DNS failure, timeout), the cached copy is kept and its TTL refreshed. HTTP error responses (4xx, 5xx) are not treated as network failures and proceed with normal expiry.
|
||||
|
||||
### Quarantine (supply-chain protection)
|
||||
|
||||
Set `quarantine_new: true` and `quarantine_days: N` on a remote to block immutable artifacts published within the last N days. Requests return `404` until the quarantine period expires, giving time to detect malicious packages before they are consumed.
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
pypi:
|
||||
base_url: "https://files.pythonhosted.org"
|
||||
package: "pypi"
|
||||
quarantine_new: true
|
||||
quarantine_days: 3 # block packages published in the last 3 days
|
||||
immutable_patterns:
|
||||
- "packages/.*\\.whl$"
|
||||
- "packages/.*\\.tar\\.gz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600
|
||||
```
|
||||
|
||||
The upstream `Last-Modified` response header is used as the publish date proxy. Artifacts that have no `Last-Modified` header are allowed through (fail-open). Mutable files (index pages, tag manifests) are never quarantined.
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
# ArtifactAPI Specification
|
||||
|
||||
## Repository model
|
||||
|
||||
Every repository entry in `remotes.yaml` has two orthogonal fields:
|
||||
|
||||
| field | values | meaning |
|
||||
|---|---|---|
|
||||
| `type` | `local`, `remote`, `virtual` | repository kind — how the repo is served |
|
||||
| `package` | `docker`, `rpm`, `alpine`, `generic` | package format — what protocol and caching rules to apply |
|
||||
|
||||
**type**
|
||||
|
||||
- `local` — files are uploaded directly to the API and stored in S3; no upstream.
|
||||
- `remote` — proxies and caches content from an upstream URL (`base_url`).
|
||||
- `virtual` — aggregates multiple repositories (not yet implemented).
|
||||
|
||||
**package**
|
||||
|
||||
- `docker` — upstream speaks the OCI Distribution API (Bearer auth, manifest/blob paths).
|
||||
- `rpm` — upstream is an RPM repository; repodata files are index files.
|
||||
- `alpine` — upstream is an Alpine APK repository; `APKINDEX.tar.gz` is an index file.
|
||||
- `generic` — plain HTTP file download; no format-specific logic.
|
||||
|
||||
---
|
||||
|
||||
## Caching
|
||||
|
||||
Two cache classes determine retention:
|
||||
|
||||
| class | stored | TTL |
|
||||
|---|---|---|
|
||||
| **file** | S3 object, no Redis entry | `file_ttl` — `0` means indefinite |
|
||||
| **index** | S3 object + Redis TTL key | `index_ttl` — when the Redis key expires the S3 object is deleted and re-fetched |
|
||||
|
||||
Index files are mutable metadata that must expire. File-class objects are treated as immutable and cached indefinitely (unless `file_ttl` is set).
|
||||
|
||||
---
|
||||
|
||||
## Docker package rules
|
||||
|
||||
### URL construction
|
||||
|
||||
Remote URLs are prefixed with `/v2/` for `package: docker` remotes:
|
||||
|
||||
```
|
||||
{base_url}/v2/{path}
|
||||
```
|
||||
|
||||
e.g. `library/nginx/manifests/latest` → `https://registry-1.docker.io/v2/library/nginx/manifests/latest`
|
||||
|
||||
### Authentication
|
||||
|
||||
Docker registries use Bearer token challenges. On a `401 Unauthorized` response, the API:
|
||||
|
||||
1. Parses the `WWW-Authenticate: Bearer` header for `realm`, `service`, and `scope`.
|
||||
2. Fetches a token from the auth realm, supplying `username`/`password` from the remote config if present.
|
||||
3. Retries the request with `Authorization: Bearer <token>`.
|
||||
|
||||
Tokens are cached in-memory keyed by `(realm, service, scope, username)` and expire 30 seconds before their stated `expires_in`.
|
||||
|
||||
### Cache classification
|
||||
|
||||
| path pattern | mutable | class | TTL source |
|
||||
|---|---|---|---|
|
||||
| `/manifests/<tag>` | yes | index | `index_ttl` |
|
||||
| `/tags/list` | yes | index | `index_ttl` |
|
||||
| `/manifests/sha256:<digest>` | no | file | `file_ttl` |
|
||||
| `/blobs/sha256:<digest>` | no | file | `file_ttl` |
|
||||
|
||||
Tag-based manifests and tag lists are mutable and cached as index. Digest-pinned manifests and blobs are content-addressed and cached indefinitely as files.
|
||||
|
||||
### Blob deduplication
|
||||
|
||||
Blobs are stored under a digest-keyed path shared across all images on the same remote:
|
||||
|
||||
```
|
||||
{remote_name}/blobs/sha256/{digest}
|
||||
```
|
||||
|
||||
The same layer pulled by different images is stored once.
|
||||
|
||||
### Accept headers
|
||||
|
||||
| path | `Accept` header sent upstream |
|
||||
|---|---|
|
||||
| `/manifests/…` | `application/vnd.docker.distribution.manifest.v2+json`, `application/vnd.oci.image.manifest.v1+json`, `application/vnd.oci.image.index.v1+json`, `application/vnd.docker.distribution.manifest.list.v2+json` |
|
||||
| `/blobs/…` | `application/octet-stream` |
|
||||
|
||||
---
|
||||
|
||||
## OCI Distribution API endpoint
|
||||
|
||||
The API exposes a native Docker registry interface so clients can use `docker pull` directly:
|
||||
|
||||
```
|
||||
GET /v2/ — version ping
|
||||
GET /v2/{remote}/{image}/manifests/{ref} — fetch manifest
|
||||
HEAD /v2/{remote}/{image}/manifests/{ref} — manifest metadata
|
||||
GET /v2/{remote}/{image}/blobs/{digest} — fetch blob
|
||||
HEAD /v2/{remote}/{image}/blobs/{digest} — blob metadata
|
||||
```
|
||||
|
||||
Responses include `Docker-Distribution-Api-Version`, `Docker-Content-Digest`, and the correct OCI `Content-Type` (detected from the manifest `mediaType` field).
|
||||
|
||||
Only remotes with `package: docker` are accessible via this endpoint. All other remotes return `400`.
|
||||
|
||||
---
|
||||
|
||||
## include_patterns
|
||||
|
||||
`include_patterns` is a list of Python regexes applied to every request before any upstream fetch or cache lookup.
|
||||
|
||||
**Generic remotes (`/api/v1/remote/…`):**
|
||||
- Patterns match against the file path and the full path.
|
||||
- Index files (mutable metadata) bypass pattern checks and are always allowed.
|
||||
|
||||
**Docker remotes (`/v2/…`):**
|
||||
- Patterns match against the image name (first two path segments, e.g. `library/nginx`) and the full path.
|
||||
- The index-file exemption does **not** apply — patterns restrict whole images, including their manifests and tag lists.
|
||||
- No patterns configured → all images allowed.
|
||||
|
||||
Returns `403` when a request is blocked.
|
||||
|
||||
---
|
||||
|
||||
## Versioning
|
||||
|
||||
The package version is derived from git tags via `hatch-vcs`. Tags follow the format `v{MAJOR}.{MINOR}.{PATCH}`.
|
||||
|
||||
Docker images are built with the version injected at build time:
|
||||
|
||||
```
|
||||
SETUPTOOLS_SCM_PRETEND_VERSION=<version> uv sync --frozen
|
||||
```
|
||||
|
||||
The `Makefile` provides `patch`, `minor`, and `major` targets that tag the current commit and rebuild the container image.
|
||||
@@ -1,57 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
||||
"git.unkin.net/unkin/artifactapi/internal/server"
|
||||
"git.unkin.net/unkin/artifactapi/internal/tui"
|
||||
)
|
||||
|
||||
var version = "dev"
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == "tui" {
|
||||
endpoint := os.Getenv("ARTIFACTAPI_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
endpoint = "http://localhost:8000"
|
||||
}
|
||||
for i, arg := range os.Args {
|
||||
if arg == "--endpoint" && i+1 < len(os.Args) {
|
||||
endpoint = os.Args[i+1]
|
||||
}
|
||||
}
|
||||
|
||||
app := tui.New(endpoint)
|
||||
if err := app.Run(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "tui error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
slog.Error("failed to load config", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
srv, err := server.New(cfg, version)
|
||||
if err != nil {
|
||||
slog.Error("failed to create server", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := srv.Run(ctx); err != nil {
|
||||
slog.Error("server exited with error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
# Overlay for the dockerised end-to-end suite (scripts/docker-e2e.sh).
|
||||
# Adds a static mock upstream that the artifactapi container proxies, so the
|
||||
# caching tests are hermetic and need no internet access.
|
||||
services:
|
||||
mockupstream:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./e2e-docker/fixtures:/usr/share/nginx/html:ro,z
|
||||
# No host port needed: only the artifactapi container talks to it, and the
|
||||
# tests compare served bytes against the on-disk fixtures.
|
||||
|
||||
artifactapi:
|
||||
# The host port is set via ARTIFACTAPI_PORT (see scripts/docker-e2e.sh),
|
||||
# defaulting to 8000; the e2e run uses 8001 to avoid colliding with a
|
||||
# locally-running instance.
|
||||
depends_on:
|
||||
mockupstream:
|
||||
condition: service_started
|
||||
+56
-56
@@ -1,22 +1,31 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
artifactapi:
|
||||
build: .
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
- VERSION=2.2.2.dev0
|
||||
ports:
|
||||
- "${ARTIFACTAPI_PORT:-8000}:8000"
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- ./examples/single-file/remotes.yaml:/app/remotes.yaml:ro,z
|
||||
- ./ca-bundle.pem:/app/ca-bundle.pem:ro,z
|
||||
environment:
|
||||
LISTEN_ADDR: ":8000"
|
||||
DBHOST: postgres
|
||||
DBPORT: "5432"
|
||||
DBUSER: artifacts
|
||||
DBPASS: artifacts123
|
||||
DBNAME: artifacts
|
||||
DBSSL: disable
|
||||
REDIS_URL: redis://redis:6379
|
||||
MINIO_ENDPOINT: minio:9000
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
MINIO_BUCKET: artifacts
|
||||
MINIO_SECURE: "false"
|
||||
- CONFIG_PATH=/app/remotes.yaml
|
||||
- DBHOST=postgres
|
||||
- DBPORT=5432
|
||||
- DBUSER=artifacts
|
||||
- DBPASS=artifacts123
|
||||
- DBNAME=artifacts
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- MINIO_ENDPOINT=minio:9000
|
||||
- MINIO_ACCESS_KEY=minioadmin
|
||||
- MINIO_SECRET_KEY=minioadmin
|
||||
- MINIO_BUCKET=artifacts
|
||||
- MINIO_SECURE=false
|
||||
- REQUESTS_CA_BUNDLE=/app/ca-bundle.pem
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -25,35 +34,27 @@ services:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8000/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
ui:
|
||||
build:
|
||||
context: ui
|
||||
dockerfile: Dockerfile.ui
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
ports:
|
||||
- "8080:80"
|
||||
depends_on:
|
||||
- artifactapi
|
||||
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
ports:
|
||||
- "5432:5432"
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
environment:
|
||||
POSTGRES_USER: artifacts
|
||||
POSTGRES_PASSWORD: artifacts123
|
||||
POSTGRES_DB: artifacts
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
command: server /data --console-address ":9001"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U artifacts -d artifacts"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
@@ -64,28 +65,27 @@ services:
|
||||
command: redis-server --save 20 1
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
- "5432:5432"
|
||||
environment:
|
||||
POSTGRES_DB: artifacts
|
||||
POSTGRES_USER: artifacts
|
||||
POSTGRES_PASSWORD: artifacts123
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
test: ["CMD-SHELL", "pg_isready -U artifacts -d artifacts"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
redis_data:
|
||||
postgres_data:
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
# Dockerised end-to-end suite
|
||||
|
||||
Black-box tests that run against a fully **containerised** artifactapi stack
|
||||
(built image + Postgres + Redis + MinIO) plus a static mock upstream. Unlike the
|
||||
in-process `e2e/` suite (testcontainers, server run in-process), these only speak
|
||||
HTTP to the running product, so they exercise the shipped container image.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
make docker-e2e # build image, compose up, run suite, compose down
|
||||
```
|
||||
|
||||
`scripts/docker-e2e.sh` builds and starts `docker-compose.yml` +
|
||||
`docker-compose.e2e.yml`, waits for `/health`, then runs
|
||||
`go test -tags=dockere2e ./e2e-docker/...` and tears everything down.
|
||||
|
||||
The stack publishes artifactapi on host port **8001** (to avoid colliding with a
|
||||
local instance on 8000). Override with `ARTIFACTAPI_URL` to point the tests at an
|
||||
already-running stack.
|
||||
|
||||
## Coverage
|
||||
|
||||
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
|
||||
- **Caching** — one immutable artifact per remote package type (generic, docker,
|
||||
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
|
||||
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
||||
verified against the origin fixture.
|
||||
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
|
||||
index), rpm (real package + **automatic repodata** generation).
|
||||
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
|
||||
across two members.
|
||||
|
||||
## Fixtures
|
||||
|
||||
`fixtures/` is served by the mock upstream at its web root. Paths mirror each
|
||||
provider's upstream URL layout (e.g. `v2/...` for docker, `v1/providers/...` for
|
||||
terraform). The RPM under `fixtures/rpmrepo/Packages/` is a real package so the
|
||||
rpm provider can parse its metadata for repodata generation.
|
||||
@@ -1,76 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestCachingPerProvider proxies one immutable artifact for every remote
|
||||
// package type through the mock upstream and asserts: first fetch is served
|
||||
// from the remote, the second from cache, and the bytes match the origin.
|
||||
func TestCachingPerProvider(t *testing.T) {
|
||||
cases := []struct {
|
||||
pkgType string
|
||||
// path is the request path under /api/v1/remote/<name>/. The provider
|
||||
// derives the upstream URL from it (docker prepends /v2/, terraform
|
||||
// prepends /v1/providers/), and the fixture lives at that resolved path.
|
||||
path string
|
||||
fixture string
|
||||
}{
|
||||
{"generic", "blobs/hello.bin", "blobs/hello.bin"},
|
||||
{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"},
|
||||
{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"},
|
||||
{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"},
|
||||
{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"},
|
||||
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
|
||||
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
|
||||
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
|
||||
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
|
||||
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.pkgType, func(t *testing.T) {
|
||||
name := "cache-" + tc.pkgType
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": %q,
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"stale_on_error": true
|
||||
}`, name, tc.pkgType, mockUpstream()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
want := fixtureBytes(t, tc.fixture)
|
||||
url := api("/api/v1/remote/" + name + "/" + tc.path)
|
||||
|
||||
// First fetch: from remote.
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("first fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if src := resp.Header.Get("X-Artifact-Source"); src != "remote" {
|
||||
t.Fatalf("first fetch source = %q, want remote", src)
|
||||
}
|
||||
if !bytes.Equal(body, want) {
|
||||
t.Fatalf("first fetch body mismatch: got %d bytes, want %d", len(body), len(want))
|
||||
}
|
||||
|
||||
// Second fetch: from cache, identical bytes.
|
||||
resp, body = doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("second fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if src := resp.Header.Get("X-Artifact-Source"); src != "cache" {
|
||||
t.Fatalf("second fetch source = %q, want cache", src)
|
||||
}
|
||||
if !bytes.Equal(body, want) {
|
||||
t.Fatalf("cached body mismatch: got %d bytes, want %d", len(body), len(want))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
hello artifactapi generic blob
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,8 +0,0 @@
|
||||
apiVersion: v1
|
||||
entries:
|
||||
alpha:
|
||||
- name: alpha
|
||||
version: 1.0.0
|
||||
urls:
|
||||
- charts/alpha-1.0.0.tgz
|
||||
generated: "2026-01-01T00:00:00Z"
|
||||
@@ -1,8 +0,0 @@
|
||||
apiVersion: v1
|
||||
entries:
|
||||
beta:
|
||||
- name: beta
|
||||
version: 2.0.0
|
||||
urls:
|
||||
- charts/beta-2.0.0.tgz
|
||||
generated: "2026-01-01T00:00:00Z"
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,108 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
// Package e2edocker holds the black-box end-to-end suite that runs against a
|
||||
// fully dockerised artifactapi stack (see scripts/docker-e2e.sh). Unlike the
|
||||
// in-process e2e suite, these tests only speak HTTP to the running container.
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func baseURL() string {
|
||||
if v := os.Getenv("ARTIFACTAPI_URL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://localhost:8000"
|
||||
}
|
||||
|
||||
// mockUpstream is the base URL the artifactapi *container* uses to reach the
|
||||
// static mock upstream. It is resolved on the compose network, not the host.
|
||||
func mockUpstream() string {
|
||||
if v := os.Getenv("MOCK_UPSTREAM_INTERNAL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://mockupstream"
|
||||
}
|
||||
|
||||
func api(path string) string { return baseURL() + path }
|
||||
|
||||
func fixtureBytes(t *testing.T, rel string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join("fixtures", rel))
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture %s: %v", rel, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func doRequest(t *testing.T, method, url string, body []byte, contentType string) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
var r io.Reader
|
||||
if body != nil {
|
||||
r = bytes.NewReader(body)
|
||||
}
|
||||
req, err := http.NewRequest(method, url, r)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, url, err)
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return resp, respBody
|
||||
}
|
||||
|
||||
func createRepo(t *testing.T, jsonBody string) {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodPost, api("/api/v2/remotes"), []byte(jsonBody), "application/json")
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create repo: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRepo(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
doRequest(t, http.MethodDelete, api("/api/v2/remotes/"+name), nil, "")
|
||||
}
|
||||
|
||||
func createVirtual(t *testing.T, jsonBody string) {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodPost, api("/api/v2/virtuals"), []byte(jsonBody), "application/json")
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteVirtual(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
doRequest(t, http.MethodDelete, api("/api/v2/virtuals/"+name), nil, "")
|
||||
}
|
||||
|
||||
// getEventually retries a GET until it returns 200 or the deadline passes. Used
|
||||
// for asynchronously-generated artifacts (e.g. rpm repodata after upload).
|
||||
func getEventually(t *testing.T, url string, timeout time.Duration) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
var resp *http.Response
|
||||
var body []byte
|
||||
for {
|
||||
resp, body = doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode == http.StatusOK || time.Now().After(deadline) {
|
||||
return resp, body
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func uploadFile(t *testing.T, repo, filePath string, body []byte, contentType string) {
|
||||
t.Helper()
|
||||
url := api("/api/v2/remotes/" + repo + "/files/" + filePath)
|
||||
resp, respBody := doRequest(t, http.MethodPut, url, body, contentType)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s: status %d: %s", filePath, resp.StatusCode, respBody)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalGenericUpload uploads a generic file and downloads it back.
|
||||
func TestLocalGenericUpload(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-generic","package_type":"generic","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-generic")
|
||||
|
||||
content := []byte("artifactapi local generic upload payload")
|
||||
uploadFile(t, "local-generic", "data/hello.bin", content, "application/octet-stream")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-generic/data/hello.bin"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, content) {
|
||||
t.Fatalf("downloaded content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalPyPIUpload uploads a wheel and validates the generated simple index.
|
||||
func TestLocalPyPIUpload(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-pypi","package_type":"pypi","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-pypi")
|
||||
|
||||
wheel := fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl")
|
||||
uploadFile(t, "local-pypi", "foo-1.0-py3-none-any.whl", wheel, "application/zip")
|
||||
|
||||
// Root index lists the package.
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("simple index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !strings.Contains(string(body), "foo") {
|
||||
t.Fatalf("simple index missing package 'foo': %s", body)
|
||||
}
|
||||
|
||||
// Per-package index lists the wheel file.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/foo/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("package index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !strings.Contains(string(body), "foo-1.0-py3-none-any.whl") {
|
||||
t.Fatalf("package index missing wheel: %s", body)
|
||||
}
|
||||
|
||||
// The wheel downloads back byte-identical.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/foo/foo-1.0-py3-none-any.whl"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download wheel: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, wheel) {
|
||||
t.Fatalf("wheel content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalRPMRepodata uploads a real RPM and validates that repodata is
|
||||
// generated automatically (the special rpm-local feature).
|
||||
func TestLocalRPMRepodata(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-rpm","package_type":"rpm","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-rpm")
|
||||
|
||||
rpm := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
|
||||
uploadFile(t, "local-rpm", "e2e-testpkg-1.0-1.noarch.rpm", rpm, "application/x-rpm")
|
||||
|
||||
// repodata is generated asynchronously after upload; poll for it.
|
||||
resp, body := getEventually(t, api("/api/v1/local/local-rpm/repodata/repomd.xml"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("repomd.xml: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "<repomd") || !strings.Contains(s, "primary") {
|
||||
t.Fatalf("repomd.xml not a valid repodata document: %s", s)
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHealth(t *testing.T) {
|
||||
resp, body := doRequest(t, http.MethodGet, api("/health"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("health: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRemoteLifecycle covers add/change/delete for a remote repository.
|
||||
func TestRemoteLifecycle(t *testing.T) {
|
||||
createRepo(t, `{
|
||||
"name": "crud-remote",
|
||||
"package_type": "generic",
|
||||
"repo_type": "remote",
|
||||
"base_url": "https://example.com",
|
||||
"mutable_ttl": 600,
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRepo(t, "crud-remote")
|
||||
|
||||
got := getRepo(t, "crud-remote")
|
||||
if got["base_url"] != "https://example.com" || got["mutable_ttl"].(float64) != 600 {
|
||||
t.Fatalf("unexpected created remote: %v", got)
|
||||
}
|
||||
|
||||
// change
|
||||
resp, body := doRequest(t, http.MethodPut, api("/api/v2/remotes/crud-remote"), []byte(`{
|
||||
"package_type": "generic",
|
||||
"base_url": "https://updated.example.com",
|
||||
"mutable_ttl": 120,
|
||||
"stale_on_error": true
|
||||
}`), "application/json")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update remote: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
got = getRepo(t, "crud-remote")
|
||||
if got["base_url"] != "https://updated.example.com" || got["mutable_ttl"].(float64) != 120 {
|
||||
t.Fatalf("update not applied: %v", got)
|
||||
}
|
||||
|
||||
// delete
|
||||
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-remote"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete remote: status %d", resp.StatusCode)
|
||||
}
|
||||
resp, _ = doRequest(t, http.MethodGet, api("/api/v2/remotes/crud-remote"), nil, "")
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("expected 404 after delete, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalLifecycle covers add/delete for a local repository.
|
||||
func TestLocalLifecycle(t *testing.T) {
|
||||
createRepo(t, `{
|
||||
"name": "crud-local",
|
||||
"package_type": "generic",
|
||||
"repo_type": "local"
|
||||
}`)
|
||||
defer deleteRepo(t, "crud-local")
|
||||
|
||||
got := getRepo(t, "crud-local")
|
||||
if got["repo_type"] != "local" {
|
||||
t.Fatalf("expected repo_type local, got %v", got["repo_type"])
|
||||
}
|
||||
|
||||
resp, _ := doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-local"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete local: status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVirtualLifecycle covers add/change/delete for a virtual repository.
|
||||
func TestVirtualLifecycle(t *testing.T) {
|
||||
createRepo(t, `{"name":"vmem-a","package_type":"helm","repo_type":"remote","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRepo(t, `{"name":"vmem-b","package_type":"helm","repo_type":"remote","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRepo(t, "vmem-a")
|
||||
defer deleteRepo(t, "vmem-b")
|
||||
|
||||
createVirtual(t, `{
|
||||
"name": "crud-virtual",
|
||||
"package_type": "helm",
|
||||
"members": ["vmem-a"]
|
||||
}`)
|
||||
defer deleteVirtual(t, "crud-virtual")
|
||||
|
||||
// change members
|
||||
resp, body := doRequest(t, http.MethodPut, api("/api/v2/virtuals/crud-virtual"), []byte(`{
|
||||
"package_type": "helm",
|
||||
"members": ["vmem-a", "vmem-b"]
|
||||
}`), "application/json")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v2/virtuals/crud-virtual"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("get virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
var v map[string]any
|
||||
if err := json.Unmarshal(body, &v); err != nil {
|
||||
t.Fatalf("decode virtual: %v", err)
|
||||
}
|
||||
members, _ := v["members"].([]any)
|
||||
if len(members) != 2 {
|
||||
t.Fatalf("expected 2 members after update, got %v", v["members"])
|
||||
}
|
||||
|
||||
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/virtuals/crud-virtual"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete virtual: status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func getRepo(t *testing.T, name string) map[string]any {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v2/remotes/"+name), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("get remote %s: status %d: %s", name, resp.StatusCode, body)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
t.Fatalf("decode remote %s: %v", name, err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
|
||||
// checks that a virtual over them serves a merged simple index.
|
||||
func TestVirtualPyPIMerge(t *testing.T) {
|
||||
createRepo(t, `{"name":"pmerge-a","package_type":"pypi","repo_type":"local"}`)
|
||||
createRepo(t, `{"name":"pmerge-b","package_type":"pypi","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "pmerge-a")
|
||||
defer deleteRepo(t, "pmerge-b")
|
||||
|
||||
uploadFile(t, "pmerge-a", "foo-1.0-py3-none-any.whl", fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl"), "application/zip")
|
||||
uploadFile(t, "pmerge-b", "bar-2.0-py3-none-any.whl", []byte("bar wheel payload"), "application/zip")
|
||||
|
||||
createVirtual(t, `{"name":"pmerge-v","package_type":"pypi","members":["pmerge-a","pmerge-b"]}`)
|
||||
defer deleteVirtual(t, "pmerge-v")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/pmerge-v/simple/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("virtual simple index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "foo") || !strings.Contains(s, "bar") {
|
||||
t.Fatalf("merged index missing a member package (want foo and bar): %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVirtualHelmMerge points two helm remotes at mock index.yaml documents
|
||||
// with distinct charts and checks the virtual merges both into one index.
|
||||
func TestVirtualHelmMerge(t *testing.T) {
|
||||
createRepo(t, `{"name":"hmerge-a","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-a","stale_on_error":true}`)
|
||||
createRepo(t, `{"name":"hmerge-b","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-b","stale_on_error":true}`)
|
||||
defer deleteRepo(t, "hmerge-a")
|
||||
defer deleteRepo(t, "hmerge-b")
|
||||
|
||||
createVirtual(t, `{"name":"hmerge-v","package_type":"helm","members":["hmerge-a","hmerge-b"]}`)
|
||||
defer deleteVirtual(t, "hmerge-v")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/hmerge-v/index.yaml"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("virtual index.yaml: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "alpha") || !strings.Contains(s, "beta") {
|
||||
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
|
||||
}
|
||||
}
|
||||
-137
@@ -1,137 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres"
|
||||
tcredis "github.com/testcontainers/testcontainers-go/modules/redis"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
||||
"git.unkin.net/unkin/artifactapi/internal/server"
|
||||
)
|
||||
|
||||
var baseURL string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
|
||||
pgContainer, err := tcpostgres.Run(ctx,
|
||||
"postgres:17-alpine",
|
||||
tcpostgres.WithDatabase("artifacts"),
|
||||
tcpostgres.WithUsername("artifacts"),
|
||||
tcpostgres.WithPassword("artifacts123"),
|
||||
testcontainers.WithWaitStrategy(
|
||||
wait.ForListeningPort("5432/tcp").WithStartupTimeout(30*time.Second),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer pgContainer.Terminate(ctx)
|
||||
|
||||
redisContainer, err := tcredis.Run(ctx,
|
||||
"redis:7-alpine",
|
||||
testcontainers.WithWaitStrategy(
|
||||
wait.ForListeningPort("6379/tcp").WithStartupTimeout(30*time.Second),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("redis: %v", err)
|
||||
}
|
||||
defer redisContainer.Terminate(ctx)
|
||||
|
||||
minioContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
ContainerRequest: testcontainers.ContainerRequest{
|
||||
Image: "minio/minio:latest",
|
||||
ExposedPorts: []string{"9000/tcp"},
|
||||
Cmd: []string{"server", "/data"},
|
||||
Env: map[string]string{
|
||||
"MINIO_ROOT_USER": "minioadmin",
|
||||
"MINIO_ROOT_PASSWORD": "minioadmin",
|
||||
},
|
||||
WaitingFor: wait.ForHTTP("/minio/health/live").WithPort("9000/tcp").WithStartupTimeout(30 * time.Second),
|
||||
},
|
||||
Started: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("minio: %v", err)
|
||||
}
|
||||
defer minioContainer.Terminate(ctx)
|
||||
|
||||
pgHost, _ := pgContainer.Host(ctx)
|
||||
pgPort, _ := pgContainer.MappedPort(ctx, "5432/tcp")
|
||||
redisHost, _ := redisContainer.Host(ctx)
|
||||
redisPort, _ := redisContainer.MappedPort(ctx, "6379/tcp")
|
||||
minioHost, _ := minioContainer.Host(ctx)
|
||||
minioPort, _ := minioContainer.MappedPort(ctx, "9000/tcp")
|
||||
|
||||
os.Setenv("DBHOST", pgHost)
|
||||
os.Setenv("DBPORT", pgPort.Port())
|
||||
os.Setenv("DBUSER", "artifacts")
|
||||
os.Setenv("DBPASS", "artifacts123")
|
||||
os.Setenv("DBNAME", "artifacts")
|
||||
os.Setenv("DBSSL", "disable")
|
||||
os.Setenv("REDIS_URL", fmt.Sprintf("redis://%s:%s", redisHost, redisPort.Port()))
|
||||
os.Setenv("MINIO_ENDPOINT", fmt.Sprintf("%s:%s", minioHost, minioPort.Port()))
|
||||
os.Setenv("MINIO_ACCESS_KEY", "minioadmin")
|
||||
os.Setenv("MINIO_SECRET_KEY", "minioadmin")
|
||||
os.Setenv("MINIO_BUCKET", "artifacts-test")
|
||||
os.Setenv("MINIO_SECURE", "false")
|
||||
os.Setenv("LISTEN_ADDR", ":0")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
cfg.ListenAddr = "127.0.0.1:0"
|
||||
|
||||
srv, err := server.New(cfg, "e2e-test")
|
||||
if err != nil {
|
||||
log.Fatalf("server: %v", err)
|
||||
}
|
||||
|
||||
srvCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
addr := startServer(srvCtx, srv)
|
||||
baseURL = "http://" + addr
|
||||
|
||||
code := m.Run()
|
||||
cancel()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func startServer(ctx context.Context, srv *server.Server) string {
|
||||
ln, err := findListener()
|
||||
if err != nil {
|
||||
log.Fatalf("listener: %v", err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
|
||||
go srv.RunOnListener(ctx, ln)
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := http.Get("http://" + addr + "/health")
|
||||
if err == nil && resp.StatusCode == 200 {
|
||||
resp.Body.Close()
|
||||
return addr
|
||||
}
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
log.Fatalf("server did not start in time at %s", addr)
|
||||
return ""
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func findListener() (net.Listener, error) {
|
||||
return net.Listen("tcp", "127.0.0.1:0")
|
||||
}
|
||||
|
||||
func apiURL(path string) string {
|
||||
return baseURL + path
|
||||
}
|
||||
|
||||
func createRemote(t *testing.T, body string) {
|
||||
t.Helper()
|
||||
resp, err := http.Post(apiURL("/api/v2/remotes"), "application/json", strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("create remote: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("create remote: status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRemote(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(http.MethodDelete, apiURL("/api/v2/remotes/"+name), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("delete remote: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
func getJSON(t *testing.T, url string) map[string]any {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("GET %s: status %d: %s", url, resp.StatusCode, b)
|
||||
}
|
||||
var result map[string]any
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func getBody(t *testing.T, url string) ([]byte, int) {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return b, resp.StatusCode
|
||||
}
|
||||
|
||||
func getString(t *testing.T, url string) string {
|
||||
t.Helper()
|
||||
b, status := getBody(t, url)
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("GET %s: status %d: %s", url, status, b)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func assertStatus(t *testing.T, url string, wantStatus int) {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != wantStatus {
|
||||
t.Errorf("GET %s: got %d, want %d", url, resp.StatusCode, wantStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRequest(t *testing.T, url string) int {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(http.MethodDelete, url, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("DELETE %s: %v", url, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
func mustFormat(format string, args ...any) string {
|
||||
return fmt.Sprintf(format, args...)
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHealth(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/health"))
|
||||
if result["status"] != "ok" {
|
||||
t.Errorf("expected ok, got %v", result["status"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoot(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/"))
|
||||
if result["name"] != "artifactapi" {
|
||||
t.Errorf("expected artifactapi, got %v", result["name"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUpstreamTimeouts(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "timeout-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"stale_on_error": true,
|
||||
"upstream_dial_timeout": 3,
|
||||
"upstream_tls_timeout": 4,
|
||||
"upstream_response_header_timeout": 5
|
||||
}`)
|
||||
defer deleteRemote(t, "timeout-test")
|
||||
|
||||
remote := getJSON(t, apiURL("/api/v2/remotes/timeout-test"))
|
||||
for field, want := range map[string]float64{
|
||||
"upstream_dial_timeout": 3,
|
||||
"upstream_tls_timeout": 4,
|
||||
"upstream_response_header_timeout": 5,
|
||||
} {
|
||||
if got, _ := remote[field].(float64); got != want {
|
||||
t.Errorf("%s: got %v, want %v", field, remote[field], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteCRUD(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "test-generic",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"description": "test remote",
|
||||
"mutable_ttl": 600,
|
||||
"check_mutable": true,
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "test-generic")
|
||||
|
||||
remote := getJSON(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if remote["name"] != "test-generic" {
|
||||
t.Errorf("expected test-generic, got %v", remote["name"])
|
||||
}
|
||||
if remote["package_type"] != "generic" {
|
||||
t.Errorf("expected generic, got %v", remote["package_type"])
|
||||
}
|
||||
|
||||
req, _ := http.NewRequest(http.MethodPut, apiURL("/api/v2/remotes/test-generic"),
|
||||
strings.NewReader(`{
|
||||
"package_type": "generic",
|
||||
"base_url": "https://updated.example.com",
|
||||
"description": "updated",
|
||||
"mutable_ttl": 300,
|
||||
"check_mutable": true,
|
||||
"stale_on_error": true
|
||||
}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
updated := getJSON(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if updated["base_url"] != "https://updated.example.com" {
|
||||
t.Errorf("expected updated URL, got %v", updated["base_url"])
|
||||
}
|
||||
|
||||
status := deleteRequest(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if status != http.StatusNoContent {
|
||||
t.Errorf("delete: got %d, want 204", status)
|
||||
}
|
||||
|
||||
assertStatus(t, apiURL("/api/v2/remotes/test-generic"), http.StatusNotFound)
|
||||
}
|
||||
|
||||
func TestRemoteList(t *testing.T) {
|
||||
createRemote(t, `{"name":"list-a","package_type":"generic","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRemote(t, `{"name":"list-b","package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRemote(t, "list-a")
|
||||
defer deleteRemote(t, "list-b")
|
||||
|
||||
resp, err := http.Get(apiURL("/api/v2/remotes"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
|
||||
var remotes []map[string]any
|
||||
json.Unmarshal(body, &remotes)
|
||||
|
||||
if len(remotes) < 2 {
|
||||
t.Fatalf("expected at least 2 remotes, got %d", len(remotes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualCRUD(t *testing.T) {
|
||||
createRemote(t, `{"name":"virt-member-a","package_type":"helm","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRemote(t, `{"name":"virt-member-b","package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRemote(t, "virt-member-a")
|
||||
defer deleteRemote(t, "virt-member-b")
|
||||
|
||||
resp, err := http.Post(apiURL("/api/v2/virtuals"), "application/json",
|
||||
strings.NewReader(`{
|
||||
"name": "test-virtual",
|
||||
"package_type": "helm",
|
||||
"description": "test virtual",
|
||||
"members": ["virt-member-a", "virt-member-b"]
|
||||
}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create virtual: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
virt := getJSON(t, apiURL("/api/v2/virtuals/test-virtual"))
|
||||
if virt["name"] != "test-virtual" {
|
||||
t.Errorf("expected test-virtual, got %v", virt["name"])
|
||||
}
|
||||
|
||||
status := deleteRequest(t, apiURL("/api/v2/virtuals/test-virtual"))
|
||||
if status != http.StatusNoContent {
|
||||
t.Errorf("delete virtual: got %d, want 204", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsEndpoint(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/api/v2/stats"))
|
||||
if _, ok := result["total_remotes"]; !ok {
|
||||
t.Error("expected total_remotes in stats")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthV2(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/api/v2/health"))
|
||||
if result["status"] != "ok" {
|
||||
t.Errorf("expected ok, got %v", result["status"])
|
||||
}
|
||||
if result["postgres"] != "ok" {
|
||||
t.Errorf("expected postgres ok, got %v", result["postgres"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidPackageType(t *testing.T) {
|
||||
resp, err := http.Post(apiURL("/api/v2/remotes"), "application/json",
|
||||
strings.NewReader(`{"name":"bad","package_type":"bogus","base_url":"https://x.com"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("expected 400 for invalid package type, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProxyUnknownRemote(t *testing.T) {
|
||||
assertStatus(t, apiURL("/api/v1/remote/nonexistent/some/path"), http.StatusNotFound)
|
||||
}
|
||||
|
||||
func TestProxyBlocklist(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "blocklist-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"blocklist": ["\\.exe$"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "blocklist-test")
|
||||
|
||||
assertStatus(t, apiURL("/api/v1/remote/blocklist-test/malware.exe"), http.StatusForbidden)
|
||||
}
|
||||
|
||||
func TestProxyHeadBlocklist(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "head-block-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"blocklist": ["\\.exe$"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "head-block-test")
|
||||
|
||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/head-block-test/malware.exe"), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("HEAD blocklisted path: got %d, want 403", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyHeadUnknownRemote(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/nonexistent/some/path"), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("HEAD unknown remote: got %d, want 404", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyPatterns(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "patterns-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"patterns": ["^releases/"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "patterns-test")
|
||||
|
||||
assertStatus(t, apiURL("/api/v1/remote/patterns-test/uploads/file.tar.gz"), http.StatusForbidden)
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
remotes:
|
||||
alpine:
|
||||
base_url: "https://dl-cdn.alpinelinux.org"
|
||||
package: "alpine"
|
||||
description: "Alpine Linux APK package repository"
|
||||
immutable_patterns:
|
||||
- ".*/x86_64/.*\\.apk$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 7200
|
||||
@@ -0,0 +1,11 @@
|
||||
remotes:
|
||||
github:
|
||||
base_url: "https://github.com"
|
||||
package: "generic"
|
||||
description: "GitHub releases and files"
|
||||
immutable_patterns:
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 0
|
||||
@@ -0,0 +1,16 @@
|
||||
remotes:
|
||||
pypi:
|
||||
base_url: "https://files.pythonhosted.org"
|
||||
package: "pypi"
|
||||
description: "Python Package Index"
|
||||
check_mutable_updates: true
|
||||
quarantine_new: true
|
||||
quarantine_days: 3
|
||||
immutable_patterns:
|
||||
- "packages/.*\\.whl$"
|
||||
- "packages/.*\\.whl\\.metadata$"
|
||||
- "packages/.*\\.tar\\.gz$"
|
||||
- "packages/.*\\.zip$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600
|
||||
@@ -0,0 +1,487 @@
|
||||
# Example remotes configuration — copy and adapt for your environment.
|
||||
#
|
||||
# immutable_patterns: artifacts cached forever (e.g. release binaries, versioned tags).
|
||||
# mutable_patterns: artifacts that expire after cache.mutable_ttl seconds and are
|
||||
# re-fetched from upstream on next request (e.g. index files,
|
||||
# branch archives). Defaults to the package-type built-ins when
|
||||
# not set (APKINDEX, repomd.xml, Docker manifests, etc.).
|
||||
# cache:
|
||||
# immutable_ttl: TTL for immutable files (0 = forever, rarely needed to change).
|
||||
# mutable_ttl: TTL in seconds for mutable files. Omit to use the default (3600).
|
||||
#
|
||||
# quarantine_new: Set to true to block immutable artifacts published within the last
|
||||
# quarantine_days days. Requests return 404 until the quarantine period
|
||||
# expires. Fails open when the publish date cannot be determined.
|
||||
# quarantine_days: Number of days to quarantine newly published artifacts (requires
|
||||
# quarantine_new: true). The upstream Last-Modified header is used as
|
||||
# the publish date.
|
||||
#
|
||||
# WARNING: this file may contain credentials — do not commit real values.
|
||||
#
|
||||
# Global configuration
|
||||
#s3:
|
||||
# endpoint: "localhost:9000"
|
||||
# access_key: "minioadmin"
|
||||
# secret_key: "minioadmin"
|
||||
# bucket: "artifacts"
|
||||
# secure: false
|
||||
#
|
||||
#redis:
|
||||
# url: "redis://localhost:6379/0"
|
||||
#
|
||||
#database:
|
||||
# url: "postgresql://artifacts:artifacts123@localhost:5432/artifacts"
|
||||
#
|
||||
remotes:
|
||||
github:
|
||||
base_url: "https://github.com"
|
||||
package: "generic"
|
||||
description: "GitHub releases and files"
|
||||
immutable_patterns:
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
- "lxc/incus/.*\\.tar\\.gz$"
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "VictoriaMetrics/VictoriaMetrics/.*/vmutils-linux-amd64-.*\\.tar\\.gz$"
|
||||
- "VictoriaMetrics/VictoriaMetrics/.*/victoria-metrics-linux-amd64-.*-cluster\\.tar\\.gz$"
|
||||
- "VictoriaMetrics/VictoriaMetrics/.*/victoria-logs-linux-amd64-.*\\.tar\\.gz$"
|
||||
- "VictoriaMetrics/VictoriaMetrics/.*/vlutils-linux-amd64-.*\\.tar\\.gz$"
|
||||
- "prometheus-community/bind_exporter/.*/bind_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "prometheus-community/pgbouncer_exporter/.*/pgbouncer_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "prometheus-community/postgres_exporter/.*/postgres_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "onedr0p/exportarr/.*/exportarr_.*_linux_amd64\\.tar\\.gz$"
|
||||
- "tynany/frr_exporter/.*/frr_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "camptocamp/prometheus-puppetdb-exporter/.*/prometheus-puppetdb-exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
- "grafana/jsonnet-language-server/.*/jsonnet-language-server_.*_linux_amd64$"
|
||||
- "helmfile/helmfile/.*/helmfile_.*_linux_amd64\\.tar\\.gz$"
|
||||
- "helmfile/vals/.*/vals_.*_linux_amd64\\.tar\\.gz$"
|
||||
- "openbao/openbao-plugins/.*/openbao-plugin-secrets-consul_linux_amd64_.*\\.tar\\.gz$"
|
||||
- "openbao/openbao-plugins/.*/openbao-plugin-secrets-nomad_linux_amd64_.*\\.tar\\.gz$"
|
||||
- "apple/foundationdb/.*/libfdb_c\\.x86_64\\.so$"
|
||||
- "stalwartlabs/stalwart/.*/stalwart-cli-x86_64-unknown-linux-gnu\\.tar\\.gz$"
|
||||
- "stalwartlabs/stalwart/.*/stalwart-foundationdb-x86_64-unknown-linux-gnu\\.tar\\.gz$"
|
||||
- "stalwartlabs/stalwart/.*/stalwart-x86_64-unknown-linux-gnu\\.tar\\.gz$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 0
|
||||
|
||||
github-archive:
|
||||
base_url: "https://github.com"
|
||||
package: "generic"
|
||||
description: "GitHub repository archive tarballs"
|
||||
immutable_patterns:
|
||||
# Tag archives are immutable — a tag never changes
|
||||
- ".*/archive/refs/tags/.*\\.tar\\.gz$"
|
||||
mutable_patterns:
|
||||
# Branch archives can change on every push
|
||||
- ".*/archive/refs/heads/main\\.tar\\.gz$"
|
||||
- ".*/archive/refs/heads/master\\.tar\\.gz$"
|
||||
# Before re-downloading an expired branch archive, check whether it has
|
||||
# actually changed (304 Not Modified → just refresh the TTL, no transfer).
|
||||
# Only applies to user-defined mutable_patterns, not package-type defaults.
|
||||
check_mutable_updates: true
|
||||
cache:
|
||||
immutable_ttl: 0 # Tag archives cached indefinitely
|
||||
mutable_ttl: 86400 # Branch archives refreshed after 1 day
|
||||
|
||||
gitea-dl:
|
||||
base_url: "https://dl.gitea.com"
|
||||
package: "generic"
|
||||
description: "Gitea download site"
|
||||
immutable_patterns:
|
||||
- "act_runner/.*/act_runner-.*-linux-amd64$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 0
|
||||
|
||||
hashicorp-releases:
|
||||
base_url: "https://releases.hashicorp.com"
|
||||
package: "generic"
|
||||
description: "HashiCorp product releases"
|
||||
immutable_patterns:
|
||||
- "terraform/.*terraform_.*_linux_amd64\\.zip$"
|
||||
- "terraform/.*terraform_.*_windows_amd64\\.zip$"
|
||||
- "terraform/.*terraform_.*_darwin_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_linux_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_windows_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_darwin_amd64\\.zip$"
|
||||
- "consul-cni/.*/consul-cni_.*_linux_amd64\\.zip$"
|
||||
- "consul/.*/consul_.*_linux_amd64\\.zip$"
|
||||
- "nomad-autoscaler/.*/nomad-autoscaler_.*_linux_amd64\\.zip$"
|
||||
- "nomad/.*/nomad_.*_linux_amd64\\.zip$"
|
||||
- "packer/.*/packer_.*_linux_amd64\\.zip$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 0
|
||||
|
||||
alpine:
|
||||
base_url: "https://dl-cdn.alpinelinux.org"
|
||||
package: "alpine"
|
||||
description: "Alpine Linux APK package repository"
|
||||
immutable_patterns:
|
||||
- ".*/x86_64/.*\\.apk$"
|
||||
# check_mutable_updates not set: APKINDEX.tar.gz is a package-type default
|
||||
# and is always re-fetched on expiry — conditional checks are skipped for
|
||||
# built-in mutable patterns regardless of this flag.
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 7200 # Index files (APKINDEX.tar.gz) cached for 2 hours
|
||||
|
||||
almalinux:
|
||||
base_url: "https://gsl-syd.mm.fcix.net/almalinux"
|
||||
package: "rpm"
|
||||
description: "AlmaLinux RPM package repository"
|
||||
immutable_patterns:
|
||||
- ".*/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
- ".*/repodata/.*$"
|
||||
- ".*\\.rpm$" # Allow all RPM files
|
||||
# repomd.xml / repodata are package-type defaults — always re-fetched on
|
||||
# expiry. check_mutable_updates would only apply to any custom
|
||||
# mutable_patterns added here.
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 7200 # Metadata files cached for 2 hours
|
||||
|
||||
epel:
|
||||
base_url: "http://mirror.aarnet.edu.au/pub/epel"
|
||||
package: "rpm"
|
||||
description: "EPEL (Extra Packages for Enterprise Linux)"
|
||||
immutable_patterns:
|
||||
- "8/Everything/x86_64/.*\\.rpm$"
|
||||
- "9/Everything/x86_64/.*\\.rpm$"
|
||||
- "10/Everything/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
- ".*/repodata/.*$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 7200 # Metadata files cached for 2 hours
|
||||
|
||||
fedora:
|
||||
base_url: "https://gsl-syd.mm.fcix.net/fedora/linux"
|
||||
package: "rpm"
|
||||
description: "Fedora Linux RPM package repository"
|
||||
immutable_patterns:
|
||||
- "releases/.*/Everything/x86_64/.*\\.rpm$"
|
||||
- "updates/.*/Everything/x86_64/.*\\.rpm$"
|
||||
- "development/.*/Everything/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
- "updates/.*/Everything/x86_64/repodata/.*$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 300 # Metadata files cached for 5 minutes
|
||||
|
||||
ghcr:
|
||||
base_url: "https://ghcr.io"
|
||||
package: "docker"
|
||||
description: "GitHub Container Registry"
|
||||
# username: "your-github-username"
|
||||
# password: "your-github-pat" # needs read:packages scope
|
||||
# Docker manifest/tag-list patterns are package-type defaults — always
|
||||
# re-fetched on expiry. check_mutable_updates only applies to any custom
|
||||
# mutable_patterns you add (e.g. a metadata endpoint).
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 300
|
||||
|
||||
dockerhub:
|
||||
base_url: "https://registry-1.docker.io"
|
||||
package: "docker"
|
||||
description: "Docker Hub registry"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 300
|
||||
|
||||
pypi:
|
||||
base_url: "https://files.pythonhosted.org"
|
||||
package: "pypi"
|
||||
description: "Python Package Index — simple index and package files via a single remote"
|
||||
# simple/ requests are transparently fetched from pypi.org; package files come from
|
||||
# files.pythonhosted.org (base_url). URLs in the simple index are rewritten to this remote.
|
||||
check_mutable_updates: true
|
||||
# Block packages published within the last 3 days (supply-chain attack mitigation).
|
||||
# Immutable artifacts (wheel/sdist) newer than quarantine_days return 404 until
|
||||
# the window passes. Disable by setting quarantine_new: false or removing both keys.
|
||||
quarantine_new: true
|
||||
quarantine_days: 3
|
||||
immutable_patterns:
|
||||
- "packages/.*\\.whl$"
|
||||
- "packages/.*\\.whl\\.metadata$"
|
||||
- "packages/.*\\.tar\\.gz$"
|
||||
- "packages/.*\\.zip$"
|
||||
- "packages/.*\\.egg$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600 # Simple index pages refreshed after 10 minutes
|
||||
|
||||
pypi-gitea:
|
||||
base_url: "https://gitea.example.com/api/packages/myorg/pypi"
|
||||
package: "pypi"
|
||||
description: "Private Gitea PyPI registry — simple index and files at the same host"
|
||||
# username: "your-gitea-username"
|
||||
# password: "your-personal-access-token" # needs package:read scope
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "files/.*\\.whl$"
|
||||
- "files/.*\\.whl\\.metadata$"
|
||||
- "files/.*\\.tar\\.gz$"
|
||||
- "files/.*\\.zip$"
|
||||
- "files/.*\\.egg$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600
|
||||
|
||||
npm:
|
||||
base_url: "https://registry.npmjs.org"
|
||||
package: "npm"
|
||||
description: "npm registry — package metadata with tarball URL rewriting"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- \.tgz$
|
||||
mutable_patterns:
|
||||
- ^(?!.*\.tgz$).*
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600 # Package metadata refreshed after 10 minutes
|
||||
|
||||
hashicorp-helm:
|
||||
base_url: "https://helm.releases.hashicorp.com"
|
||||
package: "helm"
|
||||
description: "HashiCorp Helm chart repository (Vault, Consul, Nomad, etc.)"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Chart tarballs are versioned — cache forever
|
||||
mutable_ttl: 3600 # index.yaml refreshed after 1 hour
|
||||
|
||||
metallb:
|
||||
base_url: "https://metallb.github.io/metallb"
|
||||
package: "helm"
|
||||
description: "MetalLB load balancer Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
jetstack:
|
||||
base_url: "https://charts.jetstack.io"
|
||||
package: "helm"
|
||||
description: "Jetstack Helm charts (cert-manager)"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
rancher-stable:
|
||||
base_url: "https://releases.rancher.com/server-charts/stable"
|
||||
package: "helm"
|
||||
description: "Rancher stable Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
purelb:
|
||||
base_url: "https://gitlab.com/api/v4/projects/20400619/packages/helm/stable"
|
||||
package: "helm"
|
||||
description: "PureLB load balancer Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
istio:
|
||||
base_url: "https://istio-release.storage.googleapis.com/charts"
|
||||
package: "helm"
|
||||
description: "Istio service mesh Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
cnpg:
|
||||
base_url: "https://cloudnative-pg.github.io/charts"
|
||||
package: "helm"
|
||||
description: "CloudNativePG operator Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
ceph-csi:
|
||||
base_url: "https://ceph.github.io/csi-charts"
|
||||
package: "helm"
|
||||
description: "Ceph CSI driver Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
external-dns:
|
||||
base_url: "https://kubernetes-sigs.github.io/external-dns/"
|
||||
package: "helm"
|
||||
description: "ExternalDNS Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
intel-helm:
|
||||
base_url: "https://intel.github.io/helm-charts/"
|
||||
package: "helm"
|
||||
description: "Intel Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
elastic:
|
||||
base_url: "https://helm.elastic.co"
|
||||
package: "helm"
|
||||
description: "Elastic stack Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
k8up-io:
|
||||
base_url: "https://k8up-io.github.io/k8up"
|
||||
package: "helm"
|
||||
description: "K8up backup operator Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
victoriametrics:
|
||||
base_url: "https://victoriametrics.github.io/helm-charts/"
|
||||
package: "helm"
|
||||
description: "VictoriaMetrics observability Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
grafana:
|
||||
base_url: "https://grafana.github.io/helm-charts"
|
||||
package: "helm"
|
||||
description: "Grafana observability Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
helm-openldap:
|
||||
base_url: "https://jp-gouin.github.io/helm-openldap/"
|
||||
package: "helm"
|
||||
description: "OpenLDAP Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
woodpecker:
|
||||
base_url: "https://woodpecker-ci.org/"
|
||||
package: "helm"
|
||||
description: "Woodpecker CI Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
stakater:
|
||||
base_url: "https://stakater.github.io/stakater-charts"
|
||||
package: "helm"
|
||||
description: "Stakater Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
jfrog:
|
||||
base_url: "https://charts.jfrog.io/"
|
||||
package: "helm"
|
||||
description: "JFrog Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
openvox:
|
||||
base_url: "https://openvoxproject.github.io/openvox-helm-chart"
|
||||
package: "helm"
|
||||
description: "OpenVox Helm charts"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 3600
|
||||
|
||||
|
||||
virtuals:
|
||||
helm-all:
|
||||
package: "helm"
|
||||
description: "Virtual repository merging all helm remotes — member order is priority order for duplicate chart+version"
|
||||
members:
|
||||
- hashicorp-helm
|
||||
- metallb
|
||||
- jetstack
|
||||
- rancher-stable
|
||||
- purelb
|
||||
- istio
|
||||
- cnpg
|
||||
- ceph-csi
|
||||
- external-dns
|
||||
- intel-helm
|
||||
- elastic
|
||||
- k8up-io
|
||||
- victoriametrics
|
||||
- grafana
|
||||
- helm-openldap
|
||||
- woodpecker
|
||||
- stakater
|
||||
- jfrog
|
||||
- openvox
|
||||
|
||||
locals:
|
||||
local-generic:
|
||||
package: "generic"
|
||||
description: "Local generic file repository"
|
||||
cache:
|
||||
immutable_ttl: 0 # Files cached indefinitely
|
||||
mutable_ttl: 0
|
||||
@@ -1,105 +0,0 @@
|
||||
module git.unkin.net/unkin/artifactapi
|
||||
|
||||
go 1.25.9
|
||||
|
||||
require (
|
||||
github.com/cavaliergopher/rpm v1.3.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/go-chi/chi/v5 v5.3.0
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/minio/minio-go/v7 v7.2.0
|
||||
github.com/redis/go-redis/v9 v9.20.0
|
||||
github.com/testcontainers/testcontainers-go v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.4.3 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.11.7 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
|
||||
github.com/charmbracelet/x/term v0.2.2 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.11.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v0.2.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.23 // indirect
|
||||
github.com/mdelapenya/tlscert v0.2.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/moby/api v1.54.1 // indirect
|
||||
github.com/moby/moby/client v0.4.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/stretchr/testify v1.11.1 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.51.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.2 // indirect
|
||||
)
|
||||
@@ -1,249 +0,0 @@
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cavaliergopher/rpm v1.3.0 h1:UHX46sasX8MesUXXQ+UbkFLUX4eUWTlEcX8jcnRBIgI=
|
||||
github.com/cavaliergopher/rpm v1.3.0/go.mod h1:vEumo1vvtrHM1Ov86f6+k8j7zNKOxQfHDCAIcR/36ZI=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
|
||||
github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
|
||||
github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=
|
||||
github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q=
|
||||
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
|
||||
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
|
||||
github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI=
|
||||
github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
|
||||
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
|
||||
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
|
||||
github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8=
|
||||
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
|
||||
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw=
|
||||
github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
|
||||
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs=
|
||||
github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
||||
github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4=
|
||||
github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||
github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw=
|
||||
github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g=
|
||||
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
|
||||
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
|
||||
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
|
||||
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0=
|
||||
github.com/redis/go-redis/v9 v9.20.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
||||
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
|
||||
github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 h1:id/6LH8ZeDrtAUVSuNvZUAJ1kVpb82y1pr9yweAWsRg=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0/go.mod h1:uF0jI8FITagQpBNOgweGBmPf6rP4K0SeL1XFPbsZSSY=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
||||
go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
|
||||
go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
|
||||
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss=
|
||||
gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||
pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
|
||||
pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||
@@ -1,209 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
type ProxyHandler struct {
|
||||
engine *proxy.Engine
|
||||
virtualEngine *virtual.Engine
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
local *v2.LocalHandler
|
||||
}
|
||||
|
||||
func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *database.DB, store *storage.S3, local *v2.LocalHandler) *ProxyHandler {
|
||||
return &ProxyHandler{engine: engine, virtualEngine: virtualEngine, db: db, store: store, local: local}
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/remote/{remoteName}/*", h.handleProxy)
|
||||
r.Get("/local/{localName}/*", h.handleLocal)
|
||||
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) DockerV2Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.handleDockerPing)
|
||||
r.Head("/", h.handleDockerPing)
|
||||
r.Get("/{remoteName}/*", h.handleProxy)
|
||||
r.Head("/{remoteName}/*", h.handleProxyHead)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Docker-Distribution-Api-Version", "registry/2.0")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy fetch failed", "remote", remoteName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer result.Reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, result.Reader)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.engine.Head(r.Context(), *remote, path, prov)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
||||
virtualName := chi.URLParam(r, "virtualName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
virt, err := h.db.GetVirtual(r.Context(), virtualName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("virtual %q not found", virtualName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||
|
||||
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
||||
if err != nil {
|
||||
slog.Error("virtual fetch failed", "virtual", virtualName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("X-Artifact-Source", "virtual")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
|
||||
localName := chi.URLParam(r, "localName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), localName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("local %q not found", localName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
if indexer, ok := prov.(provider.LocalIndexer); ok {
|
||||
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
h.serveLocalFile(w, r, localName, path)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, repoName, path string) {
|
||||
file, err := h.db.GetLocalFile(r.Context(), repoName, path)
|
||||
if err != nil {
|
||||
slog.Error("local file lookup failed", "repo", repoName, "path", path, "error", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if file == nil {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):])
|
||||
reader, info, err := h.store.Download(r.Context(), s3Key)
|
||||
if err != nil {
|
||||
slog.Error("local file download failed", "repo", repoName, "path", path, "error", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", info.ContentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func scheme(r *http.Request) string {
|
||||
if r.TLS != nil {
|
||||
return "https"
|
||||
}
|
||||
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
|
||||
return fwd
|
||||
}
|
||||
return "http"
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestScheme(t *testing.T) {
|
||||
if got := scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
|
||||
t.Errorf("TLS request scheme = %q, want https", got)
|
||||
}
|
||||
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {"https"}}}
|
||||
if got := scheme(r); got != "https" {
|
||||
t.Errorf("X-Forwarded-Proto scheme = %q, want https", got)
|
||||
}
|
||||
if got := scheme(&http.Request{Header: http.Header{}}); got != "http" {
|
||||
t.Errorf("default scheme = %q, want http", got)
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
var testDSN string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, terminate, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN = dsn
|
||||
code := m.Run()
|
||||
terminate()
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
// closedDB returns a DB whose pool has been closed, so every query fails —
|
||||
// used to drive the handlers' error branches.
|
||||
func closedDB(t *testing.T) *database.DB {
|
||||
t.Helper()
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatalf("new db: %v", err)
|
||||
}
|
||||
db.Close()
|
||||
return db
|
||||
}
|
||||
|
||||
func do(t *testing.T, h http.Handler, method, path, body string) int {
|
||||
t.Helper()
|
||||
var r io.Reader
|
||||
if body != "" {
|
||||
r = strings.NewReader(body)
|
||||
}
|
||||
req := httptest.NewRequest(method, path, r)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
return w.Code
|
||||
}
|
||||
|
||||
func TestRemotesErrorPaths(t *testing.T) {
|
||||
h := NewRemotesHandler(closedDB(t)).Routes()
|
||||
if c := do(t, h, "GET", "/", ""); c != 500 {
|
||||
t.Errorf("list with dead db = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "POST", "/", `{"name":"x","package_type":"generic","repo_type":"remote","base_url":"https://x"}`); c != 500 {
|
||||
t.Errorf("create with dead db = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "PUT", "/x", `{"package_type":"generic","base_url":"https://x"}`); c != 500 {
|
||||
t.Errorf("update with dead db = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "GET", "/x", ""); c != 404 {
|
||||
t.Errorf("get missing = %d, want 404", c)
|
||||
}
|
||||
if c := do(t, h, "DELETE", "/x", ""); c != 500 {
|
||||
t.Errorf("delete with dead db = %d, want 500", c)
|
||||
}
|
||||
// Bad request bodies never reach the db.
|
||||
if c := do(t, h, "POST", "/", `not json`); c != 400 {
|
||||
t.Errorf("invalid json = %d, want 400", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualsErrorPaths(t *testing.T) {
|
||||
h := NewVirtualsHandler(closedDB(t)).Routes()
|
||||
if c := do(t, h, "GET", "/", ""); c != 500 {
|
||||
t.Errorf("list = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "GET", "/x", ""); c != 404 {
|
||||
t.Errorf("get missing = %d, want 404", c)
|
||||
}
|
||||
if c := do(t, h, "POST", "/", `{"name":"v","package_type":"helm","members":["a"]}`); c != 500 {
|
||||
t.Errorf("create = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "PUT", "/v", `{"package_type":"helm","members":["a"]}`); c != 500 {
|
||||
t.Errorf("update = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "DELETE", "/v", ""); c != 500 {
|
||||
t.Errorf("delete = %d, want 500", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsErrorPaths(t *testing.T) {
|
||||
h := NewStatsHandler(closedDB(t)).Routes()
|
||||
for _, p := range []string{"/", "/top-remotes", "/top-files-by-hits", "/top-files-by-bandwidth"} {
|
||||
if c := do(t, h, "GET", p, ""); c != 500 {
|
||||
t.Errorf("stats %s = %d, want 500", p, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalErrorPaths(t *testing.T) {
|
||||
h := NewLocalHandler(closedDB(t), nil).Routes()
|
||||
// GetRemote fails on the closed db -> not found.
|
||||
if c := do(t, h, "PUT", "/x/files/a.bin", "data"); c != 404 {
|
||||
t.Errorf("upload unknown repo = %d, want 404", c)
|
||||
}
|
||||
// download / remove hit the db and 500.
|
||||
if c := do(t, h, "GET", "/x/files/a.bin", ""); c != 500 {
|
||||
t.Errorf("download = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "DELETE", "/x/files/a.bin", ""); c != 500 {
|
||||
t.Errorf("remove = %d, want 500", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalHandlerDBAccessor(t *testing.T) {
|
||||
db := closedDB(t)
|
||||
if NewLocalHandler(db, nil).DB() != db {
|
||||
t.Error("DB() should return the handler's database")
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
type EventsHandler struct{}
|
||||
|
||||
func NewEventsHandler() *EventsHandler {
|
||||
return &EventsHandler{}
|
||||
}
|
||||
|
||||
func (h *EventsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.stream)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *EventsHandler) stream(w http.ResponseWriter, r *http.Request) {
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
http.Error(w, "streaming not supported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("X-Accel-Buffering", "no")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
flusher.Flush()
|
||||
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
fmt.Fprintf(w, ": keepalive\n\n")
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
)
|
||||
|
||||
type HealthHandler struct {
|
||||
db *database.DB
|
||||
cache *cache.Redis
|
||||
store *storage.S3
|
||||
}
|
||||
|
||||
func NewHealthHandler(db *database.DB, c *cache.Redis, s *storage.S3) *HealthHandler {
|
||||
return &HealthHandler{db: db, cache: c, store: s}
|
||||
}
|
||||
|
||||
func (h *HealthHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.health)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *HealthHandler) health(w http.ResponseWriter, r *http.Request) {
|
||||
status := map[string]string{
|
||||
"status": "ok",
|
||||
"postgres": "ok",
|
||||
"redis": "ok",
|
||||
"s3": "ok",
|
||||
}
|
||||
|
||||
if err := h.db.Pool.Ping(r.Context()); err != nil {
|
||||
status["postgres"] = "error"
|
||||
status["status"] = "degraded"
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
@@ -1,227 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type LocalHandler struct {
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
cas *storage.CAS
|
||||
}
|
||||
|
||||
func NewLocalHandler(db *database.DB, store *storage.S3) *LocalHandler {
|
||||
return &LocalHandler{
|
||||
db: db,
|
||||
store: store,
|
||||
cas: storage.NewCAS(store),
|
||||
}
|
||||
}
|
||||
|
||||
func (h *LocalHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Put("/*", h.upload)
|
||||
r.Get("/*", h.download)
|
||||
r.Delete("/*", h.remove)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *LocalHandler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
if filePath == "" {
|
||||
http.Error(w, "file path required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", repoName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if remote.RepoType != models.RepoTypeLocal {
|
||||
http.Error(w, "upload only allowed for local repository types", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
|
||||
if uploader, ok := prov.(provider.LocalUploader); ok {
|
||||
h.uploadValidated(w, r, remote, filePath, prov, uploader)
|
||||
return
|
||||
}
|
||||
|
||||
h.uploadGeneric(w, r, remote, filePath)
|
||||
}
|
||||
|
||||
func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string, prov provider.Provider, uploader provider.LocalUploader) {
|
||||
storagePath, contentType, err := uploader.ValidateUpload(filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, storagePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if existing != nil {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.cas.Store(r.Context(), r.Body, contentType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.CreateLocalFile(r.Context(), remote.Name, storagePath, result.ContentHash); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if hook, ok := prov.(provider.PostUploadHook); ok {
|
||||
go hook.AfterUpload(context.Background(), remote.Name, storagePath, result.ContentHash, h, h.db)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, uploader.UploadResponse(storagePath, result.ContentHash, result.SizeBytes))
|
||||
}
|
||||
|
||||
func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) {
|
||||
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if existing != nil {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", filePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := "application/octet-stream"
|
||||
if ct := r.Header.Get("Content-Type"); ct != "" && ct != "application/octet-stream" {
|
||||
contentType = ct
|
||||
}
|
||||
|
||||
result, err := h.cas.Store(r.Context(), r.Body, contentType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.CreateLocalFile(r.Context(), remote.Name, filePath, result.ContentHash); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", filePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, map[string]any{
|
||||
"path": filePath,
|
||||
"content_hash": result.ContentHash,
|
||||
"size_bytes": result.SizeBytes,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *LocalHandler) download(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
file, err := h.db.GetLocalFile(r.Context(), repoName, filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if file == nil {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):])
|
||||
reader, info, err := h.store.Download(r.Context(), s3Key)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("download failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", info.ContentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func (h *LocalHandler) remove(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
if err := deleteLocalFile(r.Context(), h.db, repoName, filePath); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// deleteLocalFile removes a local file and runs the provider's post-delete hook,
|
||||
// so provider-derived state (e.g. RPM metadata that feeds generated repodata)
|
||||
// stops referencing a package that no longer exists.
|
||||
func deleteLocalFile(ctx context.Context, db *database.DB, repoName, filePath string) error {
|
||||
if err := db.DeleteLocalFile(ctx, repoName, filePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
remote, err := db.GetRemote(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil // file is gone; no repo left to resolve a cleanup hook from
|
||||
}
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if hook, ok := prov.(provider.PostDeleteHook); ok {
|
||||
return hook.AfterDelete(ctx, repoName, filePath, db)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *LocalHandler) DB() *database.DB {
|
||||
return h.db
|
||||
}
|
||||
|
||||
func (h *LocalHandler) Download(ctx context.Context, key string) (io.ReadCloser, int64, error) {
|
||||
reader, info, err := h.store.Download(ctx, key)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return reader, info.Size, nil
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/rpm" // register the rpm provider so its PostDeleteHook runs
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestLocalEvictCleansRPMMetadata verifies that evicting an RPM from a local
|
||||
// repo also removes the derived rpm_metadata row, so generated repodata stops
|
||||
// listing the deleted package.
|
||||
func TestLocalEvictCleansRPMMetadata(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const repo = "rpm-evict-cleanup"
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageRPM, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const hash = "sha256:bb22"
|
||||
const path = "Packages/example-0.1.0-1.x86_64.rpm"
|
||||
if err := db.UpsertBlob(ctx, hash, "blobs/bb/22", 2048, "application/x-rpm"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.CreateLocalFile(ctx, repo, path, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.InsertRPMMetadata(ctx, &provider.RPMMetadata{
|
||||
RepoName: repo, FilePath: path, ContentHash: hash,
|
||||
Name: "example", Version: "0.1.0", Release: "1", Arch: "x86_64",
|
||||
Requires: []provider.RPMDep{}, Provides: []provider.RPMDep{},
|
||||
Files: []provider.RPMFile{}, Changelogs: []provider.RPMChangelog{},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewObjectsHandler(db)
|
||||
router := chi.NewRouter()
|
||||
router.Route("/locals/{name}/objects", func(r chi.Router) {
|
||||
r.Delete("/*", h.LocalRoutes().ServeHTTP)
|
||||
})
|
||||
|
||||
del := httptest.NewRequest("DELETE", "/locals/"+repo+"/objects/"+path, nil)
|
||||
dw := httptest.NewRecorder()
|
||||
router.ServeHTTP(dw, del)
|
||||
if dw.Code != 204 {
|
||||
t.Fatalf("evict = %d, want 204", dw.Code)
|
||||
}
|
||||
|
||||
if f, _ := db.GetLocalFile(ctx, repo, path); f != nil {
|
||||
t.Fatalf("local file still present after evict: %+v", f)
|
||||
}
|
||||
entries, err := db.ListRPMMetadataEntries(ctx, repo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("rpm_metadata still present after evict: %+v", entries)
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestLocalUploadStoreFailure covers the upload handlers' store-error branches
|
||||
// by killing the object store after a successful upload.
|
||||
func TestLocalUploadStoreFailure(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
conn, termMinio, err := testsupport.StartMinio(ctx)
|
||||
if err != nil {
|
||||
t.Skip("minio unavailable")
|
||||
}
|
||||
var store *storage.S3
|
||||
for i := 0; i < 20; i++ {
|
||||
if store, err = storage.NewS3(conn.Endpoint, conn.AccessKey, conn.SecretKey, "fault", false, ""); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
termMinio()
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, pt := range []models.PackageType{models.PackageGeneric, models.PackagePyPI} {
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: "fault-" + string(pt), PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
h := NewLocalHandler(db, store)
|
||||
router := chi.NewRouter()
|
||||
router.Route("/remotes/{name}/files", func(r chi.Router) {
|
||||
r.Put("/*", h.Routes().ServeHTTP)
|
||||
})
|
||||
srv := httptest.NewServer(router)
|
||||
defer srv.Close()
|
||||
|
||||
put := func(name, path, body string) int {
|
||||
rq, _ := http.NewRequest("PUT", srv.URL+"/remotes/"+name+"/files/"+path, strings.NewReader(body))
|
||||
resp, err := http.DefaultClient.Do(rq)
|
||||
if err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
// Sanity: uploads succeed while the store is up.
|
||||
if c := put("fault-generic", "ok.bin", "data"); c != 201 {
|
||||
t.Fatalf("generic upload while up = %d", c)
|
||||
}
|
||||
if c := put("fault-pypi", "foo-1.0-py3-none-any.whl", "wheel"); c != 201 {
|
||||
t.Fatalf("pypi upload while up = %d", c)
|
||||
}
|
||||
|
||||
// Kill the store; subsequent CAS.Store calls fail -> 500.
|
||||
termMinio()
|
||||
if c := put("fault-generic", "after.bin", "data"); c != 500 {
|
||||
t.Errorf("generic upload after store down = %d, want 500", c)
|
||||
}
|
||||
if c := put("fault-pypi", "bar-1.0-py3-none-any.whl", "wheel"); c != 500 {
|
||||
t.Errorf("pypi upload after store down = %d, want 500", c)
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestLocalObjectsListing verifies that files uploaded to a local repo (which
|
||||
// live in local_files, not artifacts) are listed by the local objects endpoint
|
||||
// and can be evicted through it.
|
||||
func TestLocalObjectsListing(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const repo = "rpm-local-objs"
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageRPM, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const hash = "sha256:aa11"
|
||||
const path = "Packages/example-0.1.0-1.x86_64.rpm"
|
||||
if err := db.UpsertBlob(ctx, hash, "blobs/aa/11", 1234, "application/x-rpm"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.CreateLocalFile(ctx, repo, path, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewObjectsHandler(db)
|
||||
router := chi.NewRouter()
|
||||
router.Route("/locals/{name}/objects", func(r chi.Router) {
|
||||
r.Get("/", h.LocalRoutes().ServeHTTP)
|
||||
r.Delete("/*", h.LocalRoutes().ServeHTTP)
|
||||
})
|
||||
|
||||
// The uploaded package must appear in the listing with its blob size.
|
||||
req := httptest.NewRequest("GET", "/locals/"+repo+"/objects", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("list = %d, want 200", w.Code)
|
||||
}
|
||||
var got []models.Artifact
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d objects, want 1", len(got))
|
||||
}
|
||||
if got[0].Path != path || got[0].SizeBytes != 1234 || got[0].ContentHash != hash {
|
||||
t.Fatalf("unexpected object: %+v", got[0])
|
||||
}
|
||||
|
||||
// Eviction removes it from local_files.
|
||||
del := httptest.NewRequest("DELETE", "/locals/"+repo+"/objects/"+path, nil)
|
||||
dw := httptest.NewRecorder()
|
||||
router.ServeHTTP(dw, del)
|
||||
if dw.Code != 204 {
|
||||
t.Fatalf("evict = %d, want 204", dw.Code)
|
||||
}
|
||||
if f, _ := db.GetLocalFile(ctx, repo, path); f != nil {
|
||||
t.Fatalf("file still present after evict: %+v", f)
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
type ObjectsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewObjectsHandler(db *database.DB) *ObjectsHandler {
|
||||
return &ObjectsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Delete("/*", h.evict)
|
||||
return r
|
||||
}
|
||||
|
||||
// LocalRoutes lists and evicts objects for local repos, which live in the
|
||||
// local_files table rather than the artifacts table used by remotes.
|
||||
func (h *ObjectsHandler) LocalRoutes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.listLocal)
|
||||
r.Delete("/*", h.evictLocal)
|
||||
return r
|
||||
}
|
||||
|
||||
// pageBounds parses the shared page/per_page query params into a SQL limit and offset.
|
||||
func pageBounds(r *http.Request) (limit, offset int) {
|
||||
limit, _ = strconv.Atoi(r.URL.Query().Get("per_page"))
|
||||
if limit <= 0 || limit > 5000 {
|
||||
limit = 50
|
||||
}
|
||||
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
return limit, (page - 1) * limit
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
limit, offset := pageBounds(r)
|
||||
|
||||
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, artifacts)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) listLocal(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
limit, offset := pageBounds(r)
|
||||
|
||||
artifacts, err := h.db.ListLocalArtifacts(r.Context(), repoName, limit, offset)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, artifacts)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) evictLocal(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
if err := deleteLocalFile(r.Context(), h.db, repoName, path); err != nil {
|
||||
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
if err := h.db.DeleteArtifact(r.Context(), remoteName, path); err != nil {
|
||||
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
)
|
||||
|
||||
type ProbeHandler struct {
|
||||
engine *proxy.Engine
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewProbeHandler(engine *proxy.Engine, db *database.DB) *ProbeHandler {
|
||||
return &ProbeHandler{engine: engine, db: db}
|
||||
}
|
||||
|
||||
func (h *ProbeHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Post("/", h.probe)
|
||||
return r
|
||||
}
|
||||
|
||||
type probeRequest struct {
|
||||
Remote string `json:"remote"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
type probeResponse struct {
|
||||
Status int `json:"status"`
|
||||
Source string `json:"source,omitempty"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (h *ProbeHandler) probe(w http.ResponseWriter, r *http.Request) {
|
||||
var req probeRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if req.Remote == "" || req.Path == "" {
|
||||
http.Error(w, "remote and path are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), req.Remote)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 404,
|
||||
Error: fmt.Sprintf("remote %q not found", req.Remote),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 500,
|
||||
Error: fmt.Sprintf("no provider for %q", remote.PackageType),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
result, err := h.engine.Fetch(ctx, *remote, req.Path, prov)
|
||||
duration := time.Since(start).Milliseconds()
|
||||
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 502,
|
||||
DurationMS: duration,
|
||||
Error: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
io.Copy(io.Discard, result.Reader)
|
||||
result.Reader.Close()
|
||||
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 200,
|
||||
Source: result.Source,
|
||||
ContentType: result.ContentType,
|
||||
SizeBytes: result.Size,
|
||||
DurationMS: duration,
|
||||
Headers: map[string]string{
|
||||
"X-Artifact-Source": result.Source,
|
||||
"X-Artifact-Size": fmt.Sprintf("%d", result.Size),
|
||||
"Content-Type": result.ContentType,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type RemotesHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewRemotesHandler(db *database.DB) *RemotesHandler {
|
||||
return &RemotesHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Post("/", h.create)
|
||||
r.Get("/{name}", h.get)
|
||||
r.Put("/{name}", h.update)
|
||||
r.Delete("/{name}", h.del)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remotes, err := h.db.ListRemotes(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remotes)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
remote, err := h.db.GetRemote(r.Context(), name)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", name), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
var remote models.Remote
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !remote.PackageType.Valid() {
|
||||
http.Error(w, fmt.Sprintf("invalid package type: %q", remote.PackageType), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if remote.RepoType == "" {
|
||||
remote.RepoType = models.RepoTypeRemote
|
||||
}
|
||||
if !remote.RepoType.Valid() {
|
||||
http.Error(w, fmt.Sprintf("invalid repo type: %q", remote.RepoType), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if remote.RepoType == models.RepoTypeRemote && remote.BaseURL == "" {
|
||||
http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.CreateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
var remote models.Remote
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
remote.Name = name
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
if err := h.db.DeleteRemote(r.Context(), name); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
type StatsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewStatsHandler(db *database.DB) *StatsHandler {
|
||||
return &StatsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *StatsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.overview)
|
||||
r.Get("/top-remotes", h.topRemotes)
|
||||
r.Get("/top-files-by-hits", h.topFilesByHits)
|
||||
r.Get("/top-files-by-bandwidth", h.topFilesByBandwidth)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *StatsHandler) overview(w http.ResponseWriter, r *http.Request) {
|
||||
stats, err := h.db.GetOverviewStats(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, stats)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topRemotes(w http.ResponseWriter, r *http.Request) {
|
||||
remotes, err := h.db.GetTopRemotes(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remotes)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topFilesByHits(w http.ResponseWriter, r *http.Request) {
|
||||
files, err := h.db.GetTopFilesByHits(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, files)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topFilesByBandwidth(w http.ResponseWriter, r *http.Request) {
|
||||
files, err := h.db.GetTopFilesByBandwidth(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, files)
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type VirtualsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewVirtualsHandler(db *database.DB) *VirtualsHandler {
|
||||
return &VirtualsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Post("/", h.create)
|
||||
r.Get("/{name}", h.get)
|
||||
r.Put("/{name}", h.update)
|
||||
r.Delete("/{name}", h.del)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
virtuals, err := h.db.ListVirtuals(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virtuals)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
virt, err := h.db.GetVirtual(r.Context(), name)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("virtual %q not found", name), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
var virt models.Virtual
|
||||
if err := json.NewDecoder(r.Body).Decode(&virt); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.CreateVirtual(r.Context(), &virt); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
var virt models.Virtual
|
||||
if err := json.NewDecoder(r.Body).Decode(&virt); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
virt.Name = name
|
||||
if err := h.db.UpdateVirtual(r.Context(), &virt); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) del(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
if err := h.db.DeleteVirtual(r.Context(), name); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func BasicHeaders(remote models.Remote) http.Header {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString(
|
||||
[]byte(remote.Username+":"+remote.Password),
|
||||
))
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestBasicHeaders(t *testing.T) {
|
||||
h := BasicHeaders(models.Remote{Username: "alice", Password: "secret"})
|
||||
got := h.Get("Authorization")
|
||||
want := "Basic " + base64.StdEncoding.EncodeToString([]byte("alice:secret"))
|
||||
if got != want {
|
||||
t.Errorf("Authorization = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBasicHeadersNoUser(t *testing.T) {
|
||||
if h := BasicHeaders(models.Remote{}); h.Get("Authorization") != "" {
|
||||
t.Error("expected no Authorization header without a username")
|
||||
}
|
||||
}
|
||||
Vendored
-133
@@ -1,133 +0,0 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
var testRedis *Redis
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
url, terminate, err := testsupport.StartRedis(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
r, err := NewRedis(url)
|
||||
if err != nil {
|
||||
terminate()
|
||||
panic(err)
|
||||
}
|
||||
testRedis = r
|
||||
code := m.Run()
|
||||
r.Close()
|
||||
terminate()
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
func requireRedis(t *testing.T) {
|
||||
t.Helper()
|
||||
if testRedis == nil {
|
||||
t.Skip("Docker unavailable; skipping cache integration test")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRedisInvalid(t *testing.T) {
|
||||
if _, err := NewRedis("://bad-url"); err == nil {
|
||||
t.Error("expected error for invalid redis URL")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTTL(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
if fresh, _ := testRedis.CheckTTL(ctx, "r", "missing"); fresh {
|
||||
t.Error("missing key should not be fresh")
|
||||
}
|
||||
if err := testRedis.SetTTL(ctx, "r", "p", time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fresh, err := testRedis.CheckTTL(ctx, "r", "p"); err != nil || !fresh {
|
||||
t.Errorf("expected fresh after SetTTL: %v %v", fresh, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLock(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
ok, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("first acquire should succeed: %v %v", ok, err)
|
||||
}
|
||||
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); ok {
|
||||
t.Error("second acquire should fail while held")
|
||||
}
|
||||
if err := testRedis.ReleaseLock(ctx, "r", "lockpath"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); !ok {
|
||||
t.Error("acquire should succeed after release")
|
||||
}
|
||||
}
|
||||
|
||||
func TestETagAndToken(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
if v, _ := testRedis.GetETag(ctx, "r", "missing"); v != "" {
|
||||
t.Error("missing etag should be empty")
|
||||
}
|
||||
testRedis.SetETag(ctx, "r", "p", `"abc"`, time.Minute)
|
||||
if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` {
|
||||
t.Errorf("etag = %q", v)
|
||||
}
|
||||
|
||||
if v, _ := testRedis.GetToken(ctx, "missing"); v != "" {
|
||||
t.Error("missing token should be empty")
|
||||
}
|
||||
testRedis.SetToken(ctx, "key", "tok", time.Minute)
|
||||
if v, _ := testRedis.GetToken(ctx, "key"); v != "tok" {
|
||||
t.Errorf("token = %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCircuit(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 0 {
|
||||
t.Errorf("initial failures = %d", n)
|
||||
}
|
||||
n1, err := testRedis.IncrCircuitFailure(ctx, "cr", time.Minute)
|
||||
if err != nil || n1 != 1 {
|
||||
t.Fatalf("first incr = %d %v", n1, err)
|
||||
}
|
||||
n2, _ := testRedis.IncrCircuitFailure(ctx, "cr", time.Minute)
|
||||
if n2 != 2 {
|
||||
t.Errorf("second incr = %d", n2)
|
||||
}
|
||||
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 2 {
|
||||
t.Errorf("get failures = %d", n)
|
||||
}
|
||||
testRedis.ResetCircuit(ctx, "cr")
|
||||
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 0 {
|
||||
t.Errorf("failures after reset = %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushRemote(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
testRedis.SetTTL(ctx, "flushme", "a", time.Hour)
|
||||
testRedis.SetETag(ctx, "flushme", "a", "x", time.Hour)
|
||||
if err := testRedis.FlushRemote(ctx, "flushme"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fresh, _ := testRedis.CheckTTL(ctx, "flushme", "a"); fresh {
|
||||
t.Error("expected keys flushed")
|
||||
}
|
||||
}
|
||||
Vendored
-117
@@ -1,117 +0,0 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type Redis struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
func NewRedis(url string) (*Redis, error) {
|
||||
opts, err := redis.ParseURL(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse redis url: %w", err)
|
||||
}
|
||||
|
||||
client := redis.NewClient(opts)
|
||||
|
||||
if err := client.Ping(context.Background()).Err(); err != nil {
|
||||
return nil, fmt.Errorf("ping redis: %w", err)
|
||||
}
|
||||
|
||||
return &Redis{client: client}, nil
|
||||
}
|
||||
|
||||
func (r *Redis) Close() error {
|
||||
return r.client.Close()
|
||||
}
|
||||
|
||||
func (r *Redis) SetTTL(ctx context.Context, remote, path string, ttl time.Duration) error {
|
||||
key := fmt.Sprintf("ttl:%s:%s", remote, path)
|
||||
return r.client.Set(ctx, key, "1", ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) CheckTTL(ctx context.Context, remote, path string) (bool, error) {
|
||||
key := fmt.Sprintf("ttl:%s:%s", remote, path)
|
||||
exists, err := r.client.Exists(ctx, key).Result()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exists > 0, nil
|
||||
}
|
||||
|
||||
func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (bool, error) {
|
||||
key := fmt.Sprintf("lock:%s:%s", remote, path)
|
||||
ok, err := r.client.SetNX(ctx, key, "1", ttl).Result()
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error {
|
||||
key := fmt.Sprintf("lock:%s:%s", remote, path)
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error {
|
||||
key := fmt.Sprintf("etag:%s:%s", remote, path)
|
||||
return r.client.Set(ctx, key, etag, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) GetETag(ctx context.Context, remote, path string) (string, error) {
|
||||
key := fmt.Sprintf("etag:%s:%s", remote, path)
|
||||
val, err := r.client.Get(ctx, key).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) GetToken(ctx context.Context, key string) (string, error) {
|
||||
val, err := r.client.Get(ctx, "token:"+key).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) SetToken(ctx context.Context, key, token string, ttl time.Duration) error {
|
||||
return r.client.Set(ctx, "token:"+key, token, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) IncrCircuitFailure(ctx context.Context, remote string, cooldown time.Duration) (int64, error) {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
pipe := r.client.Pipeline()
|
||||
incr := pipe.Incr(ctx, key)
|
||||
pipe.Expire(ctx, key, cooldown)
|
||||
_, err := pipe.Exec(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return incr.Val(), nil
|
||||
}
|
||||
|
||||
func (r *Redis) ResetCircuit(ctx context.Context, remote string) error {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) GetCircuitFailures(ctx context.Context, remote string) (int64, error) {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
val, err := r.client.Get(ctx, key).Int64()
|
||||
if err == redis.Nil {
|
||||
return 0, nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) FlushRemote(ctx context.Context, remote string) error {
|
||||
iter := r.client.Scan(ctx, 0, fmt.Sprintf("*:%s:*", remote), 100).Iterator()
|
||||
for iter.Next(ctx) {
|
||||
r.client.Del(ctx, iter.Val())
|
||||
}
|
||||
return iter.Err()
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ListenAddr string
|
||||
|
||||
DBHost string
|
||||
DBPort int
|
||||
DBUser string
|
||||
DBPass string
|
||||
DBName string
|
||||
DBSSL string
|
||||
|
||||
RedisURL string
|
||||
|
||||
S3Endpoint string
|
||||
S3AccessKey string
|
||||
S3SecretKey string
|
||||
S3Bucket string
|
||||
S3Secure bool
|
||||
S3Region string
|
||||
}
|
||||
|
||||
func (c *Config) DatabaseDSN() string {
|
||||
return fmt.Sprintf(
|
||||
"postgres://%s:%s@%s:%d/%s?sslmode=%s",
|
||||
c.DBUser, c.DBPass, c.DBHost, c.DBPort, c.DBName, c.DBSSL,
|
||||
)
|
||||
}
|
||||
|
||||
func Load() (*Config, error) {
|
||||
dbPort, err := strconv.Atoi(getenv("DBPORT", "5432"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid DBPORT: %w", err)
|
||||
}
|
||||
|
||||
s3Secure, _ := strconv.ParseBool(getenv("MINIO_SECURE", "false"))
|
||||
|
||||
cfg := &Config{
|
||||
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
|
||||
|
||||
DBHost: getenv("DBHOST", "localhost"),
|
||||
DBPort: dbPort,
|
||||
DBUser: getenv("DBUSER", "artifacts"),
|
||||
DBPass: getenv("DBPASS", ""),
|
||||
DBName: getenv("DBNAME", "artifacts"),
|
||||
DBSSL: getenv("DBSSL", "disable"),
|
||||
|
||||
RedisURL: getenv("REDIS_URL", "redis://localhost:6379"),
|
||||
|
||||
S3Endpoint: getenv("MINIO_ENDPOINT", "localhost:9000"),
|
||||
S3AccessKey: getenv("MINIO_ACCESS_KEY", ""),
|
||||
S3SecretKey: getenv("MINIO_SECRET_KEY", ""),
|
||||
S3Bucket: getenv("MINIO_BUCKET", "artifacts"),
|
||||
S3Secure: s3Secure,
|
||||
S3Region: getenv("MINIO_REGION", ""),
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadDefaults(t *testing.T) {
|
||||
// Unset the vars Load reads so the fallback defaults are exercised.
|
||||
for _, k := range []string{
|
||||
"LISTEN_ADDR", "DBHOST", "DBPORT", "DBUSER", "DBPASS", "DBNAME", "DBSSL",
|
||||
"REDIS_URL", "MINIO_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY",
|
||||
"MINIO_BUCKET", "MINIO_SECURE", "MINIO_REGION",
|
||||
} {
|
||||
old, ok := os.LookupEnv(k)
|
||||
os.Unsetenv(k)
|
||||
if ok {
|
||||
t.Cleanup(func() { os.Setenv(k, old) })
|
||||
}
|
||||
}
|
||||
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if cfg.ListenAddr != ":8000" || cfg.DBPort != 5432 || cfg.DBUser != "artifacts" {
|
||||
t.Errorf("unexpected defaults: %+v", cfg)
|
||||
}
|
||||
if cfg.RedisURL != "redis://localhost:6379" || cfg.S3Bucket != "artifacts" || cfg.S3Secure {
|
||||
t.Errorf("unexpected defaults: %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadOverrides(t *testing.T) {
|
||||
t.Setenv("LISTEN_ADDR", ":9999")
|
||||
t.Setenv("DBHOST", "db.example.com")
|
||||
t.Setenv("DBPORT", "6000")
|
||||
t.Setenv("DBUSER", "u")
|
||||
t.Setenv("DBPASS", "pw")
|
||||
t.Setenv("DBNAME", "n")
|
||||
t.Setenv("DBSSL", "require")
|
||||
t.Setenv("MINIO_SECURE", "true")
|
||||
t.Setenv("MINIO_REGION", "us-east-1")
|
||||
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if cfg.ListenAddr != ":9999" || cfg.DBHost != "db.example.com" || cfg.DBPort != 6000 {
|
||||
t.Errorf("overrides not applied: %+v", cfg)
|
||||
}
|
||||
if !cfg.S3Secure {
|
||||
t.Error("MINIO_SECURE=true not parsed")
|
||||
}
|
||||
want := "postgres://u:pw@db.example.com:6000/n?sslmode=require"
|
||||
if got := cfg.DatabaseDSN(); got != want {
|
||||
t.Errorf("DSN = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadInvalidPort(t *testing.T) {
|
||||
t.Setenv("DBPORT", "not-a-number")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Error("expected error for invalid DBPORT")
|
||||
}
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) UpsertBlob(ctx context.Context, contentHash, s3Key string, sizeBytes int64, contentType string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO blobs (content_hash, s3_key, size_bytes, content_type)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (content_hash) DO NOTHING
|
||||
`, contentHash, s3Key, sizeBytes, contentType)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpsertArtifact(ctx context.Context, remoteName, path, contentHash, upstreamETag string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO artifacts (remote_name, path, content_hash, upstream_etag)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (remote_name, path) DO UPDATE SET
|
||||
content_hash = EXCLUDED.content_hash,
|
||||
upstream_etag = EXCLUDED.upstream_etag,
|
||||
last_fetched_at = NOW(),
|
||||
fetch_count = artifacts.fetch_count + 1
|
||||
`, remoteName, path, contentHash, upstreamETag)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) GetArtifact(ctx context.Context, remoteName, path string) (*models.Artifact, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||
a.last_accessed_at, a.fetch_count, a.access_count,
|
||||
b.size_bytes, b.content_type
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
WHERE a.remote_name = $1 AND a.path = $2
|
||||
`, remoteName, path)
|
||||
|
||||
var a models.Artifact
|
||||
err := row.Scan(
|
||||
&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &a.UpstreamETag,
|
||||
&a.UpstreamLastModified, &a.FirstSeenAt, &a.LastFetchedAt,
|
||||
&a.LastAccessedAt, &a.FetchCount, &a.AccessCount,
|
||||
&a.SizeBytes, &a.ContentType,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func (db *DB) TouchArtifactAccess(ctx context.Context, remoteName, path string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE artifacts SET
|
||||
last_accessed_at = NOW(),
|
||||
access_count = access_count + 1
|
||||
WHERE remote_name = $1 AND path = $2
|
||||
`, remoteName, path)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListArtifacts(ctx context.Context, remoteName string, limit, offset int) ([]models.Artifact, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||
a.last_accessed_at, a.fetch_count, a.access_count,
|
||||
b.size_bytes, b.content_type
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
WHERE a.remote_name = $1
|
||||
ORDER BY a.path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, remoteName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var artifacts []models.Artifact
|
||||
for rows.Next() {
|
||||
var a models.Artifact
|
||||
if err := rows.Scan(
|
||||
&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &a.UpstreamETag,
|
||||
&a.UpstreamLastModified, &a.FirstSeenAt, &a.LastFetchedAt,
|
||||
&a.LastAccessedAt, &a.FetchCount, &a.AccessCount,
|
||||
&a.SizeBytes, &a.ContentType,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
artifacts = append(artifacts, a)
|
||||
}
|
||||
return artifacts, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) DeleteArtifact(ctx context.Context, remoteName, path string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM artifacts WHERE remote_name = $1 AND path = $2`, remoteName, path)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cacheHit bool, sizeBytes int64, upstreamMS int, clientIP string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO access_log (remote_name, path, cache_hit, size_bytes, upstream_ms, client_ip)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
`, remoteName, path, cacheHit, sizeBytes, upstreamMS, clientIP)
|
||||
return err
|
||||
}
|
||||
|
||||
// AccessLogEntry is one buffered access-log record.
|
||||
type AccessLogEntry struct {
|
||||
RemoteName string
|
||||
Path string
|
||||
CacheHit bool
|
||||
SizeBytes int64
|
||||
UpstreamMS int
|
||||
ClientIP string
|
||||
}
|
||||
|
||||
// InsertAccessLogBatch bulk-inserts access-log rows with a single COPY.
|
||||
func (db *DB) InsertAccessLogBatch(ctx context.Context, entries []AccessLogEntry) error {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows := make([][]any, len(entries))
|
||||
for i, e := range entries {
|
||||
rows[i] = []any{e.RemoteName, e.Path, e.CacheHit, e.SizeBytes, e.UpstreamMS, e.ClientIP}
|
||||
}
|
||||
_, err := db.Pool.CopyFrom(ctx,
|
||||
pgx.Identifier{"access_log"},
|
||||
[]string{"remote_name", "path", "cache_hit", "size_bytes", "upstream_ms", "client_ip"},
|
||||
pgx.CopyFromRows(rows),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// FindOrphanedBlobs returns blobs no longer referenced by any artifact or
|
||||
// local file, restricted to those created before now()-minAge. The age cutoff
|
||||
// is a grace period that avoids a TOCTOU race with in-flight dedup uploads,
|
||||
// which insert the blob row before the referencing artifact/local_files row.
|
||||
func (db *DB) FindOrphanedBlobs(ctx context.Context, minAge time.Duration) ([]models.Blob, error) {
|
||||
cutoff := time.Now().Add(-minAge)
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT b.content_hash, b.s3_key, b.size_bytes, b.content_type, b.created_at
|
||||
FROM blobs b
|
||||
WHERE b.created_at < $1
|
||||
AND b.content_hash NOT IN (
|
||||
SELECT content_hash FROM artifacts
|
||||
UNION
|
||||
SELECT content_hash FROM local_files
|
||||
)
|
||||
`, cutoff)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var blobs []models.Blob
|
||||
for rows.Next() {
|
||||
var b models.Blob
|
||||
if err := rows.Scan(&b.ContentHash, &b.S3Key, &b.SizeBytes, &b.ContentType, &b.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blobs = append(blobs, b)
|
||||
}
|
||||
return blobs, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) DeleteBlob(ctx context.Context, contentHash string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM blobs WHERE content_hash = $1`, contentHash)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteColdArtifacts(ctx context.Context, remoteName string, olderThan time.Duration) (int64, error) {
|
||||
cutoff := time.Now().Add(-olderThan)
|
||||
tag, err := db.Pool.Exec(ctx, `
|
||||
DELETE FROM artifacts
|
||||
WHERE remote_name = $1 AND last_accessed_at < $2
|
||||
`, remoteName, cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
@@ -1,334 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
var (
|
||||
testDB *DB
|
||||
testDSN string
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
c := context.Background()
|
||||
dsn, terminate, err := testsupport.StartPostgres(c)
|
||||
if err != nil {
|
||||
// Docker unavailable: run anyway so tests self-skip via requireDB.
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN = dsn
|
||||
db, err := New(dsn)
|
||||
if err != nil {
|
||||
terminate()
|
||||
panic(err)
|
||||
}
|
||||
testDB = db
|
||||
|
||||
code := m.Run()
|
||||
|
||||
db.Close()
|
||||
terminate()
|
||||
// Return normally on success so the coverage profile is flushed; os.Exit
|
||||
// would truncate it.
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
func requireDB(t *testing.T) {
|
||||
t.Helper()
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable; skipping database integration test")
|
||||
}
|
||||
}
|
||||
|
||||
func ctx() context.Context { return context.Background() }
|
||||
|
||||
func seedRemote(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: name, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://example.com", MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed remote: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// seedBlob inserts a blob and returns its full content hash (sha256:<hash>),
|
||||
// matching the reference convention used by artifacts and local files.
|
||||
func seedBlob(t *testing.T, hash string) string {
|
||||
t.Helper()
|
||||
full := "sha256:" + hash
|
||||
if err := testDB.UpsertBlob(ctx(), full, "blobs/sha256/"+hash, 10, "application/octet-stream"); err != nil {
|
||||
t.Fatalf("seed blob: %v", err)
|
||||
}
|
||||
return full
|
||||
}
|
||||
|
||||
func TestRemotesCRUD(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-crud")
|
||||
got, err := testDB.GetRemote(ctx(), "r-crud")
|
||||
if err != nil || got.BaseURL != "https://example.com" {
|
||||
t.Fatalf("get: %v %v", got, err)
|
||||
}
|
||||
got.BaseURL = "https://updated.example.com"
|
||||
if err := testDB.UpdateRemote(ctx(), got); err != nil {
|
||||
t.Fatalf("update: %v", err)
|
||||
}
|
||||
got, _ = testDB.GetRemote(ctx(), "r-crud")
|
||||
if got.BaseURL != "https://updated.example.com" {
|
||||
t.Errorf("update not applied: %v", got.BaseURL)
|
||||
}
|
||||
list, err := testDB.ListRemotes(ctx())
|
||||
if err != nil || len(list) == 0 {
|
||||
t.Fatalf("list: %v %v", len(list), err)
|
||||
}
|
||||
if err := testDB.DeleteRemote(ctx(), "r-crud"); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetRemote(ctx(), "r-crud"); err == nil {
|
||||
t.Error("expected error after delete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactsAndBlobs(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-art")
|
||||
seedBlob(t, "aaaa")
|
||||
hash := "sha256:aaaa"
|
||||
if err := testDB.UpsertBlob(ctx(), hash, "blobs/sha256/aaaa", 10, "text/plain"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Upsert again to exercise the ON CONFLICT update branch.
|
||||
if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
art, err := testDB.GetArtifact(ctx(), "r-art", "path/a.txt")
|
||||
if err != nil || art.ContentHash != hash {
|
||||
t.Fatalf("get artifact: %v %v", art, err)
|
||||
}
|
||||
if err := testDB.TouchArtifactAccess(ctx(), "r-art", "path/a.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arts, err := testDB.ListArtifacts(ctx(), "r-art", 10, 0)
|
||||
if err != nil || len(arts) != 1 {
|
||||
t.Fatalf("list artifacts: %v %v", len(arts), err)
|
||||
}
|
||||
if err := testDB.InsertAccessLog(ctx(), "r-art", "path/a.txt", true, 10, 5, "1.2.3.4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.InsertAccessLogBatch(ctx(), []AccessLogEntry{
|
||||
{RemoteName: "r-art", Path: "b", CacheHit: false, SizeBytes: 20, UpstreamMS: 3},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.InsertAccessLogBatch(ctx(), nil); err != nil {
|
||||
t.Fatalf("empty batch should be a no-op: %v", err)
|
||||
}
|
||||
if err := testDB.DeleteArtifact(ctx(), "r-art", "path/a.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrphanAndColdCleanup(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedBlob(t, "orphanhash")
|
||||
// A blob with no artifact/local_file reference is orphaned, but only past
|
||||
// the grace period.
|
||||
if got, _ := testDB.FindOrphanedBlobs(ctx(), time.Hour); containsHash(got, "sha256:orphanhash") {
|
||||
t.Error("fresh orphan should be excluded by grace period")
|
||||
}
|
||||
orphans, err := testDB.FindOrphanedBlobs(ctx(), -time.Hour) // cutoff in the future => include fresh
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !containsHash(orphans, "sha256:orphanhash") {
|
||||
t.Error("expected orphan to be found with zero grace")
|
||||
}
|
||||
if err := testDB.DeleteBlob(ctx(), "sha256:orphanhash"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
seedRemote(t, "r-cold")
|
||||
seedBlob(t, "coldhash")
|
||||
testDB.UpsertArtifact(ctx(), "r-cold", "cold.txt", "sha256:coldhash", "")
|
||||
n, err := testDB.DeleteColdArtifacts(ctx(), "r-cold", -time.Hour) // negative => everything is "cold"
|
||||
if err != nil || n < 1 {
|
||||
t.Fatalf("delete cold: n=%d err=%v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func containsHash(blobs []models.Blob, hash string) bool {
|
||||
for _, b := range blobs {
|
||||
if b.ContentHash == hash {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestLocalFiles(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-local")
|
||||
seedBlob(t, "localhash")
|
||||
hash := "sha256:localhash"
|
||||
if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Duplicate create must be rejected.
|
||||
if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err == nil {
|
||||
t.Error("expected duplicate local file error")
|
||||
}
|
||||
f, err := testDB.GetLocalFile(ctx(), "r-local", "foo/foo-1.0.whl")
|
||||
if err != nil || f == nil {
|
||||
t.Fatalf("get local file: %v %v", f, err)
|
||||
}
|
||||
if files, err := testDB.ListLocalFiles(ctx(), "r-local", 10, 0); err != nil || len(files) != 1 {
|
||||
t.Fatalf("list: %v %v", len(files), err)
|
||||
}
|
||||
if files, err := testDB.ListLocalFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(files) != 1 {
|
||||
t.Fatalf("list by prefix: %v %v", len(files), err)
|
||||
}
|
||||
if entries, err := testDB.ListFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(entries) != 1 {
|
||||
t.Fatalf("provider list by prefix: %v %v", len(entries), err)
|
||||
}
|
||||
if pkgs, err := testDB.ListLocalFilePackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 {
|
||||
t.Fatalf("list packages: %v %v", pkgs, err)
|
||||
}
|
||||
if pkgs, err := testDB.ListPackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 {
|
||||
t.Fatalf("provider list packages: %v %v", pkgs, err)
|
||||
}
|
||||
if err := testDB.DeleteLocalFile(ctx(), "r-local", "foo/foo-1.0.whl"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualsCRUD(t *testing.T) {
|
||||
requireDB(t)
|
||||
if err := testDB.CreateVirtual(ctx(), &models.Virtual{
|
||||
Name: "v-crud", PackageType: models.PackageHelm, Members: []string{"a", "b"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := testDB.GetVirtual(ctx(), "v-crud")
|
||||
if err != nil || len(v.Members) != 2 {
|
||||
t.Fatalf("get virtual: %v %v", v, err)
|
||||
}
|
||||
v.Members = []string{"a"}
|
||||
if err := testDB.UpdateVirtual(ctx(), v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if vs, err := testDB.ListVirtuals(ctx()); err != nil || len(vs) == 0 {
|
||||
t.Fatalf("list virtuals: %v %v", len(vs), err)
|
||||
}
|
||||
if err := testDB.DeleteVirtual(ctx(), "v-crud"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStats(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-stats")
|
||||
seedBlob(t, "statshash")
|
||||
testDB.UpsertArtifact(ctx(), "r-stats", "s.txt", "sha256:statshash", "")
|
||||
testDB.InsertAccessLog(ctx(), "r-stats", "s.txt", true, 100, 2, "")
|
||||
|
||||
if _, err := testDB.GetOverviewStats(ctx()); err != nil {
|
||||
t.Fatalf("overview: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetTopRemotes(ctx(), 5); err != nil {
|
||||
t.Fatalf("top remotes: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetTopFilesByHits(ctx(), 5); err != nil {
|
||||
t.Fatalf("top files by hits: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetTopFilesByBandwidth(ctx(), 5); err != nil {
|
||||
t.Fatalf("top files by bandwidth: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatabaseErrorPaths(t *testing.T) {
|
||||
requireDB(t)
|
||||
bad, err := New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bad.Close() // every query now fails
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := bad.ListRemotes(ctx); err == nil {
|
||||
t.Error("ListRemotes should error on closed db")
|
||||
}
|
||||
if _, err := bad.ListVirtuals(ctx); err == nil {
|
||||
t.Error("ListVirtuals should error")
|
||||
}
|
||||
if _, err := bad.ListArtifacts(ctx, "r", 10, 0); err == nil {
|
||||
t.Error("ListArtifacts should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFiles(ctx, "r", 10, 0); err == nil {
|
||||
t.Error("ListLocalFiles should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFilesByPrefix(ctx, "r", "p"); err == nil {
|
||||
t.Error("ListLocalFilesByPrefix should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFilePackages(ctx, "r"); err == nil {
|
||||
t.Error("ListLocalFilePackages should error")
|
||||
}
|
||||
if _, err := bad.ListFilesByPrefix(ctx, "r", "p"); err == nil {
|
||||
t.Error("ListFilesByPrefix should error")
|
||||
}
|
||||
if _, err := bad.ListPackages(ctx, "r"); err == nil {
|
||||
t.Error("ListPackages should error")
|
||||
}
|
||||
if _, err := bad.FindOrphanedBlobs(ctx, 0); err == nil {
|
||||
t.Error("FindOrphanedBlobs should error")
|
||||
}
|
||||
if _, err := bad.GetOverviewStats(ctx); err == nil {
|
||||
t.Error("GetOverviewStats should error")
|
||||
}
|
||||
if _, err := bad.GetTopRemotes(ctx, 5); err == nil {
|
||||
t.Error("GetTopRemotes should error")
|
||||
}
|
||||
if _, err := bad.GetTopFilesByHits(ctx, 5); err == nil {
|
||||
t.Error("GetTopFilesByHits should error")
|
||||
}
|
||||
if _, err := bad.GetTopFilesByBandwidth(ctx, 5); err == nil {
|
||||
t.Error("GetTopFilesByBandwidth should error")
|
||||
}
|
||||
if _, err := bad.ListRPMMetadataEntries(ctx, "r"); err == nil {
|
||||
t.Error("ListRPMMetadataEntries should error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMMetadata(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-rpm")
|
||||
meta := &provider.RPMMetadata{
|
||||
RepoName: "r-rpm", FilePath: "Packages/x.rpm", ContentHash: "sha256:rpm",
|
||||
Name: "x", Version: "1.0", Release: "1", Arch: "noarch",
|
||||
Requires: []provider.RPMDep{{Name: "libc"}},
|
||||
Provides: []provider.RPMDep{{Name: "x"}},
|
||||
Files: []provider.RPMFile{},
|
||||
}
|
||||
if err := testDB.InsertRPMMetadata(ctx(), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, err := testDB.ListRPMMetadataEntries(ctx(), "r-rpm")
|
||||
if err != nil || len(entries) != 1 {
|
||||
t.Fatalf("list rpm entries: %v %v", len(entries), err)
|
||||
}
|
||||
if rows, err := testDB.ListRPMMetadata(ctx(), "r-rpm"); err != nil || len(rows) != 1 {
|
||||
t.Fatalf("list rpm rows: %v %v", len(rows), err)
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type LocalFile struct {
|
||||
ID int64 `json:"id"`
|
||||
RepoName string `json:"repo_name"`
|
||||
FilePath string `json:"file_path"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
var ErrAlreadyExists = fmt.Errorf("file already exists")
|
||||
|
||||
func (db *DB) CreateLocalFile(ctx context.Context, repoName, filePath, contentHash string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO local_files (repo_name, file_path, content_hash)
|
||||
VALUES ($1, $2, $3)
|
||||
`, repoName, filePath, contentHash)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return ErrAlreadyExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) GetLocalFile(ctx context.Context, repoName, filePath string) (*LocalFile, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1 AND file_path = $2
|
||||
`, repoName, filePath)
|
||||
|
||||
var f LocalFile
|
||||
if err := row.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &f, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFiles(ctx context.Context, repoName string, limit, offset int) ([]LocalFile, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1
|
||||
ORDER BY file_path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, repoName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var files []LocalFile
|
||||
for rows.Next() {
|
||||
var f LocalFile
|
||||
if err := rows.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, f)
|
||||
}
|
||||
return files, rows.Err()
|
||||
}
|
||||
|
||||
// ListLocalArtifacts returns a repo's local files shaped as models.Artifact so
|
||||
// the UI's cached-objects view can render them the same way as remote artifacts.
|
||||
// Local files carry no access/fetch counters, so those are left at zero and the
|
||||
// timestamps are all derived from created_at.
|
||||
func (db *DB) ListLocalArtifacts(ctx context.Context, repoName string, limit, offset int) ([]models.Artifact, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT lf.id, lf.repo_name, lf.file_path, lf.content_hash,
|
||||
lf.created_at, b.size_bytes, b.content_type
|
||||
FROM local_files lf
|
||||
JOIN blobs b ON lf.content_hash = b.content_hash
|
||||
WHERE lf.repo_name = $1
|
||||
ORDER BY lf.file_path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, repoName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var artifacts []models.Artifact
|
||||
for rows.Next() {
|
||||
var a models.Artifact
|
||||
var createdAt time.Time
|
||||
if err := rows.Scan(&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &createdAt, &a.SizeBytes, &a.ContentType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.FirstSeenAt = createdAt
|
||||
a.LastFetchedAt = createdAt
|
||||
a.LastAccessedAt = createdAt
|
||||
artifacts = append(artifacts, a)
|
||||
}
|
||||
return artifacts, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFilesByPrefix(ctx context.Context, repoName, prefix string) ([]LocalFile, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1 AND file_path LIKE $2
|
||||
ORDER BY file_path
|
||||
`, repoName, prefix+"%")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var files []LocalFile
|
||||
for rows.Next() {
|
||||
var f LocalFile
|
||||
if err := rows.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, f)
|
||||
}
|
||||
return files, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFilePackages(ctx context.Context, repoName string) ([]string, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT DISTINCT split_part(file_path, '/', 1)
|
||||
FROM local_files
|
||||
WHERE repo_name = $1
|
||||
ORDER BY 1
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var packages []string
|
||||
for rows.Next() {
|
||||
var pkg string
|
||||
if err := rows.Scan(&pkg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
return packages, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]provider.FileEntry, error) {
|
||||
files, err := db.ListLocalFilesByPrefix(ctx, repoName, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]provider.FileEntry, len(files))
|
||||
for i, f := range files {
|
||||
result[i] = provider.FileEntry{FilePath: f.FilePath, ContentHash: f.ContentHash}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListPackages(ctx context.Context, repoName string) ([]string, error) {
|
||||
return db.ListLocalFilePackages(ctx, repoName)
|
||||
}
|
||||
|
||||
func (db *DB) DeleteLocalFile(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM local_files WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type DB struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func New(dsn string) (*DB, error) {
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect to postgres: %w", err)
|
||||
}
|
||||
|
||||
if err := pool.Ping(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("ping postgres: %w", err)
|
||||
}
|
||||
|
||||
db := &DB{Pool: pool}
|
||||
if err := db.migrate(); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("run migrations: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (db *DB) Close() {
|
||||
db.Pool.Close()
|
||||
}
|
||||
|
||||
func (db *DB) migrate() error {
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS remotes (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
repo_type TEXT DEFAULT 'remote',
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
immutable_ttl INTEGER DEFAULT 0,
|
||||
mutable_ttl INTEGER DEFAULT 3600,
|
||||
check_mutable BOOLEAN DEFAULT TRUE,
|
||||
patterns TEXT[] DEFAULT '{}',
|
||||
blocklist TEXT[] DEFAULT '{}',
|
||||
mutable_patterns TEXT[] DEFAULT '{}',
|
||||
immutable_patterns TEXT[] DEFAULT '{}',
|
||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
||||
ban_tags TEXT[] DEFAULT '{}',
|
||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
||||
quarantine_days INTEGER DEFAULT 3,
|
||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
||||
releases_remote TEXT DEFAULT '',
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS virtuals (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
members TEXT[] NOT NULL,
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS blobs (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
s3_key TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
content_type TEXT DEFAULT 'application/octet-stream',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artifacts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
upstream_etag TEXT DEFAULT '',
|
||||
upstream_last_modified TIMESTAMPTZ,
|
||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
fetch_count BIGINT DEFAULT 1,
|
||||
access_count BIGINT DEFAULT 1,
|
||||
UNIQUE(remote_name, path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS local_files (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS access_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
cache_hit BOOLEAN NOT NULL,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
upstream_ms INTEGER DEFAULT 0,
|
||||
client_ip TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rpm_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
epoch INTEGER DEFAULT 0,
|
||||
version TEXT NOT NULL,
|
||||
release TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
summary TEXT DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
rpm_size BIGINT DEFAULT 0,
|
||||
installed_size BIGINT DEFAULT 0,
|
||||
license TEXT DEFAULT '',
|
||||
vendor TEXT DEFAULT '',
|
||||
build_group TEXT DEFAULT '',
|
||||
build_host TEXT DEFAULT '',
|
||||
source_rpm TEXT DEFAULT '',
|
||||
url TEXT DEFAULT '',
|
||||
packager TEXT DEFAULT '',
|
||||
requires JSONB DEFAULT '[]',
|
||||
provides JSONB DEFAULT '[]',
|
||||
files JSONB DEFAULT '[]',
|
||||
changelogs JSONB DEFAULT '[]',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
|
||||
`)
|
||||
return err
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const remoteCols = `name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
created_at, updated_at`
|
||||
|
||||
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
|
||||
return scanner.Scan(
|
||||
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Description, &r.Username, &r.Password,
|
||||
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
|
||||
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
|
||||
&r.BanTagsEnabled, &r.BanTags,
|
||||
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
|
||||
&r.ReleasesRemote, &r.ManagedBy,
|
||||
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
|
||||
&r.CreatedAt, &r.UpdatedAt,
|
||||
)
|
||||
}
|
||||
|
||||
func (db *DB) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
|
||||
row := db.Pool.QueryRow(ctx, `SELECT `+remoteCols+` FROM remotes WHERE name = $1`, name)
|
||||
var r models.Remote
|
||||
if err := scanRemote(row, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes ORDER BY name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var remotes []models.Remote
|
||||
for rows.Next() {
|
||||
var r models.Remote
|
||||
if err := scanRemote(rows, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remotes = append(remotes, r)
|
||||
}
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO remotes (
|
||||
name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24)
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE remotes SET
|
||||
package_type=$2, repo_type=$3, base_url=$4, description=$5, username=$6, password=$7,
|
||||
immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
|
||||
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
|
||||
ban_tags_enabled=$15, ban_tags=$16,
|
||||
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
|
||||
releases_remote=$20, managed_by=$21,
|
||||
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
|
||||
updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteRemote(ctx context.Context, name string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM remotes WHERE name = $1`, name)
|
||||
return err
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
|
||||
requiresJSON, _ := json.Marshal(meta.Requires)
|
||||
providesJSON, _ := json.Marshal(meta.Provides)
|
||||
filesJSON, _ := json.Marshal(meta.Files)
|
||||
changelogsJSON, _ := json.Marshal(meta.Changelogs)
|
||||
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO rpm_metadata (
|
||||
repo_name, file_path, content_hash,
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, files, changelogs
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash,
|
||||
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
|
||||
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
|
||||
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
|
||||
requiresJSON, providesJSON, filesJSON, changelogsJSON,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM rpm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
type RPMMetadataRow struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Epoch int
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
Summary string
|
||||
Description string
|
||||
RPMSize int64
|
||||
InstalledSize int64
|
||||
License string
|
||||
Vendor string
|
||||
Group string
|
||||
BuildHost string
|
||||
SourceRPM string
|
||||
URL string
|
||||
Packager string
|
||||
Requires json.RawMessage
|
||||
Provides json.RawMessage
|
||||
Files json.RawMessage
|
||||
Changelogs json.RawMessage
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]provider.RPMMetadata, error) {
|
||||
rows, err := db.ListRPMMetadata(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]provider.RPMMetadata, len(rows))
|
||||
for i, r := range rows {
|
||||
meta := provider.RPMMetadata{
|
||||
RepoName: r.RepoName,
|
||||
FilePath: r.FilePath,
|
||||
ContentHash: r.ContentHash,
|
||||
Name: r.Name,
|
||||
Epoch: r.Epoch,
|
||||
Version: r.Version,
|
||||
Release: r.Release,
|
||||
Arch: r.Arch,
|
||||
Summary: r.Summary,
|
||||
Description: r.Description,
|
||||
RPMSize: r.RPMSize,
|
||||
InstalledSize: r.InstalledSize,
|
||||
License: r.License,
|
||||
Vendor: r.Vendor,
|
||||
Group: r.Group,
|
||||
BuildHost: r.BuildHost,
|
||||
SourceRPM: r.SourceRPM,
|
||||
URL: r.URL,
|
||||
Packager: r.Packager,
|
||||
}
|
||||
json.Unmarshal(r.Requires, &meta.Requires)
|
||||
json.Unmarshal(r.Provides, &meta.Provides)
|
||||
json.Unmarshal(r.Files, &meta.Files)
|
||||
json.Unmarshal(r.Changelogs, &meta.Changelogs)
|
||||
result[i] = meta
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetadataRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash,
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, files, changelogs
|
||||
FROM rpm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, epoch, version, release, arch
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []RPMMetadataRow
|
||||
for rows.Next() {
|
||||
var r RPMMetadataRow
|
||||
if err := rows.Scan(
|
||||
&r.RepoName, &r.FilePath, &r.ContentHash,
|
||||
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
|
||||
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
|
||||
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
|
||||
&r.Requires, &r.Provides, &r.Files, &r.Changelogs,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) GetOverviewStats(ctx context.Context) (*models.OverviewStats, error) {
|
||||
var stats models.OverviewStats
|
||||
|
||||
err := db.Pool.QueryRow(ctx, `SELECT COUNT(*) FROM remotes`).Scan(&stats.TotalRemotes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `SELECT COALESCE(COUNT(*), 0), COALESCE(SUM(b.size_bytes), 0)
|
||||
FROM artifacts a JOIN blobs b ON a.content_hash = b.content_hash`).
|
||||
Scan(&stats.TotalObjects, &stats.TotalBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(
|
||||
(SELECT COUNT(*) FROM artifacts) - (SELECT COUNT(DISTINCT content_hash) FROM artifacts),
|
||||
0
|
||||
)`).Scan(&stats.TotalBlobsDeduped)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(size_bytes), 0)
|
||||
FROM access_log
|
||||
WHERE cache_hit = TRUE AND created_at > NOW() - INTERVAL '30 days'
|
||||
`).Scan(&stats.BandwidthSaved30d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
type RemoteStatRow struct {
|
||||
Name string `json:"name"`
|
||||
ObjectCount int64 `json:"object_count"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
Requests30d int64 `json:"requests_30d"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopRemotes(ctx context.Context, limit int) ([]RemoteStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT r.name,
|
||||
COALESCE(a.cnt, 0) AS object_count,
|
||||
COALESCE(a.total_bytes, 0) AS total_bytes,
|
||||
COALESCE(l.req_count, 0) AS requests_30d
|
||||
FROM remotes r
|
||||
LEFT JOIN (
|
||||
SELECT remote_name, COUNT(*) AS cnt, SUM(b.size_bytes) AS total_bytes
|
||||
FROM artifacts a JOIN blobs b ON a.content_hash = b.content_hash
|
||||
GROUP BY remote_name
|
||||
) a ON r.name = a.remote_name
|
||||
LEFT JOIN (
|
||||
SELECT remote_name, COUNT(*) AS req_count
|
||||
FROM access_log
|
||||
WHERE created_at > NOW() - INTERVAL '30 days'
|
||||
GROUP BY remote_name
|
||||
) l ON r.name = l.remote_name
|
||||
ORDER BY COALESCE(a.total_bytes, 0) DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []RemoteStatRow
|
||||
for rows.Next() {
|
||||
var r RemoteStatRow
|
||||
if err := rows.Scan(&r.Name, &r.ObjectCount, &r.TotalBytes, &r.Requests30d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
type FileStatRow struct {
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
AccessCount int64 `json:"access_count"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopFilesByHits(ctx context.Context, limit int) ([]FileStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT a.remote_name, a.path, a.access_count, b.size_bytes
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
ORDER BY a.access_count DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []FileStatRow
|
||||
for rows.Next() {
|
||||
var r FileStatRow
|
||||
if err := rows.Scan(&r.RemoteName, &r.Path, &r.AccessCount, &r.SizeBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
type BandwidthStatRow struct {
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
Bandwidth int64 `json:"bandwidth"`
|
||||
Requests int64 `json:"requests"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopFilesByBandwidth(ctx context.Context, limit int) ([]BandwidthStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT remote_name, path,
|
||||
COALESCE(SUM(size_bytes), 0) AS bandwidth,
|
||||
COUNT(*) AS requests
|
||||
FROM access_log
|
||||
WHERE created_at > NOW() - INTERVAL '30 days'
|
||||
GROUP BY remote_name, path
|
||||
ORDER BY bandwidth DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []BandwidthStatRow
|
||||
for rows.Next() {
|
||||
var r BandwidthStatRow
|
||||
if err := rows.Scan(&r.RemoteName, &r.Path, &r.Bandwidth, &r.Requests); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) GetVirtual(ctx context.Context, name string) (*models.Virtual, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT name, package_type, description, members, managed_by, created_at, updated_at
|
||||
FROM virtuals WHERE name = $1
|
||||
`, name)
|
||||
|
||||
var v models.Virtual
|
||||
err := row.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListVirtuals(ctx context.Context) ([]models.Virtual, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT name, package_type, description, members, managed_by, created_at, updated_at
|
||||
FROM virtuals ORDER BY name
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var virtuals []models.Virtual
|
||||
for rows.Next() {
|
||||
var v models.Virtual
|
||||
if err := rows.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
virtuals = append(virtuals, v)
|
||||
}
|
||||
return virtuals, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) CreateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO virtuals (name, package_type, description, members, managed_by)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpdateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE virtuals SET
|
||||
package_type=$2, description=$3, members=$4, managed_by=$5, updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteVirtual(ctx context.Context, name string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM virtuals WHERE name = $1`, name)
|
||||
return err
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
)
|
||||
|
||||
// blobGracePeriod is how old an orphaned blob must be before GC will delete
|
||||
// it. This avoids racing in-flight dedup uploads that insert the blob row
|
||||
// before the referencing artifact/local_files row exists.
|
||||
const blobGracePeriod = 1 * time.Hour
|
||||
|
||||
type Collector struct {
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func New(db *database.DB, store *storage.S3, interval time.Duration) *Collector {
|
||||
return &Collector{db: db, store: store, interval: interval}
|
||||
}
|
||||
|
||||
func (c *Collector) Run(ctx context.Context) {
|
||||
slog.Info("gc started", "interval", c.interval)
|
||||
ticker := time.NewTicker(c.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("gc stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
c.sweep(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Collector) sweep(ctx context.Context) {
|
||||
start := time.Now()
|
||||
|
||||
orphaned, err := c.db.FindOrphanedBlobs(ctx, blobGracePeriod)
|
||||
if err != nil {
|
||||
slog.Error("gc: find orphaned blobs", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for _, blob := range orphaned {
|
||||
if err := c.store.Delete(ctx, blob.S3Key); err != nil {
|
||||
slog.Warn("gc: delete s3 object", "key", blob.S3Key, "error", err)
|
||||
continue
|
||||
}
|
||||
if err := c.db.DeleteBlob(ctx, blob.ContentHash); err != nil {
|
||||
slog.Warn("gc: delete blob row", "hash", blob.ContentHash, "error", err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
|
||||
if deleted > 0 || len(orphaned) > 0 {
|
||||
slog.Info("gc sweep complete",
|
||||
"orphaned_found", len(orphaned),
|
||||
"deleted", deleted,
|
||||
"duration_ms", time.Since(start).Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
var (
|
||||
testDB *database.DB
|
||||
testStore *storage.S3
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, termPG, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
minio, termMinio, err := testsupport.StartMinio(ctx)
|
||||
if err != nil {
|
||||
termPG()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
db, err := database.New(dsn)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var s3 *storage.S3
|
||||
for i := 0; i < 20; i++ {
|
||||
if s3, err = storage.NewS3(minio.Endpoint, minio.AccessKey, minio.SecretKey, "gc-test", false, ""); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
testDB = db
|
||||
testStore = s3
|
||||
|
||||
code := m.Run()
|
||||
db.Close()
|
||||
termMinio()
|
||||
termPG()
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepDeletesOldOrphan(t *testing.T) {
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
hash := "sha256:gcorphan"
|
||||
key := storage.BlobKey("gcorphan")
|
||||
|
||||
if err := testStore.Upload(ctx, key, bytes.NewReader([]byte("orphan")), 6, "application/octet-stream"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.UpsertBlob(ctx, hash, key, 6, "application/octet-stream"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Age the blob past the grace period.
|
||||
if _, err := testDB.Pool.Exec(ctx, `UPDATE blobs SET created_at = now() - interval '2 hours' WHERE content_hash = $1`, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c := New(testDB, testStore, time.Hour)
|
||||
c.sweep(ctx)
|
||||
|
||||
if exists, _ := testStore.Exists(ctx, key); exists {
|
||||
t.Error("expected orphan object deleted from store")
|
||||
}
|
||||
orphans, _ := testDB.FindOrphanedBlobs(ctx, 0)
|
||||
for _, b := range orphans {
|
||||
if b.ContentHash == hash {
|
||||
t.Error("expected orphan blob row deleted")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepNoOrphans(t *testing.T) {
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
// A sweep with nothing to collect should be a clean no-op.
|
||||
New(testDB, testStore, time.Hour).sweep(context.Background())
|
||||
}
|
||||
|
||||
func TestRunStopsOnContextCancel(t *testing.T) {
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
New(testDB, testStore, time.Hour).Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Run did not return after context cancel")
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package gc_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/gc"
|
||||
)
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
c := gc.New(nil, nil, 1*time.Hour)
|
||||
if c == nil {
|
||||
t.Fatal("expected non-nil collector")
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageAlpine }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "APKINDEX.tar.gz") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".apk") {
|
||||
return "application/vnd.android.package-archive"
|
||||
}
|
||||
if strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestType(t *testing.T) {
|
||||
if (&Provider{}).Type() != models.PackageAlpine {
|
||||
t.Fatal("wrong type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("v3.19/main/x86_64/APKINDEX.tar.gz") != provider.Mutable {
|
||||
t.Error("APKINDEX should be mutable")
|
||||
}
|
||||
if p.Classify("v3.19/main/x86_64/curl-8.0-r0.apk") != provider.Immutable {
|
||||
t.Error("apk should be immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
cases := map[string]string{
|
||||
"pkg.apk": "application/vnd.android.package-archive",
|
||||
"APKINDEX.tar.gz": "application/gzip",
|
||||
"something.random": "application/octet-stream",
|
||||
}
|
||||
for path, want := range cases {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
p := &Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://dl-cdn.alpinelinux.org/alpine/"}, "/v3.19/main/x86_64/curl.apk")
|
||||
if got != "https://dl-cdn.alpinelinux.org/alpine/v3.19/main/x86_64/curl.apk" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
if out, err := (&Provider{}).RewriteResponse([]byte("x"), models.Remote{}, "http://proxy"); out != nil || err != nil {
|
||||
t.Error("alpine never rewrites")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthHeaders(t *testing.T) {
|
||||
h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("expected auth header")
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var (
|
||||
tagManifestRe = regexp.MustCompile(`/manifests/[^/]+$`)
|
||||
digestManifestRe = regexp.MustCompile(`/manifests/sha256:[0-9a-fA-F]+$`)
|
||||
tagsListRe = regexp.MustCompile(`/tags/list$`)
|
||||
)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageDocker }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if tagsListRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
if tagManifestRe.MatchString(path) && !digestManifestRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.Contains(path, "/blobs/") {
|
||||
return "application/octet-stream"
|
||||
}
|
||||
if strings.Contains(path, "/manifests/") {
|
||||
return "application/vnd.docker.distribution.manifest.v2+json"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/v2/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" && remote.Password != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestDockerClassifyBranches(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("library/nginx/tags/list") != provider.Mutable {
|
||||
t.Error("tags/list should be mutable")
|
||||
}
|
||||
if p.Classify("library/nginx/manifests/latest") != provider.Mutable {
|
||||
t.Error("tag manifest should be mutable")
|
||||
}
|
||||
if p.Classify("library/nginx/manifests/sha256:abcdef") != provider.Immutable {
|
||||
t.Error("digest manifest should be immutable")
|
||||
}
|
||||
if p.Classify("library/nginx/blobs/sha256:abc") != provider.Immutable {
|
||||
t.Error("blob should be immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.ContentType("x/blobs/sha256:abc") != "application/octet-stream" {
|
||||
t.Error("blob content type")
|
||||
}
|
||||
if p.ContentType("x/manifests/latest") != "application/vnd.docker.distribution.manifest.v2+json" {
|
||||
t.Error("manifest content type")
|
||||
}
|
||||
if p.ContentType("x/tags/list") != "application/json" {
|
||||
t.Error("default content type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerRewriteAndAuth(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if out, err := p.RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
|
||||
t.Error("docker never rewrites")
|
||||
}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("expected basic auth header")
|
||||
}
|
||||
h, _ = p.AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("no creds, no header")
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package docker_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
if p.Type() != models.PackageDocker {
|
||||
t.Errorf("expected docker, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"library/nginx/manifests/latest", provider.Mutable},
|
||||
{"library/nginx/manifests/v1.25", provider.Mutable},
|
||||
{"library/nginx/manifests/sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", provider.Immutable},
|
||||
{"library/nginx/tags/list", provider.Mutable},
|
||||
{"library/nginx/blobs/sha256:abc123", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_UpstreamURL(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://registry-1.docker.io"}, "library/nginx/manifests/latest")
|
||||
want := "https://registry-1.docker.io/v2/library/nginx/manifests/latest"
|
||||
if got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
if p.ContentType("x/blobs/sha256:abc") != "application/octet-stream" {
|
||||
t.Error("blobs should be octet-stream")
|
||||
}
|
||||
if p.ContentType("x/manifests/latest") != "application/vnd.docker.distribution.manifest.v2+json" {
|
||||
t.Error("manifests should be manifest type")
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package generic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageGeneric }
|
||||
|
||||
func (p *Provider) Classify(_ string) provider.Mutability {
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
var contentTypeMap = map[string]string{
|
||||
".tar.gz": "application/gzip",
|
||||
".tgz": "application/gzip",
|
||||
".gz": "application/gzip",
|
||||
".zip": "application/zip",
|
||||
".whl": "application/zip",
|
||||
".exe": "application/x-msdownload",
|
||||
".rpm": "application/x-rpm",
|
||||
".xml": "application/xml",
|
||||
".yaml": "text/yaml",
|
||||
".yml": "text/yaml",
|
||||
".json": "application/json",
|
||||
".sig": "application/octet-stream",
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(filePath string) string {
|
||||
lower := strings.ToLower(filePath)
|
||||
if strings.HasSuffix(lower, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
ext := path.Ext(lower)
|
||||
if ct, ok := contentTypeMap[ext]; ok {
|
||||
return ct
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, reqPath string) string {
|
||||
base := strings.TrimRight(remote.BaseURL, "/")
|
||||
return base + "/" + strings.TrimLeft(reqPath, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package generic
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestGenericRewriteResponse(t *testing.T) {
|
||||
if out, err := (&Provider{}).RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
|
||||
t.Error("generic never rewrites")
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package generic_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
if p.Type() != models.PackageGeneric {
|
||||
t.Errorf("expected generic, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify_AllImmutable(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
paths := []string{"file.tar.gz", "path/to/binary", "index.html", "data.json"}
|
||||
for _, path := range paths {
|
||||
if p.Classify(path) != provider.Immutable {
|
||||
t.Errorf("generic should classify %q as immutable", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
tests := []struct{ path, want string }{
|
||||
{"file.tar.gz", "application/gzip"},
|
||||
{"file.tgz", "application/gzip"},
|
||||
{"file.zip", "application/zip"},
|
||||
{"file.rpm", "application/x-rpm"},
|
||||
{"file.json", "application/json"},
|
||||
{"file.unknown", "application/octet-stream"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.ContentType(tt.path); got != tt.want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_UpstreamURL(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://example.com/repo"}, "path/to/file.tar.gz")
|
||||
want := "https://example.com/repo/path/to/file.tar.gz"
|
||||
if got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_AuthHeaders_BasicAuth(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "user", Password: "pass"})
|
||||
if h.Get("Authorization") != "Basic dXNlcjpwYXNz" {
|
||||
t.Errorf("unexpected auth header: %q", h.Get("Authorization"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_AuthHeaders_NoAuth(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("expected no auth header")
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package goproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageGoProxy }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "/@v/list") || strings.HasSuffix(path, "/@latest") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(path, ".mod") {
|
||||
return "text/plain"
|
||||
}
|
||||
if strings.HasSuffix(path, ".info") {
|
||||
return "application/json"
|
||||
}
|
||||
if strings.HasSuffix(path, "/list") {
|
||||
return "text/plain"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package goproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestGoProxyURLAuthRewrite(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://proxy.golang.org/"}, "/mod/@v/list"); got != "https://proxy.golang.org/mod/@v/list" {
|
||||
t.Errorf("upstream url %q", got)
|
||||
}
|
||||
if out, err := p.RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
|
||||
t.Error("goproxy never rewrites")
|
||||
}
|
||||
if h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"}); h.Get("Authorization") == "" {
|
||||
t.Error("expected basic auth header")
|
||||
}
|
||||
if got := p.ContentType("mod/@v/v1.0.0.info"); got != "application/json" {
|
||||
t.Errorf("info content type %q", got)
|
||||
}
|
||||
if got := p.ContentType("mod/@v/v1.0.0.mod"); got != "text/plain" {
|
||||
t.Errorf("mod content type %q", got)
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package goproxy_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
if p.Type() != models.PackageGoProxy {
|
||||
t.Errorf("expected goproxy, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"golang.org/x/net/@v/list", provider.Mutable},
|
||||
{"golang.org/x/net/@latest", provider.Mutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.info", provider.Immutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.mod", provider.Immutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.zip", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
tests := []struct{ path, want string }{
|
||||
{"m/@v/v1.0.0.zip", "application/zip"},
|
||||
{"m/@v/v1.0.0.mod", "text/plain"},
|
||||
{"m/@v/v1.0.0.info", "application/json"},
|
||||
{"m/@v/list", "text/plain"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.ContentType(tt.path); got != tt.want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package helm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageHelm }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "index.yaml") || strings.HasSuffix(path, "index.yml") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tgz") || strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") {
|
||||
return "text/yaml"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
rewritten := strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
if rewritten == content {
|
||||
return nil, nil
|
||||
}
|
||||
return []byte(rewritten), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package helm
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestHelmContentTypeBranches(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for path, want := range map[string]string{
|
||||
"charts/x-1.0.0.tgz": "application/gzip",
|
||||
"x.tar.gz": "application/gzip",
|
||||
"index.yaml": "text/yaml",
|
||||
"x.yml": "text/yaml",
|
||||
"other": "application/octet-stream",
|
||||
} {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package helm_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/helm"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
if p.Type() != models.PackageHelm {
|
||||
t.Errorf("expected helm, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"index.yaml", provider.Mutable},
|
||||
{"index.yml", provider.Mutable},
|
||||
{"chart-1.0.tgz", provider.Immutable},
|
||||
{"charts/nginx-1.0.tgz", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_RewriteResponse(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
body := []byte("urls:\n- https://charts.example.com/chart-1.0.tgz")
|
||||
remote := models.Remote{Name: "helm-test", BaseURL: "https://charts.example.com"}
|
||||
rewritten, err := p.RewriteResponse(body, remote, "https://proxy.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rewritten == nil {
|
||||
t.Fatal("expected rewrite")
|
||||
}
|
||||
if !strings.Contains(string(rewritten), "proxy.example.com/api/v1/remote/helm-test") {
|
||||
t.Errorf("expected proxy URL in body: %s", rewritten)
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package npm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageNPM }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, ".tgz") {
|
||||
return provider.Immutable
|
||||
}
|
||||
return provider.Mutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tgz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" || !json.Valid(body) {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
rewritten := strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
if rewritten == content {
|
||||
return nil, nil
|
||||
}
|
||||
return []byte(rewritten), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package npm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestType(t *testing.T) {
|
||||
if (&Provider{}).Type() != models.PackageNPM {
|
||||
t.Fatal("wrong type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("pkg/-/pkg-1.0.0.tgz") != provider.Immutable {
|
||||
t.Error("tgz should be immutable")
|
||||
}
|
||||
if p.Classify("pkg") != provider.Mutable {
|
||||
t.Error("metadata should be mutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.ContentType("pkg/-/pkg-1.0.0.tgz") != "application/gzip" {
|
||||
t.Error("tgz content type")
|
||||
}
|
||||
if p.ContentType("pkg") != "application/json" {
|
||||
t.Error("metadata content type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
p := &Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://registry.npmjs.org/"}, "/pkg")
|
||||
if got != "https://registry.npmjs.org/pkg" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
p := &Provider{}
|
||||
remote := models.Remote{Name: "npmjs", BaseURL: "https://registry.npmjs.org"}
|
||||
|
||||
if out, _ := p.RewriteResponse([]byte(`{"a":1}`), remote, ""); out != nil {
|
||||
t.Error("empty proxyBaseURL should be a no-op")
|
||||
}
|
||||
if out, _ := p.RewriteResponse([]byte("not json"), remote, "http://proxy"); out != nil {
|
||||
t.Error("invalid json should be a no-op")
|
||||
}
|
||||
body := []byte(`{"tarball":"https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz"}`)
|
||||
out, err := p.RewriteResponse(body, remote, "http://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(out) != `{"tarball":"http://proxy/api/v1/remote/npmjs/pkg/-/pkg-1.0.0.tgz"}` {
|
||||
t.Errorf("rewrite: %s", out)
|
||||
}
|
||||
if out, _ := p.RewriteResponse([]byte(`{"x":"unrelated"}`), remote, "http://proxy"); out != nil {
|
||||
t.Error("no matching base URL should be a no-op")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthHeaders(t *testing.T) {
|
||||
p := &Provider{}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "pw"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("expected auth header when credentials set")
|
||||
}
|
||||
h, _ = p.AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("expected no auth header without credentials")
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type Mutability int
|
||||
|
||||
const (
|
||||
Immutable Mutability = iota
|
||||
Mutable
|
||||
)
|
||||
|
||||
type Provider interface {
|
||||
Type() models.PackageType
|
||||
Classify(path string) Mutability
|
||||
ContentType(path string) string
|
||||
UpstreamURL(remote models.Remote, path string) string
|
||||
RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error)
|
||||
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
|
||||
}
|
||||
|
||||
type FileEntry struct {
|
||||
FilePath string
|
||||
ContentHash string
|
||||
}
|
||||
|
||||
type FileStore interface {
|
||||
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
|
||||
ListPackages(ctx context.Context, repoName string) ([]string, error)
|
||||
}
|
||||
|
||||
type LocalUploader interface {
|
||||
ValidateUpload(filePath string) (storagePath, contentType string, err error)
|
||||
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
|
||||
}
|
||||
|
||||
type LocalIndexer interface {
|
||||
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
|
||||
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
|
||||
}
|
||||
|
||||
type BlobReader interface {
|
||||
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
||||
}
|
||||
|
||||
type PostUploadHook interface {
|
||||
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
|
||||
}
|
||||
|
||||
// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that
|
||||
// feeds generated repodata) after a local file is removed.
|
||||
type PostDeleteHook interface {
|
||||
AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error
|
||||
}
|
||||
|
||||
type MetadataStore interface {
|
||||
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
|
||||
}
|
||||
|
||||
type MetadataDeleter interface {
|
||||
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
|
||||
type RPMMetadataReader interface {
|
||||
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
|
||||
}
|
||||
|
||||
type RPMMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Epoch int
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
Summary string
|
||||
Description string
|
||||
RPMSize int64
|
||||
InstalledSize int64
|
||||
License string
|
||||
Vendor string
|
||||
Group string
|
||||
BuildHost string
|
||||
SourceRPM string
|
||||
URL string
|
||||
Packager string
|
||||
Requires []RPMDep
|
||||
Provides []RPMDep
|
||||
Files []RPMFile
|
||||
Changelogs []RPMChangelog
|
||||
}
|
||||
|
||||
type RPMDep struct {
|
||||
Name string `json:"name"`
|
||||
Flags string `json:"flags,omitempty"`
|
||||
Epoch string `json:"epoch,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Release string `json:"release,omitempty"`
|
||||
}
|
||||
|
||||
type RPMFile struct {
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
type RPMChangelog struct {
|
||||
Author string `json:"author"`
|
||||
Date int64 `json:"date"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
|
||||
type MemberIndex struct {
|
||||
RemoteName string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
var registry = map[models.PackageType]Provider{}
|
||||
|
||||
func Register(p Provider) {
|
||||
registry[p.Type()] = p
|
||||
}
|
||||
|
||||
func Get(t models.PackageType) (Provider, error) {
|
||||
p, ok := registry[t]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no provider registered for package type %q", t)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func All() map[models.PackageType]Provider {
|
||||
return registry
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package puppet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackagePuppet }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasPrefix(path, "v3/modules/") || strings.HasPrefix(path, "v3/releases") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.HasPrefix(path, "v3/") {
|
||||
return "application/json"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
content = strings.ReplaceAll(content, `"/v3/files/`, `"`+proxyURL+`/v3/files/`)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
content = strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
return []byte(content), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package puppet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestType(t *testing.T) {
|
||||
if (&Provider{}).Type() != models.PackagePuppet {
|
||||
t.Fatal("wrong type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("v3/modules/puppetlabs-stdlib") != provider.Mutable {
|
||||
t.Error("modules should be mutable")
|
||||
}
|
||||
if p.Classify("v3/releases?module=x") != provider.Mutable {
|
||||
t.Error("releases should be mutable")
|
||||
}
|
||||
if p.Classify("v3/files/puppetlabs-stdlib-1.0.0.tar.gz") != provider.Immutable {
|
||||
t.Error("files should be immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.ContentType("x/mod-1.0.0.tar.gz") != "application/gzip" {
|
||||
t.Error("tar.gz")
|
||||
}
|
||||
if p.ContentType("v3/modules/x") != "application/json" {
|
||||
t.Error("v3 json")
|
||||
}
|
||||
if p.ContentType("other") != "application/octet-stream" {
|
||||
t.Error("default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
got := (&Provider{}).UpstreamURL(models.Remote{BaseURL: "https://forgeapi.puppet.com/"}, "/v3/modules/x")
|
||||
if got != "https://forgeapi.puppet.com/v3/modules/x" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
p := &Provider{}
|
||||
remote := models.Remote{Name: "forge", BaseURL: "https://forgeapi.puppet.com"}
|
||||
|
||||
if out, _ := p.RewriteResponse([]byte("x"), remote, ""); out != nil {
|
||||
t.Error("empty proxyBaseURL is a no-op")
|
||||
}
|
||||
|
||||
body := []byte(`{"file_uri":"/v3/files/mod.tar.gz","home":"https://forgeapi.puppet.com/x"}`)
|
||||
out, err := p.RewriteResponse(body, remote, "http://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(out)
|
||||
if !strings.Contains(s, "http://proxy/api/v1/remote/forge/v3/files/mod.tar.gz") {
|
||||
t.Errorf("v3/files not rewritten: %s", s)
|
||||
}
|
||||
if !strings.Contains(s, "http://proxy/api/v1/remote/forge/x") {
|
||||
t.Errorf("base URL not rewritten: %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthHeaders(t *testing.T) {
|
||||
h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("no credentials, no header")
|
||||
}
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
package pypi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var fileRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*\.(whl|tar\.gz|zip)$`)
|
||||
var normalizeRe = regexp.MustCompile(`[-_.]+`)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackagePyPI }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.Contains(path, "simple/") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
lower := strings.ToLower(path)
|
||||
if strings.HasSuffix(lower, ".whl") || strings.HasSuffix(lower, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.Contains(path, "simple/") {
|
||||
return "text/html"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
if strings.HasPrefix(path, "simple/") {
|
||||
return "https://pypi.org/" + path
|
||||
}
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + "/"
|
||||
content = strings.ReplaceAll(content, "https://files.pythonhosted.org/", proxyURL)
|
||||
content = strings.ReplaceAll(content, "../../", proxyURL)
|
||||
return []byte(content), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func normalize(name string) string {
|
||||
return strings.ToLower(normalizeRe.ReplaceAllString(name, "-"))
|
||||
}
|
||||
|
||||
func packageFromWheel(filename string) string {
|
||||
parts := strings.SplitN(filename, "-", 3)
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return normalize(parts[0])
|
||||
}
|
||||
|
||||
func packageFromSdist(filename string) string {
|
||||
name := filename
|
||||
for _, suffix := range []string{".tar.gz", ".zip"} {
|
||||
if strings.HasSuffix(name, suffix) {
|
||||
name = strings.TrimSuffix(name, suffix)
|
||||
break
|
||||
}
|
||||
}
|
||||
idx := strings.LastIndex(name, "-")
|
||||
if idx <= 0 {
|
||||
return ""
|
||||
}
|
||||
return normalize(name[:idx])
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !fileRe.MatchString(filename) {
|
||||
return "", "", fmt.Errorf("filename %q must be a .whl, .tar.gz, or .zip file", filename)
|
||||
}
|
||||
|
||||
var pkgName string
|
||||
if strings.HasSuffix(filename, ".whl") {
|
||||
pkgName = packageFromWheel(filename)
|
||||
} else {
|
||||
pkgName = packageFromSdist(filename)
|
||||
}
|
||||
if pkgName == "" {
|
||||
return "", "", fmt.Errorf("cannot parse package name from %q", filename)
|
||||
}
|
||||
|
||||
ct := "application/zip"
|
||||
if strings.HasSuffix(filename, ".tar.gz") {
|
||||
ct = "application/gzip"
|
||||
}
|
||||
|
||||
return pkgName + "/" + filename, ct, nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
parts := strings.SplitN(storagePath, "/", 2)
|
||||
filename := storagePath
|
||||
if len(parts) == 2 {
|
||||
filename = parts[1]
|
||||
}
|
||||
return map[string]any{
|
||||
"package": parts[0],
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
if path == "simple" || path == "simple/" {
|
||||
p.servePackageList(w, r, files, repoName)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(path, "simple/") {
|
||||
pkg := strings.TrimPrefix(path, "simple/")
|
||||
pkg = strings.TrimSuffix(pkg, "/")
|
||||
if pkg != "" && !strings.Contains(pkg, "/") {
|
||||
p.servePackageFiles(w, r, files, repoName, pkg)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
if !strings.HasPrefix(path, "simple/") {
|
||||
return nil, fmt.Errorf("unsupported index path: %q", path)
|
||||
}
|
||||
|
||||
pkg := strings.TrimPrefix(path, "simple/")
|
||||
pkg = strings.TrimSuffix(pkg, "/")
|
||||
if pkg == "" {
|
||||
return p.generatePackageListHTML(ctx, files, repoName)
|
||||
}
|
||||
return p.generatePackageFilesHTML(ctx, files, repoName, pkg)
|
||||
}
|
||||
|
||||
func (p *Provider) servePackageList(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName string) {
|
||||
body, err := p.generatePackageListHTML(r.Context(), files, repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
func (p *Provider) servePackageFiles(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, packageName string) {
|
||||
normalized := normalize(packageName)
|
||||
prefix := normalized + "/"
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
|
||||
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n",
|
||||
normalized, filename, hash, filename)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.WriteString(w, b.String())
|
||||
}
|
||||
|
||||
func (p *Provider) generatePackageListHTML(ctx context.Context, files provider.FileStore, repoName string) ([]byte, error) {
|
||||
packages, err := files.ListPackages(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, pkg := range packages {
|
||||
fmt.Fprintf(&b, "<a href=\"%s/\">%s</a>\n", pkg, pkg)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
|
||||
func (p *Provider) generatePackageFilesHTML(ctx context.Context, files provider.FileStore, repoName, packageName string) ([]byte, error) {
|
||||
normalized := normalize(packageName)
|
||||
prefix := normalized + "/"
|
||||
entries, err := files.ListFilesByPrefix(ctx, repoName, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
|
||||
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&b, "<a href=\"%s/%s#sha256=%s\">%s</a>\n",
|
||||
normalized, filename, hash, filename)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
package pypi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeFileStore is an in-memory provider.FileStore for exercising local index
|
||||
// generation without a database.
|
||||
type fakeFileStore struct {
|
||||
packages []string
|
||||
files map[string][]provider.FileEntry
|
||||
}
|
||||
|
||||
func (f *fakeFileStore) ListPackages(_ context.Context, _ string) ([]string, error) {
|
||||
return f.packages, nil
|
||||
}
|
||||
|
||||
func (f *fakeFileStore) ListFilesByPrefix(_ context.Context, _, prefix string) ([]provider.FileEntry, error) {
|
||||
return f.files[prefix], nil
|
||||
}
|
||||
|
||||
func TestTypeClassifyContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Type() != models.PackagePyPI {
|
||||
t.Fatal("type")
|
||||
}
|
||||
if p.Classify("simple/foo/") != provider.Mutable {
|
||||
t.Error("simple index should be mutable")
|
||||
}
|
||||
if p.Classify("packages/foo-1.0.whl") != provider.Immutable {
|
||||
t.Error("wheel should be immutable")
|
||||
}
|
||||
cases := map[string]string{
|
||||
"foo-1.0-py3-none-any.whl": "application/zip",
|
||||
"foo-1.0.zip": "application/zip",
|
||||
"foo-1.0.tar.gz": "application/gzip",
|
||||
"simple/foo/": "text/html",
|
||||
"weird": "application/octet-stream",
|
||||
}
|
||||
for path, want := range cases {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://files.example.com"}, "packages/foo.whl"); got != "https://files.example.com/packages/foo.whl" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://x"}, "simple/foo/"); got != "https://pypi.org/simple/foo/" {
|
||||
t.Errorf("simple should hit pypi.org, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("numpy-1.26.0-cp311-cp311-linux_x86_64.whl")
|
||||
if err != nil || sp != "numpy/numpy-1.26.0-cp311-cp311-linux_x86_64.whl" || ct != "application/zip" {
|
||||
t.Errorf("wheel: sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
sp, ct, err = p.ValidateUpload("requests-2.31.0.tar.gz")
|
||||
if err != nil || sp != "requests/requests-2.31.0.tar.gz" || ct != "application/gzip" {
|
||||
t.Errorf("sdist: sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("not-a-package.txt"); err == nil {
|
||||
t.Error("expected error for bad extension")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackageNameParsing(t *testing.T) {
|
||||
if got := packageFromWheel("Foo_Bar-1.0-py3-none-any.whl"); got != "foo-bar" {
|
||||
t.Errorf("wheel name = %q", got)
|
||||
}
|
||||
if got := packageFromWheel("noseparator.whl"); got != "" {
|
||||
t.Errorf("expected empty for unparseable wheel, got %q", got)
|
||||
}
|
||||
if got := packageFromSdist("My.Pkg-2.0.tar.gz"); got != "my-pkg" {
|
||||
t.Errorf("sdist name = %q", got)
|
||||
}
|
||||
if got := packageFromSdist("noseparator.zip"); got != "" {
|
||||
t.Errorf("expected empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadResponse(t *testing.T) {
|
||||
resp := (&Provider{}).UploadResponse("foo/foo-1.0.whl", "sha256:abc", 123)
|
||||
if resp["filename"] != "foo-1.0.whl" || resp["package"] != "foo" || resp["content_hash"] != "sha256:abc" {
|
||||
t.Errorf("unexpected upload response: %v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if out, _ := p.RewriteResponse([]byte("x"), models.Remote{Name: "pypi"}, ""); out != nil {
|
||||
t.Error("empty proxyBaseURL is a no-op")
|
||||
}
|
||||
body := []byte(`<a href="https://files.pythonhosted.org/packages/foo.whl">foo.whl</a>`)
|
||||
out, err := p.RewriteResponse(body, models.Remote{Name: "pypi"}, "http://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(out), "http://proxy/api/v1/remote/pypi/") {
|
||||
t.Errorf("not rewritten: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateLocalIndex(t *testing.T) {
|
||||
p := &Provider{}
|
||||
fs := &fakeFileStore{
|
||||
packages: []string{"foo", "bar"},
|
||||
files: map[string][]provider.FileEntry{
|
||||
"foo/": {{FilePath: "foo/foo-1.0-py3-none-any.whl", ContentHash: "sha256:aaa"}},
|
||||
},
|
||||
}
|
||||
list, err := p.GenerateLocalIndex(context.Background(), fs, "local", "simple/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(list), "foo") || !strings.Contains(string(list), "bar") {
|
||||
t.Errorf("package list missing entries: %s", list)
|
||||
}
|
||||
|
||||
files, err := p.GenerateLocalIndex(context.Background(), fs, "local", "simple/foo/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(files), "foo-1.0-py3-none-any.whl") {
|
||||
t.Errorf("file list missing wheel: %s", files)
|
||||
}
|
||||
|
||||
if _, err := p.GenerateLocalIndex(context.Background(), fs, "local", "notsimple"); err == nil {
|
||||
t.Error("expected error for non-simple path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeLocalIndexHTTP(t *testing.T) {
|
||||
p := &Provider{}
|
||||
fs := &fakeFileStore{
|
||||
packages: []string{"foo"},
|
||||
files: map[string][]provider.FileEntry{
|
||||
"foo/": {{FilePath: "foo/foo-1.0-py3-none-any.whl", ContentHash: "sha256:aaa"}},
|
||||
},
|
||||
}
|
||||
serve := func(path string) (*httptest.ResponseRecorder, bool) {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
handled := p.ServeLocalIndex(w, r, fs, "local", path)
|
||||
return w, handled
|
||||
}
|
||||
|
||||
if w, ok := serve("simple/"); !ok || w.Code != 200 || !strings.Contains(w.Body.String(), "foo") {
|
||||
t.Errorf("simple index: handled=%v code=%d body=%s", ok, w.Code, w.Body.String())
|
||||
}
|
||||
if w, ok := serve("simple/foo/"); !ok || w.Code != 200 || !strings.Contains(w.Body.String(), "foo-1.0-py3-none-any.whl") {
|
||||
t.Errorf("package index: handled=%v code=%d body=%s", ok, w.Code, w.Body.String())
|
||||
}
|
||||
// Non-simple paths are not handled.
|
||||
if _, ok := serve("packages/foo.whl"); ok {
|
||||
t.Error("non-index path should not be handled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthHeaders(t *testing.T) {
|
||||
h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("expected auth header")
|
||||
}
|
||||
}
|
||||
@@ -1,453 +0,0 @@
|
||||
package rpm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
rpmlib "github.com/cavaliergopher/rpm"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var mutableRe = []*regexp.Regexp{
|
||||
regexp.MustCompile(`repomd\.xml$`),
|
||||
regexp.MustCompile(`repodata/`),
|
||||
regexp.MustCompile(`Packages\.gz$`),
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageRPM }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
for _, re := range mutableRe {
|
||||
if re.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".rpm") {
|
||||
return "application/x-rpm"
|
||||
}
|
||||
if strings.HasSuffix(path, ".xml") || strings.HasSuffix(path, ".xml.gz") || strings.HasSuffix(path, ".xml.xz") {
|
||||
return "application/xml"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(strings.ToLower(filename), ".rpm") {
|
||||
return "", "", fmt.Errorf("file must be an .rpm package")
|
||||
}
|
||||
|
||||
return "Packages/" + filename, "application/x-rpm", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
filename := strings.TrimPrefix(storagePath, "Packages/")
|
||||
return map[string]any{
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
|
||||
|
||||
reader, blobSize, err := blobs.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
slog.Error("rpm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
pkg, err := rpmlib.Read(reader)
|
||||
if err != nil {
|
||||
slog.Error("rpm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
meta := &provider.RPMMetadata{
|
||||
RepoName: repoName,
|
||||
FilePath: storagePath,
|
||||
ContentHash: contentHash,
|
||||
Name: pkg.Name(),
|
||||
Epoch: pkg.Epoch(),
|
||||
Version: pkg.Version(),
|
||||
Release: pkg.Release(),
|
||||
Arch: pkg.Architecture(),
|
||||
Summary: pkg.Summary(),
|
||||
Description: pkg.Description(),
|
||||
RPMSize: blobSize,
|
||||
InstalledSize: int64(pkg.Size()),
|
||||
License: pkg.License(),
|
||||
Vendor: pkg.Vendor(),
|
||||
Group: firstGroup(pkg.Groups()),
|
||||
BuildHost: pkg.BuildHost(),
|
||||
SourceRPM: pkg.SourceRPM(),
|
||||
URL: pkg.URL(),
|
||||
Packager: pkg.Packager(),
|
||||
}
|
||||
|
||||
for _, req := range pkg.Requires() {
|
||||
meta.Requires = append(meta.Requires, rpmDepFromEntry(req))
|
||||
}
|
||||
for _, prov := range pkg.Provides() {
|
||||
meta.Provides = append(meta.Provides, rpmDepFromEntry(prov))
|
||||
}
|
||||
|
||||
if meta.Requires == nil {
|
||||
meta.Requires = []provider.RPMDep{}
|
||||
}
|
||||
if meta.Provides == nil {
|
||||
meta.Provides = []provider.RPMDep{}
|
||||
}
|
||||
meta.Files = []provider.RPMFile{}
|
||||
meta.Changelogs = []provider.RPMChangelog{}
|
||||
|
||||
if err := db.InsertRPMMetadata(ctx, meta); err != nil {
|
||||
slog.Error("rpm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("rpm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
}
|
||||
|
||||
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
|
||||
if err := db.DeleteRPMMetadata(ctx, repoName, storagePath); err != nil {
|
||||
slog.Error("rpm metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return err
|
||||
}
|
||||
slog.Info("rpm metadata: deleted", "repo", repoName, "path", storagePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func rpmDepFromEntry(e rpmlib.Dependency) provider.RPMDep {
|
||||
dep := provider.RPMDep{Name: e.Name()}
|
||||
if e.Flags() != 0 {
|
||||
dep.Flags = rpmFlagString(e.Flags())
|
||||
dep.Version = e.Version()
|
||||
dep.Release = e.Release()
|
||||
if e.Epoch() > 0 {
|
||||
dep.Epoch = fmt.Sprintf("%d", e.Epoch())
|
||||
}
|
||||
}
|
||||
return dep
|
||||
}
|
||||
|
||||
func rpmFlagString(f int) string {
|
||||
switch {
|
||||
case f&0x08 != 0 && f&0x04 != 0:
|
||||
return "GE"
|
||||
case f&0x02 != 0 && f&0x04 != 0:
|
||||
return "LE"
|
||||
case f&0x08 != 0:
|
||||
return "GT"
|
||||
case f&0x02 != 0:
|
||||
return "LT"
|
||||
case f&0x04 != 0:
|
||||
return "EQ"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func firstGroup(groups []string) string {
|
||||
if len(groups) > 0 {
|
||||
return groups[0]
|
||||
}
|
||||
return "Unspecified"
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
if !strings.HasPrefix(path, "repodata/") {
|
||||
return false
|
||||
}
|
||||
|
||||
rpmReader, ok := files.(provider.RPMMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "rpm metadata not available", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
tail := strings.TrimPrefix(path, "repodata/")
|
||||
|
||||
switch {
|
||||
case tail == "repomd.xml":
|
||||
p.serveRepomd(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-primary.xml.gz"):
|
||||
p.servePrimary(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-filelists.xml.gz"):
|
||||
p.serveFilelists(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-other.xml.gz"):
|
||||
p.serveOther(w, r, rpmReader, repoName)
|
||||
default:
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("rpm local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
primary := generatePrimaryXMLGZ(metas)
|
||||
filelists := generateFilelistsXMLGZ(metas)
|
||||
other := generateOtherXMLGZ(metas)
|
||||
|
||||
primaryHash := sha256Hex(primary)
|
||||
filelistsHash := sha256Hex(filelists)
|
||||
otherHash := sha256Hex(other)
|
||||
|
||||
repomd := generateRepomd(primaryHash, len(primary), filelistsHash, len(filelists), otherHash, len(other))
|
||||
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(repomd)
|
||||
}
|
||||
|
||||
func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generatePrimaryXMLGZ(metas))
|
||||
}
|
||||
|
||||
func (p *Provider) serveFilelists(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateFilelistsXMLGZ(metas))
|
||||
}
|
||||
|
||||
func (p *Provider) serveOther(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateOtherXMLGZ(metas))
|
||||
}
|
||||
|
||||
func generateRepomd(primaryHash string, primarySize int, filelistsHash string, filelistsSize int, otherHash string, otherSize int) []byte {
|
||||
ts := fmt.Sprintf("%d", time.Now().Unix())
|
||||
var b bytes.Buffer
|
||||
b.WriteString(xml.Header)
|
||||
b.WriteString(`<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">` + "\n")
|
||||
fmt.Fprintf(&b, " <revision>%s</revision>\n", ts)
|
||||
|
||||
writeRepomdData(&b, "primary", primaryHash, primarySize, ts)
|
||||
writeRepomdData(&b, "filelists", filelistsHash, filelistsSize, ts)
|
||||
writeRepomdData(&b, "other", otherHash, otherSize, ts)
|
||||
|
||||
b.WriteString("</repomd>\n")
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func writeRepomdData(b *bytes.Buffer, dtype, hash string, size int, ts string) {
|
||||
fmt.Fprintf(b, " <data type=\"%s\">\n", dtype)
|
||||
fmt.Fprintf(b, " <checksum type=\"sha256\">%s</checksum>\n", hash)
|
||||
fmt.Fprintf(b, " <location href=\"repodata/%s-%s.xml.gz\"/>\n", hash, dtype)
|
||||
fmt.Fprintf(b, " <timestamp>%s</timestamp>\n", ts)
|
||||
fmt.Fprintf(b, " <size>%d</size>\n", size)
|
||||
fmt.Fprintf(b, " </data>\n")
|
||||
}
|
||||
|
||||
func generatePrimaryXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package type=\"rpm\">\n")
|
||||
fmt.Fprintf(&xmlBuf, " <name>%s</name>\n", xmlEscape(m.Name))
|
||||
fmt.Fprintf(&xmlBuf, " <arch>%s</arch>\n", xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
fmt.Fprintf(&xmlBuf, " <checksum type=\"sha256\" pkgid=\"YES\">%s</checksum>\n", pkgHash)
|
||||
fmt.Fprintf(&xmlBuf, " <summary>%s</summary>\n", xmlEscape(m.Summary))
|
||||
fmt.Fprintf(&xmlBuf, " <description>%s</description>\n", xmlEscape(m.Description))
|
||||
if m.Packager != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <packager>%s</packager>\n", xmlEscape(m.Packager))
|
||||
}
|
||||
if m.URL != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <url>%s</url>\n", xmlEscape(m.URL))
|
||||
}
|
||||
fmt.Fprintf(&xmlBuf, " <time file=\"%d\" build=\"0\"/>\n", time.Now().Unix())
|
||||
fmt.Fprintf(&xmlBuf, " <size package=\"%d\" installed=\"%d\" archive=\"0\"/>\n", m.RPMSize, m.InstalledSize)
|
||||
fmt.Fprintf(&xmlBuf, " <location href=\"%s\"/>\n", xmlEscape(m.FilePath))
|
||||
fmt.Fprintf(&xmlBuf, " <format>\n")
|
||||
if m.License != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:license>%s</rpm:license>\n", xmlEscape(m.License))
|
||||
}
|
||||
if m.Vendor != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:vendor>%s</rpm:vendor>\n", xmlEscape(m.Vendor))
|
||||
}
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:group>%s</rpm:group>\n", xmlEscape(m.Group))
|
||||
if m.BuildHost != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:buildhost>%s</rpm:buildhost>\n", xmlEscape(m.BuildHost))
|
||||
}
|
||||
if m.SourceRPM != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:sourcerpm>%s</rpm:sourcerpm>\n", xmlEscape(m.SourceRPM))
|
||||
}
|
||||
|
||||
if len(m.Provides) > 0 {
|
||||
xmlBuf.WriteString(" <rpm:provides>\n")
|
||||
for _, d := range m.Provides {
|
||||
writeRPMEntry(&xmlBuf, d)
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:provides>\n")
|
||||
}
|
||||
if len(m.Requires) > 0 {
|
||||
xmlBuf.WriteString(" <rpm:requires>\n")
|
||||
for _, d := range m.Requires {
|
||||
writeRPMEntry(&xmlBuf, d)
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:requires>\n")
|
||||
}
|
||||
|
||||
fmt.Fprintf(&xmlBuf, " </format>\n")
|
||||
fmt.Fprintf(&xmlBuf, "</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</metadata>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func generateFilelistsXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<filelists xmlns=\"http://linux.duke.edu/metadata/filelists\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
for _, f := range m.Files {
|
||||
if f.Type != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <file type=\"%s\">%s</file>\n", f.Type, xmlEscape(f.Path))
|
||||
} else {
|
||||
fmt.Fprintf(&xmlBuf, " <file>%s</file>\n", xmlEscape(f.Path))
|
||||
}
|
||||
}
|
||||
xmlBuf.WriteString("</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</filelists>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func generateOtherXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<otherdata xmlns=\"http://linux.duke.edu/metadata/other\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
for _, cl := range m.Changelogs {
|
||||
fmt.Fprintf(&xmlBuf, " <changelog author=\"%s\" date=\"%d\">%s</changelog>\n",
|
||||
xmlEscape(cl.Author), cl.Date, xmlEscape(cl.Text))
|
||||
}
|
||||
xmlBuf.WriteString("</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</otherdata>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func writeRPMEntry(b *bytes.Buffer, d provider.RPMDep) {
|
||||
if d.Flags != "" {
|
||||
fmt.Fprintf(b, " <rpm:entry name=\"%s\" flags=\"%s\"", xmlEscape(d.Name), d.Flags)
|
||||
if d.Epoch != "" {
|
||||
fmt.Fprintf(b, " epoch=\"%s\"", d.Epoch)
|
||||
}
|
||||
if d.Version != "" {
|
||||
fmt.Fprintf(b, " ver=\"%s\"", xmlEscape(d.Version))
|
||||
}
|
||||
if d.Release != "" {
|
||||
fmt.Fprintf(b, " rel=\"%s\"", xmlEscape(d.Release))
|
||||
}
|
||||
b.WriteString("/>\n")
|
||||
} else {
|
||||
fmt.Fprintf(b, " <rpm:entry name=\"%s\"/>\n", xmlEscape(d.Name))
|
||||
}
|
||||
}
|
||||
|
||||
func xmlEscape(s string) string {
|
||||
var b bytes.Buffer
|
||||
xml.EscapeText(&b, []byte(s))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func gzipBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
gz.Write(data)
|
||||
gz.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func sha256Hex(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
package rpm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type fakeBlobReader struct{ data []byte }
|
||||
|
||||
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
|
||||
}
|
||||
|
||||
type fakeMetaStore struct{ inserted *provider.RPMMetadata }
|
||||
|
||||
func (f *fakeMetaStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
|
||||
f.inserted = m
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeRPMReader struct{ metas []provider.RPMMetadata }
|
||||
|
||||
func (f fakeRPMReader) ListRPMMetadataEntries(_ context.Context, _ string) ([]provider.RPMMetadata, error) {
|
||||
return f.metas, nil
|
||||
}
|
||||
func (f fakeRPMReader) ListFilesByPrefix(_ context.Context, _, _ string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f fakeRPMReader) ListPackages(_ context.Context, _ string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestRPMPureFuncs(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Type() != models.PackageRPM {
|
||||
t.Error("type")
|
||||
}
|
||||
if p.Classify("repodata/repomd.xml") != provider.Mutable {
|
||||
t.Error("repomd should be mutable")
|
||||
}
|
||||
if p.Classify("Packages/foo.rpm") != provider.Immutable {
|
||||
t.Error("rpm should be immutable")
|
||||
}
|
||||
if p.ContentType("x.rpm") != "application/x-rpm" {
|
||||
t.Error("rpm content type")
|
||||
}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://mirror/"}, "/Packages/x.rpm"); got != "https://mirror/Packages/x.rpm" {
|
||||
t.Errorf("upstream url %q", got)
|
||||
}
|
||||
if out, _ := p.RewriteResponse(nil, models.Remote{}, "http://p"); out != nil {
|
||||
t.Error("rpm never rewrites")
|
||||
}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("auth header")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("dir/foo-1.0.noarch.rpm")
|
||||
if err != nil || sp != "Packages/foo-1.0.noarch.rpm" || ct != "application/x-rpm" {
|
||||
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("foo.txt"); err == nil {
|
||||
t.Error("expected error for non-rpm")
|
||||
}
|
||||
resp := p.UploadResponse("Packages/foo.rpm", "sha256:abc", 10)
|
||||
if resp["content_hash"] != "sha256:abc" {
|
||||
t.Errorf("upload response %v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMAfterUpload(t *testing.T) {
|
||||
data := testsupport.MinimalRPM("e2e-testpkg", "1.0", "1", "noarch")
|
||||
store := &fakeMetaStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "myrepo", "Packages/e2e-testpkg-1.0-1.noarch.rpm",
|
||||
"sha256:deadbeef", fakeBlobReader{data: data}, store)
|
||||
|
||||
m := store.inserted
|
||||
if m == nil {
|
||||
t.Fatal("no metadata inserted")
|
||||
}
|
||||
if m.Name != "e2e-testpkg" || m.Version != "1.0" || m.Release != "1" || m.Arch != "noarch" {
|
||||
t.Errorf("unexpected metadata: %+v", m)
|
||||
}
|
||||
if m.RPMSize != int64(len(data)) {
|
||||
t.Errorf("RPMSize = %d, want %d", m.RPMSize, len(data))
|
||||
}
|
||||
if len(m.Provides) == 0 {
|
||||
t.Error("expected the package to provide itself")
|
||||
}
|
||||
}
|
||||
|
||||
type errBlobReader struct{}
|
||||
|
||||
func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return nil, 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
func TestRPMAfterUploadErrors(t *testing.T) {
|
||||
// Download failure: no metadata inserted, no panic.
|
||||
store := &fakeMetaStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", errBlobReader{}, store)
|
||||
if store.inserted != nil {
|
||||
t.Error("no metadata should be inserted on download error")
|
||||
}
|
||||
// Parse failure: garbage bytes are not a valid RPM.
|
||||
store2 := &fakeMetaStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", fakeBlobReader{data: []byte("not an rpm")}, store2)
|
||||
if store2.inserted != nil {
|
||||
t.Error("no metadata should be inserted on parse error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMServeRepodata(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeRPMReader{metas: []provider.RPMMetadata{{
|
||||
Name: "e2e-testpkg", Version: "1.0", Release: "1", Arch: "noarch",
|
||||
Summary: "test & <special>",
|
||||
ContentHash: "sha256:abc",
|
||||
Requires: []provider.RPMDep{{Name: "libc", Flags: "GE", Version: "2.0"}},
|
||||
Provides: []provider.RPMDep{{Name: "e2e-testpkg"}},
|
||||
Files: []provider.RPMFile{{Path: "/usr/share/e2e/README", Type: "file"}},
|
||||
Changelogs: []provider.RPMChangelog{{Author: "e2e", Date: 1, Text: "init"}},
|
||||
}}}
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if !p.ServeLocalIndex(w, r, reader, "myrepo", path) {
|
||||
t.Fatalf("ServeLocalIndex returned false for %q", path)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
if w := serve("repodata/repomd.xml"); w.Code != 200 || !strings.Contains(w.Body.String(), "<repomd") {
|
||||
t.Errorf("repomd: code=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
for _, name := range []string{"repodata/h-primary.xml.gz", "repodata/h-filelists.xml.gz", "repodata/h-other.xml.gz"} {
|
||||
w := serve(name)
|
||||
if w.Code != 200 {
|
||||
t.Errorf("%s: code %d", name, w.Code)
|
||||
}
|
||||
if _, err := gzip.NewReader(bytes.NewReader(w.Body.Bytes())); err != nil {
|
||||
t.Errorf("%s: not gzip: %v", name, err)
|
||||
}
|
||||
}
|
||||
// Unknown repodata file -> 404.
|
||||
if w := serve("repodata/bogus"); w.Code != http.StatusNotFound {
|
||||
t.Errorf("bogus repodata: code %d", w.Code)
|
||||
}
|
||||
// Non-repodata path -> not handled.
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/Packages/x.rpm", nil)
|
||||
if p.ServeLocalIndex(w, r, reader, "myrepo", "Packages/x.rpm") {
|
||||
t.Error("expected ServeLocalIndex false for non-repodata path")
|
||||
}
|
||||
}
|
||||
|
||||
type errRPMReader struct{}
|
||||
|
||||
func (errRPMReader) ListRPMMetadataEntries(context.Context, string) ([]provider.RPMMetadata, error) {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
func (errRPMReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (errRPMReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestRPMServeMetadataError(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for _, path := range []string{"repodata/repomd.xml", "repodata/h-primary.xml.gz", "repodata/h-filelists.xml.gz", "repodata/h-other.xml.gz"} {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
p.ServeLocalIndex(w, r, errRPMReader{}, "repo", path)
|
||||
if w.Code != 500 {
|
||||
t.Errorf("%s with failing reader = %d, want 500", path, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMFullMetadataXML(t *testing.T) {
|
||||
// A fully-populated entry exercises every optional-field branch in the
|
||||
// primary/filelists/other XML generators.
|
||||
metas := []provider.RPMMetadata{{
|
||||
Name: "full", Epoch: 1, Version: "2.0", Release: "3", Arch: "x86_64",
|
||||
Summary: "s", Description: "d", License: "MIT", Vendor: "acme",
|
||||
Group: "System", BuildHost: "build.example.com", SourceRPM: "full-2.0.src.rpm",
|
||||
URL: "https://example.com", Packager: "pkgr", ContentHash: "sha256:abc",
|
||||
RPMSize: 100, InstalledSize: 200,
|
||||
Requires: []provider.RPMDep{{Name: "libc", Flags: "GE", Epoch: "0", Version: "2.0", Release: "1"}},
|
||||
Provides: []provider.RPMDep{{Name: "full", Flags: "EQ", Version: "2.0"}},
|
||||
Files: []provider.RPMFile{{Path: "/usr/bin/full", Type: "file"}, {Path: "/etc/full", Type: "dir"}},
|
||||
Changelogs: []provider.RPMChangelog{{Author: "a", Date: 100, Text: "changed"}},
|
||||
}}
|
||||
for _, gen := range []func([]provider.RPMMetadata) []byte{generatePrimaryXMLGZ, generateFilelistsXMLGZ, generateOtherXMLGZ} {
|
||||
zr, err := gzip.NewReader(bytes.NewReader(gen(metas)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := io.ReadAll(zr); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMPrimaryXMLContents(t *testing.T) {
|
||||
// Exercise xmlEscape and dependency entry writing through the gzip'd XML.
|
||||
metas := []provider.RPMMetadata{{
|
||||
Name: "pkg", Version: "1", Release: "1", Arch: "x86_64", Summary: "a & b",
|
||||
Requires: []provider.RPMDep{{Name: "dep", Flags: "EQ", Version: "1.0", Epoch: "0"}},
|
||||
}}
|
||||
gz := generatePrimaryXMLGZ(metas)
|
||||
zr, err := gzip.NewReader(bytes.NewReader(gz))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, _ := io.ReadAll(zr)
|
||||
s := string(out)
|
||||
if !strings.Contains(s, "a & b") {
|
||||
t.Errorf("summary not xml-escaped: %s", s)
|
||||
}
|
||||
if !strings.Contains(s, "<name>pkg</name>") {
|
||||
t.Errorf("package name missing: %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMContentTypeAndHelpers(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for path, want := range map[string]string{
|
||||
"x.rpm": "application/x-rpm",
|
||||
"repodata/repomd.xml": "application/xml",
|
||||
"repodata/h-primary.xml.gz": "application/xml",
|
||||
"repodata/h-primary.xml.xz": "application/xml",
|
||||
"Packages/other": "application/octet-stream",
|
||||
} {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
for flag, want := range map[int]string{
|
||||
0x08 | 0x04: "GE",
|
||||
0x02 | 0x04: "LE",
|
||||
0x08: "GT",
|
||||
0x02: "LT",
|
||||
0x04: "EQ",
|
||||
0x00: "",
|
||||
} {
|
||||
if got := rpmFlagString(flag); got != want {
|
||||
t.Errorf("rpmFlagString(%d)=%q want %q", flag, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
if firstGroup(nil) != "Unspecified" {
|
||||
t.Error("empty groups should be Unspecified")
|
||||
}
|
||||
if firstGroup([]string{"System", "Base"}) != "System" {
|
||||
t.Error("firstGroup should return the first")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateLocalIndexUnsupported(t *testing.T) {
|
||||
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeRPMReader{}, "r", "simple/"); err == nil {
|
||||
t.Error("expected unsupported error")
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package rpm_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &rpm.Provider{}
|
||||
if p.Type() != models.PackageRPM {
|
||||
t.Errorf("expected rpm, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &rpm.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"repomd.xml", provider.Mutable},
|
||||
{"repodata/primary.xml.gz", provider.Mutable},
|
||||
{"Packages.gz", provider.Mutable},
|
||||
{"package-1.0.rpm", provider.Immutable},
|
||||
{"RPM-GPG-KEY-almalinux", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
package terraform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var versionsRe = regexp.MustCompile(`[^/]+/[^/]+/versions$`)
|
||||
|
||||
var providerZipRe = regexp.MustCompile(
|
||||
`^terraform-provider-([a-zA-Z0-9_-]+)_([0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`,
|
||||
)
|
||||
|
||||
var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageTerraform }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if versionsRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
lower := strings.ToLower(path)
|
||||
if strings.HasSuffix(lower, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".sig") {
|
||||
return "application/octet-stream"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/v1/providers/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if remote.ReleasesRemote == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if !json.Valid(body) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var data map[string]any
|
||||
if err := json.Unmarshal(body, &data); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
changed := false
|
||||
for _, field := range []string{"download_url", "shasums_url", "shasums_signature_url"} {
|
||||
if val, ok := data[field].(string); ok && val != "" {
|
||||
rewritten := rewriteDownloadURL(val, remote.ReleasesRemote, proxyBaseURL)
|
||||
if rewritten != val {
|
||||
data[field] = rewritten
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return nil, nil
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
|
||||
func rewriteDownloadURL(originalURL, releasesRemote, proxyBaseURL string) string {
|
||||
parsed, err := url.Parse(originalURL)
|
||||
if err != nil || proxyBaseURL == "" {
|
||||
return originalURL
|
||||
}
|
||||
return strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + releasesRemote + parsed.Path
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
parts := strings.Split(filePath, "/")
|
||||
if len(parts) != 3 {
|
||||
return "", "", fmt.Errorf("path must be {namespace}/{type}/{filename}.zip")
|
||||
}
|
||||
namespace, typeName, filename := parts[0], parts[1], parts[2]
|
||||
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil {
|
||||
return "", "", fmt.Errorf("filename %q does not match terraform-provider-{type}_{version}_{os}_{arch}.zip", filename)
|
||||
}
|
||||
|
||||
if m[1] != typeName {
|
||||
return "", "", fmt.Errorf("provider type in filename %q does not match path type %q", m[1], typeName)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s/%s/%s", namespace, typeName, filename), "application/zip", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
parts := strings.Split(storagePath, "/")
|
||||
if len(parts) != 3 {
|
||||
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
|
||||
}
|
||||
|
||||
m := providerZipRe.FindStringSubmatch(parts[2])
|
||||
if m == nil {
|
||||
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"namespace": parts[0],
|
||||
"type": parts[1],
|
||||
"version": m[2],
|
||||
"os": m[3],
|
||||
"arch": m[4],
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
type terraformIndex struct {
|
||||
Versions map[string]json.RawMessage `json:"versions"`
|
||||
}
|
||||
|
||||
type terraformVersionDoc struct {
|
||||
Archives map[string]terraformArchive `json:"archives"`
|
||||
}
|
||||
|
||||
type terraformArchive struct {
|
||||
URL string `json:"url"`
|
||||
Hashes []string `json:"hashes,omitempty"`
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) < 3 {
|
||||
return false
|
||||
}
|
||||
|
||||
namespace, typeName := parts[0], parts[1]
|
||||
tail := parts[2]
|
||||
|
||||
if tail == "index.json" {
|
||||
p.serveIndex(w, r, files, repoName, namespace, typeName)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasSuffix(tail, ".json") {
|
||||
version := strings.TrimSuffix(tail, ".json")
|
||||
if semverRe.MatchString(version) {
|
||||
p.serveVersionDoc(w, r, files, repoName, namespace, typeName, version)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("terraform local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
func (p *Provider) serveIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName string) {
|
||||
prefix := fmt.Sprintf("%s/%s/", namespace, typeName)
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
versions := map[string]json.RawMessage{}
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, prefix)
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
versions[m[2]] = json.RawMessage(`{}`)
|
||||
}
|
||||
|
||||
if len(versions) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(terraformIndex{Versions: versions})
|
||||
}
|
||||
|
||||
func (p *Provider) serveVersionDoc(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName, version string) {
|
||||
prefix := fmt.Sprintf("%s/%s/terraform-provider-%s_%s_", namespace, typeName, typeName, version)
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
archives := map[string]terraformArchive{}
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, fmt.Sprintf("%s/%s/", namespace, typeName))
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil || m[2] != version {
|
||||
continue
|
||||
}
|
||||
platform := m[3] + "_" + m[4]
|
||||
archive := terraformArchive{URL: filename}
|
||||
if f.ContentHash != "" {
|
||||
archive.Hashes = []string{"zh:" + strings.TrimPrefix(f.ContentHash, "sha256:")}
|
||||
}
|
||||
archives[platform] = archive
|
||||
}
|
||||
|
||||
if len(archives) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(terraformVersionDoc{Archives: archives})
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package terraform_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
if p.Type() != models.PackageTerraform {
|
||||
t.Errorf("expected terraform, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"hashicorp/vault/versions", provider.Mutable},
|
||||
{"hashicorp/vault/0.28.0/download/linux/amd64", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_RewriteResponse_DownloadInfo(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
remote := models.Remote{Name: "tf", ReleasesRemote: "hashicorp-releases"}
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"download_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/file.zip",
|
||||
"shasums_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/SHA256SUMS",
|
||||
})
|
||||
rewritten, err := p.RewriteResponse(body, remote, "https://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rewritten == nil {
|
||||
t.Fatal("expected rewrite")
|
||||
}
|
||||
var result map[string]any
|
||||
json.Unmarshal(rewritten, &result)
|
||||
if !strings.Contains(result["download_url"].(string), "proxy/api/v1/remote/hashicorp-releases") {
|
||||
t.Errorf("download_url not rewritten: %s", result["download_url"])
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultCircuitThreshold = 5
|
||||
defaultCircuitCooldown = 60 * time.Second
|
||||
)
|
||||
|
||||
type CircuitBreaker struct {
|
||||
cache *cache.Redis
|
||||
threshold int64
|
||||
cooldown time.Duration
|
||||
}
|
||||
|
||||
func NewCircuitBreaker(c *cache.Redis) *CircuitBreaker {
|
||||
return &CircuitBreaker{
|
||||
cache: c,
|
||||
threshold: defaultCircuitThreshold,
|
||||
cooldown: defaultCircuitCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) IsOpen(ctx context.Context, remote string) bool {
|
||||
failures, err := cb.cache.GetCircuitFailures(ctx, remote)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return failures >= cb.threshold
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) RecordFailure(ctx context.Context, remote string) {
|
||||
cb.cache.IncrCircuitFailure(ctx, remote, cb.cooldown)
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) RecordSuccess(ctx context.Context, remote string) {
|
||||
cb.cache.ResetCircuit(ctx, remote)
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) Health(ctx context.Context, remote string) models.RemoteHealth {
|
||||
failures, err := cb.cache.GetCircuitFailures(ctx, remote)
|
||||
if err != nil {
|
||||
return models.RemoteHealth{Status: "unknown"}
|
||||
}
|
||||
|
||||
switch {
|
||||
case failures == 0:
|
||||
return models.RemoteHealth{Status: "healthy", ConsecutiveFailures: int(failures)}
|
||||
case failures < cb.threshold:
|
||||
return models.RemoteHealth{Status: "degraded", ConsecutiveFailures: int(failures)}
|
||||
default:
|
||||
return models.RemoteHealth{Status: "down", ConsecutiveFailures: int(failures)}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user