Compare commits
64 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 221f3a7402 | |||
| 1b585af14e | |||
| e7c9387bcc | |||
| 7e07eaa758 | |||
| f61ab99ae8 | |||
| c39703ed0d | |||
| 5261af4c63 | |||
| 45d6cdbc64 | |||
| b59cc45765 | |||
| e7027c8ccc | |||
| f3680951b7 | |||
| 61a1a99112 | |||
| f0e44d6810 | |||
| 0a89b2005c | |||
| f23bf2a6d9 | |||
| b9098bf19c | |||
| 8d9bc1c422 | |||
| 30b7cef026 | |||
| 603be5b989 | |||
| 9eba49500c | |||
| 0083d67272 | |||
| 8ec7de50e3 | |||
| 9c465cbd4c | |||
| ee6e581b9d | |||
| 2a8e544de3 | |||
| 847eeb839f | |||
| 74d9c0fa84 | |||
| 097fbf0016 | |||
| 6f8e70c27a | |||
| 3a6721c2a7 | |||
| 7b13644421 | |||
| de96637122 | |||
| 1e91a5fb72 | |||
| a481a5c3b7 | |||
| b46c116f6b | |||
| f25bf6cb29 | |||
| 99cc71f56c | |||
| 9287cf7cf2 | |||
| ff2aefeef4 | |||
| a115904bbc | |||
| 8a7f26b193 | |||
| 15f934cd0b | |||
| 7b6c69b70f | |||
| 624d858062 | |||
| 1656664dfa | |||
| c7baae8d0d | |||
| 4789635e87 | |||
| ba52fedd27 | |||
| 76633403b2 | |||
| cae3503ac4 | |||
| 3f098df428 | |||
| 64266f40e9 | |||
| be25fc19f7 | |||
| 3bd3ca8b74 | |||
| 373366e695 | |||
| e6d9b175ce | |||
| 0daca40156 | |||
| 0df726467a | |||
| b8bc7f8714 | |||
| 0c780c1bd1 | |||
| 173b5d8b10 | |||
| 3352a3e886 | |||
| 8adcbac405 | |||
| 4ca89b9159 |
@@ -1,15 +0,0 @@
|
||||
.git/
|
||||
.venv/
|
||||
dist/
|
||||
tests/
|
||||
remotes.yaml
|
||||
ca-bundle.pem
|
||||
.env
|
||||
*.log
|
||||
docker-compose.yml
|
||||
.woodpecker/
|
||||
.tox/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
.pre-commit-cache/
|
||||
minio_data/
|
||||
+5
-60
@@ -1,61 +1,6 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
bin/
|
||||
terraform/
|
||||
|
||||
# Virtual environment
|
||||
.venv/
|
||||
venv/
|
||||
ENV/
|
||||
env/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# uv
|
||||
uv.lock
|
||||
|
||||
# tox
|
||||
.tox/
|
||||
|
||||
# pytest
|
||||
.pytest_cache/
|
||||
|
||||
# pre-commit
|
||||
.pre-commit-cache/
|
||||
|
||||
# ruff
|
||||
.ruff_cache/
|
||||
|
||||
# Docker volumes
|
||||
minio_data/
|
||||
|
||||
# Local configuration overrides
|
||||
ca-bundle.pem
|
||||
# e2e-docker fixtures are real package files (.rpm, .tgz, .whl, .zip, ...) that
|
||||
# are intentionally tracked, overriding any global ignore of those extensions.
|
||||
!e2e-docker/fixtures/**
|
||||
|
||||
+22
-5
@@ -1,7 +1,24 @@
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.12
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v5.0.0
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: [--fix, --exit-non-zero-on-fix]
|
||||
- id: ruff-format
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
- id: check-added-large-files
|
||||
- id: check-merge-conflict
|
||||
|
||||
- repo: https://github.com/dnephin/pre-commit-golang
|
||||
rev: v0.5.1
|
||||
hooks:
|
||||
- id: go-fmt
|
||||
- id: go-mod-tidy
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: go-vet
|
||||
name: go vet
|
||||
entry: go vet ./...
|
||||
language: system
|
||||
types: [go]
|
||||
pass_filenames: false
|
||||
|
||||
+18
-2
@@ -3,16 +3,32 @@ when:
|
||||
ref: refs/tags/v*
|
||||
|
||||
steps:
|
||||
- name: docker
|
||||
- name: docker-api
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi
|
||||
build_args:
|
||||
VERSION: ${CI_COMMIT_TAG}
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
|
||||
- name: docker-web
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi-ui
|
||||
dockerfile: ui/Dockerfile.ui
|
||||
context: ui
|
||||
build_args:
|
||||
- VERSION=${CI_COMMIT_TAG##v}
|
||||
BASE_PATH: /ui
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
|
||||
@@ -3,7 +3,15 @@ when:
|
||||
|
||||
steps:
|
||||
- name: pre-commit
|
||||
image: git.unkin.net/unkin/almalinux9-base:20260308
|
||||
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
||||
commands:
|
||||
- uvx pre-commit run --all-files
|
||||
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
|
||||
@@ -3,6 +3,6 @@ when:
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
image: git.unkin.net/unkin/almalinux9-base:20260308
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- uvx --python 3.11 --with tox-uv tox
|
||||
- go test -race -count=1 ./pkg/... ./internal/...
|
||||
|
||||
+14
-16
@@ -1,23 +1,21 @@
|
||||
FROM git.unkin.net/unkin/almalinux9-base:latest
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
ARG VERSION=0.0.0.dev0
|
||||
RUN apk add --no-cache git
|
||||
|
||||
COPY . /build
|
||||
WORKDIR /build
|
||||
|
||||
RUN HATCH_VCS_PRETEND_VERSION=${VERSION} \
|
||||
SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} \
|
||||
uv build --wheel --directory /build && \
|
||||
useradd -m -r -s /bin/sh appuser
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
USER appuser
|
||||
RUN uv tool install --from /build/dist/*.whl artifactapi
|
||||
COPY . .
|
||||
|
||||
USER root
|
||||
RUN rm -rf /build
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=${VERSION}" -o artifactapi ./cmd/artifactapi
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:nonroot
|
||||
|
||||
COPY --from=builder /build/artifactapi /usr/local/bin/artifactapi
|
||||
|
||||
EXPOSE 8000
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 CMD curl -f http://localhost:8000/health || exit 1
|
||||
USER appuser
|
||||
ENV PATH="/home/appuser/.local/bin:$PATH"
|
||||
WORKDIR /app
|
||||
CMD ["artifactapi"]
|
||||
|
||||
ENTRYPOINT ["artifactapi"]
|
||||
|
||||
@@ -1,59 +1,54 @@
|
||||
.PHONY: build install dev clean test lint format pre-commit tox docker-build docker-up docker-down docker-logs docker-rebuild docker-clean docker-restart
|
||||
.PHONY: build test lint fmt e2e docker-e2e docker docker-ui compose clean tidy check-go
|
||||
|
||||
build:
|
||||
docker build -t artifactapi:dev .
|
||||
BINARY := bin/artifactapi
|
||||
MODULE := git.unkin.net/unkin/artifactapi
|
||||
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0.0-dev")
|
||||
GO_VERSION_REQUIRED := 1.23
|
||||
GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/')
|
||||
|
||||
install: build
|
||||
check-go:
|
||||
@if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \
|
||||
echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \
|
||||
fi
|
||||
|
||||
docker-build: build
|
||||
build: check-go tidy
|
||||
go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(BINARY) ./cmd/artifactapi
|
||||
|
||||
dev: build
|
||||
uv sync --dev
|
||||
test: check-go
|
||||
go test -race -count=1 ./pkg/... ./internal/...
|
||||
|
||||
lint: check-go
|
||||
golangci-lint run ./...
|
||||
go vet ./...
|
||||
|
||||
fmt: check-go
|
||||
gofmt -w .
|
||||
goimports -w .
|
||||
|
||||
e2e: check-go
|
||||
TESTCONTAINERS_RYUK_DISABLED=true go test -tags=e2e -race -count=1 -timeout=5m ./e2e/...
|
||||
|
||||
# Build the container, bring up the full docker-compose stack + a mock upstream,
|
||||
# and run the black-box suite against the running product.
|
||||
docker-e2e: check-go
|
||||
./scripts/docker-e2e.sh
|
||||
|
||||
docker:
|
||||
docker build -t artifactapi:$(VERSION) .
|
||||
|
||||
docker-ui:
|
||||
docker build -t artifactapi-ui:$(VERSION) -f ui/Dockerfile.ui ui/
|
||||
|
||||
compose:
|
||||
docker compose up -d
|
||||
|
||||
clean:
|
||||
rm -rf .venv
|
||||
rm -rf build/
|
||||
rm -rf dist/
|
||||
rm -rf *.egg-info/
|
||||
rm -rf bin/
|
||||
|
||||
test:
|
||||
uvx --python 3.11 --with tox-uv tox
|
||||
|
||||
tox:
|
||||
uvx --python 3.11 --with tox-uv tox
|
||||
|
||||
pre-commit:
|
||||
uvx --python 3.11 pre-commit run --all-files
|
||||
|
||||
lint:
|
||||
uv run ruff check --fix .
|
||||
|
||||
format:
|
||||
uv run ruff format .
|
||||
|
||||
run:
|
||||
uv run python -m src.artifactapi.main
|
||||
|
||||
docker-up:
|
||||
docker-compose up --build --force-recreate -d
|
||||
|
||||
docker-down:
|
||||
docker-compose down
|
||||
|
||||
docker-logs:
|
||||
docker-compose logs -f
|
||||
|
||||
docker-rebuild:
|
||||
docker-compose build --no-cache
|
||||
|
||||
docker-clean:
|
||||
docker-compose down -v --remove-orphans
|
||||
docker system prune -f
|
||||
|
||||
docker-restart: docker-down docker-up
|
||||
tidy:
|
||||
go mod tidy
|
||||
|
||||
# Bump helpers — reads the latest semver tag and creates the next one.
|
||||
# If no tag exists yet, starts from v0.0.0.
|
||||
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
||||
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
||||
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
||||
|
||||
@@ -0,0 +1,880 @@
|
||||
# ArtifactAPI v3 — Go Rewrite Plan
|
||||
|
||||
## Context
|
||||
|
||||
ArtifactAPI is a production artifact proxy/cache serving ~42 remotes (Docker registries, Helm repos, RPM/Alpine repos, GitHub releases, PyPI, npm, Puppet Forge, Terraform registries, Go module proxies) across a Kubernetes cluster. The current Python (FastAPI) implementation works but has architectural debt: opaque hashed S3 paths, no UI for visibility, YAML config files that drift, no garbage collection, no access logging, and virtual repos limited to Helm only.
|
||||
|
||||
The v3 rewrite targets: a single Go binary (API + TUI), a separate React frontend (own Dockerfile), a Terraform provider (separate repo), content-addressable storage, and a cleaner data model that makes the cache inspectable and manageable.
|
||||
|
||||
**Repo**: Same repo (`git.unkin.net/unkin/artifactapi`), new branch.
|
||||
**Module**: `git.unkin.net/unkin/artifactapi`
|
||||
**Frontend**: React + Vite, separate Dockerfile, talks to API
|
||||
**Terraform provider**: Separate repo (`terraform-provider-artifactapi`)
|
||||
|
||||
---
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
┌───────────────────────────────────┐ ┌──────────────────────┐
|
||||
│ Go Binary (API + TUI) │ │ Frontend Container │
|
||||
│ │ │ │
|
||||
│ ┌──────────┐ ┌───────────────┐ │ │ React + Vite SPA │
|
||||
│ │ REST API │ │ Proxy Engine │ │◄───│ nginx / node serve │
|
||||
│ │ /api/v2 │ │ /api/v1/... │ │ │ Dockerfile.ui │
|
||||
│ │ │ │ /v2/... (OCI) │ │ └──────────────────────┘
|
||||
│ └────┬─────┘ └──────┬────────┘ │
|
||||
│ │ │ │ ┌──────────────────────┐
|
||||
│ ┌────┴───────────────┴────────┐ │ │ Terraform Provider │
|
||||
│ │ Data Layer │ │◄───│ (separate repo) │
|
||||
│ │ PostgreSQL · Redis · S3 │ │ └──────────────────────┘
|
||||
│ └─────────────────────────────┘ │
|
||||
│ │ ┌──────────────────────┐
|
||||
│ ┌─────────────────────────────┐ │ │ TUI (subcommand) │
|
||||
│ │ artifactapi tui │──│───►│ artifactapi tui │
|
||||
│ └─────────────────────────────┘ │ │ --endpoint <url> │
|
||||
└───────────────────────────────────┘ └──────────────────────┘
|
||||
```
|
||||
|
||||
Three independent deployment units:
|
||||
1. **Go binary** — API server + TUI subcommand (single `Dockerfile`)
|
||||
2. **React frontend** — SPA served by nginx (`Dockerfile.ui`), talks to `/api/v2`
|
||||
3. **Terraform provider** — separate repo, calls `/api/v2` CRUD
|
||||
|
||||
---
|
||||
|
||||
## Project Structure (Modular)
|
||||
|
||||
```
|
||||
artifactapi/
|
||||
├── cmd/
|
||||
│ └── artifactapi/
|
||||
│ └── main.go # entrypoint: serve / tui subcommands
|
||||
│
|
||||
├── pkg/ # PUBLIC — importable by terraform provider, CLI tools
|
||||
│ ├── models/ # shared domain types
|
||||
│ │ ├── remote.go # Remote, RemoteConfig, PackageType enum
|
||||
│ │ ├── virtual.go # Virtual, VirtualConfig
|
||||
│ │ ├── artifact.go # Artifact, Blob, AccessLogEntry
|
||||
│ │ ├── local.go # LocalFile, LocalRepo
|
||||
│ │ └── stats.go # RemoteStats, OverviewStats
|
||||
│ └── client/ # typed Go API client (used by TUI + Terraform provider)
|
||||
│ ├── client.go # Client struct, base HTTP
|
||||
│ ├── remotes.go # remote CRUD methods
|
||||
│ ├── virtuals.go # virtual CRUD methods
|
||||
│ ├── objects.go # object browse/evict methods
|
||||
│ └── stats.go # stats methods
|
||||
│
|
||||
├── internal/ # PRIVATE — server internals
|
||||
│ ├── server/
|
||||
│ │ ├── server.go # HTTP server setup, router
|
||||
│ │ └── middleware.go # logging, recovery, request-id, access logging
|
||||
│ │
|
||||
│ ├── api/
|
||||
│ │ ├── v1/ # proxy endpoints (v1 compat)
|
||||
│ │ │ ├── proxy.go # GET /api/v1/remote/{name}/{path}
|
||||
│ │ │ ├── docker.go # /v2/{name}/{path}
|
||||
│ │ │ ├── virtual.go # GET /api/v1/virtual/{name}/{path}
|
||||
│ │ │ └── local.go # CRUD /api/v1/local/{name}/{path}
|
||||
│ │ └── v2/ # management API
|
||||
│ │ ├── remotes.go # CRUD + stats
|
||||
│ │ ├── virtuals.go # CRUD
|
||||
│ │ ├── objects.go # browse/evict cached objects
|
||||
│ │ ├── stats.go # overview, top-remotes
|
||||
│ │ ├── events.go # SSE stream
|
||||
│ │ └── health.go # health, metrics
|
||||
│ │
|
||||
│ ├── provider/ # package-type providers (registry protocol handlers)
|
||||
│ │ ├── provider.go # Provider interface + registry
|
||||
│ │ ├── generic/
|
||||
│ │ │ ├── generic.go
|
||||
│ │ │ └── generic_test.go
|
||||
│ │ ├── docker/
|
||||
│ │ │ ├── docker.go # OCI Distribution v2 via go-containerregistry
|
||||
│ │ │ ├── auth.go # Bearer token fetch + cache
|
||||
│ │ │ └── docker_test.go
|
||||
│ │ ├── helm/
|
||||
│ │ │ ├── helm.go # index rewriting via helm.sh/helm/v3/pkg/repo
|
||||
│ │ │ ├── merger.go # virtual index merge
|
||||
│ │ │ └── helm_test.go
|
||||
│ │ ├── pypi/
|
||||
│ │ │ ├── pypi.go # simple index HTML rewriting
|
||||
│ │ │ ├── merger.go # virtual simple index merge
|
||||
│ │ │ └── pypi_test.go
|
||||
│ │ ├── npm/
|
||||
│ │ │ ├── npm.go # metadata JSON rewriting
|
||||
│ │ │ └── npm_test.go
|
||||
│ │ ├── rpm/
|
||||
│ │ │ ├── rpm.go # repodata patterns
|
||||
│ │ │ └── rpm_test.go
|
||||
│ │ ├── alpine/
|
||||
│ │ │ ├── alpine.go # APKINDEX patterns
|
||||
│ │ │ └── alpine_test.go
|
||||
│ │ ├── puppet/
|
||||
│ │ │ ├── puppet.go # file_uri JSON rewriting
|
||||
│ │ │ └── puppet_test.go
|
||||
│ │ ├── terraform/
|
||||
│ │ │ ├── terraform.go # registry protocol, download URL rewriting
|
||||
│ │ │ └── terraform_test.go
|
||||
│ │ └── goproxy/
|
||||
│ │ ├── goproxy.go # Go module proxy protocol (GOPROXY)
|
||||
│ │ └── goproxy_test.go
|
||||
│ │
|
||||
│ ├── proxy/
|
||||
│ │ ├── engine.go # core fetch-or-cache logic
|
||||
│ │ ├── engine_test.go
|
||||
│ │ ├── classifier.go # immutable vs mutable classification
|
||||
│ │ ├── classifier_test.go
|
||||
│ │ ├── revalidator.go # conditional HEAD requests (ETag/Last-Modified)
|
||||
│ │ └── circuit.go # per-remote circuit breaker
|
||||
│ │
|
||||
│ ├── storage/
|
||||
│ │ ├── s3.go # S3 client (minio-go — works with MinIO, Ceph, AWS)
|
||||
│ │ ├── s3_test.go
|
||||
│ │ ├── cas.go # content-addressable store logic
|
||||
│ │ └── cas_test.go
|
||||
│ │
|
||||
│ ├── cache/
|
||||
│ │ ├── redis.go # TTL management, fetch locks
|
||||
│ │ ├── redis_test.go
|
||||
│ │ └── lock.go # distributed lock abstraction
|
||||
│ │
|
||||
│ ├── database/
|
||||
│ │ ├── postgres.go # connection pool, migration runner
|
||||
│ │ ├── queries/ # SQL query files or sqlc-generated code
|
||||
│ │ │ ├── remotes.sql.go
|
||||
│ │ │ ├── virtuals.sql.go
|
||||
│ │ │ ├── artifacts.sql.go
|
||||
│ │ │ └── access_log.sql.go
|
||||
│ │ └── migrations/ # golang-migrate SQL files
|
||||
│ │ ├── 001_initial.up.sql
|
||||
│ │ └── 001_initial.down.sql
|
||||
│ │
|
||||
│ ├── metrics/
|
||||
│ │ └── prometheus.go # counters, gauges, histograms
|
||||
│ │
|
||||
│ ├── gc/
|
||||
│ │ ├── gc.go # background garbage collection goroutine
|
||||
│ │ └── gc_test.go
|
||||
│ │
|
||||
│ ├── tui/
|
||||
│ │ ├── app.go # Bubble Tea main model
|
||||
│ │ ├── views/
|
||||
│ │ │ ├── dashboard.go
|
||||
│ │ │ ├── remotes.go
|
||||
│ │ │ ├── objects.go
|
||||
│ │ │ └── virtuals.go
|
||||
│ │ └── components/
|
||||
│ │ ├── table.go
|
||||
│ │ └── statusbar.go
|
||||
│ │
|
||||
│ └── config/
|
||||
│ └── env.go # environment variable parsing + validation
|
||||
│
|
||||
├── ui/ # React frontend — SEPARATE DOCKERFILE
|
||||
│ ├── src/
|
||||
│ │ ├── App.tsx
|
||||
│ │ ├── pages/
|
||||
│ │ │ ├── Dashboard.tsx
|
||||
│ │ │ ├── Remotes.tsx
|
||||
│ │ │ ├── RemoteDetail.tsx
|
||||
│ │ │ ├── Virtuals.tsx
|
||||
│ │ │ └── Objects.tsx
|
||||
│ │ ├── components/
|
||||
│ │ │ ├── RemoteTable.tsx
|
||||
│ │ │ ├── ObjectBrowser.tsx
|
||||
│ │ │ ├── StatsCard.tsx
|
||||
│ │ │ └── EventFeed.tsx
|
||||
│ │ └── api/
|
||||
│ │ └── client.ts # typed API client
|
||||
│ ├── package.json
|
||||
│ ├── vite.config.ts
|
||||
│ ├── tsconfig.json
|
||||
│ ├── Dockerfile.ui # multi-stage: node build → nginx
|
||||
│ └── nginx.conf # proxy /api/* to backend, serve SPA
|
||||
│
|
||||
├── e2e/ # end-to-end integration tests
|
||||
│ ├── e2e_test.go # TestMain spins up docker-compose stack
|
||||
│ ├── proxy_test.go # proxy through real remotes
|
||||
│ ├── docker_test.go # Docker v2 protocol e2e
|
||||
│ ├── management_test.go # v2 API CRUD
|
||||
│ ├── virtual_test.go # virtual repo merge e2e
|
||||
│ └── docker-compose.e2e.yml # postgres + redis + minio for tests
|
||||
│
|
||||
├── go.mod
|
||||
├── go.sum
|
||||
├── Makefile
|
||||
├── Dockerfile # Go binary (API server + TUI)
|
||||
├── Dockerfile.ui # symlink or copy → ui/Dockerfile.ui
|
||||
└── docker-compose.yml
|
||||
```
|
||||
|
||||
### Key Modularisation Decisions
|
||||
|
||||
- **`pkg/models/`** — Shared domain types importable by the Terraform provider and any external tooling. No dependencies on internal packages
|
||||
- **`pkg/client/`** — Typed Go API client used by both the TUI and the Terraform provider. Depends only on `pkg/models/` and stdlib
|
||||
- **`internal/provider/`** — Each package type is its own subpackage with isolated tests. A provider registry maps `PackageType → Provider`
|
||||
- **`internal/database/queries/`** — Use [sqlc](https://sqlc.dev/) to generate type-safe query functions from SQL, or hand-written query files
|
||||
- **`e2e/`** — Separate test binary that spins up a real docker-compose stack
|
||||
|
||||
---
|
||||
|
||||
## Go Ecosystem Libraries
|
||||
|
||||
Prefer existing, maintained Go modules over writing protocol handlers from scratch.
|
||||
|
||||
### Package-Type Libraries
|
||||
|
||||
| Package Type | Go Module | What It Gives Us |
|
||||
|---|---|---|
|
||||
| **Docker/OCI** | `github.com/google/go-containerregistry` | Full Registry v2/OCI client: manifest parsing, auth challenges, blob operations. `pkg/registry` can implement a v2 server. Reference: `github.com/regclient/regclient` |
|
||||
| **Helm** | `helm.sh/helm/v3/pkg/repo` | Parse/generate `index.yaml`, `IndexFile`/`ChartVersion` types, URL entries. Used directly for merge |
|
||||
| **Terraform** | `github.com/hashicorp/terraform-registry-address` | Provider/module address parsing, `ForRegistryProtocol()` URL generation. Protocol spec: provider registry protocol v1 |
|
||||
| **Go Modules** | `github.com/goproxy/goproxy` | Minimalist GOPROXY protocol handler, implements full spec as `http.Handler`. Handles `/@v/list`, `/@v/{v}.info`, `/@v/{v}.mod`, `/@v/{v}.zip`, `/@latest` |
|
||||
| **RPM** | `rs3.io/go/rpm/repomd` | Parse `repomd.xml`, `primary.xml` with proper XML namespace handling |
|
||||
| **Alpine** | `gitlab.alpinelinux.org/alpine/go` | Official Alpine library: parse APKINDEX, `.apk` files |
|
||||
| **PyPI** | stdlib `golang.org/x/net/html` | No dedicated Go PyPI library exists. Parse simple index HTML with `x/net/html`, extract `<a>` tags. Minimal — the rewriting is just href replacement |
|
||||
| **npm** | stdlib `encoding/json` | npm metadata is JSON — parse with stdlib, rewrite `dist.tarball` URLs. No special library needed |
|
||||
| **Puppet Forge** | stdlib `encoding/json` | Forge API is JSON — parse and rewrite `file_uri` fields. Community lib `github.com/johnmccabe/go-puppetforge` exists but is thin; stdlib suffices |
|
||||
|
||||
### Infrastructure Libraries
|
||||
|
||||
| Purpose | Go Module | Why This One |
|
||||
|---|---|---|
|
||||
| **HTTP router** | `github.com/go-chi/chi/v5` | Lightweight, stdlib `http.Handler` compatible, middleware chain |
|
||||
| **PostgreSQL** | `github.com/jackc/pgx/v5` | Pure Go, connection pooling, COPY support, prepared statements |
|
||||
| **SQL generation** | `github.com/sqlc-dev/sqlc` | Generate type-safe Go from SQL queries — no ORM, no reflection |
|
||||
| **Redis** | `github.com/redis/go-redis/v9` | Full Redis client, pipelining, pub/sub |
|
||||
| **S3 (MinIO/Ceph/AWS)** | `github.com/minio/minio-go/v7` | Native S3-compatible client. Works with MinIO, Ceph RGW, AWS S3, any S3-compatible backend out of the box. Lighter than aws-sdk-go-v2, purpose-built for S3 compat |
|
||||
| **DB migrations** | `github.com/golang-migrate/migrate/v4` | SQL file-based migrations, CLI + library |
|
||||
| **Prometheus** | `github.com/prometheus/client_golang` | Counters, gauges, histograms |
|
||||
| **TUI** | `github.com/charmbracelet/bubbletea` | Elm-architecture TUI framework |
|
||||
| **TUI styling** | `github.com/charmbracelet/lipgloss` | Terminal styling |
|
||||
| **TUI components** | `github.com/charmbracelet/bubbles` | Table, text input, spinner, etc. |
|
||||
| **Structured logging** | `log/slog` (stdlib) | Go 1.21+ structured logging, zero dependencies |
|
||||
| **Testing** | `github.com/stretchr/testify` | Assertions + require for unit tests |
|
||||
| **Test containers** | `github.com/testcontainers/testcontainers-go` | Spin up Postgres/Redis/MinIO in e2e tests |
|
||||
|
||||
### S3 Client: Multi-Backend Support
|
||||
|
||||
Using `minio-go/v7` as the S3 client because it natively supports:
|
||||
- **MinIO** — primary development/production target
|
||||
- **Ceph RGW** — S3-compatible via endpoint config
|
||||
- **AWS S3** — via region + credential config
|
||||
- **Any S3-compatible** — GCS (interop mode), Wasabi, DigitalOcean Spaces, etc.
|
||||
|
||||
No abstraction layer needed — `minio-go` handles endpoint differences internally. Config:
|
||||
```go
|
||||
client, _ := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: useTLS,
|
||||
Region: region, // optional, for AWS
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Data Layer
|
||||
|
||||
### PostgreSQL Schema
|
||||
|
||||
```sql
|
||||
-- Remotes: managed exclusively by Terraform
|
||||
CREATE TABLE remotes (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL, -- generic, docker, helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy
|
||||
base_url TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
immutable_ttl INTEGER DEFAULT 0,
|
||||
mutable_ttl INTEGER DEFAULT 3600,
|
||||
check_mutable BOOLEAN DEFAULT TRUE,
|
||||
immutable_patterns TEXT[] DEFAULT '{}', -- user-defined immutable patterns
|
||||
mutable_patterns TEXT[] DEFAULT '{}', -- user-defined mutable patterns (merged with provider built-ins)
|
||||
allowlist TEXT[] DEFAULT '{}', -- if empty, allow all paths; if non-empty, only matching paths proxied
|
||||
blocklist TEXT[] DEFAULT '{}', -- always denied, checked before allowlist
|
||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
||||
ban_tags TEXT[] DEFAULT '{}',
|
||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
||||
quarantine_days INTEGER DEFAULT 3,
|
||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
||||
releases_remote TEXT DEFAULT '', -- terraform type: name of CDN remote for download URL rewriting
|
||||
managed_by TEXT DEFAULT '', -- 'terraform' or empty
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Virtual repositories
|
||||
CREATE TABLE virtuals (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
members TEXT[] NOT NULL,
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Content-addressable blob storage tracking
|
||||
CREATE TABLE blobs (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
s3_key TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
content_type TEXT DEFAULT 'application/octet-stream',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Artifact metadata: maps (remote, path) → content blob
|
||||
CREATE TABLE artifacts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
upstream_etag TEXT DEFAULT '',
|
||||
upstream_last_modified TIMESTAMPTZ,
|
||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
fetch_count BIGINT DEFAULT 1,
|
||||
access_count BIGINT DEFAULT 1,
|
||||
UNIQUE(remote_name, path)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_artifacts_remote ON artifacts(remote_name);
|
||||
CREATE INDEX idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
||||
|
||||
-- Local file uploads
|
||||
CREATE TABLE local_files (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
-- Access log (append-only, powers dashboards)
|
||||
CREATE TABLE access_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
cache_hit BOOLEAN NOT NULL,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
upstream_ms INTEGER DEFAULT 0,
|
||||
client_ip TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
```
|
||||
|
||||
### Redis Usage (Ephemeral Only)
|
||||
|
||||
| Key pattern | Type | TTL | Purpose |
|
||||
|---|---|---|---|
|
||||
| `ttl:{remote}:{path}` | STRING | remote's immutable/mutable TTL | Artifact freshness — existence = still fresh |
|
||||
| `lock:{remote}:{path}` | STRING (NX) | 30s | Fetch lock — prevents thundering herd |
|
||||
| `etag:{remote}:{path}` | STRING | same as TTL key | Cached ETag for conditional revalidation |
|
||||
| `circuit:{remote}` | STRING | configurable | Circuit breaker — consecutive failure count |
|
||||
|
||||
Losing Redis = all TTLs expire = next request re-validates upstream. No data loss.
|
||||
|
||||
### S3 Layout (Content-Addressable)
|
||||
|
||||
```
|
||||
artifacts-bucket/
|
||||
├── blobs/sha256/{content_hash} # immutable CAS blobs
|
||||
├── indexes/{remote}/{path} # mutable index files (helm, pypi, rpm, etc.)
|
||||
├── indexes/{virtual}/{path} # merged virtual indexes
|
||||
└── local/{repo}/{path} # user uploads (CAS-backed via blobs table)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Terraform Remote Type (New in v2)
|
||||
|
||||
The `terraform` package type proxies the Terraform Provider Registry Protocol:
|
||||
|
||||
- **URL construction**: prepends `/v1/providers/` to request paths
|
||||
- **Built-in mutable pattern**: `[^/]+/[^/]+/versions$` (version listings change over time)
|
||||
- **Built-in immutable pattern**: `[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$` (per-version download info is fixed)
|
||||
- **Response rewriting**: download info JSON — rewrites `download_url`, `shasums_url`, `shasums_signature_url` to route through a companion `releases_remote` (e.g., `hashicorp-releases` generic remote)
|
||||
- **Config**: requires `releases_remote` field pointing to the CDN remote that serves the actual binaries
|
||||
|
||||
Uses `github.com/hashicorp/terraform-registry-address` for address parsing and protocol-compliant URL generation.
|
||||
|
||||
---
|
||||
|
||||
## Go Module Proxy Remote Type (New)
|
||||
|
||||
The `goproxy` package type implements the GOPROXY protocol (Go module proxy):
|
||||
|
||||
| Endpoint | Mutability | Description |
|
||||
|---|---|---|
|
||||
| `{module}/@v/list` | Mutable | Plain text list of known versions |
|
||||
| `{module}/@latest` | Mutable | JSON metadata for latest version |
|
||||
| `{module}/@v/{version}.info` | Immutable | JSON version metadata (`Version`, `Time`) |
|
||||
| `{module}/@v/{version}.mod` | Immutable | `go.mod` file for that version |
|
||||
| `{module}/@v/{version}.zip` | Immutable | Source archive for that version |
|
||||
|
||||
- **No URL rewriting needed** — responses are self-contained (no embedded URLs)
|
||||
- **Config**: `base_url` points to upstream proxy (e.g., `https://proxy.golang.org`)
|
||||
- **Client usage**: set `GOPROXY=https://artifactapi.example.com/api/v1/remote/goproxy`
|
||||
- Uses `github.com/goproxy/goproxy` for protocol handling
|
||||
|
||||
---
|
||||
|
||||
## Allowlist / Blocklist / Automatic Mutable Patterns
|
||||
|
||||
### Access Control (Per-Remote)
|
||||
|
||||
| Field | Default | Behavior |
|
||||
|---|---|---|
|
||||
| `blocklist` | `[]` (empty) | If a path matches any blocklist pattern → **403 Forbidden**. Checked first |
|
||||
| `allowlist` | `[]` (empty) | If empty → **allow everything**. If non-empty → only matching paths are proxied; everything else → **403** |
|
||||
|
||||
Evaluation order: blocklist → allowlist → proxy. No allowlist + no blocklist = open proxy (default).
|
||||
|
||||
### Automatic Mutable Patterns (Per-Provider Built-ins)
|
||||
|
||||
Each provider declares built-in mutable patterns that are **always merged** with user-defined `mutable_patterns`. Users never need to configure these — the provider knows which paths change over time.
|
||||
|
||||
| Provider | Built-in Mutable Patterns | Rationale |
|
||||
|---|---|---|
|
||||
| **generic** | *(none)* | No convention for what's mutable |
|
||||
| **docker** | `/manifests/(?!sha256:)[^/]+$`, `/tags/list$` | Tag manifests change; digest manifests don't |
|
||||
| **helm** | `index\.yaml$` | Chart index changes when new charts are published |
|
||||
| **pypi** | `simple/` | Package index pages change with new releases |
|
||||
| **npm** | `^[^/]+$` (package metadata, not `.tgz`) | Package metadata changes; tarballs are immutable |
|
||||
| **rpm** | `repomd\.xml$`, `repodata/.*`, `Packages\.gz$` | Repo metadata rebuilt on every publish |
|
||||
| **alpine** | `APKINDEX\.tar\.gz$` | Package index rebuilt on every publish |
|
||||
| **puppet** | `^v3/modules/`, `^v3/releases` | Module metadata changes with new releases |
|
||||
| **terraform** | `[^/]+/[^/]+/versions$` | Provider version listings grow over time |
|
||||
| **goproxy** | `@v/list$`, `@latest$` | Version list and latest pointer change |
|
||||
|
||||
These are returned by `Provider.BuiltinMutablePatterns()` and merged at classification time:
|
||||
```
|
||||
effective_mutable = provider.BuiltinMutablePatterns() ∪ remote.mutable_patterns
|
||||
```
|
||||
|
||||
If a path matches `effective_mutable` → use `mutable_ttl`. If it matches `remote.immutable_patterns` → use `immutable_ttl`. Immutable patterns take precedence over mutable when both match.
|
||||
|
||||
---
|
||||
|
||||
## API Design
|
||||
|
||||
### v1 Proxy Endpoints (Backwards Compatible)
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| `GET` | `/api/v1/remote/{name}/{path}` | Proxy/cache artifact |
|
||||
| `GET` | `/api/v1/virtual/{name}/{path}` | Virtual repo proxy |
|
||||
| `GET/HEAD` | `/v2/{name}/{path}` | Docker Registry v2 |
|
||||
| `GET` | `/v2/` | Docker v2 ping |
|
||||
| `GET/PUT/HEAD/DELETE` | `/api/v1/local/{name}/{path}` | Local repo CRUD |
|
||||
|
||||
### v2 Management API (New)
|
||||
|
||||
```
|
||||
GET /api/v2/remotes → [{name, package_type, base_url, description, stats}]
|
||||
GET /api/v2/remotes/{name} → {full config + stats + health}
|
||||
POST /api/v2/remotes → create remote (Terraform provider)
|
||||
PUT /api/v2/remotes/{name} → update remote (Terraform provider)
|
||||
DELETE /api/v2/remotes/{name} → delete remote — cascades artifacts, GC cleans S3
|
||||
|
||||
GET /api/v2/virtuals → [{name, package_type, members, stats}]
|
||||
GET /api/v2/virtuals/{name} → {full config + member details}
|
||||
POST /api/v2/virtuals → create virtual
|
||||
PUT /api/v2/virtuals/{name} → update virtual
|
||||
DELETE /api/v2/virtuals/{name} → delete virtual
|
||||
|
||||
GET /api/v2/remotes/{name}/objects → paginated objects
|
||||
?q=pattern&sort=size|accessed|age&page=1&per_page=50
|
||||
DELETE /api/v2/remotes/{name}/objects/{path} → evict specific cached object
|
||||
DELETE /api/v2/remotes/{name}/cache → flush cache
|
||||
?type=all|indexes|blobs
|
||||
|
||||
GET /api/v2/stats → overview stats
|
||||
GET /api/v2/stats/top-remotes → top remotes by size/requests/hit-rate
|
||||
|
||||
GET /api/v2/health → {status, postgres, redis, s3, uptime}
|
||||
GET /metrics → Prometheus format
|
||||
GET /api/v2/events → SSE stream
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Proxy Engine
|
||||
|
||||
### Request Flow
|
||||
|
||||
```
|
||||
Client Request
|
||||
│
|
||||
▼
|
||||
Classify (immutable/mutable/denied)
|
||||
│
|
||||
├── blocklist match → 403
|
||||
├── allowlist non-empty + no match → 403
|
||||
│
|
||||
▼
|
||||
Check Redis TTL key
|
||||
│
|
||||
├── exists (fresh) → serve from S3, log access
|
||||
│
|
||||
├── missing (expired or uncached)
|
||||
│ │
|
||||
│ ▼
|
||||
│ Acquire fetch lock (Redis SETNX, 30s TTL)
|
||||
│ │
|
||||
│ ├── lock acquired
|
||||
│ │ ├── mutable + check_mutable + have ETag → HEAD upstream
|
||||
│ │ │ ├── 304 → refresh TTL, serve from S3
|
||||
│ │ │ └── changed → full fetch
|
||||
│ │ └── full fetch from upstream
|
||||
│ │ → provider.RewriteResponse() if needed
|
||||
│ │ → CAS store (hash → check blobs → upload if new)
|
||||
│ │ → upsert artifact in Postgres
|
||||
│ │ → set Redis TTL + release lock
|
||||
│ │ → on upstream error + stale_on_error → refresh TTL, serve stale
|
||||
│ │
|
||||
│ └── lock not acquired → poll S3 briefly, serve if another pod fetched it
|
||||
│
|
||||
▼
|
||||
Stream response from S3, log access
|
||||
```
|
||||
|
||||
### Circuit Breaker
|
||||
|
||||
Per-remote, tracked in Redis. Closed → Open (after N failures) → Half-open (after cooldown). Exposed via `GET /api/v2/remotes/{name}` health field.
|
||||
|
||||
### Content-Addressable Storage
|
||||
|
||||
1. Stream upstream → temp file, compute SHA256 inline
|
||||
2. Check `blobs` table for hash
|
||||
3. Exists → skip S3 upload, upsert `artifacts` row only
|
||||
4. New → upload to `blobs/sha256/{hash}`, insert both rows
|
||||
|
||||
### Garbage Collection
|
||||
|
||||
Background goroutine (configurable interval, default 1h):
|
||||
1. Orphaned blobs: delete S3 objects whose `content_hash` has no referencing `artifacts` or `local_files` rows
|
||||
2. Cold artifacts: optional per-remote, delete artifacts not accessed in N days
|
||||
3. Remote deletion: `ON DELETE CASCADE` handles Postgres; GC sweeps orphaned blobs
|
||||
|
||||
---
|
||||
|
||||
## Package Providers
|
||||
|
||||
### Provider Interface
|
||||
|
||||
```go
|
||||
type Provider interface {
|
||||
Type() models.PackageType
|
||||
BuiltinMutablePatterns() []*regexp.Regexp
|
||||
BuiltinImmutablePatterns() []*regexp.Regexp
|
||||
ContentType(path string) string
|
||||
UpstreamURL(remote models.Remote, path string) string
|
||||
RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error)
|
||||
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
```
|
||||
|
||||
### Provider Registry
|
||||
|
||||
```go
|
||||
var registry = map[models.PackageType]Provider{
|
||||
models.PackageGeneric: &generic.Provider{},
|
||||
models.PackageDocker: &docker.Provider{},
|
||||
models.PackageHelm: &helm.Provider{},
|
||||
models.PackagePyPI: &pypi.Provider{},
|
||||
models.PackageNPM: &npm.Provider{},
|
||||
models.PackageRPM: &rpm.Provider{},
|
||||
models.PackageAlpine: &alpine.Provider{},
|
||||
models.PackagePuppet: &puppet.Provider{},
|
||||
models.PackageTerraform: &terraform.Provider{},
|
||||
models.PackageGoProxy: &goproxy.Provider{},
|
||||
}
|
||||
|
||||
func Get(t models.PackageType) (Provider, error) { ... }
|
||||
```
|
||||
|
||||
Each provider lives in its own subpackage under `internal/provider/` with its own `_test.go`.
|
||||
|
||||
---
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
Every package gets `_test.go` files alongside the source. Run with `go test ./...`.
|
||||
|
||||
| Package | What's Tested |
|
||||
|---|---|
|
||||
| `internal/provider/docker/` | Auth token parsing/caching, manifest classification, tag banning, URL construction, blob key generation |
|
||||
| `internal/provider/helm/` | `index.yaml` parsing (using `helm.sh/helm/v3/pkg/repo`), URL rewriting, index merging |
|
||||
| `internal/provider/pypi/` | Simple index HTML parsing, URL rewriting, index merging |
|
||||
| `internal/provider/npm/` | Metadata JSON rewriting (`dist.tarball` URLs) |
|
||||
| `internal/provider/terraform/` | Registry URL construction, download info JSON rewriting, `releases_remote` URL extraction |
|
||||
| `internal/provider/rpm/` | Mutable pattern matching (repodata) |
|
||||
| `internal/provider/alpine/` | Mutable pattern matching (APKINDEX) |
|
||||
| `internal/provider/puppet/` | `file_uri` JSON rewriting |
|
||||
| `internal/proxy/` | Classifier (immutable vs mutable vs denied), circuit breaker state transitions, revalidator logic |
|
||||
| `internal/storage/` | CAS key generation, dedup detection, S3 operation mocking |
|
||||
| `internal/cache/` | Redis TTL set/check, fetch lock acquire/release/contention |
|
||||
| `internal/gc/` | Orphan detection queries, cold artifact selection |
|
||||
| `pkg/models/` | Model validation, PackageType enum |
|
||||
| `pkg/client/` | API client request/response serialization |
|
||||
|
||||
### End-to-End Tests
|
||||
|
||||
Located in `e2e/`. Use `testcontainers-go` to spin up real Postgres, Redis, and MinIO containers. The test binary starts the actual `artifactapi` server against these backends.
|
||||
|
||||
```go
|
||||
// e2e/e2e_test.go
|
||||
func TestMain(m *testing.M) {
|
||||
// Start postgres, redis, minio via testcontainers-go
|
||||
// Run migrations
|
||||
// Start artifactapi server on random port
|
||||
// Run tests
|
||||
// Tear down
|
||||
}
|
||||
```
|
||||
|
||||
| Test File | What's Tested |
|
||||
|---|---|
|
||||
| `e2e/proxy_test.go` | Proxy a real GitHub release through generic remote, verify S3 storage, verify Redis TTL, verify Postgres artifact row, verify cache hit on second request |
|
||||
| `e2e/docker_test.go` | Pull a real image manifest + blob through Docker v2 proxy, verify blob deduplication, tag banning |
|
||||
| `e2e/management_test.go` | Full CRUD lifecycle: create remote via v2 API, proxy through it, list objects, evict object, flush cache, delete remote |
|
||||
| `e2e/virtual_test.go` | Create two helm remotes + virtual, fetch merged index, verify priority ordering |
|
||||
| `e2e/terraform_test.go` | Proxy terraform provider version listing + download info, verify URL rewriting to releases_remote |
|
||||
| `e2e/goproxy_test.go` | Proxy Go module `@v/list`, `.info`, `.mod`, `.zip` through GOPROXY remote, verify mutable vs immutable classification |
|
||||
| `e2e/gc_test.go` | Create artifact, delete remote, trigger GC, verify S3 blob cleaned up |
|
||||
|
||||
### Code Quality
|
||||
|
||||
- `gofmt` / `goimports` — enforced in CI, run on save
|
||||
- `golangci-lint` — comprehensive linter suite (staticcheck, errcheck, govet, etc.)
|
||||
- `go vet ./...` — run in CI
|
||||
- Makefile targets: `make test`, `make lint`, `make e2e`, `make fmt`
|
||||
|
||||
---
|
||||
|
||||
## Terraform Provider (Separate Repo)
|
||||
|
||||
**Repo**: `terraform-provider-artifactapi`
|
||||
**Uses**: `pkg/client/` and `pkg/models/` from the main module
|
||||
|
||||
```hcl
|
||||
provider "artifactapi" {
|
||||
endpoint = "https://artifactapi.k8s.syd1.au.unkin.net"
|
||||
}
|
||||
|
||||
resource "artifactapi_remote" "terraform_registry" {
|
||||
name = "terraform-registry"
|
||||
package_type = "terraform"
|
||||
base_url = "https://registry.terraform.io"
|
||||
description = "Terraform provider registry"
|
||||
releases_remote = artifactapi_remote.hashicorp_releases.name
|
||||
|
||||
immutable_patterns = [
|
||||
"[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$",
|
||||
]
|
||||
|
||||
cache {
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
}
|
||||
}
|
||||
|
||||
resource "artifactapi_remote" "hashicorp_releases" {
|
||||
name = "hashicorp-releases"
|
||||
package_type = "generic"
|
||||
base_url = "https://releases.hashicorp.com"
|
||||
|
||||
immutable_patterns = [
|
||||
".*\\.zip$",
|
||||
".*SHA256SUMS(\\.sig)?$",
|
||||
]
|
||||
|
||||
cache {
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 0
|
||||
}
|
||||
}
|
||||
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
description = "All helm repos merged"
|
||||
members = [
|
||||
artifactapi_remote.jetstack.name,
|
||||
artifactapi_remote.hashicorp_helm.name,
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Web UI (React + Vite — Separate Container)
|
||||
|
||||
### Deployment
|
||||
|
||||
Separate `Dockerfile.ui`: multi-stage build (node → nginx). Served as its own container/pod. nginx proxies `/api/*` to the Go backend.
|
||||
|
||||
### Pages
|
||||
|
||||
| Route | Content |
|
||||
|---|---|
|
||||
| `/` | Dashboard: total objects, storage used, dedup savings, bandwidth saved, top remotes chart, live SSE event feed, health indicators |
|
||||
| `/remotes` | Remote table: name, type, description, object count, size, hit rate, health. Filter by type, sort any column |
|
||||
| `/remotes/:name` | Config (read-only, "Managed by Terraform" badge), stats, object browser with search/sort/evict, flush actions |
|
||||
| `/virtuals` | Virtual table: name, type, members, merged object count |
|
||||
| `/virtuals/:name` | Member list with individual stats |
|
||||
|
||||
All config is read-only — managed by Terraform.
|
||||
|
||||
---
|
||||
|
||||
## TUI (Bubble Tea — Subcommand)
|
||||
|
||||
`artifactapi tui --endpoint http://localhost:8000` or via `ARTIFACTAPI_ENDPOINT` env.
|
||||
|
||||
Uses `pkg/client/` for all API calls (same client as Terraform provider).
|
||||
|
||||
| View | Key bindings |
|
||||
|---|---|
|
||||
| Dashboard | summary stats, top remotes |
|
||||
| Remotes list | `j`/`k` navigate, `/` filter, `Enter` detail |
|
||||
| Remote detail | config + stats, `Enter` → object browser |
|
||||
| Object browser | `/` search, `d` evict, `f` flush |
|
||||
| Virtuals | `j`/`k`, `Enter` detail |
|
||||
|
||||
---
|
||||
|
||||
## Improvements Over v2
|
||||
|
||||
| Area | v2 (Python) | v3 (Go) |
|
||||
|---|---|---|
|
||||
| S3 paths | Hashed, opaque | Content-addressed CAS |
|
||||
| Config | YAML files, mtime reload | Terraform via API |
|
||||
| Package types | 8 types | 10 types (+ terraform, goproxy) |
|
||||
| Virtual repos | Helm only | Helm + PyPI, extensible |
|
||||
| Deduplication | Docker blobs only | All types via CAS |
|
||||
| Revalidation | Opt-in flag | Default for all mutable |
|
||||
| Access logging | None | Per-artifact in Postgres |
|
||||
| GC | None | Background goroutine |
|
||||
| Upstream health | Per-request | Circuit breaker |
|
||||
| S3 backends | MinIO only | MinIO, Ceph, AWS (minio-go) |
|
||||
| UI | None | Web dashboard + TUI |
|
||||
| Binary | Python + venv | Static Go binary |
|
||||
| Frontend | N/A | Separate container (React) |
|
||||
| Testing | Mocked unit tests | Unit + e2e with real backends |
|
||||
|
||||
---
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Core Engine + Models
|
||||
- Go module, Makefile (`make build test lint fmt e2e`), Dockerfile, docker-compose
|
||||
- `pkg/models/` — all domain types
|
||||
- PostgreSQL schema + migrations
|
||||
- S3 storage layer with CAS (`minio-go/v7`)
|
||||
- Redis cache layer (TTL, locks)
|
||||
- Proxy engine: fetch-or-cache, classifier, revalidator
|
||||
- Generic + Docker providers (most complexity: OCI auth, CAS, tag banning)
|
||||
- Health + metrics endpoints
|
||||
- Unit tests for all packages
|
||||
- **Milestone**: proxy Docker + generic, cache in S3, track in Postgres
|
||||
|
||||
### Phase 2: All Providers
|
||||
- Helm (using `helm.sh/helm/v3/pkg/repo`)
|
||||
- PyPI (stdlib `x/net/html`)
|
||||
- npm (stdlib `encoding/json`)
|
||||
- RPM (using `rs3.io/go/rpm/repomd`)
|
||||
- Alpine (using `gitlab.alpinelinux.org/alpine/go`)
|
||||
- Puppet Forge (stdlib `encoding/json`)
|
||||
- Terraform (using `hashicorp/terraform-registry-address`)
|
||||
- Go Modules / GOPROXY (using `github.com/goproxy/goproxy`)
|
||||
- Unit tests per provider
|
||||
- **Milestone**: feature parity with v2 + goproxy
|
||||
|
||||
### Phase 3: Management API + Virtual Repos + GC
|
||||
- `pkg/client/` — shared Go API client
|
||||
- v2 CRUD endpoints
|
||||
- Virtual repo engine: `IndexMerger` for Helm + PyPI
|
||||
- Circuit breaker
|
||||
- Access logging middleware
|
||||
- GC goroutine
|
||||
- **Milestone**: full API, virtuals, GC
|
||||
|
||||
### Phase 4: End-to-End Tests
|
||||
- `e2e/` test suite with `testcontainers-go`
|
||||
- Proxy, Docker, management, virtual, terraform, GC tests
|
||||
- CI pipeline: `make e2e`
|
||||
- **Milestone**: comprehensive e2e coverage
|
||||
|
||||
### Phase 5: Terraform Provider
|
||||
- Separate repo: `terraform-provider-artifactapi`
|
||||
- Imports `pkg/client/` and `pkg/models/`
|
||||
- `artifactapi_remote` + `artifactapi_virtual` resources + data sources
|
||||
- Import support
|
||||
- **Milestone**: manage all config via Terraform
|
||||
|
||||
### Phase 6: Web UI
|
||||
- React + Vite in `ui/`
|
||||
- `Dockerfile.ui` (multi-stage → nginx)
|
||||
- Dashboard, remotes, objects, virtuals pages
|
||||
- SSE event feed
|
||||
- **Milestone**: full web UI in separate container
|
||||
|
||||
### Phase 7: TUI
|
||||
- Bubble Tea in `internal/tui/`
|
||||
- Uses `pkg/client/`
|
||||
- Dashboard, remotes, objects, virtuals views
|
||||
- **Milestone**: TUI feature parity with web UI
|
||||
|
||||
### Phase 8: Migration + Cutover
|
||||
- Migration tool: v2 YAML → Terraform HCL + `terraform import` commands
|
||||
- S3 rehash script: `{remote}/{hash16}/{file}` → `blobs/sha256/{content_hash}`
|
||||
- Parallel run, response comparison
|
||||
- Cutover
|
||||
|
||||
---
|
||||
|
||||
## Makefile Targets
|
||||
|
||||
```makefile
|
||||
.PHONY: build test lint fmt e2e docker docker-ui
|
||||
|
||||
build: ## Build Go binary
|
||||
go build -o bin/artifactapi ./cmd/artifactapi
|
||||
|
||||
test: ## Run unit tests
|
||||
go test ./...
|
||||
|
||||
lint: ## Run golangci-lint + go vet
|
||||
golangci-lint run ./...
|
||||
go vet ./...
|
||||
|
||||
fmt: ## Format code (gofmt + goimports)
|
||||
gofmt -w .
|
||||
goimports -w .
|
||||
|
||||
e2e: ## Run end-to-end tests (requires Docker)
|
||||
go test -tags=e2e -count=1 -timeout=5m ./e2e/...
|
||||
|
||||
docker: ## Build API server Docker image
|
||||
docker build -t artifactapi .
|
||||
|
||||
docker-ui: ## Build frontend Docker image
|
||||
docker build -t artifactapi-ui -f ui/Dockerfile.ui ui/
|
||||
|
||||
compose: ## Start full stack (API + UI + Postgres + Redis + MinIO)
|
||||
docker compose up -d
|
||||
```
|
||||
@@ -1,137 +0,0 @@
|
||||
# ArtifactAPI Specification
|
||||
|
||||
## Repository model
|
||||
|
||||
Every repository entry in `remotes.yaml` has two orthogonal fields:
|
||||
|
||||
| field | values | meaning |
|
||||
|---|---|---|
|
||||
| `type` | `local`, `remote`, `virtual` | repository kind — how the repo is served |
|
||||
| `package` | `docker`, `rpm`, `alpine`, `generic` | package format — what protocol and caching rules to apply |
|
||||
|
||||
**type**
|
||||
|
||||
- `local` — files are uploaded directly to the API and stored in S3; no upstream.
|
||||
- `remote` — proxies and caches content from an upstream URL (`base_url`).
|
||||
- `virtual` — aggregates multiple repositories (not yet implemented).
|
||||
|
||||
**package**
|
||||
|
||||
- `docker` — upstream speaks the OCI Distribution API (Bearer auth, manifest/blob paths).
|
||||
- `rpm` — upstream is an RPM repository; repodata files are index files.
|
||||
- `alpine` — upstream is an Alpine APK repository; `APKINDEX.tar.gz` is an index file.
|
||||
- `generic` — plain HTTP file download; no format-specific logic.
|
||||
|
||||
---
|
||||
|
||||
## Caching
|
||||
|
||||
Two cache classes determine retention:
|
||||
|
||||
| class | stored | TTL |
|
||||
|---|---|---|
|
||||
| **file** | S3 object, no Redis entry | `file_ttl` — `0` means indefinite |
|
||||
| **index** | S3 object + Redis TTL key | `index_ttl` — when the Redis key expires the S3 object is deleted and re-fetched |
|
||||
|
||||
Index files are mutable metadata that must expire. File-class objects are treated as immutable and cached indefinitely (unless `file_ttl` is set).
|
||||
|
||||
---
|
||||
|
||||
## Docker package rules
|
||||
|
||||
### URL construction
|
||||
|
||||
Remote URLs are prefixed with `/v2/` for `package: docker` remotes:
|
||||
|
||||
```
|
||||
{base_url}/v2/{path}
|
||||
```
|
||||
|
||||
e.g. `library/nginx/manifests/latest` → `https://registry-1.docker.io/v2/library/nginx/manifests/latest`
|
||||
|
||||
### Authentication
|
||||
|
||||
Docker registries use Bearer token challenges. On a `401 Unauthorized` response, the API:
|
||||
|
||||
1. Parses the `WWW-Authenticate: Bearer` header for `realm`, `service`, and `scope`.
|
||||
2. Fetches a token from the auth realm, supplying `username`/`password` from the remote config if present.
|
||||
3. Retries the request with `Authorization: Bearer <token>`.
|
||||
|
||||
Tokens are cached in-memory keyed by `(realm, service, scope, username)` and expire 30 seconds before their stated `expires_in`.
|
||||
|
||||
### Cache classification
|
||||
|
||||
| path pattern | mutable | class | TTL source |
|
||||
|---|---|---|---|
|
||||
| `/manifests/<tag>` | yes | index | `index_ttl` |
|
||||
| `/tags/list` | yes | index | `index_ttl` |
|
||||
| `/manifests/sha256:<digest>` | no | file | `file_ttl` |
|
||||
| `/blobs/sha256:<digest>` | no | file | `file_ttl` |
|
||||
|
||||
Tag-based manifests and tag lists are mutable and cached as index. Digest-pinned manifests and blobs are content-addressed and cached indefinitely as files.
|
||||
|
||||
### Blob deduplication
|
||||
|
||||
Blobs are stored under a digest-keyed path shared across all images on the same remote:
|
||||
|
||||
```
|
||||
{remote_name}/blobs/sha256/{digest}
|
||||
```
|
||||
|
||||
The same layer pulled by different images is stored once.
|
||||
|
||||
### Accept headers
|
||||
|
||||
| path | `Accept` header sent upstream |
|
||||
|---|---|
|
||||
| `/manifests/…` | `application/vnd.docker.distribution.manifest.v2+json`, `application/vnd.oci.image.manifest.v1+json`, `application/vnd.oci.image.index.v1+json`, `application/vnd.docker.distribution.manifest.list.v2+json` |
|
||||
| `/blobs/…` | `application/octet-stream` |
|
||||
|
||||
---
|
||||
|
||||
## OCI Distribution API endpoint
|
||||
|
||||
The API exposes a native Docker registry interface so clients can use `docker pull` directly:
|
||||
|
||||
```
|
||||
GET /v2/ — version ping
|
||||
GET /v2/{remote}/{image}/manifests/{ref} — fetch manifest
|
||||
HEAD /v2/{remote}/{image}/manifests/{ref} — manifest metadata
|
||||
GET /v2/{remote}/{image}/blobs/{digest} — fetch blob
|
||||
HEAD /v2/{remote}/{image}/blobs/{digest} — blob metadata
|
||||
```
|
||||
|
||||
Responses include `Docker-Distribution-Api-Version`, `Docker-Content-Digest`, and the correct OCI `Content-Type` (detected from the manifest `mediaType` field).
|
||||
|
||||
Only remotes with `package: docker` are accessible via this endpoint. All other remotes return `400`.
|
||||
|
||||
---
|
||||
|
||||
## include_patterns
|
||||
|
||||
`include_patterns` is a list of Python regexes applied to every request before any upstream fetch or cache lookup.
|
||||
|
||||
**Generic remotes (`/api/v1/remote/…`):**
|
||||
- Patterns match against the file path and the full path.
|
||||
- Index files (mutable metadata) bypass pattern checks and are always allowed.
|
||||
|
||||
**Docker remotes (`/v2/…`):**
|
||||
- Patterns match against the image name (first two path segments, e.g. `library/nginx`) and the full path.
|
||||
- The index-file exemption does **not** apply — patterns restrict whole images, including their manifests and tag lists.
|
||||
- No patterns configured → all images allowed.
|
||||
|
||||
Returns `403` when a request is blocked.
|
||||
|
||||
---
|
||||
|
||||
## Versioning
|
||||
|
||||
The package version is derived from git tags via `hatch-vcs`. Tags follow the format `v{MAJOR}.{MINOR}.{PATCH}`.
|
||||
|
||||
Docker images are built with the version injected at build time:
|
||||
|
||||
```
|
||||
SETUPTOOLS_SCM_PRETEND_VERSION=<version> uv sync --frozen
|
||||
```
|
||||
|
||||
The `Makefile` provides `patch`, `minor`, and `major` targets that tag the current commit and rebuild the container image.
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
||||
"git.unkin.net/unkin/artifactapi/internal/server"
|
||||
"git.unkin.net/unkin/artifactapi/internal/tui"
|
||||
)
|
||||
|
||||
var version = "dev"
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == "tui" {
|
||||
endpoint := os.Getenv("ARTIFACTAPI_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
endpoint = "http://localhost:8000"
|
||||
}
|
||||
for i, arg := range os.Args {
|
||||
if arg == "--endpoint" && i+1 < len(os.Args) {
|
||||
endpoint = os.Args[i+1]
|
||||
}
|
||||
}
|
||||
|
||||
app := tui.New(endpoint)
|
||||
if err := app.Run(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "tui error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
slog.Error("failed to load config", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
srv, err := server.New(cfg, version)
|
||||
if err != nil {
|
||||
slog.Error("failed to create server", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := srv.Run(ctx); err != nil {
|
||||
slog.Error("server exited with error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
# Overlay for the dockerised end-to-end suite (scripts/docker-e2e.sh).
|
||||
# Adds a static mock upstream that the artifactapi container proxies, so the
|
||||
# caching tests are hermetic and need no internet access.
|
||||
services:
|
||||
mockupstream:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./e2e-docker/fixtures:/usr/share/nginx/html:ro,z
|
||||
# No host port needed: only the artifactapi container talks to it, and the
|
||||
# tests compare served bytes against the on-disk fixtures.
|
||||
|
||||
artifactapi:
|
||||
# The host port is set via ARTIFACTAPI_PORT (see scripts/docker-e2e.sh),
|
||||
# defaulting to 8000; the e2e run uses 8001 to avoid colliding with a
|
||||
# locally-running instance.
|
||||
depends_on:
|
||||
mockupstream:
|
||||
condition: service_started
|
||||
+56
-56
@@ -1,31 +1,22 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
artifactapi:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
- VERSION=2.2.2.dev0
|
||||
build: .
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- ./remotes.yaml:/app/remotes.yaml:ro,z
|
||||
- ./ca-bundle.pem:/app/ca-bundle.pem:ro,z
|
||||
- "${ARTIFACTAPI_PORT:-8000}:8000"
|
||||
environment:
|
||||
- CONFIG_PATH=/app/remotes.yaml
|
||||
- DBHOST=postgres
|
||||
- DBPORT=5432
|
||||
- DBUSER=artifacts
|
||||
- DBPASS=artifacts123
|
||||
- DBNAME=artifacts
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- MINIO_ENDPOINT=minio:9000
|
||||
- MINIO_ACCESS_KEY=minioadmin
|
||||
- MINIO_SECRET_KEY=minioadmin
|
||||
- MINIO_BUCKET=artifacts
|
||||
- MINIO_SECURE=false
|
||||
- REQUESTS_CA_BUNDLE=/app/ca-bundle.pem
|
||||
LISTEN_ADDR: ":8000"
|
||||
DBHOST: postgres
|
||||
DBPORT: "5432"
|
||||
DBUSER: artifacts
|
||||
DBPASS: artifacts123
|
||||
DBNAME: artifacts
|
||||
DBSSL: disable
|
||||
REDIS_URL: redis://redis:6379
|
||||
MINIO_ENDPOINT: minio:9000
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
MINIO_BUCKET: artifacts
|
||||
MINIO_SECURE: "false"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -34,27 +25,35 @@ services:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8000/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
ui:
|
||||
build:
|
||||
context: ui
|
||||
dockerfile: Dockerfile.ui
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
- "8080:80"
|
||||
depends_on:
|
||||
- artifactapi
|
||||
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
ports:
|
||||
- "5432:5432"
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
command: server /data --console-address ":9001"
|
||||
POSTGRES_USER: artifacts
|
||||
POSTGRES_PASSWORD: artifacts123
|
||||
POSTGRES_DB: artifacts
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
test: ["CMD-SHELL", "pg_isready -U artifacts -d artifacts"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
@@ -65,27 +64,28 @@ services:
|
||||
command: redis-server --save 20 1
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
ports:
|
||||
- "5432:5432"
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
POSTGRES_DB: artifacts
|
||||
POSTGRES_USER: artifacts
|
||||
POSTGRES_PASSWORD: artifacts123
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U artifacts -d artifacts"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
volumes:
|
||||
minio_data:
|
||||
redis_data:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Dockerised end-to-end suite
|
||||
|
||||
Black-box tests that run against a fully **containerised** artifactapi stack
|
||||
(built image + Postgres + Redis + MinIO) plus a static mock upstream. Unlike the
|
||||
in-process `e2e/` suite (testcontainers, server run in-process), these only speak
|
||||
HTTP to the running product, so they exercise the shipped container image.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
make docker-e2e # build image, compose up, run suite, compose down
|
||||
```
|
||||
|
||||
`scripts/docker-e2e.sh` builds and starts `docker-compose.yml` +
|
||||
`docker-compose.e2e.yml`, waits for `/health`, then runs
|
||||
`go test -tags=dockere2e ./e2e-docker/...` and tears everything down.
|
||||
|
||||
The stack publishes artifactapi on host port **8001** (to avoid colliding with a
|
||||
local instance on 8000). Override with `ARTIFACTAPI_URL` to point the tests at an
|
||||
already-running stack.
|
||||
|
||||
## Coverage
|
||||
|
||||
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
|
||||
- **Caching** — one immutable artifact per remote package type (generic, docker,
|
||||
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
|
||||
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
||||
verified against the origin fixture.
|
||||
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
|
||||
index), rpm (real package + **automatic repodata** generation).
|
||||
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
|
||||
across two members.
|
||||
|
||||
## Fixtures
|
||||
|
||||
`fixtures/` is served by the mock upstream at its web root. Paths mirror each
|
||||
provider's upstream URL layout (e.g. `v2/...` for docker, `v1/providers/...` for
|
||||
terraform). The RPM under `fixtures/rpmrepo/Packages/` is a real package so the
|
||||
rpm provider can parse its metadata for repodata generation.
|
||||
@@ -0,0 +1,76 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestCachingPerProvider proxies one immutable artifact for every remote
|
||||
// package type through the mock upstream and asserts: first fetch is served
|
||||
// from the remote, the second from cache, and the bytes match the origin.
|
||||
func TestCachingPerProvider(t *testing.T) {
|
||||
cases := []struct {
|
||||
pkgType string
|
||||
// path is the request path under /api/v1/remote/<name>/. The provider
|
||||
// derives the upstream URL from it (docker prepends /v2/, terraform
|
||||
// prepends /v1/providers/), and the fixture lives at that resolved path.
|
||||
path string
|
||||
fixture string
|
||||
}{
|
||||
{"generic", "blobs/hello.bin", "blobs/hello.bin"},
|
||||
{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"},
|
||||
{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"},
|
||||
{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"},
|
||||
{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"},
|
||||
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
|
||||
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
|
||||
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
|
||||
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
|
||||
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.pkgType, func(t *testing.T) {
|
||||
name := "cache-" + tc.pkgType
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": %q,
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"stale_on_error": true
|
||||
}`, name, tc.pkgType, mockUpstream()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
want := fixtureBytes(t, tc.fixture)
|
||||
url := api("/api/v1/remote/" + name + "/" + tc.path)
|
||||
|
||||
// First fetch: from remote.
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("first fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if src := resp.Header.Get("X-Artifact-Source"); src != "remote" {
|
||||
t.Fatalf("first fetch source = %q, want remote", src)
|
||||
}
|
||||
if !bytes.Equal(body, want) {
|
||||
t.Fatalf("first fetch body mismatch: got %d bytes, want %d", len(body), len(want))
|
||||
}
|
||||
|
||||
// Second fetch: from cache, identical bytes.
|
||||
resp, body = doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("second fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if src := resp.Header.Get("X-Artifact-Source"); src != "cache" {
|
||||
t.Fatalf("second fetch source = %q, want cache", src)
|
||||
}
|
||||
if !bytes.Equal(body, want) {
|
||||
t.Fatalf("cached body mismatch: got %d bytes, want %d", len(body), len(want))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
hello artifactapi generic blob
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,8 @@
|
||||
apiVersion: v1
|
||||
entries:
|
||||
alpha:
|
||||
- name: alpha
|
||||
version: 1.0.0
|
||||
urls:
|
||||
- charts/alpha-1.0.0.tgz
|
||||
generated: "2026-01-01T00:00:00Z"
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: v1
|
||||
entries:
|
||||
beta:
|
||||
- name: beta
|
||||
version: 2.0.0
|
||||
urls:
|
||||
- charts/beta-2.0.0.tgz
|
||||
generated: "2026-01-01T00:00:00Z"
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,108 @@
|
||||
//go:build dockere2e
|
||||
|
||||
// Package e2edocker holds the black-box end-to-end suite that runs against a
|
||||
// fully dockerised artifactapi stack (see scripts/docker-e2e.sh). Unlike the
|
||||
// in-process e2e suite, these tests only speak HTTP to the running container.
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func baseURL() string {
|
||||
if v := os.Getenv("ARTIFACTAPI_URL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://localhost:8000"
|
||||
}
|
||||
|
||||
// mockUpstream is the base URL the artifactapi *container* uses to reach the
|
||||
// static mock upstream. It is resolved on the compose network, not the host.
|
||||
func mockUpstream() string {
|
||||
if v := os.Getenv("MOCK_UPSTREAM_INTERNAL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://mockupstream"
|
||||
}
|
||||
|
||||
func api(path string) string { return baseURL() + path }
|
||||
|
||||
func fixtureBytes(t *testing.T, rel string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join("fixtures", rel))
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture %s: %v", rel, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func doRequest(t *testing.T, method, url string, body []byte, contentType string) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
var r io.Reader
|
||||
if body != nil {
|
||||
r = bytes.NewReader(body)
|
||||
}
|
||||
req, err := http.NewRequest(method, url, r)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, url, err)
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return resp, respBody
|
||||
}
|
||||
|
||||
func createRepo(t *testing.T, jsonBody string) {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodPost, api("/api/v2/remotes"), []byte(jsonBody), "application/json")
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create repo: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRepo(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
doRequest(t, http.MethodDelete, api("/api/v2/remotes/"+name), nil, "")
|
||||
}
|
||||
|
||||
func createVirtual(t *testing.T, jsonBody string) {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodPost, api("/api/v2/virtuals"), []byte(jsonBody), "application/json")
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteVirtual(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
doRequest(t, http.MethodDelete, api("/api/v2/virtuals/"+name), nil, "")
|
||||
}
|
||||
|
||||
// getEventually retries a GET until it returns 200 or the deadline passes. Used
|
||||
// for asynchronously-generated artifacts (e.g. rpm repodata after upload).
|
||||
func getEventually(t *testing.T, url string, timeout time.Duration) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
var resp *http.Response
|
||||
var body []byte
|
||||
for {
|
||||
resp, body = doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode == http.StatusOK || time.Now().After(deadline) {
|
||||
return resp, body
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func uploadFile(t *testing.T, repo, filePath string, body []byte, contentType string) {
|
||||
t.Helper()
|
||||
url := api("/api/v2/remotes/" + repo + "/files/" + filePath)
|
||||
resp, respBody := doRequest(t, http.MethodPut, url, body, contentType)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s: status %d: %s", filePath, resp.StatusCode, respBody)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalGenericUpload uploads a generic file and downloads it back.
|
||||
func TestLocalGenericUpload(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-generic","package_type":"generic","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-generic")
|
||||
|
||||
content := []byte("artifactapi local generic upload payload")
|
||||
uploadFile(t, "local-generic", "data/hello.bin", content, "application/octet-stream")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-generic/data/hello.bin"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, content) {
|
||||
t.Fatalf("downloaded content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalPyPIUpload uploads a wheel and validates the generated simple index.
|
||||
func TestLocalPyPIUpload(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-pypi","package_type":"pypi","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-pypi")
|
||||
|
||||
wheel := fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl")
|
||||
uploadFile(t, "local-pypi", "foo-1.0-py3-none-any.whl", wheel, "application/zip")
|
||||
|
||||
// Root index lists the package.
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("simple index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !strings.Contains(string(body), "foo") {
|
||||
t.Fatalf("simple index missing package 'foo': %s", body)
|
||||
}
|
||||
|
||||
// Per-package index lists the wheel file.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/foo/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("package index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !strings.Contains(string(body), "foo-1.0-py3-none-any.whl") {
|
||||
t.Fatalf("package index missing wheel: %s", body)
|
||||
}
|
||||
|
||||
// The wheel downloads back byte-identical.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/foo/foo-1.0-py3-none-any.whl"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download wheel: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, wheel) {
|
||||
t.Fatalf("wheel content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalRPMRepodata uploads a real RPM and validates that repodata is
|
||||
// generated automatically (the special rpm-local feature).
|
||||
func TestLocalRPMRepodata(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-rpm","package_type":"rpm","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-rpm")
|
||||
|
||||
rpm := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
|
||||
uploadFile(t, "local-rpm", "e2e-testpkg-1.0-1.noarch.rpm", rpm, "application/x-rpm")
|
||||
|
||||
// repodata is generated asynchronously after upload; poll for it.
|
||||
resp, body := getEventually(t, api("/api/v1/local/local-rpm/repodata/repomd.xml"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("repomd.xml: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "<repomd") || !strings.Contains(s, "primary") {
|
||||
t.Fatalf("repomd.xml not a valid repodata document: %s", s)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHealth(t *testing.T) {
|
||||
resp, body := doRequest(t, http.MethodGet, api("/health"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("health: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRemoteLifecycle covers add/change/delete for a remote repository.
|
||||
func TestRemoteLifecycle(t *testing.T) {
|
||||
createRepo(t, `{
|
||||
"name": "crud-remote",
|
||||
"package_type": "generic",
|
||||
"repo_type": "remote",
|
||||
"base_url": "https://example.com",
|
||||
"mutable_ttl": 600,
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRepo(t, "crud-remote")
|
||||
|
||||
got := getRepo(t, "crud-remote")
|
||||
if got["base_url"] != "https://example.com" || got["mutable_ttl"].(float64) != 600 {
|
||||
t.Fatalf("unexpected created remote: %v", got)
|
||||
}
|
||||
|
||||
// change
|
||||
resp, body := doRequest(t, http.MethodPut, api("/api/v2/remotes/crud-remote"), []byte(`{
|
||||
"package_type": "generic",
|
||||
"base_url": "https://updated.example.com",
|
||||
"mutable_ttl": 120,
|
||||
"stale_on_error": true
|
||||
}`), "application/json")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update remote: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
got = getRepo(t, "crud-remote")
|
||||
if got["base_url"] != "https://updated.example.com" || got["mutable_ttl"].(float64) != 120 {
|
||||
t.Fatalf("update not applied: %v", got)
|
||||
}
|
||||
|
||||
// delete
|
||||
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-remote"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete remote: status %d", resp.StatusCode)
|
||||
}
|
||||
resp, _ = doRequest(t, http.MethodGet, api("/api/v2/remotes/crud-remote"), nil, "")
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("expected 404 after delete, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalLifecycle covers add/delete for a local repository.
|
||||
func TestLocalLifecycle(t *testing.T) {
|
||||
createRepo(t, `{
|
||||
"name": "crud-local",
|
||||
"package_type": "generic",
|
||||
"repo_type": "local"
|
||||
}`)
|
||||
defer deleteRepo(t, "crud-local")
|
||||
|
||||
got := getRepo(t, "crud-local")
|
||||
if got["repo_type"] != "local" {
|
||||
t.Fatalf("expected repo_type local, got %v", got["repo_type"])
|
||||
}
|
||||
|
||||
resp, _ := doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-local"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete local: status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVirtualLifecycle covers add/change/delete for a virtual repository.
|
||||
func TestVirtualLifecycle(t *testing.T) {
|
||||
createRepo(t, `{"name":"vmem-a","package_type":"helm","repo_type":"remote","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRepo(t, `{"name":"vmem-b","package_type":"helm","repo_type":"remote","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRepo(t, "vmem-a")
|
||||
defer deleteRepo(t, "vmem-b")
|
||||
|
||||
createVirtual(t, `{
|
||||
"name": "crud-virtual",
|
||||
"package_type": "helm",
|
||||
"members": ["vmem-a"]
|
||||
}`)
|
||||
defer deleteVirtual(t, "crud-virtual")
|
||||
|
||||
// change members
|
||||
resp, body := doRequest(t, http.MethodPut, api("/api/v2/virtuals/crud-virtual"), []byte(`{
|
||||
"package_type": "helm",
|
||||
"members": ["vmem-a", "vmem-b"]
|
||||
}`), "application/json")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v2/virtuals/crud-virtual"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("get virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
var v map[string]any
|
||||
if err := json.Unmarshal(body, &v); err != nil {
|
||||
t.Fatalf("decode virtual: %v", err)
|
||||
}
|
||||
members, _ := v["members"].([]any)
|
||||
if len(members) != 2 {
|
||||
t.Fatalf("expected 2 members after update, got %v", v["members"])
|
||||
}
|
||||
|
||||
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/virtuals/crud-virtual"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete virtual: status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func getRepo(t *testing.T, name string) map[string]any {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v2/remotes/"+name), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("get remote %s: status %d: %s", name, resp.StatusCode, body)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
t.Fatalf("decode remote %s: %v", name, err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
|
||||
// checks that a virtual over them serves a merged simple index.
|
||||
func TestVirtualPyPIMerge(t *testing.T) {
|
||||
createRepo(t, `{"name":"pmerge-a","package_type":"pypi","repo_type":"local"}`)
|
||||
createRepo(t, `{"name":"pmerge-b","package_type":"pypi","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "pmerge-a")
|
||||
defer deleteRepo(t, "pmerge-b")
|
||||
|
||||
uploadFile(t, "pmerge-a", "foo-1.0-py3-none-any.whl", fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl"), "application/zip")
|
||||
uploadFile(t, "pmerge-b", "bar-2.0-py3-none-any.whl", []byte("bar wheel payload"), "application/zip")
|
||||
|
||||
createVirtual(t, `{"name":"pmerge-v","package_type":"pypi","members":["pmerge-a","pmerge-b"]}`)
|
||||
defer deleteVirtual(t, "pmerge-v")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/pmerge-v/simple/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("virtual simple index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "foo") || !strings.Contains(s, "bar") {
|
||||
t.Fatalf("merged index missing a member package (want foo and bar): %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVirtualHelmMerge points two helm remotes at mock index.yaml documents
|
||||
// with distinct charts and checks the virtual merges both into one index.
|
||||
func TestVirtualHelmMerge(t *testing.T) {
|
||||
createRepo(t, `{"name":"hmerge-a","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-a","stale_on_error":true}`)
|
||||
createRepo(t, `{"name":"hmerge-b","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-b","stale_on_error":true}`)
|
||||
defer deleteRepo(t, "hmerge-a")
|
||||
defer deleteRepo(t, "hmerge-b")
|
||||
|
||||
createVirtual(t, `{"name":"hmerge-v","package_type":"helm","members":["hmerge-a","hmerge-b"]}`)
|
||||
defer deleteVirtual(t, "hmerge-v")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/hmerge-v/index.yaml"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("virtual index.yaml: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "alpha") || !strings.Contains(s, "beta") {
|
||||
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
|
||||
}
|
||||
}
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres"
|
||||
tcredis "github.com/testcontainers/testcontainers-go/modules/redis"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
||||
"git.unkin.net/unkin/artifactapi/internal/server"
|
||||
)
|
||||
|
||||
var baseURL string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
|
||||
pgContainer, err := tcpostgres.Run(ctx,
|
||||
"postgres:17-alpine",
|
||||
tcpostgres.WithDatabase("artifacts"),
|
||||
tcpostgres.WithUsername("artifacts"),
|
||||
tcpostgres.WithPassword("artifacts123"),
|
||||
testcontainers.WithWaitStrategy(
|
||||
wait.ForListeningPort("5432/tcp").WithStartupTimeout(30*time.Second),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer pgContainer.Terminate(ctx)
|
||||
|
||||
redisContainer, err := tcredis.Run(ctx,
|
||||
"redis:7-alpine",
|
||||
testcontainers.WithWaitStrategy(
|
||||
wait.ForListeningPort("6379/tcp").WithStartupTimeout(30*time.Second),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("redis: %v", err)
|
||||
}
|
||||
defer redisContainer.Terminate(ctx)
|
||||
|
||||
minioContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
ContainerRequest: testcontainers.ContainerRequest{
|
||||
Image: "minio/minio:latest",
|
||||
ExposedPorts: []string{"9000/tcp"},
|
||||
Cmd: []string{"server", "/data"},
|
||||
Env: map[string]string{
|
||||
"MINIO_ROOT_USER": "minioadmin",
|
||||
"MINIO_ROOT_PASSWORD": "minioadmin",
|
||||
},
|
||||
WaitingFor: wait.ForHTTP("/minio/health/live").WithPort("9000/tcp").WithStartupTimeout(30 * time.Second),
|
||||
},
|
||||
Started: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("minio: %v", err)
|
||||
}
|
||||
defer minioContainer.Terminate(ctx)
|
||||
|
||||
pgHost, _ := pgContainer.Host(ctx)
|
||||
pgPort, _ := pgContainer.MappedPort(ctx, "5432/tcp")
|
||||
redisHost, _ := redisContainer.Host(ctx)
|
||||
redisPort, _ := redisContainer.MappedPort(ctx, "6379/tcp")
|
||||
minioHost, _ := minioContainer.Host(ctx)
|
||||
minioPort, _ := minioContainer.MappedPort(ctx, "9000/tcp")
|
||||
|
||||
os.Setenv("DBHOST", pgHost)
|
||||
os.Setenv("DBPORT", pgPort.Port())
|
||||
os.Setenv("DBUSER", "artifacts")
|
||||
os.Setenv("DBPASS", "artifacts123")
|
||||
os.Setenv("DBNAME", "artifacts")
|
||||
os.Setenv("DBSSL", "disable")
|
||||
os.Setenv("REDIS_URL", fmt.Sprintf("redis://%s:%s", redisHost, redisPort.Port()))
|
||||
os.Setenv("MINIO_ENDPOINT", fmt.Sprintf("%s:%s", minioHost, minioPort.Port()))
|
||||
os.Setenv("MINIO_ACCESS_KEY", "minioadmin")
|
||||
os.Setenv("MINIO_SECRET_KEY", "minioadmin")
|
||||
os.Setenv("MINIO_BUCKET", "artifacts-test")
|
||||
os.Setenv("MINIO_SECURE", "false")
|
||||
os.Setenv("LISTEN_ADDR", ":0")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
cfg.ListenAddr = "127.0.0.1:0"
|
||||
|
||||
srv, err := server.New(cfg, "e2e-test")
|
||||
if err != nil {
|
||||
log.Fatalf("server: %v", err)
|
||||
}
|
||||
|
||||
srvCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
addr := startServer(srvCtx, srv)
|
||||
baseURL = "http://" + addr
|
||||
|
||||
code := m.Run()
|
||||
cancel()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func startServer(ctx context.Context, srv *server.Server) string {
|
||||
ln, err := findListener()
|
||||
if err != nil {
|
||||
log.Fatalf("listener: %v", err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
|
||||
go srv.RunOnListener(ctx, ln)
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := http.Get("http://" + addr + "/health")
|
||||
if err == nil && resp.StatusCode == 200 {
|
||||
resp.Body.Close()
|
||||
return addr
|
||||
}
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
log.Fatalf("server did not start in time at %s", addr)
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func findListener() (net.Listener, error) {
|
||||
return net.Listen("tcp", "127.0.0.1:0")
|
||||
}
|
||||
|
||||
func apiURL(path string) string {
|
||||
return baseURL + path
|
||||
}
|
||||
|
||||
func createRemote(t *testing.T, body string) {
|
||||
t.Helper()
|
||||
resp, err := http.Post(apiURL("/api/v2/remotes"), "application/json", strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("create remote: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("create remote: status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRemote(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(http.MethodDelete, apiURL("/api/v2/remotes/"+name), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("delete remote: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
func getJSON(t *testing.T, url string) map[string]any {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("GET %s: status %d: %s", url, resp.StatusCode, b)
|
||||
}
|
||||
var result map[string]any
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func getBody(t *testing.T, url string) ([]byte, int) {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return b, resp.StatusCode
|
||||
}
|
||||
|
||||
func getString(t *testing.T, url string) string {
|
||||
t.Helper()
|
||||
b, status := getBody(t, url)
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("GET %s: status %d: %s", url, status, b)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func assertStatus(t *testing.T, url string, wantStatus int) {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != wantStatus {
|
||||
t.Errorf("GET %s: got %d, want %d", url, resp.StatusCode, wantStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRequest(t *testing.T, url string) int {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(http.MethodDelete, url, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("DELETE %s: %v", url, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
func mustFormat(format string, args ...any) string {
|
||||
return fmt.Sprintf(format, args...)
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHealth(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/health"))
|
||||
if result["status"] != "ok" {
|
||||
t.Errorf("expected ok, got %v", result["status"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoot(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/"))
|
||||
if result["name"] != "artifactapi" {
|
||||
t.Errorf("expected artifactapi, got %v", result["name"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUpstreamTimeouts(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "timeout-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"stale_on_error": true,
|
||||
"upstream_dial_timeout": 3,
|
||||
"upstream_tls_timeout": 4,
|
||||
"upstream_response_header_timeout": 5
|
||||
}`)
|
||||
defer deleteRemote(t, "timeout-test")
|
||||
|
||||
remote := getJSON(t, apiURL("/api/v2/remotes/timeout-test"))
|
||||
for field, want := range map[string]float64{
|
||||
"upstream_dial_timeout": 3,
|
||||
"upstream_tls_timeout": 4,
|
||||
"upstream_response_header_timeout": 5,
|
||||
} {
|
||||
if got, _ := remote[field].(float64); got != want {
|
||||
t.Errorf("%s: got %v, want %v", field, remote[field], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteCRUD(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "test-generic",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"description": "test remote",
|
||||
"mutable_ttl": 600,
|
||||
"check_mutable": true,
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "test-generic")
|
||||
|
||||
remote := getJSON(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if remote["name"] != "test-generic" {
|
||||
t.Errorf("expected test-generic, got %v", remote["name"])
|
||||
}
|
||||
if remote["package_type"] != "generic" {
|
||||
t.Errorf("expected generic, got %v", remote["package_type"])
|
||||
}
|
||||
|
||||
req, _ := http.NewRequest(http.MethodPut, apiURL("/api/v2/remotes/test-generic"),
|
||||
strings.NewReader(`{
|
||||
"package_type": "generic",
|
||||
"base_url": "https://updated.example.com",
|
||||
"description": "updated",
|
||||
"mutable_ttl": 300,
|
||||
"check_mutable": true,
|
||||
"stale_on_error": true
|
||||
}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
updated := getJSON(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if updated["base_url"] != "https://updated.example.com" {
|
||||
t.Errorf("expected updated URL, got %v", updated["base_url"])
|
||||
}
|
||||
|
||||
status := deleteRequest(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if status != http.StatusNoContent {
|
||||
t.Errorf("delete: got %d, want 204", status)
|
||||
}
|
||||
|
||||
assertStatus(t, apiURL("/api/v2/remotes/test-generic"), http.StatusNotFound)
|
||||
}
|
||||
|
||||
func TestRemoteList(t *testing.T) {
|
||||
createRemote(t, `{"name":"list-a","package_type":"generic","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRemote(t, `{"name":"list-b","package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRemote(t, "list-a")
|
||||
defer deleteRemote(t, "list-b")
|
||||
|
||||
resp, err := http.Get(apiURL("/api/v2/remotes"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
|
||||
var remotes []map[string]any
|
||||
json.Unmarshal(body, &remotes)
|
||||
|
||||
if len(remotes) < 2 {
|
||||
t.Fatalf("expected at least 2 remotes, got %d", len(remotes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualCRUD(t *testing.T) {
|
||||
createRemote(t, `{"name":"virt-member-a","package_type":"helm","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRemote(t, `{"name":"virt-member-b","package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRemote(t, "virt-member-a")
|
||||
defer deleteRemote(t, "virt-member-b")
|
||||
|
||||
resp, err := http.Post(apiURL("/api/v2/virtuals"), "application/json",
|
||||
strings.NewReader(`{
|
||||
"name": "test-virtual",
|
||||
"package_type": "helm",
|
||||
"description": "test virtual",
|
||||
"members": ["virt-member-a", "virt-member-b"]
|
||||
}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create virtual: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
virt := getJSON(t, apiURL("/api/v2/virtuals/test-virtual"))
|
||||
if virt["name"] != "test-virtual" {
|
||||
t.Errorf("expected test-virtual, got %v", virt["name"])
|
||||
}
|
||||
|
||||
status := deleteRequest(t, apiURL("/api/v2/virtuals/test-virtual"))
|
||||
if status != http.StatusNoContent {
|
||||
t.Errorf("delete virtual: got %d, want 204", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsEndpoint(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/api/v2/stats"))
|
||||
if _, ok := result["total_remotes"]; !ok {
|
||||
t.Error("expected total_remotes in stats")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthV2(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/api/v2/health"))
|
||||
if result["status"] != "ok" {
|
||||
t.Errorf("expected ok, got %v", result["status"])
|
||||
}
|
||||
if result["postgres"] != "ok" {
|
||||
t.Errorf("expected postgres ok, got %v", result["postgres"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidPackageType(t *testing.T) {
|
||||
resp, err := http.Post(apiURL("/api/v2/remotes"), "application/json",
|
||||
strings.NewReader(`{"name":"bad","package_type":"bogus","base_url":"https://x.com"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("expected 400 for invalid package type, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProxyUnknownRemote(t *testing.T) {
|
||||
assertStatus(t, apiURL("/api/v1/remote/nonexistent/some/path"), http.StatusNotFound)
|
||||
}
|
||||
|
||||
func TestProxyBlocklist(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "blocklist-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"blocklist": ["\\.exe$"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "blocklist-test")
|
||||
|
||||
assertStatus(t, apiURL("/api/v1/remote/blocklist-test/malware.exe"), http.StatusForbidden)
|
||||
}
|
||||
|
||||
func TestProxyHeadBlocklist(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "head-block-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"blocklist": ["\\.exe$"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "head-block-test")
|
||||
|
||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/head-block-test/malware.exe"), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("HEAD blocklisted path: got %d, want 403", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyHeadUnknownRemote(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/nonexistent/some/path"), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("HEAD unknown remote: got %d, want 404", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyPatterns(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "patterns-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"patterns": ["^releases/"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "patterns-test")
|
||||
|
||||
assertStatus(t, apiURL("/api/v1/remote/patterns-test/uploads/file.tar.gz"), http.StatusForbidden)
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
module git.unkin.net/unkin/artifactapi
|
||||
|
||||
go 1.25.9
|
||||
|
||||
require (
|
||||
github.com/cavaliergopher/rpm v1.3.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/go-chi/chi/v5 v5.3.0
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/minio/minio-go/v7 v7.2.0
|
||||
github.com/redis/go-redis/v9 v9.20.0
|
||||
github.com/testcontainers/testcontainers-go v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.4.3 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.11.7 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
|
||||
github.com/charmbracelet/x/term v0.2.2 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.11.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v0.2.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.23 // indirect
|
||||
github.com/mdelapenya/tlscert v0.2.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/moby/api v1.54.1 // indirect
|
||||
github.com/moby/moby/client v0.4.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/stretchr/testify v1.11.1 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.51.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.2 // indirect
|
||||
)
|
||||
@@ -0,0 +1,249 @@
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cavaliergopher/rpm v1.3.0 h1:UHX46sasX8MesUXXQ+UbkFLUX4eUWTlEcX8jcnRBIgI=
|
||||
github.com/cavaliergopher/rpm v1.3.0/go.mod h1:vEumo1vvtrHM1Ov86f6+k8j7zNKOxQfHDCAIcR/36ZI=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
|
||||
github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
|
||||
github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=
|
||||
github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q=
|
||||
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
|
||||
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
|
||||
github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI=
|
||||
github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
|
||||
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
|
||||
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
|
||||
github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8=
|
||||
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
|
||||
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw=
|
||||
github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
|
||||
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs=
|
||||
github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
||||
github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4=
|
||||
github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||
github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw=
|
||||
github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g=
|
||||
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
|
||||
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
|
||||
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
|
||||
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0=
|
||||
github.com/redis/go-redis/v9 v9.20.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
||||
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
|
||||
github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 h1:id/6LH8ZeDrtAUVSuNvZUAJ1kVpb82y1pr9yweAWsRg=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0/go.mod h1:uF0jI8FITagQpBNOgweGBmPf6rP4K0SeL1XFPbsZSSY=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
||||
go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
|
||||
go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
|
||||
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss=
|
||||
gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||
pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
|
||||
pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||
@@ -0,0 +1,209 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
type ProxyHandler struct {
|
||||
engine *proxy.Engine
|
||||
virtualEngine *virtual.Engine
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
local *v2.LocalHandler
|
||||
}
|
||||
|
||||
func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *database.DB, store *storage.S3, local *v2.LocalHandler) *ProxyHandler {
|
||||
return &ProxyHandler{engine: engine, virtualEngine: virtualEngine, db: db, store: store, local: local}
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/remote/{remoteName}/*", h.handleProxy)
|
||||
r.Get("/local/{localName}/*", h.handleLocal)
|
||||
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) DockerV2Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.handleDockerPing)
|
||||
r.Head("/", h.handleDockerPing)
|
||||
r.Get("/{remoteName}/*", h.handleProxy)
|
||||
r.Head("/{remoteName}/*", h.handleProxyHead)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Docker-Distribution-Api-Version", "registry/2.0")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy fetch failed", "remote", remoteName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer result.Reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, result.Reader)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.engine.Head(r.Context(), *remote, path, prov)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
||||
virtualName := chi.URLParam(r, "virtualName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
virt, err := h.db.GetVirtual(r.Context(), virtualName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("virtual %q not found", virtualName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||
|
||||
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
||||
if err != nil {
|
||||
slog.Error("virtual fetch failed", "virtual", virtualName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("X-Artifact-Source", "virtual")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
|
||||
localName := chi.URLParam(r, "localName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), localName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("local %q not found", localName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
if indexer, ok := prov.(provider.LocalIndexer); ok {
|
||||
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
h.serveLocalFile(w, r, localName, path)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, repoName, path string) {
|
||||
file, err := h.db.GetLocalFile(r.Context(), repoName, path)
|
||||
if err != nil {
|
||||
slog.Error("local file lookup failed", "repo", repoName, "path", path, "error", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if file == nil {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):])
|
||||
reader, info, err := h.store.Download(r.Context(), s3Key)
|
||||
if err != nil {
|
||||
slog.Error("local file download failed", "repo", repoName, "path", path, "error", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", info.ContentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func scheme(r *http.Request) string {
|
||||
if r.TLS != nil {
|
||||
return "https"
|
||||
}
|
||||
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
|
||||
return fwd
|
||||
}
|
||||
return "http"
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
type EventsHandler struct{}
|
||||
|
||||
func NewEventsHandler() *EventsHandler {
|
||||
return &EventsHandler{}
|
||||
}
|
||||
|
||||
func (h *EventsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.stream)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *EventsHandler) stream(w http.ResponseWriter, r *http.Request) {
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
http.Error(w, "streaming not supported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("X-Accel-Buffering", "no")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
flusher.Flush()
|
||||
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
fmt.Fprintf(w, ": keepalive\n\n")
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
)
|
||||
|
||||
type HealthHandler struct {
|
||||
db *database.DB
|
||||
cache *cache.Redis
|
||||
store *storage.S3
|
||||
}
|
||||
|
||||
func NewHealthHandler(db *database.DB, c *cache.Redis, s *storage.S3) *HealthHandler {
|
||||
return &HealthHandler{db: db, cache: c, store: s}
|
||||
}
|
||||
|
||||
func (h *HealthHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.health)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *HealthHandler) health(w http.ResponseWriter, r *http.Request) {
|
||||
status := map[string]string{
|
||||
"status": "ok",
|
||||
"postgres": "ok",
|
||||
"redis": "ok",
|
||||
"s3": "ok",
|
||||
}
|
||||
|
||||
if err := h.db.Pool.Ping(r.Context()); err != nil {
|
||||
status["postgres"] = "error"
|
||||
status["status"] = "degraded"
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type LocalHandler struct {
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
cas *storage.CAS
|
||||
}
|
||||
|
||||
func NewLocalHandler(db *database.DB, store *storage.S3) *LocalHandler {
|
||||
return &LocalHandler{
|
||||
db: db,
|
||||
store: store,
|
||||
cas: storage.NewCAS(store),
|
||||
}
|
||||
}
|
||||
|
||||
func (h *LocalHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Put("/*", h.upload)
|
||||
r.Get("/*", h.download)
|
||||
r.Delete("/*", h.remove)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *LocalHandler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
if filePath == "" {
|
||||
http.Error(w, "file path required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", repoName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if remote.RepoType != models.RepoTypeLocal {
|
||||
http.Error(w, "upload only allowed for local repository types", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
|
||||
if uploader, ok := prov.(provider.LocalUploader); ok {
|
||||
h.uploadValidated(w, r, remote, filePath, prov, uploader)
|
||||
return
|
||||
}
|
||||
|
||||
h.uploadGeneric(w, r, remote, filePath)
|
||||
}
|
||||
|
||||
func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string, prov provider.Provider, uploader provider.LocalUploader) {
|
||||
storagePath, contentType, err := uploader.ValidateUpload(filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, storagePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if existing != nil {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.cas.Store(r.Context(), r.Body, contentType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.CreateLocalFile(r.Context(), remote.Name, storagePath, result.ContentHash); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if hook, ok := prov.(provider.PostUploadHook); ok {
|
||||
go hook.AfterUpload(context.Background(), remote.Name, storagePath, result.ContentHash, h, h.db)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, uploader.UploadResponse(storagePath, result.ContentHash, result.SizeBytes))
|
||||
}
|
||||
|
||||
func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) {
|
||||
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if existing != nil {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", filePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := "application/octet-stream"
|
||||
if ct := r.Header.Get("Content-Type"); ct != "" && ct != "application/octet-stream" {
|
||||
contentType = ct
|
||||
}
|
||||
|
||||
result, err := h.cas.Store(r.Context(), r.Body, contentType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.CreateLocalFile(r.Context(), remote.Name, filePath, result.ContentHash); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", filePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, map[string]any{
|
||||
"path": filePath,
|
||||
"content_hash": result.ContentHash,
|
||||
"size_bytes": result.SizeBytes,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *LocalHandler) download(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
file, err := h.db.GetLocalFile(r.Context(), repoName, filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if file == nil {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):])
|
||||
reader, info, err := h.store.Download(r.Context(), s3Key)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("download failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", info.ContentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func (h *LocalHandler) remove(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
if err := h.db.DeleteLocalFile(r.Context(), repoName, filePath); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *LocalHandler) DB() *database.DB {
|
||||
return h.db
|
||||
}
|
||||
|
||||
func (h *LocalHandler) Download(ctx context.Context, key string) (io.ReadCloser, int64, error) {
|
||||
reader, info, err := h.store.Download(ctx, key)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return reader, info.Size, nil
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
type ObjectsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewObjectsHandler(db *database.DB) *ObjectsHandler {
|
||||
return &ObjectsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Delete("/*", h.evict)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("per_page"))
|
||||
if limit <= 0 || limit > 5000 {
|
||||
limit = 50
|
||||
}
|
||||
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
offset := (page - 1) * limit
|
||||
|
||||
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, artifacts)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
if err := h.db.DeleteArtifact(r.Context(), remoteName, path); err != nil {
|
||||
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
)
|
||||
|
||||
type ProbeHandler struct {
|
||||
engine *proxy.Engine
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewProbeHandler(engine *proxy.Engine, db *database.DB) *ProbeHandler {
|
||||
return &ProbeHandler{engine: engine, db: db}
|
||||
}
|
||||
|
||||
func (h *ProbeHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Post("/", h.probe)
|
||||
return r
|
||||
}
|
||||
|
||||
type probeRequest struct {
|
||||
Remote string `json:"remote"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
type probeResponse struct {
|
||||
Status int `json:"status"`
|
||||
Source string `json:"source,omitempty"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (h *ProbeHandler) probe(w http.ResponseWriter, r *http.Request) {
|
||||
var req probeRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if req.Remote == "" || req.Path == "" {
|
||||
http.Error(w, "remote and path are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), req.Remote)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 404,
|
||||
Error: fmt.Sprintf("remote %q not found", req.Remote),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 500,
|
||||
Error: fmt.Sprintf("no provider for %q", remote.PackageType),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
result, err := h.engine.Fetch(ctx, *remote, req.Path, prov)
|
||||
duration := time.Since(start).Milliseconds()
|
||||
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 502,
|
||||
DurationMS: duration,
|
||||
Error: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
io.Copy(io.Discard, result.Reader)
|
||||
result.Reader.Close()
|
||||
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 200,
|
||||
Source: result.Source,
|
||||
ContentType: result.ContentType,
|
||||
SizeBytes: result.Size,
|
||||
DurationMS: duration,
|
||||
Headers: map[string]string{
|
||||
"X-Artifact-Source": result.Source,
|
||||
"X-Artifact-Size": fmt.Sprintf("%d", result.Size),
|
||||
"Content-Type": result.ContentType,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type RemotesHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewRemotesHandler(db *database.DB) *RemotesHandler {
|
||||
return &RemotesHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Post("/", h.create)
|
||||
r.Get("/{name}", h.get)
|
||||
r.Put("/{name}", h.update)
|
||||
r.Delete("/{name}", h.del)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remotes, err := h.db.ListRemotes(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remotes)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
remote, err := h.db.GetRemote(r.Context(), name)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", name), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
var remote models.Remote
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !remote.PackageType.Valid() {
|
||||
http.Error(w, fmt.Sprintf("invalid package type: %q", remote.PackageType), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if remote.RepoType == "" {
|
||||
remote.RepoType = models.RepoTypeRemote
|
||||
}
|
||||
if !remote.RepoType.Valid() {
|
||||
http.Error(w, fmt.Sprintf("invalid repo type: %q", remote.RepoType), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if remote.RepoType == models.RepoTypeRemote && remote.BaseURL == "" {
|
||||
http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.CreateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
var remote models.Remote
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
remote.Name = name
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
if err := h.db.DeleteRemote(r.Context(), name); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
type StatsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewStatsHandler(db *database.DB) *StatsHandler {
|
||||
return &StatsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *StatsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.overview)
|
||||
r.Get("/top-remotes", h.topRemotes)
|
||||
r.Get("/top-files-by-hits", h.topFilesByHits)
|
||||
r.Get("/top-files-by-bandwidth", h.topFilesByBandwidth)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *StatsHandler) overview(w http.ResponseWriter, r *http.Request) {
|
||||
stats, err := h.db.GetOverviewStats(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, stats)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topRemotes(w http.ResponseWriter, r *http.Request) {
|
||||
remotes, err := h.db.GetTopRemotes(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remotes)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topFilesByHits(w http.ResponseWriter, r *http.Request) {
|
||||
files, err := h.db.GetTopFilesByHits(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, files)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topFilesByBandwidth(w http.ResponseWriter, r *http.Request) {
|
||||
files, err := h.db.GetTopFilesByBandwidth(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, files)
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type VirtualsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewVirtualsHandler(db *database.DB) *VirtualsHandler {
|
||||
return &VirtualsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Post("/", h.create)
|
||||
r.Get("/{name}", h.get)
|
||||
r.Put("/{name}", h.update)
|
||||
r.Delete("/{name}", h.del)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
virtuals, err := h.db.ListVirtuals(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virtuals)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
virt, err := h.db.GetVirtual(r.Context(), name)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("virtual %q not found", name), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
var virt models.Virtual
|
||||
if err := json.NewDecoder(r.Body).Decode(&virt); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.CreateVirtual(r.Context(), &virt); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
var virt models.Virtual
|
||||
if err := json.NewDecoder(r.Body).Decode(&virt); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
virt.Name = name
|
||||
if err := h.db.UpdateVirtual(r.Context(), &virt); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) del(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
if err := h.db.DeleteVirtual(r.Context(), name); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func BasicHeaders(remote models.Remote) http.Header {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString(
|
||||
[]byte(remote.Username+":"+remote.Password),
|
||||
))
|
||||
}
|
||||
return h
|
||||
}
|
||||
Vendored
+117
@@ -0,0 +1,117 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type Redis struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
func NewRedis(url string) (*Redis, error) {
|
||||
opts, err := redis.ParseURL(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse redis url: %w", err)
|
||||
}
|
||||
|
||||
client := redis.NewClient(opts)
|
||||
|
||||
if err := client.Ping(context.Background()).Err(); err != nil {
|
||||
return nil, fmt.Errorf("ping redis: %w", err)
|
||||
}
|
||||
|
||||
return &Redis{client: client}, nil
|
||||
}
|
||||
|
||||
func (r *Redis) Close() error {
|
||||
return r.client.Close()
|
||||
}
|
||||
|
||||
func (r *Redis) SetTTL(ctx context.Context, remote, path string, ttl time.Duration) error {
|
||||
key := fmt.Sprintf("ttl:%s:%s", remote, path)
|
||||
return r.client.Set(ctx, key, "1", ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) CheckTTL(ctx context.Context, remote, path string) (bool, error) {
|
||||
key := fmt.Sprintf("ttl:%s:%s", remote, path)
|
||||
exists, err := r.client.Exists(ctx, key).Result()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exists > 0, nil
|
||||
}
|
||||
|
||||
func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (bool, error) {
|
||||
key := fmt.Sprintf("lock:%s:%s", remote, path)
|
||||
ok, err := r.client.SetNX(ctx, key, "1", ttl).Result()
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error {
|
||||
key := fmt.Sprintf("lock:%s:%s", remote, path)
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error {
|
||||
key := fmt.Sprintf("etag:%s:%s", remote, path)
|
||||
return r.client.Set(ctx, key, etag, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) GetETag(ctx context.Context, remote, path string) (string, error) {
|
||||
key := fmt.Sprintf("etag:%s:%s", remote, path)
|
||||
val, err := r.client.Get(ctx, key).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) GetToken(ctx context.Context, key string) (string, error) {
|
||||
val, err := r.client.Get(ctx, "token:"+key).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) SetToken(ctx context.Context, key, token string, ttl time.Duration) error {
|
||||
return r.client.Set(ctx, "token:"+key, token, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) IncrCircuitFailure(ctx context.Context, remote string, cooldown time.Duration) (int64, error) {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
pipe := r.client.Pipeline()
|
||||
incr := pipe.Incr(ctx, key)
|
||||
pipe.Expire(ctx, key, cooldown)
|
||||
_, err := pipe.Exec(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return incr.Val(), nil
|
||||
}
|
||||
|
||||
func (r *Redis) ResetCircuit(ctx context.Context, remote string) error {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) GetCircuitFailures(ctx context.Context, remote string) (int64, error) {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
val, err := r.client.Get(ctx, key).Int64()
|
||||
if err == redis.Nil {
|
||||
return 0, nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) FlushRemote(ctx context.Context, remote string) error {
|
||||
iter := r.client.Scan(ctx, 0, fmt.Sprintf("*:%s:*", remote), 100).Iterator()
|
||||
for iter.Next(ctx) {
|
||||
r.client.Del(ctx, iter.Val())
|
||||
}
|
||||
return iter.Err()
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ListenAddr string
|
||||
|
||||
DBHost string
|
||||
DBPort int
|
||||
DBUser string
|
||||
DBPass string
|
||||
DBName string
|
||||
DBSSL string
|
||||
|
||||
RedisURL string
|
||||
|
||||
S3Endpoint string
|
||||
S3AccessKey string
|
||||
S3SecretKey string
|
||||
S3Bucket string
|
||||
S3Secure bool
|
||||
S3Region string
|
||||
}
|
||||
|
||||
func (c *Config) DatabaseDSN() string {
|
||||
return fmt.Sprintf(
|
||||
"postgres://%s:%s@%s:%d/%s?sslmode=%s",
|
||||
c.DBUser, c.DBPass, c.DBHost, c.DBPort, c.DBName, c.DBSSL,
|
||||
)
|
||||
}
|
||||
|
||||
func Load() (*Config, error) {
|
||||
dbPort, err := strconv.Atoi(getenv("DBPORT", "5432"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid DBPORT: %w", err)
|
||||
}
|
||||
|
||||
s3Secure, _ := strconv.ParseBool(getenv("MINIO_SECURE", "false"))
|
||||
|
||||
cfg := &Config{
|
||||
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
|
||||
|
||||
DBHost: getenv("DBHOST", "localhost"),
|
||||
DBPort: dbPort,
|
||||
DBUser: getenv("DBUSER", "artifacts"),
|
||||
DBPass: getenv("DBPASS", ""),
|
||||
DBName: getenv("DBNAME", "artifacts"),
|
||||
DBSSL: getenv("DBSSL", "disable"),
|
||||
|
||||
RedisURL: getenv("REDIS_URL", "redis://localhost:6379"),
|
||||
|
||||
S3Endpoint: getenv("MINIO_ENDPOINT", "localhost:9000"),
|
||||
S3AccessKey: getenv("MINIO_ACCESS_KEY", ""),
|
||||
S3SecretKey: getenv("MINIO_SECRET_KEY", ""),
|
||||
S3Bucket: getenv("MINIO_BUCKET", "artifacts"),
|
||||
S3Secure: s3Secure,
|
||||
S3Region: getenv("MINIO_REGION", ""),
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) UpsertBlob(ctx context.Context, contentHash, s3Key string, sizeBytes int64, contentType string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO blobs (content_hash, s3_key, size_bytes, content_type)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (content_hash) DO NOTHING
|
||||
`, contentHash, s3Key, sizeBytes, contentType)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpsertArtifact(ctx context.Context, remoteName, path, contentHash, upstreamETag string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO artifacts (remote_name, path, content_hash, upstream_etag)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (remote_name, path) DO UPDATE SET
|
||||
content_hash = EXCLUDED.content_hash,
|
||||
upstream_etag = EXCLUDED.upstream_etag,
|
||||
last_fetched_at = NOW(),
|
||||
fetch_count = artifacts.fetch_count + 1
|
||||
`, remoteName, path, contentHash, upstreamETag)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) GetArtifact(ctx context.Context, remoteName, path string) (*models.Artifact, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||
a.last_accessed_at, a.fetch_count, a.access_count,
|
||||
b.size_bytes, b.content_type
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
WHERE a.remote_name = $1 AND a.path = $2
|
||||
`, remoteName, path)
|
||||
|
||||
var a models.Artifact
|
||||
err := row.Scan(
|
||||
&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &a.UpstreamETag,
|
||||
&a.UpstreamLastModified, &a.FirstSeenAt, &a.LastFetchedAt,
|
||||
&a.LastAccessedAt, &a.FetchCount, &a.AccessCount,
|
||||
&a.SizeBytes, &a.ContentType,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func (db *DB) TouchArtifactAccess(ctx context.Context, remoteName, path string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE artifacts SET
|
||||
last_accessed_at = NOW(),
|
||||
access_count = access_count + 1
|
||||
WHERE remote_name = $1 AND path = $2
|
||||
`, remoteName, path)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListArtifacts(ctx context.Context, remoteName string, limit, offset int) ([]models.Artifact, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||
a.last_accessed_at, a.fetch_count, a.access_count,
|
||||
b.size_bytes, b.content_type
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
WHERE a.remote_name = $1
|
||||
ORDER BY a.path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, remoteName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var artifacts []models.Artifact
|
||||
for rows.Next() {
|
||||
var a models.Artifact
|
||||
if err := rows.Scan(
|
||||
&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &a.UpstreamETag,
|
||||
&a.UpstreamLastModified, &a.FirstSeenAt, &a.LastFetchedAt,
|
||||
&a.LastAccessedAt, &a.FetchCount, &a.AccessCount,
|
||||
&a.SizeBytes, &a.ContentType,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
artifacts = append(artifacts, a)
|
||||
}
|
||||
return artifacts, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) DeleteArtifact(ctx context.Context, remoteName, path string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM artifacts WHERE remote_name = $1 AND path = $2`, remoteName, path)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cacheHit bool, sizeBytes int64, upstreamMS int, clientIP string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO access_log (remote_name, path, cache_hit, size_bytes, upstream_ms, client_ip)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
`, remoteName, path, cacheHit, sizeBytes, upstreamMS, clientIP)
|
||||
return err
|
||||
}
|
||||
|
||||
// AccessLogEntry is one buffered access-log record.
|
||||
type AccessLogEntry struct {
|
||||
RemoteName string
|
||||
Path string
|
||||
CacheHit bool
|
||||
SizeBytes int64
|
||||
UpstreamMS int
|
||||
ClientIP string
|
||||
}
|
||||
|
||||
// InsertAccessLogBatch bulk-inserts access-log rows with a single COPY.
|
||||
func (db *DB) InsertAccessLogBatch(ctx context.Context, entries []AccessLogEntry) error {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows := make([][]any, len(entries))
|
||||
for i, e := range entries {
|
||||
rows[i] = []any{e.RemoteName, e.Path, e.CacheHit, e.SizeBytes, e.UpstreamMS, e.ClientIP}
|
||||
}
|
||||
_, err := db.Pool.CopyFrom(ctx,
|
||||
pgx.Identifier{"access_log"},
|
||||
[]string{"remote_name", "path", "cache_hit", "size_bytes", "upstream_ms", "client_ip"},
|
||||
pgx.CopyFromRows(rows),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// FindOrphanedBlobs returns blobs no longer referenced by any artifact or
|
||||
// local file, restricted to those created before now()-minAge. The age cutoff
|
||||
// is a grace period that avoids a TOCTOU race with in-flight dedup uploads,
|
||||
// which insert the blob row before the referencing artifact/local_files row.
|
||||
func (db *DB) FindOrphanedBlobs(ctx context.Context, minAge time.Duration) ([]models.Blob, error) {
|
||||
cutoff := time.Now().Add(-minAge)
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT b.content_hash, b.s3_key, b.size_bytes, b.content_type, b.created_at
|
||||
FROM blobs b
|
||||
WHERE b.created_at < $1
|
||||
AND b.content_hash NOT IN (
|
||||
SELECT content_hash FROM artifacts
|
||||
UNION
|
||||
SELECT content_hash FROM local_files
|
||||
)
|
||||
`, cutoff)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var blobs []models.Blob
|
||||
for rows.Next() {
|
||||
var b models.Blob
|
||||
if err := rows.Scan(&b.ContentHash, &b.S3Key, &b.SizeBytes, &b.ContentType, &b.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blobs = append(blobs, b)
|
||||
}
|
||||
return blobs, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) DeleteBlob(ctx context.Context, contentHash string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM blobs WHERE content_hash = $1`, contentHash)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteColdArtifacts(ctx context.Context, remoteName string, olderThan time.Duration) (int64, error) {
|
||||
cutoff := time.Now().Add(-olderThan)
|
||||
tag, err := db.Pool.Exec(ctx, `
|
||||
DELETE FROM artifacts
|
||||
WHERE remote_name = $1 AND last_accessed_at < $2
|
||||
`, remoteName, cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
type LocalFile struct {
|
||||
ID int64 `json:"id"`
|
||||
RepoName string `json:"repo_name"`
|
||||
FilePath string `json:"file_path"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
var ErrAlreadyExists = fmt.Errorf("file already exists")
|
||||
|
||||
func (db *DB) CreateLocalFile(ctx context.Context, repoName, filePath, contentHash string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO local_files (repo_name, file_path, content_hash)
|
||||
VALUES ($1, $2, $3)
|
||||
`, repoName, filePath, contentHash)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return ErrAlreadyExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) GetLocalFile(ctx context.Context, repoName, filePath string) (*LocalFile, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1 AND file_path = $2
|
||||
`, repoName, filePath)
|
||||
|
||||
var f LocalFile
|
||||
if err := row.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &f, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFiles(ctx context.Context, repoName string, limit, offset int) ([]LocalFile, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1
|
||||
ORDER BY file_path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, repoName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var files []LocalFile
|
||||
for rows.Next() {
|
||||
var f LocalFile
|
||||
if err := rows.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, f)
|
||||
}
|
||||
return files, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFilesByPrefix(ctx context.Context, repoName, prefix string) ([]LocalFile, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1 AND file_path LIKE $2
|
||||
ORDER BY file_path
|
||||
`, repoName, prefix+"%")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var files []LocalFile
|
||||
for rows.Next() {
|
||||
var f LocalFile
|
||||
if err := rows.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, f)
|
||||
}
|
||||
return files, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFilePackages(ctx context.Context, repoName string) ([]string, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT DISTINCT split_part(file_path, '/', 1)
|
||||
FROM local_files
|
||||
WHERE repo_name = $1
|
||||
ORDER BY 1
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var packages []string
|
||||
for rows.Next() {
|
||||
var pkg string
|
||||
if err := rows.Scan(&pkg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
return packages, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]provider.FileEntry, error) {
|
||||
files, err := db.ListLocalFilesByPrefix(ctx, repoName, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]provider.FileEntry, len(files))
|
||||
for i, f := range files {
|
||||
result[i] = provider.FileEntry{FilePath: f.FilePath, ContentHash: f.ContentHash}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListPackages(ctx context.Context, repoName string) ([]string, error) {
|
||||
return db.ListLocalFilePackages(ctx, repoName)
|
||||
}
|
||||
|
||||
func (db *DB) DeleteLocalFile(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM local_files WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type DB struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func New(dsn string) (*DB, error) {
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect to postgres: %w", err)
|
||||
}
|
||||
|
||||
if err := pool.Ping(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("ping postgres: %w", err)
|
||||
}
|
||||
|
||||
db := &DB{Pool: pool}
|
||||
if err := db.migrate(); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("run migrations: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (db *DB) Close() {
|
||||
db.Pool.Close()
|
||||
}
|
||||
|
||||
func (db *DB) migrate() error {
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS remotes (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
repo_type TEXT DEFAULT 'remote',
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
immutable_ttl INTEGER DEFAULT 0,
|
||||
mutable_ttl INTEGER DEFAULT 3600,
|
||||
check_mutable BOOLEAN DEFAULT TRUE,
|
||||
patterns TEXT[] DEFAULT '{}',
|
||||
blocklist TEXT[] DEFAULT '{}',
|
||||
mutable_patterns TEXT[] DEFAULT '{}',
|
||||
immutable_patterns TEXT[] DEFAULT '{}',
|
||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
||||
ban_tags TEXT[] DEFAULT '{}',
|
||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
||||
quarantine_days INTEGER DEFAULT 3,
|
||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
||||
releases_remote TEXT DEFAULT '',
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS virtuals (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
members TEXT[] NOT NULL,
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS blobs (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
s3_key TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
content_type TEXT DEFAULT 'application/octet-stream',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artifacts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
upstream_etag TEXT DEFAULT '',
|
||||
upstream_last_modified TIMESTAMPTZ,
|
||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
fetch_count BIGINT DEFAULT 1,
|
||||
access_count BIGINT DEFAULT 1,
|
||||
UNIQUE(remote_name, path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS local_files (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS access_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
cache_hit BOOLEAN NOT NULL,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
upstream_ms INTEGER DEFAULT 0,
|
||||
client_ip TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rpm_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
epoch INTEGER DEFAULT 0,
|
||||
version TEXT NOT NULL,
|
||||
release TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
summary TEXT DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
rpm_size BIGINT DEFAULT 0,
|
||||
installed_size BIGINT DEFAULT 0,
|
||||
license TEXT DEFAULT '',
|
||||
vendor TEXT DEFAULT '',
|
||||
build_group TEXT DEFAULT '',
|
||||
build_host TEXT DEFAULT '',
|
||||
source_rpm TEXT DEFAULT '',
|
||||
url TEXT DEFAULT '',
|
||||
packager TEXT DEFAULT '',
|
||||
requires JSONB DEFAULT '[]',
|
||||
provides JSONB DEFAULT '[]',
|
||||
files JSONB DEFAULT '[]',
|
||||
changelogs JSONB DEFAULT '[]',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
|
||||
`)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const remoteCols = `name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
created_at, updated_at`
|
||||
|
||||
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
|
||||
return scanner.Scan(
|
||||
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Description, &r.Username, &r.Password,
|
||||
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
|
||||
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
|
||||
&r.BanTagsEnabled, &r.BanTags,
|
||||
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
|
||||
&r.ReleasesRemote, &r.ManagedBy,
|
||||
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
|
||||
&r.CreatedAt, &r.UpdatedAt,
|
||||
)
|
||||
}
|
||||
|
||||
func (db *DB) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
|
||||
row := db.Pool.QueryRow(ctx, `SELECT `+remoteCols+` FROM remotes WHERE name = $1`, name)
|
||||
var r models.Remote
|
||||
if err := scanRemote(row, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes ORDER BY name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var remotes []models.Remote
|
||||
for rows.Next() {
|
||||
var r models.Remote
|
||||
if err := scanRemote(rows, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remotes = append(remotes, r)
|
||||
}
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO remotes (
|
||||
name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24)
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE remotes SET
|
||||
package_type=$2, repo_type=$3, base_url=$4, description=$5, username=$6, password=$7,
|
||||
immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
|
||||
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
|
||||
ban_tags_enabled=$15, ban_tags=$16,
|
||||
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
|
||||
releases_remote=$20, managed_by=$21,
|
||||
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
|
||||
updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteRemote(ctx context.Context, name string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM remotes WHERE name = $1`, name)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
|
||||
requiresJSON, _ := json.Marshal(meta.Requires)
|
||||
providesJSON, _ := json.Marshal(meta.Provides)
|
||||
filesJSON, _ := json.Marshal(meta.Files)
|
||||
changelogsJSON, _ := json.Marshal(meta.Changelogs)
|
||||
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO rpm_metadata (
|
||||
repo_name, file_path, content_hash,
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, files, changelogs
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash,
|
||||
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
|
||||
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
|
||||
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
|
||||
requiresJSON, providesJSON, filesJSON, changelogsJSON,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
type RPMMetadataRow struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Epoch int
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
Summary string
|
||||
Description string
|
||||
RPMSize int64
|
||||
InstalledSize int64
|
||||
License string
|
||||
Vendor string
|
||||
Group string
|
||||
BuildHost string
|
||||
SourceRPM string
|
||||
URL string
|
||||
Packager string
|
||||
Requires json.RawMessage
|
||||
Provides json.RawMessage
|
||||
Files json.RawMessage
|
||||
Changelogs json.RawMessage
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]provider.RPMMetadata, error) {
|
||||
rows, err := db.ListRPMMetadata(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]provider.RPMMetadata, len(rows))
|
||||
for i, r := range rows {
|
||||
meta := provider.RPMMetadata{
|
||||
RepoName: r.RepoName,
|
||||
FilePath: r.FilePath,
|
||||
ContentHash: r.ContentHash,
|
||||
Name: r.Name,
|
||||
Epoch: r.Epoch,
|
||||
Version: r.Version,
|
||||
Release: r.Release,
|
||||
Arch: r.Arch,
|
||||
Summary: r.Summary,
|
||||
Description: r.Description,
|
||||
RPMSize: r.RPMSize,
|
||||
InstalledSize: r.InstalledSize,
|
||||
License: r.License,
|
||||
Vendor: r.Vendor,
|
||||
Group: r.Group,
|
||||
BuildHost: r.BuildHost,
|
||||
SourceRPM: r.SourceRPM,
|
||||
URL: r.URL,
|
||||
Packager: r.Packager,
|
||||
}
|
||||
json.Unmarshal(r.Requires, &meta.Requires)
|
||||
json.Unmarshal(r.Provides, &meta.Provides)
|
||||
json.Unmarshal(r.Files, &meta.Files)
|
||||
json.Unmarshal(r.Changelogs, &meta.Changelogs)
|
||||
result[i] = meta
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetadataRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash,
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, files, changelogs
|
||||
FROM rpm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, epoch, version, release, arch
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []RPMMetadataRow
|
||||
for rows.Next() {
|
||||
var r RPMMetadataRow
|
||||
if err := rows.Scan(
|
||||
&r.RepoName, &r.FilePath, &r.ContentHash,
|
||||
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
|
||||
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
|
||||
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
|
||||
&r.Requires, &r.Provides, &r.Files, &r.Changelogs,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) GetOverviewStats(ctx context.Context) (*models.OverviewStats, error) {
|
||||
var stats models.OverviewStats
|
||||
|
||||
err := db.Pool.QueryRow(ctx, `SELECT COUNT(*) FROM remotes`).Scan(&stats.TotalRemotes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `SELECT COALESCE(COUNT(*), 0), COALESCE(SUM(b.size_bytes), 0)
|
||||
FROM artifacts a JOIN blobs b ON a.content_hash = b.content_hash`).
|
||||
Scan(&stats.TotalObjects, &stats.TotalBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(
|
||||
(SELECT COUNT(*) FROM artifacts) - (SELECT COUNT(DISTINCT content_hash) FROM artifacts),
|
||||
0
|
||||
)`).Scan(&stats.TotalBlobsDeduped)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(size_bytes), 0)
|
||||
FROM access_log
|
||||
WHERE cache_hit = TRUE AND created_at > NOW() - INTERVAL '30 days'
|
||||
`).Scan(&stats.BandwidthSaved30d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
type RemoteStatRow struct {
|
||||
Name string `json:"name"`
|
||||
ObjectCount int64 `json:"object_count"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
Requests30d int64 `json:"requests_30d"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopRemotes(ctx context.Context, limit int) ([]RemoteStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT r.name,
|
||||
COALESCE(a.cnt, 0) AS object_count,
|
||||
COALESCE(a.total_bytes, 0) AS total_bytes,
|
||||
COALESCE(l.req_count, 0) AS requests_30d
|
||||
FROM remotes r
|
||||
LEFT JOIN (
|
||||
SELECT remote_name, COUNT(*) AS cnt, SUM(b.size_bytes) AS total_bytes
|
||||
FROM artifacts a JOIN blobs b ON a.content_hash = b.content_hash
|
||||
GROUP BY remote_name
|
||||
) a ON r.name = a.remote_name
|
||||
LEFT JOIN (
|
||||
SELECT remote_name, COUNT(*) AS req_count
|
||||
FROM access_log
|
||||
WHERE created_at > NOW() - INTERVAL '30 days'
|
||||
GROUP BY remote_name
|
||||
) l ON r.name = l.remote_name
|
||||
ORDER BY COALESCE(a.total_bytes, 0) DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []RemoteStatRow
|
||||
for rows.Next() {
|
||||
var r RemoteStatRow
|
||||
if err := rows.Scan(&r.Name, &r.ObjectCount, &r.TotalBytes, &r.Requests30d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
type FileStatRow struct {
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
AccessCount int64 `json:"access_count"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopFilesByHits(ctx context.Context, limit int) ([]FileStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT a.remote_name, a.path, a.access_count, b.size_bytes
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
ORDER BY a.access_count DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []FileStatRow
|
||||
for rows.Next() {
|
||||
var r FileStatRow
|
||||
if err := rows.Scan(&r.RemoteName, &r.Path, &r.AccessCount, &r.SizeBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
type BandwidthStatRow struct {
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
Bandwidth int64 `json:"bandwidth"`
|
||||
Requests int64 `json:"requests"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopFilesByBandwidth(ctx context.Context, limit int) ([]BandwidthStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT remote_name, path,
|
||||
COALESCE(SUM(size_bytes), 0) AS bandwidth,
|
||||
COUNT(*) AS requests
|
||||
FROM access_log
|
||||
WHERE created_at > NOW() - INTERVAL '30 days'
|
||||
GROUP BY remote_name, path
|
||||
ORDER BY bandwidth DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []BandwidthStatRow
|
||||
for rows.Next() {
|
||||
var r BandwidthStatRow
|
||||
if err := rows.Scan(&r.RemoteName, &r.Path, &r.Bandwidth, &r.Requests); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) GetVirtual(ctx context.Context, name string) (*models.Virtual, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT name, package_type, description, members, managed_by, created_at, updated_at
|
||||
FROM virtuals WHERE name = $1
|
||||
`, name)
|
||||
|
||||
var v models.Virtual
|
||||
err := row.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListVirtuals(ctx context.Context) ([]models.Virtual, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT name, package_type, description, members, managed_by, created_at, updated_at
|
||||
FROM virtuals ORDER BY name
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var virtuals []models.Virtual
|
||||
for rows.Next() {
|
||||
var v models.Virtual
|
||||
if err := rows.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
virtuals = append(virtuals, v)
|
||||
}
|
||||
return virtuals, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) CreateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO virtuals (name, package_type, description, members, managed_by)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpdateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE virtuals SET
|
||||
package_type=$2, description=$3, members=$4, managed_by=$5, updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteVirtual(ctx context.Context, name string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM virtuals WHERE name = $1`, name)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
)
|
||||
|
||||
// blobGracePeriod is how old an orphaned blob must be before GC will delete
|
||||
// it. This avoids racing in-flight dedup uploads that insert the blob row
|
||||
// before the referencing artifact/local_files row exists.
|
||||
const blobGracePeriod = 1 * time.Hour
|
||||
|
||||
type Collector struct {
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func New(db *database.DB, store *storage.S3, interval time.Duration) *Collector {
|
||||
return &Collector{db: db, store: store, interval: interval}
|
||||
}
|
||||
|
||||
func (c *Collector) Run(ctx context.Context) {
|
||||
slog.Info("gc started", "interval", c.interval)
|
||||
ticker := time.NewTicker(c.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("gc stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
c.sweep(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Collector) sweep(ctx context.Context) {
|
||||
start := time.Now()
|
||||
|
||||
orphaned, err := c.db.FindOrphanedBlobs(ctx, blobGracePeriod)
|
||||
if err != nil {
|
||||
slog.Error("gc: find orphaned blobs", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for _, blob := range orphaned {
|
||||
if err := c.store.Delete(ctx, blob.S3Key); err != nil {
|
||||
slog.Warn("gc: delete s3 object", "key", blob.S3Key, "error", err)
|
||||
continue
|
||||
}
|
||||
if err := c.db.DeleteBlob(ctx, blob.ContentHash); err != nil {
|
||||
slog.Warn("gc: delete blob row", "hash", blob.ContentHash, "error", err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
|
||||
if deleted > 0 || len(orphaned) > 0 {
|
||||
slog.Info("gc sweep complete",
|
||||
"orphaned_found", len(orphaned),
|
||||
"deleted", deleted,
|
||||
"duration_ms", time.Since(start).Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package gc_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/gc"
|
||||
)
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
c := gc.New(nil, nil, 1*time.Hour)
|
||||
if c == nil {
|
||||
t.Fatal("expected non-nil collector")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageAlpine }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "APKINDEX.tar.gz") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".apk") {
|
||||
return "application/vnd.android.package-archive"
|
||||
}
|
||||
if strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var (
|
||||
tagManifestRe = regexp.MustCompile(`/manifests/[^/]+$`)
|
||||
digestManifestRe = regexp.MustCompile(`/manifests/sha256:[0-9a-fA-F]+$`)
|
||||
tagsListRe = regexp.MustCompile(`/tags/list$`)
|
||||
)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageDocker }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if tagsListRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
if tagManifestRe.MatchString(path) && !digestManifestRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.Contains(path, "/blobs/") {
|
||||
return "application/octet-stream"
|
||||
}
|
||||
if strings.Contains(path, "/manifests/") {
|
||||
return "application/vnd.docker.distribution.manifest.v2+json"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/v2/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" && remote.Password != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package docker_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
if p.Type() != models.PackageDocker {
|
||||
t.Errorf("expected docker, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"library/nginx/manifests/latest", provider.Mutable},
|
||||
{"library/nginx/manifests/v1.25", provider.Mutable},
|
||||
{"library/nginx/manifests/sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", provider.Immutable},
|
||||
{"library/nginx/tags/list", provider.Mutable},
|
||||
{"library/nginx/blobs/sha256:abc123", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_UpstreamURL(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://registry-1.docker.io"}, "library/nginx/manifests/latest")
|
||||
want := "https://registry-1.docker.io/v2/library/nginx/manifests/latest"
|
||||
if got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
if p.ContentType("x/blobs/sha256:abc") != "application/octet-stream" {
|
||||
t.Error("blobs should be octet-stream")
|
||||
}
|
||||
if p.ContentType("x/manifests/latest") != "application/vnd.docker.distribution.manifest.v2+json" {
|
||||
t.Error("manifests should be manifest type")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package generic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageGeneric }
|
||||
|
||||
func (p *Provider) Classify(_ string) provider.Mutability {
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
var contentTypeMap = map[string]string{
|
||||
".tar.gz": "application/gzip",
|
||||
".tgz": "application/gzip",
|
||||
".gz": "application/gzip",
|
||||
".zip": "application/zip",
|
||||
".whl": "application/zip",
|
||||
".exe": "application/x-msdownload",
|
||||
".rpm": "application/x-rpm",
|
||||
".xml": "application/xml",
|
||||
".yaml": "text/yaml",
|
||||
".yml": "text/yaml",
|
||||
".json": "application/json",
|
||||
".sig": "application/octet-stream",
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(filePath string) string {
|
||||
lower := strings.ToLower(filePath)
|
||||
if strings.HasSuffix(lower, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
ext := path.Ext(lower)
|
||||
if ct, ok := contentTypeMap[ext]; ok {
|
||||
return ct
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, reqPath string) string {
|
||||
base := strings.TrimRight(remote.BaseURL, "/")
|
||||
return base + "/" + strings.TrimLeft(reqPath, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package generic_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
if p.Type() != models.PackageGeneric {
|
||||
t.Errorf("expected generic, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify_AllImmutable(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
paths := []string{"file.tar.gz", "path/to/binary", "index.html", "data.json"}
|
||||
for _, path := range paths {
|
||||
if p.Classify(path) != provider.Immutable {
|
||||
t.Errorf("generic should classify %q as immutable", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
tests := []struct{ path, want string }{
|
||||
{"file.tar.gz", "application/gzip"},
|
||||
{"file.tgz", "application/gzip"},
|
||||
{"file.zip", "application/zip"},
|
||||
{"file.rpm", "application/x-rpm"},
|
||||
{"file.json", "application/json"},
|
||||
{"file.unknown", "application/octet-stream"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.ContentType(tt.path); got != tt.want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_UpstreamURL(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://example.com/repo"}, "path/to/file.tar.gz")
|
||||
want := "https://example.com/repo/path/to/file.tar.gz"
|
||||
if got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_AuthHeaders_BasicAuth(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "user", Password: "pass"})
|
||||
if h.Get("Authorization") != "Basic dXNlcjpwYXNz" {
|
||||
t.Errorf("unexpected auth header: %q", h.Get("Authorization"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_AuthHeaders_NoAuth(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("expected no auth header")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package goproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageGoProxy }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "/@v/list") || strings.HasSuffix(path, "/@latest") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(path, ".mod") {
|
||||
return "text/plain"
|
||||
}
|
||||
if strings.HasSuffix(path, ".info") {
|
||||
return "application/json"
|
||||
}
|
||||
if strings.HasSuffix(path, "/list") {
|
||||
return "text/plain"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package goproxy_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
if p.Type() != models.PackageGoProxy {
|
||||
t.Errorf("expected goproxy, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"golang.org/x/net/@v/list", provider.Mutable},
|
||||
{"golang.org/x/net/@latest", provider.Mutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.info", provider.Immutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.mod", provider.Immutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.zip", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
tests := []struct{ path, want string }{
|
||||
{"m/@v/v1.0.0.zip", "application/zip"},
|
||||
{"m/@v/v1.0.0.mod", "text/plain"},
|
||||
{"m/@v/v1.0.0.info", "application/json"},
|
||||
{"m/@v/list", "text/plain"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.ContentType(tt.path); got != tt.want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package helm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageHelm }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "index.yaml") || strings.HasSuffix(path, "index.yml") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tgz") || strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") {
|
||||
return "text/yaml"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
rewritten := strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
if rewritten == content {
|
||||
return nil, nil
|
||||
}
|
||||
return []byte(rewritten), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package helm_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/helm"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
if p.Type() != models.PackageHelm {
|
||||
t.Errorf("expected helm, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"index.yaml", provider.Mutable},
|
||||
{"index.yml", provider.Mutable},
|
||||
{"chart-1.0.tgz", provider.Immutable},
|
||||
{"charts/nginx-1.0.tgz", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_RewriteResponse(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
body := []byte("urls:\n- https://charts.example.com/chart-1.0.tgz")
|
||||
remote := models.Remote{Name: "helm-test", BaseURL: "https://charts.example.com"}
|
||||
rewritten, err := p.RewriteResponse(body, remote, "https://proxy.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rewritten == nil {
|
||||
t.Fatal("expected rewrite")
|
||||
}
|
||||
if !strings.Contains(string(rewritten), "proxy.example.com/api/v1/remote/helm-test") {
|
||||
t.Errorf("expected proxy URL in body: %s", rewritten)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package npm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageNPM }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, ".tgz") {
|
||||
return provider.Immutable
|
||||
}
|
||||
return provider.Mutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tgz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" || !json.Valid(body) {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
rewritten := strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
if rewritten == content {
|
||||
return nil, nil
|
||||
}
|
||||
return []byte(rewritten), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type Mutability int
|
||||
|
||||
const (
|
||||
Immutable Mutability = iota
|
||||
Mutable
|
||||
)
|
||||
|
||||
type Provider interface {
|
||||
Type() models.PackageType
|
||||
Classify(path string) Mutability
|
||||
ContentType(path string) string
|
||||
UpstreamURL(remote models.Remote, path string) string
|
||||
RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error)
|
||||
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
|
||||
}
|
||||
|
||||
type FileEntry struct {
|
||||
FilePath string
|
||||
ContentHash string
|
||||
}
|
||||
|
||||
type FileStore interface {
|
||||
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
|
||||
ListPackages(ctx context.Context, repoName string) ([]string, error)
|
||||
}
|
||||
|
||||
type LocalUploader interface {
|
||||
ValidateUpload(filePath string) (storagePath, contentType string, err error)
|
||||
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
|
||||
}
|
||||
|
||||
type LocalIndexer interface {
|
||||
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
|
||||
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
|
||||
}
|
||||
|
||||
type BlobReader interface {
|
||||
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
||||
}
|
||||
|
||||
type PostUploadHook interface {
|
||||
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
|
||||
}
|
||||
|
||||
type MetadataStore interface {
|
||||
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
|
||||
}
|
||||
|
||||
type RPMMetadataReader interface {
|
||||
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
|
||||
}
|
||||
|
||||
type RPMMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Epoch int
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
Summary string
|
||||
Description string
|
||||
RPMSize int64
|
||||
InstalledSize int64
|
||||
License string
|
||||
Vendor string
|
||||
Group string
|
||||
BuildHost string
|
||||
SourceRPM string
|
||||
URL string
|
||||
Packager string
|
||||
Requires []RPMDep
|
||||
Provides []RPMDep
|
||||
Files []RPMFile
|
||||
Changelogs []RPMChangelog
|
||||
}
|
||||
|
||||
type RPMDep struct {
|
||||
Name string `json:"name"`
|
||||
Flags string `json:"flags,omitempty"`
|
||||
Epoch string `json:"epoch,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Release string `json:"release,omitempty"`
|
||||
}
|
||||
|
||||
type RPMFile struct {
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
type RPMChangelog struct {
|
||||
Author string `json:"author"`
|
||||
Date int64 `json:"date"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
|
||||
type MemberIndex struct {
|
||||
RemoteName string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
var registry = map[models.PackageType]Provider{}
|
||||
|
||||
func Register(p Provider) {
|
||||
registry[p.Type()] = p
|
||||
}
|
||||
|
||||
func Get(t models.PackageType) (Provider, error) {
|
||||
p, ok := registry[t]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no provider registered for package type %q", t)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func All() map[models.PackageType]Provider {
|
||||
return registry
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package puppet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackagePuppet }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasPrefix(path, "v3/modules/") || strings.HasPrefix(path, "v3/releases") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.HasPrefix(path, "v3/") {
|
||||
return "application/json"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
content = strings.ReplaceAll(content, `"/v3/files/`, `"`+proxyURL+`/v3/files/`)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
content = strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
return []byte(content), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package pypi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var fileRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*\.(whl|tar\.gz|zip)$`)
|
||||
var normalizeRe = regexp.MustCompile(`[-_.]+`)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackagePyPI }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.Contains(path, "simple/") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
lower := strings.ToLower(path)
|
||||
if strings.HasSuffix(lower, ".whl") || strings.HasSuffix(lower, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.Contains(path, "simple/") {
|
||||
return "text/html"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
if strings.HasPrefix(path, "simple/") {
|
||||
return "https://pypi.org/" + path
|
||||
}
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + "/"
|
||||
content = strings.ReplaceAll(content, "https://files.pythonhosted.org/", proxyURL)
|
||||
content = strings.ReplaceAll(content, "../../", proxyURL)
|
||||
return []byte(content), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func normalize(name string) string {
|
||||
return strings.ToLower(normalizeRe.ReplaceAllString(name, "-"))
|
||||
}
|
||||
|
||||
func packageFromWheel(filename string) string {
|
||||
parts := strings.SplitN(filename, "-", 3)
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return normalize(parts[0])
|
||||
}
|
||||
|
||||
func packageFromSdist(filename string) string {
|
||||
name := filename
|
||||
for _, suffix := range []string{".tar.gz", ".zip"} {
|
||||
if strings.HasSuffix(name, suffix) {
|
||||
name = strings.TrimSuffix(name, suffix)
|
||||
break
|
||||
}
|
||||
}
|
||||
idx := strings.LastIndex(name, "-")
|
||||
if idx <= 0 {
|
||||
return ""
|
||||
}
|
||||
return normalize(name[:idx])
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !fileRe.MatchString(filename) {
|
||||
return "", "", fmt.Errorf("filename %q must be a .whl, .tar.gz, or .zip file", filename)
|
||||
}
|
||||
|
||||
var pkgName string
|
||||
if strings.HasSuffix(filename, ".whl") {
|
||||
pkgName = packageFromWheel(filename)
|
||||
} else {
|
||||
pkgName = packageFromSdist(filename)
|
||||
}
|
||||
if pkgName == "" {
|
||||
return "", "", fmt.Errorf("cannot parse package name from %q", filename)
|
||||
}
|
||||
|
||||
ct := "application/zip"
|
||||
if strings.HasSuffix(filename, ".tar.gz") {
|
||||
ct = "application/gzip"
|
||||
}
|
||||
|
||||
return pkgName + "/" + filename, ct, nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
parts := strings.SplitN(storagePath, "/", 2)
|
||||
filename := storagePath
|
||||
if len(parts) == 2 {
|
||||
filename = parts[1]
|
||||
}
|
||||
return map[string]any{
|
||||
"package": parts[0],
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
if path == "simple" || path == "simple/" {
|
||||
p.servePackageList(w, r, files, repoName)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(path, "simple/") {
|
||||
pkg := strings.TrimPrefix(path, "simple/")
|
||||
pkg = strings.TrimSuffix(pkg, "/")
|
||||
if pkg != "" && !strings.Contains(pkg, "/") {
|
||||
p.servePackageFiles(w, r, files, repoName, pkg)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
if !strings.HasPrefix(path, "simple/") {
|
||||
return nil, fmt.Errorf("unsupported index path: %q", path)
|
||||
}
|
||||
|
||||
pkg := strings.TrimPrefix(path, "simple/")
|
||||
pkg = strings.TrimSuffix(pkg, "/")
|
||||
if pkg == "" {
|
||||
return p.generatePackageListHTML(ctx, files, repoName)
|
||||
}
|
||||
return p.generatePackageFilesHTML(ctx, files, repoName, pkg)
|
||||
}
|
||||
|
||||
func (p *Provider) servePackageList(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName string) {
|
||||
body, err := p.generatePackageListHTML(r.Context(), files, repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
func (p *Provider) servePackageFiles(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, packageName string) {
|
||||
normalized := normalize(packageName)
|
||||
prefix := normalized + "/"
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
|
||||
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n",
|
||||
normalized, filename, hash, filename)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.WriteString(w, b.String())
|
||||
}
|
||||
|
||||
func (p *Provider) generatePackageListHTML(ctx context.Context, files provider.FileStore, repoName string) ([]byte, error) {
|
||||
packages, err := files.ListPackages(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, pkg := range packages {
|
||||
fmt.Fprintf(&b, "<a href=\"%s/\">%s</a>\n", pkg, pkg)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
|
||||
func (p *Provider) generatePackageFilesHTML(ctx context.Context, files provider.FileStore, repoName, packageName string) ([]byte, error) {
|
||||
normalized := normalize(packageName)
|
||||
prefix := normalized + "/"
|
||||
entries, err := files.ListFilesByPrefix(ctx, repoName, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
|
||||
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&b, "<a href=\"%s/%s#sha256=%s\">%s</a>\n",
|
||||
normalized, filename, hash, filename)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
@@ -0,0 +1,444 @@
|
||||
package rpm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
rpmlib "github.com/cavaliergopher/rpm"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var mutableRe = []*regexp.Regexp{
|
||||
regexp.MustCompile(`repomd\.xml$`),
|
||||
regexp.MustCompile(`repodata/`),
|
||||
regexp.MustCompile(`Packages\.gz$`),
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageRPM }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
for _, re := range mutableRe {
|
||||
if re.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".rpm") {
|
||||
return "application/x-rpm"
|
||||
}
|
||||
if strings.HasSuffix(path, ".xml") || strings.HasSuffix(path, ".xml.gz") || strings.HasSuffix(path, ".xml.xz") {
|
||||
return "application/xml"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(strings.ToLower(filename), ".rpm") {
|
||||
return "", "", fmt.Errorf("file must be an .rpm package")
|
||||
}
|
||||
|
||||
return "Packages/" + filename, "application/x-rpm", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
filename := strings.TrimPrefix(storagePath, "Packages/")
|
||||
return map[string]any{
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
|
||||
|
||||
reader, blobSize, err := blobs.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
slog.Error("rpm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
pkg, err := rpmlib.Read(reader)
|
||||
if err != nil {
|
||||
slog.Error("rpm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
meta := &provider.RPMMetadata{
|
||||
RepoName: repoName,
|
||||
FilePath: storagePath,
|
||||
ContentHash: contentHash,
|
||||
Name: pkg.Name(),
|
||||
Epoch: pkg.Epoch(),
|
||||
Version: pkg.Version(),
|
||||
Release: pkg.Release(),
|
||||
Arch: pkg.Architecture(),
|
||||
Summary: pkg.Summary(),
|
||||
Description: pkg.Description(),
|
||||
RPMSize: blobSize,
|
||||
InstalledSize: int64(pkg.Size()),
|
||||
License: pkg.License(),
|
||||
Vendor: pkg.Vendor(),
|
||||
Group: firstGroup(pkg.Groups()),
|
||||
BuildHost: pkg.BuildHost(),
|
||||
SourceRPM: pkg.SourceRPM(),
|
||||
URL: pkg.URL(),
|
||||
Packager: pkg.Packager(),
|
||||
}
|
||||
|
||||
for _, req := range pkg.Requires() {
|
||||
meta.Requires = append(meta.Requires, rpmDepFromEntry(req))
|
||||
}
|
||||
for _, prov := range pkg.Provides() {
|
||||
meta.Provides = append(meta.Provides, rpmDepFromEntry(prov))
|
||||
}
|
||||
|
||||
if meta.Requires == nil {
|
||||
meta.Requires = []provider.RPMDep{}
|
||||
}
|
||||
if meta.Provides == nil {
|
||||
meta.Provides = []provider.RPMDep{}
|
||||
}
|
||||
meta.Files = []provider.RPMFile{}
|
||||
meta.Changelogs = []provider.RPMChangelog{}
|
||||
|
||||
if err := db.InsertRPMMetadata(ctx, meta); err != nil {
|
||||
slog.Error("rpm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("rpm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
}
|
||||
|
||||
func rpmDepFromEntry(e rpmlib.Dependency) provider.RPMDep {
|
||||
dep := provider.RPMDep{Name: e.Name()}
|
||||
if e.Flags() != 0 {
|
||||
dep.Flags = rpmFlagString(e.Flags())
|
||||
dep.Version = e.Version()
|
||||
dep.Release = e.Release()
|
||||
if e.Epoch() > 0 {
|
||||
dep.Epoch = fmt.Sprintf("%d", e.Epoch())
|
||||
}
|
||||
}
|
||||
return dep
|
||||
}
|
||||
|
||||
func rpmFlagString(f int) string {
|
||||
switch {
|
||||
case f&0x08 != 0 && f&0x04 != 0:
|
||||
return "GE"
|
||||
case f&0x02 != 0 && f&0x04 != 0:
|
||||
return "LE"
|
||||
case f&0x08 != 0:
|
||||
return "GT"
|
||||
case f&0x02 != 0:
|
||||
return "LT"
|
||||
case f&0x04 != 0:
|
||||
return "EQ"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func firstGroup(groups []string) string {
|
||||
if len(groups) > 0 {
|
||||
return groups[0]
|
||||
}
|
||||
return "Unspecified"
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
if !strings.HasPrefix(path, "repodata/") {
|
||||
return false
|
||||
}
|
||||
|
||||
rpmReader, ok := files.(provider.RPMMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "rpm metadata not available", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
tail := strings.TrimPrefix(path, "repodata/")
|
||||
|
||||
switch {
|
||||
case tail == "repomd.xml":
|
||||
p.serveRepomd(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-primary.xml.gz"):
|
||||
p.servePrimary(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-filelists.xml.gz"):
|
||||
p.serveFilelists(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-other.xml.gz"):
|
||||
p.serveOther(w, r, rpmReader, repoName)
|
||||
default:
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("rpm local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
primary := generatePrimaryXMLGZ(metas)
|
||||
filelists := generateFilelistsXMLGZ(metas)
|
||||
other := generateOtherXMLGZ(metas)
|
||||
|
||||
primaryHash := sha256Hex(primary)
|
||||
filelistsHash := sha256Hex(filelists)
|
||||
otherHash := sha256Hex(other)
|
||||
|
||||
repomd := generateRepomd(primaryHash, len(primary), filelistsHash, len(filelists), otherHash, len(other))
|
||||
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(repomd)
|
||||
}
|
||||
|
||||
func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generatePrimaryXMLGZ(metas))
|
||||
}
|
||||
|
||||
func (p *Provider) serveFilelists(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateFilelistsXMLGZ(metas))
|
||||
}
|
||||
|
||||
func (p *Provider) serveOther(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateOtherXMLGZ(metas))
|
||||
}
|
||||
|
||||
func generateRepomd(primaryHash string, primarySize int, filelistsHash string, filelistsSize int, otherHash string, otherSize int) []byte {
|
||||
ts := fmt.Sprintf("%d", time.Now().Unix())
|
||||
var b bytes.Buffer
|
||||
b.WriteString(xml.Header)
|
||||
b.WriteString(`<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">` + "\n")
|
||||
fmt.Fprintf(&b, " <revision>%s</revision>\n", ts)
|
||||
|
||||
writeRepomdData(&b, "primary", primaryHash, primarySize, ts)
|
||||
writeRepomdData(&b, "filelists", filelistsHash, filelistsSize, ts)
|
||||
writeRepomdData(&b, "other", otherHash, otherSize, ts)
|
||||
|
||||
b.WriteString("</repomd>\n")
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func writeRepomdData(b *bytes.Buffer, dtype, hash string, size int, ts string) {
|
||||
fmt.Fprintf(b, " <data type=\"%s\">\n", dtype)
|
||||
fmt.Fprintf(b, " <checksum type=\"sha256\">%s</checksum>\n", hash)
|
||||
fmt.Fprintf(b, " <location href=\"repodata/%s-%s.xml.gz\"/>\n", hash, dtype)
|
||||
fmt.Fprintf(b, " <timestamp>%s</timestamp>\n", ts)
|
||||
fmt.Fprintf(b, " <size>%d</size>\n", size)
|
||||
fmt.Fprintf(b, " </data>\n")
|
||||
}
|
||||
|
||||
func generatePrimaryXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package type=\"rpm\">\n")
|
||||
fmt.Fprintf(&xmlBuf, " <name>%s</name>\n", xmlEscape(m.Name))
|
||||
fmt.Fprintf(&xmlBuf, " <arch>%s</arch>\n", xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
fmt.Fprintf(&xmlBuf, " <checksum type=\"sha256\" pkgid=\"YES\">%s</checksum>\n", pkgHash)
|
||||
fmt.Fprintf(&xmlBuf, " <summary>%s</summary>\n", xmlEscape(m.Summary))
|
||||
fmt.Fprintf(&xmlBuf, " <description>%s</description>\n", xmlEscape(m.Description))
|
||||
if m.Packager != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <packager>%s</packager>\n", xmlEscape(m.Packager))
|
||||
}
|
||||
if m.URL != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <url>%s</url>\n", xmlEscape(m.URL))
|
||||
}
|
||||
fmt.Fprintf(&xmlBuf, " <time file=\"%d\" build=\"0\"/>\n", time.Now().Unix())
|
||||
fmt.Fprintf(&xmlBuf, " <size package=\"%d\" installed=\"%d\" archive=\"0\"/>\n", m.RPMSize, m.InstalledSize)
|
||||
fmt.Fprintf(&xmlBuf, " <location href=\"%s\"/>\n", xmlEscape(m.FilePath))
|
||||
fmt.Fprintf(&xmlBuf, " <format>\n")
|
||||
if m.License != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:license>%s</rpm:license>\n", xmlEscape(m.License))
|
||||
}
|
||||
if m.Vendor != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:vendor>%s</rpm:vendor>\n", xmlEscape(m.Vendor))
|
||||
}
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:group>%s</rpm:group>\n", xmlEscape(m.Group))
|
||||
if m.BuildHost != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:buildhost>%s</rpm:buildhost>\n", xmlEscape(m.BuildHost))
|
||||
}
|
||||
if m.SourceRPM != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:sourcerpm>%s</rpm:sourcerpm>\n", xmlEscape(m.SourceRPM))
|
||||
}
|
||||
|
||||
if len(m.Provides) > 0 {
|
||||
xmlBuf.WriteString(" <rpm:provides>\n")
|
||||
for _, d := range m.Provides {
|
||||
writeRPMEntry(&xmlBuf, d)
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:provides>\n")
|
||||
}
|
||||
if len(m.Requires) > 0 {
|
||||
xmlBuf.WriteString(" <rpm:requires>\n")
|
||||
for _, d := range m.Requires {
|
||||
writeRPMEntry(&xmlBuf, d)
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:requires>\n")
|
||||
}
|
||||
|
||||
fmt.Fprintf(&xmlBuf, " </format>\n")
|
||||
fmt.Fprintf(&xmlBuf, "</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</metadata>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func generateFilelistsXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<filelists xmlns=\"http://linux.duke.edu/metadata/filelists\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
for _, f := range m.Files {
|
||||
if f.Type != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <file type=\"%s\">%s</file>\n", f.Type, xmlEscape(f.Path))
|
||||
} else {
|
||||
fmt.Fprintf(&xmlBuf, " <file>%s</file>\n", xmlEscape(f.Path))
|
||||
}
|
||||
}
|
||||
xmlBuf.WriteString("</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</filelists>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func generateOtherXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<otherdata xmlns=\"http://linux.duke.edu/metadata/other\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
for _, cl := range m.Changelogs {
|
||||
fmt.Fprintf(&xmlBuf, " <changelog author=\"%s\" date=\"%d\">%s</changelog>\n",
|
||||
xmlEscape(cl.Author), cl.Date, xmlEscape(cl.Text))
|
||||
}
|
||||
xmlBuf.WriteString("</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</otherdata>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func writeRPMEntry(b *bytes.Buffer, d provider.RPMDep) {
|
||||
if d.Flags != "" {
|
||||
fmt.Fprintf(b, " <rpm:entry name=\"%s\" flags=\"%s\"", xmlEscape(d.Name), d.Flags)
|
||||
if d.Epoch != "" {
|
||||
fmt.Fprintf(b, " epoch=\"%s\"", d.Epoch)
|
||||
}
|
||||
if d.Version != "" {
|
||||
fmt.Fprintf(b, " ver=\"%s\"", xmlEscape(d.Version))
|
||||
}
|
||||
if d.Release != "" {
|
||||
fmt.Fprintf(b, " rel=\"%s\"", xmlEscape(d.Release))
|
||||
}
|
||||
b.WriteString("/>\n")
|
||||
} else {
|
||||
fmt.Fprintf(b, " <rpm:entry name=\"%s\"/>\n", xmlEscape(d.Name))
|
||||
}
|
||||
}
|
||||
|
||||
func xmlEscape(s string) string {
|
||||
var b bytes.Buffer
|
||||
xml.EscapeText(&b, []byte(s))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func gzipBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
gz.Write(data)
|
||||
gz.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func sha256Hex(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package rpm_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &rpm.Provider{}
|
||||
if p.Type() != models.PackageRPM {
|
||||
t.Errorf("expected rpm, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &rpm.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"repomd.xml", provider.Mutable},
|
||||
{"repodata/primary.xml.gz", provider.Mutable},
|
||||
{"Packages.gz", provider.Mutable},
|
||||
{"package-1.0.rpm", provider.Immutable},
|
||||
{"RPM-GPG-KEY-almalinux", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package terraform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var versionsRe = regexp.MustCompile(`[^/]+/[^/]+/versions$`)
|
||||
|
||||
var providerZipRe = regexp.MustCompile(
|
||||
`^terraform-provider-([a-zA-Z0-9_-]+)_([0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`,
|
||||
)
|
||||
|
||||
var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageTerraform }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if versionsRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
lower := strings.ToLower(path)
|
||||
if strings.HasSuffix(lower, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".sig") {
|
||||
return "application/octet-stream"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/v1/providers/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if remote.ReleasesRemote == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if !json.Valid(body) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var data map[string]any
|
||||
if err := json.Unmarshal(body, &data); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
changed := false
|
||||
for _, field := range []string{"download_url", "shasums_url", "shasums_signature_url"} {
|
||||
if val, ok := data[field].(string); ok && val != "" {
|
||||
rewritten := rewriteDownloadURL(val, remote.ReleasesRemote, proxyBaseURL)
|
||||
if rewritten != val {
|
||||
data[field] = rewritten
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return nil, nil
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
|
||||
func rewriteDownloadURL(originalURL, releasesRemote, proxyBaseURL string) string {
|
||||
parsed, err := url.Parse(originalURL)
|
||||
if err != nil || proxyBaseURL == "" {
|
||||
return originalURL
|
||||
}
|
||||
return strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + releasesRemote + parsed.Path
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
parts := strings.Split(filePath, "/")
|
||||
if len(parts) != 3 {
|
||||
return "", "", fmt.Errorf("path must be {namespace}/{type}/{filename}.zip")
|
||||
}
|
||||
namespace, typeName, filename := parts[0], parts[1], parts[2]
|
||||
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil {
|
||||
return "", "", fmt.Errorf("filename %q does not match terraform-provider-{type}_{version}_{os}_{arch}.zip", filename)
|
||||
}
|
||||
|
||||
if m[1] != typeName {
|
||||
return "", "", fmt.Errorf("provider type in filename %q does not match path type %q", m[1], typeName)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s/%s/%s", namespace, typeName, filename), "application/zip", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
parts := strings.Split(storagePath, "/")
|
||||
if len(parts) != 3 {
|
||||
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
|
||||
}
|
||||
|
||||
m := providerZipRe.FindStringSubmatch(parts[2])
|
||||
if m == nil {
|
||||
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"namespace": parts[0],
|
||||
"type": parts[1],
|
||||
"version": m[2],
|
||||
"os": m[3],
|
||||
"arch": m[4],
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
type terraformIndex struct {
|
||||
Versions map[string]json.RawMessage `json:"versions"`
|
||||
}
|
||||
|
||||
type terraformVersionDoc struct {
|
||||
Archives map[string]terraformArchive `json:"archives"`
|
||||
}
|
||||
|
||||
type terraformArchive struct {
|
||||
URL string `json:"url"`
|
||||
Hashes []string `json:"hashes,omitempty"`
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) < 3 {
|
||||
return false
|
||||
}
|
||||
|
||||
namespace, typeName := parts[0], parts[1]
|
||||
tail := parts[2]
|
||||
|
||||
if tail == "index.json" {
|
||||
p.serveIndex(w, r, files, repoName, namespace, typeName)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasSuffix(tail, ".json") {
|
||||
version := strings.TrimSuffix(tail, ".json")
|
||||
if semverRe.MatchString(version) {
|
||||
p.serveVersionDoc(w, r, files, repoName, namespace, typeName, version)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("terraform local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
func (p *Provider) serveIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName string) {
|
||||
prefix := fmt.Sprintf("%s/%s/", namespace, typeName)
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
versions := map[string]json.RawMessage{}
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, prefix)
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
versions[m[2]] = json.RawMessage(`{}`)
|
||||
}
|
||||
|
||||
if len(versions) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(terraformIndex{Versions: versions})
|
||||
}
|
||||
|
||||
func (p *Provider) serveVersionDoc(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName, version string) {
|
||||
prefix := fmt.Sprintf("%s/%s/terraform-provider-%s_%s_", namespace, typeName, typeName, version)
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
archives := map[string]terraformArchive{}
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, fmt.Sprintf("%s/%s/", namespace, typeName))
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil || m[2] != version {
|
||||
continue
|
||||
}
|
||||
platform := m[3] + "_" + m[4]
|
||||
archive := terraformArchive{URL: filename}
|
||||
if f.ContentHash != "" {
|
||||
archive.Hashes = []string{"zh:" + strings.TrimPrefix(f.ContentHash, "sha256:")}
|
||||
}
|
||||
archives[platform] = archive
|
||||
}
|
||||
|
||||
if len(archives) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(terraformVersionDoc{Archives: archives})
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package terraform_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
if p.Type() != models.PackageTerraform {
|
||||
t.Errorf("expected terraform, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"hashicorp/vault/versions", provider.Mutable},
|
||||
{"hashicorp/vault/0.28.0/download/linux/amd64", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_RewriteResponse_DownloadInfo(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
remote := models.Remote{Name: "tf", ReleasesRemote: "hashicorp-releases"}
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"download_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/file.zip",
|
||||
"shasums_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/SHA256SUMS",
|
||||
})
|
||||
rewritten, err := p.RewriteResponse(body, remote, "https://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rewritten == nil {
|
||||
t.Fatal("expected rewrite")
|
||||
}
|
||||
var result map[string]any
|
||||
json.Unmarshal(rewritten, &result)
|
||||
if !strings.Contains(result["download_url"].(string), "proxy/api/v1/remote/hashicorp-releases") {
|
||||
t.Errorf("download_url not rewritten: %s", result["download_url"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultCircuitThreshold = 5
|
||||
defaultCircuitCooldown = 60 * time.Second
|
||||
)
|
||||
|
||||
type CircuitBreaker struct {
|
||||
cache *cache.Redis
|
||||
threshold int64
|
||||
cooldown time.Duration
|
||||
}
|
||||
|
||||
func NewCircuitBreaker(c *cache.Redis) *CircuitBreaker {
|
||||
return &CircuitBreaker{
|
||||
cache: c,
|
||||
threshold: defaultCircuitThreshold,
|
||||
cooldown: defaultCircuitCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) IsOpen(ctx context.Context, remote string) bool {
|
||||
failures, err := cb.cache.GetCircuitFailures(ctx, remote)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return failures >= cb.threshold
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) RecordFailure(ctx context.Context, remote string) {
|
||||
cb.cache.IncrCircuitFailure(ctx, remote, cb.cooldown)
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) RecordSuccess(ctx context.Context, remote string) {
|
||||
cb.cache.ResetCircuit(ctx, remote)
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) Health(ctx context.Context, remote string) models.RemoteHealth {
|
||||
failures, err := cb.cache.GetCircuitFailures(ctx, remote)
|
||||
if err != nil {
|
||||
return models.RemoteHealth{Status: "unknown"}
|
||||
}
|
||||
|
||||
switch {
|
||||
case failures == 0:
|
||||
return models.RemoteHealth{Status: "healthy", ConsecutiveFailures: int(failures)}
|
||||
case failures < cb.threshold:
|
||||
return models.RemoteHealth{Status: "degraded", ConsecutiveFailures: int(failures)}
|
||||
default:
|
||||
return models.RemoteHealth{Status: "down", ConsecutiveFailures: int(failures)}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package proxy_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
)
|
||||
|
||||
func TestCircuitBreaker_New(t *testing.T) {
|
||||
cb := proxy.NewCircuitBreaker(nil)
|
||||
if cb == nil {
|
||||
t.Fatal("expected non-nil circuit breaker")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sync"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type Classification int
|
||||
|
||||
const (
|
||||
ClassImmutable Classification = iota
|
||||
ClassMutable
|
||||
ClassDenied
|
||||
)
|
||||
|
||||
func (c Classification) String() string {
|
||||
switch c {
|
||||
case ClassImmutable:
|
||||
return "immutable"
|
||||
case ClassMutable:
|
||||
return "mutable"
|
||||
case ClassDenied:
|
||||
return "denied"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
type Classifier struct {
|
||||
provider provider.Provider
|
||||
}
|
||||
|
||||
func NewClassifier(p provider.Provider) *Classifier {
|
||||
return &Classifier{provider: p}
|
||||
}
|
||||
|
||||
func (c *Classifier) Classify(remote models.Remote, path string) Classification {
|
||||
if matchesAny(path, compilePatterns(remote.Blocklist)) {
|
||||
return ClassDenied
|
||||
}
|
||||
|
||||
if len(remote.Patterns) > 0 && !matchesAny(path, compilePatterns(remote.Patterns)) {
|
||||
return ClassDenied
|
||||
}
|
||||
|
||||
if matchesAny(path, compilePatterns(remote.ImmutablePatterns)) {
|
||||
return ClassImmutable
|
||||
}
|
||||
|
||||
if matchesAny(path, compilePatterns(remote.MutablePatterns)) {
|
||||
return ClassMutable
|
||||
}
|
||||
|
||||
if c.provider.Classify(path) == provider.Mutable {
|
||||
return ClassMutable
|
||||
}
|
||||
|
||||
return ClassImmutable
|
||||
}
|
||||
|
||||
// patternCache memoises regex compilation. Classify runs on every proxied
|
||||
// request and previously recompiled each remote's pattern lists every time;
|
||||
// keying by the pattern string lets each distinct pattern compile once and
|
||||
// then be reused, with no invalidation needed (the pattern text is the key).
|
||||
// A pattern that fails to compile is cached as a typed nil so we don't retry.
|
||||
var patternCache sync.Map // map[string]*regexp.Regexp
|
||||
|
||||
func compileCached(pattern string) *regexp.Regexp {
|
||||
if v, ok := patternCache.Load(pattern); ok {
|
||||
return v.(*regexp.Regexp)
|
||||
}
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
re = nil
|
||||
}
|
||||
patternCache.Store(pattern, re)
|
||||
return re
|
||||
}
|
||||
|
||||
func compilePatterns(patterns []string) []*regexp.Regexp {
|
||||
compiled := make([]*regexp.Regexp, 0, len(patterns))
|
||||
for _, p := range patterns {
|
||||
if re := compileCached(p); re != nil {
|
||||
compiled = append(compiled, re)
|
||||
}
|
||||
}
|
||||
return compiled
|
||||
}
|
||||
|
||||
func matchesAny(path string, patterns []*regexp.Regexp) bool {
|
||||
for _, re := range patterns {
|
||||
if re.MatchString(path) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package proxy_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/helm"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestClassifier_EmptyPatternsAllowsAll(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{Name: "test"}
|
||||
if c.Classify(remote, "any/path") == proxy.ClassDenied {
|
||||
t.Error("empty patterns should allow all paths")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_PatternsActAsAllowlist(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{
|
||||
Name: "test",
|
||||
Patterns: []string{`^releases/`},
|
||||
}
|
||||
if c.Classify(remote, "releases/v1.0/app.tar.gz") == proxy.ClassDenied {
|
||||
t.Error("path matching patterns should be allowed")
|
||||
}
|
||||
if c.Classify(remote, "uploads/other.tar.gz") != proxy.ClassDenied {
|
||||
t.Error("path not matching patterns should be denied")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_BlocklistDenies(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{
|
||||
Name: "test",
|
||||
Blocklist: []string{`\.exe$`},
|
||||
}
|
||||
if c.Classify(remote, "malware.exe") != proxy.ClassDenied {
|
||||
t.Error("blocklist match should deny")
|
||||
}
|
||||
if c.Classify(remote, "legit.tar.gz") == proxy.ClassDenied {
|
||||
t.Error("non-blocked path should be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_BlocklistBeforePatterns(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{
|
||||
Name: "test",
|
||||
Patterns: []string{`^releases/`},
|
||||
Blocklist: []string{`releases/v0\.1/`},
|
||||
}
|
||||
if c.Classify(remote, "releases/v0.1/app.tar.gz") != proxy.ClassDenied {
|
||||
t.Error("blocklist should take priority")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_GenericAllImmutable(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{Name: "test"}
|
||||
if c.Classify(remote, "any/file.tar.gz") != proxy.ClassImmutable {
|
||||
t.Error("generic provider should classify everything as immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_GenericMutableOverride(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{
|
||||
Name: "test",
|
||||
MutablePatterns: []string{`/archive/refs/heads/`},
|
||||
}
|
||||
if c.Classify(remote, "repo/archive/refs/heads/main.tar.gz") != proxy.ClassMutable {
|
||||
t.Error("mutable_patterns should override provider default")
|
||||
}
|
||||
if c.Classify(remote, "repo/releases/v1.0.tar.gz") != proxy.ClassImmutable {
|
||||
t.Error("non-mutable path should stay immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_ImmutableOverride(t *testing.T) {
|
||||
c := proxy.NewClassifier(&helm.Provider{})
|
||||
remote := models.Remote{
|
||||
Name: "test",
|
||||
ImmutablePatterns: []string{`special-index\.yaml$`},
|
||||
}
|
||||
if c.Classify(remote, "special-index.yaml") != proxy.ClassImmutable {
|
||||
t.Error("immutable_patterns should force immutable even for normally mutable paths")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_HelmAutoClassifies(t *testing.T) {
|
||||
c := proxy.NewClassifier(&helm.Provider{})
|
||||
remote := models.Remote{Name: "test"}
|
||||
if c.Classify(remote, "index.yaml") != proxy.ClassMutable {
|
||||
t.Error("helm should auto-classify index.yaml as mutable")
|
||||
}
|
||||
if c.Classify(remote, "chart-1.0.tgz") != proxy.ClassImmutable {
|
||||
t.Error("helm should auto-classify .tgz as immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_DockerAutoClassifies(t *testing.T) {
|
||||
c := proxy.NewClassifier(&docker.Provider{})
|
||||
remote := models.Remote{Name: "test"}
|
||||
if c.Classify(remote, "library/nginx/manifests/latest") != proxy.ClassMutable {
|
||||
t.Error("docker should classify tag manifest as mutable")
|
||||
}
|
||||
if c.Classify(remote, "library/nginx/manifests/sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890") != proxy.ClassImmutable {
|
||||
t.Error("docker should classify digest manifest as immutable")
|
||||
}
|
||||
if c.Classify(remote, "library/nginx/blobs/sha256:abc") != proxy.ClassImmutable {
|
||||
t.Error("docker should classify blobs as immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_RPMAutoClassifies(t *testing.T) {
|
||||
c := proxy.NewClassifier(&rpm.Provider{})
|
||||
remote := models.Remote{Name: "test"}
|
||||
if c.Classify(remote, "repodata/primary.xml.gz") != proxy.ClassMutable {
|
||||
t.Error("rpm should classify repodata as mutable")
|
||||
}
|
||||
if c.Classify(remote, "packages/foo-1.0.rpm") != proxy.ClassImmutable {
|
||||
t.Error("rpm should classify .rpm as immutable")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,651 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const fetchLockTTL = 30 * time.Second
|
||||
|
||||
const (
|
||||
accessLogBufferSize = 4096
|
||||
accessLogBatchSize = 128
|
||||
accessLogFlushEvery = 2 * time.Second
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
db *database.DB
|
||||
cache *cache.Redis
|
||||
store *storage.S3
|
||||
cas *storage.CAS
|
||||
circuit *CircuitBreaker
|
||||
accessLog chan database.AccessLogEntry
|
||||
}
|
||||
|
||||
func NewEngine(db *database.DB, c *cache.Redis, s *storage.S3) *Engine {
|
||||
e := &Engine{
|
||||
db: db,
|
||||
cache: c,
|
||||
store: s,
|
||||
cas: storage.NewCAS(s),
|
||||
circuit: NewCircuitBreaker(c),
|
||||
accessLog: make(chan database.AccessLogEntry, accessLogBufferSize),
|
||||
}
|
||||
go e.runAccessLogWriter()
|
||||
return e
|
||||
}
|
||||
|
||||
// runAccessLogWriter drains the access-log channel and writes rows in batches,
|
||||
// replacing a goroutine-per-request insert. It runs for the process lifetime;
|
||||
// access logs are best-effort telemetry, so a small tail may be lost on abrupt
|
||||
// shutdown.
|
||||
func (e *Engine) runAccessLogWriter() {
|
||||
ticker := time.NewTicker(accessLogFlushEvery)
|
||||
defer ticker.Stop()
|
||||
|
||||
batch := make([]database.AccessLogEntry, 0, accessLogBatchSize)
|
||||
flush := func() {
|
||||
if len(batch) == 0 {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if err := e.db.InsertAccessLogBatch(ctx, batch); err != nil {
|
||||
slog.Warn("access log batch insert failed", "error", err, "count", len(batch))
|
||||
}
|
||||
cancel()
|
||||
batch = batch[:0]
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case entry := <-e.accessLog:
|
||||
batch = append(batch, entry)
|
||||
if len(batch) >= accessLogBatchSize {
|
||||
flush()
|
||||
}
|
||||
case <-ticker.C:
|
||||
flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type FetchResult struct {
|
||||
Reader io.ReadCloser
|
||||
ContentType string
|
||||
Size int64
|
||||
Source string // "cache" or "remote"
|
||||
}
|
||||
|
||||
func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, prov provider.Provider, clientHeaders ...http.Header) (*FetchResult, error) {
|
||||
classifier := NewClassifier(prov)
|
||||
class := classifier.Classify(remote, path)
|
||||
|
||||
if class == ClassDenied {
|
||||
return nil, &ProxyError{Status: http.StatusForbidden, Message: "access denied"}
|
||||
}
|
||||
|
||||
ttl := e.ttlFor(remote, class)
|
||||
|
||||
fresh, err := e.cache.CheckTTL(ctx, remote.Name, path)
|
||||
if err != nil {
|
||||
slog.Warn("redis check failed, treating as miss", "error", err)
|
||||
}
|
||||
|
||||
if fresh {
|
||||
result, err := e.serveFromStore(ctx, remote, path)
|
||||
if err == nil {
|
||||
result.Source = "cache"
|
||||
e.logAccess(remote.Name, path, true, result.Size, 0)
|
||||
return result, nil
|
||||
}
|
||||
slog.Warn("cache hit but S3 miss, re-fetching", "remote", remote.Name, "path", path)
|
||||
}
|
||||
|
||||
locked, err := e.cache.AcquireLock(ctx, remote.Name, path, fetchLockTTL)
|
||||
if err != nil {
|
||||
slog.Warn("lock acquire failed", "error", err)
|
||||
}
|
||||
|
||||
if !locked {
|
||||
// Another request holds the fetch lock. Poll the store until the leader
|
||||
// populates it rather than immediately racing to fetch upstream too; a
|
||||
// cold-cache stampede otherwise hits upstream once per waiter.
|
||||
if result := e.waitForStore(ctx, remote, path); result != nil {
|
||||
result.Source = "cache"
|
||||
e.logAccess(remote.Name, path, true, result.Size, 0)
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
if locked {
|
||||
defer e.cache.ReleaseLock(ctx, remote.Name, path)
|
||||
}
|
||||
|
||||
if class == ClassMutable && remote.CheckMutable {
|
||||
etag, _ := e.cache.GetETag(ctx, remote.Name, path)
|
||||
if etag != "" {
|
||||
notModified, err := e.checkUpstream(ctx, remote, path, etag, prov)
|
||||
if err == nil && notModified {
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
|
||||
result, err := e.serveFromStore(ctx, remote, path)
|
||||
if err == nil {
|
||||
result.Source = "cache"
|
||||
e.logAccess(remote.Name, path, true, result.Size, 0)
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var fwdHeaders http.Header
|
||||
if len(clientHeaders) > 0 && clientHeaders[0] != nil {
|
||||
fwdHeaders = clientHeaders[0]
|
||||
}
|
||||
|
||||
// Short-circuit upstream calls when the remote's breaker is open: serve
|
||||
// stale from the store if we have it, otherwise fail fast rather than
|
||||
// hammering a known-bad upstream.
|
||||
if e.circuit.IsOpen(ctx, remote.Name) {
|
||||
if stale, serr := e.serveFromStore(ctx, remote, path); serr == nil {
|
||||
slog.Warn("circuit open, serving stale", "remote", remote.Name, "path", path)
|
||||
stale.Source = "cache"
|
||||
e.logAccess(remote.Name, path, true, stale.Size, 0)
|
||||
return stale, nil
|
||||
}
|
||||
return nil, &ProxyError{Status: http.StatusServiceUnavailable, Message: "upstream circuit open"}
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
result, err := e.fetchFromUpstream(ctx, remote, path, prov, class, ttl, fwdHeaders)
|
||||
upstreamMS := int(time.Since(start).Milliseconds())
|
||||
if err != nil {
|
||||
if isNetworkError(err) {
|
||||
e.circuit.RecordFailure(ctx, remote.Name)
|
||||
}
|
||||
if remote.StaleOnError && isNetworkError(err) {
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
stale, serr := e.serveFromStore(ctx, remote, path)
|
||||
if serr == nil {
|
||||
slog.Warn("serving stale on upstream error", "remote", remote.Name, "path", path, "error", err)
|
||||
stale.Source = "cache"
|
||||
e.logAccess(remote.Name, path, true, stale.Size, 0)
|
||||
return stale, nil
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
e.circuit.RecordSuccess(ctx, remote.Name)
|
||||
e.logAccess(remote.Name, path, false, result.Size, upstreamMS)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// HeadResult carries artifact metadata for a HEAD request. There is no body.
|
||||
type HeadResult struct {
|
||||
ContentType string
|
||||
Size int64
|
||||
Source string // "cache" or "remote"
|
||||
}
|
||||
|
||||
// Head resolves artifact metadata without fetching or streaming the body.
|
||||
// Cached artifacts/indexes are answered from the store metadata; on a miss it
|
||||
// issues an upstream HEAD. It never downloads or caches the body.
|
||||
func (e *Engine) Head(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*HeadResult, error) {
|
||||
class := NewClassifier(prov).Classify(remote, path)
|
||||
if class == ClassDenied {
|
||||
return nil, &ProxyError{Status: http.StatusForbidden, Message: "access denied"}
|
||||
}
|
||||
|
||||
if artifact, err := e.db.GetArtifact(ctx, remote.Name, path); err == nil && artifact != nil {
|
||||
return &HeadResult{ContentType: artifact.ContentType, Size: artifact.SizeBytes, Source: "cache"}, nil
|
||||
}
|
||||
if info, err := e.store.Stat(ctx, storage.IndexKey(remote.Name, path)); err == nil {
|
||||
return &HeadResult{ContentType: info.ContentType, Size: info.Size, Source: "cache"}, nil
|
||||
}
|
||||
|
||||
return e.headUpstream(ctx, remote, path, prov)
|
||||
}
|
||||
|
||||
func (e *Engine) headUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*HeadResult, error) {
|
||||
url := prov.UpstreamURL(remote, path)
|
||||
|
||||
authHeaders, err := prov.AuthHeaders(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("auth headers: %w", err)
|
||||
}
|
||||
|
||||
doHead := func(extra http.Header) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
for k, vv := range authHeaders {
|
||||
for _, v := range vv {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
for k, vv := range extra {
|
||||
for _, v := range vv {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
}
|
||||
return http.DefaultClient.Do(req)
|
||||
}
|
||||
|
||||
resp, err := doHead(nil)
|
||||
if err != nil {
|
||||
return nil, &UpstreamError{Err: err}
|
||||
}
|
||||
if resp.StatusCode == http.StatusUnauthorized {
|
||||
resp.Body.Close()
|
||||
token, _, terr := fetchBearerToken(ctx, resp.Header.Get("Www-Authenticate"), remote)
|
||||
if terr == nil && token != "" {
|
||||
resp, err = doHead(http.Header{"Authorization": []string{"Bearer " + token}})
|
||||
if err != nil {
|
||||
return nil, &UpstreamError{Err: err}
|
||||
}
|
||||
} else {
|
||||
return nil, &ProxyError{Status: http.StatusUnauthorized, Message: "upstream returned 401"}
|
||||
}
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, &ProxyError{Status: resp.StatusCode, Message: fmt.Sprintf("upstream returned %d", resp.StatusCode)}
|
||||
}
|
||||
|
||||
contentType := prov.ContentType(path)
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "" {
|
||||
contentType = ct
|
||||
}
|
||||
return &HeadResult{ContentType: contentType, Size: resp.ContentLength, Source: "remote"}, nil
|
||||
}
|
||||
|
||||
func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration, clientHeaders http.Header) (*FetchResult, error) {
|
||||
url := prov.UpstreamURL(remote, path)
|
||||
|
||||
authHeaders, err := prov.AuthHeaders(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("auth headers: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
for k, vv := range authHeaders {
|
||||
for _, v := range vv {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
if clientHeaders != nil {
|
||||
if accept := clientHeaders.Get("Accept"); accept != "" {
|
||||
req.Header.Set("Accept", accept)
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := clientForRemote(remote).Do(req)
|
||||
if err != nil {
|
||||
return nil, &UpstreamError{Err: err}
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusUnauthorized {
|
||||
resp.Body.Close()
|
||||
token, err := e.cachedBearerToken(ctx, resp.Header.Get("Www-Authenticate"), remote)
|
||||
if err == nil && token != "" {
|
||||
req2, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
req2.Header.Set("Authorization", "Bearer "+token)
|
||||
if clientHeaders != nil {
|
||||
if accept := clientHeaders.Get("Accept"); accept != "" {
|
||||
req2.Header.Set("Accept", accept)
|
||||
}
|
||||
}
|
||||
resp, err = clientForRemote(remote).Do(req2)
|
||||
if err != nil {
|
||||
return nil, &UpstreamError{Err: err}
|
||||
}
|
||||
} else {
|
||||
return nil, &ProxyError{Status: http.StatusUnauthorized, Message: "upstream returned 401"}
|
||||
}
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
return nil, &ProxyError{Status: resp.StatusCode, Message: fmt.Sprintf("upstream returned %d", resp.StatusCode)}
|
||||
}
|
||||
|
||||
contentType := prov.ContentType(path)
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "" {
|
||||
contentType = ct
|
||||
}
|
||||
|
||||
// Mutable indexes are small and may be rewritten, so buffer them in memory.
|
||||
if class == ClassMutable {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read upstream body: %w", err)
|
||||
}
|
||||
|
||||
rewritten, err := prov.RewriteResponse(body, remote, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rewrite response: %w", err)
|
||||
}
|
||||
if rewritten != nil {
|
||||
body = rewritten
|
||||
}
|
||||
|
||||
s3Key := storage.IndexKey(remote.Name, path)
|
||||
if err := e.store.Upload(ctx, s3Key, bytesReader(body), int64(len(body)), contentType); err != nil {
|
||||
return nil, fmt.Errorf("upload index: %w", err)
|
||||
}
|
||||
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
if etag := resp.Header.Get("ETag"); etag != "" {
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
|
||||
}
|
||||
|
||||
return &FetchResult{
|
||||
Reader: io.NopCloser(bytesReader(body)),
|
||||
ContentType: contentType,
|
||||
Size: int64(len(body)),
|
||||
Source: "remote",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Immutable blobs are streamed through the content-addressable store
|
||||
// (tempfile -> sha256 -> S3) so arbitrarily large artifacts never sit
|
||||
// fully in memory. Immutable content is never rewritten in the proxy path.
|
||||
casResult, err := e.cas.Store(ctx, resp.Body, contentType)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("store blob: %w", err)
|
||||
}
|
||||
|
||||
if err := e.db.UpsertBlob(ctx, casResult.ContentHash, casResult.S3Key, casResult.SizeBytes, contentType); err != nil {
|
||||
slog.Warn("upsert blob failed", "error", err)
|
||||
}
|
||||
if err := e.db.UpsertArtifact(ctx, remote.Name, path, casResult.ContentHash, resp.Header.Get("ETag")); err != nil {
|
||||
slog.Warn("upsert artifact failed", "error", err)
|
||||
}
|
||||
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
if etag := resp.Header.Get("ETag"); etag != "" {
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
|
||||
}
|
||||
|
||||
reader, info, err := e.store.Download(ctx, casResult.S3Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serve stored blob: %w", err)
|
||||
}
|
||||
return &FetchResult{
|
||||
Reader: reader,
|
||||
ContentType: info.ContentType,
|
||||
Size: casResult.SizeBytes,
|
||||
Source: "remote",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// waitForStore polls the store for an artifact populated by the request that
|
||||
// holds the fetch lock, returning it once available or nil if it does not
|
||||
// appear within the wait budget (after which the caller fetches upstream
|
||||
// itself). It stops early if the request context is cancelled.
|
||||
func (e *Engine) waitForStore(ctx context.Context, remote models.Remote, path string) *FetchResult {
|
||||
const (
|
||||
pollInterval = 100 * time.Millisecond
|
||||
maxWait = 5 * time.Second
|
||||
)
|
||||
deadline := time.Now().Add(maxWait)
|
||||
for {
|
||||
if result, err := e.serveFromStore(ctx, remote, path); err == nil {
|
||||
return result
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(pollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) serveFromStore(ctx context.Context, remote models.Remote, path string) (*FetchResult, error) {
|
||||
artifact, err := e.db.GetArtifact(ctx, remote.Name, path)
|
||||
if err == nil && artifact != nil {
|
||||
s3Key := storage.BlobKey(artifact.ContentHash[len("sha256:"):])
|
||||
reader, info, err := e.store.Download(ctx, s3Key)
|
||||
if err == nil {
|
||||
_ = e.db.TouchArtifactAccess(ctx, remote.Name, path)
|
||||
return &FetchResult{
|
||||
Reader: reader,
|
||||
ContentType: info.ContentType,
|
||||
Size: info.Size,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
s3Key := storage.IndexKey(remote.Name, path)
|
||||
reader, info, err := e.store.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("not in store: %w", err)
|
||||
}
|
||||
return &FetchResult{
|
||||
Reader: reader,
|
||||
ContentType: info.ContentType,
|
||||
Size: info.Size,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *Engine) checkUpstream(ctx context.Context, remote models.Remote, path, etag string, prov provider.Provider) (bool, error) {
|
||||
url := prov.UpstreamURL(remote, path)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
|
||||
authHeaders, err := prov.AuthHeaders(ctx, remote)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for k, vv := range authHeaders {
|
||||
for _, v := range vv {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := clientForRemote(remote).Do(req)
|
||||
if err != nil {
|
||||
return false, &UpstreamError{Err: err}
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
return resp.StatusCode == http.StatusNotModified, nil
|
||||
}
|
||||
|
||||
func (e *Engine) ttlFor(remote models.Remote, class Classification) time.Duration {
|
||||
switch class {
|
||||
case ClassImmutable:
|
||||
if remote.ImmutableTTL == 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(remote.ImmutableTTL) * time.Second
|
||||
default:
|
||||
return time.Duration(remote.MutableTTL) * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
// logAccess enqueues an access-log entry for the batch writer. It never blocks
|
||||
// the request path: if the buffer is full the entry is dropped.
|
||||
func (e *Engine) logAccess(remoteName, path string, cacheHit bool, size int64, upstreamMS int) {
|
||||
select {
|
||||
case e.accessLog <- database.AccessLogEntry{
|
||||
RemoteName: remoteName,
|
||||
Path: path,
|
||||
CacheHit: cacheHit,
|
||||
SizeBytes: size,
|
||||
UpstreamMS: upstreamMS,
|
||||
}:
|
||||
default:
|
||||
slog.Warn("access log buffer full, dropping entry", "remote", remoteName, "path", path)
|
||||
}
|
||||
}
|
||||
|
||||
func bytesReader(data []byte) io.Reader {
|
||||
return io.NewSectionReader(readerAt(data), 0, int64(len(data)))
|
||||
}
|
||||
|
||||
type readerAt []byte
|
||||
|
||||
func (r readerAt) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
if off >= int64(len(r)) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n = copy(p, r[off:])
|
||||
if off+int64(n) >= int64(len(r)) {
|
||||
err = io.EOF
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// bearerTokenTTLDefault/Margin bound how long a token is cached: the default
|
||||
// is used when the token endpoint omits expires_in, and the margin is
|
||||
// subtracted so a cached token is refreshed slightly before it actually expires.
|
||||
const (
|
||||
bearerTokenTTLDefault = 60 * time.Second
|
||||
bearerTokenTTLMargin = 10 * time.Second
|
||||
)
|
||||
|
||||
func sha256Hash(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// cachedBearerToken returns a bearer token for the given challenge, reusing a
|
||||
// Redis-cached token for the same remote+challenge while it is still valid.
|
||||
func (e *Engine) cachedBearerToken(ctx context.Context, wwwAuth string, remote models.Remote) (string, error) {
|
||||
key := remote.Name + ":" + sha256Hash([]byte(wwwAuth))
|
||||
if tok, err := e.cache.GetToken(ctx, key); err == nil && tok != "" {
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
tok, ttl, err := fetchBearerToken(ctx, wwwAuth, remote)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if tok != "" {
|
||||
if ttl <= 0 {
|
||||
ttl = bearerTokenTTLDefault
|
||||
}
|
||||
if ttl > bearerTokenTTLMargin {
|
||||
ttl -= bearerTokenTTLMargin
|
||||
}
|
||||
_ = e.cache.SetToken(ctx, key, tok, ttl)
|
||||
}
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
func fetchBearerToken(ctx context.Context, wwwAuth string, remote models.Remote) (string, time.Duration, error) {
|
||||
if !strings.HasPrefix(wwwAuth, "Bearer ") {
|
||||
return "", 0, fmt.Errorf("not a Bearer challenge")
|
||||
}
|
||||
|
||||
params := map[string]string{}
|
||||
for _, part := range strings.Split(wwwAuth[7:], ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
eq := strings.Index(part, "=")
|
||||
if eq < 0 {
|
||||
continue
|
||||
}
|
||||
key := part[:eq]
|
||||
val := strings.Trim(part[eq+1:], `"`)
|
||||
params[key] = val
|
||||
}
|
||||
|
||||
realm := params["realm"]
|
||||
if realm == "" {
|
||||
return "", 0, fmt.Errorf("no realm in Bearer challenge")
|
||||
}
|
||||
|
||||
tokenURL := realm
|
||||
sep := "?"
|
||||
if s, ok := params["service"]; ok {
|
||||
tokenURL += sep + "service=" + s
|
||||
sep = "&"
|
||||
}
|
||||
if s, ok := params["scope"]; ok {
|
||||
tokenURL += sep + "scope=" + s
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL, nil)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
if remote.Username != "" && remote.Password != "" {
|
||||
req.SetBasicAuth(remote.Username, remote.Password)
|
||||
}
|
||||
|
||||
resp, err := clientForRemote(remote).Do(req)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", 0, fmt.Errorf("token endpoint returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var tokenResp struct {
|
||||
Token string `json:"token"`
|
||||
AccessToken string `json:"access_token"`
|
||||
ExpiresIn int `json:"expires_in"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
ttl := time.Duration(tokenResp.ExpiresIn) * time.Second
|
||||
if tokenResp.Token != "" {
|
||||
return tokenResp.Token, ttl, nil
|
||||
}
|
||||
return tokenResp.AccessToken, ttl, nil
|
||||
}
|
||||
|
||||
type ProxyError struct {
|
||||
Status int
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *ProxyError) Error() string { return e.Message }
|
||||
|
||||
type UpstreamError struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *UpstreamError) Error() string { return fmt.Sprintf("upstream error: %v", e.Err) }
|
||||
func (e *UpstreamError) Unwrap() error { return e.Err }
|
||||
|
||||
func isNetworkError(err error) bool {
|
||||
var ue *UpstreamError
|
||||
return errors.As(err, &ue)
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// Default upstream timeouts. A remote may override any of these; a zero
|
||||
// override falls back to the default here. There is deliberately no overall
|
||||
// Client.Timeout: the proxy streams arbitrarily large artifacts and total time
|
||||
// is bounded by the request context instead. We only constrain the phases that
|
||||
// must never hang — connect, TLS handshake, and time-to-first-response-header —
|
||||
// so a slow or wedged upstream cannot pin a goroutine and connection.
|
||||
const (
|
||||
defaultDialTimeout = 10 * time.Second
|
||||
defaultTLSTimeout = 10 * time.Second
|
||||
defaultResponseHeaderTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
type clientKey struct {
|
||||
dial time.Duration
|
||||
tls time.Duration
|
||||
respHeader time.Duration
|
||||
}
|
||||
|
||||
var (
|
||||
clientCacheMu sync.Mutex
|
||||
clientCache = map[clientKey]*http.Client{}
|
||||
)
|
||||
|
||||
// upstreamClientFor returns an HTTP client configured with the given timeouts,
|
||||
// reusing a cached client (and its connection pool) for identical timeout sets.
|
||||
// Zero values fall back to the defaults.
|
||||
func upstreamClientFor(dial, tls, respHeader time.Duration) *http.Client {
|
||||
if dial <= 0 {
|
||||
dial = defaultDialTimeout
|
||||
}
|
||||
if tls <= 0 {
|
||||
tls = defaultTLSTimeout
|
||||
}
|
||||
if respHeader <= 0 {
|
||||
respHeader = defaultResponseHeaderTimeout
|
||||
}
|
||||
key := clientKey{dial: dial, tls: tls, respHeader: respHeader}
|
||||
|
||||
clientCacheMu.Lock()
|
||||
defer clientCacheMu.Unlock()
|
||||
if c, ok := clientCache[key]; ok {
|
||||
return c
|
||||
}
|
||||
|
||||
c := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: dial,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: tls,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
ResponseHeaderTimeout: respHeader,
|
||||
},
|
||||
}
|
||||
clientCache[key] = c
|
||||
return c
|
||||
}
|
||||
|
||||
// clientForRemote returns the upstream client for a remote, applying its
|
||||
// per-remote timeout overrides (in seconds) on top of the defaults.
|
||||
func clientForRemote(remote models.Remote) *http.Client {
|
||||
return upstreamClientFor(
|
||||
time.Duration(remote.UpstreamDialTimeout)*time.Second,
|
||||
time.Duration(remote.UpstreamTLSTimeout)*time.Second,
|
||||
time.Duration(remote.UpstreamResponseHeaderTimeout)*time.Second,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
)
|
||||
|
||||
func cors(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func NewStructuredLogger() func(next http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
|
||||
|
||||
defer func() {
|
||||
slog.Info("request",
|
||||
"method", r.Method,
|
||||
"path", r.URL.Path,
|
||||
"status", ww.Status(),
|
||||
"bytes", ww.BytesWritten(),
|
||||
"duration_ms", time.Since(start).Milliseconds(),
|
||||
"remote", r.RemoteAddr,
|
||||
"request_id", middleware.GetReqID(r.Context()),
|
||||
)
|
||||
}()
|
||||
|
||||
next.ServeHTTP(ww, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
|
||||
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/gc"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
cfg *config.Config
|
||||
version string
|
||||
router chi.Router
|
||||
db *database.DB
|
||||
cache *cache.Redis
|
||||
store *storage.S3
|
||||
engine *proxy.Engine
|
||||
virtEngine *virtual.Engine
|
||||
localHandler *v2.LocalHandler
|
||||
gc *gc.Collector
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, version string) (*Server, error) {
|
||||
db, err := database.New(cfg.DatabaseDSN())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("database: %w", err)
|
||||
}
|
||||
|
||||
redis, err := cache.NewRedis(cfg.RedisURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("redis: %w", err)
|
||||
}
|
||||
|
||||
s3, err := storage.NewS3(cfg.S3Endpoint, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3Bucket, cfg.S3Secure, cfg.S3Region)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("s3: %w", err)
|
||||
}
|
||||
|
||||
engine := proxy.NewEngine(db, redis, s3)
|
||||
localHandler := v2.NewLocalHandler(db, s3)
|
||||
virtEngine := virtual.NewEngine(db, engine)
|
||||
collector := gc.New(db, s3, 1*time.Hour)
|
||||
|
||||
s := &Server{
|
||||
cfg: cfg,
|
||||
version: version,
|
||||
db: db,
|
||||
cache: redis,
|
||||
store: s3,
|
||||
engine: engine,
|
||||
virtEngine: virtEngine,
|
||||
localHandler: localHandler,
|
||||
gc: collector,
|
||||
}
|
||||
|
||||
s.router = s.routes()
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Server) routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
|
||||
r.Use(middleware.RequestID)
|
||||
r.Use(middleware.RealIP)
|
||||
r.Use(NewStructuredLogger())
|
||||
r.Use(middleware.Recoverer)
|
||||
|
||||
r.Use(cors)
|
||||
|
||||
r.Get("/health", s.handleHealth)
|
||||
r.Get("/", s.handleRoot)
|
||||
|
||||
proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler)
|
||||
r.Mount("/api/v1", proxyHandler.Routes())
|
||||
r.Mount("/v2", proxyHandler.DockerV2Routes())
|
||||
|
||||
remotesHandler := v2.NewRemotesHandler(s.db)
|
||||
virtualsHandler := v2.NewVirtualsHandler(s.db)
|
||||
healthHandler := v2.NewHealthHandler(s.db, s.cache, s.store)
|
||||
statsHandler := v2.NewStatsHandler(s.db)
|
||||
eventsHandler := v2.NewEventsHandler()
|
||||
probeHandler := v2.NewProbeHandler(s.engine, s.db)
|
||||
|
||||
r.Route("/api/v2", func(r chi.Router) {
|
||||
r.Mount("/remotes", remotesHandler.Routes())
|
||||
r.Mount("/virtuals", virtualsHandler.Routes())
|
||||
r.Mount("/health", healthHandler.Routes())
|
||||
r.Mount("/stats", statsHandler.Routes())
|
||||
r.Mount("/events", eventsHandler.Routes())
|
||||
r.Mount("/probe", probeHandler.Routes())
|
||||
|
||||
r.Route("/remotes/{name}/objects", func(r chi.Router) {
|
||||
objHandler := v2.NewObjectsHandler(s.db)
|
||||
r.Get("/", objHandler.Routes().ServeHTTP)
|
||||
r.Delete("/*", objHandler.Routes().ServeHTTP)
|
||||
})
|
||||
|
||||
r.Route("/remotes/{name}/files", func(r chi.Router) {
|
||||
r.Put("/*", s.localHandler.Routes().ServeHTTP)
|
||||
r.Get("/*", s.localHandler.Routes().ServeHTTP)
|
||||
r.Delete("/*", s.localHandler.Routes().ServeHTTP)
|
||||
})
|
||||
})
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprint(w, `{"status":"ok"}`)
|
||||
}
|
||||
|
||||
func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, `{"name":"artifactapi","version":"%s"}`, s.version)
|
||||
}
|
||||
|
||||
func (s *Server) newHTTPServer() *http.Server {
|
||||
return &http.Server{
|
||||
Addr: s.cfg.ListenAddr,
|
||||
Handler: s.router,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 300 * time.Second,
|
||||
IdleTimeout: 120 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) Run(ctx context.Context) error {
|
||||
go s.gc.Run(ctx)
|
||||
|
||||
httpServer := s.newHTTPServer()
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
slog.Info("shutting down server")
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = httpServer.Shutdown(shutdownCtx)
|
||||
}()
|
||||
|
||||
slog.Info("starting server", "addr", s.cfg.ListenAddr)
|
||||
if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) RunOnListener(ctx context.Context, ln net.Listener) error {
|
||||
go s.gc.Run(ctx)
|
||||
|
||||
httpServer := s.newHTTPServer()
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = httpServer.Shutdown(shutdownCtx)
|
||||
}()
|
||||
|
||||
slog.Info("starting server", "addr", ln.Addr().String())
|
||||
if err := httpServer.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type CAS struct {
|
||||
s3 *S3
|
||||
}
|
||||
|
||||
func NewCAS(s3 *S3) *CAS {
|
||||
return &CAS{s3: s3}
|
||||
}
|
||||
|
||||
type CASResult struct {
|
||||
ContentHash string
|
||||
S3Key string
|
||||
SizeBytes int64
|
||||
AlreadyExists bool
|
||||
}
|
||||
|
||||
func (c *CAS) Store(ctx context.Context, reader io.Reader, contentType string) (*CASResult, error) {
|
||||
tmp, err := os.CreateTemp("", "artifact-*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
defer tmp.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
size, err := io.Copy(io.MultiWriter(tmp, hasher), reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("write temp file: %w", err)
|
||||
}
|
||||
|
||||
hash := hex.EncodeToString(hasher.Sum(nil))
|
||||
s3Key := BlobKey(hash)
|
||||
|
||||
exists, err := c.s3.Exists(ctx, s3Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("check blob exists: %w", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, fmt.Errorf("seek temp file: %w", err)
|
||||
}
|
||||
if err := c.s3.Upload(ctx, s3Key, tmp, size, contentType); err != nil {
|
||||
return nil, fmt.Errorf("upload blob: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &CASResult{
|
||||
ContentHash: fmt.Sprintf("sha256:%s", hash),
|
||||
S3Key: s3Key,
|
||||
SizeBytes: size,
|
||||
AlreadyExists: exists,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func BlobKey(hash string) string {
|
||||
return fmt.Sprintf("blobs/sha256/%s", hash)
|
||||
}
|
||||
|
||||
func IndexKey(remote, path string) string {
|
||||
return fmt.Sprintf("indexes/%s/%s", remote, path)
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type S3 struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
func NewS3(endpoint, accessKey, secretKey, bucket string, secure bool, region string) (*S3, error) {
|
||||
opts := &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: secure,
|
||||
}
|
||||
if region != "" {
|
||||
opts.Region = region
|
||||
}
|
||||
|
||||
client, err := minio.New(endpoint, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create s3 client: %w", err)
|
||||
}
|
||||
|
||||
s := &S3{client: client, bucket: bucket}
|
||||
|
||||
if err := s.ensureBucket(context.Background()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *S3) ensureBucket(ctx context.Context) error {
|
||||
exists, err := s.client.BucketExists(ctx, s.bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
return fmt.Errorf("create bucket: %w", err)
|
||||
}
|
||||
slog.Info("created bucket", "bucket", s.bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3) Upload(ctx context.Context, key string, reader io.Reader, size int64, contentType string) error {
|
||||
_, err := s.client.PutObject(ctx, s.bucket, key, reader, size, minio.PutObjectOptions{
|
||||
ContentType: contentType,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3) Download(ctx context.Context, key string) (io.ReadCloser, *minio.ObjectInfo, error) {
|
||||
obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
info, err := obj.Stat()
|
||||
if err != nil {
|
||||
obj.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return obj, &info, nil
|
||||
}
|
||||
|
||||
func (s *S3) Exists(ctx context.Context, key string) (bool, error) {
|
||||
_, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
resp := minio.ToErrorResponse(err)
|
||||
if resp.Code == "NoSuchKey" {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *S3) Delete(ctx context.Context, key string) error {
|
||||
return s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (s *S3) Stat(ctx context.Context, key string) (*minio.ObjectInfo, error) {
|
||||
info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &info, nil
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/client"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type view int
|
||||
|
||||
const (
|
||||
viewDashboard view = iota
|
||||
viewRemotes
|
||||
viewRemoteDetail
|
||||
viewObjects
|
||||
viewVirtuals
|
||||
)
|
||||
|
||||
type model struct {
|
||||
client *client.Client
|
||||
view view
|
||||
width int
|
||||
height int
|
||||
err error
|
||||
loading bool
|
||||
|
||||
stats *models.OverviewStats
|
||||
remotes []models.Remote
|
||||
virtuals []models.Virtual
|
||||
objects []models.Artifact
|
||||
|
||||
selectedRemote string
|
||||
cursor int
|
||||
page int
|
||||
}
|
||||
|
||||
func New(endpoint string) *model {
|
||||
return &model{
|
||||
client: client.New(endpoint),
|
||||
view: viewDashboard,
|
||||
loading: true,
|
||||
page: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) Run() error {
|
||||
p := tea.NewProgram(m, tea.WithAltScreen())
|
||||
_, err := p.Run()
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *model) Init() tea.Cmd {
|
||||
return m.loadDashboard()
|
||||
}
|
||||
|
||||
type dashboardLoaded struct {
|
||||
stats *models.OverviewStats
|
||||
remotes []models.Remote
|
||||
virtuals []models.Virtual
|
||||
}
|
||||
|
||||
type remotesLoaded struct{ remotes []models.Remote }
|
||||
type virtualsLoaded struct{ virtuals []models.Virtual }
|
||||
type objectsLoaded struct{ objects []models.Artifact }
|
||||
type errMsg struct{ err error }
|
||||
|
||||
func (m *model) loadDashboard() tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
ctx := context.Background()
|
||||
stats, err := m.client.Stats(ctx)
|
||||
if err != nil {
|
||||
return errMsg{err}
|
||||
}
|
||||
remotes, _ := m.client.ListRemotes(ctx)
|
||||
virtuals, _ := m.client.ListVirtuals(ctx)
|
||||
return dashboardLoaded{stats: stats, remotes: remotes, virtuals: virtuals}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) loadRemotes() tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
remotes, err := m.client.ListRemotes(context.Background())
|
||||
if err != nil {
|
||||
return errMsg{err}
|
||||
}
|
||||
return remotesLoaded{remotes}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) loadVirtuals() tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
virtuals, err := m.client.ListVirtuals(context.Background())
|
||||
if err != nil {
|
||||
return errMsg{err}
|
||||
}
|
||||
return virtualsLoaded{virtuals}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) loadObjects() tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
objects, err := m.client.ListObjects(context.Background(), m.selectedRemote, m.page, 30)
|
||||
if err != nil {
|
||||
return errMsg{err}
|
||||
}
|
||||
return objectsLoaded{objects}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.WindowSizeMsg:
|
||||
m.width = msg.Width
|
||||
m.height = msg.Height
|
||||
return m, nil
|
||||
|
||||
case tea.KeyMsg:
|
||||
return m.handleKey(msg)
|
||||
|
||||
case dashboardLoaded:
|
||||
m.loading = false
|
||||
m.stats = msg.stats
|
||||
m.remotes = msg.remotes
|
||||
m.virtuals = msg.virtuals
|
||||
return m, nil
|
||||
|
||||
case remotesLoaded:
|
||||
m.loading = false
|
||||
m.remotes = msg.remotes
|
||||
m.cursor = 0
|
||||
return m, nil
|
||||
|
||||
case virtualsLoaded:
|
||||
m.loading = false
|
||||
m.virtuals = msg.virtuals
|
||||
m.cursor = 0
|
||||
return m, nil
|
||||
|
||||
case objectsLoaded:
|
||||
m.loading = false
|
||||
m.objects = msg.objects
|
||||
m.cursor = 0
|
||||
return m, nil
|
||||
|
||||
case errMsg:
|
||||
m.loading = false
|
||||
m.err = msg.err
|
||||
return m, nil
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *model) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.String() {
|
||||
case "q", "ctrl+c":
|
||||
if m.view == viewDashboard {
|
||||
return m, tea.Quit
|
||||
}
|
||||
m.view = viewDashboard
|
||||
m.cursor = 0
|
||||
m.loading = true
|
||||
return m, m.loadDashboard()
|
||||
|
||||
case "esc":
|
||||
switch m.view {
|
||||
case viewRemoteDetail, viewObjects:
|
||||
m.view = viewRemotes
|
||||
m.cursor = 0
|
||||
m.loading = true
|
||||
return m, m.loadRemotes()
|
||||
case viewRemotes, viewVirtuals:
|
||||
m.view = viewDashboard
|
||||
m.cursor = 0
|
||||
m.loading = true
|
||||
return m, m.loadDashboard()
|
||||
default:
|
||||
return m, tea.Quit
|
||||
}
|
||||
|
||||
case "1":
|
||||
m.view = viewDashboard
|
||||
m.loading = true
|
||||
return m, m.loadDashboard()
|
||||
|
||||
case "2":
|
||||
m.view = viewRemotes
|
||||
m.loading = true
|
||||
return m, m.loadRemotes()
|
||||
|
||||
case "3":
|
||||
m.view = viewVirtuals
|
||||
m.loading = true
|
||||
return m, m.loadVirtuals()
|
||||
|
||||
case "j", "down":
|
||||
m.cursor++
|
||||
m.clampCursor()
|
||||
return m, nil
|
||||
|
||||
case "k", "up":
|
||||
if m.cursor > 0 {
|
||||
m.cursor--
|
||||
}
|
||||
return m, nil
|
||||
|
||||
case "enter":
|
||||
return m.handleEnter()
|
||||
|
||||
case "r":
|
||||
m.loading = true
|
||||
switch m.view {
|
||||
case viewDashboard:
|
||||
return m, m.loadDashboard()
|
||||
case viewRemotes:
|
||||
return m, m.loadRemotes()
|
||||
case viewVirtuals:
|
||||
return m, m.loadVirtuals()
|
||||
case viewObjects:
|
||||
return m, m.loadObjects()
|
||||
}
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *model) handleEnter() (tea.Model, tea.Cmd) {
|
||||
switch m.view {
|
||||
case viewRemotes:
|
||||
if m.cursor < len(m.remotes) {
|
||||
m.selectedRemote = m.remotes[m.cursor].Name
|
||||
m.view = viewRemoteDetail
|
||||
return m, nil
|
||||
}
|
||||
case viewRemoteDetail:
|
||||
m.view = viewObjects
|
||||
m.page = 1
|
||||
m.loading = true
|
||||
return m, m.loadObjects()
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *model) clampCursor() {
|
||||
max := 0
|
||||
switch m.view {
|
||||
case viewRemotes:
|
||||
max = len(m.remotes) - 1
|
||||
case viewVirtuals:
|
||||
max = len(m.virtuals) - 1
|
||||
case viewObjects:
|
||||
max = len(m.objects) - 1
|
||||
}
|
||||
if m.cursor > max {
|
||||
m.cursor = max
|
||||
}
|
||||
if m.cursor < 0 {
|
||||
m.cursor = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) View() string {
|
||||
if m.loading {
|
||||
return m.chrome("Loading...")
|
||||
}
|
||||
if m.err != nil {
|
||||
return m.chrome(errStyle.Render(fmt.Sprintf("Error: %v", m.err)))
|
||||
}
|
||||
|
||||
var body string
|
||||
switch m.view {
|
||||
case viewDashboard:
|
||||
body = m.viewDashboard()
|
||||
case viewRemotes:
|
||||
body = m.viewRemotesList()
|
||||
case viewRemoteDetail:
|
||||
body = m.viewRemoteDetail()
|
||||
case viewObjects:
|
||||
body = m.viewObjectsList()
|
||||
case viewVirtuals:
|
||||
body = m.viewVirtualsList()
|
||||
}
|
||||
|
||||
return m.chrome(body)
|
||||
}
|
||||
|
||||
func (m *model) chrome(body string) string {
|
||||
nav := navStyle.Render(
|
||||
"[1] Dashboard [2] Remotes [3] Virtuals │ [r] Refresh [q] Quit",
|
||||
)
|
||||
return lipgloss.JoinVertical(lipgloss.Left, body, "", nav)
|
||||
}
|
||||
|
||||
var (
|
||||
titleStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("12"))
|
||||
navStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8"))
|
||||
errStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("9"))
|
||||
mutedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8"))
|
||||
selStyle = lipgloss.NewStyle().Background(lipgloss.Color("4")).Foreground(lipgloss.Color("15"))
|
||||
)
|
||||
@@ -0,0 +1,140 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/tui/views"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (m *model) viewDashboard() string {
|
||||
return titleStyle.Render("ArtifactAPI Dashboard") + "\n\n" +
|
||||
views.RenderDashboard(m.stats, len(m.remotes), len(m.virtuals)) +
|
||||
"\n\n" + mutedStyle.Render("Press [2] for remotes, [3] for virtuals")
|
||||
}
|
||||
|
||||
func (m *model) viewRemotesList() string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(titleStyle.Render("Remotes") + "\n\n")
|
||||
|
||||
if len(m.remotes) == 0 {
|
||||
sb.WriteString(mutedStyle.Render("No remotes configured"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
for i, r := range m.remotes {
|
||||
line := fmt.Sprintf(" %-25s %-12s %s", r.Name, r.PackageType, r.Description)
|
||||
if i == m.cursor {
|
||||
sb.WriteString(selStyle.Render(line))
|
||||
} else {
|
||||
sb.WriteString(line)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
sb.WriteString("\n" + mutedStyle.Render("j/k navigate · enter detail · esc back"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func (m *model) viewRemoteDetail() string {
|
||||
var r *remoteView
|
||||
for i := range m.remotes {
|
||||
if m.remotes[i].Name == m.selectedRemote {
|
||||
r = &remoteView{m.remotes[i]}
|
||||
break
|
||||
}
|
||||
}
|
||||
if r == nil {
|
||||
return mutedStyle.Render("Remote not found")
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString(titleStyle.Render(r.Name) + "\n\n")
|
||||
sb.WriteString(fmt.Sprintf(" Type: %s\n", r.PackageType))
|
||||
sb.WriteString(fmt.Sprintf(" Base URL: %s\n", r.BaseURL))
|
||||
sb.WriteString(fmt.Sprintf(" Description: %s\n", r.Description))
|
||||
sb.WriteString(fmt.Sprintf(" Immutable TTL: %s\n", ttlStr(r.ImmutableTTL)))
|
||||
sb.WriteString(fmt.Sprintf(" Mutable TTL: %ds\n", r.MutableTTL))
|
||||
sb.WriteString(fmt.Sprintf(" Revalidation: %v\n", r.CheckMutable))
|
||||
sb.WriteString(fmt.Sprintf(" Stale on Error: %v\n", r.StaleOnError))
|
||||
|
||||
if len(r.Patterns) > 0 {
|
||||
sb.WriteString(fmt.Sprintf(" Patterns: %s\n", strings.Join(r.Patterns, ", ")))
|
||||
}
|
||||
if len(r.Blocklist) > 0 {
|
||||
sb.WriteString(fmt.Sprintf(" Blocklist: %s\n", strings.Join(r.Blocklist, ", ")))
|
||||
}
|
||||
if r.ManagedBy != "" {
|
||||
sb.WriteString(fmt.Sprintf(" Managed by: %s\n", r.ManagedBy))
|
||||
}
|
||||
|
||||
sb.WriteString("\n" + mutedStyle.Render("enter → browse objects · esc back"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func (m *model) viewObjectsList() string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(titleStyle.Render(fmt.Sprintf("Objects: %s (page %d)", m.selectedRemote, m.page)) + "\n\n")
|
||||
|
||||
if len(m.objects) == 0 {
|
||||
sb.WriteString(mutedStyle.Render("No cached objects"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
for i, a := range m.objects {
|
||||
size := views.FormatBytes(a.SizeBytes)
|
||||
line := fmt.Sprintf(" %-50s %10s %5d hits", truncate(a.Path, 50), size, a.AccessCount)
|
||||
if i == m.cursor {
|
||||
sb.WriteString(selStyle.Render(line))
|
||||
} else {
|
||||
sb.WriteString(line)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
sb.WriteString("\n" + mutedStyle.Render("j/k navigate · esc back"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func (m *model) viewVirtualsList() string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(titleStyle.Render("Virtual Repositories") + "\n\n")
|
||||
|
||||
if len(m.virtuals) == 0 {
|
||||
sb.WriteString(mutedStyle.Render("No virtual repositories configured"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
for i, v := range m.virtuals {
|
||||
line := fmt.Sprintf(" %-25s %-12s %d members %s",
|
||||
v.Name, v.PackageType, len(v.Members), v.Description)
|
||||
if i == m.cursor {
|
||||
sb.WriteString(selStyle.Render(line))
|
||||
} else {
|
||||
sb.WriteString(line)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
sb.WriteString("\n" + mutedStyle.Render("j/k navigate · esc back"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
type remoteView struct {
|
||||
models.Remote
|
||||
}
|
||||
|
||||
func ttlStr(ttl int) string {
|
||||
if ttl == 0 {
|
||||
return "forever"
|
||||
}
|
||||
return fmt.Sprintf("%ds", ttl)
|
||||
}
|
||||
|
||||
func truncate(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max-3] + "..."
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package views
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func FormatBytes(bytes int64) string {
|
||||
if bytes == 0 {
|
||||
return "0 B"
|
||||
}
|
||||
units := []string{"B", "KB", "MB", "GB", "TB"}
|
||||
i := 0
|
||||
b := float64(bytes)
|
||||
for b >= 1024 && i < len(units)-1 {
|
||||
b /= 1024
|
||||
i++
|
||||
}
|
||||
if i == 0 {
|
||||
return fmt.Sprintf("%.0f %s", b, units[i])
|
||||
}
|
||||
return fmt.Sprintf("%.1f %s", b, units[i])
|
||||
}
|
||||
|
||||
func RenderDashboard(stats *models.OverviewStats, remoteCount, virtualCount int) string {
|
||||
if stats == nil {
|
||||
return "No stats available"
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"╭─ Dashboard ──────────────────────────────╮\n"+
|
||||
"│ Remotes: %-24d│\n"+
|
||||
"│ Cached Objects: %-24d│\n"+
|
||||
"│ Storage Used: %-24s│\n"+
|
||||
"│ Dedup Savings: %-20d blobs │\n"+
|
||||
"│ Virtuals: %-24d│\n"+
|
||||
"╰──────────────────────────────────────────╯",
|
||||
stats.TotalRemotes,
|
||||
stats.TotalObjects,
|
||||
FormatBytes(stats.TotalBytes),
|
||||
stats.TotalBlobsDeduped,
|
||||
virtualCount,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
db *database.DB
|
||||
proxyEngine *proxy.Engine
|
||||
}
|
||||
|
||||
func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
|
||||
return &Engine{db: db, proxyEngine: proxyEngine}
|
||||
}
|
||||
|
||||
func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) {
|
||||
merger, err := GetMerger(virt.PackageType)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("unsupported virtual type %q: %w", virt.PackageType, err)
|
||||
}
|
||||
|
||||
members, err := e.fetchMemberIndexes(ctx, virt, path)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
if len(members) == 0 {
|
||||
return nil, "", fmt.Errorf("no members reachable for virtual %q", virt.Name)
|
||||
}
|
||||
|
||||
merged, err := merger.MergeIndexes(members, proxyBaseURL)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("merge indexes: %w", err)
|
||||
}
|
||||
|
||||
contentType := "application/octet-stream"
|
||||
switch virt.PackageType {
|
||||
case models.PackageHelm:
|
||||
contentType = "text/yaml"
|
||||
case models.PackagePyPI:
|
||||
contentType = "text/html"
|
||||
}
|
||||
|
||||
return merged, contentType, nil
|
||||
}
|
||||
|
||||
func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, path string) ([]MemberIndex, error) {
|
||||
type result struct {
|
||||
index MemberIndex
|
||||
err error
|
||||
}
|
||||
|
||||
results := make([]result, len(virt.Members))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i, memberName := range virt.Members {
|
||||
wg.Add(1)
|
||||
go func(idx int, name string) {
|
||||
defer wg.Done()
|
||||
|
||||
remote, err := e.db.GetRemote(ctx, name)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("remote %q: %w", name, err)}
|
||||
return
|
||||
}
|
||||
|
||||
if remote.RepoType == models.RepoTypeLocal {
|
||||
body, err := e.fetchLocalIndex(ctx, *remote, path)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("local index %q: %w", name, err)}
|
||||
return
|
||||
}
|
||||
results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, BaseURL: remote.BaseURL, Body: body}}
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("provider %q: %w", remote.PackageType, err)}
|
||||
return
|
||||
}
|
||||
|
||||
fetchResult, err := e.proxyEngine.Fetch(ctx, *remote, path, prov)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("fetch %q/%s: %w", name, path, err)}
|
||||
return
|
||||
}
|
||||
defer fetchResult.Reader.Close()
|
||||
|
||||
body, err := io.ReadAll(fetchResult.Reader)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("read %q: %w", name, err)}
|
||||
return
|
||||
}
|
||||
|
||||
results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, BaseURL: remote.BaseURL, Body: body}}
|
||||
}(i, memberName)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
var members []MemberIndex
|
||||
for _, r := range results {
|
||||
if r.err != nil {
|
||||
slog.Warn("virtual member fetch failed", "error", r.err)
|
||||
continue
|
||||
}
|
||||
members = append(members, r.index)
|
||||
}
|
||||
|
||||
return members, nil
|
||||
}
|
||||
|
||||
func (e *Engine) fetchLocalIndex(ctx context.Context, remote models.Remote, path string) ([]byte, error) {
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no provider for %q: %w", remote.PackageType, err)
|
||||
}
|
||||
|
||||
indexer, ok := prov.(provider.LocalIndexer)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("provider %q does not support local index generation", remote.PackageType)
|
||||
}
|
||||
|
||||
return indexer.GenerateLocalIndex(ctx, e.db, remote.Name, path)
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterMerger(models.PackageHelm, &HelmMerger{})
|
||||
}
|
||||
|
||||
type HelmMerger struct{}
|
||||
|
||||
type helmIndex struct {
|
||||
APIVersion string `yaml:"apiVersion"`
|
||||
Entries map[string][]helmChartVersion `yaml:"entries"`
|
||||
Generated string `yaml:"generated,omitempty"`
|
||||
}
|
||||
|
||||
type helmChartVersion struct {
|
||||
Name string `yaml:"name"`
|
||||
Version string `yaml:"version"`
|
||||
URLs []string `yaml:"urls"`
|
||||
rest map[string]any
|
||||
}
|
||||
|
||||
func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
|
||||
merged := &helmIndex{
|
||||
APIVersion: "v1",
|
||||
Entries: make(map[string][]helmChartVersion),
|
||||
}
|
||||
|
||||
seen := map[string]map[string]bool{}
|
||||
|
||||
for _, member := range members {
|
||||
var idx helmIndex
|
||||
if err := yaml.Unmarshal(member.Body, &idx); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for chart, versions := range idx.Entries {
|
||||
if seen[chart] == nil {
|
||||
seen[chart] = map[string]bool{}
|
||||
}
|
||||
for _, ver := range versions {
|
||||
key := chart + ":" + ver.Version
|
||||
if seen[chart][ver.Version] {
|
||||
continue
|
||||
}
|
||||
seen[chart][ver.Version] = true
|
||||
|
||||
if proxyBaseURL != "" {
|
||||
routePrefix := "remote"
|
||||
if member.RepoType == "local" {
|
||||
routePrefix = "local"
|
||||
}
|
||||
baseHost := extractHost(member.BaseURL)
|
||||
|
||||
for i, u := range ver.URLs {
|
||||
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
|
||||
if baseHost != "" && extractHost(u) != baseHost {
|
||||
continue
|
||||
}
|
||||
relPath := extractPathRelativeToBase(u, member.BaseURL)
|
||||
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
|
||||
strings.TrimRight(proxyBaseURL, "/"),
|
||||
routePrefix,
|
||||
member.RemoteName,
|
||||
relPath)
|
||||
} else {
|
||||
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
|
||||
strings.TrimRight(proxyBaseURL, "/"),
|
||||
routePrefix,
|
||||
member.RemoteName,
|
||||
u)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
merged.Entries[chart] = append(merged.Entries[chart], ver)
|
||||
_ = key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return yaml.Marshal(merged)
|
||||
}
|
||||
|
||||
func extractHost(rawURL string) string {
|
||||
idx := strings.Index(rawURL, "://")
|
||||
if idx == -1 {
|
||||
return ""
|
||||
}
|
||||
rest := rawURL[idx+3:]
|
||||
slashIdx := strings.Index(rest, "/")
|
||||
if slashIdx == -1 {
|
||||
return rest
|
||||
}
|
||||
return rest[:slashIdx]
|
||||
}
|
||||
|
||||
func extractPathRelativeToBase(rawURL, baseURL string) string {
|
||||
fullPath := extractPath(rawURL)
|
||||
basePath := extractPath(baseURL)
|
||||
if basePath != "" {
|
||||
basePath = strings.TrimRight(basePath, "/") + "/"
|
||||
if strings.HasPrefix(fullPath, basePath) {
|
||||
return fullPath[len(basePath):]
|
||||
}
|
||||
}
|
||||
return fullPath
|
||||
}
|
||||
|
||||
func extractPath(rawURL string) string {
|
||||
idx := strings.Index(rawURL, "://")
|
||||
if idx == -1 {
|
||||
return rawURL
|
||||
}
|
||||
rest := rawURL[idx+3:]
|
||||
slashIdx := strings.Index(rest, "/")
|
||||
if slashIdx == -1 {
|
||||
return ""
|
||||
}
|
||||
return rest[slashIdx+1:]
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package virtual_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
func TestHelmMerger_BasicMerge(t *testing.T) {
|
||||
m := &virtual.HelmMerger{}
|
||||
|
||||
member1 := virtual.MemberIndex{
|
||||
RemoteName: "repo-a",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
nginx:
|
||||
- name: nginx
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- https://charts-a.example.com/nginx-1.0.0.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
member2 := virtual.MemberIndex{
|
||||
RemoteName: "repo-b",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
redis:
|
||||
- name: redis
|
||||
version: "2.0.0"
|
||||
urls:
|
||||
- https://charts-b.example.com/redis-2.0.0.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
if !strings.Contains(body, "nginx") {
|
||||
t.Error("expected nginx in merged index")
|
||||
}
|
||||
if !strings.Contains(body, "redis") {
|
||||
t.Error("expected redis in merged index")
|
||||
}
|
||||
if !strings.Contains(body, "proxy.example.com/api/v1/remote/repo-a") {
|
||||
t.Error("expected proxy URL for repo-a")
|
||||
}
|
||||
if !strings.Contains(body, "proxy.example.com/api/v1/remote/repo-b") {
|
||||
t.Error("expected proxy URL for repo-b")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmMerger_Dedup(t *testing.T) {
|
||||
m := &virtual.HelmMerger{}
|
||||
|
||||
idx := []byte(`apiVersion: v1
|
||||
entries:
|
||||
nginx:
|
||||
- name: nginx
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- nginx-1.0.0.tgz
|
||||
`)
|
||||
|
||||
members := []virtual.MemberIndex{
|
||||
{RemoteName: "repo-a", Body: idx},
|
||||
{RemoteName: "repo-b", Body: idx},
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes(members, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
count := strings.Count(string(result), "name: nginx")
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 entry for nginx, got %d\n%s", count, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmMerger_PriorityOrder(t *testing.T) {
|
||||
m := &virtual.HelmMerger{}
|
||||
|
||||
member1 := virtual.MemberIndex{
|
||||
RemoteName: "priority-repo",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
chart:
|
||||
- name: chart
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- chart-from-priority.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
member2 := virtual.MemberIndex{
|
||||
RemoteName: "fallback-repo",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
chart:
|
||||
- name: chart
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- chart-from-fallback.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
if !strings.Contains(body, "priority-repo") {
|
||||
t.Error("expected priority repo URL to win")
|
||||
}
|
||||
if strings.Contains(body, "fallback-repo") {
|
||||
t.Error("expected fallback repo to be excluded for duplicate")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type MemberIndex struct {
|
||||
RemoteName string
|
||||
RepoType models.RepoType
|
||||
BaseURL string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
|
||||
var mergers = map[models.PackageType]IndexMerger{}
|
||||
|
||||
func RegisterMerger(pt models.PackageType, m IndexMerger) {
|
||||
mergers[pt] = m
|
||||
}
|
||||
|
||||
func GetMerger(pt models.PackageType) (IndexMerger, error) {
|
||||
m, ok := mergers[pt]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no merger registered for package type %q", pt)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterMerger(models.PackagePyPI, &PyPIMerger{})
|
||||
}
|
||||
|
||||
type PyPIMerger struct{}
|
||||
|
||||
func (m *PyPIMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
|
||||
links := map[string]string{}
|
||||
|
||||
for _, member := range members {
|
||||
body := string(member.Body)
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "<a ") {
|
||||
continue
|
||||
}
|
||||
|
||||
href := extractHref(line)
|
||||
text := extractLinkText(line)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, exists := links[text]; exists {
|
||||
continue
|
||||
}
|
||||
|
||||
if proxyBaseURL != "" && href != "" {
|
||||
routePrefix := "remote"
|
||||
if member.RepoType == "local" {
|
||||
routePrefix = "local"
|
||||
}
|
||||
href = fmt.Sprintf("%s/api/v1/%s/%s/%s",
|
||||
strings.TrimRight(proxyBaseURL, "/"),
|
||||
routePrefix,
|
||||
member.RemoteName,
|
||||
strings.TrimLeft(href, "/"))
|
||||
}
|
||||
|
||||
links[text] = href
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(links))
|
||||
for k := range links {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, name := range keys {
|
||||
sb.WriteString(fmt.Sprintf(" <a href=\"%s\">%s</a>\n", links[name], name))
|
||||
}
|
||||
sb.WriteString("</body></html>\n")
|
||||
|
||||
return []byte(sb.String()), nil
|
||||
}
|
||||
|
||||
func extractHref(tag string) string {
|
||||
idx := strings.Index(tag, `href="`)
|
||||
if idx == -1 {
|
||||
return ""
|
||||
}
|
||||
rest := tag[idx+6:]
|
||||
end := strings.Index(rest, `"`)
|
||||
if end == -1 {
|
||||
return rest
|
||||
}
|
||||
return rest[:end]
|
||||
}
|
||||
|
||||
func extractLinkText(tag string) string {
|
||||
start := strings.Index(tag, ">")
|
||||
if start == -1 {
|
||||
return ""
|
||||
}
|
||||
rest := tag[start+1:]
|
||||
end := strings.Index(rest, "</a>")
|
||||
if end == -1 {
|
||||
return strings.TrimSpace(rest)
|
||||
}
|
||||
return strings.TrimSpace(rest[:end])
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package virtual_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
func TestPyPIMerger_BasicMerge(t *testing.T) {
|
||||
m := &virtual.PyPIMerger{}
|
||||
|
||||
member1 := virtual.MemberIndex{
|
||||
RemoteName: "pypi-a",
|
||||
Body: []byte(`<!DOCTYPE html>
|
||||
<html><body>
|
||||
<a href="/simple/requests/">requests</a>
|
||||
<a href="/simple/flask/">flask</a>
|
||||
</body></html>`),
|
||||
}
|
||||
|
||||
member2 := virtual.MemberIndex{
|
||||
RemoteName: "pypi-b",
|
||||
Body: []byte(`<!DOCTYPE html>
|
||||
<html><body>
|
||||
<a href="/simple/django/">django</a>
|
||||
</body></html>`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
if !strings.Contains(body, "requests") {
|
||||
t.Error("expected requests")
|
||||
}
|
||||
if !strings.Contains(body, "flask") {
|
||||
t.Error("expected flask")
|
||||
}
|
||||
if !strings.Contains(body, "django") {
|
||||
t.Error("expected django")
|
||||
}
|
||||
if !strings.Contains(body, "proxy.example.com/api/v1/remote/pypi-a") {
|
||||
t.Error("expected proxy URL for pypi-a")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyPIMerger_Dedup(t *testing.T) {
|
||||
m := &virtual.PyPIMerger{}
|
||||
|
||||
idx := []byte(`<html><body>
|
||||
<a href="/simple/requests/">requests</a>
|
||||
</body></html>`)
|
||||
|
||||
members := []virtual.MemberIndex{
|
||||
{RemoteName: "a", Body: idx},
|
||||
{RemoteName: "b", Body: idx},
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes(members, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
count := strings.Count(string(result), "<a ")
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 <a> tag for deduplicated requests, got %d\n%s", count, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyPIMerger_Sorted(t *testing.T) {
|
||||
m := &virtual.PyPIMerger{}
|
||||
|
||||
member := virtual.MemberIndex{
|
||||
RemoteName: "pypi",
|
||||
Body: []byte(`<html><body>
|
||||
<a href="/z/">zebra</a>
|
||||
<a href="/a/">alpha</a>
|
||||
<a href="/m/">middle</a>
|
||||
</body></html>`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
alphaIdx := strings.Index(body, "alpha")
|
||||
middleIdx := strings.Index(body, "middle")
|
||||
zebraIdx := strings.Index(body, "zebra")
|
||||
|
||||
if alphaIdx > middleIdx || middleIdx > zebraIdx {
|
||||
t.Error("expected sorted output")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
baseURL string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func New(baseURL string) *Client {
|
||||
return &Client{
|
||||
baseURL: baseURL,
|
||||
httpClient: http.DefaultClient,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) get(ctx context.Context, path string, out any) error {
|
||||
return c.do(ctx, http.MethodGet, path, nil, out)
|
||||
}
|
||||
|
||||
func (c *Client) post(ctx context.Context, path string, body any, out any) error {
|
||||
return c.do(ctx, http.MethodPost, path, body, out)
|
||||
}
|
||||
|
||||
func (c *Client) put(ctx context.Context, path string, body any, out any) error {
|
||||
return c.do(ctx, http.MethodPut, path, body, out)
|
||||
}
|
||||
|
||||
func (c *Client) delete(ctx context.Context, path string) error {
|
||||
return c.do(ctx, http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *Client) do(ctx context.Context, method, path string, body any, out any) error {
|
||||
var bodyReader io.Reader
|
||||
if body != nil {
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal: %w", err)
|
||||
}
|
||||
bodyReader = bytes.NewReader(b)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, bodyReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request: %w", err)
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("do: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("api error %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
|
||||
if out != nil && resp.StatusCode != http.StatusNoContent {
|
||||
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
|
||||
return fmt.Errorf("decode: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (c *Client) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
var remotes []models.Remote
|
||||
err := c.get(ctx, "/api/v2/remotes", &remotes)
|
||||
return remotes, err
|
||||
}
|
||||
|
||||
func (c *Client) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
|
||||
var remote models.Remote
|
||||
err := c.get(ctx, fmt.Sprintf("/api/v2/remotes/%s", name), &remote)
|
||||
return &remote, err
|
||||
}
|
||||
|
||||
func (c *Client) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
return c.post(ctx, "/api/v2/remotes", r, r)
|
||||
}
|
||||
|
||||
func (c *Client) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
return c.put(ctx, fmt.Sprintf("/api/v2/remotes/%s", r.Name), r, r)
|
||||
}
|
||||
|
||||
func (c *Client) DeleteRemote(ctx context.Context, name string) error {
|
||||
return c.delete(ctx, fmt.Sprintf("/api/v2/remotes/%s", name))
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (c *Client) Stats(ctx context.Context) (*models.OverviewStats, error) {
|
||||
var stats models.OverviewStats
|
||||
err := c.get(ctx, "/api/v2/stats", &stats)
|
||||
return &stats, err
|
||||
}
|
||||
|
||||
func (c *Client) Health(ctx context.Context) (*models.RemoteHealth, error) {
|
||||
var health models.RemoteHealth
|
||||
err := c.get(ctx, "/api/v2/health", &health)
|
||||
return &health, err
|
||||
}
|
||||
|
||||
func (c *Client) ListObjects(ctx context.Context, remote string, page, perPage int) ([]models.Artifact, error) {
|
||||
var artifacts []models.Artifact
|
||||
err := c.get(ctx, fmt.Sprintf("/api/v2/remotes/%s/objects?page=%d&per_page=%d", remote, page, perPage), &artifacts)
|
||||
return artifacts, err
|
||||
}
|
||||
|
||||
func (c *Client) EvictObject(ctx context.Context, remote, path string) error {
|
||||
return c.delete(ctx, fmt.Sprintf("/api/v2/remotes/%s/objects/%s", remote, path))
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (c *Client) ListVirtuals(ctx context.Context) ([]models.Virtual, error) {
|
||||
var virtuals []models.Virtual
|
||||
err := c.get(ctx, "/api/v2/virtuals", &virtuals)
|
||||
return virtuals, err
|
||||
}
|
||||
|
||||
func (c *Client) GetVirtual(ctx context.Context, name string) (*models.Virtual, error) {
|
||||
var virt models.Virtual
|
||||
err := c.get(ctx, fmt.Sprintf("/api/v2/virtuals/%s", name), &virt)
|
||||
return &virt, err
|
||||
}
|
||||
|
||||
func (c *Client) CreateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
return c.post(ctx, "/api/v2/virtuals", v, v)
|
||||
}
|
||||
|
||||
func (c *Client) UpdateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
return c.put(ctx, fmt.Sprintf("/api/v2/virtuals/%s", v.Name), v, v)
|
||||
}
|
||||
|
||||
func (c *Client) DeleteVirtual(ctx context.Context, name string) error {
|
||||
return c.delete(ctx, fmt.Sprintf("/api/v2/virtuals/%s", name))
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Blob struct {
|
||||
ContentHash string `json:"content_hash"`
|
||||
S3Key string `json:"s3_key"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ContentType string `json:"content_type"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type Artifact struct {
|
||||
ID int64 `json:"id"`
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
UpstreamETag string `json:"upstream_etag,omitempty"`
|
||||
UpstreamLastModified *time.Time `json:"upstream_last_modified,omitempty"`
|
||||
FirstSeenAt time.Time `json:"first_seen_at"`
|
||||
LastFetchedAt time.Time `json:"last_fetched_at"`
|
||||
LastAccessedAt time.Time `json:"last_accessed_at"`
|
||||
FetchCount int64 `json:"fetch_count"`
|
||||
AccessCount int64 `json:"access_count"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
}
|
||||
|
||||
type AccessLogEntry struct {
|
||||
ID int64 `json:"id"`
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
CacheHit bool `json:"cache_hit"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
UpstreamMS int `json:"upstream_ms"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user