Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 unkinben
|
1cca9fef00 |
+46
-5
@@ -1,6 +1,47 @@
|
||||
bin/
|
||||
/terraform/
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# e2e-docker fixtures are real package files (.rpm, .tgz, .whl, .zip, ...) that
|
||||
# are intentionally tracked, overriding any global ignore of those extensions.
|
||||
!e2e-docker/fixtures/**
|
||||
# Virtual environment
|
||||
.venv/
|
||||
venv/
|
||||
ENV/
|
||||
env/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
remotes.yaml
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# uv
|
||||
uv.lock
|
||||
|
||||
# Docker volumes
|
||||
minio_data/
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v5.0.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
- id: check-added-large-files
|
||||
- id: check-merge-conflict
|
||||
|
||||
- repo: https://github.com/dnephin/pre-commit-golang
|
||||
rev: v0.5.1
|
||||
hooks:
|
||||
- id: go-fmt
|
||||
- id: go-mod-tidy
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: go-vet
|
||||
name: go vet
|
||||
entry: go vet ./...
|
||||
language: system
|
||||
types: [go]
|
||||
pass_filenames: false
|
||||
@@ -1,9 +0,0 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: docker-build
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
repo: git.unkin.net/unkin/artifactapi
|
||||
dry_run: true
|
||||
@@ -1,34 +0,0 @@
|
||||
when:
|
||||
- event: tag
|
||||
ref: refs/tags/v*
|
||||
|
||||
steps:
|
||||
- name: docker-api
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi
|
||||
build_args:
|
||||
VERSION: ${CI_COMMIT_TAG}
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
|
||||
- name: docker-web
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi-ui
|
||||
dockerfile: ui/Dockerfile.ui
|
||||
context: ui
|
||||
build_args:
|
||||
BASE_PATH: /ui
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
@@ -1,17 +0,0 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: pre-commit
|
||||
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
||||
commands:
|
||||
- uvx pre-commit run --all-files
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
@@ -1,8 +0,0 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- go test -race -count=1 ./pkg/... ./internal/...
|
||||
+43
-11
@@ -1,21 +1,53 @@
|
||||
FROM golang:1.25-alpine AS builder
|
||||
# Use Alpine Linux as base image
|
||||
FROM python:3.11-alpine
|
||||
|
||||
RUN apk add --no-cache git
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
WORKDIR /build
|
||||
# Install system dependencies
|
||||
RUN apk add --no-cache \
|
||||
gcc \
|
||||
musl-dev \
|
||||
libffi-dev \
|
||||
postgresql-dev \
|
||||
curl \
|
||||
wget \
|
||||
tar
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
# Install uv
|
||||
ARG PACKAGE_VERSION=0.9.21
|
||||
RUN wget -O /app/uv-x86_64-unknown-linux-musl.tar.gz https://github.com/astral-sh/uv/releases/download/${PACKAGE_VERSION}/uv-x86_64-unknown-linux-musl.tar.gz && \
|
||||
tar xf /app/uv-x86_64-unknown-linux-musl.tar.gz -C /app && \
|
||||
mv /app/uv-x86_64-unknown-linux-musl/uv /usr/local/bin/uv && \
|
||||
rm -rf /app/uv-x86_64-unknown-linux-musl* && \
|
||||
chmod +x /usr/local/bin/uv && \
|
||||
uv --version
|
||||
|
||||
COPY . .
|
||||
# Copy CA bundle from host
|
||||
COPY ca-bundle.pem /app/ca-bundle.pem
|
||||
RUN chmod 644 /app/ca-bundle.pem
|
||||
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=${VERSION}" -o artifactapi ./cmd/artifactapi
|
||||
# Create non-root user first
|
||||
RUN adduser -D -s /bin/sh appuser && \
|
||||
chown -R appuser:appuser /app
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:nonroot
|
||||
# Copy dependency files and change ownership
|
||||
COPY --chown=appuser:appuser pyproject.toml uv.lock README.md ./
|
||||
|
||||
COPY --from=builder /build/artifactapi /usr/local/bin/artifactapi
|
||||
# Switch to appuser and install Python dependencies
|
||||
USER appuser
|
||||
RUN uv sync --frozen
|
||||
|
||||
# Copy application source
|
||||
COPY --chown=appuser:appuser src/ ./src/
|
||||
COPY --chown=appuser:appuser remotes.yaml ./
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8000
|
||||
|
||||
ENTRYPOINT ["artifactapi"]
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD curl -f http://localhost:8000/health || exit 1
|
||||
|
||||
# Run the application
|
||||
CMD ["uv", "run", "python", "-m", "src.artifactapi.main"]
|
||||
@@ -1,71 +1,49 @@
|
||||
.PHONY: build test lint fmt e2e docker-e2e docker docker-ui compose clean tidy check-go
|
||||
.PHONY: build install dev clean test lint format docker-build docker-up docker-down docker-logs docker-rebuild docker-clean docker-restart
|
||||
|
||||
BINARY := bin/artifactapi
|
||||
MODULE := git.unkin.net/unkin/artifactapi
|
||||
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0.0-dev")
|
||||
GO_VERSION_REQUIRED := 1.23
|
||||
GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/')
|
||||
build:
|
||||
docker build --no-cache -t artifactapi:latest .
|
||||
|
||||
check-go:
|
||||
@if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \
|
||||
echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \
|
||||
fi
|
||||
install: build
|
||||
|
||||
build: check-go tidy
|
||||
go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(BINARY) ./cmd/artifactapi
|
||||
docker-build: build
|
||||
|
||||
test: check-go
|
||||
go test -race -count=1 ./pkg/... ./internal/...
|
||||
|
||||
lint: check-go
|
||||
golangci-lint run ./...
|
||||
go vet ./...
|
||||
|
||||
fmt: check-go
|
||||
gofmt -w .
|
||||
goimports -w .
|
||||
|
||||
e2e: check-go
|
||||
TESTCONTAINERS_RYUK_DISABLED=true go test -tags=e2e -race -count=1 -timeout=5m ./e2e/...
|
||||
|
||||
# Build the container, bring up the full docker-compose stack + a mock upstream,
|
||||
# and run the black-box suite against the running product.
|
||||
docker-e2e: check-go
|
||||
./scripts/docker-e2e.sh
|
||||
|
||||
docker:
|
||||
docker build -t artifactapi:$(VERSION) .
|
||||
|
||||
docker-ui:
|
||||
docker build -t artifactapi-ui:$(VERSION) -f ui/Dockerfile.ui ui/
|
||||
|
||||
compose:
|
||||
docker compose up -d
|
||||
dev: build
|
||||
uv sync --dev
|
||||
|
||||
clean:
|
||||
rm -rf bin/
|
||||
rm -rf .venv
|
||||
rm -rf build/
|
||||
rm -rf dist/
|
||||
rm -rf *.egg-info/
|
||||
|
||||
tidy:
|
||||
go mod tidy
|
||||
test:
|
||||
uv run pytest
|
||||
|
||||
# Bump helpers — reads the latest semver tag and creates the next one.
|
||||
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
||||
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
||||
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
||||
_MIN := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f2)
|
||||
_PAT := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f3)
|
||||
lint:
|
||||
uv run ruff check --fix .
|
||||
|
||||
patch:
|
||||
@NEW=v$(_MAJ).$(_MIN).$(shell expr $(_PAT) + 1); \
|
||||
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||
format:
|
||||
uv run ruff format .
|
||||
|
||||
minor:
|
||||
@NEW=v$(_MAJ).$(shell expr $(_MIN) + 1).0; \
|
||||
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||
run:
|
||||
uv venv --python 3.11 && \
|
||||
source .venv/bin/activate && \
|
||||
uv run python -m src.artifactapi.main
|
||||
|
||||
major:
|
||||
@NEW=v$(shell expr $(_MAJ) + 1).0.0; \
|
||||
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||
docker-up:
|
||||
docker-compose up --build --force-recreate -d
|
||||
|
||||
_tag:
|
||||
git push origin $(TAG)
|
||||
docker-down:
|
||||
docker-compose down
|
||||
|
||||
docker-logs:
|
||||
docker-compose logs -f
|
||||
|
||||
docker-rebuild:
|
||||
docker-compose build --no-cache
|
||||
|
||||
docker-clean:
|
||||
docker-compose down -v --remove-orphans
|
||||
docker system prune -f
|
||||
|
||||
docker-restart: docker-down docker-up
|
||||
|
||||
@@ -1,880 +0,0 @@
|
||||
# ArtifactAPI v3 — Go Rewrite Plan
|
||||
|
||||
## Context
|
||||
|
||||
ArtifactAPI is a production artifact proxy/cache serving ~42 remotes (Docker registries, Helm repos, RPM/Alpine repos, GitHub releases, PyPI, npm, Puppet Forge, Terraform registries, Go module proxies) across a Kubernetes cluster. The current Python (FastAPI) implementation works but has architectural debt: opaque hashed S3 paths, no UI for visibility, YAML config files that drift, no garbage collection, no access logging, and virtual repos limited to Helm only.
|
||||
|
||||
The v3 rewrite targets: a single Go binary (API + TUI), a separate React frontend (own Dockerfile), a Terraform provider (separate repo), content-addressable storage, and a cleaner data model that makes the cache inspectable and manageable.
|
||||
|
||||
**Repo**: Same repo (`git.unkin.net/unkin/artifactapi`), new branch.
|
||||
**Module**: `git.unkin.net/unkin/artifactapi`
|
||||
**Frontend**: React + Vite, separate Dockerfile, talks to API
|
||||
**Terraform provider**: Separate repo (`terraform-provider-artifactapi`)
|
||||
|
||||
---
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
┌───────────────────────────────────┐ ┌──────────────────────┐
|
||||
│ Go Binary (API + TUI) │ │ Frontend Container │
|
||||
│ │ │ │
|
||||
│ ┌──────────┐ ┌───────────────┐ │ │ React + Vite SPA │
|
||||
│ │ REST API │ │ Proxy Engine │ │◄───│ nginx / node serve │
|
||||
│ │ /api/v2 │ │ /api/v1/... │ │ │ Dockerfile.ui │
|
||||
│ │ │ │ /v2/... (OCI) │ │ └──────────────────────┘
|
||||
│ └────┬─────┘ └──────┬────────┘ │
|
||||
│ │ │ │ ┌──────────────────────┐
|
||||
│ ┌────┴───────────────┴────────┐ │ │ Terraform Provider │
|
||||
│ │ Data Layer │ │◄───│ (separate repo) │
|
||||
│ │ PostgreSQL · Redis · S3 │ │ └──────────────────────┘
|
||||
│ └─────────────────────────────┘ │
|
||||
│ │ ┌──────────────────────┐
|
||||
│ ┌─────────────────────────────┐ │ │ TUI (subcommand) │
|
||||
│ │ artifactapi tui │──│───►│ artifactapi tui │
|
||||
│ └─────────────────────────────┘ │ │ --endpoint <url> │
|
||||
└───────────────────────────────────┘ └──────────────────────┘
|
||||
```
|
||||
|
||||
Three independent deployment units:
|
||||
1. **Go binary** — API server + TUI subcommand (single `Dockerfile`)
|
||||
2. **React frontend** — SPA served by nginx (`Dockerfile.ui`), talks to `/api/v2`
|
||||
3. **Terraform provider** — separate repo, calls `/api/v2` CRUD
|
||||
|
||||
---
|
||||
|
||||
## Project Structure (Modular)
|
||||
|
||||
```
|
||||
artifactapi/
|
||||
├── cmd/
|
||||
│ └── artifactapi/
|
||||
│ └── main.go # entrypoint: serve / tui subcommands
|
||||
│
|
||||
├── pkg/ # PUBLIC — importable by terraform provider, CLI tools
|
||||
│ ├── models/ # shared domain types
|
||||
│ │ ├── remote.go # Remote, RemoteConfig, PackageType enum
|
||||
│ │ ├── virtual.go # Virtual, VirtualConfig
|
||||
│ │ ├── artifact.go # Artifact, Blob, AccessLogEntry
|
||||
│ │ ├── local.go # LocalFile, LocalRepo
|
||||
│ │ └── stats.go # RemoteStats, OverviewStats
|
||||
│ └── client/ # typed Go API client (used by TUI + Terraform provider)
|
||||
│ ├── client.go # Client struct, base HTTP
|
||||
│ ├── remotes.go # remote CRUD methods
|
||||
│ ├── virtuals.go # virtual CRUD methods
|
||||
│ ├── objects.go # object browse/evict methods
|
||||
│ └── stats.go # stats methods
|
||||
│
|
||||
├── internal/ # PRIVATE — server internals
|
||||
│ ├── server/
|
||||
│ │ ├── server.go # HTTP server setup, router
|
||||
│ │ └── middleware.go # logging, recovery, request-id, access logging
|
||||
│ │
|
||||
│ ├── api/
|
||||
│ │ ├── v1/ # proxy endpoints (v1 compat)
|
||||
│ │ │ ├── proxy.go # GET /api/v1/remote/{name}/{path}
|
||||
│ │ │ ├── docker.go # /v2/{name}/{path}
|
||||
│ │ │ ├── virtual.go # GET /api/v1/virtual/{name}/{path}
|
||||
│ │ │ └── local.go # CRUD /api/v1/local/{name}/{path}
|
||||
│ │ └── v2/ # management API
|
||||
│ │ ├── remotes.go # CRUD + stats
|
||||
│ │ ├── virtuals.go # CRUD
|
||||
│ │ ├── objects.go # browse/evict cached objects
|
||||
│ │ ├── stats.go # overview, top-remotes
|
||||
│ │ ├── events.go # SSE stream
|
||||
│ │ └── health.go # health, metrics
|
||||
│ │
|
||||
│ ├── provider/ # package-type providers (registry protocol handlers)
|
||||
│ │ ├── provider.go # Provider interface + registry
|
||||
│ │ ├── generic/
|
||||
│ │ │ ├── generic.go
|
||||
│ │ │ └── generic_test.go
|
||||
│ │ ├── docker/
|
||||
│ │ │ ├── docker.go # OCI Distribution v2 via go-containerregistry
|
||||
│ │ │ ├── auth.go # Bearer token fetch + cache
|
||||
│ │ │ └── docker_test.go
|
||||
│ │ ├── helm/
|
||||
│ │ │ ├── helm.go # index rewriting via helm.sh/helm/v3/pkg/repo
|
||||
│ │ │ ├── merger.go # virtual index merge
|
||||
│ │ │ └── helm_test.go
|
||||
│ │ ├── pypi/
|
||||
│ │ │ ├── pypi.go # simple index HTML rewriting
|
||||
│ │ │ ├── merger.go # virtual simple index merge
|
||||
│ │ │ └── pypi_test.go
|
||||
│ │ ├── npm/
|
||||
│ │ │ ├── npm.go # metadata JSON rewriting
|
||||
│ │ │ └── npm_test.go
|
||||
│ │ ├── rpm/
|
||||
│ │ │ ├── rpm.go # repodata patterns
|
||||
│ │ │ └── rpm_test.go
|
||||
│ │ ├── alpine/
|
||||
│ │ │ ├── alpine.go # APKINDEX patterns
|
||||
│ │ │ └── alpine_test.go
|
||||
│ │ ├── puppet/
|
||||
│ │ │ ├── puppet.go # file_uri JSON rewriting
|
||||
│ │ │ └── puppet_test.go
|
||||
│ │ ├── terraform/
|
||||
│ │ │ ├── terraform.go # registry protocol, download URL rewriting
|
||||
│ │ │ └── terraform_test.go
|
||||
│ │ └── goproxy/
|
||||
│ │ ├── goproxy.go # Go module proxy protocol (GOPROXY)
|
||||
│ │ └── goproxy_test.go
|
||||
│ │
|
||||
│ ├── proxy/
|
||||
│ │ ├── engine.go # core fetch-or-cache logic
|
||||
│ │ ├── engine_test.go
|
||||
│ │ ├── classifier.go # immutable vs mutable classification
|
||||
│ │ ├── classifier_test.go
|
||||
│ │ ├── revalidator.go # conditional HEAD requests (ETag/Last-Modified)
|
||||
│ │ └── circuit.go # per-remote circuit breaker
|
||||
│ │
|
||||
│ ├── storage/
|
||||
│ │ ├── s3.go # S3 client (minio-go — works with MinIO, Ceph, AWS)
|
||||
│ │ ├── s3_test.go
|
||||
│ │ ├── cas.go # content-addressable store logic
|
||||
│ │ └── cas_test.go
|
||||
│ │
|
||||
│ ├── cache/
|
||||
│ │ ├── redis.go # TTL management, fetch locks
|
||||
│ │ ├── redis_test.go
|
||||
│ │ └── lock.go # distributed lock abstraction
|
||||
│ │
|
||||
│ ├── database/
|
||||
│ │ ├── postgres.go # connection pool, migration runner
|
||||
│ │ ├── queries/ # SQL query files or sqlc-generated code
|
||||
│ │ │ ├── remotes.sql.go
|
||||
│ │ │ ├── virtuals.sql.go
|
||||
│ │ │ ├── artifacts.sql.go
|
||||
│ │ │ └── access_log.sql.go
|
||||
│ │ └── migrations/ # golang-migrate SQL files
|
||||
│ │ ├── 001_initial.up.sql
|
||||
│ │ └── 001_initial.down.sql
|
||||
│ │
|
||||
│ ├── metrics/
|
||||
│ │ └── prometheus.go # counters, gauges, histograms
|
||||
│ │
|
||||
│ ├── gc/
|
||||
│ │ ├── gc.go # background garbage collection goroutine
|
||||
│ │ └── gc_test.go
|
||||
│ │
|
||||
│ ├── tui/
|
||||
│ │ ├── app.go # Bubble Tea main model
|
||||
│ │ ├── views/
|
||||
│ │ │ ├── dashboard.go
|
||||
│ │ │ ├── remotes.go
|
||||
│ │ │ ├── objects.go
|
||||
│ │ │ └── virtuals.go
|
||||
│ │ └── components/
|
||||
│ │ ├── table.go
|
||||
│ │ └── statusbar.go
|
||||
│ │
|
||||
│ └── config/
|
||||
│ └── env.go # environment variable parsing + validation
|
||||
│
|
||||
├── ui/ # React frontend — SEPARATE DOCKERFILE
|
||||
│ ├── src/
|
||||
│ │ ├── App.tsx
|
||||
│ │ ├── pages/
|
||||
│ │ │ ├── Dashboard.tsx
|
||||
│ │ │ ├── Remotes.tsx
|
||||
│ │ │ ├── RemoteDetail.tsx
|
||||
│ │ │ ├── Virtuals.tsx
|
||||
│ │ │ └── Objects.tsx
|
||||
│ │ ├── components/
|
||||
│ │ │ ├── RemoteTable.tsx
|
||||
│ │ │ ├── ObjectBrowser.tsx
|
||||
│ │ │ ├── StatsCard.tsx
|
||||
│ │ │ └── EventFeed.tsx
|
||||
│ │ └── api/
|
||||
│ │ └── client.ts # typed API client
|
||||
│ ├── package.json
|
||||
│ ├── vite.config.ts
|
||||
│ ├── tsconfig.json
|
||||
│ ├── Dockerfile.ui # multi-stage: node build → nginx
|
||||
│ └── nginx.conf # proxy /api/* to backend, serve SPA
|
||||
│
|
||||
├── e2e/ # end-to-end integration tests
|
||||
│ ├── e2e_test.go # TestMain spins up docker-compose stack
|
||||
│ ├── proxy_test.go # proxy through real remotes
|
||||
│ ├── docker_test.go # Docker v2 protocol e2e
|
||||
│ ├── management_test.go # v2 API CRUD
|
||||
│ ├── virtual_test.go # virtual repo merge e2e
|
||||
│ └── docker-compose.e2e.yml # postgres + redis + minio for tests
|
||||
│
|
||||
├── go.mod
|
||||
├── go.sum
|
||||
├── Makefile
|
||||
├── Dockerfile # Go binary (API server + TUI)
|
||||
├── Dockerfile.ui # symlink or copy → ui/Dockerfile.ui
|
||||
└── docker-compose.yml
|
||||
```
|
||||
|
||||
### Key Modularisation Decisions
|
||||
|
||||
- **`pkg/models/`** — Shared domain types importable by the Terraform provider and any external tooling. No dependencies on internal packages
|
||||
- **`pkg/client/`** — Typed Go API client used by both the TUI and the Terraform provider. Depends only on `pkg/models/` and stdlib
|
||||
- **`internal/provider/`** — Each package type is its own subpackage with isolated tests. A provider registry maps `PackageType → Provider`
|
||||
- **`internal/database/queries/`** — Use [sqlc](https://sqlc.dev/) to generate type-safe query functions from SQL, or hand-written query files
|
||||
- **`e2e/`** — Separate test binary that spins up a real docker-compose stack
|
||||
|
||||
---
|
||||
|
||||
## Go Ecosystem Libraries
|
||||
|
||||
Prefer existing, maintained Go modules over writing protocol handlers from scratch.
|
||||
|
||||
### Package-Type Libraries
|
||||
|
||||
| Package Type | Go Module | What It Gives Us |
|
||||
|---|---|---|
|
||||
| **Docker/OCI** | `github.com/google/go-containerregistry` | Full Registry v2/OCI client: manifest parsing, auth challenges, blob operations. `pkg/registry` can implement a v2 server. Reference: `github.com/regclient/regclient` |
|
||||
| **Helm** | `helm.sh/helm/v3/pkg/repo` | Parse/generate `index.yaml`, `IndexFile`/`ChartVersion` types, URL entries. Used directly for merge |
|
||||
| **Terraform** | `github.com/hashicorp/terraform-registry-address` | Provider/module address parsing, `ForRegistryProtocol()` URL generation. Protocol spec: provider registry protocol v1 |
|
||||
| **Go Modules** | `github.com/goproxy/goproxy` | Minimalist GOPROXY protocol handler, implements full spec as `http.Handler`. Handles `/@v/list`, `/@v/{v}.info`, `/@v/{v}.mod`, `/@v/{v}.zip`, `/@latest` |
|
||||
| **RPM** | `rs3.io/go/rpm/repomd` | Parse `repomd.xml`, `primary.xml` with proper XML namespace handling |
|
||||
| **Alpine** | `gitlab.alpinelinux.org/alpine/go` | Official Alpine library: parse APKINDEX, `.apk` files |
|
||||
| **PyPI** | stdlib `golang.org/x/net/html` | No dedicated Go PyPI library exists. Parse simple index HTML with `x/net/html`, extract `<a>` tags. Minimal — the rewriting is just href replacement |
|
||||
| **npm** | stdlib `encoding/json` | npm metadata is JSON — parse with stdlib, rewrite `dist.tarball` URLs. No special library needed |
|
||||
| **Puppet Forge** | stdlib `encoding/json` | Forge API is JSON — parse and rewrite `file_uri` fields. Community lib `github.com/johnmccabe/go-puppetforge` exists but is thin; stdlib suffices |
|
||||
|
||||
### Infrastructure Libraries
|
||||
|
||||
| Purpose | Go Module | Why This One |
|
||||
|---|---|---|
|
||||
| **HTTP router** | `github.com/go-chi/chi/v5` | Lightweight, stdlib `http.Handler` compatible, middleware chain |
|
||||
| **PostgreSQL** | `github.com/jackc/pgx/v5` | Pure Go, connection pooling, COPY support, prepared statements |
|
||||
| **SQL generation** | `github.com/sqlc-dev/sqlc` | Generate type-safe Go from SQL queries — no ORM, no reflection |
|
||||
| **Redis** | `github.com/redis/go-redis/v9` | Full Redis client, pipelining, pub/sub |
|
||||
| **S3 (MinIO/Ceph/AWS)** | `github.com/minio/minio-go/v7` | Native S3-compatible client. Works with MinIO, Ceph RGW, AWS S3, any S3-compatible backend out of the box. Lighter than aws-sdk-go-v2, purpose-built for S3 compat |
|
||||
| **DB migrations** | `github.com/golang-migrate/migrate/v4` | SQL file-based migrations, CLI + library |
|
||||
| **Prometheus** | `github.com/prometheus/client_golang` | Counters, gauges, histograms |
|
||||
| **TUI** | `github.com/charmbracelet/bubbletea` | Elm-architecture TUI framework |
|
||||
| **TUI styling** | `github.com/charmbracelet/lipgloss` | Terminal styling |
|
||||
| **TUI components** | `github.com/charmbracelet/bubbles` | Table, text input, spinner, etc. |
|
||||
| **Structured logging** | `log/slog` (stdlib) | Go 1.21+ structured logging, zero dependencies |
|
||||
| **Testing** | `github.com/stretchr/testify` | Assertions + require for unit tests |
|
||||
| **Test containers** | `github.com/testcontainers/testcontainers-go` | Spin up Postgres/Redis/MinIO in e2e tests |
|
||||
|
||||
### S3 Client: Multi-Backend Support
|
||||
|
||||
Using `minio-go/v7` as the S3 client because it natively supports:
|
||||
- **MinIO** — primary development/production target
|
||||
- **Ceph RGW** — S3-compatible via endpoint config
|
||||
- **AWS S3** — via region + credential config
|
||||
- **Any S3-compatible** — GCS (interop mode), Wasabi, DigitalOcean Spaces, etc.
|
||||
|
||||
No abstraction layer needed — `minio-go` handles endpoint differences internally. Config:
|
||||
```go
|
||||
client, _ := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: useTLS,
|
||||
Region: region, // optional, for AWS
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Data Layer
|
||||
|
||||
### PostgreSQL Schema
|
||||
|
||||
```sql
|
||||
-- Remotes: managed exclusively by Terraform
|
||||
CREATE TABLE remotes (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL, -- generic, docker, helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy
|
||||
base_url TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
immutable_ttl INTEGER DEFAULT 0,
|
||||
mutable_ttl INTEGER DEFAULT 3600,
|
||||
check_mutable BOOLEAN DEFAULT TRUE,
|
||||
immutable_patterns TEXT[] DEFAULT '{}', -- user-defined immutable patterns
|
||||
mutable_patterns TEXT[] DEFAULT '{}', -- user-defined mutable patterns (merged with provider built-ins)
|
||||
allowlist TEXT[] DEFAULT '{}', -- if empty, allow all paths; if non-empty, only matching paths proxied
|
||||
blocklist TEXT[] DEFAULT '{}', -- always denied, checked before allowlist
|
||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
||||
ban_tags TEXT[] DEFAULT '{}',
|
||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
||||
quarantine_days INTEGER DEFAULT 3,
|
||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
||||
releases_remote TEXT DEFAULT '', -- terraform type: name of CDN remote for download URL rewriting
|
||||
managed_by TEXT DEFAULT '', -- 'terraform' or empty
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Virtual repositories
|
||||
CREATE TABLE virtuals (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
members TEXT[] NOT NULL,
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Content-addressable blob storage tracking
|
||||
CREATE TABLE blobs (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
s3_key TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
content_type TEXT DEFAULT 'application/octet-stream',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Artifact metadata: maps (remote, path) → content blob
|
||||
CREATE TABLE artifacts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
upstream_etag TEXT DEFAULT '',
|
||||
upstream_last_modified TIMESTAMPTZ,
|
||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
fetch_count BIGINT DEFAULT 1,
|
||||
access_count BIGINT DEFAULT 1,
|
||||
UNIQUE(remote_name, path)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_artifacts_remote ON artifacts(remote_name);
|
||||
CREATE INDEX idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
||||
|
||||
-- Local file uploads
|
||||
CREATE TABLE local_files (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
-- Access log (append-only, powers dashboards)
|
||||
CREATE TABLE access_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
cache_hit BOOLEAN NOT NULL,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
upstream_ms INTEGER DEFAULT 0,
|
||||
client_ip TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
```
|
||||
|
||||
### Redis Usage (Ephemeral Only)
|
||||
|
||||
| Key pattern | Type | TTL | Purpose |
|
||||
|---|---|---|---|
|
||||
| `ttl:{remote}:{path}` | STRING | remote's immutable/mutable TTL | Artifact freshness — existence = still fresh |
|
||||
| `lock:{remote}:{path}` | STRING (NX) | 30s | Fetch lock — prevents thundering herd |
|
||||
| `etag:{remote}:{path}` | STRING | same as TTL key | Cached ETag for conditional revalidation |
|
||||
| `circuit:{remote}` | STRING | configurable | Circuit breaker — consecutive failure count |
|
||||
|
||||
Losing Redis = all TTLs expire = next request re-validates upstream. No data loss.
|
||||
|
||||
### S3 Layout (Content-Addressable)
|
||||
|
||||
```
|
||||
artifacts-bucket/
|
||||
├── blobs/sha256/{content_hash} # immutable CAS blobs
|
||||
├── indexes/{remote}/{path} # mutable index files (helm, pypi, rpm, etc.)
|
||||
├── indexes/{virtual}/{path} # merged virtual indexes
|
||||
└── local/{repo}/{path} # user uploads (CAS-backed via blobs table)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Terraform Remote Type (New in v2)
|
||||
|
||||
The `terraform` package type proxies the Terraform Provider Registry Protocol:
|
||||
|
||||
- **URL construction**: prepends `/v1/providers/` to request paths
|
||||
- **Built-in mutable pattern**: `[^/]+/[^/]+/versions$` (version listings change over time)
|
||||
- **Built-in immutable pattern**: `[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$` (per-version download info is fixed)
|
||||
- **Response rewriting**: download info JSON — rewrites `download_url`, `shasums_url`, `shasums_signature_url` to route through a companion `releases_remote` (e.g., `hashicorp-releases` generic remote)
|
||||
- **Config**: requires `releases_remote` field pointing to the CDN remote that serves the actual binaries
|
||||
|
||||
Uses `github.com/hashicorp/terraform-registry-address` for address parsing and protocol-compliant URL generation.
|
||||
|
||||
---
|
||||
|
||||
## Go Module Proxy Remote Type (New)
|
||||
|
||||
The `goproxy` package type implements the GOPROXY protocol (Go module proxy):
|
||||
|
||||
| Endpoint | Mutability | Description |
|
||||
|---|---|---|
|
||||
| `{module}/@v/list` | Mutable | Plain text list of known versions |
|
||||
| `{module}/@latest` | Mutable | JSON metadata for latest version |
|
||||
| `{module}/@v/{version}.info` | Immutable | JSON version metadata (`Version`, `Time`) |
|
||||
| `{module}/@v/{version}.mod` | Immutable | `go.mod` file for that version |
|
||||
| `{module}/@v/{version}.zip` | Immutable | Source archive for that version |
|
||||
|
||||
- **No URL rewriting needed** — responses are self-contained (no embedded URLs)
|
||||
- **Config**: `base_url` points to upstream proxy (e.g., `https://proxy.golang.org`)
|
||||
- **Client usage**: set `GOPROXY=https://artifactapi.example.com/api/v1/remote/goproxy`
|
||||
- Uses `github.com/goproxy/goproxy` for protocol handling
|
||||
|
||||
---
|
||||
|
||||
## Allowlist / Blocklist / Automatic Mutable Patterns
|
||||
|
||||
### Access Control (Per-Remote)
|
||||
|
||||
| Field | Default | Behavior |
|
||||
|---|---|---|
|
||||
| `blocklist` | `[]` (empty) | If a path matches any blocklist pattern → **403 Forbidden**. Checked first |
|
||||
| `allowlist` | `[]` (empty) | If empty → **allow everything**. If non-empty → only matching paths are proxied; everything else → **403** |
|
||||
|
||||
Evaluation order: blocklist → allowlist → proxy. No allowlist + no blocklist = open proxy (default).
|
||||
|
||||
### Automatic Mutable Patterns (Per-Provider Built-ins)
|
||||
|
||||
Each provider declares built-in mutable patterns that are **always merged** with user-defined `mutable_patterns`. Users never need to configure these — the provider knows which paths change over time.
|
||||
|
||||
| Provider | Built-in Mutable Patterns | Rationale |
|
||||
|---|---|---|
|
||||
| **generic** | *(none)* | No convention for what's mutable |
|
||||
| **docker** | `/manifests/(?!sha256:)[^/]+$`, `/tags/list$` | Tag manifests change; digest manifests don't |
|
||||
| **helm** | `index\.yaml$` | Chart index changes when new charts are published |
|
||||
| **pypi** | `simple/` | Package index pages change with new releases |
|
||||
| **npm** | `^[^/]+$` (package metadata, not `.tgz`) | Package metadata changes; tarballs are immutable |
|
||||
| **rpm** | `repomd\.xml$`, `repodata/.*`, `Packages\.gz$` | Repo metadata rebuilt on every publish |
|
||||
| **alpine** | `APKINDEX\.tar\.gz$` | Package index rebuilt on every publish |
|
||||
| **puppet** | `^v3/modules/`, `^v3/releases` | Module metadata changes with new releases |
|
||||
| **terraform** | `[^/]+/[^/]+/versions$` | Provider version listings grow over time |
|
||||
| **goproxy** | `@v/list$`, `@latest$` | Version list and latest pointer change |
|
||||
|
||||
These are returned by `Provider.BuiltinMutablePatterns()` and merged at classification time:
|
||||
```
|
||||
effective_mutable = provider.BuiltinMutablePatterns() ∪ remote.mutable_patterns
|
||||
```
|
||||
|
||||
If a path matches `effective_mutable` → use `mutable_ttl`. If it matches `remote.immutable_patterns` → use `immutable_ttl`. Immutable patterns take precedence over mutable when both match.
|
||||
|
||||
---
|
||||
|
||||
## API Design
|
||||
|
||||
### v1 Proxy Endpoints (Backwards Compatible)
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| `GET` | `/api/v1/remote/{name}/{path}` | Proxy/cache artifact |
|
||||
| `GET` | `/api/v1/virtual/{name}/{path}` | Virtual repo proxy |
|
||||
| `GET/HEAD` | `/v2/{name}/{path}` | Docker Registry v2 |
|
||||
| `GET` | `/v2/` | Docker v2 ping |
|
||||
| `GET/PUT/HEAD/DELETE` | `/api/v1/local/{name}/{path}` | Local repo CRUD |
|
||||
|
||||
### v2 Management API (New)
|
||||
|
||||
```
|
||||
GET /api/v2/remotes → [{name, package_type, base_url, description, stats}]
|
||||
GET /api/v2/remotes/{name} → {full config + stats + health}
|
||||
POST /api/v2/remotes → create remote (Terraform provider)
|
||||
PUT /api/v2/remotes/{name} → update remote (Terraform provider)
|
||||
DELETE /api/v2/remotes/{name} → delete remote — cascades artifacts, GC cleans S3
|
||||
|
||||
GET /api/v2/virtuals → [{name, package_type, members, stats}]
|
||||
GET /api/v2/virtuals/{name} → {full config + member details}
|
||||
POST /api/v2/virtuals → create virtual
|
||||
PUT /api/v2/virtuals/{name} → update virtual
|
||||
DELETE /api/v2/virtuals/{name} → delete virtual
|
||||
|
||||
GET /api/v2/remotes/{name}/objects → paginated objects
|
||||
?q=pattern&sort=size|accessed|age&page=1&per_page=50
|
||||
DELETE /api/v2/remotes/{name}/objects/{path} → evict specific cached object
|
||||
DELETE /api/v2/remotes/{name}/cache → flush cache
|
||||
?type=all|indexes|blobs
|
||||
|
||||
GET /api/v2/stats → overview stats
|
||||
GET /api/v2/stats/top-remotes → top remotes by size/requests/hit-rate
|
||||
|
||||
GET /api/v2/health → {status, postgres, redis, s3, uptime}
|
||||
GET /metrics → Prometheus format
|
||||
GET /api/v2/events → SSE stream
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Proxy Engine
|
||||
|
||||
### Request Flow
|
||||
|
||||
```
|
||||
Client Request
|
||||
│
|
||||
▼
|
||||
Classify (immutable/mutable/denied)
|
||||
│
|
||||
├── blocklist match → 403
|
||||
├── allowlist non-empty + no match → 403
|
||||
│
|
||||
▼
|
||||
Check Redis TTL key
|
||||
│
|
||||
├── exists (fresh) → serve from S3, log access
|
||||
│
|
||||
├── missing (expired or uncached)
|
||||
│ │
|
||||
│ ▼
|
||||
│ Acquire fetch lock (Redis SETNX, 30s TTL)
|
||||
│ │
|
||||
│ ├── lock acquired
|
||||
│ │ ├── mutable + check_mutable + have ETag → HEAD upstream
|
||||
│ │ │ ├── 304 → refresh TTL, serve from S3
|
||||
│ │ │ └── changed → full fetch
|
||||
│ │ └── full fetch from upstream
|
||||
│ │ → provider.RewriteResponse() if needed
|
||||
│ │ → CAS store (hash → check blobs → upload if new)
|
||||
│ │ → upsert artifact in Postgres
|
||||
│ │ → set Redis TTL + release lock
|
||||
│ │ → on upstream error + stale_on_error → refresh TTL, serve stale
|
||||
│ │
|
||||
│ └── lock not acquired → poll S3 briefly, serve if another pod fetched it
|
||||
│
|
||||
▼
|
||||
Stream response from S3, log access
|
||||
```
|
||||
|
||||
### Circuit Breaker
|
||||
|
||||
Per-remote, tracked in Redis. Closed → Open (after N failures) → Half-open (after cooldown). Exposed via `GET /api/v2/remotes/{name}` health field.
|
||||
|
||||
### Content-Addressable Storage
|
||||
|
||||
1. Stream upstream → temp file, compute SHA256 inline
|
||||
2. Check `blobs` table for hash
|
||||
3. Exists → skip S3 upload, upsert `artifacts` row only
|
||||
4. New → upload to `blobs/sha256/{hash}`, insert both rows
|
||||
|
||||
### Garbage Collection
|
||||
|
||||
Background goroutine (configurable interval, default 1h):
|
||||
1. Orphaned blobs: delete S3 objects whose `content_hash` has no referencing `artifacts` or `local_files` rows
|
||||
2. Cold artifacts: optional per-remote, delete artifacts not accessed in N days
|
||||
3. Remote deletion: `ON DELETE CASCADE` handles Postgres; GC sweeps orphaned blobs
|
||||
|
||||
---
|
||||
|
||||
## Package Providers
|
||||
|
||||
### Provider Interface
|
||||
|
||||
```go
|
||||
type Provider interface {
|
||||
Type() models.PackageType
|
||||
BuiltinMutablePatterns() []*regexp.Regexp
|
||||
BuiltinImmutablePatterns() []*regexp.Regexp
|
||||
ContentType(path string) string
|
||||
UpstreamURL(remote models.Remote, path string) string
|
||||
RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error)
|
||||
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
```
|
||||
|
||||
### Provider Registry
|
||||
|
||||
```go
|
||||
var registry = map[models.PackageType]Provider{
|
||||
models.PackageGeneric: &generic.Provider{},
|
||||
models.PackageDocker: &docker.Provider{},
|
||||
models.PackageHelm: &helm.Provider{},
|
||||
models.PackagePyPI: &pypi.Provider{},
|
||||
models.PackageNPM: &npm.Provider{},
|
||||
models.PackageRPM: &rpm.Provider{},
|
||||
models.PackageAlpine: &alpine.Provider{},
|
||||
models.PackagePuppet: &puppet.Provider{},
|
||||
models.PackageTerraform: &terraform.Provider{},
|
||||
models.PackageGoProxy: &goproxy.Provider{},
|
||||
}
|
||||
|
||||
func Get(t models.PackageType) (Provider, error) { ... }
|
||||
```
|
||||
|
||||
Each provider lives in its own subpackage under `internal/provider/` with its own `_test.go`.
|
||||
|
||||
---
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
Every package gets `_test.go` files alongside the source. Run with `go test ./...`.
|
||||
|
||||
| Package | What's Tested |
|
||||
|---|---|
|
||||
| `internal/provider/docker/` | Auth token parsing/caching, manifest classification, tag banning, URL construction, blob key generation |
|
||||
| `internal/provider/helm/` | `index.yaml` parsing (using `helm.sh/helm/v3/pkg/repo`), URL rewriting, index merging |
|
||||
| `internal/provider/pypi/` | Simple index HTML parsing, URL rewriting, index merging |
|
||||
| `internal/provider/npm/` | Metadata JSON rewriting (`dist.tarball` URLs) |
|
||||
| `internal/provider/terraform/` | Registry URL construction, download info JSON rewriting, `releases_remote` URL extraction |
|
||||
| `internal/provider/rpm/` | Mutable pattern matching (repodata) |
|
||||
| `internal/provider/alpine/` | Mutable pattern matching (APKINDEX) |
|
||||
| `internal/provider/puppet/` | `file_uri` JSON rewriting |
|
||||
| `internal/proxy/` | Classifier (immutable vs mutable vs denied), circuit breaker state transitions, revalidator logic |
|
||||
| `internal/storage/` | CAS key generation, dedup detection, S3 operation mocking |
|
||||
| `internal/cache/` | Redis TTL set/check, fetch lock acquire/release/contention |
|
||||
| `internal/gc/` | Orphan detection queries, cold artifact selection |
|
||||
| `pkg/models/` | Model validation, PackageType enum |
|
||||
| `pkg/client/` | API client request/response serialization |
|
||||
|
||||
### End-to-End Tests
|
||||
|
||||
Located in `e2e/`. Use `testcontainers-go` to spin up real Postgres, Redis, and MinIO containers. The test binary starts the actual `artifactapi` server against these backends.
|
||||
|
||||
```go
|
||||
// e2e/e2e_test.go
|
||||
func TestMain(m *testing.M) {
|
||||
// Start postgres, redis, minio via testcontainers-go
|
||||
// Run migrations
|
||||
// Start artifactapi server on random port
|
||||
// Run tests
|
||||
// Tear down
|
||||
}
|
||||
```
|
||||
|
||||
| Test File | What's Tested |
|
||||
|---|---|
|
||||
| `e2e/proxy_test.go` | Proxy a real GitHub release through generic remote, verify S3 storage, verify Redis TTL, verify Postgres artifact row, verify cache hit on second request |
|
||||
| `e2e/docker_test.go` | Pull a real image manifest + blob through Docker v2 proxy, verify blob deduplication, tag banning |
|
||||
| `e2e/management_test.go` | Full CRUD lifecycle: create remote via v2 API, proxy through it, list objects, evict object, flush cache, delete remote |
|
||||
| `e2e/virtual_test.go` | Create two helm remotes + virtual, fetch merged index, verify priority ordering |
|
||||
| `e2e/terraform_test.go` | Proxy terraform provider version listing + download info, verify URL rewriting to releases_remote |
|
||||
| `e2e/goproxy_test.go` | Proxy Go module `@v/list`, `.info`, `.mod`, `.zip` through GOPROXY remote, verify mutable vs immutable classification |
|
||||
| `e2e/gc_test.go` | Create artifact, delete remote, trigger GC, verify S3 blob cleaned up |
|
||||
|
||||
### Code Quality
|
||||
|
||||
- `gofmt` / `goimports` — enforced in CI, run on save
|
||||
- `golangci-lint` — comprehensive linter suite (staticcheck, errcheck, govet, etc.)
|
||||
- `go vet ./...` — run in CI
|
||||
- Makefile targets: `make test`, `make lint`, `make e2e`, `make fmt`
|
||||
|
||||
---
|
||||
|
||||
## Terraform Provider (Separate Repo)
|
||||
|
||||
**Repo**: `terraform-provider-artifactapi`
|
||||
**Uses**: `pkg/client/` and `pkg/models/` from the main module
|
||||
|
||||
```hcl
|
||||
provider "artifactapi" {
|
||||
endpoint = "https://artifactapi.k8s.syd1.au.unkin.net"
|
||||
}
|
||||
|
||||
resource "artifactapi_remote" "terraform_registry" {
|
||||
name = "terraform-registry"
|
||||
package_type = "terraform"
|
||||
base_url = "https://registry.terraform.io"
|
||||
description = "Terraform provider registry"
|
||||
releases_remote = artifactapi_remote.hashicorp_releases.name
|
||||
|
||||
immutable_patterns = [
|
||||
"[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$",
|
||||
]
|
||||
|
||||
cache {
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
}
|
||||
}
|
||||
|
||||
resource "artifactapi_remote" "hashicorp_releases" {
|
||||
name = "hashicorp-releases"
|
||||
package_type = "generic"
|
||||
base_url = "https://releases.hashicorp.com"
|
||||
|
||||
immutable_patterns = [
|
||||
".*\\.zip$",
|
||||
".*SHA256SUMS(\\.sig)?$",
|
||||
]
|
||||
|
||||
cache {
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 0
|
||||
}
|
||||
}
|
||||
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
description = "All helm repos merged"
|
||||
members = [
|
||||
artifactapi_remote.jetstack.name,
|
||||
artifactapi_remote.hashicorp_helm.name,
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Web UI (React + Vite — Separate Container)
|
||||
|
||||
### Deployment
|
||||
|
||||
Separate `Dockerfile.ui`: multi-stage build (node → nginx). Served as its own container/pod. nginx proxies `/api/*` to the Go backend.
|
||||
|
||||
### Pages
|
||||
|
||||
| Route | Content |
|
||||
|---|---|
|
||||
| `/` | Dashboard: total objects, storage used, dedup savings, bandwidth saved, top remotes chart, live SSE event feed, health indicators |
|
||||
| `/remotes` | Remote table: name, type, description, object count, size, hit rate, health. Filter by type, sort any column |
|
||||
| `/remotes/:name` | Config (read-only, "Managed by Terraform" badge), stats, object browser with search/sort/evict, flush actions |
|
||||
| `/virtuals` | Virtual table: name, type, members, merged object count |
|
||||
| `/virtuals/:name` | Member list with individual stats |
|
||||
|
||||
All config is read-only — managed by Terraform.
|
||||
|
||||
---
|
||||
|
||||
## TUI (Bubble Tea — Subcommand)
|
||||
|
||||
`artifactapi tui --endpoint http://localhost:8000` or via `ARTIFACTAPI_ENDPOINT` env.
|
||||
|
||||
Uses `pkg/client/` for all API calls (same client as Terraform provider).
|
||||
|
||||
| View | Key bindings |
|
||||
|---|---|
|
||||
| Dashboard | summary stats, top remotes |
|
||||
| Remotes list | `j`/`k` navigate, `/` filter, `Enter` detail |
|
||||
| Remote detail | config + stats, `Enter` → object browser |
|
||||
| Object browser | `/` search, `d` evict, `f` flush |
|
||||
| Virtuals | `j`/`k`, `Enter` detail |
|
||||
|
||||
---
|
||||
|
||||
## Improvements Over v2
|
||||
|
||||
| Area | v2 (Python) | v3 (Go) |
|
||||
|---|---|---|
|
||||
| S3 paths | Hashed, opaque | Content-addressed CAS |
|
||||
| Config | YAML files, mtime reload | Terraform via API |
|
||||
| Package types | 8 types | 10 types (+ terraform, goproxy) |
|
||||
| Virtual repos | Helm only | Helm + PyPI, extensible |
|
||||
| Deduplication | Docker blobs only | All types via CAS |
|
||||
| Revalidation | Opt-in flag | Default for all mutable |
|
||||
| Access logging | None | Per-artifact in Postgres |
|
||||
| GC | None | Background goroutine |
|
||||
| Upstream health | Per-request | Circuit breaker |
|
||||
| S3 backends | MinIO only | MinIO, Ceph, AWS (minio-go) |
|
||||
| UI | None | Web dashboard + TUI |
|
||||
| Binary | Python + venv | Static Go binary |
|
||||
| Frontend | N/A | Separate container (React) |
|
||||
| Testing | Mocked unit tests | Unit + e2e with real backends |
|
||||
|
||||
---
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Core Engine + Models
|
||||
- Go module, Makefile (`make build test lint fmt e2e`), Dockerfile, docker-compose
|
||||
- `pkg/models/` — all domain types
|
||||
- PostgreSQL schema + migrations
|
||||
- S3 storage layer with CAS (`minio-go/v7`)
|
||||
- Redis cache layer (TTL, locks)
|
||||
- Proxy engine: fetch-or-cache, classifier, revalidator
|
||||
- Generic + Docker providers (most complexity: OCI auth, CAS, tag banning)
|
||||
- Health + metrics endpoints
|
||||
- Unit tests for all packages
|
||||
- **Milestone**: proxy Docker + generic, cache in S3, track in Postgres
|
||||
|
||||
### Phase 2: All Providers
|
||||
- Helm (using `helm.sh/helm/v3/pkg/repo`)
|
||||
- PyPI (stdlib `x/net/html`)
|
||||
- npm (stdlib `encoding/json`)
|
||||
- RPM (using `rs3.io/go/rpm/repomd`)
|
||||
- Alpine (using `gitlab.alpinelinux.org/alpine/go`)
|
||||
- Puppet Forge (stdlib `encoding/json`)
|
||||
- Terraform (using `hashicorp/terraform-registry-address`)
|
||||
- Go Modules / GOPROXY (using `github.com/goproxy/goproxy`)
|
||||
- Unit tests per provider
|
||||
- **Milestone**: feature parity with v2 + goproxy
|
||||
|
||||
### Phase 3: Management API + Virtual Repos + GC
|
||||
- `pkg/client/` — shared Go API client
|
||||
- v2 CRUD endpoints
|
||||
- Virtual repo engine: `IndexMerger` for Helm + PyPI
|
||||
- Circuit breaker
|
||||
- Access logging middleware
|
||||
- GC goroutine
|
||||
- **Milestone**: full API, virtuals, GC
|
||||
|
||||
### Phase 4: End-to-End Tests
|
||||
- `e2e/` test suite with `testcontainers-go`
|
||||
- Proxy, Docker, management, virtual, terraform, GC tests
|
||||
- CI pipeline: `make e2e`
|
||||
- **Milestone**: comprehensive e2e coverage
|
||||
|
||||
### Phase 5: Terraform Provider
|
||||
- Separate repo: `terraform-provider-artifactapi`
|
||||
- Imports `pkg/client/` and `pkg/models/`
|
||||
- `artifactapi_remote` + `artifactapi_virtual` resources + data sources
|
||||
- Import support
|
||||
- **Milestone**: manage all config via Terraform
|
||||
|
||||
### Phase 6: Web UI
|
||||
- React + Vite in `ui/`
|
||||
- `Dockerfile.ui` (multi-stage → nginx)
|
||||
- Dashboard, remotes, objects, virtuals pages
|
||||
- SSE event feed
|
||||
- **Milestone**: full web UI in separate container
|
||||
|
||||
### Phase 7: TUI
|
||||
- Bubble Tea in `internal/tui/`
|
||||
- Uses `pkg/client/`
|
||||
- Dashboard, remotes, objects, virtuals views
|
||||
- **Milestone**: TUI feature parity with web UI
|
||||
|
||||
### Phase 8: Migration + Cutover
|
||||
- Migration tool: v2 YAML → Terraform HCL + `terraform import` commands
|
||||
- S3 rehash script: `{remote}/{hash16}/{file}` → `blobs/sha256/{content_hash}`
|
||||
- Parallel run, response comparison
|
||||
- Cutover
|
||||
|
||||
---
|
||||
|
||||
## Makefile Targets
|
||||
|
||||
```makefile
|
||||
.PHONY: build test lint fmt e2e docker docker-ui
|
||||
|
||||
build: ## Build Go binary
|
||||
go build -o bin/artifactapi ./cmd/artifactapi
|
||||
|
||||
test: ## Run unit tests
|
||||
go test ./...
|
||||
|
||||
lint: ## Run golangci-lint + go vet
|
||||
golangci-lint run ./...
|
||||
go vet ./...
|
||||
|
||||
fmt: ## Format code (gofmt + goimports)
|
||||
gofmt -w .
|
||||
goimports -w .
|
||||
|
||||
e2e: ## Run end-to-end tests (requires Docker)
|
||||
go test -tags=e2e -count=1 -timeout=5m ./e2e/...
|
||||
|
||||
docker: ## Build API server Docker image
|
||||
docker build -t artifactapi .
|
||||
|
||||
docker-ui: ## Build frontend Docker image
|
||||
docker build -t artifactapi-ui -f ui/Dockerfile.ui ui/
|
||||
|
||||
compose: ## Start full stack (API + UI + Postgres + Redis + MinIO)
|
||||
docker compose up -d
|
||||
```
|
||||
@@ -1,197 +1,664 @@
|
||||
# ArtifactAPI
|
||||
# Artifact Storage System
|
||||
|
||||
Caching proxy for package repositories. Single Go binary, 10 package types, content-addressable storage, managed by Terraform.
|
||||
A generic FastAPI-based artifact caching system that downloads and stores files from remote sources (GitHub, Gitea, HashiCorp, etc.) in S3-compatible storage with configuration-based access control.
|
||||
|
||||
## Quick Start
|
||||
## Features
|
||||
|
||||
```bash
|
||||
# Start backing services
|
||||
docker compose up -d postgres redis minio
|
||||
|
||||
# Build and run
|
||||
make build
|
||||
./bin/artifactapi
|
||||
|
||||
# Frontend (separate container or dev server)
|
||||
cd ui && npm install && npm run dev
|
||||
```
|
||||
|
||||
API: `http://localhost:8000` | Frontend: `http://localhost:5173`
|
||||
|
||||
## Package Types
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `generic` | nothing | everything |
|
||||
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
|
||||
| `helm` | `index.yaml` | `.tgz` charts |
|
||||
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
|
||||
| `npm` | package metadata | `.tgz` tarballs |
|
||||
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
|
||||
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
|
||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||
|
||||
Providers classify paths automatically. Users only configure what to proxy and TTLs.
|
||||
|
||||
## Terraform
|
||||
|
||||
Remotes and virtuals are managed by Terraform. Each package type has its own resource:
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 7200
|
||||
|
||||
patterns = [
|
||||
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
|
||||
"mikefarah/yq/.*/yq_linux_amd64$",
|
||||
]
|
||||
|
||||
mutable_patterns = [
|
||||
".*/archive/refs/heads/.*\\.tar\\.gz$",
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_docker" "dockerhub" {
|
||||
name = "dockerhub"
|
||||
base_url = "https://registry-1.docker.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
ban_tags_enabled = true
|
||||
ban_tags = ["latest"]
|
||||
|
||||
patterns = [
|
||||
"^library/postgres",
|
||||
"^library/redis",
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_helm" "jetstack" {
|
||||
name = "jetstack"
|
||||
base_url = "https://charts.jetstack.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 3600
|
||||
}
|
||||
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
members = [artifactapi_remote_helm.jetstack.name]
|
||||
}
|
||||
```
|
||||
|
||||
Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi)
|
||||
|
||||
### Serving providers as a registry
|
||||
|
||||
A local `terraform` repo is a real provider registry: upload
|
||||
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
|
||||
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
|
||||
no `.terraformrc` mirror config:
|
||||
|
||||
```hcl
|
||||
terraform {
|
||||
required_providers {
|
||||
artifactapi = {
|
||||
source = "artifactapi.k8s.syd1.au.unkin.net/<repo>/<type>"
|
||||
version = "0.1.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The Terraform *namespace* segment is the artifactapi repo name; the provider is
|
||||
matched by *type*. The registry serves service discovery
|
||||
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
|
||||
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
|
||||
|
||||
Signing needs a GPG key. By default artifactapi generates one on first start and
|
||||
stores it in the database (`signing_keys` table), so every replica shares it and
|
||||
there's nothing to provision. To bring your own key instead, point
|
||||
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
|
||||
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
|
||||
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
|
||||
|
||||
## Access Control
|
||||
|
||||
| Field | Default | Behaviour |
|
||||
|---|---|---|
|
||||
| `patterns` | empty (proxy all) | If set, only matching paths are proxied. Acts as allowlist. |
|
||||
| `blocklist` | empty | Matching paths always denied. Checked first. |
|
||||
| `mutable_patterns` | empty | Override: force paths to mutable TTL. |
|
||||
| `immutable_patterns` | empty | Override: force paths to immutable TTL. |
|
||||
|
||||
No patterns + no blocklist = open proxy. Provider handles mutability classification automatically.
|
||||
|
||||
## API
|
||||
|
||||
### Proxy (v1)
|
||||
|
||||
```
|
||||
GET /api/v1/remote/{name}/{path} Proxy/cache artifact
|
||||
GET /api/v1/virtual/{name}/{path} Virtual repo (merged index)
|
||||
GET /v2/{name}/{path} Docker Registry v2
|
||||
```
|
||||
|
||||
### Management (v2)
|
||||
|
||||
```
|
||||
GET/POST /api/v2/remotes List / create remotes
|
||||
GET/PUT/DELETE /api/v2/remotes/{name} Read / update / delete remote
|
||||
GET/DELETE /api/v2/remotes/{name}/objects Browse / evict cached objects
|
||||
GET /api/v2/stats Overview stats
|
||||
GET /api/v2/health Service health
|
||||
POST /api/v2/probe Test a remote (fetch without streaming to client)
|
||||
GET /api/v2/events SSE event stream
|
||||
```
|
||||
- **Generic Remote Support**: Works with any HTTP-based file server (GitHub, Gitea, HashiCorp, custom servers)
|
||||
- **Configuration-Based**: YAML configuration for remotes, patterns, and access control
|
||||
- **Direct URL API**: Access cached files via clean URLs like `/api/github/owner/repo/path/file.tar.gz`
|
||||
- **Pattern Filtering**: Regex-based inclusion patterns for security and organization
|
||||
- **Smart Caching**: Automatic download and cache on first access, serve from cache afterward
|
||||
- **S3 Storage**: MinIO/S3 backend with predictable paths
|
||||
- **Content-Type Detection**: Automatic MIME type detection for downloads
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
PostgreSQL ─── config (remotes, virtuals), artifact metadata, access log
|
||||
Redis ─── TTL keys, fetch locks, circuit breaker state
|
||||
S3/MinIO ─── content-addressable blob storage (blobs/sha256/{hash})
|
||||
```
|
||||
The system acts as a caching proxy that:
|
||||
1. Receives requests via the `/api/{remote}/{path}` endpoint
|
||||
2. Checks if the file is already cached
|
||||
3. If not cached, downloads from the configured remote and caches it
|
||||
4. Serves the file with appropriate headers and content types
|
||||
5. Enforces access control via configurable regex patterns
|
||||
|
||||
S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go).
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `LISTEN_ADDR` | `:8000` | Server listen address |
|
||||
| `DBHOST` | `localhost` | PostgreSQL host |
|
||||
| `DBPORT` | `5432` | PostgreSQL port |
|
||||
| `DBUSER` | `artifacts` | PostgreSQL user |
|
||||
| `DBPASS` | | PostgreSQL password |
|
||||
| `DBNAME` | `artifacts` | PostgreSQL database |
|
||||
| `REDIS_URL` | `redis://localhost:6379` | Redis URL |
|
||||
| `MINIO_ENDPOINT` | `localhost:9000` | S3 endpoint |
|
||||
| `MINIO_ACCESS_KEY` | | S3 access key |
|
||||
| `MINIO_SECRET_KEY` | | S3 secret key |
|
||||
| `MINIO_BUCKET` | `artifacts` | S3 bucket |
|
||||
| `MINIO_SECURE` | `false` | Use HTTPS for S3 |
|
||||
| `MINIO_REGION` | | S3 region (AWS) |
|
||||
|
||||
## Development
|
||||
## Quick Start
|
||||
|
||||
1. Start MinIO container:
|
||||
```bash
|
||||
make build # Build binary
|
||||
make test # Unit tests
|
||||
make e2e # E2E tests (needs Docker)
|
||||
make lint # golangci-lint + go vet
|
||||
make fmt # gofmt + goimports
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
### TUI
|
||||
|
||||
2. Create virtual environment and install dependencies:
|
||||
```bash
|
||||
./bin/artifactapi tui --endpoint http://localhost:8000
|
||||
uv venv
|
||||
source .venv/bin/activate
|
||||
uv pip install -r requirements.txt
|
||||
```
|
||||
|
||||
3. Start the API:
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
|
||||
4. Access artifacts directly via URL:
|
||||
```bash
|
||||
# This will download and cache the file on first access
|
||||
xh GET localhost:8000/api/github/gruntwork-io/terragrunt/releases/download/v0.96.1/terragrunt_linux_amd64.tar.gz
|
||||
|
||||
# Subsequent requests serve from cache (see X-Artifact-Source: cache header)
|
||||
curl -I localhost:8000/api/github/gruntwork-io/terragrunt/releases/download/v0.96.1/terragrunt_linux_amd64.tar.gz
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Direct Access
|
||||
- `GET /api/{remote}/{path}` - Direct access to artifacts with auto-caching
|
||||
|
||||
### Management
|
||||
- `GET /` - API info and available remotes
|
||||
- `GET /health` - Health check
|
||||
- `GET /config` - View current configuration
|
||||
- `POST /cache-artifact` - Batch cache artifacts matching pattern
|
||||
- `GET /artifacts/{remote}` - List cached artifacts
|
||||
|
||||
## Configuration
|
||||
|
||||
The system uses `remotes.yaml` to define remote repositories and access patterns. All other configuration is provided via environment variables.
|
||||
|
||||
### remotes.yaml Structure
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
remote-name:
|
||||
base_url: "https://example.com" # Base URL for the remote
|
||||
type: "remote" # Type: "remote" or "local"
|
||||
package: "generic" # Package type: "generic", "alpine", "rpm"
|
||||
description: "Human readable description"
|
||||
include_patterns: # Regex patterns for allowed files
|
||||
- "pattern1"
|
||||
- "pattern2"
|
||||
cache: # Cache configuration (optional)
|
||||
file_ttl: 0 # File cache TTL (0 = indefinite)
|
||||
index_ttl: 300 # Index file TTL in seconds
|
||||
```
|
||||
|
||||
### Remote Types
|
||||
|
||||
#### Generic Remotes
|
||||
For general file hosting (GitHub releases, custom servers):
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
github:
|
||||
base_url: "https://github.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "GitHub releases and files"
|
||||
include_patterns:
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
- "lxc/incus/.*\\.tar\\.gz$"
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
cache:
|
||||
file_ttl: 0 # Cache files indefinitely
|
||||
index_ttl: 0 # No index files for generic remotes
|
||||
|
||||
hashicorp-releases:
|
||||
base_url: "https://releases.hashicorp.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "HashiCorp product releases"
|
||||
include_patterns:
|
||||
- "terraform/.*terraform_.*_linux_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_linux_amd64\\.zip$"
|
||||
- "consul/.*/consul_.*_linux_amd64\\.zip$"
|
||||
cache:
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
```
|
||||
|
||||
#### Package Repository Remotes
|
||||
For Linux package repositories with index files:
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
alpine:
|
||||
base_url: "https://dl-cdn.alpinelinux.org"
|
||||
type: "remote"
|
||||
package: "alpine"
|
||||
description: "Alpine Linux APK package repository"
|
||||
include_patterns:
|
||||
- ".*/x86_64/.*\\.apk$" # Only x86_64 packages
|
||||
cache:
|
||||
file_ttl: 0 # Cache packages indefinitely
|
||||
index_ttl: 7200 # Cache APKINDEX.tar.gz for 2 hours
|
||||
|
||||
almalinux:
|
||||
base_url: "http://mirror.aarnet.edu.au/pub/almalinux"
|
||||
type: "remote"
|
||||
package: "rpm"
|
||||
description: "AlmaLinux RPM package repository"
|
||||
include_patterns:
|
||||
- ".*/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
cache:
|
||||
file_ttl: 0
|
||||
index_ttl: 7200 # Cache metadata files for 2 hours
|
||||
```
|
||||
|
||||
#### Local Repositories
|
||||
For storing custom artifacts:
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
local-generic:
|
||||
type: "local"
|
||||
package: "generic"
|
||||
description: "Local generic file repository"
|
||||
cache:
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
```
|
||||
|
||||
### Include Patterns
|
||||
|
||||
Include patterns are regular expressions that control which files can be accessed:
|
||||
|
||||
```yaml
|
||||
include_patterns:
|
||||
# Specific project patterns
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
|
||||
# File extension patterns
|
||||
- ".*\\.tar\\.gz$"
|
||||
- ".*\\.zip$"
|
||||
- ".*\\.rpm$"
|
||||
|
||||
# Architecture-specific patterns
|
||||
- ".*/x86_64/.*"
|
||||
- ".*/linux-amd64/.*"
|
||||
|
||||
# Version-specific patterns
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
```
|
||||
|
||||
**Security Note**: Only files matching at least one include pattern are accessible. Files not matching any pattern return HTTP 403.
|
||||
|
||||
### Cache Configuration
|
||||
|
||||
Control how long different file types are cached:
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
file_ttl: 0 # Regular files (0 = cache indefinitely)
|
||||
index_ttl: 300 # Index files like APKINDEX.tar.gz (seconds)
|
||||
```
|
||||
|
||||
**Index Files**: Repository metadata files that change frequently:
|
||||
- Alpine: `APKINDEX.tar.gz`
|
||||
- RPM: `repomd.xml`, `*-primary.xml.gz`, etc.
|
||||
- These are automatically detected and use `index_ttl`
|
||||
|
||||
### Environment Variables
|
||||
|
||||
All runtime configuration comes from environment variables:
|
||||
|
||||
**Database Configuration:**
|
||||
- `DBHOST` - PostgreSQL host
|
||||
- `DBPORT` - PostgreSQL port
|
||||
- `DBUSER` - PostgreSQL username
|
||||
- `DBPASS` - PostgreSQL password
|
||||
- `DBNAME` - PostgreSQL database name
|
||||
|
||||
**Redis Configuration:**
|
||||
- `REDIS_URL` - Redis connection URL (e.g., `redis://localhost:6379`)
|
||||
|
||||
**S3/MinIO Configuration:**
|
||||
- `MINIO_ENDPOINT` - MinIO/S3 endpoint
|
||||
- `MINIO_ACCESS_KEY` - S3 access key
|
||||
- `MINIO_SECRET_KEY` - S3 secret key
|
||||
- `MINIO_BUCKET` - S3 bucket name
|
||||
- `MINIO_SECURE` - Use HTTPS (`true`/`false`)
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Direct File Access
|
||||
```bash
|
||||
# Access GitHub releases
|
||||
curl localhost:8000/api/github/gruntwork-io/terragrunt/releases/download/v0.96.1/terragrunt_linux_amd64.tar.gz
|
||||
|
||||
# Access HashiCorp releases (when configured)
|
||||
curl localhost:8000/api/hashicorp/terraform/1.6.0/terraform_1.6.0_linux_amd64.zip
|
||||
|
||||
# Access custom remotes
|
||||
curl localhost:8000/api/custom/path/to/file.tar.gz
|
||||
```
|
||||
|
||||
### Response Headers
|
||||
- `X-Artifact-Source: cache|remote` - Indicates if served from cache or freshly downloaded
|
||||
- `Content-Type` - Automatically detected (application/gzip, application/zip, etc.)
|
||||
- `Content-Disposition` - Download filename
|
||||
- `Content-Length` - File size
|
||||
|
||||
### Pattern Enforcement
|
||||
Access is controlled by regex patterns in the configuration. Requests for files not matching any pattern return HTTP 403.
|
||||
|
||||
## Storage Path Format
|
||||
|
||||
Files are stored with keys like:
|
||||
- `{remote_name}/{path_hash}/{filename}` for direct API access
|
||||
- `{hostname}/{url_hash}/{filename}` for legacy batch operations
|
||||
|
||||
Example: `github/a1b2c3d4e5f6g7h8/terragrunt_linux_amd64.tar.gz`
|
||||
|
||||
## Kubernetes Deployment
|
||||
|
||||
Deploy the artifact storage system to Kubernetes using the following manifests:
|
||||
|
||||
### 1. Namespace
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: artifact-storage
|
||||
```
|
||||
|
||||
### 2. ConfigMap for remotes.yaml
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: artifactapi-config
|
||||
namespace: artifact-storage
|
||||
data:
|
||||
remotes.yaml: |
|
||||
remotes:
|
||||
github:
|
||||
base_url: "https://github.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "GitHub releases and files"
|
||||
include_patterns:
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
- "lxc/incus/.*\\.tar\\.gz$"
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
cache:
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
|
||||
hashicorp-releases:
|
||||
base_url: "https://releases.hashicorp.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "HashiCorp product releases"
|
||||
include_patterns:
|
||||
- "terraform/.*terraform_.*_linux_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_linux_amd64\\.zip$"
|
||||
- "consul/.*/consul_.*_linux_amd64\\.zip$"
|
||||
cache:
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
```
|
||||
|
||||
### 3. Secret for Environment Variables
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: artifactapi-secret
|
||||
namespace: artifact-storage
|
||||
type: Opaque
|
||||
stringData:
|
||||
DBHOST: "postgres-service"
|
||||
DBPORT: "5432"
|
||||
DBUSER: "artifacts"
|
||||
DBPASS: "artifacts123"
|
||||
DBNAME: "artifacts"
|
||||
REDIS_URL: "redis://redis-service:6379"
|
||||
MINIO_ENDPOINT: "minio-service:9000"
|
||||
MINIO_ACCESS_KEY: "minioadmin"
|
||||
MINIO_SECRET_KEY: "minioadmin"
|
||||
MINIO_BUCKET: "artifacts"
|
||||
MINIO_SECURE: "false"
|
||||
```
|
||||
|
||||
### 4. PostgreSQL Deployment
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: postgres
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: postgres
|
||||
spec:
|
||||
containers:
|
||||
- name: postgres
|
||||
image: postgres:15-alpine
|
||||
env:
|
||||
- name: POSTGRES_DB
|
||||
value: artifacts
|
||||
- name: POSTGRES_USER
|
||||
value: artifacts
|
||||
- name: POSTGRES_PASSWORD
|
||||
value: artifacts123
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
volumeMounts:
|
||||
- name: postgres-storage
|
||||
mountPath: /var/lib/postgresql/data
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["pg_isready", "-U", "artifacts", "-d", "artifacts"]
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: postgres-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: postgres-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: postgres-service
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
selector:
|
||||
app: postgres
|
||||
ports:
|
||||
- port: 5432
|
||||
targetPort: 5432
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: postgres-pvc
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
```
|
||||
|
||||
### 5. Redis Deployment
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: redis
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:7-alpine
|
||||
command: ["redis-server", "--save", "20", "1"]
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
volumeMounts:
|
||||
- name: redis-storage
|
||||
mountPath: /data
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: redis-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: redis-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redis-service
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
selector:
|
||||
app: redis
|
||||
ports:
|
||||
- port: 6379
|
||||
targetPort: 6379
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: redis-pvc
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
```
|
||||
|
||||
### 6. MinIO Deployment
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: minio
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: minio
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: minio
|
||||
spec:
|
||||
containers:
|
||||
- name: minio
|
||||
image: minio/minio:latest
|
||||
command: ["minio", "server", "/data", "--console-address", ":9001"]
|
||||
env:
|
||||
- name: MINIO_ROOT_USER
|
||||
value: minioadmin
|
||||
- name: MINIO_ROOT_PASSWORD
|
||||
value: minioadmin
|
||||
ports:
|
||||
- containerPort: 9000
|
||||
- containerPort: 9001
|
||||
volumeMounts:
|
||||
- name: minio-storage
|
||||
mountPath: /data
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /minio/health/live
|
||||
port: 9000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: minio-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: minio-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: minio-service
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
selector:
|
||||
app: minio
|
||||
ports:
|
||||
- name: api
|
||||
port: 9000
|
||||
targetPort: 9000
|
||||
- name: console
|
||||
port: 9001
|
||||
targetPort: 9001
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: minio-pvc
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
```
|
||||
|
||||
### 7. Artifact API Deployment
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: artifactapi
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: artifactapi
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: artifactapi
|
||||
spec:
|
||||
containers:
|
||||
- name: artifactapi
|
||||
image: artifactapi:latest
|
||||
ports:
|
||||
- containerPort: 8000
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: artifactapi-secret
|
||||
volumeMounts:
|
||||
- name: config-volume
|
||||
mountPath: /app/remotes.yaml
|
||||
subPath: remotes.yaml
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
volumes:
|
||||
- name: config-volume
|
||||
configMap:
|
||||
name: artifactapi-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: artifactapi-service
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
selector:
|
||||
app: artifactapi
|
||||
ports:
|
||||
- port: 8000
|
||||
targetPort: 8000
|
||||
type: ClusterIP
|
||||
```
|
||||
|
||||
### 8. Ingress (Optional)
|
||||
```yaml
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: artifactapi-ingress
|
||||
namespace: artifact-storage
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/rewrite-target: /
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "10g"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||
spec:
|
||||
rules:
|
||||
- host: artifacts.example.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: artifactapi-service
|
||||
port:
|
||||
number: 8000
|
||||
```
|
||||
|
||||
### Deployment Commands
|
||||
```bash
|
||||
# Create namespace
|
||||
kubectl apply -f namespace.yaml
|
||||
|
||||
# Deploy PostgreSQL, Redis, and MinIO
|
||||
kubectl apply -f postgres.yaml
|
||||
kubectl apply -f redis.yaml
|
||||
kubectl apply -f minio.yaml
|
||||
|
||||
# Wait for databases to be ready
|
||||
kubectl wait --for=condition=ready pod -l app=postgres -n artifact-storage --timeout=300s
|
||||
kubectl wait --for=condition=ready pod -l app=redis -n artifact-storage --timeout=300s
|
||||
kubectl wait --for=condition=ready pod -l app=minio -n artifact-storage --timeout=300s
|
||||
|
||||
# Deploy configuration and application
|
||||
kubectl apply -f configmap.yaml
|
||||
kubectl apply -f secret.yaml
|
||||
kubectl apply -f artifactapi.yaml
|
||||
|
||||
# Optional: Deploy ingress
|
||||
kubectl apply -f ingress.yaml
|
||||
|
||||
# Check deployment status
|
||||
kubectl get pods -n artifact-storage
|
||||
kubectl logs -f deployment/artifactapi -n artifact-storage
|
||||
```
|
||||
|
||||
### Access the API
|
||||
```bash
|
||||
# Port-forward to access locally
|
||||
kubectl port-forward service/artifactapi-service 8000:8000 -n artifact-storage
|
||||
|
||||
# Test the API
|
||||
curl http://localhost:8000/health
|
||||
curl http://localhost:8000/
|
||||
|
||||
# Access artifacts
|
||||
curl "http://localhost:8000/api/github/gruntwork-io/terragrunt/releases/download/v0.96.1/terragrunt_linux_amd64"
|
||||
```
|
||||
|
||||
### Notes for Production
|
||||
- Use proper secrets management (e.g., Vault, Sealed Secrets)
|
||||
- Configure resource limits and requests appropriately
|
||||
- Set up monitoring and alerting
|
||||
- Use external managed databases for production workloads
|
||||
- Configure backup strategies for persistent volumes
|
||||
- Set up proper TLS certificates for ingress
|
||||
- Consider using StatefulSets for databases with persistent storage
|
||||
@@ -1,57 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
||||
"git.unkin.net/unkin/artifactapi/internal/server"
|
||||
"git.unkin.net/unkin/artifactapi/internal/tui"
|
||||
)
|
||||
|
||||
var version = "dev"
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == "tui" {
|
||||
endpoint := os.Getenv("ARTIFACTAPI_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
endpoint = "http://localhost:8000"
|
||||
}
|
||||
for i, arg := range os.Args {
|
||||
if arg == "--endpoint" && i+1 < len(os.Args) {
|
||||
endpoint = os.Args[i+1]
|
||||
}
|
||||
}
|
||||
|
||||
app := tui.New(endpoint)
|
||||
if err := app.Run(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "tui error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
slog.Error("failed to load config", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
srv, err := server.New(cfg, version)
|
||||
if err != nil {
|
||||
slog.Error("failed to create server", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := srv.Run(ctx); err != nil {
|
||||
slog.Error("server exited with error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
# Overlay for the dockerised end-to-end suite (scripts/docker-e2e.sh).
|
||||
# Adds a static mock upstream that the artifactapi container proxies, so the
|
||||
# caching tests are hermetic and need no internet access.
|
||||
services:
|
||||
mockupstream:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./e2e-docker/fixtures:/usr/share/nginx/html:ro,z
|
||||
# No host port needed: only the artifactapi container talks to it, and the
|
||||
# tests compare served bytes against the on-disk fixtures.
|
||||
|
||||
artifactapi:
|
||||
# The host port is set via ARTIFACTAPI_PORT (see scripts/docker-e2e.sh),
|
||||
# defaulting to 8000; the e2e run uses 8001 to avoid colliding with a
|
||||
# locally-running instance.
|
||||
depends_on:
|
||||
mockupstream:
|
||||
condition: service_started
|
||||
+51
-56
@@ -1,22 +1,26 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
artifactapi:
|
||||
build: .
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
no_cache: true
|
||||
ports:
|
||||
- "${ARTIFACTAPI_PORT:-8000}:8000"
|
||||
- "8000:8000"
|
||||
environment:
|
||||
LISTEN_ADDR: ":8000"
|
||||
DBHOST: postgres
|
||||
DBPORT: "5432"
|
||||
DBUSER: artifacts
|
||||
DBPASS: artifacts123
|
||||
DBNAME: artifacts
|
||||
DBSSL: disable
|
||||
REDIS_URL: redis://redis:6379
|
||||
MINIO_ENDPOINT: minio:9000
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
MINIO_BUCKET: artifacts
|
||||
MINIO_SECURE: "false"
|
||||
- CONFIG_PATH=/app/remotes.yaml
|
||||
- DBHOST=postgres
|
||||
- DBPORT=5432
|
||||
- DBUSER=artifacts
|
||||
- DBPASS=artifacts123
|
||||
- DBNAME=artifacts
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- MINIO_ENDPOINT=minio:9000
|
||||
- MINIO_ACCESS_KEY=minioadmin
|
||||
- MINIO_SECRET_KEY=minioadmin
|
||||
- MINIO_BUCKET=artifacts
|
||||
- MINIO_SECURE=false
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -25,35 +29,27 @@ services:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8000/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
ui:
|
||||
build:
|
||||
context: ui
|
||||
dockerfile: Dockerfile.ui
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
ports:
|
||||
- "8080:80"
|
||||
depends_on:
|
||||
- artifactapi
|
||||
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
ports:
|
||||
- "5432:5432"
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
environment:
|
||||
POSTGRES_USER: artifacts
|
||||
POSTGRES_PASSWORD: artifacts123
|
||||
POSTGRES_DB: artifacts
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
command: server /data --console-address ":9001"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U artifacts -d artifacts"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
@@ -64,28 +60,27 @@ services:
|
||||
command: redis-server --save 20 1
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
- "5432:5432"
|
||||
environment:
|
||||
POSTGRES_DB: artifacts
|
||||
POSTGRES_USER: artifacts
|
||||
POSTGRES_PASSWORD: artifacts123
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
test: ["CMD-SHELL", "pg_isready -U artifacts -d artifacts"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
redis_data:
|
||||
postgres_data:
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
# Dockerised end-to-end suite
|
||||
|
||||
Black-box tests that run against a fully **containerised** artifactapi stack
|
||||
(built image + Postgres + Redis + MinIO) plus a static mock upstream. Unlike the
|
||||
in-process `e2e/` suite (testcontainers, server run in-process), these only speak
|
||||
HTTP to the running product, so they exercise the shipped container image.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
make docker-e2e # build image, compose up, run suite, compose down
|
||||
```
|
||||
|
||||
`scripts/docker-e2e.sh` builds and starts `docker-compose.yml` +
|
||||
`docker-compose.e2e.yml`, waits for `/health`, then runs
|
||||
`go test -tags=dockere2e ./e2e-docker/...` and tears everything down.
|
||||
|
||||
The stack publishes artifactapi on host port **8001** (to avoid colliding with a
|
||||
local instance on 8000). Override with `ARTIFACTAPI_URL` to point the tests at an
|
||||
already-running stack.
|
||||
|
||||
## Coverage
|
||||
|
||||
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
|
||||
- **Caching** — one immutable artifact per remote package type (generic, docker,
|
||||
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
|
||||
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
||||
verified against the origin fixture.
|
||||
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
|
||||
index), rpm (real package + **automatic repodata** generation).
|
||||
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
|
||||
across two members.
|
||||
|
||||
## Fixtures
|
||||
|
||||
`fixtures/` is served by the mock upstream at its web root. Paths mirror each
|
||||
provider's upstream URL layout (e.g. `v2/...` for docker, `v1/providers/...` for
|
||||
terraform). The RPM under `fixtures/rpmrepo/Packages/` is a real package so the
|
||||
rpm provider can parse its metadata for repodata generation.
|
||||
@@ -1,76 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestCachingPerProvider proxies one immutable artifact for every remote
|
||||
// package type through the mock upstream and asserts: first fetch is served
|
||||
// from the remote, the second from cache, and the bytes match the origin.
|
||||
func TestCachingPerProvider(t *testing.T) {
|
||||
cases := []struct {
|
||||
pkgType string
|
||||
// path is the request path under /api/v1/remote/<name>/. The provider
|
||||
// derives the upstream URL from it (docker prepends /v2/, terraform
|
||||
// prepends /v1/providers/), and the fixture lives at that resolved path.
|
||||
path string
|
||||
fixture string
|
||||
}{
|
||||
{"generic", "blobs/hello.bin", "blobs/hello.bin"},
|
||||
{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"},
|
||||
{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"},
|
||||
{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"},
|
||||
{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"},
|
||||
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
|
||||
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
|
||||
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
|
||||
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
|
||||
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.pkgType, func(t *testing.T) {
|
||||
name := "cache-" + tc.pkgType
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": %q,
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"stale_on_error": true
|
||||
}`, name, tc.pkgType, mockUpstream()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
want := fixtureBytes(t, tc.fixture)
|
||||
url := api("/api/v1/remote/" + name + "/" + tc.path)
|
||||
|
||||
// First fetch: from remote.
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("first fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if src := resp.Header.Get("X-Artifact-Source"); src != "remote" {
|
||||
t.Fatalf("first fetch source = %q, want remote", src)
|
||||
}
|
||||
if !bytes.Equal(body, want) {
|
||||
t.Fatalf("first fetch body mismatch: got %d bytes, want %d", len(body), len(want))
|
||||
}
|
||||
|
||||
// Second fetch: from cache, identical bytes.
|
||||
resp, body = doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("second fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if src := resp.Header.Get("X-Artifact-Source"); src != "cache" {
|
||||
t.Fatalf("second fetch source = %q, want cache", src)
|
||||
}
|
||||
if !bytes.Equal(body, want) {
|
||||
t.Fatalf("cached body mismatch: got %d bytes, want %d", len(body), len(want))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
hello artifactapi generic blob
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,8 +0,0 @@
|
||||
apiVersion: v1
|
||||
entries:
|
||||
alpha:
|
||||
- name: alpha
|
||||
version: 1.0.0
|
||||
urls:
|
||||
- charts/alpha-1.0.0.tgz
|
||||
generated: "2026-01-01T00:00:00Z"
|
||||
@@ -1,8 +0,0 @@
|
||||
apiVersion: v1
|
||||
entries:
|
||||
beta:
|
||||
- name: beta
|
||||
version: 2.0.0
|
||||
urls:
|
||||
- charts/beta-2.0.0.tgz
|
||||
generated: "2026-01-01T00:00:00Z"
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,108 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
// Package e2edocker holds the black-box end-to-end suite that runs against a
|
||||
// fully dockerised artifactapi stack (see scripts/docker-e2e.sh). Unlike the
|
||||
// in-process e2e suite, these tests only speak HTTP to the running container.
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func baseURL() string {
|
||||
if v := os.Getenv("ARTIFACTAPI_URL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://localhost:8000"
|
||||
}
|
||||
|
||||
// mockUpstream is the base URL the artifactapi *container* uses to reach the
|
||||
// static mock upstream. It is resolved on the compose network, not the host.
|
||||
func mockUpstream() string {
|
||||
if v := os.Getenv("MOCK_UPSTREAM_INTERNAL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://mockupstream"
|
||||
}
|
||||
|
||||
func api(path string) string { return baseURL() + path }
|
||||
|
||||
func fixtureBytes(t *testing.T, rel string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join("fixtures", rel))
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture %s: %v", rel, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func doRequest(t *testing.T, method, url string, body []byte, contentType string) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
var r io.Reader
|
||||
if body != nil {
|
||||
r = bytes.NewReader(body)
|
||||
}
|
||||
req, err := http.NewRequest(method, url, r)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, url, err)
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return resp, respBody
|
||||
}
|
||||
|
||||
func createRepo(t *testing.T, jsonBody string) {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodPost, api("/api/v2/remotes"), []byte(jsonBody), "application/json")
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create repo: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRepo(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
doRequest(t, http.MethodDelete, api("/api/v2/remotes/"+name), nil, "")
|
||||
}
|
||||
|
||||
func createVirtual(t *testing.T, jsonBody string) {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodPost, api("/api/v2/virtuals"), []byte(jsonBody), "application/json")
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteVirtual(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
doRequest(t, http.MethodDelete, api("/api/v2/virtuals/"+name), nil, "")
|
||||
}
|
||||
|
||||
// getEventually retries a GET until it returns 200 or the deadline passes. Used
|
||||
// for asynchronously-generated artifacts (e.g. rpm repodata after upload).
|
||||
func getEventually(t *testing.T, url string, timeout time.Duration) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
var resp *http.Response
|
||||
var body []byte
|
||||
for {
|
||||
resp, body = doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode == http.StatusOK || time.Now().After(deadline) {
|
||||
return resp, body
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func uploadFile(t *testing.T, repo, filePath string, body []byte, contentType string) {
|
||||
t.Helper()
|
||||
url := api("/api/v2/remotes/" + repo + "/files/" + filePath)
|
||||
resp, respBody := doRequest(t, http.MethodPut, url, body, contentType)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s: status %d: %s", filePath, resp.StatusCode, respBody)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalGenericUpload uploads a generic file and downloads it back.
|
||||
func TestLocalGenericUpload(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-generic","package_type":"generic","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-generic")
|
||||
|
||||
content := []byte("artifactapi local generic upload payload")
|
||||
uploadFile(t, "local-generic", "data/hello.bin", content, "application/octet-stream")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-generic/data/hello.bin"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, content) {
|
||||
t.Fatalf("downloaded content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalPyPIUpload uploads a wheel and validates the generated simple index.
|
||||
func TestLocalPyPIUpload(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-pypi","package_type":"pypi","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-pypi")
|
||||
|
||||
wheel := fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl")
|
||||
uploadFile(t, "local-pypi", "foo-1.0-py3-none-any.whl", wheel, "application/zip")
|
||||
|
||||
// Root index lists the package.
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("simple index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !strings.Contains(string(body), "foo") {
|
||||
t.Fatalf("simple index missing package 'foo': %s", body)
|
||||
}
|
||||
|
||||
// Per-package index lists the wheel file.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/foo/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("package index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !strings.Contains(string(body), "foo-1.0-py3-none-any.whl") {
|
||||
t.Fatalf("package index missing wheel: %s", body)
|
||||
}
|
||||
|
||||
// The wheel downloads back byte-identical.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/foo/foo-1.0-py3-none-any.whl"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download wheel: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, wheel) {
|
||||
t.Fatalf("wheel content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalRPMRepodata uploads a real RPM and validates that repodata is
|
||||
// generated automatically (the special rpm-local feature).
|
||||
func TestLocalRPMRepodata(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-rpm","package_type":"rpm","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-rpm")
|
||||
|
||||
rpm := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
|
||||
uploadFile(t, "local-rpm", "e2e-testpkg-1.0-1.noarch.rpm", rpm, "application/x-rpm")
|
||||
|
||||
// repodata is generated asynchronously after upload; poll for it.
|
||||
resp, body := getEventually(t, api("/api/v1/local/local-rpm/repodata/repomd.xml"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("repomd.xml: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "<repomd") || !strings.Contains(s, "primary") {
|
||||
t.Fatalf("repomd.xml not a valid repodata document: %s", s)
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHealth(t *testing.T) {
|
||||
resp, body := doRequest(t, http.MethodGet, api("/health"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("health: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRemoteLifecycle covers add/change/delete for a remote repository.
|
||||
func TestRemoteLifecycle(t *testing.T) {
|
||||
createRepo(t, `{
|
||||
"name": "crud-remote",
|
||||
"package_type": "generic",
|
||||
"repo_type": "remote",
|
||||
"base_url": "https://example.com",
|
||||
"mutable_ttl": 600,
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRepo(t, "crud-remote")
|
||||
|
||||
got := getRepo(t, "crud-remote")
|
||||
if got["base_url"] != "https://example.com" || got["mutable_ttl"].(float64) != 600 {
|
||||
t.Fatalf("unexpected created remote: %v", got)
|
||||
}
|
||||
|
||||
// change
|
||||
resp, body := doRequest(t, http.MethodPut, api("/api/v2/remotes/crud-remote"), []byte(`{
|
||||
"package_type": "generic",
|
||||
"base_url": "https://updated.example.com",
|
||||
"mutable_ttl": 120,
|
||||
"stale_on_error": true
|
||||
}`), "application/json")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update remote: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
got = getRepo(t, "crud-remote")
|
||||
if got["base_url"] != "https://updated.example.com" || got["mutable_ttl"].(float64) != 120 {
|
||||
t.Fatalf("update not applied: %v", got)
|
||||
}
|
||||
|
||||
// delete
|
||||
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-remote"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete remote: status %d", resp.StatusCode)
|
||||
}
|
||||
resp, _ = doRequest(t, http.MethodGet, api("/api/v2/remotes/crud-remote"), nil, "")
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("expected 404 after delete, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalLifecycle covers add/delete for a local repository.
|
||||
func TestLocalLifecycle(t *testing.T) {
|
||||
createRepo(t, `{
|
||||
"name": "crud-local",
|
||||
"package_type": "generic",
|
||||
"repo_type": "local"
|
||||
}`)
|
||||
defer deleteRepo(t, "crud-local")
|
||||
|
||||
got := getRepo(t, "crud-local")
|
||||
if got["repo_type"] != "local" {
|
||||
t.Fatalf("expected repo_type local, got %v", got["repo_type"])
|
||||
}
|
||||
|
||||
resp, _ := doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-local"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete local: status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVirtualLifecycle covers add/change/delete for a virtual repository.
|
||||
func TestVirtualLifecycle(t *testing.T) {
|
||||
createRepo(t, `{"name":"vmem-a","package_type":"helm","repo_type":"remote","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRepo(t, `{"name":"vmem-b","package_type":"helm","repo_type":"remote","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRepo(t, "vmem-a")
|
||||
defer deleteRepo(t, "vmem-b")
|
||||
|
||||
createVirtual(t, `{
|
||||
"name": "crud-virtual",
|
||||
"package_type": "helm",
|
||||
"members": ["vmem-a"]
|
||||
}`)
|
||||
defer deleteVirtual(t, "crud-virtual")
|
||||
|
||||
// change members
|
||||
resp, body := doRequest(t, http.MethodPut, api("/api/v2/virtuals/crud-virtual"), []byte(`{
|
||||
"package_type": "helm",
|
||||
"members": ["vmem-a", "vmem-b"]
|
||||
}`), "application/json")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v2/virtuals/crud-virtual"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("get virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
var v map[string]any
|
||||
if err := json.Unmarshal(body, &v); err != nil {
|
||||
t.Fatalf("decode virtual: %v", err)
|
||||
}
|
||||
members, _ := v["members"].([]any)
|
||||
if len(members) != 2 {
|
||||
t.Fatalf("expected 2 members after update, got %v", v["members"])
|
||||
}
|
||||
|
||||
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/virtuals/crud-virtual"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete virtual: status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func getRepo(t *testing.T, name string) map[string]any {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v2/remotes/"+name), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("get remote %s: status %d: %s", name, resp.StatusCode, body)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
t.Fatalf("decode remote %s: %v", name, err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
|
||||
// checks that a virtual over them serves a merged simple index.
|
||||
func TestVirtualPyPIMerge(t *testing.T) {
|
||||
createRepo(t, `{"name":"pmerge-a","package_type":"pypi","repo_type":"local"}`)
|
||||
createRepo(t, `{"name":"pmerge-b","package_type":"pypi","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "pmerge-a")
|
||||
defer deleteRepo(t, "pmerge-b")
|
||||
|
||||
uploadFile(t, "pmerge-a", "foo-1.0-py3-none-any.whl", fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl"), "application/zip")
|
||||
uploadFile(t, "pmerge-b", "bar-2.0-py3-none-any.whl", []byte("bar wheel payload"), "application/zip")
|
||||
|
||||
createVirtual(t, `{"name":"pmerge-v","package_type":"pypi","members":["pmerge-a","pmerge-b"]}`)
|
||||
defer deleteVirtual(t, "pmerge-v")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/pmerge-v/simple/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("virtual simple index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "foo") || !strings.Contains(s, "bar") {
|
||||
t.Fatalf("merged index missing a member package (want foo and bar): %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVirtualHelmMerge points two helm remotes at mock index.yaml documents
|
||||
// with distinct charts and checks the virtual merges both into one index.
|
||||
func TestVirtualHelmMerge(t *testing.T) {
|
||||
createRepo(t, `{"name":"hmerge-a","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-a","stale_on_error":true}`)
|
||||
createRepo(t, `{"name":"hmerge-b","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-b","stale_on_error":true}`)
|
||||
defer deleteRepo(t, "hmerge-a")
|
||||
defer deleteRepo(t, "hmerge-b")
|
||||
|
||||
createVirtual(t, `{"name":"hmerge-v","package_type":"helm","members":["hmerge-a","hmerge-b"]}`)
|
||||
defer deleteVirtual(t, "hmerge-v")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/hmerge-v/index.yaml"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("virtual index.yaml: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "alpha") || !strings.Contains(s, "beta") {
|
||||
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
|
||||
}
|
||||
}
|
||||
-137
@@ -1,137 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres"
|
||||
tcredis "github.com/testcontainers/testcontainers-go/modules/redis"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
||||
"git.unkin.net/unkin/artifactapi/internal/server"
|
||||
)
|
||||
|
||||
var baseURL string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
|
||||
pgContainer, err := tcpostgres.Run(ctx,
|
||||
"postgres:17-alpine",
|
||||
tcpostgres.WithDatabase("artifacts"),
|
||||
tcpostgres.WithUsername("artifacts"),
|
||||
tcpostgres.WithPassword("artifacts123"),
|
||||
testcontainers.WithWaitStrategy(
|
||||
wait.ForListeningPort("5432/tcp").WithStartupTimeout(30*time.Second),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer pgContainer.Terminate(ctx)
|
||||
|
||||
redisContainer, err := tcredis.Run(ctx,
|
||||
"redis:7-alpine",
|
||||
testcontainers.WithWaitStrategy(
|
||||
wait.ForListeningPort("6379/tcp").WithStartupTimeout(30*time.Second),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("redis: %v", err)
|
||||
}
|
||||
defer redisContainer.Terminate(ctx)
|
||||
|
||||
minioContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
ContainerRequest: testcontainers.ContainerRequest{
|
||||
Image: "minio/minio:latest",
|
||||
ExposedPorts: []string{"9000/tcp"},
|
||||
Cmd: []string{"server", "/data"},
|
||||
Env: map[string]string{
|
||||
"MINIO_ROOT_USER": "minioadmin",
|
||||
"MINIO_ROOT_PASSWORD": "minioadmin",
|
||||
},
|
||||
WaitingFor: wait.ForHTTP("/minio/health/live").WithPort("9000/tcp").WithStartupTimeout(30 * time.Second),
|
||||
},
|
||||
Started: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("minio: %v", err)
|
||||
}
|
||||
defer minioContainer.Terminate(ctx)
|
||||
|
||||
pgHost, _ := pgContainer.Host(ctx)
|
||||
pgPort, _ := pgContainer.MappedPort(ctx, "5432/tcp")
|
||||
redisHost, _ := redisContainer.Host(ctx)
|
||||
redisPort, _ := redisContainer.MappedPort(ctx, "6379/tcp")
|
||||
minioHost, _ := minioContainer.Host(ctx)
|
||||
minioPort, _ := minioContainer.MappedPort(ctx, "9000/tcp")
|
||||
|
||||
os.Setenv("DBHOST", pgHost)
|
||||
os.Setenv("DBPORT", pgPort.Port())
|
||||
os.Setenv("DBUSER", "artifacts")
|
||||
os.Setenv("DBPASS", "artifacts123")
|
||||
os.Setenv("DBNAME", "artifacts")
|
||||
os.Setenv("DBSSL", "disable")
|
||||
os.Setenv("REDIS_URL", fmt.Sprintf("redis://%s:%s", redisHost, redisPort.Port()))
|
||||
os.Setenv("MINIO_ENDPOINT", fmt.Sprintf("%s:%s", minioHost, minioPort.Port()))
|
||||
os.Setenv("MINIO_ACCESS_KEY", "minioadmin")
|
||||
os.Setenv("MINIO_SECRET_KEY", "minioadmin")
|
||||
os.Setenv("MINIO_BUCKET", "artifacts-test")
|
||||
os.Setenv("MINIO_SECURE", "false")
|
||||
os.Setenv("LISTEN_ADDR", ":0")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
cfg.ListenAddr = "127.0.0.1:0"
|
||||
|
||||
srv, err := server.New(cfg, "e2e-test")
|
||||
if err != nil {
|
||||
log.Fatalf("server: %v", err)
|
||||
}
|
||||
|
||||
srvCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
addr := startServer(srvCtx, srv)
|
||||
baseURL = "http://" + addr
|
||||
|
||||
code := m.Run()
|
||||
cancel()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func startServer(ctx context.Context, srv *server.Server) string {
|
||||
ln, err := findListener()
|
||||
if err != nil {
|
||||
log.Fatalf("listener: %v", err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
|
||||
go srv.RunOnListener(ctx, ln)
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := http.Get("http://" + addr + "/health")
|
||||
if err == nil && resp.StatusCode == 200 {
|
||||
resp.Body.Close()
|
||||
return addr
|
||||
}
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
log.Fatalf("server did not start in time at %s", addr)
|
||||
return ""
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func findListener() (net.Listener, error) {
|
||||
return net.Listen("tcp", "127.0.0.1:0")
|
||||
}
|
||||
|
||||
func apiURL(path string) string {
|
||||
return baseURL + path
|
||||
}
|
||||
|
||||
func createRemote(t *testing.T, body string) {
|
||||
t.Helper()
|
||||
resp, err := http.Post(apiURL("/api/v2/remotes"), "application/json", strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("create remote: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("create remote: status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRemote(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(http.MethodDelete, apiURL("/api/v2/remotes/"+name), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("delete remote: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
func getJSON(t *testing.T, url string) map[string]any {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("GET %s: status %d: %s", url, resp.StatusCode, b)
|
||||
}
|
||||
var result map[string]any
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func getBody(t *testing.T, url string) ([]byte, int) {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return b, resp.StatusCode
|
||||
}
|
||||
|
||||
func getString(t *testing.T, url string) string {
|
||||
t.Helper()
|
||||
b, status := getBody(t, url)
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("GET %s: status %d: %s", url, status, b)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func assertStatus(t *testing.T, url string, wantStatus int) {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != wantStatus {
|
||||
t.Errorf("GET %s: got %d, want %d", url, resp.StatusCode, wantStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRequest(t *testing.T, url string) int {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(http.MethodDelete, url, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("DELETE %s: %v", url, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
func mustFormat(format string, args ...any) string {
|
||||
return fmt.Sprintf(format, args...)
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHealth(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/health"))
|
||||
if result["status"] != "ok" {
|
||||
t.Errorf("expected ok, got %v", result["status"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoot(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/"))
|
||||
if result["name"] != "artifactapi" {
|
||||
t.Errorf("expected artifactapi, got %v", result["name"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUpstreamTimeouts(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "timeout-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"stale_on_error": true,
|
||||
"upstream_dial_timeout": 3,
|
||||
"upstream_tls_timeout": 4,
|
||||
"upstream_response_header_timeout": 5
|
||||
}`)
|
||||
defer deleteRemote(t, "timeout-test")
|
||||
|
||||
remote := getJSON(t, apiURL("/api/v2/remotes/timeout-test"))
|
||||
for field, want := range map[string]float64{
|
||||
"upstream_dial_timeout": 3,
|
||||
"upstream_tls_timeout": 4,
|
||||
"upstream_response_header_timeout": 5,
|
||||
} {
|
||||
if got, _ := remote[field].(float64); got != want {
|
||||
t.Errorf("%s: got %v, want %v", field, remote[field], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteCRUD(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "test-generic",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"description": "test remote",
|
||||
"mutable_ttl": 600,
|
||||
"check_mutable": true,
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "test-generic")
|
||||
|
||||
remote := getJSON(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if remote["name"] != "test-generic" {
|
||||
t.Errorf("expected test-generic, got %v", remote["name"])
|
||||
}
|
||||
if remote["package_type"] != "generic" {
|
||||
t.Errorf("expected generic, got %v", remote["package_type"])
|
||||
}
|
||||
|
||||
req, _ := http.NewRequest(http.MethodPut, apiURL("/api/v2/remotes/test-generic"),
|
||||
strings.NewReader(`{
|
||||
"package_type": "generic",
|
||||
"base_url": "https://updated.example.com",
|
||||
"description": "updated",
|
||||
"mutable_ttl": 300,
|
||||
"check_mutable": true,
|
||||
"stale_on_error": true
|
||||
}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
updated := getJSON(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if updated["base_url"] != "https://updated.example.com" {
|
||||
t.Errorf("expected updated URL, got %v", updated["base_url"])
|
||||
}
|
||||
|
||||
status := deleteRequest(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if status != http.StatusNoContent {
|
||||
t.Errorf("delete: got %d, want 204", status)
|
||||
}
|
||||
|
||||
assertStatus(t, apiURL("/api/v2/remotes/test-generic"), http.StatusNotFound)
|
||||
}
|
||||
|
||||
func TestRemoteList(t *testing.T) {
|
||||
createRemote(t, `{"name":"list-a","package_type":"generic","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRemote(t, `{"name":"list-b","package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRemote(t, "list-a")
|
||||
defer deleteRemote(t, "list-b")
|
||||
|
||||
resp, err := http.Get(apiURL("/api/v2/remotes"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
|
||||
var remotes []map[string]any
|
||||
json.Unmarshal(body, &remotes)
|
||||
|
||||
if len(remotes) < 2 {
|
||||
t.Fatalf("expected at least 2 remotes, got %d", len(remotes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualCRUD(t *testing.T) {
|
||||
createRemote(t, `{"name":"virt-member-a","package_type":"helm","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRemote(t, `{"name":"virt-member-b","package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRemote(t, "virt-member-a")
|
||||
defer deleteRemote(t, "virt-member-b")
|
||||
|
||||
resp, err := http.Post(apiURL("/api/v2/virtuals"), "application/json",
|
||||
strings.NewReader(`{
|
||||
"name": "test-virtual",
|
||||
"package_type": "helm",
|
||||
"description": "test virtual",
|
||||
"members": ["virt-member-a", "virt-member-b"]
|
||||
}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create virtual: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
virt := getJSON(t, apiURL("/api/v2/virtuals/test-virtual"))
|
||||
if virt["name"] != "test-virtual" {
|
||||
t.Errorf("expected test-virtual, got %v", virt["name"])
|
||||
}
|
||||
|
||||
status := deleteRequest(t, apiURL("/api/v2/virtuals/test-virtual"))
|
||||
if status != http.StatusNoContent {
|
||||
t.Errorf("delete virtual: got %d, want 204", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsEndpoint(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/api/v2/stats"))
|
||||
if _, ok := result["total_remotes"]; !ok {
|
||||
t.Error("expected total_remotes in stats")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthV2(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/api/v2/health"))
|
||||
if result["status"] != "ok" {
|
||||
t.Errorf("expected ok, got %v", result["status"])
|
||||
}
|
||||
if result["postgres"] != "ok" {
|
||||
t.Errorf("expected postgres ok, got %v", result["postgres"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidPackageType(t *testing.T) {
|
||||
resp, err := http.Post(apiURL("/api/v2/remotes"), "application/json",
|
||||
strings.NewReader(`{"name":"bad","package_type":"bogus","base_url":"https://x.com"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("expected 400 for invalid package type, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProxyUnknownRemote(t *testing.T) {
|
||||
assertStatus(t, apiURL("/api/v1/remote/nonexistent/some/path"), http.StatusNotFound)
|
||||
}
|
||||
|
||||
func TestProxyBlocklist(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "blocklist-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"blocklist": ["\\.exe$"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "blocklist-test")
|
||||
|
||||
assertStatus(t, apiURL("/api/v1/remote/blocklist-test/malware.exe"), http.StatusForbidden)
|
||||
}
|
||||
|
||||
func TestProxyHeadBlocklist(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "head-block-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"blocklist": ["\\.exe$"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "head-block-test")
|
||||
|
||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/head-block-test/malware.exe"), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("HEAD blocklisted path: got %d, want 403", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyHeadUnknownRemote(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/nonexistent/some/path"), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("HEAD unknown remote: got %d, want 404", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyPatterns(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "patterns-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"patterns": ["^releases/"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "patterns-test")
|
||||
|
||||
assertStatus(t, apiURL("/api/v1/remote/patterns-test/uploads/file.tar.gz"), http.StatusForbidden)
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
module git.unkin.net/unkin/artifactapi
|
||||
|
||||
go 1.25.9
|
||||
|
||||
require (
|
||||
github.com/cavaliergopher/rpm v1.3.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/go-chi/chi/v5 v5.3.0
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/minio/minio-go/v7 v7.2.0
|
||||
github.com/redis/go-redis/v9 v9.20.0
|
||||
github.com/testcontainers/testcontainers-go v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
|
||||
golang.org/x/crypto v0.51.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.4.3 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.11.7 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
|
||||
github.com/charmbracelet/x/term v0.2.2 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.11.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v0.2.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.23 // indirect
|
||||
github.com/mdelapenya/tlscert v0.2.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/moby/api v1.54.1 // indirect
|
||||
github.com/moby/moby/client v0.4.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/stretchr/testify v1.11.1 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.2 // indirect
|
||||
)
|
||||
@@ -1,249 +0,0 @@
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cavaliergopher/rpm v1.3.0 h1:UHX46sasX8MesUXXQ+UbkFLUX4eUWTlEcX8jcnRBIgI=
|
||||
github.com/cavaliergopher/rpm v1.3.0/go.mod h1:vEumo1vvtrHM1Ov86f6+k8j7zNKOxQfHDCAIcR/36ZI=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
|
||||
github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
|
||||
github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=
|
||||
github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q=
|
||||
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
|
||||
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
|
||||
github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI=
|
||||
github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
|
||||
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
|
||||
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
|
||||
github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8=
|
||||
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
|
||||
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw=
|
||||
github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
|
||||
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs=
|
||||
github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
||||
github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4=
|
||||
github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||
github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw=
|
||||
github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g=
|
||||
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
|
||||
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
|
||||
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
|
||||
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0=
|
||||
github.com/redis/go-redis/v9 v9.20.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
||||
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
|
||||
github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 h1:id/6LH8ZeDrtAUVSuNvZUAJ1kVpb82y1pr9yweAWsRg=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0/go.mod h1:uF0jI8FITagQpBNOgweGBmPf6rP4K0SeL1XFPbsZSSY=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
||||
go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
|
||||
go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
|
||||
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss=
|
||||
gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||
pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
|
||||
pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||
@@ -1,301 +0,0 @@
|
||||
// Package terraform serves local terraform repos as a real Terraform provider
|
||||
// registry: service discovery, version listing, and GPG-signed downloads, so
|
||||
// `terraform init` installs from a bare source address with no client config.
|
||||
package terraform
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
tfprov "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
"git.unkin.net/unkin/artifactapi/internal/tfsign"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// ProvidersV1Path is the base the service-discovery document advertises (Terraform
|
||||
// appends "{namespace}/{type}/versions" etc). MountPath is the same prefix without
|
||||
// the trailing slash, for chi.Mount.
|
||||
const (
|
||||
ProvidersV1Path = "/terraform/v1/providers/"
|
||||
MountPath = "/terraform/v1/providers"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
db *database.DB
|
||||
signer *tfsign.Signer
|
||||
protocols []string
|
||||
}
|
||||
|
||||
func NewHandler(db *database.DB, signer *tfsign.Signer, protocols string) *Handler {
|
||||
var protos []string
|
||||
for _, p := range strings.Split(protocols, ",") {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
protos = append(protos, p)
|
||||
}
|
||||
}
|
||||
if len(protos) == 0 {
|
||||
protos = []string{"5.0", "6.0"}
|
||||
}
|
||||
return &Handler{db: db, signer: signer, protocols: protos}
|
||||
}
|
||||
|
||||
// Enabled reports whether a signing key is configured. Without one the registry
|
||||
// cannot produce the signed SHA256SUMS the protocol requires, so it stays off.
|
||||
func (h *Handler) Enabled() bool { return h.signer != nil }
|
||||
|
||||
func (h *Handler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/{namespace}/{type}/versions", h.versions)
|
||||
r.Get("/{namespace}/{type}/{version}/download/{os}/{arch}", h.download)
|
||||
r.Get("/{namespace}/{type}/{version}/sha256sums", h.sha256sums)
|
||||
r.Get("/{namespace}/{type}/{version}/sha256sums.sig", h.sha256sumsSig)
|
||||
return r
|
||||
}
|
||||
|
||||
// ServiceDiscovery answers /.well-known/terraform.json, pointing Terraform at the
|
||||
// providers.v1 protocol base.
|
||||
func (h *Handler) ServiceDiscovery(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.Enabled() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]string{"providers.v1": ProvidersV1Path})
|
||||
}
|
||||
|
||||
// providerFile is one resolved platform artifact within a repo.
|
||||
type providerFile struct {
|
||||
version string
|
||||
os string
|
||||
arch string
|
||||
filePath string // path within the repo, e.g. unkin/artifactapi/...zip
|
||||
sha256 string // hex, no "sha256:" prefix
|
||||
}
|
||||
|
||||
// resolve finds every provider zip of the given type in the repo (namespace).
|
||||
// The Terraform source namespace maps to the artifactapi repo name; the provider
|
||||
// is matched by type across whatever in-repo folder it was uploaded under.
|
||||
func (h *Handler) resolve(r *http.Request, namespace, typeName string) ([]providerFile, error) {
|
||||
remote, err := h.db.GetRemote(r.Context(), namespace)
|
||||
if err != nil || remote.PackageType != models.PackageTerraform {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rows, err := h.db.ListLocalFiles(r.Context(), namespace, 10000, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var out []providerFile
|
||||
for _, row := range rows {
|
||||
parsed := tfprov.ParseProviderZip(path.Base(row.FilePath))
|
||||
if !parsed.Ok || parsed.Type != typeName {
|
||||
continue
|
||||
}
|
||||
out = append(out, providerFile{
|
||||
version: parsed.Version,
|
||||
os: parsed.OS,
|
||||
arch: parsed.Arch,
|
||||
filePath: row.FilePath,
|
||||
sha256: strings.TrimPrefix(row.ContentHash, "sha256:"),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h *Handler) versions(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.Enabled() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
namespace := chi.URLParam(r, "namespace")
|
||||
typeName := chi.URLParam(r, "type")
|
||||
|
||||
files, err := h.resolve(r, namespace, typeName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if len(files) == 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Group platforms by version, de-duplicated and stably ordered.
|
||||
type platform struct {
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
}
|
||||
platforms := map[string]map[string]platform{}
|
||||
for _, f := range files {
|
||||
if platforms[f.version] == nil {
|
||||
platforms[f.version] = map[string]platform{}
|
||||
}
|
||||
platforms[f.version][f.os+"_"+f.arch] = platform{OS: f.os, Arch: f.arch}
|
||||
}
|
||||
|
||||
type versionEntry struct {
|
||||
Version string `json:"version"`
|
||||
Protocols []string `json:"protocols"`
|
||||
Platforms []platform `json:"platforms"`
|
||||
}
|
||||
out := struct {
|
||||
Versions []versionEntry `json:"versions"`
|
||||
}{}
|
||||
for version, plats := range platforms {
|
||||
entry := versionEntry{Version: version, Protocols: h.protocols}
|
||||
for _, p := range plats {
|
||||
entry.Platforms = append(entry.Platforms, p)
|
||||
}
|
||||
sort.Slice(entry.Platforms, func(i, j int) bool {
|
||||
return entry.Platforms[i].OS+entry.Platforms[i].Arch < entry.Platforms[j].OS+entry.Platforms[j].Arch
|
||||
})
|
||||
out.Versions = append(out.Versions, entry)
|
||||
}
|
||||
sort.Slice(out.Versions, func(i, j int) bool { return out.Versions[i].Version < out.Versions[j].Version })
|
||||
|
||||
writeJSON(w, out)
|
||||
}
|
||||
|
||||
func (h *Handler) download(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.Enabled() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
namespace := chi.URLParam(r, "namespace")
|
||||
typeName := chi.URLParam(r, "type")
|
||||
version := chi.URLParam(r, "version")
|
||||
osName := chi.URLParam(r, "os")
|
||||
arch := chi.URLParam(r, "arch")
|
||||
|
||||
files, err := h.resolve(r, namespace, typeName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
var match *providerFile
|
||||
for i := range files {
|
||||
if files[i].version == version && files[i].os == osName && files[i].arch == arch {
|
||||
match = &files[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if match == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
base := baseURL(r)
|
||||
verBase := fmt.Sprintf("%s%s/%s/%s", base+ProvidersV1Path, namespace, typeName, version)
|
||||
|
||||
type gpgKey struct {
|
||||
KeyID string `json:"key_id"`
|
||||
ASCIIArmor string `json:"ascii_armor"`
|
||||
}
|
||||
resp := struct {
|
||||
Protocols []string `json:"protocols"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
Filename string `json:"filename"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
SHASumsURL string `json:"shasums_url"`
|
||||
SHASumsSignatureURL string `json:"shasums_signature_url"`
|
||||
SHASum string `json:"shasum"`
|
||||
SigningKeys struct {
|
||||
GPGPublicKeys []gpgKey `json:"gpg_public_keys"`
|
||||
} `json:"signing_keys"`
|
||||
}{
|
||||
Protocols: h.protocols,
|
||||
OS: match.os,
|
||||
Arch: match.arch,
|
||||
Filename: path.Base(match.filePath),
|
||||
DownloadURL: fmt.Sprintf("%s/api/v1/local/%s/%s", base, namespace, match.filePath),
|
||||
SHASumsURL: verBase + "/sha256sums",
|
||||
SHASumsSignatureURL: verBase + "/sha256sums.sig",
|
||||
SHASum: match.sha256,
|
||||
}
|
||||
resp.SigningKeys.GPGPublicKeys = []gpgKey{{
|
||||
KeyID: h.signer.KeyID(),
|
||||
ASCIIArmor: h.signer.PublicKeyArmor(),
|
||||
}}
|
||||
|
||||
writeJSON(w, resp)
|
||||
}
|
||||
|
||||
func (h *Handler) sha256sums(w http.ResponseWriter, r *http.Request) {
|
||||
sums, ok := h.buildSums(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Write(sums)
|
||||
}
|
||||
|
||||
func (h *Handler) sha256sumsSig(w http.ResponseWriter, r *http.Request) {
|
||||
sums, ok := h.buildSums(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
sig, err := h.signer.Sign(sums)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Write(sig)
|
||||
}
|
||||
|
||||
// buildSums renders the SHA256SUMS body for one version: one "<hex> <filename>"
|
||||
// line per platform zip, sorted by filename so the signed bytes are stable.
|
||||
func (h *Handler) buildSums(w http.ResponseWriter, r *http.Request) ([]byte, bool) {
|
||||
if !h.Enabled() {
|
||||
http.NotFound(w, r)
|
||||
return nil, false
|
||||
}
|
||||
namespace := chi.URLParam(r, "namespace")
|
||||
typeName := chi.URLParam(r, "type")
|
||||
version := chi.URLParam(r, "version")
|
||||
|
||||
files, err := h.resolve(r, namespace, typeName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return nil, false
|
||||
}
|
||||
|
||||
var lines []string
|
||||
for _, f := range files {
|
||||
if f.version != version {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("%s %s", f.sha256, path.Base(f.filePath)))
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
http.NotFound(w, r)
|
||||
return nil, false
|
||||
}
|
||||
sort.Strings(lines)
|
||||
return []byte(strings.Join(lines, "\n") + "\n"), true
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
func baseURL(r *http.Request) string {
|
||||
scheme := "http"
|
||||
if r.TLS != nil {
|
||||
scheme = "https"
|
||||
}
|
||||
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
|
||||
scheme = fwd
|
||||
}
|
||||
return scheme + "://" + r.Host
|
||||
}
|
||||
@@ -1,186 +0,0 @@
|
||||
package terraform
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"golang.org/x/crypto/openpgp"
|
||||
"golang.org/x/crypto/openpgp/armor"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/internal/tfsign"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
var testDSN string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, terminate, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN = dsn
|
||||
code := m.Run()
|
||||
terminate()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// testSigner writes a throwaway armored key and loads it.
|
||||
func testSigner(t *testing.T) *tfsign.Signer {
|
||||
t.Helper()
|
||||
e, err := openpgp.NewEntity("artifactapi test", "tf", "tf@example.com", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
w, _ := armor.Encode(&buf, openpgp.PrivateKeyType, nil)
|
||||
if err := e.SerializePrivate(w, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Close()
|
||||
p := filepath.Join(t.TempDir(), "private-key.asc")
|
||||
if err := os.WriteFile(p, buf.Bytes(), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := tfsign.Load(p, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestProviderRegistryFlow(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const repo = "tf-reg" // Terraform namespace == repo name
|
||||
const filePath = "unkin/artifactapi/terraform-provider-artifactapi_1.2.3_linux_amd64.zip"
|
||||
const hash = "sha256:983cdb25cb7b976538e4334d26e52dee5f44749b9be1500c760cf5cf66be659b"
|
||||
const wantSha = "983cdb25cb7b976538e4334d26e52dee5f44749b9be1500c760cf5cf66be659b"
|
||||
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageTerraform, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.UpsertBlob(ctx, hash, "blobs/98/3c", 6381007, "application/zip"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.CreateLocalFile(ctx, repo, filePath, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
signer := testSigner(t)
|
||||
h := NewHandler(db, signer, "5.0,6.0")
|
||||
router := chi.NewRouter()
|
||||
router.Get("/.well-known/terraform.json", h.ServiceDiscovery)
|
||||
router.Mount(MountPath, h.Routes())
|
||||
|
||||
get := func(p string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest("GET", p, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
return w
|
||||
}
|
||||
|
||||
// Service discovery.
|
||||
w := get("/.well-known/terraform.json")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("discovery = %d", w.Code)
|
||||
}
|
||||
var disc map[string]string
|
||||
json.Unmarshal(w.Body.Bytes(), &disc)
|
||||
if disc["providers.v1"] != ProvidersV1Path {
|
||||
t.Errorf("providers.v1 = %q", disc["providers.v1"])
|
||||
}
|
||||
|
||||
// Versions.
|
||||
w = get("/terraform/v1/providers/tf-reg/artifactapi/versions")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("versions = %d %s", w.Code, w.Body)
|
||||
}
|
||||
var vresp struct {
|
||||
Versions []struct {
|
||||
Version string `json:"version"`
|
||||
Protocols []string `json:"protocols"`
|
||||
Platforms []map[string]string `json:"platforms"`
|
||||
} `json:"versions"`
|
||||
}
|
||||
json.Unmarshal(w.Body.Bytes(), &vresp)
|
||||
if len(vresp.Versions) != 1 || vresp.Versions[0].Version != "1.2.3" {
|
||||
t.Fatalf("unexpected versions: %+v", vresp)
|
||||
}
|
||||
if len(vresp.Versions[0].Platforms) != 1 || vresp.Versions[0].Platforms[0]["os"] != "linux" {
|
||||
t.Fatalf("unexpected platforms: %+v", vresp.Versions[0].Platforms)
|
||||
}
|
||||
|
||||
// Download.
|
||||
w = get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/download/linux/amd64")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("download = %d %s", w.Code, w.Body)
|
||||
}
|
||||
var dl struct {
|
||||
Filename string `json:"filename"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
SHASumsURL string `json:"shasums_url"`
|
||||
SHASumsSignatureURL string `json:"shasums_signature_url"`
|
||||
SHASum string `json:"shasum"`
|
||||
SigningKeys struct {
|
||||
GPGPublicKeys []struct {
|
||||
KeyID string `json:"key_id"`
|
||||
ASCIIArmor string `json:"ascii_armor"`
|
||||
} `json:"gpg_public_keys"`
|
||||
} `json:"signing_keys"`
|
||||
}
|
||||
json.Unmarshal(w.Body.Bytes(), &dl)
|
||||
if dl.SHASum != wantSha {
|
||||
t.Errorf("shasum = %q", dl.SHASum)
|
||||
}
|
||||
wantURL := "http://example.com/api/v1/local/tf-reg/" + filePath
|
||||
if dl.DownloadURL != wantURL {
|
||||
t.Errorf("download_url = %q, want %q", dl.DownloadURL, wantURL)
|
||||
}
|
||||
if len(dl.SigningKeys.GPGPublicKeys) != 1 || dl.SigningKeys.GPGPublicKeys[0].KeyID != signer.KeyID() {
|
||||
t.Errorf("signing key mismatch: %+v", dl.SigningKeys)
|
||||
}
|
||||
|
||||
// SHA256SUMS + signature verify against the advertised key.
|
||||
sums := get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/sha256sums")
|
||||
wantLine := wantSha + " terraform-provider-artifactapi_1.2.3_linux_amd64.zip\n"
|
||||
if sums.Body.String() != wantLine {
|
||||
t.Errorf("sha256sums = %q, want %q", sums.Body.String(), wantLine)
|
||||
}
|
||||
sig := get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/sha256sums.sig")
|
||||
keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewReader([]byte(dl.SigningKeys.GPGPublicKeys[0].ASCIIArmor)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewReader(sums.Body.Bytes()), bytes.NewReader(sig.Body.Bytes())); err != nil {
|
||||
t.Errorf("sha256sums.sig did not verify: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryDisabledWithoutSigner(t *testing.T) {
|
||||
h := NewHandler(nil, nil, "")
|
||||
router := chi.NewRouter()
|
||||
router.Get("/.well-known/terraform.json", h.ServiceDiscovery)
|
||||
req := httptest.NewRequest("GET", "/.well-known/terraform.json", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 404 {
|
||||
t.Errorf("disabled discovery = %d, want 404", w.Code)
|
||||
}
|
||||
}
|
||||
@@ -1,209 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
type ProxyHandler struct {
|
||||
engine *proxy.Engine
|
||||
virtualEngine *virtual.Engine
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
local *v2.LocalHandler
|
||||
}
|
||||
|
||||
func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *database.DB, store *storage.S3, local *v2.LocalHandler) *ProxyHandler {
|
||||
return &ProxyHandler{engine: engine, virtualEngine: virtualEngine, db: db, store: store, local: local}
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/remote/{remoteName}/*", h.handleProxy)
|
||||
r.Get("/local/{localName}/*", h.handleLocal)
|
||||
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) DockerV2Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.handleDockerPing)
|
||||
r.Head("/", h.handleDockerPing)
|
||||
r.Get("/{remoteName}/*", h.handleProxy)
|
||||
r.Head("/{remoteName}/*", h.handleProxyHead)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Docker-Distribution-Api-Version", "registry/2.0")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy fetch failed", "remote", remoteName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer result.Reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, result.Reader)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.engine.Head(r.Context(), *remote, path, prov)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
||||
virtualName := chi.URLParam(r, "virtualName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
virt, err := h.db.GetVirtual(r.Context(), virtualName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("virtual %q not found", virtualName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||
|
||||
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
||||
if err != nil {
|
||||
slog.Error("virtual fetch failed", "virtual", virtualName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("X-Artifact-Source", "virtual")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
|
||||
localName := chi.URLParam(r, "localName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), localName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("local %q not found", localName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
if indexer, ok := prov.(provider.LocalIndexer); ok {
|
||||
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
h.serveLocalFile(w, r, localName, path)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, repoName, path string) {
|
||||
file, err := h.db.GetLocalFile(r.Context(), repoName, path)
|
||||
if err != nil {
|
||||
slog.Error("local file lookup failed", "repo", repoName, "path", path, "error", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if file == nil {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):])
|
||||
reader, info, err := h.store.Download(r.Context(), s3Key)
|
||||
if err != nil {
|
||||
slog.Error("local file download failed", "repo", repoName, "path", path, "error", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", info.ContentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func scheme(r *http.Request) string {
|
||||
if r.TLS != nil {
|
||||
return "https"
|
||||
}
|
||||
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
|
||||
return fwd
|
||||
}
|
||||
return "http"
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestScheme(t *testing.T) {
|
||||
if got := scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
|
||||
t.Errorf("TLS request scheme = %q, want https", got)
|
||||
}
|
||||
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {"https"}}}
|
||||
if got := scheme(r); got != "https" {
|
||||
t.Errorf("X-Forwarded-Proto scheme = %q, want https", got)
|
||||
}
|
||||
if got := scheme(&http.Request{Header: http.Header{}}); got != "http" {
|
||||
t.Errorf("default scheme = %q, want http", got)
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
var testDSN string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, terminate, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN = dsn
|
||||
code := m.Run()
|
||||
terminate()
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
// closedDB returns a DB whose pool has been closed, so every query fails —
|
||||
// used to drive the handlers' error branches.
|
||||
func closedDB(t *testing.T) *database.DB {
|
||||
t.Helper()
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatalf("new db: %v", err)
|
||||
}
|
||||
db.Close()
|
||||
return db
|
||||
}
|
||||
|
||||
func do(t *testing.T, h http.Handler, method, path, body string) int {
|
||||
t.Helper()
|
||||
var r io.Reader
|
||||
if body != "" {
|
||||
r = strings.NewReader(body)
|
||||
}
|
||||
req := httptest.NewRequest(method, path, r)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
return w.Code
|
||||
}
|
||||
|
||||
func TestRemotesErrorPaths(t *testing.T) {
|
||||
h := NewRemotesHandler(closedDB(t)).Routes()
|
||||
if c := do(t, h, "GET", "/", ""); c != 500 {
|
||||
t.Errorf("list with dead db = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "POST", "/", `{"name":"x","package_type":"generic","repo_type":"remote","base_url":"https://x"}`); c != 500 {
|
||||
t.Errorf("create with dead db = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "PUT", "/x", `{"package_type":"generic","base_url":"https://x"}`); c != 500 {
|
||||
t.Errorf("update with dead db = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "GET", "/x", ""); c != 404 {
|
||||
t.Errorf("get missing = %d, want 404", c)
|
||||
}
|
||||
if c := do(t, h, "DELETE", "/x", ""); c != 500 {
|
||||
t.Errorf("delete with dead db = %d, want 500", c)
|
||||
}
|
||||
// Bad request bodies never reach the db.
|
||||
if c := do(t, h, "POST", "/", `not json`); c != 400 {
|
||||
t.Errorf("invalid json = %d, want 400", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualsErrorPaths(t *testing.T) {
|
||||
h := NewVirtualsHandler(closedDB(t)).Routes()
|
||||
if c := do(t, h, "GET", "/", ""); c != 500 {
|
||||
t.Errorf("list = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "GET", "/x", ""); c != 404 {
|
||||
t.Errorf("get missing = %d, want 404", c)
|
||||
}
|
||||
if c := do(t, h, "POST", "/", `{"name":"v","package_type":"helm","members":["a"]}`); c != 500 {
|
||||
t.Errorf("create = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "PUT", "/v", `{"package_type":"helm","members":["a"]}`); c != 500 {
|
||||
t.Errorf("update = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "DELETE", "/v", ""); c != 500 {
|
||||
t.Errorf("delete = %d, want 500", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsErrorPaths(t *testing.T) {
|
||||
h := NewStatsHandler(closedDB(t)).Routes()
|
||||
for _, p := range []string{"/", "/top-remotes", "/top-files-by-hits", "/top-files-by-bandwidth"} {
|
||||
if c := do(t, h, "GET", p, ""); c != 500 {
|
||||
t.Errorf("stats %s = %d, want 500", p, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalErrorPaths(t *testing.T) {
|
||||
h := NewLocalHandler(closedDB(t), nil).Routes()
|
||||
// GetRemote fails on the closed db -> not found.
|
||||
if c := do(t, h, "PUT", "/x/files/a.bin", "data"); c != 404 {
|
||||
t.Errorf("upload unknown repo = %d, want 404", c)
|
||||
}
|
||||
// download / remove hit the db and 500.
|
||||
if c := do(t, h, "GET", "/x/files/a.bin", ""); c != 500 {
|
||||
t.Errorf("download = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "DELETE", "/x/files/a.bin", ""); c != 500 {
|
||||
t.Errorf("remove = %d, want 500", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalHandlerDBAccessor(t *testing.T) {
|
||||
db := closedDB(t)
|
||||
if NewLocalHandler(db, nil).DB() != db {
|
||||
t.Error("DB() should return the handler's database")
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
type EventsHandler struct{}
|
||||
|
||||
func NewEventsHandler() *EventsHandler {
|
||||
return &EventsHandler{}
|
||||
}
|
||||
|
||||
func (h *EventsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.stream)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *EventsHandler) stream(w http.ResponseWriter, r *http.Request) {
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
http.Error(w, "streaming not supported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("X-Accel-Buffering", "no")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
flusher.Flush()
|
||||
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
fmt.Fprintf(w, ": keepalive\n\n")
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
)
|
||||
|
||||
type HealthHandler struct {
|
||||
db *database.DB
|
||||
cache *cache.Redis
|
||||
store *storage.S3
|
||||
}
|
||||
|
||||
func NewHealthHandler(db *database.DB, c *cache.Redis, s *storage.S3) *HealthHandler {
|
||||
return &HealthHandler{db: db, cache: c, store: s}
|
||||
}
|
||||
|
||||
func (h *HealthHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.health)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *HealthHandler) health(w http.ResponseWriter, r *http.Request) {
|
||||
status := map[string]string{
|
||||
"status": "ok",
|
||||
"postgres": "ok",
|
||||
"redis": "ok",
|
||||
"s3": "ok",
|
||||
}
|
||||
|
||||
if err := h.db.Pool.Ping(r.Context()); err != nil {
|
||||
status["postgres"] = "error"
|
||||
status["status"] = "degraded"
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
@@ -1,227 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type LocalHandler struct {
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
cas *storage.CAS
|
||||
}
|
||||
|
||||
func NewLocalHandler(db *database.DB, store *storage.S3) *LocalHandler {
|
||||
return &LocalHandler{
|
||||
db: db,
|
||||
store: store,
|
||||
cas: storage.NewCAS(store),
|
||||
}
|
||||
}
|
||||
|
||||
func (h *LocalHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Put("/*", h.upload)
|
||||
r.Get("/*", h.download)
|
||||
r.Delete("/*", h.remove)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *LocalHandler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
if filePath == "" {
|
||||
http.Error(w, "file path required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", repoName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if remote.RepoType != models.RepoTypeLocal {
|
||||
http.Error(w, "upload only allowed for local repository types", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
|
||||
if uploader, ok := prov.(provider.LocalUploader); ok {
|
||||
h.uploadValidated(w, r, remote, filePath, prov, uploader)
|
||||
return
|
||||
}
|
||||
|
||||
h.uploadGeneric(w, r, remote, filePath)
|
||||
}
|
||||
|
||||
func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string, prov provider.Provider, uploader provider.LocalUploader) {
|
||||
storagePath, contentType, err := uploader.ValidateUpload(filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, storagePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if existing != nil {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.cas.Store(r.Context(), r.Body, contentType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.CreateLocalFile(r.Context(), remote.Name, storagePath, result.ContentHash); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if hook, ok := prov.(provider.PostUploadHook); ok {
|
||||
go hook.AfterUpload(context.Background(), remote.Name, storagePath, result.ContentHash, h, h.db)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, uploader.UploadResponse(storagePath, result.ContentHash, result.SizeBytes))
|
||||
}
|
||||
|
||||
func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) {
|
||||
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if existing != nil {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", filePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := "application/octet-stream"
|
||||
if ct := r.Header.Get("Content-Type"); ct != "" && ct != "application/octet-stream" {
|
||||
contentType = ct
|
||||
}
|
||||
|
||||
result, err := h.cas.Store(r.Context(), r.Body, contentType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.CreateLocalFile(r.Context(), remote.Name, filePath, result.ContentHash); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", filePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, map[string]any{
|
||||
"path": filePath,
|
||||
"content_hash": result.ContentHash,
|
||||
"size_bytes": result.SizeBytes,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *LocalHandler) download(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
file, err := h.db.GetLocalFile(r.Context(), repoName, filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if file == nil {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):])
|
||||
reader, info, err := h.store.Download(r.Context(), s3Key)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("download failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", info.ContentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func (h *LocalHandler) remove(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
if err := deleteLocalFile(r.Context(), h.db, repoName, filePath); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// deleteLocalFile removes a local file and runs the provider's post-delete hook,
|
||||
// so provider-derived state (e.g. RPM metadata that feeds generated repodata)
|
||||
// stops referencing a package that no longer exists.
|
||||
func deleteLocalFile(ctx context.Context, db *database.DB, repoName, filePath string) error {
|
||||
if err := db.DeleteLocalFile(ctx, repoName, filePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
remote, err := db.GetRemote(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil // file is gone; no repo left to resolve a cleanup hook from
|
||||
}
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if hook, ok := prov.(provider.PostDeleteHook); ok {
|
||||
return hook.AfterDelete(ctx, repoName, filePath, db)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *LocalHandler) DB() *database.DB {
|
||||
return h.db
|
||||
}
|
||||
|
||||
func (h *LocalHandler) Download(ctx context.Context, key string) (io.ReadCloser, int64, error) {
|
||||
reader, info, err := h.store.Download(ctx, key)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return reader, info.Size, nil
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/rpm" // register the rpm provider so its PostDeleteHook runs
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestLocalEvictCleansRPMMetadata verifies that evicting an RPM from a local
|
||||
// repo also removes the derived rpm_metadata row, so generated repodata stops
|
||||
// listing the deleted package.
|
||||
func TestLocalEvictCleansRPMMetadata(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const repo = "rpm-evict-cleanup"
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageRPM, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const hash = "sha256:bb22"
|
||||
const path = "Packages/example-0.1.0-1.x86_64.rpm"
|
||||
if err := db.UpsertBlob(ctx, hash, "blobs/bb/22", 2048, "application/x-rpm"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.CreateLocalFile(ctx, repo, path, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.InsertRPMMetadata(ctx, &provider.RPMMetadata{
|
||||
RepoName: repo, FilePath: path, ContentHash: hash,
|
||||
Name: "example", Version: "0.1.0", Release: "1", Arch: "x86_64",
|
||||
Requires: []provider.RPMDep{}, Provides: []provider.RPMDep{},
|
||||
Files: []provider.RPMFile{}, Changelogs: []provider.RPMChangelog{},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewObjectsHandler(db)
|
||||
router := chi.NewRouter()
|
||||
router.Route("/locals/{name}/objects", func(r chi.Router) {
|
||||
r.Delete("/*", h.LocalRoutes().ServeHTTP)
|
||||
})
|
||||
|
||||
del := httptest.NewRequest("DELETE", "/locals/"+repo+"/objects/"+path, nil)
|
||||
dw := httptest.NewRecorder()
|
||||
router.ServeHTTP(dw, del)
|
||||
if dw.Code != 204 {
|
||||
t.Fatalf("evict = %d, want 204", dw.Code)
|
||||
}
|
||||
|
||||
if f, _ := db.GetLocalFile(ctx, repo, path); f != nil {
|
||||
t.Fatalf("local file still present after evict: %+v", f)
|
||||
}
|
||||
entries, err := db.ListRPMMetadataEntries(ctx, repo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("rpm_metadata still present after evict: %+v", entries)
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestLocalUploadStoreFailure covers the upload handlers' store-error branches
|
||||
// by killing the object store after a successful upload.
|
||||
func TestLocalUploadStoreFailure(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
conn, termMinio, err := testsupport.StartMinio(ctx)
|
||||
if err != nil {
|
||||
t.Skip("minio unavailable")
|
||||
}
|
||||
var store *storage.S3
|
||||
for i := 0; i < 20; i++ {
|
||||
if store, err = storage.NewS3(conn.Endpoint, conn.AccessKey, conn.SecretKey, "fault", false, ""); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
termMinio()
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, pt := range []models.PackageType{models.PackageGeneric, models.PackagePyPI} {
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: "fault-" + string(pt), PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
h := NewLocalHandler(db, store)
|
||||
router := chi.NewRouter()
|
||||
router.Route("/remotes/{name}/files", func(r chi.Router) {
|
||||
r.Put("/*", h.Routes().ServeHTTP)
|
||||
})
|
||||
srv := httptest.NewServer(router)
|
||||
defer srv.Close()
|
||||
|
||||
put := func(name, path, body string) int {
|
||||
rq, _ := http.NewRequest("PUT", srv.URL+"/remotes/"+name+"/files/"+path, strings.NewReader(body))
|
||||
resp, err := http.DefaultClient.Do(rq)
|
||||
if err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
// Sanity: uploads succeed while the store is up.
|
||||
if c := put("fault-generic", "ok.bin", "data"); c != 201 {
|
||||
t.Fatalf("generic upload while up = %d", c)
|
||||
}
|
||||
if c := put("fault-pypi", "foo-1.0-py3-none-any.whl", "wheel"); c != 201 {
|
||||
t.Fatalf("pypi upload while up = %d", c)
|
||||
}
|
||||
|
||||
// Kill the store; subsequent CAS.Store calls fail -> 500.
|
||||
termMinio()
|
||||
if c := put("fault-generic", "after.bin", "data"); c != 500 {
|
||||
t.Errorf("generic upload after store down = %d, want 500", c)
|
||||
}
|
||||
if c := put("fault-pypi", "bar-1.0-py3-none-any.whl", "wheel"); c != 500 {
|
||||
t.Errorf("pypi upload after store down = %d, want 500", c)
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestLocalObjectsListing verifies that files uploaded to a local repo (which
|
||||
// live in local_files, not artifacts) are listed by the local objects endpoint
|
||||
// and can be evicted through it.
|
||||
func TestLocalObjectsListing(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const repo = "rpm-local-objs"
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageRPM, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const hash = "sha256:aa11"
|
||||
const path = "Packages/example-0.1.0-1.x86_64.rpm"
|
||||
if err := db.UpsertBlob(ctx, hash, "blobs/aa/11", 1234, "application/x-rpm"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.CreateLocalFile(ctx, repo, path, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewObjectsHandler(db)
|
||||
router := chi.NewRouter()
|
||||
router.Route("/locals/{name}/objects", func(r chi.Router) {
|
||||
r.Get("/", h.LocalRoutes().ServeHTTP)
|
||||
r.Delete("/*", h.LocalRoutes().ServeHTTP)
|
||||
})
|
||||
|
||||
// The uploaded package must appear in the listing with its blob size.
|
||||
req := httptest.NewRequest("GET", "/locals/"+repo+"/objects", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("list = %d, want 200", w.Code)
|
||||
}
|
||||
var got []models.Artifact
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d objects, want 1", len(got))
|
||||
}
|
||||
if got[0].Path != path || got[0].SizeBytes != 1234 || got[0].ContentHash != hash {
|
||||
t.Fatalf("unexpected object: %+v", got[0])
|
||||
}
|
||||
|
||||
// Eviction removes it from local_files.
|
||||
del := httptest.NewRequest("DELETE", "/locals/"+repo+"/objects/"+path, nil)
|
||||
dw := httptest.NewRecorder()
|
||||
router.ServeHTTP(dw, del)
|
||||
if dw.Code != 204 {
|
||||
t.Fatalf("evict = %d, want 204", dw.Code)
|
||||
}
|
||||
if f, _ := db.GetLocalFile(ctx, repo, path); f != nil {
|
||||
t.Fatalf("file still present after evict: %+v", f)
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
type ObjectsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewObjectsHandler(db *database.DB) *ObjectsHandler {
|
||||
return &ObjectsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Delete("/*", h.evict)
|
||||
return r
|
||||
}
|
||||
|
||||
// LocalRoutes lists and evicts objects for local repos, which live in the
|
||||
// local_files table rather than the artifacts table used by remotes.
|
||||
func (h *ObjectsHandler) LocalRoutes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.listLocal)
|
||||
r.Delete("/*", h.evictLocal)
|
||||
return r
|
||||
}
|
||||
|
||||
// pageBounds parses the shared page/per_page query params into a SQL limit and offset.
|
||||
func pageBounds(r *http.Request) (limit, offset int) {
|
||||
limit, _ = strconv.Atoi(r.URL.Query().Get("per_page"))
|
||||
if limit <= 0 || limit > 5000 {
|
||||
limit = 50
|
||||
}
|
||||
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
return limit, (page - 1) * limit
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
limit, offset := pageBounds(r)
|
||||
|
||||
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, artifacts)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) listLocal(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
limit, offset := pageBounds(r)
|
||||
|
||||
artifacts, err := h.db.ListLocalArtifacts(r.Context(), repoName, limit, offset)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, artifacts)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) evictLocal(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
if err := deleteLocalFile(r.Context(), h.db, repoName, path); err != nil {
|
||||
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
if err := h.db.DeleteArtifact(r.Context(), remoteName, path); err != nil {
|
||||
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
)
|
||||
|
||||
type ProbeHandler struct {
|
||||
engine *proxy.Engine
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewProbeHandler(engine *proxy.Engine, db *database.DB) *ProbeHandler {
|
||||
return &ProbeHandler{engine: engine, db: db}
|
||||
}
|
||||
|
||||
func (h *ProbeHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Post("/", h.probe)
|
||||
return r
|
||||
}
|
||||
|
||||
type probeRequest struct {
|
||||
Remote string `json:"remote"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
type probeResponse struct {
|
||||
Status int `json:"status"`
|
||||
Source string `json:"source,omitempty"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (h *ProbeHandler) probe(w http.ResponseWriter, r *http.Request) {
|
||||
var req probeRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if req.Remote == "" || req.Path == "" {
|
||||
http.Error(w, "remote and path are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), req.Remote)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 404,
|
||||
Error: fmt.Sprintf("remote %q not found", req.Remote),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 500,
|
||||
Error: fmt.Sprintf("no provider for %q", remote.PackageType),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
result, err := h.engine.Fetch(ctx, *remote, req.Path, prov)
|
||||
duration := time.Since(start).Milliseconds()
|
||||
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 502,
|
||||
DurationMS: duration,
|
||||
Error: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
io.Copy(io.Discard, result.Reader)
|
||||
result.Reader.Close()
|
||||
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 200,
|
||||
Source: result.Source,
|
||||
ContentType: result.ContentType,
|
||||
SizeBytes: result.Size,
|
||||
DurationMS: duration,
|
||||
Headers: map[string]string{
|
||||
"X-Artifact-Source": result.Source,
|
||||
"X-Artifact-Size": fmt.Sprintf("%d", result.Size),
|
||||
"Content-Type": result.ContentType,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type RemotesHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewRemotesHandler(db *database.DB) *RemotesHandler {
|
||||
return &RemotesHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Post("/", h.create)
|
||||
r.Get("/{name}", h.get)
|
||||
r.Put("/{name}", h.update)
|
||||
r.Delete("/{name}", h.del)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remotes, err := h.db.ListRemotes(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remotes)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
remote, err := h.db.GetRemote(r.Context(), name)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", name), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
var remote models.Remote
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !remote.PackageType.Valid() {
|
||||
http.Error(w, fmt.Sprintf("invalid package type: %q", remote.PackageType), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if remote.RepoType == "" {
|
||||
remote.RepoType = models.RepoTypeRemote
|
||||
}
|
||||
if !remote.RepoType.Valid() {
|
||||
http.Error(w, fmt.Sprintf("invalid repo type: %q", remote.RepoType), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if remote.RepoType == models.RepoTypeRemote && remote.BaseURL == "" {
|
||||
http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.CreateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
var remote models.Remote
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
remote.Name = name
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
if err := h.db.DeleteRemote(r.Context(), name); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
type StatsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewStatsHandler(db *database.DB) *StatsHandler {
|
||||
return &StatsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *StatsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.overview)
|
||||
r.Get("/top-remotes", h.topRemotes)
|
||||
r.Get("/top-files-by-hits", h.topFilesByHits)
|
||||
r.Get("/top-files-by-bandwidth", h.topFilesByBandwidth)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *StatsHandler) overview(w http.ResponseWriter, r *http.Request) {
|
||||
stats, err := h.db.GetOverviewStats(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, stats)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topRemotes(w http.ResponseWriter, r *http.Request) {
|
||||
remotes, err := h.db.GetTopRemotes(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remotes)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topFilesByHits(w http.ResponseWriter, r *http.Request) {
|
||||
files, err := h.db.GetTopFilesByHits(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, files)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topFilesByBandwidth(w http.ResponseWriter, r *http.Request) {
|
||||
files, err := h.db.GetTopFilesByBandwidth(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, files)
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type VirtualsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewVirtualsHandler(db *database.DB) *VirtualsHandler {
|
||||
return &VirtualsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Post("/", h.create)
|
||||
r.Get("/{name}", h.get)
|
||||
r.Put("/{name}", h.update)
|
||||
r.Delete("/{name}", h.del)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
virtuals, err := h.db.ListVirtuals(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virtuals)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
virt, err := h.db.GetVirtual(r.Context(), name)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("virtual %q not found", name), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
var virt models.Virtual
|
||||
if err := json.NewDecoder(r.Body).Decode(&virt); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.CreateVirtual(r.Context(), &virt); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
var virt models.Virtual
|
||||
if err := json.NewDecoder(r.Body).Decode(&virt); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
virt.Name = name
|
||||
if err := h.db.UpdateVirtual(r.Context(), &virt); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) del(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
if err := h.db.DeleteVirtual(r.Context(), name); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func BasicHeaders(remote models.Remote) http.Header {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString(
|
||||
[]byte(remote.Username+":"+remote.Password),
|
||||
))
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestBasicHeaders(t *testing.T) {
|
||||
h := BasicHeaders(models.Remote{Username: "alice", Password: "secret"})
|
||||
got := h.Get("Authorization")
|
||||
want := "Basic " + base64.StdEncoding.EncodeToString([]byte("alice:secret"))
|
||||
if got != want {
|
||||
t.Errorf("Authorization = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBasicHeadersNoUser(t *testing.T) {
|
||||
if h := BasicHeaders(models.Remote{}); h.Get("Authorization") != "" {
|
||||
t.Error("expected no Authorization header without a username")
|
||||
}
|
||||
}
|
||||
Vendored
-133
@@ -1,133 +0,0 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
var testRedis *Redis
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
url, terminate, err := testsupport.StartRedis(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
r, err := NewRedis(url)
|
||||
if err != nil {
|
||||
terminate()
|
||||
panic(err)
|
||||
}
|
||||
testRedis = r
|
||||
code := m.Run()
|
||||
r.Close()
|
||||
terminate()
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
func requireRedis(t *testing.T) {
|
||||
t.Helper()
|
||||
if testRedis == nil {
|
||||
t.Skip("Docker unavailable; skipping cache integration test")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRedisInvalid(t *testing.T) {
|
||||
if _, err := NewRedis("://bad-url"); err == nil {
|
||||
t.Error("expected error for invalid redis URL")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTTL(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
if fresh, _ := testRedis.CheckTTL(ctx, "r", "missing"); fresh {
|
||||
t.Error("missing key should not be fresh")
|
||||
}
|
||||
if err := testRedis.SetTTL(ctx, "r", "p", time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fresh, err := testRedis.CheckTTL(ctx, "r", "p"); err != nil || !fresh {
|
||||
t.Errorf("expected fresh after SetTTL: %v %v", fresh, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLock(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
ok, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("first acquire should succeed: %v %v", ok, err)
|
||||
}
|
||||
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); ok {
|
||||
t.Error("second acquire should fail while held")
|
||||
}
|
||||
if err := testRedis.ReleaseLock(ctx, "r", "lockpath"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); !ok {
|
||||
t.Error("acquire should succeed after release")
|
||||
}
|
||||
}
|
||||
|
||||
func TestETagAndToken(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
if v, _ := testRedis.GetETag(ctx, "r", "missing"); v != "" {
|
||||
t.Error("missing etag should be empty")
|
||||
}
|
||||
testRedis.SetETag(ctx, "r", "p", `"abc"`, time.Minute)
|
||||
if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` {
|
||||
t.Errorf("etag = %q", v)
|
||||
}
|
||||
|
||||
if v, _ := testRedis.GetToken(ctx, "missing"); v != "" {
|
||||
t.Error("missing token should be empty")
|
||||
}
|
||||
testRedis.SetToken(ctx, "key", "tok", time.Minute)
|
||||
if v, _ := testRedis.GetToken(ctx, "key"); v != "tok" {
|
||||
t.Errorf("token = %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCircuit(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 0 {
|
||||
t.Errorf("initial failures = %d", n)
|
||||
}
|
||||
n1, err := testRedis.IncrCircuitFailure(ctx, "cr", time.Minute)
|
||||
if err != nil || n1 != 1 {
|
||||
t.Fatalf("first incr = %d %v", n1, err)
|
||||
}
|
||||
n2, _ := testRedis.IncrCircuitFailure(ctx, "cr", time.Minute)
|
||||
if n2 != 2 {
|
||||
t.Errorf("second incr = %d", n2)
|
||||
}
|
||||
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 2 {
|
||||
t.Errorf("get failures = %d", n)
|
||||
}
|
||||
testRedis.ResetCircuit(ctx, "cr")
|
||||
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 0 {
|
||||
t.Errorf("failures after reset = %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushRemote(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
testRedis.SetTTL(ctx, "flushme", "a", time.Hour)
|
||||
testRedis.SetETag(ctx, "flushme", "a", "x", time.Hour)
|
||||
if err := testRedis.FlushRemote(ctx, "flushme"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fresh, _ := testRedis.CheckTTL(ctx, "flushme", "a"); fresh {
|
||||
t.Error("expected keys flushed")
|
||||
}
|
||||
}
|
||||
Vendored
-117
@@ -1,117 +0,0 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type Redis struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
func NewRedis(url string) (*Redis, error) {
|
||||
opts, err := redis.ParseURL(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse redis url: %w", err)
|
||||
}
|
||||
|
||||
client := redis.NewClient(opts)
|
||||
|
||||
if err := client.Ping(context.Background()).Err(); err != nil {
|
||||
return nil, fmt.Errorf("ping redis: %w", err)
|
||||
}
|
||||
|
||||
return &Redis{client: client}, nil
|
||||
}
|
||||
|
||||
func (r *Redis) Close() error {
|
||||
return r.client.Close()
|
||||
}
|
||||
|
||||
func (r *Redis) SetTTL(ctx context.Context, remote, path string, ttl time.Duration) error {
|
||||
key := fmt.Sprintf("ttl:%s:%s", remote, path)
|
||||
return r.client.Set(ctx, key, "1", ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) CheckTTL(ctx context.Context, remote, path string) (bool, error) {
|
||||
key := fmt.Sprintf("ttl:%s:%s", remote, path)
|
||||
exists, err := r.client.Exists(ctx, key).Result()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exists > 0, nil
|
||||
}
|
||||
|
||||
func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (bool, error) {
|
||||
key := fmt.Sprintf("lock:%s:%s", remote, path)
|
||||
ok, err := r.client.SetNX(ctx, key, "1", ttl).Result()
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error {
|
||||
key := fmt.Sprintf("lock:%s:%s", remote, path)
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error {
|
||||
key := fmt.Sprintf("etag:%s:%s", remote, path)
|
||||
return r.client.Set(ctx, key, etag, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) GetETag(ctx context.Context, remote, path string) (string, error) {
|
||||
key := fmt.Sprintf("etag:%s:%s", remote, path)
|
||||
val, err := r.client.Get(ctx, key).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) GetToken(ctx context.Context, key string) (string, error) {
|
||||
val, err := r.client.Get(ctx, "token:"+key).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) SetToken(ctx context.Context, key, token string, ttl time.Duration) error {
|
||||
return r.client.Set(ctx, "token:"+key, token, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) IncrCircuitFailure(ctx context.Context, remote string, cooldown time.Duration) (int64, error) {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
pipe := r.client.Pipeline()
|
||||
incr := pipe.Incr(ctx, key)
|
||||
pipe.Expire(ctx, key, cooldown)
|
||||
_, err := pipe.Exec(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return incr.Val(), nil
|
||||
}
|
||||
|
||||
func (r *Redis) ResetCircuit(ctx context.Context, remote string) error {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) GetCircuitFailures(ctx context.Context, remote string) (int64, error) {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
val, err := r.client.Get(ctx, key).Int64()
|
||||
if err == redis.Nil {
|
||||
return 0, nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) FlushRemote(ctx context.Context, remote string) error {
|
||||
iter := r.client.Scan(ctx, 0, fmt.Sprintf("*:%s:*", remote), 100).Iterator()
|
||||
for iter.Next(ctx) {
|
||||
r.client.Del(ctx, iter.Val())
|
||||
}
|
||||
return iter.Err()
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ListenAddr string
|
||||
|
||||
DBHost string
|
||||
DBPort int
|
||||
DBUser string
|
||||
DBPass string
|
||||
DBName string
|
||||
DBSSL string
|
||||
|
||||
RedisURL string
|
||||
|
||||
S3Endpoint string
|
||||
S3AccessKey string
|
||||
S3SecretKey string
|
||||
S3Bucket string
|
||||
S3Secure bool
|
||||
S3Region string
|
||||
|
||||
// Terraform provider registry signing. When TFSigningKeyPath points at a
|
||||
// readable armored GPG private key, artifactapi serves local terraform
|
||||
// repos as a real provider registry (service discovery + signed
|
||||
// SHA256SUMS). Left empty, the registry endpoints stay disabled.
|
||||
TFSigningKeyPath string
|
||||
TFSigningKeyPassphrase string
|
||||
TFProviderProtocols string
|
||||
}
|
||||
|
||||
func (c *Config) DatabaseDSN() string {
|
||||
return fmt.Sprintf(
|
||||
"postgres://%s:%s@%s:%d/%s?sslmode=%s",
|
||||
c.DBUser, c.DBPass, c.DBHost, c.DBPort, c.DBName, c.DBSSL,
|
||||
)
|
||||
}
|
||||
|
||||
func Load() (*Config, error) {
|
||||
dbPort, err := strconv.Atoi(getenv("DBPORT", "5432"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid DBPORT: %w", err)
|
||||
}
|
||||
|
||||
s3Secure, _ := strconv.ParseBool(getenv("MINIO_SECURE", "false"))
|
||||
|
||||
cfg := &Config{
|
||||
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
|
||||
|
||||
DBHost: getenv("DBHOST", "localhost"),
|
||||
DBPort: dbPort,
|
||||
DBUser: getenv("DBUSER", "artifacts"),
|
||||
DBPass: getenv("DBPASS", ""),
|
||||
DBName: getenv("DBNAME", "artifacts"),
|
||||
DBSSL: getenv("DBSSL", "disable"),
|
||||
|
||||
RedisURL: getenv("REDIS_URL", "redis://localhost:6379"),
|
||||
|
||||
S3Endpoint: getenv("MINIO_ENDPOINT", "localhost:9000"),
|
||||
S3AccessKey: getenv("MINIO_ACCESS_KEY", ""),
|
||||
S3SecretKey: getenv("MINIO_SECRET_KEY", ""),
|
||||
S3Bucket: getenv("MINIO_BUCKET", "artifacts"),
|
||||
S3Secure: s3Secure,
|
||||
S3Region: getenv("MINIO_REGION", ""),
|
||||
|
||||
TFSigningKeyPath: getenv("TF_SIGNING_KEY_PATH", ""),
|
||||
TFSigningKeyPassphrase: getenv("TF_SIGNING_KEY_PASSPHRASE", ""),
|
||||
TFProviderProtocols: getenv("TF_PROVIDER_PROTOCOLS", "5.0,6.0"),
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadDefaults(t *testing.T) {
|
||||
// Unset the vars Load reads so the fallback defaults are exercised.
|
||||
for _, k := range []string{
|
||||
"LISTEN_ADDR", "DBHOST", "DBPORT", "DBUSER", "DBPASS", "DBNAME", "DBSSL",
|
||||
"REDIS_URL", "MINIO_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY",
|
||||
"MINIO_BUCKET", "MINIO_SECURE", "MINIO_REGION",
|
||||
} {
|
||||
old, ok := os.LookupEnv(k)
|
||||
os.Unsetenv(k)
|
||||
if ok {
|
||||
t.Cleanup(func() { os.Setenv(k, old) })
|
||||
}
|
||||
}
|
||||
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if cfg.ListenAddr != ":8000" || cfg.DBPort != 5432 || cfg.DBUser != "artifacts" {
|
||||
t.Errorf("unexpected defaults: %+v", cfg)
|
||||
}
|
||||
if cfg.RedisURL != "redis://localhost:6379" || cfg.S3Bucket != "artifacts" || cfg.S3Secure {
|
||||
t.Errorf("unexpected defaults: %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadOverrides(t *testing.T) {
|
||||
t.Setenv("LISTEN_ADDR", ":9999")
|
||||
t.Setenv("DBHOST", "db.example.com")
|
||||
t.Setenv("DBPORT", "6000")
|
||||
t.Setenv("DBUSER", "u")
|
||||
t.Setenv("DBPASS", "pw")
|
||||
t.Setenv("DBNAME", "n")
|
||||
t.Setenv("DBSSL", "require")
|
||||
t.Setenv("MINIO_SECURE", "true")
|
||||
t.Setenv("MINIO_REGION", "us-east-1")
|
||||
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if cfg.ListenAddr != ":9999" || cfg.DBHost != "db.example.com" || cfg.DBPort != 6000 {
|
||||
t.Errorf("overrides not applied: %+v", cfg)
|
||||
}
|
||||
if !cfg.S3Secure {
|
||||
t.Error("MINIO_SECURE=true not parsed")
|
||||
}
|
||||
want := "postgres://u:pw@db.example.com:6000/n?sslmode=require"
|
||||
if got := cfg.DatabaseDSN(); got != want {
|
||||
t.Errorf("DSN = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadInvalidPort(t *testing.T) {
|
||||
t.Setenv("DBPORT", "not-a-number")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Error("expected error for invalid DBPORT")
|
||||
}
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) UpsertBlob(ctx context.Context, contentHash, s3Key string, sizeBytes int64, contentType string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO blobs (content_hash, s3_key, size_bytes, content_type)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (content_hash) DO NOTHING
|
||||
`, contentHash, s3Key, sizeBytes, contentType)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpsertArtifact(ctx context.Context, remoteName, path, contentHash, upstreamETag string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO artifacts (remote_name, path, content_hash, upstream_etag)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (remote_name, path) DO UPDATE SET
|
||||
content_hash = EXCLUDED.content_hash,
|
||||
upstream_etag = EXCLUDED.upstream_etag,
|
||||
last_fetched_at = NOW(),
|
||||
fetch_count = artifacts.fetch_count + 1
|
||||
`, remoteName, path, contentHash, upstreamETag)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) GetArtifact(ctx context.Context, remoteName, path string) (*models.Artifact, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||
a.last_accessed_at, a.fetch_count, a.access_count,
|
||||
b.size_bytes, b.content_type
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
WHERE a.remote_name = $1 AND a.path = $2
|
||||
`, remoteName, path)
|
||||
|
||||
var a models.Artifact
|
||||
err := row.Scan(
|
||||
&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &a.UpstreamETag,
|
||||
&a.UpstreamLastModified, &a.FirstSeenAt, &a.LastFetchedAt,
|
||||
&a.LastAccessedAt, &a.FetchCount, &a.AccessCount,
|
||||
&a.SizeBytes, &a.ContentType,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func (db *DB) TouchArtifactAccess(ctx context.Context, remoteName, path string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE artifacts SET
|
||||
last_accessed_at = NOW(),
|
||||
access_count = access_count + 1
|
||||
WHERE remote_name = $1 AND path = $2
|
||||
`, remoteName, path)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListArtifacts(ctx context.Context, remoteName string, limit, offset int) ([]models.Artifact, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||
a.last_accessed_at, a.fetch_count, a.access_count,
|
||||
b.size_bytes, b.content_type
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
WHERE a.remote_name = $1
|
||||
ORDER BY a.path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, remoteName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var artifacts []models.Artifact
|
||||
for rows.Next() {
|
||||
var a models.Artifact
|
||||
if err := rows.Scan(
|
||||
&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &a.UpstreamETag,
|
||||
&a.UpstreamLastModified, &a.FirstSeenAt, &a.LastFetchedAt,
|
||||
&a.LastAccessedAt, &a.FetchCount, &a.AccessCount,
|
||||
&a.SizeBytes, &a.ContentType,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
artifacts = append(artifacts, a)
|
||||
}
|
||||
return artifacts, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) DeleteArtifact(ctx context.Context, remoteName, path string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM artifacts WHERE remote_name = $1 AND path = $2`, remoteName, path)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cacheHit bool, sizeBytes int64, upstreamMS int, clientIP string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO access_log (remote_name, path, cache_hit, size_bytes, upstream_ms, client_ip)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
`, remoteName, path, cacheHit, sizeBytes, upstreamMS, clientIP)
|
||||
return err
|
||||
}
|
||||
|
||||
// AccessLogEntry is one buffered access-log record.
|
||||
type AccessLogEntry struct {
|
||||
RemoteName string
|
||||
Path string
|
||||
CacheHit bool
|
||||
SizeBytes int64
|
||||
UpstreamMS int
|
||||
ClientIP string
|
||||
}
|
||||
|
||||
// InsertAccessLogBatch bulk-inserts access-log rows with a single COPY.
|
||||
func (db *DB) InsertAccessLogBatch(ctx context.Context, entries []AccessLogEntry) error {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows := make([][]any, len(entries))
|
||||
for i, e := range entries {
|
||||
rows[i] = []any{e.RemoteName, e.Path, e.CacheHit, e.SizeBytes, e.UpstreamMS, e.ClientIP}
|
||||
}
|
||||
_, err := db.Pool.CopyFrom(ctx,
|
||||
pgx.Identifier{"access_log"},
|
||||
[]string{"remote_name", "path", "cache_hit", "size_bytes", "upstream_ms", "client_ip"},
|
||||
pgx.CopyFromRows(rows),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// FindOrphanedBlobs returns blobs no longer referenced by any artifact or
|
||||
// local file, restricted to those created before now()-minAge. The age cutoff
|
||||
// is a grace period that avoids a TOCTOU race with in-flight dedup uploads,
|
||||
// which insert the blob row before the referencing artifact/local_files row.
|
||||
func (db *DB) FindOrphanedBlobs(ctx context.Context, minAge time.Duration) ([]models.Blob, error) {
|
||||
cutoff := time.Now().Add(-minAge)
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT b.content_hash, b.s3_key, b.size_bytes, b.content_type, b.created_at
|
||||
FROM blobs b
|
||||
WHERE b.created_at < $1
|
||||
AND b.content_hash NOT IN (
|
||||
SELECT content_hash FROM artifacts
|
||||
UNION
|
||||
SELECT content_hash FROM local_files
|
||||
)
|
||||
`, cutoff)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var blobs []models.Blob
|
||||
for rows.Next() {
|
||||
var b models.Blob
|
||||
if err := rows.Scan(&b.ContentHash, &b.S3Key, &b.SizeBytes, &b.ContentType, &b.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blobs = append(blobs, b)
|
||||
}
|
||||
return blobs, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) DeleteBlob(ctx context.Context, contentHash string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM blobs WHERE content_hash = $1`, contentHash)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteColdArtifacts(ctx context.Context, remoteName string, olderThan time.Duration) (int64, error) {
|
||||
cutoff := time.Now().Add(-olderThan)
|
||||
tag, err := db.Pool.Exec(ctx, `
|
||||
DELETE FROM artifacts
|
||||
WHERE remote_name = $1 AND last_accessed_at < $2
|
||||
`, remoteName, cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
@@ -1,334 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
var (
|
||||
testDB *DB
|
||||
testDSN string
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
c := context.Background()
|
||||
dsn, terminate, err := testsupport.StartPostgres(c)
|
||||
if err != nil {
|
||||
// Docker unavailable: run anyway so tests self-skip via requireDB.
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN = dsn
|
||||
db, err := New(dsn)
|
||||
if err != nil {
|
||||
terminate()
|
||||
panic(err)
|
||||
}
|
||||
testDB = db
|
||||
|
||||
code := m.Run()
|
||||
|
||||
db.Close()
|
||||
terminate()
|
||||
// Return normally on success so the coverage profile is flushed; os.Exit
|
||||
// would truncate it.
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
func requireDB(t *testing.T) {
|
||||
t.Helper()
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable; skipping database integration test")
|
||||
}
|
||||
}
|
||||
|
||||
func ctx() context.Context { return context.Background() }
|
||||
|
||||
func seedRemote(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: name, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://example.com", MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed remote: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// seedBlob inserts a blob and returns its full content hash (sha256:<hash>),
|
||||
// matching the reference convention used by artifacts and local files.
|
||||
func seedBlob(t *testing.T, hash string) string {
|
||||
t.Helper()
|
||||
full := "sha256:" + hash
|
||||
if err := testDB.UpsertBlob(ctx(), full, "blobs/sha256/"+hash, 10, "application/octet-stream"); err != nil {
|
||||
t.Fatalf("seed blob: %v", err)
|
||||
}
|
||||
return full
|
||||
}
|
||||
|
||||
func TestRemotesCRUD(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-crud")
|
||||
got, err := testDB.GetRemote(ctx(), "r-crud")
|
||||
if err != nil || got.BaseURL != "https://example.com" {
|
||||
t.Fatalf("get: %v %v", got, err)
|
||||
}
|
||||
got.BaseURL = "https://updated.example.com"
|
||||
if err := testDB.UpdateRemote(ctx(), got); err != nil {
|
||||
t.Fatalf("update: %v", err)
|
||||
}
|
||||
got, _ = testDB.GetRemote(ctx(), "r-crud")
|
||||
if got.BaseURL != "https://updated.example.com" {
|
||||
t.Errorf("update not applied: %v", got.BaseURL)
|
||||
}
|
||||
list, err := testDB.ListRemotes(ctx())
|
||||
if err != nil || len(list) == 0 {
|
||||
t.Fatalf("list: %v %v", len(list), err)
|
||||
}
|
||||
if err := testDB.DeleteRemote(ctx(), "r-crud"); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetRemote(ctx(), "r-crud"); err == nil {
|
||||
t.Error("expected error after delete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactsAndBlobs(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-art")
|
||||
seedBlob(t, "aaaa")
|
||||
hash := "sha256:aaaa"
|
||||
if err := testDB.UpsertBlob(ctx(), hash, "blobs/sha256/aaaa", 10, "text/plain"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Upsert again to exercise the ON CONFLICT update branch.
|
||||
if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
art, err := testDB.GetArtifact(ctx(), "r-art", "path/a.txt")
|
||||
if err != nil || art.ContentHash != hash {
|
||||
t.Fatalf("get artifact: %v %v", art, err)
|
||||
}
|
||||
if err := testDB.TouchArtifactAccess(ctx(), "r-art", "path/a.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arts, err := testDB.ListArtifacts(ctx(), "r-art", 10, 0)
|
||||
if err != nil || len(arts) != 1 {
|
||||
t.Fatalf("list artifacts: %v %v", len(arts), err)
|
||||
}
|
||||
if err := testDB.InsertAccessLog(ctx(), "r-art", "path/a.txt", true, 10, 5, "1.2.3.4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.InsertAccessLogBatch(ctx(), []AccessLogEntry{
|
||||
{RemoteName: "r-art", Path: "b", CacheHit: false, SizeBytes: 20, UpstreamMS: 3},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.InsertAccessLogBatch(ctx(), nil); err != nil {
|
||||
t.Fatalf("empty batch should be a no-op: %v", err)
|
||||
}
|
||||
if err := testDB.DeleteArtifact(ctx(), "r-art", "path/a.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrphanAndColdCleanup(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedBlob(t, "orphanhash")
|
||||
// A blob with no artifact/local_file reference is orphaned, but only past
|
||||
// the grace period.
|
||||
if got, _ := testDB.FindOrphanedBlobs(ctx(), time.Hour); containsHash(got, "sha256:orphanhash") {
|
||||
t.Error("fresh orphan should be excluded by grace period")
|
||||
}
|
||||
orphans, err := testDB.FindOrphanedBlobs(ctx(), -time.Hour) // cutoff in the future => include fresh
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !containsHash(orphans, "sha256:orphanhash") {
|
||||
t.Error("expected orphan to be found with zero grace")
|
||||
}
|
||||
if err := testDB.DeleteBlob(ctx(), "sha256:orphanhash"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
seedRemote(t, "r-cold")
|
||||
seedBlob(t, "coldhash")
|
||||
testDB.UpsertArtifact(ctx(), "r-cold", "cold.txt", "sha256:coldhash", "")
|
||||
n, err := testDB.DeleteColdArtifacts(ctx(), "r-cold", -time.Hour) // negative => everything is "cold"
|
||||
if err != nil || n < 1 {
|
||||
t.Fatalf("delete cold: n=%d err=%v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func containsHash(blobs []models.Blob, hash string) bool {
|
||||
for _, b := range blobs {
|
||||
if b.ContentHash == hash {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestLocalFiles(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-local")
|
||||
seedBlob(t, "localhash")
|
||||
hash := "sha256:localhash"
|
||||
if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Duplicate create must be rejected.
|
||||
if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err == nil {
|
||||
t.Error("expected duplicate local file error")
|
||||
}
|
||||
f, err := testDB.GetLocalFile(ctx(), "r-local", "foo/foo-1.0.whl")
|
||||
if err != nil || f == nil {
|
||||
t.Fatalf("get local file: %v %v", f, err)
|
||||
}
|
||||
if files, err := testDB.ListLocalFiles(ctx(), "r-local", 10, 0); err != nil || len(files) != 1 {
|
||||
t.Fatalf("list: %v %v", len(files), err)
|
||||
}
|
||||
if files, err := testDB.ListLocalFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(files) != 1 {
|
||||
t.Fatalf("list by prefix: %v %v", len(files), err)
|
||||
}
|
||||
if entries, err := testDB.ListFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(entries) != 1 {
|
||||
t.Fatalf("provider list by prefix: %v %v", len(entries), err)
|
||||
}
|
||||
if pkgs, err := testDB.ListLocalFilePackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 {
|
||||
t.Fatalf("list packages: %v %v", pkgs, err)
|
||||
}
|
||||
if pkgs, err := testDB.ListPackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 {
|
||||
t.Fatalf("provider list packages: %v %v", pkgs, err)
|
||||
}
|
||||
if err := testDB.DeleteLocalFile(ctx(), "r-local", "foo/foo-1.0.whl"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualsCRUD(t *testing.T) {
|
||||
requireDB(t)
|
||||
if err := testDB.CreateVirtual(ctx(), &models.Virtual{
|
||||
Name: "v-crud", PackageType: models.PackageHelm, Members: []string{"a", "b"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := testDB.GetVirtual(ctx(), "v-crud")
|
||||
if err != nil || len(v.Members) != 2 {
|
||||
t.Fatalf("get virtual: %v %v", v, err)
|
||||
}
|
||||
v.Members = []string{"a"}
|
||||
if err := testDB.UpdateVirtual(ctx(), v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if vs, err := testDB.ListVirtuals(ctx()); err != nil || len(vs) == 0 {
|
||||
t.Fatalf("list virtuals: %v %v", len(vs), err)
|
||||
}
|
||||
if err := testDB.DeleteVirtual(ctx(), "v-crud"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStats(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-stats")
|
||||
seedBlob(t, "statshash")
|
||||
testDB.UpsertArtifact(ctx(), "r-stats", "s.txt", "sha256:statshash", "")
|
||||
testDB.InsertAccessLog(ctx(), "r-stats", "s.txt", true, 100, 2, "")
|
||||
|
||||
if _, err := testDB.GetOverviewStats(ctx()); err != nil {
|
||||
t.Fatalf("overview: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetTopRemotes(ctx(), 5); err != nil {
|
||||
t.Fatalf("top remotes: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetTopFilesByHits(ctx(), 5); err != nil {
|
||||
t.Fatalf("top files by hits: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetTopFilesByBandwidth(ctx(), 5); err != nil {
|
||||
t.Fatalf("top files by bandwidth: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatabaseErrorPaths(t *testing.T) {
|
||||
requireDB(t)
|
||||
bad, err := New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bad.Close() // every query now fails
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := bad.ListRemotes(ctx); err == nil {
|
||||
t.Error("ListRemotes should error on closed db")
|
||||
}
|
||||
if _, err := bad.ListVirtuals(ctx); err == nil {
|
||||
t.Error("ListVirtuals should error")
|
||||
}
|
||||
if _, err := bad.ListArtifacts(ctx, "r", 10, 0); err == nil {
|
||||
t.Error("ListArtifacts should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFiles(ctx, "r", 10, 0); err == nil {
|
||||
t.Error("ListLocalFiles should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFilesByPrefix(ctx, "r", "p"); err == nil {
|
||||
t.Error("ListLocalFilesByPrefix should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFilePackages(ctx, "r"); err == nil {
|
||||
t.Error("ListLocalFilePackages should error")
|
||||
}
|
||||
if _, err := bad.ListFilesByPrefix(ctx, "r", "p"); err == nil {
|
||||
t.Error("ListFilesByPrefix should error")
|
||||
}
|
||||
if _, err := bad.ListPackages(ctx, "r"); err == nil {
|
||||
t.Error("ListPackages should error")
|
||||
}
|
||||
if _, err := bad.FindOrphanedBlobs(ctx, 0); err == nil {
|
||||
t.Error("FindOrphanedBlobs should error")
|
||||
}
|
||||
if _, err := bad.GetOverviewStats(ctx); err == nil {
|
||||
t.Error("GetOverviewStats should error")
|
||||
}
|
||||
if _, err := bad.GetTopRemotes(ctx, 5); err == nil {
|
||||
t.Error("GetTopRemotes should error")
|
||||
}
|
||||
if _, err := bad.GetTopFilesByHits(ctx, 5); err == nil {
|
||||
t.Error("GetTopFilesByHits should error")
|
||||
}
|
||||
if _, err := bad.GetTopFilesByBandwidth(ctx, 5); err == nil {
|
||||
t.Error("GetTopFilesByBandwidth should error")
|
||||
}
|
||||
if _, err := bad.ListRPMMetadataEntries(ctx, "r"); err == nil {
|
||||
t.Error("ListRPMMetadataEntries should error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMMetadata(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-rpm")
|
||||
meta := &provider.RPMMetadata{
|
||||
RepoName: "r-rpm", FilePath: "Packages/x.rpm", ContentHash: "sha256:rpm",
|
||||
Name: "x", Version: "1.0", Release: "1", Arch: "noarch",
|
||||
Requires: []provider.RPMDep{{Name: "libc"}},
|
||||
Provides: []provider.RPMDep{{Name: "x"}},
|
||||
Files: []provider.RPMFile{},
|
||||
}
|
||||
if err := testDB.InsertRPMMetadata(ctx(), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, err := testDB.ListRPMMetadataEntries(ctx(), "r-rpm")
|
||||
if err != nil || len(entries) != 1 {
|
||||
t.Fatalf("list rpm entries: %v %v", len(entries), err)
|
||||
}
|
||||
if rows, err := testDB.ListRPMMetadata(ctx(), "r-rpm"); err != nil || len(rows) != 1 {
|
||||
t.Fatalf("list rpm rows: %v %v", len(rows), err)
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type LocalFile struct {
|
||||
ID int64 `json:"id"`
|
||||
RepoName string `json:"repo_name"`
|
||||
FilePath string `json:"file_path"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
var ErrAlreadyExists = fmt.Errorf("file already exists")
|
||||
|
||||
func (db *DB) CreateLocalFile(ctx context.Context, repoName, filePath, contentHash string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO local_files (repo_name, file_path, content_hash)
|
||||
VALUES ($1, $2, $3)
|
||||
`, repoName, filePath, contentHash)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return ErrAlreadyExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) GetLocalFile(ctx context.Context, repoName, filePath string) (*LocalFile, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1 AND file_path = $2
|
||||
`, repoName, filePath)
|
||||
|
||||
var f LocalFile
|
||||
if err := row.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &f, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFiles(ctx context.Context, repoName string, limit, offset int) ([]LocalFile, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1
|
||||
ORDER BY file_path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, repoName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var files []LocalFile
|
||||
for rows.Next() {
|
||||
var f LocalFile
|
||||
if err := rows.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, f)
|
||||
}
|
||||
return files, rows.Err()
|
||||
}
|
||||
|
||||
// ListLocalArtifacts returns a repo's local files shaped as models.Artifact so
|
||||
// the UI's cached-objects view can render them the same way as remote artifacts.
|
||||
// Local files carry no access/fetch counters, so those are left at zero and the
|
||||
// timestamps are all derived from created_at.
|
||||
func (db *DB) ListLocalArtifacts(ctx context.Context, repoName string, limit, offset int) ([]models.Artifact, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT lf.id, lf.repo_name, lf.file_path, lf.content_hash,
|
||||
lf.created_at, b.size_bytes, b.content_type
|
||||
FROM local_files lf
|
||||
JOIN blobs b ON lf.content_hash = b.content_hash
|
||||
WHERE lf.repo_name = $1
|
||||
ORDER BY lf.file_path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, repoName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var artifacts []models.Artifact
|
||||
for rows.Next() {
|
||||
var a models.Artifact
|
||||
var createdAt time.Time
|
||||
if err := rows.Scan(&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &createdAt, &a.SizeBytes, &a.ContentType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.FirstSeenAt = createdAt
|
||||
a.LastFetchedAt = createdAt
|
||||
a.LastAccessedAt = createdAt
|
||||
artifacts = append(artifacts, a)
|
||||
}
|
||||
return artifacts, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFilesByPrefix(ctx context.Context, repoName, prefix string) ([]LocalFile, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1 AND file_path LIKE $2
|
||||
ORDER BY file_path
|
||||
`, repoName, prefix+"%")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var files []LocalFile
|
||||
for rows.Next() {
|
||||
var f LocalFile
|
||||
if err := rows.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, f)
|
||||
}
|
||||
return files, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFilePackages(ctx context.Context, repoName string) ([]string, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT DISTINCT split_part(file_path, '/', 1)
|
||||
FROM local_files
|
||||
WHERE repo_name = $1
|
||||
ORDER BY 1
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var packages []string
|
||||
for rows.Next() {
|
||||
var pkg string
|
||||
if err := rows.Scan(&pkg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
return packages, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]provider.FileEntry, error) {
|
||||
files, err := db.ListLocalFilesByPrefix(ctx, repoName, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]provider.FileEntry, len(files))
|
||||
for i, f := range files {
|
||||
result[i] = provider.FileEntry{FilePath: f.FilePath, ContentHash: f.ContentHash}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListPackages(ctx context.Context, repoName string) ([]string, error) {
|
||||
return db.ListLocalFilePackages(ctx, repoName)
|
||||
}
|
||||
|
||||
func (db *DB) DeleteLocalFile(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM local_files WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type DB struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func New(dsn string) (*DB, error) {
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect to postgres: %w", err)
|
||||
}
|
||||
|
||||
if err := pool.Ping(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("ping postgres: %w", err)
|
||||
}
|
||||
|
||||
db := &DB{Pool: pool}
|
||||
if err := db.migrate(); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("run migrations: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (db *DB) Close() {
|
||||
db.Pool.Close()
|
||||
}
|
||||
|
||||
func (db *DB) migrate() error {
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS remotes (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
repo_type TEXT DEFAULT 'remote',
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
immutable_ttl INTEGER DEFAULT 0,
|
||||
mutable_ttl INTEGER DEFAULT 3600,
|
||||
check_mutable BOOLEAN DEFAULT TRUE,
|
||||
patterns TEXT[] DEFAULT '{}',
|
||||
blocklist TEXT[] DEFAULT '{}',
|
||||
mutable_patterns TEXT[] DEFAULT '{}',
|
||||
immutable_patterns TEXT[] DEFAULT '{}',
|
||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
||||
ban_tags TEXT[] DEFAULT '{}',
|
||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
||||
quarantine_days INTEGER DEFAULT 3,
|
||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
||||
releases_remote TEXT DEFAULT '',
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS virtuals (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
members TEXT[] NOT NULL,
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS blobs (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
s3_key TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
content_type TEXT DEFAULT 'application/octet-stream',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artifacts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
upstream_etag TEXT DEFAULT '',
|
||||
upstream_last_modified TIMESTAMPTZ,
|
||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
fetch_count BIGINT DEFAULT 1,
|
||||
access_count BIGINT DEFAULT 1,
|
||||
UNIQUE(remote_name, path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS local_files (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS access_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
cache_hit BOOLEAN NOT NULL,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
upstream_ms INTEGER DEFAULT 0,
|
||||
client_ip TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rpm_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
epoch INTEGER DEFAULT 0,
|
||||
version TEXT NOT NULL,
|
||||
release TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
summary TEXT DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
rpm_size BIGINT DEFAULT 0,
|
||||
installed_size BIGINT DEFAULT 0,
|
||||
license TEXT DEFAULT '',
|
||||
vendor TEXT DEFAULT '',
|
||||
build_group TEXT DEFAULT '',
|
||||
build_host TEXT DEFAULT '',
|
||||
source_rpm TEXT DEFAULT '',
|
||||
url TEXT DEFAULT '',
|
||||
packager TEXT DEFAULT '',
|
||||
requires JSONB DEFAULT '[]',
|
||||
provides JSONB DEFAULT '[]',
|
||||
files JSONB DEFAULT '[]',
|
||||
changelogs JSONB DEFAULT '[]',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signing_keys (
|
||||
purpose TEXT PRIMARY KEY,
|
||||
private_key_armor TEXT NOT NULL,
|
||||
key_id TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
`)
|
||||
return err
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const remoteCols = `name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
created_at, updated_at`
|
||||
|
||||
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
|
||||
return scanner.Scan(
|
||||
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Description, &r.Username, &r.Password,
|
||||
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
|
||||
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
|
||||
&r.BanTagsEnabled, &r.BanTags,
|
||||
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
|
||||
&r.ReleasesRemote, &r.ManagedBy,
|
||||
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
|
||||
&r.CreatedAt, &r.UpdatedAt,
|
||||
)
|
||||
}
|
||||
|
||||
func (db *DB) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
|
||||
row := db.Pool.QueryRow(ctx, `SELECT `+remoteCols+` FROM remotes WHERE name = $1`, name)
|
||||
var r models.Remote
|
||||
if err := scanRemote(row, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes ORDER BY name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var remotes []models.Remote
|
||||
for rows.Next() {
|
||||
var r models.Remote
|
||||
if err := scanRemote(rows, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remotes = append(remotes, r)
|
||||
}
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO remotes (
|
||||
name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24)
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE remotes SET
|
||||
package_type=$2, repo_type=$3, base_url=$4, description=$5, username=$6, password=$7,
|
||||
immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
|
||||
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
|
||||
ban_tags_enabled=$15, ban_tags=$16,
|
||||
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
|
||||
releases_remote=$20, managed_by=$21,
|
||||
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
|
||||
updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteRemote(ctx context.Context, name string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM remotes WHERE name = $1`, name)
|
||||
return err
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
|
||||
requiresJSON, _ := json.Marshal(meta.Requires)
|
||||
providesJSON, _ := json.Marshal(meta.Provides)
|
||||
filesJSON, _ := json.Marshal(meta.Files)
|
||||
changelogsJSON, _ := json.Marshal(meta.Changelogs)
|
||||
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO rpm_metadata (
|
||||
repo_name, file_path, content_hash,
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, files, changelogs
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash,
|
||||
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
|
||||
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
|
||||
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
|
||||
requiresJSON, providesJSON, filesJSON, changelogsJSON,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM rpm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
type RPMMetadataRow struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Epoch int
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
Summary string
|
||||
Description string
|
||||
RPMSize int64
|
||||
InstalledSize int64
|
||||
License string
|
||||
Vendor string
|
||||
Group string
|
||||
BuildHost string
|
||||
SourceRPM string
|
||||
URL string
|
||||
Packager string
|
||||
Requires json.RawMessage
|
||||
Provides json.RawMessage
|
||||
Files json.RawMessage
|
||||
Changelogs json.RawMessage
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]provider.RPMMetadata, error) {
|
||||
rows, err := db.ListRPMMetadata(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]provider.RPMMetadata, len(rows))
|
||||
for i, r := range rows {
|
||||
meta := provider.RPMMetadata{
|
||||
RepoName: r.RepoName,
|
||||
FilePath: r.FilePath,
|
||||
ContentHash: r.ContentHash,
|
||||
Name: r.Name,
|
||||
Epoch: r.Epoch,
|
||||
Version: r.Version,
|
||||
Release: r.Release,
|
||||
Arch: r.Arch,
|
||||
Summary: r.Summary,
|
||||
Description: r.Description,
|
||||
RPMSize: r.RPMSize,
|
||||
InstalledSize: r.InstalledSize,
|
||||
License: r.License,
|
||||
Vendor: r.Vendor,
|
||||
Group: r.Group,
|
||||
BuildHost: r.BuildHost,
|
||||
SourceRPM: r.SourceRPM,
|
||||
URL: r.URL,
|
||||
Packager: r.Packager,
|
||||
}
|
||||
json.Unmarshal(r.Requires, &meta.Requires)
|
||||
json.Unmarshal(r.Provides, &meta.Provides)
|
||||
json.Unmarshal(r.Files, &meta.Files)
|
||||
json.Unmarshal(r.Changelogs, &meta.Changelogs)
|
||||
result[i] = meta
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetadataRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash,
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, files, changelogs
|
||||
FROM rpm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, epoch, version, release, arch
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []RPMMetadataRow
|
||||
for rows.Next() {
|
||||
var r RPMMetadataRow
|
||||
if err := rows.Scan(
|
||||
&r.RepoName, &r.FilePath, &r.ContentHash,
|
||||
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
|
||||
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
|
||||
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
|
||||
&r.Requires, &r.Provides, &r.Files, &r.Changelogs,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// GetSigningKey returns the stored armored private key and key id for a purpose.
|
||||
// found is false when no key has been generated yet.
|
||||
func (db *DB) GetSigningKey(ctx context.Context, purpose string) (armor, keyID string, found bool, err error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT private_key_armor, key_id FROM signing_keys WHERE purpose = $1
|
||||
`, purpose)
|
||||
if err := row.Scan(&armor, &keyID); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", "", false, nil
|
||||
}
|
||||
return "", "", false, err
|
||||
}
|
||||
return armor, keyID, true, nil
|
||||
}
|
||||
|
||||
// InsertSigningKeyIfAbsent stores a freshly generated key, doing nothing if
|
||||
// another replica already inserted one. Callers re-read with GetSigningKey to
|
||||
// pick up whichever key won the race.
|
||||
func (db *DB) InsertSigningKeyIfAbsent(ctx context.Context, purpose, armor, keyID string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO signing_keys (purpose, private_key_armor, key_id)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (purpose) DO NOTHING
|
||||
`, purpose, armor, keyID)
|
||||
return err
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package database
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSigningKeyRoundTripAndIdempotency(t *testing.T) {
|
||||
requireDB(t)
|
||||
|
||||
const purpose = "terraform-provider-test"
|
||||
|
||||
// Absent to start.
|
||||
if _, _, found, err := testDB.GetSigningKey(ctx(), purpose); err != nil || found {
|
||||
t.Fatalf("expected no key, got found=%v err=%v", found, err)
|
||||
}
|
||||
|
||||
if err := testDB.InsertSigningKeyIfAbsent(ctx(), purpose, "ARMOR-1", "KEYID1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A second insert must not overwrite (models the replica race).
|
||||
if err := testDB.InsertSigningKeyIfAbsent(ctx(), purpose, "ARMOR-2", "KEYID2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
armor, keyID, found, err := testDB.GetSigningKey(ctx(), purpose)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("expected key, found=%v err=%v", found, err)
|
||||
}
|
||||
if armor != "ARMOR-1" || keyID != "KEYID1" {
|
||||
t.Errorf("key was overwritten: armor=%q key_id=%q", armor, keyID)
|
||||
}
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) GetOverviewStats(ctx context.Context) (*models.OverviewStats, error) {
|
||||
var stats models.OverviewStats
|
||||
|
||||
err := db.Pool.QueryRow(ctx, `SELECT COUNT(*) FROM remotes`).Scan(&stats.TotalRemotes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `SELECT COALESCE(COUNT(*), 0), COALESCE(SUM(b.size_bytes), 0)
|
||||
FROM artifacts a JOIN blobs b ON a.content_hash = b.content_hash`).
|
||||
Scan(&stats.TotalObjects, &stats.TotalBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(
|
||||
(SELECT COUNT(*) FROM artifacts) - (SELECT COUNT(DISTINCT content_hash) FROM artifacts),
|
||||
0
|
||||
)`).Scan(&stats.TotalBlobsDeduped)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(size_bytes), 0)
|
||||
FROM access_log
|
||||
WHERE cache_hit = TRUE AND created_at > NOW() - INTERVAL '30 days'
|
||||
`).Scan(&stats.BandwidthSaved30d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
type RemoteStatRow struct {
|
||||
Name string `json:"name"`
|
||||
ObjectCount int64 `json:"object_count"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
Requests30d int64 `json:"requests_30d"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopRemotes(ctx context.Context, limit int) ([]RemoteStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT r.name,
|
||||
COALESCE(a.cnt, 0) AS object_count,
|
||||
COALESCE(a.total_bytes, 0) AS total_bytes,
|
||||
COALESCE(l.req_count, 0) AS requests_30d
|
||||
FROM remotes r
|
||||
LEFT JOIN (
|
||||
SELECT remote_name, COUNT(*) AS cnt, SUM(b.size_bytes) AS total_bytes
|
||||
FROM artifacts a JOIN blobs b ON a.content_hash = b.content_hash
|
||||
GROUP BY remote_name
|
||||
) a ON r.name = a.remote_name
|
||||
LEFT JOIN (
|
||||
SELECT remote_name, COUNT(*) AS req_count
|
||||
FROM access_log
|
||||
WHERE created_at > NOW() - INTERVAL '30 days'
|
||||
GROUP BY remote_name
|
||||
) l ON r.name = l.remote_name
|
||||
ORDER BY COALESCE(a.total_bytes, 0) DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []RemoteStatRow
|
||||
for rows.Next() {
|
||||
var r RemoteStatRow
|
||||
if err := rows.Scan(&r.Name, &r.ObjectCount, &r.TotalBytes, &r.Requests30d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
type FileStatRow struct {
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
AccessCount int64 `json:"access_count"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopFilesByHits(ctx context.Context, limit int) ([]FileStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT a.remote_name, a.path, a.access_count, b.size_bytes
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
ORDER BY a.access_count DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []FileStatRow
|
||||
for rows.Next() {
|
||||
var r FileStatRow
|
||||
if err := rows.Scan(&r.RemoteName, &r.Path, &r.AccessCount, &r.SizeBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
type BandwidthStatRow struct {
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
Bandwidth int64 `json:"bandwidth"`
|
||||
Requests int64 `json:"requests"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopFilesByBandwidth(ctx context.Context, limit int) ([]BandwidthStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT remote_name, path,
|
||||
COALESCE(SUM(size_bytes), 0) AS bandwidth,
|
||||
COUNT(*) AS requests
|
||||
FROM access_log
|
||||
WHERE created_at > NOW() - INTERVAL '30 days'
|
||||
GROUP BY remote_name, path
|
||||
ORDER BY bandwidth DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []BandwidthStatRow
|
||||
for rows.Next() {
|
||||
var r BandwidthStatRow
|
||||
if err := rows.Scan(&r.RemoteName, &r.Path, &r.Bandwidth, &r.Requests); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) GetVirtual(ctx context.Context, name string) (*models.Virtual, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT name, package_type, description, members, managed_by, created_at, updated_at
|
||||
FROM virtuals WHERE name = $1
|
||||
`, name)
|
||||
|
||||
var v models.Virtual
|
||||
err := row.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListVirtuals(ctx context.Context) ([]models.Virtual, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT name, package_type, description, members, managed_by, created_at, updated_at
|
||||
FROM virtuals ORDER BY name
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var virtuals []models.Virtual
|
||||
for rows.Next() {
|
||||
var v models.Virtual
|
||||
if err := rows.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
virtuals = append(virtuals, v)
|
||||
}
|
||||
return virtuals, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) CreateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO virtuals (name, package_type, description, members, managed_by)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpdateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE virtuals SET
|
||||
package_type=$2, description=$3, members=$4, managed_by=$5, updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteVirtual(ctx context.Context, name string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM virtuals WHERE name = $1`, name)
|
||||
return err
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
)
|
||||
|
||||
// blobGracePeriod is how old an orphaned blob must be before GC will delete
|
||||
// it. This avoids racing in-flight dedup uploads that insert the blob row
|
||||
// before the referencing artifact/local_files row exists.
|
||||
const blobGracePeriod = 1 * time.Hour
|
||||
|
||||
type Collector struct {
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func New(db *database.DB, store *storage.S3, interval time.Duration) *Collector {
|
||||
return &Collector{db: db, store: store, interval: interval}
|
||||
}
|
||||
|
||||
func (c *Collector) Run(ctx context.Context) {
|
||||
slog.Info("gc started", "interval", c.interval)
|
||||
ticker := time.NewTicker(c.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("gc stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
c.sweep(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Collector) sweep(ctx context.Context) {
|
||||
start := time.Now()
|
||||
|
||||
orphaned, err := c.db.FindOrphanedBlobs(ctx, blobGracePeriod)
|
||||
if err != nil {
|
||||
slog.Error("gc: find orphaned blobs", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for _, blob := range orphaned {
|
||||
if err := c.store.Delete(ctx, blob.S3Key); err != nil {
|
||||
slog.Warn("gc: delete s3 object", "key", blob.S3Key, "error", err)
|
||||
continue
|
||||
}
|
||||
if err := c.db.DeleteBlob(ctx, blob.ContentHash); err != nil {
|
||||
slog.Warn("gc: delete blob row", "hash", blob.ContentHash, "error", err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
|
||||
if deleted > 0 || len(orphaned) > 0 {
|
||||
slog.Info("gc sweep complete",
|
||||
"orphaned_found", len(orphaned),
|
||||
"deleted", deleted,
|
||||
"duration_ms", time.Since(start).Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
var (
|
||||
testDB *database.DB
|
||||
testStore *storage.S3
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, termPG, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
minio, termMinio, err := testsupport.StartMinio(ctx)
|
||||
if err != nil {
|
||||
termPG()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
db, err := database.New(dsn)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var s3 *storage.S3
|
||||
for i := 0; i < 20; i++ {
|
||||
if s3, err = storage.NewS3(minio.Endpoint, minio.AccessKey, minio.SecretKey, "gc-test", false, ""); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
testDB = db
|
||||
testStore = s3
|
||||
|
||||
code := m.Run()
|
||||
db.Close()
|
||||
termMinio()
|
||||
termPG()
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepDeletesOldOrphan(t *testing.T) {
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
hash := "sha256:gcorphan"
|
||||
key := storage.BlobKey("gcorphan")
|
||||
|
||||
if err := testStore.Upload(ctx, key, bytes.NewReader([]byte("orphan")), 6, "application/octet-stream"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.UpsertBlob(ctx, hash, key, 6, "application/octet-stream"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Age the blob past the grace period.
|
||||
if _, err := testDB.Pool.Exec(ctx, `UPDATE blobs SET created_at = now() - interval '2 hours' WHERE content_hash = $1`, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c := New(testDB, testStore, time.Hour)
|
||||
c.sweep(ctx)
|
||||
|
||||
if exists, _ := testStore.Exists(ctx, key); exists {
|
||||
t.Error("expected orphan object deleted from store")
|
||||
}
|
||||
orphans, _ := testDB.FindOrphanedBlobs(ctx, 0)
|
||||
for _, b := range orphans {
|
||||
if b.ContentHash == hash {
|
||||
t.Error("expected orphan blob row deleted")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepNoOrphans(t *testing.T) {
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
// A sweep with nothing to collect should be a clean no-op.
|
||||
New(testDB, testStore, time.Hour).sweep(context.Background())
|
||||
}
|
||||
|
||||
func TestRunStopsOnContextCancel(t *testing.T) {
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
New(testDB, testStore, time.Hour).Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Run did not return after context cancel")
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package gc_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/gc"
|
||||
)
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
c := gc.New(nil, nil, 1*time.Hour)
|
||||
if c == nil {
|
||||
t.Fatal("expected non-nil collector")
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageAlpine }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "APKINDEX.tar.gz") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".apk") {
|
||||
return "application/vnd.android.package-archive"
|
||||
}
|
||||
if strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestType(t *testing.T) {
|
||||
if (&Provider{}).Type() != models.PackageAlpine {
|
||||
t.Fatal("wrong type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("v3.19/main/x86_64/APKINDEX.tar.gz") != provider.Mutable {
|
||||
t.Error("APKINDEX should be mutable")
|
||||
}
|
||||
if p.Classify("v3.19/main/x86_64/curl-8.0-r0.apk") != provider.Immutable {
|
||||
t.Error("apk should be immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
cases := map[string]string{
|
||||
"pkg.apk": "application/vnd.android.package-archive",
|
||||
"APKINDEX.tar.gz": "application/gzip",
|
||||
"something.random": "application/octet-stream",
|
||||
}
|
||||
for path, want := range cases {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
p := &Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://dl-cdn.alpinelinux.org/alpine/"}, "/v3.19/main/x86_64/curl.apk")
|
||||
if got != "https://dl-cdn.alpinelinux.org/alpine/v3.19/main/x86_64/curl.apk" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
if out, err := (&Provider{}).RewriteResponse([]byte("x"), models.Remote{}, "http://proxy"); out != nil || err != nil {
|
||||
t.Error("alpine never rewrites")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthHeaders(t *testing.T) {
|
||||
h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("expected auth header")
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var (
|
||||
tagManifestRe = regexp.MustCompile(`/manifests/[^/]+$`)
|
||||
digestManifestRe = regexp.MustCompile(`/manifests/sha256:[0-9a-fA-F]+$`)
|
||||
tagsListRe = regexp.MustCompile(`/tags/list$`)
|
||||
)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageDocker }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if tagsListRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
if tagManifestRe.MatchString(path) && !digestManifestRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.Contains(path, "/blobs/") {
|
||||
return "application/octet-stream"
|
||||
}
|
||||
if strings.Contains(path, "/manifests/") {
|
||||
return "application/vnd.docker.distribution.manifest.v2+json"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/v2/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" && remote.Password != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestDockerClassifyBranches(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("library/nginx/tags/list") != provider.Mutable {
|
||||
t.Error("tags/list should be mutable")
|
||||
}
|
||||
if p.Classify("library/nginx/manifests/latest") != provider.Mutable {
|
||||
t.Error("tag manifest should be mutable")
|
||||
}
|
||||
if p.Classify("library/nginx/manifests/sha256:abcdef") != provider.Immutable {
|
||||
t.Error("digest manifest should be immutable")
|
||||
}
|
||||
if p.Classify("library/nginx/blobs/sha256:abc") != provider.Immutable {
|
||||
t.Error("blob should be immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.ContentType("x/blobs/sha256:abc") != "application/octet-stream" {
|
||||
t.Error("blob content type")
|
||||
}
|
||||
if p.ContentType("x/manifests/latest") != "application/vnd.docker.distribution.manifest.v2+json" {
|
||||
t.Error("manifest content type")
|
||||
}
|
||||
if p.ContentType("x/tags/list") != "application/json" {
|
||||
t.Error("default content type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerRewriteAndAuth(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if out, err := p.RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
|
||||
t.Error("docker never rewrites")
|
||||
}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("expected basic auth header")
|
||||
}
|
||||
h, _ = p.AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("no creds, no header")
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package docker_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
if p.Type() != models.PackageDocker {
|
||||
t.Errorf("expected docker, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"library/nginx/manifests/latest", provider.Mutable},
|
||||
{"library/nginx/manifests/v1.25", provider.Mutable},
|
||||
{"library/nginx/manifests/sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", provider.Immutable},
|
||||
{"library/nginx/tags/list", provider.Mutable},
|
||||
{"library/nginx/blobs/sha256:abc123", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_UpstreamURL(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://registry-1.docker.io"}, "library/nginx/manifests/latest")
|
||||
want := "https://registry-1.docker.io/v2/library/nginx/manifests/latest"
|
||||
if got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
if p.ContentType("x/blobs/sha256:abc") != "application/octet-stream" {
|
||||
t.Error("blobs should be octet-stream")
|
||||
}
|
||||
if p.ContentType("x/manifests/latest") != "application/vnd.docker.distribution.manifest.v2+json" {
|
||||
t.Error("manifests should be manifest type")
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package generic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageGeneric }
|
||||
|
||||
func (p *Provider) Classify(_ string) provider.Mutability {
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
var contentTypeMap = map[string]string{
|
||||
".tar.gz": "application/gzip",
|
||||
".tgz": "application/gzip",
|
||||
".gz": "application/gzip",
|
||||
".zip": "application/zip",
|
||||
".whl": "application/zip",
|
||||
".exe": "application/x-msdownload",
|
||||
".rpm": "application/x-rpm",
|
||||
".xml": "application/xml",
|
||||
".yaml": "text/yaml",
|
||||
".yml": "text/yaml",
|
||||
".json": "application/json",
|
||||
".sig": "application/octet-stream",
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(filePath string) string {
|
||||
lower := strings.ToLower(filePath)
|
||||
if strings.HasSuffix(lower, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
ext := path.Ext(lower)
|
||||
if ct, ok := contentTypeMap[ext]; ok {
|
||||
return ct
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, reqPath string) string {
|
||||
base := strings.TrimRight(remote.BaseURL, "/")
|
||||
return base + "/" + strings.TrimLeft(reqPath, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package generic
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestGenericRewriteResponse(t *testing.T) {
|
||||
if out, err := (&Provider{}).RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
|
||||
t.Error("generic never rewrites")
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package generic_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
if p.Type() != models.PackageGeneric {
|
||||
t.Errorf("expected generic, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify_AllImmutable(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
paths := []string{"file.tar.gz", "path/to/binary", "index.html", "data.json"}
|
||||
for _, path := range paths {
|
||||
if p.Classify(path) != provider.Immutable {
|
||||
t.Errorf("generic should classify %q as immutable", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
tests := []struct{ path, want string }{
|
||||
{"file.tar.gz", "application/gzip"},
|
||||
{"file.tgz", "application/gzip"},
|
||||
{"file.zip", "application/zip"},
|
||||
{"file.rpm", "application/x-rpm"},
|
||||
{"file.json", "application/json"},
|
||||
{"file.unknown", "application/octet-stream"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.ContentType(tt.path); got != tt.want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_UpstreamURL(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://example.com/repo"}, "path/to/file.tar.gz")
|
||||
want := "https://example.com/repo/path/to/file.tar.gz"
|
||||
if got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_AuthHeaders_BasicAuth(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "user", Password: "pass"})
|
||||
if h.Get("Authorization") != "Basic dXNlcjpwYXNz" {
|
||||
t.Errorf("unexpected auth header: %q", h.Get("Authorization"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_AuthHeaders_NoAuth(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("expected no auth header")
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package goproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageGoProxy }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "/@v/list") || strings.HasSuffix(path, "/@latest") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(path, ".mod") {
|
||||
return "text/plain"
|
||||
}
|
||||
if strings.HasSuffix(path, ".info") {
|
||||
return "application/json"
|
||||
}
|
||||
if strings.HasSuffix(path, "/list") {
|
||||
return "text/plain"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package goproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestGoProxyURLAuthRewrite(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://proxy.golang.org/"}, "/mod/@v/list"); got != "https://proxy.golang.org/mod/@v/list" {
|
||||
t.Errorf("upstream url %q", got)
|
||||
}
|
||||
if out, err := p.RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
|
||||
t.Error("goproxy never rewrites")
|
||||
}
|
||||
if h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"}); h.Get("Authorization") == "" {
|
||||
t.Error("expected basic auth header")
|
||||
}
|
||||
if got := p.ContentType("mod/@v/v1.0.0.info"); got != "application/json" {
|
||||
t.Errorf("info content type %q", got)
|
||||
}
|
||||
if got := p.ContentType("mod/@v/v1.0.0.mod"); got != "text/plain" {
|
||||
t.Errorf("mod content type %q", got)
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package goproxy_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
if p.Type() != models.PackageGoProxy {
|
||||
t.Errorf("expected goproxy, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"golang.org/x/net/@v/list", provider.Mutable},
|
||||
{"golang.org/x/net/@latest", provider.Mutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.info", provider.Immutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.mod", provider.Immutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.zip", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
tests := []struct{ path, want string }{
|
||||
{"m/@v/v1.0.0.zip", "application/zip"},
|
||||
{"m/@v/v1.0.0.mod", "text/plain"},
|
||||
{"m/@v/v1.0.0.info", "application/json"},
|
||||
{"m/@v/list", "text/plain"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.ContentType(tt.path); got != tt.want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package helm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageHelm }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "index.yaml") || strings.HasSuffix(path, "index.yml") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tgz") || strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") {
|
||||
return "text/yaml"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
rewritten := strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
if rewritten == content {
|
||||
return nil, nil
|
||||
}
|
||||
return []byte(rewritten), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package helm
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestHelmContentTypeBranches(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for path, want := range map[string]string{
|
||||
"charts/x-1.0.0.tgz": "application/gzip",
|
||||
"x.tar.gz": "application/gzip",
|
||||
"index.yaml": "text/yaml",
|
||||
"x.yml": "text/yaml",
|
||||
"other": "application/octet-stream",
|
||||
} {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package helm_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/helm"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
if p.Type() != models.PackageHelm {
|
||||
t.Errorf("expected helm, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"index.yaml", provider.Mutable},
|
||||
{"index.yml", provider.Mutable},
|
||||
{"chart-1.0.tgz", provider.Immutable},
|
||||
{"charts/nginx-1.0.tgz", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_RewriteResponse(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
body := []byte("urls:\n- https://charts.example.com/chart-1.0.tgz")
|
||||
remote := models.Remote{Name: "helm-test", BaseURL: "https://charts.example.com"}
|
||||
rewritten, err := p.RewriteResponse(body, remote, "https://proxy.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rewritten == nil {
|
||||
t.Fatal("expected rewrite")
|
||||
}
|
||||
if !strings.Contains(string(rewritten), "proxy.example.com/api/v1/remote/helm-test") {
|
||||
t.Errorf("expected proxy URL in body: %s", rewritten)
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package npm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageNPM }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, ".tgz") {
|
||||
return provider.Immutable
|
||||
}
|
||||
return provider.Mutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tgz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" || !json.Valid(body) {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
rewritten := strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
if rewritten == content {
|
||||
return nil, nil
|
||||
}
|
||||
return []byte(rewritten), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package npm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestType(t *testing.T) {
|
||||
if (&Provider{}).Type() != models.PackageNPM {
|
||||
t.Fatal("wrong type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("pkg/-/pkg-1.0.0.tgz") != provider.Immutable {
|
||||
t.Error("tgz should be immutable")
|
||||
}
|
||||
if p.Classify("pkg") != provider.Mutable {
|
||||
t.Error("metadata should be mutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.ContentType("pkg/-/pkg-1.0.0.tgz") != "application/gzip" {
|
||||
t.Error("tgz content type")
|
||||
}
|
||||
if p.ContentType("pkg") != "application/json" {
|
||||
t.Error("metadata content type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
p := &Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://registry.npmjs.org/"}, "/pkg")
|
||||
if got != "https://registry.npmjs.org/pkg" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
p := &Provider{}
|
||||
remote := models.Remote{Name: "npmjs", BaseURL: "https://registry.npmjs.org"}
|
||||
|
||||
if out, _ := p.RewriteResponse([]byte(`{"a":1}`), remote, ""); out != nil {
|
||||
t.Error("empty proxyBaseURL should be a no-op")
|
||||
}
|
||||
if out, _ := p.RewriteResponse([]byte("not json"), remote, "http://proxy"); out != nil {
|
||||
t.Error("invalid json should be a no-op")
|
||||
}
|
||||
body := []byte(`{"tarball":"https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz"}`)
|
||||
out, err := p.RewriteResponse(body, remote, "http://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(out) != `{"tarball":"http://proxy/api/v1/remote/npmjs/pkg/-/pkg-1.0.0.tgz"}` {
|
||||
t.Errorf("rewrite: %s", out)
|
||||
}
|
||||
if out, _ := p.RewriteResponse([]byte(`{"x":"unrelated"}`), remote, "http://proxy"); out != nil {
|
||||
t.Error("no matching base URL should be a no-op")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthHeaders(t *testing.T) {
|
||||
p := &Provider{}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "pw"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("expected auth header when credentials set")
|
||||
}
|
||||
h, _ = p.AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("expected no auth header without credentials")
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type Mutability int
|
||||
|
||||
const (
|
||||
Immutable Mutability = iota
|
||||
Mutable
|
||||
)
|
||||
|
||||
type Provider interface {
|
||||
Type() models.PackageType
|
||||
Classify(path string) Mutability
|
||||
ContentType(path string) string
|
||||
UpstreamURL(remote models.Remote, path string) string
|
||||
RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error)
|
||||
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
|
||||
}
|
||||
|
||||
type FileEntry struct {
|
||||
FilePath string
|
||||
ContentHash string
|
||||
}
|
||||
|
||||
type FileStore interface {
|
||||
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
|
||||
ListPackages(ctx context.Context, repoName string) ([]string, error)
|
||||
}
|
||||
|
||||
type LocalUploader interface {
|
||||
ValidateUpload(filePath string) (storagePath, contentType string, err error)
|
||||
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
|
||||
}
|
||||
|
||||
type LocalIndexer interface {
|
||||
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
|
||||
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
|
||||
}
|
||||
|
||||
type BlobReader interface {
|
||||
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
||||
}
|
||||
|
||||
type PostUploadHook interface {
|
||||
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
|
||||
}
|
||||
|
||||
// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that
|
||||
// feeds generated repodata) after a local file is removed.
|
||||
type PostDeleteHook interface {
|
||||
AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error
|
||||
}
|
||||
|
||||
type MetadataStore interface {
|
||||
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
|
||||
}
|
||||
|
||||
type MetadataDeleter interface {
|
||||
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
|
||||
type RPMMetadataReader interface {
|
||||
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
|
||||
}
|
||||
|
||||
type RPMMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Epoch int
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
Summary string
|
||||
Description string
|
||||
RPMSize int64
|
||||
InstalledSize int64
|
||||
License string
|
||||
Vendor string
|
||||
Group string
|
||||
BuildHost string
|
||||
SourceRPM string
|
||||
URL string
|
||||
Packager string
|
||||
Requires []RPMDep
|
||||
Provides []RPMDep
|
||||
Files []RPMFile
|
||||
Changelogs []RPMChangelog
|
||||
}
|
||||
|
||||
type RPMDep struct {
|
||||
Name string `json:"name"`
|
||||
Flags string `json:"flags,omitempty"`
|
||||
Epoch string `json:"epoch,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Release string `json:"release,omitempty"`
|
||||
}
|
||||
|
||||
type RPMFile struct {
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
type RPMChangelog struct {
|
||||
Author string `json:"author"`
|
||||
Date int64 `json:"date"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
|
||||
type MemberIndex struct {
|
||||
RemoteName string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
var registry = map[models.PackageType]Provider{}
|
||||
|
||||
func Register(p Provider) {
|
||||
registry[p.Type()] = p
|
||||
}
|
||||
|
||||
func Get(t models.PackageType) (Provider, error) {
|
||||
p, ok := registry[t]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no provider registered for package type %q", t)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func All() map[models.PackageType]Provider {
|
||||
return registry
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package puppet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackagePuppet }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasPrefix(path, "v3/modules/") || strings.HasPrefix(path, "v3/releases") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.HasPrefix(path, "v3/") {
|
||||
return "application/json"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
content = strings.ReplaceAll(content, `"/v3/files/`, `"`+proxyURL+`/v3/files/`)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
content = strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
return []byte(content), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package puppet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestType(t *testing.T) {
|
||||
if (&Provider{}).Type() != models.PackagePuppet {
|
||||
t.Fatal("wrong type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("v3/modules/puppetlabs-stdlib") != provider.Mutable {
|
||||
t.Error("modules should be mutable")
|
||||
}
|
||||
if p.Classify("v3/releases?module=x") != provider.Mutable {
|
||||
t.Error("releases should be mutable")
|
||||
}
|
||||
if p.Classify("v3/files/puppetlabs-stdlib-1.0.0.tar.gz") != provider.Immutable {
|
||||
t.Error("files should be immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.ContentType("x/mod-1.0.0.tar.gz") != "application/gzip" {
|
||||
t.Error("tar.gz")
|
||||
}
|
||||
if p.ContentType("v3/modules/x") != "application/json" {
|
||||
t.Error("v3 json")
|
||||
}
|
||||
if p.ContentType("other") != "application/octet-stream" {
|
||||
t.Error("default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
got := (&Provider{}).UpstreamURL(models.Remote{BaseURL: "https://forgeapi.puppet.com/"}, "/v3/modules/x")
|
||||
if got != "https://forgeapi.puppet.com/v3/modules/x" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
p := &Provider{}
|
||||
remote := models.Remote{Name: "forge", BaseURL: "https://forgeapi.puppet.com"}
|
||||
|
||||
if out, _ := p.RewriteResponse([]byte("x"), remote, ""); out != nil {
|
||||
t.Error("empty proxyBaseURL is a no-op")
|
||||
}
|
||||
|
||||
body := []byte(`{"file_uri":"/v3/files/mod.tar.gz","home":"https://forgeapi.puppet.com/x"}`)
|
||||
out, err := p.RewriteResponse(body, remote, "http://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(out)
|
||||
if !strings.Contains(s, "http://proxy/api/v1/remote/forge/v3/files/mod.tar.gz") {
|
||||
t.Errorf("v3/files not rewritten: %s", s)
|
||||
}
|
||||
if !strings.Contains(s, "http://proxy/api/v1/remote/forge/x") {
|
||||
t.Errorf("base URL not rewritten: %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthHeaders(t *testing.T) {
|
||||
h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("no credentials, no header")
|
||||
}
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
package pypi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var fileRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*\.(whl|tar\.gz|zip)$`)
|
||||
var normalizeRe = regexp.MustCompile(`[-_.]+`)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackagePyPI }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.Contains(path, "simple/") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
lower := strings.ToLower(path)
|
||||
if strings.HasSuffix(lower, ".whl") || strings.HasSuffix(lower, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.Contains(path, "simple/") {
|
||||
return "text/html"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
if strings.HasPrefix(path, "simple/") {
|
||||
return "https://pypi.org/" + path
|
||||
}
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + "/"
|
||||
content = strings.ReplaceAll(content, "https://files.pythonhosted.org/", proxyURL)
|
||||
content = strings.ReplaceAll(content, "../../", proxyURL)
|
||||
return []byte(content), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func normalize(name string) string {
|
||||
return strings.ToLower(normalizeRe.ReplaceAllString(name, "-"))
|
||||
}
|
||||
|
||||
func packageFromWheel(filename string) string {
|
||||
parts := strings.SplitN(filename, "-", 3)
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return normalize(parts[0])
|
||||
}
|
||||
|
||||
func packageFromSdist(filename string) string {
|
||||
name := filename
|
||||
for _, suffix := range []string{".tar.gz", ".zip"} {
|
||||
if strings.HasSuffix(name, suffix) {
|
||||
name = strings.TrimSuffix(name, suffix)
|
||||
break
|
||||
}
|
||||
}
|
||||
idx := strings.LastIndex(name, "-")
|
||||
if idx <= 0 {
|
||||
return ""
|
||||
}
|
||||
return normalize(name[:idx])
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !fileRe.MatchString(filename) {
|
||||
return "", "", fmt.Errorf("filename %q must be a .whl, .tar.gz, or .zip file", filename)
|
||||
}
|
||||
|
||||
var pkgName string
|
||||
if strings.HasSuffix(filename, ".whl") {
|
||||
pkgName = packageFromWheel(filename)
|
||||
} else {
|
||||
pkgName = packageFromSdist(filename)
|
||||
}
|
||||
if pkgName == "" {
|
||||
return "", "", fmt.Errorf("cannot parse package name from %q", filename)
|
||||
}
|
||||
|
||||
ct := "application/zip"
|
||||
if strings.HasSuffix(filename, ".tar.gz") {
|
||||
ct = "application/gzip"
|
||||
}
|
||||
|
||||
return pkgName + "/" + filename, ct, nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
parts := strings.SplitN(storagePath, "/", 2)
|
||||
filename := storagePath
|
||||
if len(parts) == 2 {
|
||||
filename = parts[1]
|
||||
}
|
||||
return map[string]any{
|
||||
"package": parts[0],
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
if path == "simple" || path == "simple/" {
|
||||
p.servePackageList(w, r, files, repoName)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(path, "simple/") {
|
||||
pkg := strings.TrimPrefix(path, "simple/")
|
||||
pkg = strings.TrimSuffix(pkg, "/")
|
||||
if pkg != "" && !strings.Contains(pkg, "/") {
|
||||
p.servePackageFiles(w, r, files, repoName, pkg)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
if !strings.HasPrefix(path, "simple/") {
|
||||
return nil, fmt.Errorf("unsupported index path: %q", path)
|
||||
}
|
||||
|
||||
pkg := strings.TrimPrefix(path, "simple/")
|
||||
pkg = strings.TrimSuffix(pkg, "/")
|
||||
if pkg == "" {
|
||||
return p.generatePackageListHTML(ctx, files, repoName)
|
||||
}
|
||||
return p.generatePackageFilesHTML(ctx, files, repoName, pkg)
|
||||
}
|
||||
|
||||
func (p *Provider) servePackageList(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName string) {
|
||||
body, err := p.generatePackageListHTML(r.Context(), files, repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
func (p *Provider) servePackageFiles(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, packageName string) {
|
||||
normalized := normalize(packageName)
|
||||
prefix := normalized + "/"
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
|
||||
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n",
|
||||
normalized, filename, hash, filename)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.WriteString(w, b.String())
|
||||
}
|
||||
|
||||
func (p *Provider) generatePackageListHTML(ctx context.Context, files provider.FileStore, repoName string) ([]byte, error) {
|
||||
packages, err := files.ListPackages(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, pkg := range packages {
|
||||
fmt.Fprintf(&b, "<a href=\"%s/\">%s</a>\n", pkg, pkg)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
|
||||
func (p *Provider) generatePackageFilesHTML(ctx context.Context, files provider.FileStore, repoName, packageName string) ([]byte, error) {
|
||||
normalized := normalize(packageName)
|
||||
prefix := normalized + "/"
|
||||
entries, err := files.ListFilesByPrefix(ctx, repoName, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
|
||||
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&b, "<a href=\"%s/%s#sha256=%s\">%s</a>\n",
|
||||
normalized, filename, hash, filename)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
package pypi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeFileStore is an in-memory provider.FileStore for exercising local index
|
||||
// generation without a database.
|
||||
type fakeFileStore struct {
|
||||
packages []string
|
||||
files map[string][]provider.FileEntry
|
||||
}
|
||||
|
||||
func (f *fakeFileStore) ListPackages(_ context.Context, _ string) ([]string, error) {
|
||||
return f.packages, nil
|
||||
}
|
||||
|
||||
func (f *fakeFileStore) ListFilesByPrefix(_ context.Context, _, prefix string) ([]provider.FileEntry, error) {
|
||||
return f.files[prefix], nil
|
||||
}
|
||||
|
||||
func TestTypeClassifyContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Type() != models.PackagePyPI {
|
||||
t.Fatal("type")
|
||||
}
|
||||
if p.Classify("simple/foo/") != provider.Mutable {
|
||||
t.Error("simple index should be mutable")
|
||||
}
|
||||
if p.Classify("packages/foo-1.0.whl") != provider.Immutable {
|
||||
t.Error("wheel should be immutable")
|
||||
}
|
||||
cases := map[string]string{
|
||||
"foo-1.0-py3-none-any.whl": "application/zip",
|
||||
"foo-1.0.zip": "application/zip",
|
||||
"foo-1.0.tar.gz": "application/gzip",
|
||||
"simple/foo/": "text/html",
|
||||
"weird": "application/octet-stream",
|
||||
}
|
||||
for path, want := range cases {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://files.example.com"}, "packages/foo.whl"); got != "https://files.example.com/packages/foo.whl" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://x"}, "simple/foo/"); got != "https://pypi.org/simple/foo/" {
|
||||
t.Errorf("simple should hit pypi.org, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("numpy-1.26.0-cp311-cp311-linux_x86_64.whl")
|
||||
if err != nil || sp != "numpy/numpy-1.26.0-cp311-cp311-linux_x86_64.whl" || ct != "application/zip" {
|
||||
t.Errorf("wheel: sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
sp, ct, err = p.ValidateUpload("requests-2.31.0.tar.gz")
|
||||
if err != nil || sp != "requests/requests-2.31.0.tar.gz" || ct != "application/gzip" {
|
||||
t.Errorf("sdist: sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("not-a-package.txt"); err == nil {
|
||||
t.Error("expected error for bad extension")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackageNameParsing(t *testing.T) {
|
||||
if got := packageFromWheel("Foo_Bar-1.0-py3-none-any.whl"); got != "foo-bar" {
|
||||
t.Errorf("wheel name = %q", got)
|
||||
}
|
||||
if got := packageFromWheel("noseparator.whl"); got != "" {
|
||||
t.Errorf("expected empty for unparseable wheel, got %q", got)
|
||||
}
|
||||
if got := packageFromSdist("My.Pkg-2.0.tar.gz"); got != "my-pkg" {
|
||||
t.Errorf("sdist name = %q", got)
|
||||
}
|
||||
if got := packageFromSdist("noseparator.zip"); got != "" {
|
||||
t.Errorf("expected empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadResponse(t *testing.T) {
|
||||
resp := (&Provider{}).UploadResponse("foo/foo-1.0.whl", "sha256:abc", 123)
|
||||
if resp["filename"] != "foo-1.0.whl" || resp["package"] != "foo" || resp["content_hash"] != "sha256:abc" {
|
||||
t.Errorf("unexpected upload response: %v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if out, _ := p.RewriteResponse([]byte("x"), models.Remote{Name: "pypi"}, ""); out != nil {
|
||||
t.Error("empty proxyBaseURL is a no-op")
|
||||
}
|
||||
body := []byte(`<a href="https://files.pythonhosted.org/packages/foo.whl">foo.whl</a>`)
|
||||
out, err := p.RewriteResponse(body, models.Remote{Name: "pypi"}, "http://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(out), "http://proxy/api/v1/remote/pypi/") {
|
||||
t.Errorf("not rewritten: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateLocalIndex(t *testing.T) {
|
||||
p := &Provider{}
|
||||
fs := &fakeFileStore{
|
||||
packages: []string{"foo", "bar"},
|
||||
files: map[string][]provider.FileEntry{
|
||||
"foo/": {{FilePath: "foo/foo-1.0-py3-none-any.whl", ContentHash: "sha256:aaa"}},
|
||||
},
|
||||
}
|
||||
list, err := p.GenerateLocalIndex(context.Background(), fs, "local", "simple/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(list), "foo") || !strings.Contains(string(list), "bar") {
|
||||
t.Errorf("package list missing entries: %s", list)
|
||||
}
|
||||
|
||||
files, err := p.GenerateLocalIndex(context.Background(), fs, "local", "simple/foo/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(files), "foo-1.0-py3-none-any.whl") {
|
||||
t.Errorf("file list missing wheel: %s", files)
|
||||
}
|
||||
|
||||
if _, err := p.GenerateLocalIndex(context.Background(), fs, "local", "notsimple"); err == nil {
|
||||
t.Error("expected error for non-simple path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeLocalIndexHTTP(t *testing.T) {
|
||||
p := &Provider{}
|
||||
fs := &fakeFileStore{
|
||||
packages: []string{"foo"},
|
||||
files: map[string][]provider.FileEntry{
|
||||
"foo/": {{FilePath: "foo/foo-1.0-py3-none-any.whl", ContentHash: "sha256:aaa"}},
|
||||
},
|
||||
}
|
||||
serve := func(path string) (*httptest.ResponseRecorder, bool) {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
handled := p.ServeLocalIndex(w, r, fs, "local", path)
|
||||
return w, handled
|
||||
}
|
||||
|
||||
if w, ok := serve("simple/"); !ok || w.Code != 200 || !strings.Contains(w.Body.String(), "foo") {
|
||||
t.Errorf("simple index: handled=%v code=%d body=%s", ok, w.Code, w.Body.String())
|
||||
}
|
||||
if w, ok := serve("simple/foo/"); !ok || w.Code != 200 || !strings.Contains(w.Body.String(), "foo-1.0-py3-none-any.whl") {
|
||||
t.Errorf("package index: handled=%v code=%d body=%s", ok, w.Code, w.Body.String())
|
||||
}
|
||||
// Non-simple paths are not handled.
|
||||
if _, ok := serve("packages/foo.whl"); ok {
|
||||
t.Error("non-index path should not be handled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthHeaders(t *testing.T) {
|
||||
h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("expected auth header")
|
||||
}
|
||||
}
|
||||
@@ -1,453 +0,0 @@
|
||||
package rpm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
rpmlib "github.com/cavaliergopher/rpm"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var mutableRe = []*regexp.Regexp{
|
||||
regexp.MustCompile(`repomd\.xml$`),
|
||||
regexp.MustCompile(`repodata/`),
|
||||
regexp.MustCompile(`Packages\.gz$`),
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageRPM }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
for _, re := range mutableRe {
|
||||
if re.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".rpm") {
|
||||
return "application/x-rpm"
|
||||
}
|
||||
if strings.HasSuffix(path, ".xml") || strings.HasSuffix(path, ".xml.gz") || strings.HasSuffix(path, ".xml.xz") {
|
||||
return "application/xml"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(strings.ToLower(filename), ".rpm") {
|
||||
return "", "", fmt.Errorf("file must be an .rpm package")
|
||||
}
|
||||
|
||||
return "Packages/" + filename, "application/x-rpm", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
filename := strings.TrimPrefix(storagePath, "Packages/")
|
||||
return map[string]any{
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
|
||||
|
||||
reader, blobSize, err := blobs.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
slog.Error("rpm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
pkg, err := rpmlib.Read(reader)
|
||||
if err != nil {
|
||||
slog.Error("rpm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
meta := &provider.RPMMetadata{
|
||||
RepoName: repoName,
|
||||
FilePath: storagePath,
|
||||
ContentHash: contentHash,
|
||||
Name: pkg.Name(),
|
||||
Epoch: pkg.Epoch(),
|
||||
Version: pkg.Version(),
|
||||
Release: pkg.Release(),
|
||||
Arch: pkg.Architecture(),
|
||||
Summary: pkg.Summary(),
|
||||
Description: pkg.Description(),
|
||||
RPMSize: blobSize,
|
||||
InstalledSize: int64(pkg.Size()),
|
||||
License: pkg.License(),
|
||||
Vendor: pkg.Vendor(),
|
||||
Group: firstGroup(pkg.Groups()),
|
||||
BuildHost: pkg.BuildHost(),
|
||||
SourceRPM: pkg.SourceRPM(),
|
||||
URL: pkg.URL(),
|
||||
Packager: pkg.Packager(),
|
||||
}
|
||||
|
||||
for _, req := range pkg.Requires() {
|
||||
meta.Requires = append(meta.Requires, rpmDepFromEntry(req))
|
||||
}
|
||||
for _, prov := range pkg.Provides() {
|
||||
meta.Provides = append(meta.Provides, rpmDepFromEntry(prov))
|
||||
}
|
||||
|
||||
if meta.Requires == nil {
|
||||
meta.Requires = []provider.RPMDep{}
|
||||
}
|
||||
if meta.Provides == nil {
|
||||
meta.Provides = []provider.RPMDep{}
|
||||
}
|
||||
meta.Files = []provider.RPMFile{}
|
||||
meta.Changelogs = []provider.RPMChangelog{}
|
||||
|
||||
if err := db.InsertRPMMetadata(ctx, meta); err != nil {
|
||||
slog.Error("rpm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("rpm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
}
|
||||
|
||||
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
|
||||
if err := db.DeleteRPMMetadata(ctx, repoName, storagePath); err != nil {
|
||||
slog.Error("rpm metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return err
|
||||
}
|
||||
slog.Info("rpm metadata: deleted", "repo", repoName, "path", storagePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func rpmDepFromEntry(e rpmlib.Dependency) provider.RPMDep {
|
||||
dep := provider.RPMDep{Name: e.Name()}
|
||||
if e.Flags() != 0 {
|
||||
dep.Flags = rpmFlagString(e.Flags())
|
||||
dep.Version = e.Version()
|
||||
dep.Release = e.Release()
|
||||
if e.Epoch() > 0 {
|
||||
dep.Epoch = fmt.Sprintf("%d", e.Epoch())
|
||||
}
|
||||
}
|
||||
return dep
|
||||
}
|
||||
|
||||
func rpmFlagString(f int) string {
|
||||
switch {
|
||||
case f&0x08 != 0 && f&0x04 != 0:
|
||||
return "GE"
|
||||
case f&0x02 != 0 && f&0x04 != 0:
|
||||
return "LE"
|
||||
case f&0x08 != 0:
|
||||
return "GT"
|
||||
case f&0x02 != 0:
|
||||
return "LT"
|
||||
case f&0x04 != 0:
|
||||
return "EQ"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func firstGroup(groups []string) string {
|
||||
if len(groups) > 0 {
|
||||
return groups[0]
|
||||
}
|
||||
return "Unspecified"
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
if !strings.HasPrefix(path, "repodata/") {
|
||||
return false
|
||||
}
|
||||
|
||||
rpmReader, ok := files.(provider.RPMMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "rpm metadata not available", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
tail := strings.TrimPrefix(path, "repodata/")
|
||||
|
||||
switch {
|
||||
case tail == "repomd.xml":
|
||||
p.serveRepomd(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-primary.xml.gz"):
|
||||
p.servePrimary(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-filelists.xml.gz"):
|
||||
p.serveFilelists(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-other.xml.gz"):
|
||||
p.serveOther(w, r, rpmReader, repoName)
|
||||
default:
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("rpm local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
primary := generatePrimaryXMLGZ(metas)
|
||||
filelists := generateFilelistsXMLGZ(metas)
|
||||
other := generateOtherXMLGZ(metas)
|
||||
|
||||
primaryHash := sha256Hex(primary)
|
||||
filelistsHash := sha256Hex(filelists)
|
||||
otherHash := sha256Hex(other)
|
||||
|
||||
repomd := generateRepomd(primaryHash, len(primary), filelistsHash, len(filelists), otherHash, len(other))
|
||||
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(repomd)
|
||||
}
|
||||
|
||||
func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generatePrimaryXMLGZ(metas))
|
||||
}
|
||||
|
||||
func (p *Provider) serveFilelists(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateFilelistsXMLGZ(metas))
|
||||
}
|
||||
|
||||
func (p *Provider) serveOther(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateOtherXMLGZ(metas))
|
||||
}
|
||||
|
||||
func generateRepomd(primaryHash string, primarySize int, filelistsHash string, filelistsSize int, otherHash string, otherSize int) []byte {
|
||||
ts := fmt.Sprintf("%d", time.Now().Unix())
|
||||
var b bytes.Buffer
|
||||
b.WriteString(xml.Header)
|
||||
b.WriteString(`<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">` + "\n")
|
||||
fmt.Fprintf(&b, " <revision>%s</revision>\n", ts)
|
||||
|
||||
writeRepomdData(&b, "primary", primaryHash, primarySize, ts)
|
||||
writeRepomdData(&b, "filelists", filelistsHash, filelistsSize, ts)
|
||||
writeRepomdData(&b, "other", otherHash, otherSize, ts)
|
||||
|
||||
b.WriteString("</repomd>\n")
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func writeRepomdData(b *bytes.Buffer, dtype, hash string, size int, ts string) {
|
||||
fmt.Fprintf(b, " <data type=\"%s\">\n", dtype)
|
||||
fmt.Fprintf(b, " <checksum type=\"sha256\">%s</checksum>\n", hash)
|
||||
fmt.Fprintf(b, " <location href=\"repodata/%s-%s.xml.gz\"/>\n", hash, dtype)
|
||||
fmt.Fprintf(b, " <timestamp>%s</timestamp>\n", ts)
|
||||
fmt.Fprintf(b, " <size>%d</size>\n", size)
|
||||
fmt.Fprintf(b, " </data>\n")
|
||||
}
|
||||
|
||||
func generatePrimaryXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package type=\"rpm\">\n")
|
||||
fmt.Fprintf(&xmlBuf, " <name>%s</name>\n", xmlEscape(m.Name))
|
||||
fmt.Fprintf(&xmlBuf, " <arch>%s</arch>\n", xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
fmt.Fprintf(&xmlBuf, " <checksum type=\"sha256\" pkgid=\"YES\">%s</checksum>\n", pkgHash)
|
||||
fmt.Fprintf(&xmlBuf, " <summary>%s</summary>\n", xmlEscape(m.Summary))
|
||||
fmt.Fprintf(&xmlBuf, " <description>%s</description>\n", xmlEscape(m.Description))
|
||||
if m.Packager != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <packager>%s</packager>\n", xmlEscape(m.Packager))
|
||||
}
|
||||
if m.URL != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <url>%s</url>\n", xmlEscape(m.URL))
|
||||
}
|
||||
fmt.Fprintf(&xmlBuf, " <time file=\"%d\" build=\"0\"/>\n", time.Now().Unix())
|
||||
fmt.Fprintf(&xmlBuf, " <size package=\"%d\" installed=\"%d\" archive=\"0\"/>\n", m.RPMSize, m.InstalledSize)
|
||||
fmt.Fprintf(&xmlBuf, " <location href=\"%s\"/>\n", xmlEscape(m.FilePath))
|
||||
fmt.Fprintf(&xmlBuf, " <format>\n")
|
||||
if m.License != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:license>%s</rpm:license>\n", xmlEscape(m.License))
|
||||
}
|
||||
if m.Vendor != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:vendor>%s</rpm:vendor>\n", xmlEscape(m.Vendor))
|
||||
}
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:group>%s</rpm:group>\n", xmlEscape(m.Group))
|
||||
if m.BuildHost != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:buildhost>%s</rpm:buildhost>\n", xmlEscape(m.BuildHost))
|
||||
}
|
||||
if m.SourceRPM != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:sourcerpm>%s</rpm:sourcerpm>\n", xmlEscape(m.SourceRPM))
|
||||
}
|
||||
|
||||
if len(m.Provides) > 0 {
|
||||
xmlBuf.WriteString(" <rpm:provides>\n")
|
||||
for _, d := range m.Provides {
|
||||
writeRPMEntry(&xmlBuf, d)
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:provides>\n")
|
||||
}
|
||||
if len(m.Requires) > 0 {
|
||||
xmlBuf.WriteString(" <rpm:requires>\n")
|
||||
for _, d := range m.Requires {
|
||||
writeRPMEntry(&xmlBuf, d)
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:requires>\n")
|
||||
}
|
||||
|
||||
fmt.Fprintf(&xmlBuf, " </format>\n")
|
||||
fmt.Fprintf(&xmlBuf, "</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</metadata>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func generateFilelistsXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<filelists xmlns=\"http://linux.duke.edu/metadata/filelists\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
for _, f := range m.Files {
|
||||
if f.Type != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <file type=\"%s\">%s</file>\n", f.Type, xmlEscape(f.Path))
|
||||
} else {
|
||||
fmt.Fprintf(&xmlBuf, " <file>%s</file>\n", xmlEscape(f.Path))
|
||||
}
|
||||
}
|
||||
xmlBuf.WriteString("</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</filelists>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func generateOtherXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<otherdata xmlns=\"http://linux.duke.edu/metadata/other\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
for _, cl := range m.Changelogs {
|
||||
fmt.Fprintf(&xmlBuf, " <changelog author=\"%s\" date=\"%d\">%s</changelog>\n",
|
||||
xmlEscape(cl.Author), cl.Date, xmlEscape(cl.Text))
|
||||
}
|
||||
xmlBuf.WriteString("</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</otherdata>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func writeRPMEntry(b *bytes.Buffer, d provider.RPMDep) {
|
||||
if d.Flags != "" {
|
||||
fmt.Fprintf(b, " <rpm:entry name=\"%s\" flags=\"%s\"", xmlEscape(d.Name), d.Flags)
|
||||
if d.Epoch != "" {
|
||||
fmt.Fprintf(b, " epoch=\"%s\"", d.Epoch)
|
||||
}
|
||||
if d.Version != "" {
|
||||
fmt.Fprintf(b, " ver=\"%s\"", xmlEscape(d.Version))
|
||||
}
|
||||
if d.Release != "" {
|
||||
fmt.Fprintf(b, " rel=\"%s\"", xmlEscape(d.Release))
|
||||
}
|
||||
b.WriteString("/>\n")
|
||||
} else {
|
||||
fmt.Fprintf(b, " <rpm:entry name=\"%s\"/>\n", xmlEscape(d.Name))
|
||||
}
|
||||
}
|
||||
|
||||
func xmlEscape(s string) string {
|
||||
var b bytes.Buffer
|
||||
xml.EscapeText(&b, []byte(s))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func gzipBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
gz.Write(data)
|
||||
gz.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func sha256Hex(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
package rpm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type fakeBlobReader struct{ data []byte }
|
||||
|
||||
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
|
||||
}
|
||||
|
||||
type fakeMetaStore struct{ inserted *provider.RPMMetadata }
|
||||
|
||||
func (f *fakeMetaStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
|
||||
f.inserted = m
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeRPMReader struct{ metas []provider.RPMMetadata }
|
||||
|
||||
func (f fakeRPMReader) ListRPMMetadataEntries(_ context.Context, _ string) ([]provider.RPMMetadata, error) {
|
||||
return f.metas, nil
|
||||
}
|
||||
func (f fakeRPMReader) ListFilesByPrefix(_ context.Context, _, _ string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f fakeRPMReader) ListPackages(_ context.Context, _ string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestRPMPureFuncs(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Type() != models.PackageRPM {
|
||||
t.Error("type")
|
||||
}
|
||||
if p.Classify("repodata/repomd.xml") != provider.Mutable {
|
||||
t.Error("repomd should be mutable")
|
||||
}
|
||||
if p.Classify("Packages/foo.rpm") != provider.Immutable {
|
||||
t.Error("rpm should be immutable")
|
||||
}
|
||||
if p.ContentType("x.rpm") != "application/x-rpm" {
|
||||
t.Error("rpm content type")
|
||||
}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://mirror/"}, "/Packages/x.rpm"); got != "https://mirror/Packages/x.rpm" {
|
||||
t.Errorf("upstream url %q", got)
|
||||
}
|
||||
if out, _ := p.RewriteResponse(nil, models.Remote{}, "http://p"); out != nil {
|
||||
t.Error("rpm never rewrites")
|
||||
}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("auth header")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("dir/foo-1.0.noarch.rpm")
|
||||
if err != nil || sp != "Packages/foo-1.0.noarch.rpm" || ct != "application/x-rpm" {
|
||||
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("foo.txt"); err == nil {
|
||||
t.Error("expected error for non-rpm")
|
||||
}
|
||||
resp := p.UploadResponse("Packages/foo.rpm", "sha256:abc", 10)
|
||||
if resp["content_hash"] != "sha256:abc" {
|
||||
t.Errorf("upload response %v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMAfterUpload(t *testing.T) {
|
||||
data := testsupport.MinimalRPM("e2e-testpkg", "1.0", "1", "noarch")
|
||||
store := &fakeMetaStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "myrepo", "Packages/e2e-testpkg-1.0-1.noarch.rpm",
|
||||
"sha256:deadbeef", fakeBlobReader{data: data}, store)
|
||||
|
||||
m := store.inserted
|
||||
if m == nil {
|
||||
t.Fatal("no metadata inserted")
|
||||
}
|
||||
if m.Name != "e2e-testpkg" || m.Version != "1.0" || m.Release != "1" || m.Arch != "noarch" {
|
||||
t.Errorf("unexpected metadata: %+v", m)
|
||||
}
|
||||
if m.RPMSize != int64(len(data)) {
|
||||
t.Errorf("RPMSize = %d, want %d", m.RPMSize, len(data))
|
||||
}
|
||||
if len(m.Provides) == 0 {
|
||||
t.Error("expected the package to provide itself")
|
||||
}
|
||||
}
|
||||
|
||||
type errBlobReader struct{}
|
||||
|
||||
func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return nil, 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
func TestRPMAfterUploadErrors(t *testing.T) {
|
||||
// Download failure: no metadata inserted, no panic.
|
||||
store := &fakeMetaStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", errBlobReader{}, store)
|
||||
if store.inserted != nil {
|
||||
t.Error("no metadata should be inserted on download error")
|
||||
}
|
||||
// Parse failure: garbage bytes are not a valid RPM.
|
||||
store2 := &fakeMetaStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", fakeBlobReader{data: []byte("not an rpm")}, store2)
|
||||
if store2.inserted != nil {
|
||||
t.Error("no metadata should be inserted on parse error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMServeRepodata(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeRPMReader{metas: []provider.RPMMetadata{{
|
||||
Name: "e2e-testpkg", Version: "1.0", Release: "1", Arch: "noarch",
|
||||
Summary: "test & <special>",
|
||||
ContentHash: "sha256:abc",
|
||||
Requires: []provider.RPMDep{{Name: "libc", Flags: "GE", Version: "2.0"}},
|
||||
Provides: []provider.RPMDep{{Name: "e2e-testpkg"}},
|
||||
Files: []provider.RPMFile{{Path: "/usr/share/e2e/README", Type: "file"}},
|
||||
Changelogs: []provider.RPMChangelog{{Author: "e2e", Date: 1, Text: "init"}},
|
||||
}}}
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if !p.ServeLocalIndex(w, r, reader, "myrepo", path) {
|
||||
t.Fatalf("ServeLocalIndex returned false for %q", path)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
if w := serve("repodata/repomd.xml"); w.Code != 200 || !strings.Contains(w.Body.String(), "<repomd") {
|
||||
t.Errorf("repomd: code=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
for _, name := range []string{"repodata/h-primary.xml.gz", "repodata/h-filelists.xml.gz", "repodata/h-other.xml.gz"} {
|
||||
w := serve(name)
|
||||
if w.Code != 200 {
|
||||
t.Errorf("%s: code %d", name, w.Code)
|
||||
}
|
||||
if _, err := gzip.NewReader(bytes.NewReader(w.Body.Bytes())); err != nil {
|
||||
t.Errorf("%s: not gzip: %v", name, err)
|
||||
}
|
||||
}
|
||||
// Unknown repodata file -> 404.
|
||||
if w := serve("repodata/bogus"); w.Code != http.StatusNotFound {
|
||||
t.Errorf("bogus repodata: code %d", w.Code)
|
||||
}
|
||||
// Non-repodata path -> not handled.
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/Packages/x.rpm", nil)
|
||||
if p.ServeLocalIndex(w, r, reader, "myrepo", "Packages/x.rpm") {
|
||||
t.Error("expected ServeLocalIndex false for non-repodata path")
|
||||
}
|
||||
}
|
||||
|
||||
type errRPMReader struct{}
|
||||
|
||||
func (errRPMReader) ListRPMMetadataEntries(context.Context, string) ([]provider.RPMMetadata, error) {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
func (errRPMReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (errRPMReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestRPMServeMetadataError(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for _, path := range []string{"repodata/repomd.xml", "repodata/h-primary.xml.gz", "repodata/h-filelists.xml.gz", "repodata/h-other.xml.gz"} {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
p.ServeLocalIndex(w, r, errRPMReader{}, "repo", path)
|
||||
if w.Code != 500 {
|
||||
t.Errorf("%s with failing reader = %d, want 500", path, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMFullMetadataXML(t *testing.T) {
|
||||
// A fully-populated entry exercises every optional-field branch in the
|
||||
// primary/filelists/other XML generators.
|
||||
metas := []provider.RPMMetadata{{
|
||||
Name: "full", Epoch: 1, Version: "2.0", Release: "3", Arch: "x86_64",
|
||||
Summary: "s", Description: "d", License: "MIT", Vendor: "acme",
|
||||
Group: "System", BuildHost: "build.example.com", SourceRPM: "full-2.0.src.rpm",
|
||||
URL: "https://example.com", Packager: "pkgr", ContentHash: "sha256:abc",
|
||||
RPMSize: 100, InstalledSize: 200,
|
||||
Requires: []provider.RPMDep{{Name: "libc", Flags: "GE", Epoch: "0", Version: "2.0", Release: "1"}},
|
||||
Provides: []provider.RPMDep{{Name: "full", Flags: "EQ", Version: "2.0"}},
|
||||
Files: []provider.RPMFile{{Path: "/usr/bin/full", Type: "file"}, {Path: "/etc/full", Type: "dir"}},
|
||||
Changelogs: []provider.RPMChangelog{{Author: "a", Date: 100, Text: "changed"}},
|
||||
}}
|
||||
for _, gen := range []func([]provider.RPMMetadata) []byte{generatePrimaryXMLGZ, generateFilelistsXMLGZ, generateOtherXMLGZ} {
|
||||
zr, err := gzip.NewReader(bytes.NewReader(gen(metas)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := io.ReadAll(zr); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMPrimaryXMLContents(t *testing.T) {
|
||||
// Exercise xmlEscape and dependency entry writing through the gzip'd XML.
|
||||
metas := []provider.RPMMetadata{{
|
||||
Name: "pkg", Version: "1", Release: "1", Arch: "x86_64", Summary: "a & b",
|
||||
Requires: []provider.RPMDep{{Name: "dep", Flags: "EQ", Version: "1.0", Epoch: "0"}},
|
||||
}}
|
||||
gz := generatePrimaryXMLGZ(metas)
|
||||
zr, err := gzip.NewReader(bytes.NewReader(gz))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, _ := io.ReadAll(zr)
|
||||
s := string(out)
|
||||
if !strings.Contains(s, "a & b") {
|
||||
t.Errorf("summary not xml-escaped: %s", s)
|
||||
}
|
||||
if !strings.Contains(s, "<name>pkg</name>") {
|
||||
t.Errorf("package name missing: %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMContentTypeAndHelpers(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for path, want := range map[string]string{
|
||||
"x.rpm": "application/x-rpm",
|
||||
"repodata/repomd.xml": "application/xml",
|
||||
"repodata/h-primary.xml.gz": "application/xml",
|
||||
"repodata/h-primary.xml.xz": "application/xml",
|
||||
"Packages/other": "application/octet-stream",
|
||||
} {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
for flag, want := range map[int]string{
|
||||
0x08 | 0x04: "GE",
|
||||
0x02 | 0x04: "LE",
|
||||
0x08: "GT",
|
||||
0x02: "LT",
|
||||
0x04: "EQ",
|
||||
0x00: "",
|
||||
} {
|
||||
if got := rpmFlagString(flag); got != want {
|
||||
t.Errorf("rpmFlagString(%d)=%q want %q", flag, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
if firstGroup(nil) != "Unspecified" {
|
||||
t.Error("empty groups should be Unspecified")
|
||||
}
|
||||
if firstGroup([]string{"System", "Base"}) != "System" {
|
||||
t.Error("firstGroup should return the first")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateLocalIndexUnsupported(t *testing.T) {
|
||||
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeRPMReader{}, "r", "simple/"); err == nil {
|
||||
t.Error("expected unsupported error")
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package rpm_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &rpm.Provider{}
|
||||
if p.Type() != models.PackageRPM {
|
||||
t.Errorf("expected rpm, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &rpm.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"repomd.xml", provider.Mutable},
|
||||
{"repodata/primary.xml.gz", provider.Mutable},
|
||||
{"Packages.gz", provider.Mutable},
|
||||
{"package-1.0.rpm", provider.Immutable},
|
||||
{"RPM-GPG-KEY-almalinux", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,258 +0,0 @@
|
||||
package terraform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var versionsRe = regexp.MustCompile(`[^/]+/[^/]+/versions$`)
|
||||
|
||||
var providerZipRe = regexp.MustCompile(
|
||||
`^terraform-provider-([a-zA-Z0-9_-]+)_([0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`,
|
||||
)
|
||||
|
||||
var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`)
|
||||
|
||||
// ParsedProviderZip describes a terraform-provider-{type}_{version}_{os}_{arch}.zip
|
||||
// filename. Ok is false when the name doesn't match that convention.
|
||||
type ParsedProviderZip struct {
|
||||
Type string
|
||||
Version string
|
||||
OS string
|
||||
Arch string
|
||||
Ok bool
|
||||
}
|
||||
|
||||
// ParseProviderZip extracts the type, version and platform from a provider zip
|
||||
// filename (the base name, not a full path). It's the canonical parser shared by
|
||||
// the network-mirror index and the provider registry handler.
|
||||
func ParseProviderZip(filename string) ParsedProviderZip {
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil {
|
||||
return ParsedProviderZip{}
|
||||
}
|
||||
return ParsedProviderZip{Type: m[1], Version: m[2], OS: m[3], Arch: m[4], Ok: true}
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageTerraform }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if versionsRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
lower := strings.ToLower(path)
|
||||
if strings.HasSuffix(lower, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".sig") {
|
||||
return "application/octet-stream"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/v1/providers/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if remote.ReleasesRemote == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if !json.Valid(body) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var data map[string]any
|
||||
if err := json.Unmarshal(body, &data); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
changed := false
|
||||
for _, field := range []string{"download_url", "shasums_url", "shasums_signature_url"} {
|
||||
if val, ok := data[field].(string); ok && val != "" {
|
||||
rewritten := rewriteDownloadURL(val, remote.ReleasesRemote, proxyBaseURL)
|
||||
if rewritten != val {
|
||||
data[field] = rewritten
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return nil, nil
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
|
||||
func rewriteDownloadURL(originalURL, releasesRemote, proxyBaseURL string) string {
|
||||
parsed, err := url.Parse(originalURL)
|
||||
if err != nil || proxyBaseURL == "" {
|
||||
return originalURL
|
||||
}
|
||||
return strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + releasesRemote + parsed.Path
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
parts := strings.Split(filePath, "/")
|
||||
if len(parts) != 3 {
|
||||
return "", "", fmt.Errorf("path must be {namespace}/{type}/{filename}.zip")
|
||||
}
|
||||
namespace, typeName, filename := parts[0], parts[1], parts[2]
|
||||
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil {
|
||||
return "", "", fmt.Errorf("filename %q does not match terraform-provider-{type}_{version}_{os}_{arch}.zip", filename)
|
||||
}
|
||||
|
||||
if m[1] != typeName {
|
||||
return "", "", fmt.Errorf("provider type in filename %q does not match path type %q", m[1], typeName)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s/%s/%s", namespace, typeName, filename), "application/zip", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
parts := strings.Split(storagePath, "/")
|
||||
if len(parts) != 3 {
|
||||
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
|
||||
}
|
||||
|
||||
m := providerZipRe.FindStringSubmatch(parts[2])
|
||||
if m == nil {
|
||||
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"namespace": parts[0],
|
||||
"type": parts[1],
|
||||
"version": m[2],
|
||||
"os": m[3],
|
||||
"arch": m[4],
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
type terraformIndex struct {
|
||||
Versions map[string]json.RawMessage `json:"versions"`
|
||||
}
|
||||
|
||||
type terraformVersionDoc struct {
|
||||
Archives map[string]terraformArchive `json:"archives"`
|
||||
}
|
||||
|
||||
type terraformArchive struct {
|
||||
URL string `json:"url"`
|
||||
Hashes []string `json:"hashes,omitempty"`
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) < 3 {
|
||||
return false
|
||||
}
|
||||
|
||||
namespace, typeName := parts[0], parts[1]
|
||||
tail := parts[2]
|
||||
|
||||
if tail == "index.json" {
|
||||
p.serveIndex(w, r, files, repoName, namespace, typeName)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasSuffix(tail, ".json") {
|
||||
version := strings.TrimSuffix(tail, ".json")
|
||||
if semverRe.MatchString(version) {
|
||||
p.serveVersionDoc(w, r, files, repoName, namespace, typeName, version)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("terraform local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
func (p *Provider) serveIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName string) {
|
||||
prefix := fmt.Sprintf("%s/%s/", namespace, typeName)
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
versions := map[string]json.RawMessage{}
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, prefix)
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
versions[m[2]] = json.RawMessage(`{}`)
|
||||
}
|
||||
|
||||
if len(versions) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(terraformIndex{Versions: versions})
|
||||
}
|
||||
|
||||
func (p *Provider) serveVersionDoc(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName, version string) {
|
||||
prefix := fmt.Sprintf("%s/%s/terraform-provider-%s_%s_", namespace, typeName, typeName, version)
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
archives := map[string]terraformArchive{}
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, fmt.Sprintf("%s/%s/", namespace, typeName))
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil || m[2] != version {
|
||||
continue
|
||||
}
|
||||
platform := m[3] + "_" + m[4]
|
||||
archive := terraformArchive{URL: filename}
|
||||
if f.ContentHash != "" {
|
||||
archive.Hashes = []string{"zh:" + strings.TrimPrefix(f.ContentHash, "sha256:")}
|
||||
}
|
||||
archives[platform] = archive
|
||||
}
|
||||
|
||||
if len(archives) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(terraformVersionDoc{Archives: archives})
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
package terraform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type fakeFileStore struct{ entries []provider.FileEntry }
|
||||
|
||||
func (f fakeFileStore) ListFilesByPrefix(_ context.Context, _, prefix string) ([]provider.FileEntry, error) {
|
||||
var out []provider.FileEntry
|
||||
for _, e := range f.entries {
|
||||
if strings.HasPrefix(e.FilePath, prefix) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (f fakeFileStore) ListPackages(_ context.Context, _ string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestTFPureFuncs(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("hashicorp/aws/versions") != provider.Mutable {
|
||||
t.Error("versions should be mutable")
|
||||
}
|
||||
if p.Classify("hashicorp/aws/terraform-provider-aws_1.0.0_linux_amd64.zip") != provider.Immutable {
|
||||
t.Error("zip should be immutable")
|
||||
}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://registry.terraform.io"}, "hashicorp/aws/versions"); got != "https://registry.terraform.io/v1/providers/hashicorp/aws/versions" {
|
||||
t.Errorf("upstream url %q", got)
|
||||
}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("auth header")
|
||||
}
|
||||
_ = p.ContentType("x.json")
|
||||
}
|
||||
|
||||
func TestTFValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("hashicorp/aws/terraform-provider-aws_1.2.3_linux_amd64.zip")
|
||||
if err != nil || sp != "hashicorp/aws/terraform-provider-aws_1.2.3_linux_amd64.zip" || ct != "application/zip" {
|
||||
t.Errorf("valid: sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("too/few"); err == nil {
|
||||
t.Error("expected error for wrong path depth")
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("ns/aws/not-a-provider.zip"); err == nil {
|
||||
t.Error("expected error for bad filename")
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("ns/gcp/terraform-provider-aws_1.0.0_linux_amd64.zip"); err == nil {
|
||||
t.Error("expected error for type mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTFUploadResponse(t *testing.T) {
|
||||
p := &Provider{}
|
||||
resp := p.UploadResponse("hashicorp/aws/terraform-provider-aws_1.2.3_linux_amd64.zip", "sha256:abc", 100)
|
||||
if resp["namespace"] != "hashicorp" || resp["type"] != "aws" || resp["version"] != "1.2.3" || resp["os"] != "linux" || resp["arch"] != "amd64" {
|
||||
t.Errorf("structured response wrong: %v", resp)
|
||||
}
|
||||
fallback := p.UploadResponse("weird/path", "sha256:x", 1)
|
||||
if fallback["path"] != "weird/path" {
|
||||
t.Errorf("fallback response wrong: %v", fallback)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTFRewriteResponse(t *testing.T) {
|
||||
p := &Provider{}
|
||||
remote := models.Remote{Name: "tf", ReleasesRemote: "hashicorp-releases"}
|
||||
|
||||
if out, _ := p.RewriteResponse([]byte(`{"download_url":"x"}`), models.Remote{}, "http://proxy"); out != nil {
|
||||
t.Error("no ReleasesRemote should be a no-op")
|
||||
}
|
||||
if out, _ := p.RewriteResponse([]byte("not json"), remote, "http://proxy"); out != nil {
|
||||
t.Error("invalid json should be a no-op")
|
||||
}
|
||||
body := []byte(`{"download_url":"https://releases.hashicorp.com/terraform-provider-aws/1.0/aws.zip"}`)
|
||||
out, err := p.RewriteResponse(body, remote, "http://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(out), "http://proxy/api/v1/remote/hashicorp-releases/") {
|
||||
t.Errorf("download_url not rewritten: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTFServeLocalIndex(t *testing.T) {
|
||||
p := &Provider{}
|
||||
fs := fakeFileStore{entries: []provider.FileEntry{
|
||||
{FilePath: "hashicorp/aws/terraform-provider-aws_1.0.0_linux_amd64.zip", ContentHash: "sha256:deadbeef"},
|
||||
{FilePath: "hashicorp/aws/terraform-provider-aws_1.0.0_darwin_arm64.zip", ContentHash: "sha256:cafe"},
|
||||
}}
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
p.ServeLocalIndex(w, r, fs, "repo", path)
|
||||
return w
|
||||
}
|
||||
|
||||
if w := serve("hashicorp/aws/index.json"); w.Code != 200 || !strings.Contains(w.Body.String(), "1.0.0") {
|
||||
t.Errorf("index.json: code=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if w := serve("hashicorp/aws/1.0.0.json"); w.Code != 200 || !strings.Contains(w.Body.String(), "linux_amd64") {
|
||||
t.Errorf("version doc: code=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
// Not a terraform index path.
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if p.ServeLocalIndex(w, r, fs, "repo", "hashicorp/aws/other.txt") {
|
||||
t.Error("non-index path should return false")
|
||||
}
|
||||
if p.ServeLocalIndex(httptest.NewRecorder(), r, fs, "repo", "too/short") {
|
||||
t.Error("short path should return false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTFContentTypeAndEmptyIndex(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for path, want := range map[string]string{
|
||||
"x.zip": "application/zip",
|
||||
"x.sig": "application/octet-stream",
|
||||
"index.json": "application/json",
|
||||
} {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
// index / version doc with no matching files -> 404.
|
||||
empty := fakeFileStore{}
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/hashicorp/aws/index.json", nil)
|
||||
p.ServeLocalIndex(w, r, empty, "repo", "hashicorp/aws/index.json")
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("empty index should be 404, got %d", w.Code)
|
||||
}
|
||||
w = httptest.NewRecorder()
|
||||
p.ServeLocalIndex(w, r, empty, "repo", "hashicorp/aws/1.0.0.json")
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("empty version doc should be 404, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteDownloadURL(t *testing.T) {
|
||||
// Empty proxy base -> unchanged.
|
||||
if got := rewriteDownloadURL("https://x/a.zip", "rel", ""); got != "https://x/a.zip" {
|
||||
t.Errorf("empty base: %q", got)
|
||||
}
|
||||
// Unparseable URL -> unchanged.
|
||||
if got := rewriteDownloadURL("://bad", "rel", "http://p"); got != "://bad" {
|
||||
t.Errorf("bad url: %q", got)
|
||||
}
|
||||
// Normal rewrite.
|
||||
if got := rewriteDownloadURL("https://cdn/path/a.zip", "rel", "http://p"); got != "http://p/api/v1/remote/rel/path/a.zip" {
|
||||
t.Errorf("rewrite: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTFGenerateLocalIndexUnsupported(t *testing.T) {
|
||||
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeFileStore{}, "r", "x"); err == nil {
|
||||
t.Error("expected unsupported error")
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package terraform_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
if p.Type() != models.PackageTerraform {
|
||||
t.Errorf("expected terraform, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"hashicorp/vault/versions", provider.Mutable},
|
||||
{"hashicorp/vault/0.28.0/download/linux/amd64", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_RewriteResponse_DownloadInfo(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
remote := models.Remote{Name: "tf", ReleasesRemote: "hashicorp-releases"}
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"download_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/file.zip",
|
||||
"shasums_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/SHA256SUMS",
|
||||
})
|
||||
rewritten, err := p.RewriteResponse(body, remote, "https://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rewritten == nil {
|
||||
t.Fatal("expected rewrite")
|
||||
}
|
||||
var result map[string]any
|
||||
json.Unmarshal(rewritten, &result)
|
||||
if !strings.Contains(result["download_url"].(string), "proxy/api/v1/remote/hashicorp-releases") {
|
||||
t.Errorf("download_url not rewritten: %s", result["download_url"])
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultCircuitThreshold = 5
|
||||
defaultCircuitCooldown = 60 * time.Second
|
||||
)
|
||||
|
||||
type CircuitBreaker struct {
|
||||
cache *cache.Redis
|
||||
threshold int64
|
||||
cooldown time.Duration
|
||||
}
|
||||
|
||||
func NewCircuitBreaker(c *cache.Redis) *CircuitBreaker {
|
||||
return &CircuitBreaker{
|
||||
cache: c,
|
||||
threshold: defaultCircuitThreshold,
|
||||
cooldown: defaultCircuitCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) IsOpen(ctx context.Context, remote string) bool {
|
||||
failures, err := cb.cache.GetCircuitFailures(ctx, remote)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return failures >= cb.threshold
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) RecordFailure(ctx context.Context, remote string) {
|
||||
cb.cache.IncrCircuitFailure(ctx, remote, cb.cooldown)
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) RecordSuccess(ctx context.Context, remote string) {
|
||||
cb.cache.ResetCircuit(ctx, remote)
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) Health(ctx context.Context, remote string) models.RemoteHealth {
|
||||
failures, err := cb.cache.GetCircuitFailures(ctx, remote)
|
||||
if err != nil {
|
||||
return models.RemoteHealth{Status: "unknown"}
|
||||
}
|
||||
|
||||
switch {
|
||||
case failures == 0:
|
||||
return models.RemoteHealth{Status: "healthy", ConsecutiveFailures: int(failures)}
|
||||
case failures < cb.threshold:
|
||||
return models.RemoteHealth{Status: "degraded", ConsecutiveFailures: int(failures)}
|
||||
default:
|
||||
return models.RemoteHealth{Status: "down", ConsecutiveFailures: int(failures)}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user