Compare commits

..

2 Commits

Author SHA1 Message Date
benvin 756846c0ba Merge branch 'master' into benvin/local-terraform-registry 2026-06-22 23:32:12 +10:00
unkinben ab44271e82 feat: add local repository type with repo_type field
Introduces repo_type (remote/local) as a separate axis from package_type
so that any package type can be hosted locally. A terraform local repo
is package_type=terraform + repo_type=local.

- Remote model gains RepoType field (defaults to "remote")
- Database schema adds repo_type column with migration for existing DBs
- V1 proxy adds /api/v1/local/{name}/* route for serving local files
- V2 upload via PUT /api/v2/remotes/{name}/files/{ns}/{type}/{file}.zip
  validates filename matches terraform-provider-{type}_{ver}_{os}_{arch}.zip
  and returns 409 on duplicate (no overwrites)
- index.json and {version}.json are computed on-the-fly from uploaded zips
  rather than stored as separate files
- V2 create validates repo_type and requires base_url only for remotes
2026-06-22 22:51:41 +10:00
108 changed files with 332 additions and 7750 deletions
+1 -5
View File
@@ -1,6 +1,2 @@
bin/ bin/
/terraform/ terraform/
# e2e-docker fixtures are real package files (.rpm, .tgz, .whl, .zip, ...) that
# are intentionally tracked, overriding any global ignore of those extensions.
!e2e-docker/fixtures/**
-24
View File
@@ -1,24 +0,0 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
- id: check-merge-conflict
- repo: https://github.com/dnephin/pre-commit-golang
rev: v0.5.1
hooks:
- id: go-fmt
- id: go-mod-tidy
- repo: local
hooks:
- id: go-vet
name: go vet
entry: go vet ./...
language: system
types: [go]
pass_filenames: false
-4
View File
@@ -8,8 +8,6 @@ steps:
settings: settings:
registry: git.unkin.net registry: git.unkin.net
repo: git.unkin.net/unkin/artifactapi repo: git.unkin.net/unkin/artifactapi
build_args:
VERSION: ${CI_COMMIT_TAG}
username: droneci username: droneci
password: password:
from_secret: DRONECI_PASSWORD from_secret: DRONECI_PASSWORD
@@ -24,8 +22,6 @@ steps:
repo: git.unkin.net/unkin/artifactapi-ui repo: git.unkin.net/unkin/artifactapi-ui
dockerfile: ui/Dockerfile.ui dockerfile: ui/Dockerfile.ui
context: ui context: ui
build_args:
BASE_PATH: /ui
username: droneci username: droneci
password: password:
from_secret: DRONECI_PASSWORD from_secret: DRONECI_PASSWORD
+3 -11
View File
@@ -3,15 +3,7 @@ when:
steps: steps:
- name: pre-commit - name: pre-commit
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606 image: golang:1.25
commands: commands:
- uvx pre-commit run --all-files - test -z "$(gofmt -l .)"
backend_options: - go vet ./...
kubernetes:
resources:
requests:
memory: 512Mi
cpu: 1
limits:
memory: 2Gi
cpu: 2
+1 -2
View File
@@ -9,8 +9,7 @@ RUN go mod download
COPY . . COPY . .
ARG VERSION=dev RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o artifactapi ./cmd/artifactapi
RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=${VERSION}" -o artifactapi ./cmd/artifactapi
FROM gcr.io/distroless/static-debian12:nonroot FROM gcr.io/distroless/static-debian12:nonroot
+2 -7
View File
@@ -1,4 +1,4 @@
.PHONY: build test lint fmt e2e docker-e2e docker docker-ui compose clean tidy check-go .PHONY: build test lint fmt e2e docker docker-ui compose clean tidy check-go
BINARY := bin/artifactapi BINARY := bin/artifactapi
MODULE := git.unkin.net/unkin/artifactapi MODULE := git.unkin.net/unkin/artifactapi
@@ -12,7 +12,7 @@ check-go:
fi fi
build: check-go tidy build: check-go tidy
go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(BINARY) ./cmd/artifactapi go build -ldflags="-s -w" -o $(BINARY) ./cmd/artifactapi
test: check-go test: check-go
go test -race -count=1 ./pkg/... ./internal/... go test -race -count=1 ./pkg/... ./internal/...
@@ -28,11 +28,6 @@ fmt: check-go
e2e: check-go e2e: check-go
TESTCONTAINERS_RYUK_DISABLED=true go test -tags=e2e -race -count=1 -timeout=5m ./e2e/... TESTCONTAINERS_RYUK_DISABLED=true go test -tags=e2e -race -count=1 -timeout=5m ./e2e/...
# Build the container, bring up the full docker-compose stack + a mock upstream,
# and run the black-box suite against the running product.
docker-e2e: check-go
./scripts/docker-e2e.sh
docker: docker:
docker build -t artifactapi:$(VERSION) . docker build -t artifactapi:$(VERSION) .
-30
View File
@@ -89,36 +89,6 @@ resource "artifactapi_virtual" "helm" {
Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi) Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi)
### Serving providers as a registry
A local `terraform` repo is a real provider registry: upload
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
no `.terraformrc` mirror config:
```hcl
terraform {
required_providers {
artifactapi = {
source = "artifactapi.k8s.syd1.au.unkin.net/<repo>/<type>"
version = "0.1.2"
}
}
}
```
The Terraform *namespace* segment is the artifactapi repo name; the provider is
matched by *type*. The registry serves service discovery
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
Signing needs a GPG key. By default artifactapi generates one on first start and
stores it in the database (`signing_keys` table), so every replica shares it and
there's nothing to provision. To bring your own key instead, point
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
## Access Control ## Access Control
| Field | Default | Behaviour | | Field | Default | Behaviour |
+1 -3
View File
@@ -13,8 +13,6 @@ import (
"git.unkin.net/unkin/artifactapi/internal/tui" "git.unkin.net/unkin/artifactapi/internal/tui"
) )
var version = "dev"
func main() { func main() {
if len(os.Args) > 1 && os.Args[1] == "tui" { if len(os.Args) > 1 && os.Args[1] == "tui" {
endpoint := os.Getenv("ARTIFACTAPI_ENDPOINT") endpoint := os.Getenv("ARTIFACTAPI_ENDPOINT")
@@ -44,7 +42,7 @@ func main() {
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel() defer cancel()
srv, err := server.New(cfg, version) srv, err := server.New(cfg)
if err != nil { if err != nil {
slog.Error("failed to create server", "error", err) slog.Error("failed to create server", "error", err)
os.Exit(1) os.Exit(1)
-18
View File
@@ -1,18 +0,0 @@
# Overlay for the dockerised end-to-end suite (scripts/docker-e2e.sh).
# Adds a static mock upstream that the artifactapi container proxies, so the
# caching tests are hermetic and need no internet access.
services:
mockupstream:
image: nginx:alpine
volumes:
- ./e2e-docker/fixtures:/usr/share/nginx/html:ro,z
# No host port needed: only the artifactapi container talks to it, and the
# tests compare served bytes against the on-disk fixtures.
artifactapi:
# The host port is set via ARTIFACTAPI_PORT (see scripts/docker-e2e.sh),
# defaulting to 8000; the e2e run uses 8001 to avoid colliding with a
# locally-running instance.
depends_on:
mockupstream:
condition: service_started
+1 -1
View File
@@ -2,7 +2,7 @@ services:
artifactapi: artifactapi:
build: . build: .
ports: ports:
- "${ARTIFACTAPI_PORT:-8000}:8000" - "8000:8000"
environment: environment:
LISTEN_ADDR: ":8000" LISTEN_ADDR: ":8000"
DBHOST: postgres DBHOST: postgres
-39
View File
@@ -1,39 +0,0 @@
# Dockerised end-to-end suite
Black-box tests that run against a fully **containerised** artifactapi stack
(built image + Postgres + Redis + MinIO) plus a static mock upstream. Unlike the
in-process `e2e/` suite (testcontainers, server run in-process), these only speak
HTTP to the running product, so they exercise the shipped container image.
## Run
```bash
make docker-e2e # build image, compose up, run suite, compose down
```
`scripts/docker-e2e.sh` builds and starts `docker-compose.yml` +
`docker-compose.e2e.yml`, waits for `/health`, then runs
`go test -tags=dockere2e ./e2e-docker/...` and tears everything down.
The stack publishes artifactapi on host port **8001** (to avoid colliding with a
local instance on 8000). Override with `ARTIFACTAPI_URL` to point the tests at an
already-running stack.
## Coverage
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
- **Caching** — one immutable artifact per remote package type (generic, docker,
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
verified against the origin fixture.
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
index), rpm (real package + **automatic repodata** generation).
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
across two members.
## Fixtures
`fixtures/` is served by the mock upstream at its web root. Paths mirror each
provider's upstream URL layout (e.g. `v2/...` for docker, `v1/providers/...` for
terraform). The RPM under `fixtures/rpmrepo/Packages/` is a real package so the
rpm provider can parse its metadata for repodata generation.
-76
View File
@@ -1,76 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"bytes"
"fmt"
"net/http"
"testing"
)
// TestCachingPerProvider proxies one immutable artifact for every remote
// package type through the mock upstream and asserts: first fetch is served
// from the remote, the second from cache, and the bytes match the origin.
func TestCachingPerProvider(t *testing.T) {
cases := []struct {
pkgType string
// path is the request path under /api/v1/remote/<name>/. The provider
// derives the upstream URL from it (docker prepends /v2/, terraform
// prepends /v1/providers/), and the fixture lives at that resolved path.
path string
fixture string
}{
{"generic", "blobs/hello.bin", "blobs/hello.bin"},
{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"},
{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"},
{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"},
{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"},
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
}
for _, tc := range cases {
t.Run(tc.pkgType, func(t *testing.T) {
name := "cache-" + tc.pkgType
createRepo(t, fmt.Sprintf(`{
"name": %q,
"package_type": %q,
"repo_type": "remote",
"base_url": %q,
"stale_on_error": true
}`, name, tc.pkgType, mockUpstream()))
defer deleteRepo(t, name)
want := fixtureBytes(t, tc.fixture)
url := api("/api/v1/remote/" + name + "/" + tc.path)
// First fetch: from remote.
resp, body := doRequest(t, http.MethodGet, url, nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("first fetch: status %d: %s", resp.StatusCode, body)
}
if src := resp.Header.Get("X-Artifact-Source"); src != "remote" {
t.Fatalf("first fetch source = %q, want remote", src)
}
if !bytes.Equal(body, want) {
t.Fatalf("first fetch body mismatch: got %d bytes, want %d", len(body), len(want))
}
// Second fetch: from cache, identical bytes.
resp, body = doRequest(t, http.MethodGet, url, nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("second fetch: status %d: %s", resp.StatusCode, body)
}
if src := resp.Header.Get("X-Artifact-Source"); src != "cache" {
t.Fatalf("second fetch source = %q, want cache", src)
}
if !bytes.Equal(body, want) {
t.Fatalf("cached body mismatch: got %d bytes, want %d", len(body), len(want))
}
})
}
}
Binary file not shown.
-1
View File
@@ -1 +0,0 @@
hello artifactapi generic blob
Binary file not shown.
-8
View File
@@ -1,8 +0,0 @@
apiVersion: v1
entries:
alpha:
- name: alpha
version: 1.0.0
urls:
- charts/alpha-1.0.0.tgz
generated: "2026-01-01T00:00:00Z"
-8
View File
@@ -1,8 +0,0 @@
apiVersion: v1
entries:
beta:
- name: beta
version: 2.0.0
urls:
- charts/beta-2.0.0.tgz
generated: "2026-01-01T00:00:00Z"
Binary file not shown.
-108
View File
@@ -1,108 +0,0 @@
//go:build dockere2e
// Package e2edocker holds the black-box end-to-end suite that runs against a
// fully dockerised artifactapi stack (see scripts/docker-e2e.sh). Unlike the
// in-process e2e suite, these tests only speak HTTP to the running container.
package e2edocker
import (
"bytes"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func baseURL() string {
if v := os.Getenv("ARTIFACTAPI_URL"); v != "" {
return strings.TrimRight(v, "/")
}
return "http://localhost:8000"
}
// mockUpstream is the base URL the artifactapi *container* uses to reach the
// static mock upstream. It is resolved on the compose network, not the host.
func mockUpstream() string {
if v := os.Getenv("MOCK_UPSTREAM_INTERNAL"); v != "" {
return strings.TrimRight(v, "/")
}
return "http://mockupstream"
}
func api(path string) string { return baseURL() + path }
func fixtureBytes(t *testing.T, rel string) []byte {
t.Helper()
b, err := os.ReadFile(filepath.Join("fixtures", rel))
if err != nil {
t.Fatalf("read fixture %s: %v", rel, err)
}
return b
}
func doRequest(t *testing.T, method, url string, body []byte, contentType string) (*http.Response, []byte) {
t.Helper()
var r io.Reader
if body != nil {
r = bytes.NewReader(body)
}
req, err := http.NewRequest(method, url, r)
if err != nil {
t.Fatalf("%s %s: %v", method, url, err)
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("%s %s: %v", method, url, err)
}
defer resp.Body.Close()
respBody, _ := io.ReadAll(resp.Body)
return resp, respBody
}
func createRepo(t *testing.T, jsonBody string) {
t.Helper()
resp, body := doRequest(t, http.MethodPost, api("/api/v2/remotes"), []byte(jsonBody), "application/json")
if resp.StatusCode != http.StatusCreated {
t.Fatalf("create repo: status %d: %s", resp.StatusCode, body)
}
}
func deleteRepo(t *testing.T, name string) {
t.Helper()
doRequest(t, http.MethodDelete, api("/api/v2/remotes/"+name), nil, "")
}
func createVirtual(t *testing.T, jsonBody string) {
t.Helper()
resp, body := doRequest(t, http.MethodPost, api("/api/v2/virtuals"), []byte(jsonBody), "application/json")
if resp.StatusCode != http.StatusCreated {
t.Fatalf("create virtual: status %d: %s", resp.StatusCode, body)
}
}
func deleteVirtual(t *testing.T, name string) {
t.Helper()
doRequest(t, http.MethodDelete, api("/api/v2/virtuals/"+name), nil, "")
}
// getEventually retries a GET until it returns 200 or the deadline passes. Used
// for asynchronously-generated artifacts (e.g. rpm repodata after upload).
func getEventually(t *testing.T, url string, timeout time.Duration) (*http.Response, []byte) {
t.Helper()
deadline := time.Now().Add(timeout)
var resp *http.Response
var body []byte
for {
resp, body = doRequest(t, http.MethodGet, url, nil, "")
if resp.StatusCode == http.StatusOK || time.Now().After(deadline) {
return resp, body
}
time.Sleep(250 * time.Millisecond)
}
}
-93
View File
@@ -1,93 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"bytes"
"net/http"
"strings"
"testing"
"time"
)
func uploadFile(t *testing.T, repo, filePath string, body []byte, contentType string) {
t.Helper()
url := api("/api/v2/remotes/" + repo + "/files/" + filePath)
resp, respBody := doRequest(t, http.MethodPut, url, body, contentType)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload %s: status %d: %s", filePath, resp.StatusCode, respBody)
}
}
// TestLocalGenericUpload uploads a generic file and downloads it back.
func TestLocalGenericUpload(t *testing.T) {
createRepo(t, `{"name":"local-generic","package_type":"generic","repo_type":"local"}`)
defer deleteRepo(t, "local-generic")
content := []byte("artifactapi local generic upload payload")
uploadFile(t, "local-generic", "data/hello.bin", content, "application/octet-stream")
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-generic/data/hello.bin"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("download: status %d: %s", resp.StatusCode, body)
}
if !bytes.Equal(body, content) {
t.Fatalf("downloaded content mismatch")
}
}
// TestLocalPyPIUpload uploads a wheel and validates the generated simple index.
func TestLocalPyPIUpload(t *testing.T) {
createRepo(t, `{"name":"local-pypi","package_type":"pypi","repo_type":"local"}`)
defer deleteRepo(t, "local-pypi")
wheel := fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl")
uploadFile(t, "local-pypi", "foo-1.0-py3-none-any.whl", wheel, "application/zip")
// Root index lists the package.
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("simple index: status %d: %s", resp.StatusCode, body)
}
if !strings.Contains(string(body), "foo") {
t.Fatalf("simple index missing package 'foo': %s", body)
}
// Per-package index lists the wheel file.
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/foo/"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("package index: status %d: %s", resp.StatusCode, body)
}
if !strings.Contains(string(body), "foo-1.0-py3-none-any.whl") {
t.Fatalf("package index missing wheel: %s", body)
}
// The wheel downloads back byte-identical.
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/foo/foo-1.0-py3-none-any.whl"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("download wheel: status %d: %s", resp.StatusCode, body)
}
if !bytes.Equal(body, wheel) {
t.Fatalf("wheel content mismatch")
}
}
// TestLocalRPMRepodata uploads a real RPM and validates that repodata is
// generated automatically (the special rpm-local feature).
func TestLocalRPMRepodata(t *testing.T) {
createRepo(t, `{"name":"local-rpm","package_type":"rpm","repo_type":"local"}`)
defer deleteRepo(t, "local-rpm")
rpm := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
uploadFile(t, "local-rpm", "e2e-testpkg-1.0-1.noarch.rpm", rpm, "application/x-rpm")
// repodata is generated asynchronously after upload; poll for it.
resp, body := getEventually(t, api("/api/v1/local/local-rpm/repodata/repomd.xml"), 15*time.Second)
if resp.StatusCode != http.StatusOK {
t.Fatalf("repomd.xml: status %d: %s", resp.StatusCode, body)
}
s := string(body)
if !strings.Contains(s, "<repomd") || !strings.Contains(s, "primary") {
t.Fatalf("repomd.xml not a valid repodata document: %s", s)
}
}
-134
View File
@@ -1,134 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"encoding/json"
"net/http"
"testing"
)
func TestHealth(t *testing.T) {
resp, body := doRequest(t, http.MethodGet, api("/health"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("health: status %d: %s", resp.StatusCode, body)
}
}
// TestRemoteLifecycle covers add/change/delete for a remote repository.
func TestRemoteLifecycle(t *testing.T) {
createRepo(t, `{
"name": "crud-remote",
"package_type": "generic",
"repo_type": "remote",
"base_url": "https://example.com",
"mutable_ttl": 600,
"stale_on_error": true
}`)
defer deleteRepo(t, "crud-remote")
got := getRepo(t, "crud-remote")
if got["base_url"] != "https://example.com" || got["mutable_ttl"].(float64) != 600 {
t.Fatalf("unexpected created remote: %v", got)
}
// change
resp, body := doRequest(t, http.MethodPut, api("/api/v2/remotes/crud-remote"), []byte(`{
"package_type": "generic",
"base_url": "https://updated.example.com",
"mutable_ttl": 120,
"stale_on_error": true
}`), "application/json")
if resp.StatusCode != http.StatusOK {
t.Fatalf("update remote: status %d: %s", resp.StatusCode, body)
}
got = getRepo(t, "crud-remote")
if got["base_url"] != "https://updated.example.com" || got["mutable_ttl"].(float64) != 120 {
t.Fatalf("update not applied: %v", got)
}
// delete
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-remote"), nil, "")
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("delete remote: status %d", resp.StatusCode)
}
resp, _ = doRequest(t, http.MethodGet, api("/api/v2/remotes/crud-remote"), nil, "")
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("expected 404 after delete, got %d", resp.StatusCode)
}
}
// TestLocalLifecycle covers add/delete for a local repository.
func TestLocalLifecycle(t *testing.T) {
createRepo(t, `{
"name": "crud-local",
"package_type": "generic",
"repo_type": "local"
}`)
defer deleteRepo(t, "crud-local")
got := getRepo(t, "crud-local")
if got["repo_type"] != "local" {
t.Fatalf("expected repo_type local, got %v", got["repo_type"])
}
resp, _ := doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-local"), nil, "")
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("delete local: status %d", resp.StatusCode)
}
}
// TestVirtualLifecycle covers add/change/delete for a virtual repository.
func TestVirtualLifecycle(t *testing.T) {
createRepo(t, `{"name":"vmem-a","package_type":"helm","repo_type":"remote","base_url":"https://a.example.com","stale_on_error":true}`)
createRepo(t, `{"name":"vmem-b","package_type":"helm","repo_type":"remote","base_url":"https://b.example.com","stale_on_error":true}`)
defer deleteRepo(t, "vmem-a")
defer deleteRepo(t, "vmem-b")
createVirtual(t, `{
"name": "crud-virtual",
"package_type": "helm",
"members": ["vmem-a"]
}`)
defer deleteVirtual(t, "crud-virtual")
// change members
resp, body := doRequest(t, http.MethodPut, api("/api/v2/virtuals/crud-virtual"), []byte(`{
"package_type": "helm",
"members": ["vmem-a", "vmem-b"]
}`), "application/json")
if resp.StatusCode != http.StatusOK {
t.Fatalf("update virtual: status %d: %s", resp.StatusCode, body)
}
resp, body = doRequest(t, http.MethodGet, api("/api/v2/virtuals/crud-virtual"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("get virtual: status %d: %s", resp.StatusCode, body)
}
var v map[string]any
if err := json.Unmarshal(body, &v); err != nil {
t.Fatalf("decode virtual: %v", err)
}
members, _ := v["members"].([]any)
if len(members) != 2 {
t.Fatalf("expected 2 members after update, got %v", v["members"])
}
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/virtuals/crud-virtual"), nil, "")
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("delete virtual: status %d", resp.StatusCode)
}
}
func getRepo(t *testing.T, name string) map[string]any {
t.Helper()
resp, body := doRequest(t, http.MethodGet, api("/api/v2/remotes/"+name), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("get remote %s: status %d: %s", name, resp.StatusCode, body)
}
var m map[string]any
if err := json.Unmarshal(body, &m); err != nil {
t.Fatalf("decode remote %s: %v", name, err)
}
return m
}
-54
View File
@@ -1,54 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"net/http"
"strings"
"testing"
)
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
// checks that a virtual over them serves a merged simple index.
func TestVirtualPyPIMerge(t *testing.T) {
createRepo(t, `{"name":"pmerge-a","package_type":"pypi","repo_type":"local"}`)
createRepo(t, `{"name":"pmerge-b","package_type":"pypi","repo_type":"local"}`)
defer deleteRepo(t, "pmerge-a")
defer deleteRepo(t, "pmerge-b")
uploadFile(t, "pmerge-a", "foo-1.0-py3-none-any.whl", fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl"), "application/zip")
uploadFile(t, "pmerge-b", "bar-2.0-py3-none-any.whl", []byte("bar wheel payload"), "application/zip")
createVirtual(t, `{"name":"pmerge-v","package_type":"pypi","members":["pmerge-a","pmerge-b"]}`)
defer deleteVirtual(t, "pmerge-v")
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/pmerge-v/simple/"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("virtual simple index: status %d: %s", resp.StatusCode, body)
}
s := string(body)
if !strings.Contains(s, "foo") || !strings.Contains(s, "bar") {
t.Fatalf("merged index missing a member package (want foo and bar): %s", s)
}
}
// TestVirtualHelmMerge points two helm remotes at mock index.yaml documents
// with distinct charts and checks the virtual merges both into one index.
func TestVirtualHelmMerge(t *testing.T) {
createRepo(t, `{"name":"hmerge-a","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-a","stale_on_error":true}`)
createRepo(t, `{"name":"hmerge-b","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-b","stale_on_error":true}`)
defer deleteRepo(t, "hmerge-a")
defer deleteRepo(t, "hmerge-b")
createVirtual(t, `{"name":"hmerge-v","package_type":"helm","members":["hmerge-a","hmerge-b"]}`)
defer deleteVirtual(t, "hmerge-v")
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/hmerge-v/index.yaml"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("virtual index.yaml: status %d: %s", resp.StatusCode, body)
}
s := string(body)
if !strings.Contains(s, "alpha") || !strings.Contains(s, "beta") {
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
}
}
+1 -1
View File
@@ -95,7 +95,7 @@ func TestMain(m *testing.M) {
} }
cfg.ListenAddr = "127.0.0.1:0" cfg.ListenAddr = "127.0.0.1:0"
srv, err := server.New(cfg, "e2e-test") srv, err := server.New(cfg)
if err != nil { if err != nil {
log.Fatalf("server: %v", err) log.Fatalf("server: %v", err)
} }
-24
View File
@@ -24,30 +24,6 @@ func TestRoot(t *testing.T) {
} }
} }
func TestRemoteUpstreamTimeouts(t *testing.T) {
createRemote(t, `{
"name": "timeout-test",
"package_type": "generic",
"base_url": "https://example.com",
"stale_on_error": true,
"upstream_dial_timeout": 3,
"upstream_tls_timeout": 4,
"upstream_response_header_timeout": 5
}`)
defer deleteRemote(t, "timeout-test")
remote := getJSON(t, apiURL("/api/v2/remotes/timeout-test"))
for field, want := range map[string]float64{
"upstream_dial_timeout": 3,
"upstream_tls_timeout": 4,
"upstream_response_header_timeout": 5,
} {
if got, _ := remote[field].(float64); got != want {
t.Errorf("%s: got %v, want %v", field, remote[field], want)
}
}
}
func TestRemoteCRUD(t *testing.T) { func TestRemoteCRUD(t *testing.T) {
createRemote(t, `{ createRemote(t, `{
"name": "test-generic", "name": "test-generic",
-33
View File
@@ -24,39 +24,6 @@ func TestProxyBlocklist(t *testing.T) {
assertStatus(t, apiURL("/api/v1/remote/blocklist-test/malware.exe"), http.StatusForbidden) assertStatus(t, apiURL("/api/v1/remote/blocklist-test/malware.exe"), http.StatusForbidden)
} }
func TestProxyHeadBlocklist(t *testing.T) {
createRemote(t, `{
"name": "head-block-test",
"package_type": "generic",
"base_url": "https://example.com",
"blocklist": ["\\.exe$"],
"stale_on_error": true
}`)
defer deleteRemote(t, "head-block-test")
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/head-block-test/malware.exe"), nil)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("HEAD: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusForbidden {
t.Fatalf("HEAD blocklisted path: got %d, want 403", resp.StatusCode)
}
}
func TestProxyHeadUnknownRemote(t *testing.T) {
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/nonexistent/some/path"), nil)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("HEAD: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("HEAD unknown remote: got %d, want 404", resp.StatusCode)
}
}
func TestProxyPatterns(t *testing.T) { func TestProxyPatterns(t *testing.T) {
createRemote(t, `{ createRemote(t, `{
"name": "patterns-test", "name": "patterns-test",
+1 -2
View File
@@ -3,7 +3,6 @@ module git.unkin.net/unkin/artifactapi
go 1.25.9 go 1.25.9
require ( require (
github.com/cavaliergopher/rpm v1.3.0
github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0 github.com/charmbracelet/lipgloss v1.1.0
github.com/go-chi/chi/v5 v5.3.0 github.com/go-chi/chi/v5 v5.3.0
@@ -13,7 +12,6 @@ require (
github.com/testcontainers/testcontainers-go v0.42.0 github.com/testcontainers/testcontainers-go v0.42.0
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
golang.org/x/crypto v0.51.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
) )
@@ -97,6 +95,7 @@ require (
go.opentelemetry.io/otel/trace v1.41.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect
go.uber.org/atomic v1.11.0 // indirect go.uber.org/atomic v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.51.0 // indirect
golang.org/x/net v0.53.0 // indirect golang.org/x/net v0.53.0 // indirect
golang.org/x/sync v0.20.0 // indirect golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.44.0 // indirect golang.org/x/sys v0.44.0 // indirect
-2
View File
@@ -12,8 +12,6 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cavaliergopher/rpm v1.3.0 h1:UHX46sasX8MesUXXQ+UbkFLUX4eUWTlEcX8jcnRBIgI=
github.com/cavaliergopher/rpm v1.3.0/go.mod h1:vEumo1vvtrHM1Ov86f6+k8j7zNKOxQfHDCAIcR/36ZI=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-301
View File
@@ -1,301 +0,0 @@
// Package terraform serves local terraform repos as a real Terraform provider
// registry: service discovery, version listing, and GPG-signed downloads, so
// `terraform init` installs from a bare source address with no client config.
package terraform
import (
"encoding/json"
"fmt"
"net/http"
"path"
"sort"
"strings"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
tfprov "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
"git.unkin.net/unkin/artifactapi/internal/tfsign"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// ProvidersV1Path is the base the service-discovery document advertises (Terraform
// appends "{namespace}/{type}/versions" etc). MountPath is the same prefix without
// the trailing slash, for chi.Mount.
const (
ProvidersV1Path = "/terraform/v1/providers/"
MountPath = "/terraform/v1/providers"
)
type Handler struct {
db *database.DB
signer *tfsign.Signer
protocols []string
}
func NewHandler(db *database.DB, signer *tfsign.Signer, protocols string) *Handler {
var protos []string
for _, p := range strings.Split(protocols, ",") {
if p = strings.TrimSpace(p); p != "" {
protos = append(protos, p)
}
}
if len(protos) == 0 {
protos = []string{"5.0", "6.0"}
}
return &Handler{db: db, signer: signer, protocols: protos}
}
// Enabled reports whether a signing key is configured. Without one the registry
// cannot produce the signed SHA256SUMS the protocol requires, so it stays off.
func (h *Handler) Enabled() bool { return h.signer != nil }
func (h *Handler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/{namespace}/{type}/versions", h.versions)
r.Get("/{namespace}/{type}/{version}/download/{os}/{arch}", h.download)
r.Get("/{namespace}/{type}/{version}/sha256sums", h.sha256sums)
r.Get("/{namespace}/{type}/{version}/sha256sums.sig", h.sha256sumsSig)
return r
}
// ServiceDiscovery answers /.well-known/terraform.json, pointing Terraform at the
// providers.v1 protocol base.
func (h *Handler) ServiceDiscovery(w http.ResponseWriter, r *http.Request) {
if !h.Enabled() {
http.NotFound(w, r)
return
}
writeJSON(w, map[string]string{"providers.v1": ProvidersV1Path})
}
// providerFile is one resolved platform artifact within a repo.
type providerFile struct {
version string
os string
arch string
filePath string // path within the repo, e.g. unkin/artifactapi/...zip
sha256 string // hex, no "sha256:" prefix
}
// resolve finds every provider zip of the given type in the repo (namespace).
// The Terraform source namespace maps to the artifactapi repo name; the provider
// is matched by type across whatever in-repo folder it was uploaded under.
func (h *Handler) resolve(r *http.Request, namespace, typeName string) ([]providerFile, error) {
remote, err := h.db.GetRemote(r.Context(), namespace)
if err != nil || remote.PackageType != models.PackageTerraform {
return nil, nil
}
rows, err := h.db.ListLocalFiles(r.Context(), namespace, 10000, 0)
if err != nil {
return nil, err
}
var out []providerFile
for _, row := range rows {
parsed := tfprov.ParseProviderZip(path.Base(row.FilePath))
if !parsed.Ok || parsed.Type != typeName {
continue
}
out = append(out, providerFile{
version: parsed.Version,
os: parsed.OS,
arch: parsed.Arch,
filePath: row.FilePath,
sha256: strings.TrimPrefix(row.ContentHash, "sha256:"),
})
}
return out, nil
}
func (h *Handler) versions(w http.ResponseWriter, r *http.Request) {
if !h.Enabled() {
http.NotFound(w, r)
return
}
namespace := chi.URLParam(r, "namespace")
typeName := chi.URLParam(r, "type")
files, err := h.resolve(r, namespace, typeName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if len(files) == 0 {
http.NotFound(w, r)
return
}
// Group platforms by version, de-duplicated and stably ordered.
type platform struct {
OS string `json:"os"`
Arch string `json:"arch"`
}
platforms := map[string]map[string]platform{}
for _, f := range files {
if platforms[f.version] == nil {
platforms[f.version] = map[string]platform{}
}
platforms[f.version][f.os+"_"+f.arch] = platform{OS: f.os, Arch: f.arch}
}
type versionEntry struct {
Version string `json:"version"`
Protocols []string `json:"protocols"`
Platforms []platform `json:"platforms"`
}
out := struct {
Versions []versionEntry `json:"versions"`
}{}
for version, plats := range platforms {
entry := versionEntry{Version: version, Protocols: h.protocols}
for _, p := range plats {
entry.Platforms = append(entry.Platforms, p)
}
sort.Slice(entry.Platforms, func(i, j int) bool {
return entry.Platforms[i].OS+entry.Platforms[i].Arch < entry.Platforms[j].OS+entry.Platforms[j].Arch
})
out.Versions = append(out.Versions, entry)
}
sort.Slice(out.Versions, func(i, j int) bool { return out.Versions[i].Version < out.Versions[j].Version })
writeJSON(w, out)
}
func (h *Handler) download(w http.ResponseWriter, r *http.Request) {
if !h.Enabled() {
http.NotFound(w, r)
return
}
namespace := chi.URLParam(r, "namespace")
typeName := chi.URLParam(r, "type")
version := chi.URLParam(r, "version")
osName := chi.URLParam(r, "os")
arch := chi.URLParam(r, "arch")
files, err := h.resolve(r, namespace, typeName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var match *providerFile
for i := range files {
if files[i].version == version && files[i].os == osName && files[i].arch == arch {
match = &files[i]
break
}
}
if match == nil {
http.NotFound(w, r)
return
}
base := baseURL(r)
verBase := fmt.Sprintf("%s%s/%s/%s", base+ProvidersV1Path, namespace, typeName, version)
type gpgKey struct {
KeyID string `json:"key_id"`
ASCIIArmor string `json:"ascii_armor"`
}
resp := struct {
Protocols []string `json:"protocols"`
OS string `json:"os"`
Arch string `json:"arch"`
Filename string `json:"filename"`
DownloadURL string `json:"download_url"`
SHASumsURL string `json:"shasums_url"`
SHASumsSignatureURL string `json:"shasums_signature_url"`
SHASum string `json:"shasum"`
SigningKeys struct {
GPGPublicKeys []gpgKey `json:"gpg_public_keys"`
} `json:"signing_keys"`
}{
Protocols: h.protocols,
OS: match.os,
Arch: match.arch,
Filename: path.Base(match.filePath),
DownloadURL: fmt.Sprintf("%s/api/v1/local/%s/%s", base, namespace, match.filePath),
SHASumsURL: verBase + "/sha256sums",
SHASumsSignatureURL: verBase + "/sha256sums.sig",
SHASum: match.sha256,
}
resp.SigningKeys.GPGPublicKeys = []gpgKey{{
KeyID: h.signer.KeyID(),
ASCIIArmor: h.signer.PublicKeyArmor(),
}}
writeJSON(w, resp)
}
func (h *Handler) sha256sums(w http.ResponseWriter, r *http.Request) {
sums, ok := h.buildSums(w, r)
if !ok {
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Write(sums)
}
func (h *Handler) sha256sumsSig(w http.ResponseWriter, r *http.Request) {
sums, ok := h.buildSums(w, r)
if !ok {
return
}
sig, err := h.signer.Sign(sums)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
w.Write(sig)
}
// buildSums renders the SHA256SUMS body for one version: one "<hex> <filename>"
// line per platform zip, sorted by filename so the signed bytes are stable.
func (h *Handler) buildSums(w http.ResponseWriter, r *http.Request) ([]byte, bool) {
if !h.Enabled() {
http.NotFound(w, r)
return nil, false
}
namespace := chi.URLParam(r, "namespace")
typeName := chi.URLParam(r, "type")
version := chi.URLParam(r, "version")
files, err := h.resolve(r, namespace, typeName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return nil, false
}
var lines []string
for _, f := range files {
if f.version != version {
continue
}
lines = append(lines, fmt.Sprintf("%s %s", f.sha256, path.Base(f.filePath)))
}
if len(lines) == 0 {
http.NotFound(w, r)
return nil, false
}
sort.Strings(lines)
return []byte(strings.Join(lines, "\n") + "\n"), true
}
func writeJSON(w http.ResponseWriter, v any) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(v)
}
func baseURL(r *http.Request) string {
scheme := "http"
if r.TLS != nil {
scheme = "https"
}
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
scheme = fwd
}
return scheme + "://" + r.Host
}
-186
View File
@@ -1,186 +0,0 @@
package terraform
import (
"bytes"
"context"
"encoding/json"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/go-chi/chi/v5"
"golang.org/x/crypto/openpgp"
"golang.org/x/crypto/openpgp/armor"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/internal/tfsign"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var testDSN string
func TestMain(m *testing.M) {
ctx := context.Background()
dsn, terminate, err := testsupport.StartPostgres(ctx)
if err != nil {
os.Exit(m.Run())
}
testDSN = dsn
code := m.Run()
terminate()
os.Exit(code)
}
// testSigner writes a throwaway armored key and loads it.
func testSigner(t *testing.T) *tfsign.Signer {
t.Helper()
e, err := openpgp.NewEntity("artifactapi test", "tf", "tf@example.com", nil)
if err != nil {
t.Fatal(err)
}
var buf bytes.Buffer
w, _ := armor.Encode(&buf, openpgp.PrivateKeyType, nil)
if err := e.SerializePrivate(w, nil); err != nil {
t.Fatal(err)
}
w.Close()
p := filepath.Join(t.TempDir(), "private-key.asc")
if err := os.WriteFile(p, buf.Bytes(), 0o600); err != nil {
t.Fatal(err)
}
s, err := tfsign.Load(p, "")
if err != nil {
t.Fatal(err)
}
return s
}
func TestProviderRegistryFlow(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
}
ctx := context.Background()
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
defer db.Close()
const repo = "tf-reg" // Terraform namespace == repo name
const filePath = "unkin/artifactapi/terraform-provider-artifactapi_1.2.3_linux_amd64.zip"
const hash = "sha256:983cdb25cb7b976538e4334d26e52dee5f44749b9be1500c760cf5cf66be659b"
const wantSha = "983cdb25cb7b976538e4334d26e52dee5f44749b9be1500c760cf5cf66be659b"
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageTerraform, RepoType: models.RepoTypeLocal}); err != nil {
t.Fatal(err)
}
if err := db.UpsertBlob(ctx, hash, "blobs/98/3c", 6381007, "application/zip"); err != nil {
t.Fatal(err)
}
if err := db.CreateLocalFile(ctx, repo, filePath, hash); err != nil {
t.Fatal(err)
}
signer := testSigner(t)
h := NewHandler(db, signer, "5.0,6.0")
router := chi.NewRouter()
router.Get("/.well-known/terraform.json", h.ServiceDiscovery)
router.Mount(MountPath, h.Routes())
get := func(p string) *httptest.ResponseRecorder {
req := httptest.NewRequest("GET", p, nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
return w
}
// Service discovery.
w := get("/.well-known/terraform.json")
if w.Code != 200 {
t.Fatalf("discovery = %d", w.Code)
}
var disc map[string]string
json.Unmarshal(w.Body.Bytes(), &disc)
if disc["providers.v1"] != ProvidersV1Path {
t.Errorf("providers.v1 = %q", disc["providers.v1"])
}
// Versions.
w = get("/terraform/v1/providers/tf-reg/artifactapi/versions")
if w.Code != 200 {
t.Fatalf("versions = %d %s", w.Code, w.Body)
}
var vresp struct {
Versions []struct {
Version string `json:"version"`
Protocols []string `json:"protocols"`
Platforms []map[string]string `json:"platforms"`
} `json:"versions"`
}
json.Unmarshal(w.Body.Bytes(), &vresp)
if len(vresp.Versions) != 1 || vresp.Versions[0].Version != "1.2.3" {
t.Fatalf("unexpected versions: %+v", vresp)
}
if len(vresp.Versions[0].Platforms) != 1 || vresp.Versions[0].Platforms[0]["os"] != "linux" {
t.Fatalf("unexpected platforms: %+v", vresp.Versions[0].Platforms)
}
// Download.
w = get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/download/linux/amd64")
if w.Code != 200 {
t.Fatalf("download = %d %s", w.Code, w.Body)
}
var dl struct {
Filename string `json:"filename"`
DownloadURL string `json:"download_url"`
SHASumsURL string `json:"shasums_url"`
SHASumsSignatureURL string `json:"shasums_signature_url"`
SHASum string `json:"shasum"`
SigningKeys struct {
GPGPublicKeys []struct {
KeyID string `json:"key_id"`
ASCIIArmor string `json:"ascii_armor"`
} `json:"gpg_public_keys"`
} `json:"signing_keys"`
}
json.Unmarshal(w.Body.Bytes(), &dl)
if dl.SHASum != wantSha {
t.Errorf("shasum = %q", dl.SHASum)
}
wantURL := "http://example.com/api/v1/local/tf-reg/" + filePath
if dl.DownloadURL != wantURL {
t.Errorf("download_url = %q, want %q", dl.DownloadURL, wantURL)
}
if len(dl.SigningKeys.GPGPublicKeys) != 1 || dl.SigningKeys.GPGPublicKeys[0].KeyID != signer.KeyID() {
t.Errorf("signing key mismatch: %+v", dl.SigningKeys)
}
// SHA256SUMS + signature verify against the advertised key.
sums := get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/sha256sums")
wantLine := wantSha + " terraform-provider-artifactapi_1.2.3_linux_amd64.zip\n"
if sums.Body.String() != wantLine {
t.Errorf("sha256sums = %q, want %q", sums.Body.String(), wantLine)
}
sig := get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/sha256sums.sig")
keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewReader([]byte(dl.SigningKeys.GPGPublicKeys[0].ASCIIArmor)))
if err != nil {
t.Fatal(err)
}
if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewReader(sums.Body.Bytes()), bytes.NewReader(sig.Body.Bytes())); err != nil {
t.Errorf("sha256sums.sig did not verify: %v", err)
}
}
func TestRegistryDisabledWithoutSigner(t *testing.T) {
h := NewHandler(nil, nil, "")
router := chi.NewRouter()
router.Get("/.well-known/terraform.json", h.ServiceDiscovery)
req := httptest.NewRequest("GET", "/.well-known/terraform.json", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 404 {
t.Errorf("disabled discovery = %d, want 404", w.Code)
}
}
+33 -54
View File
@@ -6,6 +6,8 @@ import (
"io" "io"
"log/slog" "log/slog"
"net/http" "net/http"
"regexp"
"strings"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
@@ -15,8 +17,11 @@ import (
"git.unkin.net/unkin/artifactapi/internal/proxy" "git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/virtual" "git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
) )
var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`)
type ProxyHandler struct { type ProxyHandler struct {
engine *proxy.Engine engine *proxy.Engine
virtualEngine *virtual.Engine virtualEngine *virtual.Engine
@@ -37,20 +42,6 @@ func (h *ProxyHandler) Routes() chi.Router {
return r return r
} }
func (h *ProxyHandler) DockerV2Routes() chi.Router {
r := chi.NewRouter()
r.Get("/", h.handleDockerPing)
r.Head("/", h.handleDockerPing)
r.Get("/{remoteName}/*", h.handleProxy)
r.Head("/{remoteName}/*", h.handleProxyHead)
return r
}
func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Docker-Distribution-Api-Version", "registry/2.0")
w.WriteHeader(http.StatusOK)
}
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) { func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "remoteName") remoteName := chi.URLParam(r, "remoteName")
path := chi.URLParam(r, "*") path := chi.URLParam(r, "*")
@@ -67,7 +58,7 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
return return
} }
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header) result, err := h.engine.Fetch(r.Context(), *remote, path, prov)
if err != nil { if err != nil {
var proxyErr *proxy.ProxyError var proxyErr *proxy.ProxyError
if errors.As(err, &proxyErr) { if errors.As(err, &proxyErr) {
@@ -89,42 +80,6 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
io.Copy(w, result.Reader) io.Copy(w, result.Reader)
} }
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "remoteName")
path := chi.URLParam(r, "*")
remote, err := h.db.GetRemote(r.Context(), remoteName)
if err != nil {
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
return
}
prov, err := provider.Get(remote.PackageType)
if err != nil {
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
return
}
result, err := h.engine.Head(r.Context(), *remote, path, prov)
if err != nil {
var proxyErr *proxy.ProxyError
if errors.As(err, &proxyErr) {
http.Error(w, proxyErr.Message, proxyErr.Status)
return
}
slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", result.ContentType)
w.Header().Set("X-Artifact-Source", result.Source)
if result.Size > 0 {
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
}
w.WriteHeader(http.StatusOK)
}
func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) { func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
virtualName := chi.URLParam(r, "virtualName") virtualName := chi.URLParam(r, "virtualName")
path := chi.URLParam(r, "*") path := chi.URLParam(r, "*")
@@ -160,9 +115,8 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
return return
} }
prov, _ := provider.Get(remote.PackageType) if remote.PackageType == models.PackageTerraform {
if indexer, ok := prov.(provider.LocalIndexer); ok { if h.serveTerraformMirror(w, r, remote, path) {
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
return return
} }
} }
@@ -170,6 +124,31 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
h.serveLocalFile(w, r, localName, path) h.serveLocalFile(w, r, localName, path)
} }
func (h *ProxyHandler) serveTerraformMirror(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) bool {
parts := strings.Split(path, "/")
if len(parts) < 3 {
return false
}
namespace, typeName := parts[0], parts[1]
tail := parts[2]
if tail == "index.json" {
h.local.ServeTerraformIndex(w, r, remote.Name, namespace, typeName)
return true
}
if strings.HasSuffix(tail, ".json") {
version := strings.TrimSuffix(tail, ".json")
if semverRe.MatchString(version) {
h.local.ServeTerraformVersionDoc(w, r, remote.Name, namespace, typeName, version)
return true
}
}
return false
}
func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, repoName, path string) { func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, repoName, path string) {
file, err := h.db.GetLocalFile(r.Context(), repoName, path) file, err := h.db.GetLocalFile(r.Context(), repoName, path)
if err != nil { if err != nil {
-20
View File
@@ -1,20 +0,0 @@
package v1
import (
"crypto/tls"
"net/http"
"testing"
)
func TestScheme(t *testing.T) {
if got := scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
t.Errorf("TLS request scheme = %q, want https", got)
}
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {"https"}}}
if got := scheme(r); got != "https" {
t.Errorf("X-Forwarded-Proto scheme = %q, want https", got)
}
if got := scheme(&http.Request{Header: http.Header{}}); got != "http" {
t.Errorf("default scheme = %q, want http", got)
}
}
-130
View File
@@ -1,130 +0,0 @@
package v2
import (
"context"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
var testDSN string
func TestMain(m *testing.M) {
ctx := context.Background()
dsn, terminate, err := testsupport.StartPostgres(ctx)
if err != nil {
os.Exit(m.Run())
}
testDSN = dsn
code := m.Run()
terminate()
if code != 0 {
os.Exit(code)
}
}
// closedDB returns a DB whose pool has been closed, so every query fails —
// used to drive the handlers' error branches.
func closedDB(t *testing.T) *database.DB {
t.Helper()
if testDSN == "" {
t.Skip("Docker unavailable")
}
db, err := database.New(testDSN)
if err != nil {
t.Fatalf("new db: %v", err)
}
db.Close()
return db
}
func do(t *testing.T, h http.Handler, method, path, body string) int {
t.Helper()
var r io.Reader
if body != "" {
r = strings.NewReader(body)
}
req := httptest.NewRequest(method, path, r)
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
return w.Code
}
func TestRemotesErrorPaths(t *testing.T) {
h := NewRemotesHandler(closedDB(t)).Routes()
if c := do(t, h, "GET", "/", ""); c != 500 {
t.Errorf("list with dead db = %d, want 500", c)
}
if c := do(t, h, "POST", "/", `{"name":"x","package_type":"generic","repo_type":"remote","base_url":"https://x"}`); c != 500 {
t.Errorf("create with dead db = %d, want 500", c)
}
if c := do(t, h, "PUT", "/x", `{"package_type":"generic","base_url":"https://x"}`); c != 500 {
t.Errorf("update with dead db = %d, want 500", c)
}
if c := do(t, h, "GET", "/x", ""); c != 404 {
t.Errorf("get missing = %d, want 404", c)
}
if c := do(t, h, "DELETE", "/x", ""); c != 500 {
t.Errorf("delete with dead db = %d, want 500", c)
}
// Bad request bodies never reach the db.
if c := do(t, h, "POST", "/", `not json`); c != 400 {
t.Errorf("invalid json = %d, want 400", c)
}
}
func TestVirtualsErrorPaths(t *testing.T) {
h := NewVirtualsHandler(closedDB(t)).Routes()
if c := do(t, h, "GET", "/", ""); c != 500 {
t.Errorf("list = %d, want 500", c)
}
if c := do(t, h, "GET", "/x", ""); c != 404 {
t.Errorf("get missing = %d, want 404", c)
}
if c := do(t, h, "POST", "/", `{"name":"v","package_type":"helm","members":["a"]}`); c != 500 {
t.Errorf("create = %d, want 500", c)
}
if c := do(t, h, "PUT", "/v", `{"package_type":"helm","members":["a"]}`); c != 500 {
t.Errorf("update = %d, want 500", c)
}
if c := do(t, h, "DELETE", "/v", ""); c != 500 {
t.Errorf("delete = %d, want 500", c)
}
}
func TestStatsErrorPaths(t *testing.T) {
h := NewStatsHandler(closedDB(t)).Routes()
for _, p := range []string{"/", "/top-remotes", "/top-files-by-hits", "/top-files-by-bandwidth"} {
if c := do(t, h, "GET", p, ""); c != 500 {
t.Errorf("stats %s = %d, want 500", p, c)
}
}
}
func TestLocalErrorPaths(t *testing.T) {
h := NewLocalHandler(closedDB(t), nil).Routes()
// GetRemote fails on the closed db -> not found.
if c := do(t, h, "PUT", "/x/files/a.bin", "data"); c != 404 {
t.Errorf("upload unknown repo = %d, want 404", c)
}
// download / remove hit the db and 500.
if c := do(t, h, "GET", "/x/files/a.bin", ""); c != 500 {
t.Errorf("download = %d, want 500", c)
}
if c := do(t, h, "DELETE", "/x/files/a.bin", ""); c != 500 {
t.Errorf("remove = %d, want 500", c)
}
}
func TestLocalHandlerDBAccessor(t *testing.T) {
db := closedDB(t)
if NewLocalHandler(db, nil).DB() != db {
t.Error("DB() should return the handler's database")
}
}
+116 -47
View File
@@ -1,20 +1,25 @@
package v2 package v2
import ( import (
"context" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"regexp"
"strings"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database" "git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models" "git.unkin.net/unkin/artifactapi/pkg/models"
) )
var providerZipRe = regexp.MustCompile(
`^terraform-provider-([a-zA-Z0-9_-]+)_([0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`,
)
type LocalHandler struct { type LocalHandler struct {
db *database.DB db *database.DB
store *storage.S3 store *storage.S3
@@ -56,22 +61,41 @@ func (h *LocalHandler) upload(w http.ResponseWriter, r *http.Request) {
return return
} }
prov, _ := provider.Get(remote.PackageType) if remote.PackageType == models.PackageTerraform {
h.uploadTerraformProvider(w, r, remote, filePath)
if uploader, ok := prov.(provider.LocalUploader); ok {
h.uploadValidated(w, r, remote, filePath, prov, uploader)
return return
} }
h.uploadGeneric(w, r, remote, filePath) h.uploadGeneric(w, r, remote, filePath)
} }
func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string, prov provider.Provider, uploader provider.LocalUploader) { func (h *LocalHandler) uploadTerraformProvider(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) {
storagePath, contentType, err := uploader.ValidateUpload(filePath) parts := strings.Split(filePath, "/")
if err != nil { if len(parts) != 3 {
http.Error(w, err.Error(), http.StatusBadRequest) http.Error(w, "path must be {namespace}/{type}/{filename}.zip", http.StatusBadRequest)
return return
} }
namespace, typeName, filename := parts[0], parts[1], parts[2]
m := providerZipRe.FindStringSubmatch(filename)
if m == nil {
http.Error(w, fmt.Sprintf(
"filename %q does not match terraform-provider-{type}_{version}_{os}_{arch}.zip",
filename,
), http.StatusBadRequest)
return
}
fileType, version, os, arch := m[1], m[2], m[3], m[4]
if fileType != typeName {
http.Error(w, fmt.Sprintf(
"provider type in filename %q does not match path type %q",
fileType, typeName,
), http.StatusBadRequest)
return
}
storagePath := fmt.Sprintf("%s/%s/%s", namespace, typeName, filename)
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, storagePath) existing, err := h.db.GetLocalFile(r.Context(), remote.Name, storagePath)
if err != nil { if err != nil {
@@ -79,17 +103,20 @@ func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, r
return return
} }
if existing != nil { if existing != nil {
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict) http.Error(w, fmt.Sprintf(
"provider %s/%s version %s for %s_%s already exists; overwrites are not allowed",
namespace, typeName, version, os, arch,
), http.StatusConflict)
return return
} }
result, err := h.cas.Store(r.Context(), r.Body, contentType) result, err := h.cas.Store(r.Context(), r.Body, "application/zip")
if err != nil { if err != nil {
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError) http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
return return
} }
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil { if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, "application/zip"); err != nil {
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError) http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
return return
} }
@@ -103,11 +130,15 @@ func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, r
return return
} }
if hook, ok := prov.(provider.PostUploadHook); ok { writeJSON(w, http.StatusCreated, map[string]any{
go hook.AfterUpload(context.Background(), remote.Name, storagePath, result.ContentHash, h, h.db) "namespace": namespace,
} "type": typeName,
"version": version,
writeJSON(w, http.StatusCreated, uploader.UploadResponse(storagePath, result.ContentHash, result.SizeBytes)) "os": os,
"arch": arch,
"content_hash": result.ContentHash,
"size_bytes": result.SizeBytes,
})
} }
func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) { func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) {
@@ -185,43 +216,81 @@ func (h *LocalHandler) remove(w http.ResponseWriter, r *http.Request) {
repoName := chi.URLParam(r, "name") repoName := chi.URLParam(r, "name")
filePath := chi.URLParam(r, "*") filePath := chi.URLParam(r, "*")
if err := deleteLocalFile(r.Context(), h.db, repoName, filePath); err != nil { if err := h.db.DeleteLocalFile(r.Context(), repoName, filePath); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
return return
} }
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
} }
// deleteLocalFile removes a local file and runs the provider's post-delete hook, type terraformIndex struct {
// so provider-derived state (e.g. RPM metadata that feeds generated repodata) Versions map[string]json.RawMessage `json:"versions"`
// stops referencing a package that no longer exists.
func deleteLocalFile(ctx context.Context, db *database.DB, repoName, filePath string) error {
if err := db.DeleteLocalFile(ctx, repoName, filePath); err != nil {
return err
}
remote, err := db.GetRemote(ctx, repoName)
if err != nil {
return nil // file is gone; no repo left to resolve a cleanup hook from
}
prov, err := provider.Get(remote.PackageType)
if err != nil {
return nil
}
if hook, ok := prov.(provider.PostDeleteHook); ok {
return hook.AfterDelete(ctx, repoName, filePath, db)
}
return nil
} }
func (h *LocalHandler) DB() *database.DB { type terraformVersionDoc struct {
return h.db Archives map[string]terraformArchive `json:"archives"`
} }
func (h *LocalHandler) Download(ctx context.Context, key string) (io.ReadCloser, int64, error) { type terraformArchive struct {
reader, info, err := h.store.Download(ctx, key) URL string `json:"url"`
if err != nil { Hashes []string `json:"hashes,omitempty"`
return nil, 0, err }
}
return reader, info.Size, nil func (h *LocalHandler) ServeTerraformIndex(w http.ResponseWriter, r *http.Request, repoName, namespace, typeName string) {
prefix := fmt.Sprintf("%s/%s/", namespace, typeName)
files, err := h.db.ListLocalFilesByPrefix(r.Context(), repoName, prefix)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
versions := map[string]json.RawMessage{}
for _, f := range files {
filename := strings.TrimPrefix(f.FilePath, prefix)
m := providerZipRe.FindStringSubmatch(filename)
if m == nil {
continue
}
versions[m[2]] = json.RawMessage(`{}`)
}
if len(versions) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(terraformIndex{Versions: versions})
}
func (h *LocalHandler) ServeTerraformVersionDoc(w http.ResponseWriter, r *http.Request, repoName, namespace, typeName, version string) {
prefix := fmt.Sprintf("%s/%s/terraform-provider-%s_%s_", namespace, typeName, typeName, version)
files, err := h.db.ListLocalFilesByPrefix(r.Context(), repoName, prefix)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
archives := map[string]terraformArchive{}
for _, f := range files {
filename := strings.TrimPrefix(f.FilePath, fmt.Sprintf("%s/%s/", namespace, typeName))
m := providerZipRe.FindStringSubmatch(filename)
if m == nil || m[2] != version {
continue
}
platform := m[3] + "_" + m[4]
archive := terraformArchive{URL: filename}
if f.ContentHash != "" {
archive.Hashes = []string{"zh:" + strings.TrimPrefix(f.ContentHash, "sha256:")}
}
archives[platform] = archive
}
if len(archives) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(terraformVersionDoc{Archives: archives})
} }
@@ -1,75 +0,0 @@
package v2
import (
"context"
"net/http/httptest"
"testing"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
_ "git.unkin.net/unkin/artifactapi/internal/provider/rpm" // register the rpm provider so its PostDeleteHook runs
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// TestLocalEvictCleansRPMMetadata verifies that evicting an RPM from a local
// repo also removes the derived rpm_metadata row, so generated repodata stops
// listing the deleted package.
func TestLocalEvictCleansRPMMetadata(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
}
ctx := context.Background()
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
defer db.Close()
const repo = "rpm-evict-cleanup"
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageRPM, RepoType: models.RepoTypeLocal}); err != nil {
t.Fatal(err)
}
const hash = "sha256:bb22"
const path = "Packages/example-0.1.0-1.x86_64.rpm"
if err := db.UpsertBlob(ctx, hash, "blobs/bb/22", 2048, "application/x-rpm"); err != nil {
t.Fatal(err)
}
if err := db.CreateLocalFile(ctx, repo, path, hash); err != nil {
t.Fatal(err)
}
if err := db.InsertRPMMetadata(ctx, &provider.RPMMetadata{
RepoName: repo, FilePath: path, ContentHash: hash,
Name: "example", Version: "0.1.0", Release: "1", Arch: "x86_64",
Requires: []provider.RPMDep{}, Provides: []provider.RPMDep{},
Files: []provider.RPMFile{}, Changelogs: []provider.RPMChangelog{},
}); err != nil {
t.Fatal(err)
}
h := NewObjectsHandler(db)
router := chi.NewRouter()
router.Route("/locals/{name}/objects", func(r chi.Router) {
r.Delete("/*", h.LocalRoutes().ServeHTTP)
})
del := httptest.NewRequest("DELETE", "/locals/"+repo+"/objects/"+path, nil)
dw := httptest.NewRecorder()
router.ServeHTTP(dw, del)
if dw.Code != 204 {
t.Fatalf("evict = %d, want 204", dw.Code)
}
if f, _ := db.GetLocalFile(ctx, repo, path); f != nil {
t.Fatalf("local file still present after evict: %+v", f)
}
entries, err := db.ListRPMMetadataEntries(ctx, repo)
if err != nil {
t.Fatal(err)
}
if len(entries) != 0 {
t.Fatalf("rpm_metadata still present after evict: %+v", entries)
}
}
-88
View File
@@ -1,88 +0,0 @@
package v2
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// TestLocalUploadStoreFailure covers the upload handlers' store-error branches
// by killing the object store after a successful upload.
func TestLocalUploadStoreFailure(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
}
ctx := context.Background()
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
defer db.Close()
conn, termMinio, err := testsupport.StartMinio(ctx)
if err != nil {
t.Skip("minio unavailable")
}
var store *storage.S3
for i := 0; i < 20; i++ {
if store, err = storage.NewS3(conn.Endpoint, conn.AccessKey, conn.SecretKey, "fault", false, ""); err == nil {
break
}
time.Sleep(500 * time.Millisecond)
}
if err != nil {
termMinio()
t.Fatal(err)
}
for _, pt := range []models.PackageType{models.PackageGeneric, models.PackagePyPI} {
if err := db.CreateRemote(ctx, &models.Remote{Name: "fault-" + string(pt), PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil {
t.Fatal(err)
}
}
h := NewLocalHandler(db, store)
router := chi.NewRouter()
router.Route("/remotes/{name}/files", func(r chi.Router) {
r.Put("/*", h.Routes().ServeHTTP)
})
srv := httptest.NewServer(router)
defer srv.Close()
put := func(name, path, body string) int {
rq, _ := http.NewRequest("PUT", srv.URL+"/remotes/"+name+"/files/"+path, strings.NewReader(body))
resp, err := http.DefaultClient.Do(rq)
if err != nil {
t.Fatalf("put: %v", err)
}
resp.Body.Close()
return resp.StatusCode
}
// Sanity: uploads succeed while the store is up.
if c := put("fault-generic", "ok.bin", "data"); c != 201 {
t.Fatalf("generic upload while up = %d", c)
}
if c := put("fault-pypi", "foo-1.0-py3-none-any.whl", "wheel"); c != 201 {
t.Fatalf("pypi upload while up = %d", c)
}
// Kill the store; subsequent CAS.Store calls fail -> 500.
termMinio()
if c := put("fault-generic", "after.bin", "data"); c != 500 {
t.Errorf("generic upload after store down = %d, want 500", c)
}
if c := put("fault-pypi", "bar-1.0-py3-none-any.whl", "wheel"); c != 500 {
t.Errorf("pypi upload after store down = %d, want 500", c)
}
}
-78
View File
@@ -1,78 +0,0 @@
package v2
import (
"context"
"encoding/json"
"net/http/httptest"
"testing"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// TestLocalObjectsListing verifies that files uploaded to a local repo (which
// live in local_files, not artifacts) are listed by the local objects endpoint
// and can be evicted through it.
func TestLocalObjectsListing(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
}
ctx := context.Background()
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
defer db.Close()
const repo = "rpm-local-objs"
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageRPM, RepoType: models.RepoTypeLocal}); err != nil {
t.Fatal(err)
}
const hash = "sha256:aa11"
const path = "Packages/example-0.1.0-1.x86_64.rpm"
if err := db.UpsertBlob(ctx, hash, "blobs/aa/11", 1234, "application/x-rpm"); err != nil {
t.Fatal(err)
}
if err := db.CreateLocalFile(ctx, repo, path, hash); err != nil {
t.Fatal(err)
}
h := NewObjectsHandler(db)
router := chi.NewRouter()
router.Route("/locals/{name}/objects", func(r chi.Router) {
r.Get("/", h.LocalRoutes().ServeHTTP)
r.Delete("/*", h.LocalRoutes().ServeHTTP)
})
// The uploaded package must appear in the listing with its blob size.
req := httptest.NewRequest("GET", "/locals/"+repo+"/objects", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("list = %d, want 200", w.Code)
}
var got []models.Artifact
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
t.Fatalf("decode: %v", err)
}
if len(got) != 1 {
t.Fatalf("got %d objects, want 1", len(got))
}
if got[0].Path != path || got[0].SizeBytes != 1234 || got[0].ContentHash != hash {
t.Fatalf("unexpected object: %+v", got[0])
}
// Eviction removes it from local_files.
del := httptest.NewRequest("DELETE", "/locals/"+repo+"/objects/"+path, nil)
dw := httptest.NewRecorder()
router.ServeHTTP(dw, del)
if dw.Code != 204 {
t.Fatalf("evict = %d, want 204", dw.Code)
}
if f, _ := db.GetLocalFile(ctx, repo, path); f != nil {
t.Fatalf("file still present after evict: %+v", f)
}
}
+4 -41
View File
@@ -25,18 +25,9 @@ func (h *ObjectsHandler) Routes() chi.Router {
return r return r
} }
// LocalRoutes lists and evicts objects for local repos, which live in the func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
// local_files table rather than the artifacts table used by remotes. remoteName := chi.URLParam(r, "name")
func (h *ObjectsHandler) LocalRoutes() chi.Router { limit, _ := strconv.Atoi(r.URL.Query().Get("per_page"))
r := chi.NewRouter()
r.Get("/", h.listLocal)
r.Delete("/*", h.evictLocal)
return r
}
// pageBounds parses the shared page/per_page query params into a SQL limit and offset.
func pageBounds(r *http.Request) (limit, offset int) {
limit, _ = strconv.Atoi(r.URL.Query().Get("per_page"))
if limit <= 0 || limit > 5000 { if limit <= 0 || limit > 5000 {
limit = 50 limit = 50
} }
@@ -44,12 +35,7 @@ func pageBounds(r *http.Request) (limit, offset int) {
if page <= 0 { if page <= 0 {
page = 1 page = 1
} }
return limit, (page - 1) * limit offset := (page - 1) * limit
}
func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "name")
limit, offset := pageBounds(r)
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset) artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
if err != nil { if err != nil {
@@ -59,29 +45,6 @@ func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, artifacts) writeJSON(w, http.StatusOK, artifacts)
} }
func (h *ObjectsHandler) listLocal(w http.ResponseWriter, r *http.Request) {
repoName := chi.URLParam(r, "name")
limit, offset := pageBounds(r)
artifacts, err := h.db.ListLocalArtifacts(r.Context(), repoName, limit, offset)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, artifacts)
}
func (h *ObjectsHandler) evictLocal(w http.ResponseWriter, r *http.Request) {
repoName := chi.URLParam(r, "name")
path := chi.URLParam(r, "*")
if err := deleteLocalFile(r.Context(), h.db, repoName, path); err != nil {
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) { func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "name") remoteName := chi.URLParam(r, "name")
path := chi.URLParam(r, "*") path := chi.URLParam(r, "*")
-8
View File
@@ -69,10 +69,6 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest) http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest)
return return
} }
if err := remote.ValidatePatterns(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.db.CreateRemote(r.Context(), &remote); err != nil { if err := h.db.CreateRemote(r.Context(), &remote); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
return return
@@ -88,10 +84,6 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
return return
} }
remote.Name = name remote.Name = name
if err := remote.ValidatePatterns(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil { if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
return return
-23
View File
@@ -1,23 +0,0 @@
package auth
import (
"encoding/base64"
"testing"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestBasicHeaders(t *testing.T) {
h := BasicHeaders(models.Remote{Username: "alice", Password: "secret"})
got := h.Get("Authorization")
want := "Basic " + base64.StdEncoding.EncodeToString([]byte("alice:secret"))
if got != want {
t.Errorf("Authorization = %q, want %q", got, want)
}
}
func TestBasicHeadersNoUser(t *testing.T) {
if h := BasicHeaders(models.Remote{}); h.Get("Authorization") != "" {
t.Error("expected no Authorization header without a username")
}
}
-133
View File
@@ -1,133 +0,0 @@
package cache
import (
"context"
"os"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
var testRedis *Redis
func TestMain(m *testing.M) {
ctx := context.Background()
url, terminate, err := testsupport.StartRedis(ctx)
if err != nil {
os.Exit(m.Run())
}
r, err := NewRedis(url)
if err != nil {
terminate()
panic(err)
}
testRedis = r
code := m.Run()
r.Close()
terminate()
if code != 0 {
os.Exit(code)
}
}
func requireRedis(t *testing.T) {
t.Helper()
if testRedis == nil {
t.Skip("Docker unavailable; skipping cache integration test")
}
}
func TestNewRedisInvalid(t *testing.T) {
if _, err := NewRedis("://bad-url"); err == nil {
t.Error("expected error for invalid redis URL")
}
}
func TestTTL(t *testing.T) {
requireRedis(t)
ctx := context.Background()
if fresh, _ := testRedis.CheckTTL(ctx, "r", "missing"); fresh {
t.Error("missing key should not be fresh")
}
if err := testRedis.SetTTL(ctx, "r", "p", time.Minute); err != nil {
t.Fatal(err)
}
if fresh, err := testRedis.CheckTTL(ctx, "r", "p"); err != nil || !fresh {
t.Errorf("expected fresh after SetTTL: %v %v", fresh, err)
}
}
func TestLock(t *testing.T) {
requireRedis(t)
ctx := context.Background()
ok, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
if err != nil || !ok {
t.Fatalf("first acquire should succeed: %v %v", ok, err)
}
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); ok {
t.Error("second acquire should fail while held")
}
if err := testRedis.ReleaseLock(ctx, "r", "lockpath"); err != nil {
t.Fatal(err)
}
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); !ok {
t.Error("acquire should succeed after release")
}
}
func TestETagAndToken(t *testing.T) {
requireRedis(t)
ctx := context.Background()
if v, _ := testRedis.GetETag(ctx, "r", "missing"); v != "" {
t.Error("missing etag should be empty")
}
testRedis.SetETag(ctx, "r", "p", `"abc"`, time.Minute)
if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` {
t.Errorf("etag = %q", v)
}
if v, _ := testRedis.GetToken(ctx, "missing"); v != "" {
t.Error("missing token should be empty")
}
testRedis.SetToken(ctx, "key", "tok", time.Minute)
if v, _ := testRedis.GetToken(ctx, "key"); v != "tok" {
t.Errorf("token = %q", v)
}
}
func TestCircuit(t *testing.T) {
requireRedis(t)
ctx := context.Background()
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 0 {
t.Errorf("initial failures = %d", n)
}
n1, err := testRedis.IncrCircuitFailure(ctx, "cr", time.Minute)
if err != nil || n1 != 1 {
t.Fatalf("first incr = %d %v", n1, err)
}
n2, _ := testRedis.IncrCircuitFailure(ctx, "cr", time.Minute)
if n2 != 2 {
t.Errorf("second incr = %d", n2)
}
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 2 {
t.Errorf("get failures = %d", n)
}
testRedis.ResetCircuit(ctx, "cr")
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 0 {
t.Errorf("failures after reset = %d", n)
}
}
func TestFlushRemote(t *testing.T) {
requireRedis(t)
ctx := context.Background()
testRedis.SetTTL(ctx, "flushme", "a", time.Hour)
testRedis.SetETag(ctx, "flushme", "a", "x", time.Hour)
if err := testRedis.FlushRemote(ctx, "flushme"); err != nil {
t.Fatal(err)
}
if fresh, _ := testRedis.CheckTTL(ctx, "flushme", "a"); fresh {
t.Error("expected keys flushed")
}
}
-12
View File
@@ -70,18 +70,6 @@ func (r *Redis) GetETag(ctx context.Context, remote, path string) (string, error
return val, err return val, err
} }
func (r *Redis) GetToken(ctx context.Context, key string) (string, error) {
val, err := r.client.Get(ctx, "token:"+key).Result()
if err == redis.Nil {
return "", nil
}
return val, err
}
func (r *Redis) SetToken(ctx context.Context, key, token string, ttl time.Duration) error {
return r.client.Set(ctx, "token:"+key, token, ttl).Err()
}
func (r *Redis) IncrCircuitFailure(ctx context.Context, remote string, cooldown time.Duration) (int64, error) { func (r *Redis) IncrCircuitFailure(ctx context.Context, remote string, cooldown time.Duration) (int64, error) {
key := fmt.Sprintf("circuit:%s", remote) key := fmt.Sprintf("circuit:%s", remote)
pipe := r.client.Pipeline() pipe := r.client.Pipeline()
+1 -13
View File
@@ -24,14 +24,6 @@ type Config struct {
S3Bucket string S3Bucket string
S3Secure bool S3Secure bool
S3Region string S3Region string
// Terraform provider registry signing. When TFSigningKeyPath points at a
// readable armored GPG private key, artifactapi serves local terraform
// repos as a real provider registry (service discovery + signed
// SHA256SUMS). Left empty, the registry endpoints stay disabled.
TFSigningKeyPath string
TFSigningKeyPassphrase string
TFProviderProtocols string
} }
func (c *Config) DatabaseDSN() string { func (c *Config) DatabaseDSN() string {
@@ -67,17 +59,13 @@ func Load() (*Config, error) {
S3Bucket: getenv("MINIO_BUCKET", "artifacts"), S3Bucket: getenv("MINIO_BUCKET", "artifacts"),
S3Secure: s3Secure, S3Secure: s3Secure,
S3Region: getenv("MINIO_REGION", ""), S3Region: getenv("MINIO_REGION", ""),
TFSigningKeyPath: getenv("TF_SIGNING_KEY_PATH", ""),
TFSigningKeyPassphrase: getenv("TF_SIGNING_KEY_PASSPHRASE", ""),
TFProviderProtocols: getenv("TF_PROVIDER_PROTOCOLS", "5.0,6.0"),
} }
return cfg, nil return cfg, nil
} }
func getenv(key, fallback string) string { func getenv(key, fallback string) string {
if v, ok := os.LookupEnv(key); ok { if v := os.Getenv(key); v != "" {
return v return v
} }
return fallback return fallback
-66
View File
@@ -1,66 +0,0 @@
package config
import (
"os"
"testing"
)
func TestLoadDefaults(t *testing.T) {
// Unset the vars Load reads so the fallback defaults are exercised.
for _, k := range []string{
"LISTEN_ADDR", "DBHOST", "DBPORT", "DBUSER", "DBPASS", "DBNAME", "DBSSL",
"REDIS_URL", "MINIO_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY",
"MINIO_BUCKET", "MINIO_SECURE", "MINIO_REGION",
} {
old, ok := os.LookupEnv(k)
os.Unsetenv(k)
if ok {
t.Cleanup(func() { os.Setenv(k, old) })
}
}
cfg, err := Load()
if err != nil {
t.Fatalf("load: %v", err)
}
if cfg.ListenAddr != ":8000" || cfg.DBPort != 5432 || cfg.DBUser != "artifacts" {
t.Errorf("unexpected defaults: %+v", cfg)
}
if cfg.RedisURL != "redis://localhost:6379" || cfg.S3Bucket != "artifacts" || cfg.S3Secure {
t.Errorf("unexpected defaults: %+v", cfg)
}
}
func TestLoadOverrides(t *testing.T) {
t.Setenv("LISTEN_ADDR", ":9999")
t.Setenv("DBHOST", "db.example.com")
t.Setenv("DBPORT", "6000")
t.Setenv("DBUSER", "u")
t.Setenv("DBPASS", "pw")
t.Setenv("DBNAME", "n")
t.Setenv("DBSSL", "require")
t.Setenv("MINIO_SECURE", "true")
t.Setenv("MINIO_REGION", "us-east-1")
cfg, err := Load()
if err != nil {
t.Fatalf("load: %v", err)
}
if cfg.ListenAddr != ":9999" || cfg.DBHost != "db.example.com" || cfg.DBPort != 6000 {
t.Errorf("overrides not applied: %+v", cfg)
}
if !cfg.S3Secure {
t.Error("MINIO_SECURE=true not parsed")
}
want := "postgres://u:pw@db.example.com:6000/n?sslmode=require"
if got := cfg.DatabaseDSN(); got != want {
t.Errorf("DSN = %q, want %q", got, want)
}
}
func TestLoadInvalidPort(t *testing.T) {
t.Setenv("DBPORT", "not-a-number")
if _, err := Load(); err == nil {
t.Error("expected error for invalid DBPORT")
}
}
+3 -38
View File
@@ -4,8 +4,6 @@ import (
"context" "context"
"time" "time"
"github.com/jackc/pgx/v5"
"git.unkin.net/unkin/artifactapi/pkg/models" "git.unkin.net/unkin/artifactapi/pkg/models"
) )
@@ -111,49 +109,16 @@ func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cach
return err return err
} }
// AccessLogEntry is one buffered access-log record. func (db *DB) FindOrphanedBlobs(ctx context.Context) ([]models.Blob, error) {
type AccessLogEntry struct {
RemoteName string
Path string
CacheHit bool
SizeBytes int64
UpstreamMS int
ClientIP string
}
// InsertAccessLogBatch bulk-inserts access-log rows with a single COPY.
func (db *DB) InsertAccessLogBatch(ctx context.Context, entries []AccessLogEntry) error {
if len(entries) == 0 {
return nil
}
rows := make([][]any, len(entries))
for i, e := range entries {
rows[i] = []any{e.RemoteName, e.Path, e.CacheHit, e.SizeBytes, e.UpstreamMS, e.ClientIP}
}
_, err := db.Pool.CopyFrom(ctx,
pgx.Identifier{"access_log"},
[]string{"remote_name", "path", "cache_hit", "size_bytes", "upstream_ms", "client_ip"},
pgx.CopyFromRows(rows),
)
return err
}
// FindOrphanedBlobs returns blobs no longer referenced by any artifact or
// local file, restricted to those created before now()-minAge. The age cutoff
// is a grace period that avoids a TOCTOU race with in-flight dedup uploads,
// which insert the blob row before the referencing artifact/local_files row.
func (db *DB) FindOrphanedBlobs(ctx context.Context, minAge time.Duration) ([]models.Blob, error) {
cutoff := time.Now().Add(-minAge)
rows, err := db.Pool.Query(ctx, ` rows, err := db.Pool.Query(ctx, `
SELECT b.content_hash, b.s3_key, b.size_bytes, b.content_type, b.created_at SELECT b.content_hash, b.s3_key, b.size_bytes, b.content_type, b.created_at
FROM blobs b FROM blobs b
WHERE b.created_at < $1 WHERE b.content_hash NOT IN (
AND b.content_hash NOT IN (
SELECT content_hash FROM artifacts SELECT content_hash FROM artifacts
UNION UNION
SELECT content_hash FROM local_files SELECT content_hash FROM local_files
) )
`, cutoff) `)
if err != nil { if err != nil {
return nil, err return nil, err
} }
-334
View File
@@ -1,334 +0,0 @@
package database
import (
"context"
"os"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var (
testDB *DB
testDSN string
)
func TestMain(m *testing.M) {
c := context.Background()
dsn, terminate, err := testsupport.StartPostgres(c)
if err != nil {
// Docker unavailable: run anyway so tests self-skip via requireDB.
os.Exit(m.Run())
}
testDSN = dsn
db, err := New(dsn)
if err != nil {
terminate()
panic(err)
}
testDB = db
code := m.Run()
db.Close()
terminate()
// Return normally on success so the coverage profile is flushed; os.Exit
// would truncate it.
if code != 0 {
os.Exit(code)
}
}
func requireDB(t *testing.T) {
t.Helper()
if testDB == nil {
t.Skip("Docker unavailable; skipping database integration test")
}
}
func ctx() context.Context { return context.Background() }
func seedRemote(t *testing.T, name string) {
t.Helper()
if err := testDB.CreateRemote(ctx(), &models.Remote{
Name: name, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote,
BaseURL: "https://example.com", MutableTTL: 3600,
}); err != nil {
t.Fatalf("seed remote: %v", err)
}
}
// seedBlob inserts a blob and returns its full content hash (sha256:<hash>),
// matching the reference convention used by artifacts and local files.
func seedBlob(t *testing.T, hash string) string {
t.Helper()
full := "sha256:" + hash
if err := testDB.UpsertBlob(ctx(), full, "blobs/sha256/"+hash, 10, "application/octet-stream"); err != nil {
t.Fatalf("seed blob: %v", err)
}
return full
}
func TestRemotesCRUD(t *testing.T) {
requireDB(t)
seedRemote(t, "r-crud")
got, err := testDB.GetRemote(ctx(), "r-crud")
if err != nil || got.BaseURL != "https://example.com" {
t.Fatalf("get: %v %v", got, err)
}
got.BaseURL = "https://updated.example.com"
if err := testDB.UpdateRemote(ctx(), got); err != nil {
t.Fatalf("update: %v", err)
}
got, _ = testDB.GetRemote(ctx(), "r-crud")
if got.BaseURL != "https://updated.example.com" {
t.Errorf("update not applied: %v", got.BaseURL)
}
list, err := testDB.ListRemotes(ctx())
if err != nil || len(list) == 0 {
t.Fatalf("list: %v %v", len(list), err)
}
if err := testDB.DeleteRemote(ctx(), "r-crud"); err != nil {
t.Fatalf("delete: %v", err)
}
if _, err := testDB.GetRemote(ctx(), "r-crud"); err == nil {
t.Error("expected error after delete")
}
}
func TestArtifactsAndBlobs(t *testing.T) {
requireDB(t)
seedRemote(t, "r-art")
seedBlob(t, "aaaa")
hash := "sha256:aaaa"
if err := testDB.UpsertBlob(ctx(), hash, "blobs/sha256/aaaa", 10, "text/plain"); err != nil {
t.Fatal(err)
}
if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag1"); err != nil {
t.Fatal(err)
}
// Upsert again to exercise the ON CONFLICT update branch.
if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag2"); err != nil {
t.Fatal(err)
}
art, err := testDB.GetArtifact(ctx(), "r-art", "path/a.txt")
if err != nil || art.ContentHash != hash {
t.Fatalf("get artifact: %v %v", art, err)
}
if err := testDB.TouchArtifactAccess(ctx(), "r-art", "path/a.txt"); err != nil {
t.Fatal(err)
}
arts, err := testDB.ListArtifacts(ctx(), "r-art", 10, 0)
if err != nil || len(arts) != 1 {
t.Fatalf("list artifacts: %v %v", len(arts), err)
}
if err := testDB.InsertAccessLog(ctx(), "r-art", "path/a.txt", true, 10, 5, "1.2.3.4"); err != nil {
t.Fatal(err)
}
if err := testDB.InsertAccessLogBatch(ctx(), []AccessLogEntry{
{RemoteName: "r-art", Path: "b", CacheHit: false, SizeBytes: 20, UpstreamMS: 3},
}); err != nil {
t.Fatal(err)
}
if err := testDB.InsertAccessLogBatch(ctx(), nil); err != nil {
t.Fatalf("empty batch should be a no-op: %v", err)
}
if err := testDB.DeleteArtifact(ctx(), "r-art", "path/a.txt"); err != nil {
t.Fatal(err)
}
}
func TestOrphanAndColdCleanup(t *testing.T) {
requireDB(t)
seedBlob(t, "orphanhash")
// A blob with no artifact/local_file reference is orphaned, but only past
// the grace period.
if got, _ := testDB.FindOrphanedBlobs(ctx(), time.Hour); containsHash(got, "sha256:orphanhash") {
t.Error("fresh orphan should be excluded by grace period")
}
orphans, err := testDB.FindOrphanedBlobs(ctx(), -time.Hour) // cutoff in the future => include fresh
if err != nil {
t.Fatal(err)
}
if !containsHash(orphans, "sha256:orphanhash") {
t.Error("expected orphan to be found with zero grace")
}
if err := testDB.DeleteBlob(ctx(), "sha256:orphanhash"); err != nil {
t.Fatal(err)
}
seedRemote(t, "r-cold")
seedBlob(t, "coldhash")
testDB.UpsertArtifact(ctx(), "r-cold", "cold.txt", "sha256:coldhash", "")
n, err := testDB.DeleteColdArtifacts(ctx(), "r-cold", -time.Hour) // negative => everything is "cold"
if err != nil || n < 1 {
t.Fatalf("delete cold: n=%d err=%v", n, err)
}
}
func containsHash(blobs []models.Blob, hash string) bool {
for _, b := range blobs {
if b.ContentHash == hash {
return true
}
}
return false
}
func TestLocalFiles(t *testing.T) {
requireDB(t)
seedRemote(t, "r-local")
seedBlob(t, "localhash")
hash := "sha256:localhash"
if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err != nil {
t.Fatal(err)
}
// Duplicate create must be rejected.
if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err == nil {
t.Error("expected duplicate local file error")
}
f, err := testDB.GetLocalFile(ctx(), "r-local", "foo/foo-1.0.whl")
if err != nil || f == nil {
t.Fatalf("get local file: %v %v", f, err)
}
if files, err := testDB.ListLocalFiles(ctx(), "r-local", 10, 0); err != nil || len(files) != 1 {
t.Fatalf("list: %v %v", len(files), err)
}
if files, err := testDB.ListLocalFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(files) != 1 {
t.Fatalf("list by prefix: %v %v", len(files), err)
}
if entries, err := testDB.ListFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(entries) != 1 {
t.Fatalf("provider list by prefix: %v %v", len(entries), err)
}
if pkgs, err := testDB.ListLocalFilePackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 {
t.Fatalf("list packages: %v %v", pkgs, err)
}
if pkgs, err := testDB.ListPackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 {
t.Fatalf("provider list packages: %v %v", pkgs, err)
}
if err := testDB.DeleteLocalFile(ctx(), "r-local", "foo/foo-1.0.whl"); err != nil {
t.Fatal(err)
}
}
func TestVirtualsCRUD(t *testing.T) {
requireDB(t)
if err := testDB.CreateVirtual(ctx(), &models.Virtual{
Name: "v-crud", PackageType: models.PackageHelm, Members: []string{"a", "b"},
}); err != nil {
t.Fatal(err)
}
v, err := testDB.GetVirtual(ctx(), "v-crud")
if err != nil || len(v.Members) != 2 {
t.Fatalf("get virtual: %v %v", v, err)
}
v.Members = []string{"a"}
if err := testDB.UpdateVirtual(ctx(), v); err != nil {
t.Fatal(err)
}
if vs, err := testDB.ListVirtuals(ctx()); err != nil || len(vs) == 0 {
t.Fatalf("list virtuals: %v %v", len(vs), err)
}
if err := testDB.DeleteVirtual(ctx(), "v-crud"); err != nil {
t.Fatal(err)
}
}
func TestStats(t *testing.T) {
requireDB(t)
seedRemote(t, "r-stats")
seedBlob(t, "statshash")
testDB.UpsertArtifact(ctx(), "r-stats", "s.txt", "sha256:statshash", "")
testDB.InsertAccessLog(ctx(), "r-stats", "s.txt", true, 100, 2, "")
if _, err := testDB.GetOverviewStats(ctx()); err != nil {
t.Fatalf("overview: %v", err)
}
if _, err := testDB.GetTopRemotes(ctx(), 5); err != nil {
t.Fatalf("top remotes: %v", err)
}
if _, err := testDB.GetTopFilesByHits(ctx(), 5); err != nil {
t.Fatalf("top files by hits: %v", err)
}
if _, err := testDB.GetTopFilesByBandwidth(ctx(), 5); err != nil {
t.Fatalf("top files by bandwidth: %v", err)
}
}
func TestDatabaseErrorPaths(t *testing.T) {
requireDB(t)
bad, err := New(testDSN)
if err != nil {
t.Fatal(err)
}
bad.Close() // every query now fails
ctx := context.Background()
if _, err := bad.ListRemotes(ctx); err == nil {
t.Error("ListRemotes should error on closed db")
}
if _, err := bad.ListVirtuals(ctx); err == nil {
t.Error("ListVirtuals should error")
}
if _, err := bad.ListArtifacts(ctx, "r", 10, 0); err == nil {
t.Error("ListArtifacts should error")
}
if _, err := bad.ListLocalFiles(ctx, "r", 10, 0); err == nil {
t.Error("ListLocalFiles should error")
}
if _, err := bad.ListLocalFilesByPrefix(ctx, "r", "p"); err == nil {
t.Error("ListLocalFilesByPrefix should error")
}
if _, err := bad.ListLocalFilePackages(ctx, "r"); err == nil {
t.Error("ListLocalFilePackages should error")
}
if _, err := bad.ListFilesByPrefix(ctx, "r", "p"); err == nil {
t.Error("ListFilesByPrefix should error")
}
if _, err := bad.ListPackages(ctx, "r"); err == nil {
t.Error("ListPackages should error")
}
if _, err := bad.FindOrphanedBlobs(ctx, 0); err == nil {
t.Error("FindOrphanedBlobs should error")
}
if _, err := bad.GetOverviewStats(ctx); err == nil {
t.Error("GetOverviewStats should error")
}
if _, err := bad.GetTopRemotes(ctx, 5); err == nil {
t.Error("GetTopRemotes should error")
}
if _, err := bad.GetTopFilesByHits(ctx, 5); err == nil {
t.Error("GetTopFilesByHits should error")
}
if _, err := bad.GetTopFilesByBandwidth(ctx, 5); err == nil {
t.Error("GetTopFilesByBandwidth should error")
}
if _, err := bad.ListRPMMetadataEntries(ctx, "r"); err == nil {
t.Error("ListRPMMetadataEntries should error")
}
}
func TestRPMMetadata(t *testing.T) {
requireDB(t)
seedRemote(t, "r-rpm")
meta := &provider.RPMMetadata{
RepoName: "r-rpm", FilePath: "Packages/x.rpm", ContentHash: "sha256:rpm",
Name: "x", Version: "1.0", Release: "1", Arch: "noarch",
Requires: []provider.RPMDep{{Name: "libc"}},
Provides: []provider.RPMDep{{Name: "x"}},
Files: []provider.RPMFile{},
}
if err := testDB.InsertRPMMetadata(ctx(), meta); err != nil {
t.Fatal(err)
}
entries, err := testDB.ListRPMMetadataEntries(ctx(), "r-rpm")
if err != nil || len(entries) != 1 {
t.Fatalf("list rpm entries: %v %v", len(entries), err)
}
if rows, err := testDB.ListRPMMetadata(ctx(), "r-rpm"); err != nil || len(rows) != 1 {
t.Fatalf("list rpm rows: %v %v", len(rows), err)
}
}
-76
View File
@@ -8,9 +8,6 @@ import (
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgconn"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
) )
type LocalFile struct { type LocalFile struct {
@@ -79,40 +76,6 @@ func (db *DB) ListLocalFiles(ctx context.Context, repoName string, limit, offset
return files, rows.Err() return files, rows.Err()
} }
// ListLocalArtifacts returns a repo's local files shaped as models.Artifact so
// the UI's cached-objects view can render them the same way as remote artifacts.
// Local files carry no access/fetch counters, so those are left at zero and the
// timestamps are all derived from created_at.
func (db *DB) ListLocalArtifacts(ctx context.Context, repoName string, limit, offset int) ([]models.Artifact, error) {
rows, err := db.Pool.Query(ctx, `
SELECT lf.id, lf.repo_name, lf.file_path, lf.content_hash,
lf.created_at, b.size_bytes, b.content_type
FROM local_files lf
JOIN blobs b ON lf.content_hash = b.content_hash
WHERE lf.repo_name = $1
ORDER BY lf.file_path
LIMIT $2 OFFSET $3
`, repoName, limit, offset)
if err != nil {
return nil, err
}
defer rows.Close()
var artifacts []models.Artifact
for rows.Next() {
var a models.Artifact
var createdAt time.Time
if err := rows.Scan(&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &createdAt, &a.SizeBytes, &a.ContentType); err != nil {
return nil, err
}
a.FirstSeenAt = createdAt
a.LastFetchedAt = createdAt
a.LastAccessedAt = createdAt
artifacts = append(artifacts, a)
}
return artifacts, rows.Err()
}
func (db *DB) ListLocalFilesByPrefix(ctx context.Context, repoName, prefix string) ([]LocalFile, error) { func (db *DB) ListLocalFilesByPrefix(ctx context.Context, repoName, prefix string) ([]LocalFile, error) {
rows, err := db.Pool.Query(ctx, ` rows, err := db.Pool.Query(ctx, `
SELECT id, repo_name, file_path, content_hash, created_at SELECT id, repo_name, file_path, content_hash, created_at
@@ -136,45 +99,6 @@ func (db *DB) ListLocalFilesByPrefix(ctx context.Context, repoName, prefix strin
return files, rows.Err() return files, rows.Err()
} }
func (db *DB) ListLocalFilePackages(ctx context.Context, repoName string) ([]string, error) {
rows, err := db.Pool.Query(ctx, `
SELECT DISTINCT split_part(file_path, '/', 1)
FROM local_files
WHERE repo_name = $1
ORDER BY 1
`, repoName)
if err != nil {
return nil, err
}
defer rows.Close()
var packages []string
for rows.Next() {
var pkg string
if err := rows.Scan(&pkg); err != nil {
return nil, err
}
packages = append(packages, pkg)
}
return packages, rows.Err()
}
func (db *DB) ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]provider.FileEntry, error) {
files, err := db.ListLocalFilesByPrefix(ctx, repoName, prefix)
if err != nil {
return nil, err
}
result := make([]provider.FileEntry, len(files))
for i, f := range files {
result[i] = provider.FileEntry{FilePath: f.FilePath, ContentHash: f.ContentHash}
}
return result, nil
}
func (db *DB) ListPackages(ctx context.Context, repoName string) ([]string, error) {
return db.ListLocalFilePackages(ctx, repoName)
}
func (db *DB) DeleteLocalFile(ctx context.Context, repoName, filePath string) error { func (db *DB) DeleteLocalFile(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM local_files WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) _, err := db.Pool.Exec(ctx, `DELETE FROM local_files WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err return err
-41
View File
@@ -124,47 +124,6 @@ func (db *DB) migrate() error {
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at); CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote'; ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
CREATE TABLE IF NOT EXISTS rpm_metadata (
id BIGSERIAL PRIMARY KEY,
repo_name TEXT NOT NULL,
file_path TEXT NOT NULL,
content_hash TEXT NOT NULL,
name TEXT NOT NULL,
epoch INTEGER DEFAULT 0,
version TEXT NOT NULL,
release TEXT NOT NULL,
arch TEXT NOT NULL,
summary TEXT DEFAULT '',
description TEXT DEFAULT '',
rpm_size BIGINT DEFAULT 0,
installed_size BIGINT DEFAULT 0,
license TEXT DEFAULT '',
vendor TEXT DEFAULT '',
build_group TEXT DEFAULT '',
build_host TEXT DEFAULT '',
source_rpm TEXT DEFAULT '',
url TEXT DEFAULT '',
packager TEXT DEFAULT '',
requires JSONB DEFAULT '[]',
provides JSONB DEFAULT '[]',
files JSONB DEFAULT '[]',
changelogs JSONB DEFAULT '[]',
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(repo_name, file_path)
);
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
CREATE TABLE IF NOT EXISTS signing_keys (
purpose TEXT PRIMARY KEY,
private_key_armor TEXT NOT NULL,
key_id TEXT NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW()
);
`) `)
return err return err
} }
+5 -14
View File
@@ -11,9 +11,7 @@ const remoteCols = `name, package_type, repo_type, base_url, description, userna
patterns, blocklist, mutable_patterns, immutable_patterns, patterns, blocklist, mutable_patterns, immutable_patterns,
ban_tags_enabled, ban_tags, ban_tags_enabled, ban_tags,
quarantine_enabled, quarantine_days, stale_on_error, quarantine_enabled, quarantine_days, stale_on_error,
releases_remote, managed_by, releases_remote, managed_by, created_at, updated_at`
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
created_at, updated_at`
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error { func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
return scanner.Scan( return scanner.Scan(
@@ -22,9 +20,7 @@ func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns, &r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
&r.BanTagsEnabled, &r.BanTags, &r.BanTagsEnabled, &r.BanTags,
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError, &r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
&r.ReleasesRemote, &r.ManagedBy, &r.ReleasesRemote, &r.ManagedBy, &r.CreatedAt, &r.UpdatedAt,
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
&r.CreatedAt, &r.UpdatedAt,
) )
} }
@@ -63,9 +59,8 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
patterns, blocklist, mutable_patterns, immutable_patterns, patterns, blocklist, mutable_patterns, immutable_patterns,
ban_tags_enabled, ban_tags, ban_tags_enabled, ban_tags,
quarantine_enabled, quarantine_days, stale_on_error, quarantine_enabled, quarantine_days, stale_on_error,
releases_remote, managed_by, releases_remote, managed_by
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21)
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24)
`, `,
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password, r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
r.ImmutableTTL, r.MutableTTL, r.CheckMutable, r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
@@ -73,7 +68,6 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
r.BanTagsEnabled, r.BanTags, r.BanTagsEnabled, r.BanTags,
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError, r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
r.ReleasesRemote, r.ManagedBy, r.ReleasesRemote, r.ManagedBy,
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
) )
return err return err
} }
@@ -86,9 +80,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14, patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
ban_tags_enabled=$15, ban_tags=$16, ban_tags_enabled=$15, ban_tags=$16,
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19, quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
releases_remote=$20, managed_by=$21, releases_remote=$20, managed_by=$21, updated_at=NOW()
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
updated_at=NOW()
WHERE name=$1 WHERE name=$1
`, `,
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password, r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
@@ -97,7 +89,6 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
r.BanTagsEnabled, r.BanTags, r.BanTagsEnabled, r.BanTags,
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError, r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
r.ReleasesRemote, r.ManagedBy, r.ReleasesRemote, r.ManagedBy,
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
) )
return err return err
} }
-134
View File
@@ -1,134 +0,0 @@
package database
import (
"context"
"encoding/json"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
requiresJSON, _ := json.Marshal(meta.Requires)
providesJSON, _ := json.Marshal(meta.Provides)
filesJSON, _ := json.Marshal(meta.Files)
changelogsJSON, _ := json.Marshal(meta.Changelogs)
_, err := db.Pool.Exec(ctx, `
INSERT INTO rpm_metadata (
repo_name, file_path, content_hash,
name, epoch, version, release, arch,
summary, description, rpm_size, installed_size,
license, vendor, build_group, build_host, source_rpm, url, packager,
requires, provides, files, changelogs
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23)
ON CONFLICT (repo_name, file_path) DO NOTHING
`,
meta.RepoName, meta.FilePath, meta.ContentHash,
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
requiresJSON, providesJSON, filesJSON, changelogsJSON,
)
return err
}
func (db *DB) DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM rpm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err
}
type RPMMetadataRow struct {
RepoName string
FilePath string
ContentHash string
Name string
Epoch int
Version string
Release string
Arch string
Summary string
Description string
RPMSize int64
InstalledSize int64
License string
Vendor string
Group string
BuildHost string
SourceRPM string
URL string
Packager string
Requires json.RawMessage
Provides json.RawMessage
Files json.RawMessage
Changelogs json.RawMessage
}
func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]provider.RPMMetadata, error) {
rows, err := db.ListRPMMetadata(ctx, repoName)
if err != nil {
return nil, err
}
result := make([]provider.RPMMetadata, len(rows))
for i, r := range rows {
meta := provider.RPMMetadata{
RepoName: r.RepoName,
FilePath: r.FilePath,
ContentHash: r.ContentHash,
Name: r.Name,
Epoch: r.Epoch,
Version: r.Version,
Release: r.Release,
Arch: r.Arch,
Summary: r.Summary,
Description: r.Description,
RPMSize: r.RPMSize,
InstalledSize: r.InstalledSize,
License: r.License,
Vendor: r.Vendor,
Group: r.Group,
BuildHost: r.BuildHost,
SourceRPM: r.SourceRPM,
URL: r.URL,
Packager: r.Packager,
}
json.Unmarshal(r.Requires, &meta.Requires)
json.Unmarshal(r.Provides, &meta.Provides)
json.Unmarshal(r.Files, &meta.Files)
json.Unmarshal(r.Changelogs, &meta.Changelogs)
result[i] = meta
}
return result, nil
}
func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetadataRow, error) {
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, content_hash,
name, epoch, version, release, arch,
summary, description, rpm_size, installed_size,
license, vendor, build_group, build_host, source_rpm, url, packager,
requires, provides, files, changelogs
FROM rpm_metadata
WHERE repo_name = $1
ORDER BY name, epoch, version, release, arch
`, repoName)
if err != nil {
return nil, err
}
defer rows.Close()
var result []RPMMetadataRow
for rows.Next() {
var r RPMMetadataRow
if err := rows.Scan(
&r.RepoName, &r.FilePath, &r.ContentHash,
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
&r.Requires, &r.Provides, &r.Files, &r.Changelogs,
); err != nil {
return nil, err
}
result = append(result, r)
}
return result, rows.Err()
}
-35
View File
@@ -1,35 +0,0 @@
package database
import (
"context"
"errors"
"github.com/jackc/pgx/v5"
)
// GetSigningKey returns the stored armored private key and key id for a purpose.
// found is false when no key has been generated yet.
func (db *DB) GetSigningKey(ctx context.Context, purpose string) (armor, keyID string, found bool, err error) {
row := db.Pool.QueryRow(ctx, `
SELECT private_key_armor, key_id FROM signing_keys WHERE purpose = $1
`, purpose)
if err := row.Scan(&armor, &keyID); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return "", "", false, nil
}
return "", "", false, err
}
return armor, keyID, true, nil
}
// InsertSigningKeyIfAbsent stores a freshly generated key, doing nothing if
// another replica already inserted one. Callers re-read with GetSigningKey to
// pick up whichever key won the race.
func (db *DB) InsertSigningKeyIfAbsent(ctx context.Context, purpose, armor, keyID string) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO signing_keys (purpose, private_key_armor, key_id)
VALUES ($1, $2, $3)
ON CONFLICT (purpose) DO NOTHING
`, purpose, armor, keyID)
return err
}
-31
View File
@@ -1,31 +0,0 @@
package database
import "testing"
func TestSigningKeyRoundTripAndIdempotency(t *testing.T) {
requireDB(t)
const purpose = "terraform-provider-test"
// Absent to start.
if _, _, found, err := testDB.GetSigningKey(ctx(), purpose); err != nil || found {
t.Fatalf("expected no key, got found=%v err=%v", found, err)
}
if err := testDB.InsertSigningKeyIfAbsent(ctx(), purpose, "ARMOR-1", "KEYID1"); err != nil {
t.Fatal(err)
}
// A second insert must not overwrite (models the replica race).
if err := testDB.InsertSigningKeyIfAbsent(ctx(), purpose, "ARMOR-2", "KEYID2"); err != nil {
t.Fatal(err)
}
armor, keyID, found, err := testDB.GetSigningKey(ctx(), purpose)
if err != nil || !found {
t.Fatalf("expected key, found=%v err=%v", found, err)
}
if armor != "ARMOR-1" || keyID != "KEYID1" {
t.Errorf("key was overwritten: armor=%q key_id=%q", armor, keyID)
}
}
-9
View File
@@ -30,15 +30,6 @@ func (db *DB) GetOverviewStats(ctx context.Context) (*models.OverviewStats, erro
return nil, err return nil, err
} }
err = db.Pool.QueryRow(ctx, `
SELECT COALESCE(SUM(size_bytes), 0)
FROM access_log
WHERE cache_hit = TRUE AND created_at > NOW() - INTERVAL '30 days'
`).Scan(&stats.BandwidthSaved30d)
if err != nil {
return nil, err
}
return &stats, nil return &stats, nil
} }
+1 -6
View File
@@ -9,11 +9,6 @@ import (
"git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/internal/storage"
) )
// blobGracePeriod is how old an orphaned blob must be before GC will delete
// it. This avoids racing in-flight dedup uploads that insert the blob row
// before the referencing artifact/local_files row exists.
const blobGracePeriod = 1 * time.Hour
type Collector struct { type Collector struct {
db *database.DB db *database.DB
store *storage.S3 store *storage.S3
@@ -43,7 +38,7 @@ func (c *Collector) Run(ctx context.Context) {
func (c *Collector) sweep(ctx context.Context) { func (c *Collector) sweep(ctx context.Context) {
start := time.Now() start := time.Now()
orphaned, err := c.db.FindOrphanedBlobs(ctx, blobGracePeriod) orphaned, err := c.db.FindOrphanedBlobs(ctx)
if err != nil { if err != nil {
slog.Error("gc: find orphaned blobs", "error", err) slog.Error("gc: find orphaned blobs", "error", err)
return return
-114
View File
@@ -1,114 +0,0 @@
package gc
import (
"bytes"
"context"
"os"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
var (
testDB *database.DB
testStore *storage.S3
)
func TestMain(m *testing.M) {
ctx := context.Background()
dsn, termPG, err := testsupport.StartPostgres(ctx)
if err != nil {
os.Exit(m.Run())
}
minio, termMinio, err := testsupport.StartMinio(ctx)
if err != nil {
termPG()
os.Exit(m.Run())
}
db, err := database.New(dsn)
if err != nil {
panic(err)
}
var s3 *storage.S3
for i := 0; i < 20; i++ {
if s3, err = storage.NewS3(minio.Endpoint, minio.AccessKey, minio.SecretKey, "gc-test", false, ""); err == nil {
break
}
time.Sleep(500 * time.Millisecond)
}
if err != nil {
panic(err)
}
testDB = db
testStore = s3
code := m.Run()
db.Close()
termMinio()
termPG()
if code != 0 {
os.Exit(code)
}
}
func TestSweepDeletesOldOrphan(t *testing.T) {
if testDB == nil {
t.Skip("Docker unavailable")
}
ctx := context.Background()
hash := "sha256:gcorphan"
key := storage.BlobKey("gcorphan")
if err := testStore.Upload(ctx, key, bytes.NewReader([]byte("orphan")), 6, "application/octet-stream"); err != nil {
t.Fatal(err)
}
if err := testDB.UpsertBlob(ctx, hash, key, 6, "application/octet-stream"); err != nil {
t.Fatal(err)
}
// Age the blob past the grace period.
if _, err := testDB.Pool.Exec(ctx, `UPDATE blobs SET created_at = now() - interval '2 hours' WHERE content_hash = $1`, hash); err != nil {
t.Fatal(err)
}
c := New(testDB, testStore, time.Hour)
c.sweep(ctx)
if exists, _ := testStore.Exists(ctx, key); exists {
t.Error("expected orphan object deleted from store")
}
orphans, _ := testDB.FindOrphanedBlobs(ctx, 0)
for _, b := range orphans {
if b.ContentHash == hash {
t.Error("expected orphan blob row deleted")
}
}
}
func TestSweepNoOrphans(t *testing.T) {
if testDB == nil {
t.Skip("Docker unavailable")
}
// A sweep with nothing to collect should be a clean no-op.
New(testDB, testStore, time.Hour).sweep(context.Background())
}
func TestRunStopsOnContextCancel(t *testing.T) {
if testDB == nil {
t.Skip("Docker unavailable")
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
New(testDB, testStore, time.Hour).Run(ctx)
close(done)
}()
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("Run did not return after context cancel")
}
}
-60
View File
@@ -1,60 +0,0 @@
package alpine
import (
"context"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestType(t *testing.T) {
if (&Provider{}).Type() != models.PackageAlpine {
t.Fatal("wrong type")
}
}
func TestClassify(t *testing.T) {
p := &Provider{}
if p.Classify("v3.19/main/x86_64/APKINDEX.tar.gz") != provider.Mutable {
t.Error("APKINDEX should be mutable")
}
if p.Classify("v3.19/main/x86_64/curl-8.0-r0.apk") != provider.Immutable {
t.Error("apk should be immutable")
}
}
func TestContentType(t *testing.T) {
p := &Provider{}
cases := map[string]string{
"pkg.apk": "application/vnd.android.package-archive",
"APKINDEX.tar.gz": "application/gzip",
"something.random": "application/octet-stream",
}
for path, want := range cases {
if got := p.ContentType(path); got != want {
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
}
}
}
func TestUpstreamURL(t *testing.T) {
p := &Provider{}
got := p.UpstreamURL(models.Remote{BaseURL: "https://dl-cdn.alpinelinux.org/alpine/"}, "/v3.19/main/x86_64/curl.apk")
if got != "https://dl-cdn.alpinelinux.org/alpine/v3.19/main/x86_64/curl.apk" {
t.Errorf("got %q", got)
}
}
func TestRewriteResponse(t *testing.T) {
if out, err := (&Provider{}).RewriteResponse([]byte("x"), models.Remote{}, "http://proxy"); out != nil || err != nil {
t.Error("alpine never rewrites")
}
}
func TestAuthHeaders(t *testing.T) {
h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
if h.Get("Authorization") == "" {
t.Error("expected auth header")
}
}
@@ -1,53 +0,0 @@
package docker
import (
"context"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestDockerClassifyBranches(t *testing.T) {
p := &Provider{}
if p.Classify("library/nginx/tags/list") != provider.Mutable {
t.Error("tags/list should be mutable")
}
if p.Classify("library/nginx/manifests/latest") != provider.Mutable {
t.Error("tag manifest should be mutable")
}
if p.Classify("library/nginx/manifests/sha256:abcdef") != provider.Immutable {
t.Error("digest manifest should be immutable")
}
if p.Classify("library/nginx/blobs/sha256:abc") != provider.Immutable {
t.Error("blob should be immutable")
}
}
func TestDockerContentType(t *testing.T) {
p := &Provider{}
if p.ContentType("x/blobs/sha256:abc") != "application/octet-stream" {
t.Error("blob content type")
}
if p.ContentType("x/manifests/latest") != "application/vnd.docker.distribution.manifest.v2+json" {
t.Error("manifest content type")
}
if p.ContentType("x/tags/list") != "application/json" {
t.Error("default content type")
}
}
func TestDockerRewriteAndAuth(t *testing.T) {
p := &Provider{}
if out, err := p.RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
t.Error("docker never rewrites")
}
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
if h.Get("Authorization") == "" {
t.Error("expected basic auth header")
}
h, _ = p.AuthHeaders(context.Background(), models.Remote{})
if h.Get("Authorization") != "" {
t.Error("no creds, no header")
}
}
@@ -1,13 +0,0 @@
package generic
import (
"testing"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestGenericRewriteResponse(t *testing.T) {
if out, err := (&Provider{}).RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
t.Error("generic never rewrites")
}
}
@@ -1,27 +0,0 @@
package goproxy
import (
"context"
"testing"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestGoProxyURLAuthRewrite(t *testing.T) {
p := &Provider{}
if got := p.UpstreamURL(models.Remote{BaseURL: "https://proxy.golang.org/"}, "/mod/@v/list"); got != "https://proxy.golang.org/mod/@v/list" {
t.Errorf("upstream url %q", got)
}
if out, err := p.RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
t.Error("goproxy never rewrites")
}
if h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"}); h.Get("Authorization") == "" {
t.Error("expected basic auth header")
}
if got := p.ContentType("mod/@v/v1.0.0.info"); got != "application/json" {
t.Errorf("info content type %q", got)
}
if got := p.ContentType("mod/@v/v1.0.0.mod"); got != "text/plain" {
t.Errorf("mod content type %q", got)
}
}
-18
View File
@@ -1,18 +0,0 @@
package helm
import "testing"
func TestHelmContentTypeBranches(t *testing.T) {
p := &Provider{}
for path, want := range map[string]string{
"charts/x-1.0.0.tgz": "application/gzip",
"x.tar.gz": "application/gzip",
"index.yaml": "text/yaml",
"x.yml": "text/yaml",
"other": "application/octet-stream",
} {
if got := p.ContentType(path); got != want {
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
}
}
}
-78
View File
@@ -1,78 +0,0 @@
package npm
import (
"context"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestType(t *testing.T) {
if (&Provider{}).Type() != models.PackageNPM {
t.Fatal("wrong type")
}
}
func TestClassify(t *testing.T) {
p := &Provider{}
if p.Classify("pkg/-/pkg-1.0.0.tgz") != provider.Immutable {
t.Error("tgz should be immutable")
}
if p.Classify("pkg") != provider.Mutable {
t.Error("metadata should be mutable")
}
}
func TestContentType(t *testing.T) {
p := &Provider{}
if p.ContentType("pkg/-/pkg-1.0.0.tgz") != "application/gzip" {
t.Error("tgz content type")
}
if p.ContentType("pkg") != "application/json" {
t.Error("metadata content type")
}
}
func TestUpstreamURL(t *testing.T) {
p := &Provider{}
got := p.UpstreamURL(models.Remote{BaseURL: "https://registry.npmjs.org/"}, "/pkg")
if got != "https://registry.npmjs.org/pkg" {
t.Errorf("got %q", got)
}
}
func TestRewriteResponse(t *testing.T) {
p := &Provider{}
remote := models.Remote{Name: "npmjs", BaseURL: "https://registry.npmjs.org"}
if out, _ := p.RewriteResponse([]byte(`{"a":1}`), remote, ""); out != nil {
t.Error("empty proxyBaseURL should be a no-op")
}
if out, _ := p.RewriteResponse([]byte("not json"), remote, "http://proxy"); out != nil {
t.Error("invalid json should be a no-op")
}
body := []byte(`{"tarball":"https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz"}`)
out, err := p.RewriteResponse(body, remote, "http://proxy")
if err != nil {
t.Fatal(err)
}
if string(out) != `{"tarball":"http://proxy/api/v1/remote/npmjs/pkg/-/pkg-1.0.0.tgz"}` {
t.Errorf("rewrite: %s", out)
}
if out, _ := p.RewriteResponse([]byte(`{"x":"unrelated"}`), remote, "http://proxy"); out != nil {
t.Error("no matching base URL should be a no-op")
}
}
func TestAuthHeaders(t *testing.T) {
p := &Provider{}
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "pw"})
if h.Get("Authorization") == "" {
t.Error("expected auth header when credentials set")
}
h, _ = p.AuthHeaders(context.Background(), models.Remote{})
if h.Get("Authorization") != "" {
t.Error("expected no auth header without credentials")
}
}
-92
View File
@@ -3,7 +3,6 @@ package provider
import ( import (
"context" "context"
"fmt" "fmt"
"io"
"net/http" "net/http"
"git.unkin.net/unkin/artifactapi/pkg/models" "git.unkin.net/unkin/artifactapi/pkg/models"
@@ -25,97 +24,6 @@ type Provider interface {
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error) AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
} }
type FileEntry struct {
FilePath string
ContentHash string
}
type FileStore interface {
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
ListPackages(ctx context.Context, repoName string) ([]string, error)
}
type LocalUploader interface {
ValidateUpload(filePath string) (storagePath, contentType string, err error)
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
}
type LocalIndexer interface {
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
}
type BlobReader interface {
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
}
type PostUploadHook interface {
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
}
// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that
// feeds generated repodata) after a local file is removed.
type PostDeleteHook interface {
AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error
}
type MetadataStore interface {
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
}
type MetadataDeleter interface {
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
}
type RPMMetadataReader interface {
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
}
type RPMMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Epoch int
Version string
Release string
Arch string
Summary string
Description string
RPMSize int64
InstalledSize int64
License string
Vendor string
Group string
BuildHost string
SourceRPM string
URL string
Packager string
Requires []RPMDep
Provides []RPMDep
Files []RPMFile
Changelogs []RPMChangelog
}
type RPMDep struct {
Name string `json:"name"`
Flags string `json:"flags,omitempty"`
Epoch string `json:"epoch,omitempty"`
Version string `json:"version,omitempty"`
Release string `json:"release,omitempty"`
}
type RPMFile struct {
Path string `json:"path"`
Type string `json:"type,omitempty"`
}
type RPMChangelog struct {
Author string `json:"author"`
Date int64 `json:"date"`
Text string `json:"text"`
}
type IndexMerger interface { type IndexMerger interface {
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
} }
-78
View File
@@ -1,78 +0,0 @@
package puppet
import (
"context"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestType(t *testing.T) {
if (&Provider{}).Type() != models.PackagePuppet {
t.Fatal("wrong type")
}
}
func TestClassify(t *testing.T) {
p := &Provider{}
if p.Classify("v3/modules/puppetlabs-stdlib") != provider.Mutable {
t.Error("modules should be mutable")
}
if p.Classify("v3/releases?module=x") != provider.Mutable {
t.Error("releases should be mutable")
}
if p.Classify("v3/files/puppetlabs-stdlib-1.0.0.tar.gz") != provider.Immutable {
t.Error("files should be immutable")
}
}
func TestContentType(t *testing.T) {
p := &Provider{}
if p.ContentType("x/mod-1.0.0.tar.gz") != "application/gzip" {
t.Error("tar.gz")
}
if p.ContentType("v3/modules/x") != "application/json" {
t.Error("v3 json")
}
if p.ContentType("other") != "application/octet-stream" {
t.Error("default")
}
}
func TestUpstreamURL(t *testing.T) {
got := (&Provider{}).UpstreamURL(models.Remote{BaseURL: "https://forgeapi.puppet.com/"}, "/v3/modules/x")
if got != "https://forgeapi.puppet.com/v3/modules/x" {
t.Errorf("got %q", got)
}
}
func TestRewriteResponse(t *testing.T) {
p := &Provider{}
remote := models.Remote{Name: "forge", BaseURL: "https://forgeapi.puppet.com"}
if out, _ := p.RewriteResponse([]byte("x"), remote, ""); out != nil {
t.Error("empty proxyBaseURL is a no-op")
}
body := []byte(`{"file_uri":"/v3/files/mod.tar.gz","home":"https://forgeapi.puppet.com/x"}`)
out, err := p.RewriteResponse(body, remote, "http://proxy")
if err != nil {
t.Fatal(err)
}
s := string(out)
if !strings.Contains(s, "http://proxy/api/v1/remote/forge/v3/files/mod.tar.gz") {
t.Errorf("v3/files not rewritten: %s", s)
}
if !strings.Contains(s, "http://proxy/api/v1/remote/forge/x") {
t.Errorf("base URL not rewritten: %s", s)
}
}
func TestAuthHeaders(t *testing.T) {
h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{})
if h.Get("Authorization") != "" {
t.Error("no credentials, no header")
}
}
-180
View File
@@ -2,10 +2,7 @@ package pypi
import ( import (
"context" "context"
"fmt"
"io"
"net/http" "net/http"
"regexp"
"strings" "strings"
"git.unkin.net/unkin/artifactapi/internal/auth" "git.unkin.net/unkin/artifactapi/internal/auth"
@@ -17,9 +14,6 @@ func init() {
provider.Register(&Provider{}) provider.Register(&Provider{})
} }
var fileRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*\.(whl|tar\.gz|zip)$`)
var normalizeRe = regexp.MustCompile(`[-_.]+`)
type Provider struct{} type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackagePyPI } func (p *Provider) Type() models.PackageType { return models.PackagePyPI }
@@ -66,177 +60,3 @@ func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseU
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil return auth.BasicHeaders(remote), nil
} }
func normalize(name string) string {
return strings.ToLower(normalizeRe.ReplaceAllString(name, "-"))
}
func packageFromWheel(filename string) string {
parts := strings.SplitN(filename, "-", 3)
if len(parts) < 2 {
return ""
}
return normalize(parts[0])
}
func packageFromSdist(filename string) string {
name := filename
for _, suffix := range []string{".tar.gz", ".zip"} {
if strings.HasSuffix(name, suffix) {
name = strings.TrimSuffix(name, suffix)
break
}
}
idx := strings.LastIndex(name, "-")
if idx <= 0 {
return ""
}
return normalize(name[:idx])
}
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
filename := filePath
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
filename = filePath[idx+1:]
}
if !fileRe.MatchString(filename) {
return "", "", fmt.Errorf("filename %q must be a .whl, .tar.gz, or .zip file", filename)
}
var pkgName string
if strings.HasSuffix(filename, ".whl") {
pkgName = packageFromWheel(filename)
} else {
pkgName = packageFromSdist(filename)
}
if pkgName == "" {
return "", "", fmt.Errorf("cannot parse package name from %q", filename)
}
ct := "application/zip"
if strings.HasSuffix(filename, ".tar.gz") {
ct = "application/gzip"
}
return pkgName + "/" + filename, ct, nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
parts := strings.SplitN(storagePath, "/", 2)
filename := storagePath
if len(parts) == 2 {
filename = parts[1]
}
return map[string]any{
"package": parts[0],
"filename": filename,
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
if path == "simple" || path == "simple/" {
p.servePackageList(w, r, files, repoName)
return true
}
if strings.HasPrefix(path, "simple/") {
pkg := strings.TrimPrefix(path, "simple/")
pkg = strings.TrimSuffix(pkg, "/")
if pkg != "" && !strings.Contains(pkg, "/") {
p.servePackageFiles(w, r, files, repoName, pkg)
return true
}
}
return false
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
if !strings.HasPrefix(path, "simple/") {
return nil, fmt.Errorf("unsupported index path: %q", path)
}
pkg := strings.TrimPrefix(path, "simple/")
pkg = strings.TrimSuffix(pkg, "/")
if pkg == "" {
return p.generatePackageListHTML(ctx, files, repoName)
}
return p.generatePackageFilesHTML(ctx, files, repoName, pkg)
}
func (p *Provider) servePackageList(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName string) {
body, err := p.generatePackageListHTML(r.Context(), files, repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusOK)
w.Write(body)
}
func (p *Provider) servePackageFiles(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, packageName string) {
normalized := normalize(packageName)
prefix := normalized + "/"
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if len(entries) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, f := range entries {
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n",
normalized, filename, hash, filename)
}
b.WriteString("</body></html>\n")
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusOK)
io.WriteString(w, b.String())
}
func (p *Provider) generatePackageListHTML(ctx context.Context, files provider.FileStore, repoName string) ([]byte, error) {
packages, err := files.ListPackages(ctx, repoName)
if err != nil {
return nil, err
}
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, pkg := range packages {
fmt.Fprintf(&b, "<a href=\"%s/\">%s</a>\n", pkg, pkg)
}
b.WriteString("</body></html>\n")
return []byte(b.String()), nil
}
func (p *Provider) generatePackageFilesHTML(ctx context.Context, files provider.FileStore, repoName, packageName string) ([]byte, error) {
normalized := normalize(packageName)
prefix := normalized + "/"
entries, err := files.ListFilesByPrefix(ctx, repoName, prefix)
if err != nil {
return nil, err
}
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, f := range entries {
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
fmt.Fprintf(&b, "<a href=\"%s/%s#sha256=%s\">%s</a>\n",
normalized, filename, hash, filename)
}
b.WriteString("</body></html>\n")
return []byte(b.String()), nil
}
-177
View File
@@ -1,177 +0,0 @@
package pypi
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// fakeFileStore is an in-memory provider.FileStore for exercising local index
// generation without a database.
type fakeFileStore struct {
packages []string
files map[string][]provider.FileEntry
}
func (f *fakeFileStore) ListPackages(_ context.Context, _ string) ([]string, error) {
return f.packages, nil
}
func (f *fakeFileStore) ListFilesByPrefix(_ context.Context, _, prefix string) ([]provider.FileEntry, error) {
return f.files[prefix], nil
}
func TestTypeClassifyContentType(t *testing.T) {
p := &Provider{}
if p.Type() != models.PackagePyPI {
t.Fatal("type")
}
if p.Classify("simple/foo/") != provider.Mutable {
t.Error("simple index should be mutable")
}
if p.Classify("packages/foo-1.0.whl") != provider.Immutable {
t.Error("wheel should be immutable")
}
cases := map[string]string{
"foo-1.0-py3-none-any.whl": "application/zip",
"foo-1.0.zip": "application/zip",
"foo-1.0.tar.gz": "application/gzip",
"simple/foo/": "text/html",
"weird": "application/octet-stream",
}
for path, want := range cases {
if got := p.ContentType(path); got != want {
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
}
}
}
func TestUpstreamURL(t *testing.T) {
p := &Provider{}
if got := p.UpstreamURL(models.Remote{BaseURL: "https://files.example.com"}, "packages/foo.whl"); got != "https://files.example.com/packages/foo.whl" {
t.Errorf("got %q", got)
}
if got := p.UpstreamURL(models.Remote{BaseURL: "https://x"}, "simple/foo/"); got != "https://pypi.org/simple/foo/" {
t.Errorf("simple should hit pypi.org, got %q", got)
}
}
func TestValidateUpload(t *testing.T) {
p := &Provider{}
sp, ct, err := p.ValidateUpload("numpy-1.26.0-cp311-cp311-linux_x86_64.whl")
if err != nil || sp != "numpy/numpy-1.26.0-cp311-cp311-linux_x86_64.whl" || ct != "application/zip" {
t.Errorf("wheel: sp=%q ct=%q err=%v", sp, ct, err)
}
sp, ct, err = p.ValidateUpload("requests-2.31.0.tar.gz")
if err != nil || sp != "requests/requests-2.31.0.tar.gz" || ct != "application/gzip" {
t.Errorf("sdist: sp=%q ct=%q err=%v", sp, ct, err)
}
if _, _, err := p.ValidateUpload("not-a-package.txt"); err == nil {
t.Error("expected error for bad extension")
}
}
func TestPackageNameParsing(t *testing.T) {
if got := packageFromWheel("Foo_Bar-1.0-py3-none-any.whl"); got != "foo-bar" {
t.Errorf("wheel name = %q", got)
}
if got := packageFromWheel("noseparator.whl"); got != "" {
t.Errorf("expected empty for unparseable wheel, got %q", got)
}
if got := packageFromSdist("My.Pkg-2.0.tar.gz"); got != "my-pkg" {
t.Errorf("sdist name = %q", got)
}
if got := packageFromSdist("noseparator.zip"); got != "" {
t.Errorf("expected empty, got %q", got)
}
}
func TestUploadResponse(t *testing.T) {
resp := (&Provider{}).UploadResponse("foo/foo-1.0.whl", "sha256:abc", 123)
if resp["filename"] != "foo-1.0.whl" || resp["package"] != "foo" || resp["content_hash"] != "sha256:abc" {
t.Errorf("unexpected upload response: %v", resp)
}
}
func TestRewriteResponse(t *testing.T) {
p := &Provider{}
if out, _ := p.RewriteResponse([]byte("x"), models.Remote{Name: "pypi"}, ""); out != nil {
t.Error("empty proxyBaseURL is a no-op")
}
body := []byte(`<a href="https://files.pythonhosted.org/packages/foo.whl">foo.whl</a>`)
out, err := p.RewriteResponse(body, models.Remote{Name: "pypi"}, "http://proxy")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), "http://proxy/api/v1/remote/pypi/") {
t.Errorf("not rewritten: %s", out)
}
}
func TestGenerateLocalIndex(t *testing.T) {
p := &Provider{}
fs := &fakeFileStore{
packages: []string{"foo", "bar"},
files: map[string][]provider.FileEntry{
"foo/": {{FilePath: "foo/foo-1.0-py3-none-any.whl", ContentHash: "sha256:aaa"}},
},
}
list, err := p.GenerateLocalIndex(context.Background(), fs, "local", "simple/")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(list), "foo") || !strings.Contains(string(list), "bar") {
t.Errorf("package list missing entries: %s", list)
}
files, err := p.GenerateLocalIndex(context.Background(), fs, "local", "simple/foo/")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(files), "foo-1.0-py3-none-any.whl") {
t.Errorf("file list missing wheel: %s", files)
}
if _, err := p.GenerateLocalIndex(context.Background(), fs, "local", "notsimple"); err == nil {
t.Error("expected error for non-simple path")
}
}
func TestServeLocalIndexHTTP(t *testing.T) {
p := &Provider{}
fs := &fakeFileStore{
packages: []string{"foo"},
files: map[string][]provider.FileEntry{
"foo/": {{FilePath: "foo/foo-1.0-py3-none-any.whl", ContentHash: "sha256:aaa"}},
},
}
serve := func(path string) (*httptest.ResponseRecorder, bool) {
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
handled := p.ServeLocalIndex(w, r, fs, "local", path)
return w, handled
}
if w, ok := serve("simple/"); !ok || w.Code != 200 || !strings.Contains(w.Body.String(), "foo") {
t.Errorf("simple index: handled=%v code=%d body=%s", ok, w.Code, w.Body.String())
}
if w, ok := serve("simple/foo/"); !ok || w.Code != 200 || !strings.Contains(w.Body.String(), "foo-1.0-py3-none-any.whl") {
t.Errorf("package index: handled=%v code=%d body=%s", ok, w.Code, w.Body.String())
}
// Non-simple paths are not handled.
if _, ok := serve("packages/foo.whl"); ok {
t.Error("non-index path should not be handled")
}
}
func TestAuthHeaders(t *testing.T) {
h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
if h.Get("Authorization") == "" {
t.Error("expected auth header")
}
}
-396
View File
@@ -1,24 +1,13 @@
package rpm package rpm
import ( import (
"bytes"
"compress/gzip"
"context" "context"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"fmt"
"log/slog"
"net/http" "net/http"
"regexp" "regexp"
"strings" "strings"
"time"
rpmlib "github.com/cavaliergopher/rpm"
"git.unkin.net/unkin/artifactapi/internal/auth" "git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models" "git.unkin.net/unkin/artifactapi/pkg/models"
) )
@@ -66,388 +55,3 @@ func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte,
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil return auth.BasicHeaders(remote), nil
} }
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
filename := filePath
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
filename = filePath[idx+1:]
}
if !strings.HasSuffix(strings.ToLower(filename), ".rpm") {
return "", "", fmt.Errorf("file must be an .rpm package")
}
return "Packages/" + filename, "application/x-rpm", nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
filename := strings.TrimPrefix(storagePath, "Packages/")
return map[string]any{
"filename": filename,
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
reader, blobSize, err := blobs.Download(ctx, s3Key)
if err != nil {
slog.Error("rpm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
return
}
defer reader.Close()
pkg, err := rpmlib.Read(reader)
if err != nil {
slog.Error("rpm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
return
}
meta := &provider.RPMMetadata{
RepoName: repoName,
FilePath: storagePath,
ContentHash: contentHash,
Name: pkg.Name(),
Epoch: pkg.Epoch(),
Version: pkg.Version(),
Release: pkg.Release(),
Arch: pkg.Architecture(),
Summary: pkg.Summary(),
Description: pkg.Description(),
RPMSize: blobSize,
InstalledSize: int64(pkg.Size()),
License: pkg.License(),
Vendor: pkg.Vendor(),
Group: firstGroup(pkg.Groups()),
BuildHost: pkg.BuildHost(),
SourceRPM: pkg.SourceRPM(),
URL: pkg.URL(),
Packager: pkg.Packager(),
}
for _, req := range pkg.Requires() {
meta.Requires = append(meta.Requires, rpmDepFromEntry(req))
}
for _, prov := range pkg.Provides() {
meta.Provides = append(meta.Provides, rpmDepFromEntry(prov))
}
if meta.Requires == nil {
meta.Requires = []provider.RPMDep{}
}
if meta.Provides == nil {
meta.Provides = []provider.RPMDep{}
}
meta.Files = []provider.RPMFile{}
meta.Changelogs = []provider.RPMChangelog{}
if err := db.InsertRPMMetadata(ctx, meta); err != nil {
slog.Error("rpm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
return
}
slog.Info("rpm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
}
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
if err := db.DeleteRPMMetadata(ctx, repoName, storagePath); err != nil {
slog.Error("rpm metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
return err
}
slog.Info("rpm metadata: deleted", "repo", repoName, "path", storagePath)
return nil
}
func rpmDepFromEntry(e rpmlib.Dependency) provider.RPMDep {
dep := provider.RPMDep{Name: e.Name()}
if e.Flags() != 0 {
dep.Flags = rpmFlagString(e.Flags())
dep.Version = e.Version()
dep.Release = e.Release()
if e.Epoch() > 0 {
dep.Epoch = fmt.Sprintf("%d", e.Epoch())
}
}
return dep
}
func rpmFlagString(f int) string {
switch {
case f&0x08 != 0 && f&0x04 != 0:
return "GE"
case f&0x02 != 0 && f&0x04 != 0:
return "LE"
case f&0x08 != 0:
return "GT"
case f&0x02 != 0:
return "LT"
case f&0x04 != 0:
return "EQ"
default:
return ""
}
}
func firstGroup(groups []string) string {
if len(groups) > 0 {
return groups[0]
}
return "Unspecified"
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
if !strings.HasPrefix(path, "repodata/") {
return false
}
rpmReader, ok := files.(provider.RPMMetadataReader)
if !ok {
http.Error(w, "rpm metadata not available", http.StatusInternalServerError)
return true
}
tail := strings.TrimPrefix(path, "repodata/")
switch {
case tail == "repomd.xml":
p.serveRepomd(w, r, rpmReader, repoName)
case strings.HasSuffix(tail, "-primary.xml.gz"):
p.servePrimary(w, r, rpmReader, repoName)
case strings.HasSuffix(tail, "-filelists.xml.gz"):
p.serveFilelists(w, r, rpmReader, repoName)
case strings.HasSuffix(tail, "-other.xml.gz"):
p.serveOther(w, r, rpmReader, repoName)
default:
http.Error(w, "not found", http.StatusNotFound)
}
return true
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
return nil, fmt.Errorf("rpm local index generation for virtual repos not supported")
}
func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
primary := generatePrimaryXMLGZ(metas)
filelists := generateFilelistsXMLGZ(metas)
other := generateOtherXMLGZ(metas)
primaryHash := sha256Hex(primary)
filelistsHash := sha256Hex(filelists)
otherHash := sha256Hex(other)
repomd := generateRepomd(primaryHash, len(primary), filelistsHash, len(filelists), otherHash, len(other))
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(http.StatusOK)
w.Write(repomd)
}
func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/gzip")
w.WriteHeader(http.StatusOK)
w.Write(generatePrimaryXMLGZ(metas))
}
func (p *Provider) serveFilelists(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/gzip")
w.WriteHeader(http.StatusOK)
w.Write(generateFilelistsXMLGZ(metas))
}
func (p *Provider) serveOther(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/gzip")
w.WriteHeader(http.StatusOK)
w.Write(generateOtherXMLGZ(metas))
}
func generateRepomd(primaryHash string, primarySize int, filelistsHash string, filelistsSize int, otherHash string, otherSize int) []byte {
ts := fmt.Sprintf("%d", time.Now().Unix())
var b bytes.Buffer
b.WriteString(xml.Header)
b.WriteString(`<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">` + "\n")
fmt.Fprintf(&b, " <revision>%s</revision>\n", ts)
writeRepomdData(&b, "primary", primaryHash, primarySize, ts)
writeRepomdData(&b, "filelists", filelistsHash, filelistsSize, ts)
writeRepomdData(&b, "other", otherHash, otherSize, ts)
b.WriteString("</repomd>\n")
return b.Bytes()
}
func writeRepomdData(b *bytes.Buffer, dtype, hash string, size int, ts string) {
fmt.Fprintf(b, " <data type=\"%s\">\n", dtype)
fmt.Fprintf(b, " <checksum type=\"sha256\">%s</checksum>\n", hash)
fmt.Fprintf(b, " <location href=\"repodata/%s-%s.xml.gz\"/>\n", hash, dtype)
fmt.Fprintf(b, " <timestamp>%s</timestamp>\n", ts)
fmt.Fprintf(b, " <size>%d</size>\n", size)
fmt.Fprintf(b, " </data>\n")
}
func generatePrimaryXMLGZ(metas []provider.RPMMetadata) []byte {
var xmlBuf bytes.Buffer
xmlBuf.WriteString(xml.Header)
fmt.Fprintf(&xmlBuf, "<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"%d\">\n", len(metas))
for _, m := range metas {
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
fmt.Fprintf(&xmlBuf, "<package type=\"rpm\">\n")
fmt.Fprintf(&xmlBuf, " <name>%s</name>\n", xmlEscape(m.Name))
fmt.Fprintf(&xmlBuf, " <arch>%s</arch>\n", xmlEscape(m.Arch))
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
fmt.Fprintf(&xmlBuf, " <checksum type=\"sha256\" pkgid=\"YES\">%s</checksum>\n", pkgHash)
fmt.Fprintf(&xmlBuf, " <summary>%s</summary>\n", xmlEscape(m.Summary))
fmt.Fprintf(&xmlBuf, " <description>%s</description>\n", xmlEscape(m.Description))
if m.Packager != "" {
fmt.Fprintf(&xmlBuf, " <packager>%s</packager>\n", xmlEscape(m.Packager))
}
if m.URL != "" {
fmt.Fprintf(&xmlBuf, " <url>%s</url>\n", xmlEscape(m.URL))
}
fmt.Fprintf(&xmlBuf, " <time file=\"%d\" build=\"0\"/>\n", time.Now().Unix())
fmt.Fprintf(&xmlBuf, " <size package=\"%d\" installed=\"%d\" archive=\"0\"/>\n", m.RPMSize, m.InstalledSize)
fmt.Fprintf(&xmlBuf, " <location href=\"%s\"/>\n", xmlEscape(m.FilePath))
fmt.Fprintf(&xmlBuf, " <format>\n")
if m.License != "" {
fmt.Fprintf(&xmlBuf, " <rpm:license>%s</rpm:license>\n", xmlEscape(m.License))
}
if m.Vendor != "" {
fmt.Fprintf(&xmlBuf, " <rpm:vendor>%s</rpm:vendor>\n", xmlEscape(m.Vendor))
}
fmt.Fprintf(&xmlBuf, " <rpm:group>%s</rpm:group>\n", xmlEscape(m.Group))
if m.BuildHost != "" {
fmt.Fprintf(&xmlBuf, " <rpm:buildhost>%s</rpm:buildhost>\n", xmlEscape(m.BuildHost))
}
if m.SourceRPM != "" {
fmt.Fprintf(&xmlBuf, " <rpm:sourcerpm>%s</rpm:sourcerpm>\n", xmlEscape(m.SourceRPM))
}
if len(m.Provides) > 0 {
xmlBuf.WriteString(" <rpm:provides>\n")
for _, d := range m.Provides {
writeRPMEntry(&xmlBuf, d)
}
xmlBuf.WriteString(" </rpm:provides>\n")
}
if len(m.Requires) > 0 {
xmlBuf.WriteString(" <rpm:requires>\n")
for _, d := range m.Requires {
writeRPMEntry(&xmlBuf, d)
}
xmlBuf.WriteString(" </rpm:requires>\n")
}
fmt.Fprintf(&xmlBuf, " </format>\n")
fmt.Fprintf(&xmlBuf, "</package>\n")
}
xmlBuf.WriteString("</metadata>\n")
return gzipBytes(xmlBuf.Bytes())
}
func generateFilelistsXMLGZ(metas []provider.RPMMetadata) []byte {
var xmlBuf bytes.Buffer
xmlBuf.WriteString(xml.Header)
fmt.Fprintf(&xmlBuf, "<filelists xmlns=\"http://linux.duke.edu/metadata/filelists\" packages=\"%d\">\n", len(metas))
for _, m := range metas {
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
for _, f := range m.Files {
if f.Type != "" {
fmt.Fprintf(&xmlBuf, " <file type=\"%s\">%s</file>\n", f.Type, xmlEscape(f.Path))
} else {
fmt.Fprintf(&xmlBuf, " <file>%s</file>\n", xmlEscape(f.Path))
}
}
xmlBuf.WriteString("</package>\n")
}
xmlBuf.WriteString("</filelists>\n")
return gzipBytes(xmlBuf.Bytes())
}
func generateOtherXMLGZ(metas []provider.RPMMetadata) []byte {
var xmlBuf bytes.Buffer
xmlBuf.WriteString(xml.Header)
fmt.Fprintf(&xmlBuf, "<otherdata xmlns=\"http://linux.duke.edu/metadata/other\" packages=\"%d\">\n", len(metas))
for _, m := range metas {
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
for _, cl := range m.Changelogs {
fmt.Fprintf(&xmlBuf, " <changelog author=\"%s\" date=\"%d\">%s</changelog>\n",
xmlEscape(cl.Author), cl.Date, xmlEscape(cl.Text))
}
xmlBuf.WriteString("</package>\n")
}
xmlBuf.WriteString("</otherdata>\n")
return gzipBytes(xmlBuf.Bytes())
}
func writeRPMEntry(b *bytes.Buffer, d provider.RPMDep) {
if d.Flags != "" {
fmt.Fprintf(b, " <rpm:entry name=\"%s\" flags=\"%s\"", xmlEscape(d.Name), d.Flags)
if d.Epoch != "" {
fmt.Fprintf(b, " epoch=\"%s\"", d.Epoch)
}
if d.Version != "" {
fmt.Fprintf(b, " ver=\"%s\"", xmlEscape(d.Version))
}
if d.Release != "" {
fmt.Fprintf(b, " rel=\"%s\"", xmlEscape(d.Release))
}
b.WriteString("/>\n")
} else {
fmt.Fprintf(b, " <rpm:entry name=\"%s\"/>\n", xmlEscape(d.Name))
}
}
func xmlEscape(s string) string {
var b bytes.Buffer
xml.EscapeText(&b, []byte(s))
return b.String()
}
func gzipBytes(data []byte) []byte {
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
gz.Write(data)
gz.Close()
return buf.Bytes()
}
func sha256Hex(data []byte) string {
h := sha256.Sum256(data)
return hex.EncodeToString(h[:])
}
-276
View File
@@ -1,276 +0,0 @@
package rpm
import (
"bytes"
"compress/gzip"
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type fakeBlobReader struct{ data []byte }
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
}
type fakeMetaStore struct{ inserted *provider.RPMMetadata }
func (f *fakeMetaStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
f.inserted = m
return nil
}
type fakeRPMReader struct{ metas []provider.RPMMetadata }
func (f fakeRPMReader) ListRPMMetadataEntries(_ context.Context, _ string) ([]provider.RPMMetadata, error) {
return f.metas, nil
}
func (f fakeRPMReader) ListFilesByPrefix(_ context.Context, _, _ string) ([]provider.FileEntry, error) {
return nil, nil
}
func (f fakeRPMReader) ListPackages(_ context.Context, _ string) ([]string, error) { return nil, nil }
func TestRPMPureFuncs(t *testing.T) {
p := &Provider{}
if p.Type() != models.PackageRPM {
t.Error("type")
}
if p.Classify("repodata/repomd.xml") != provider.Mutable {
t.Error("repomd should be mutable")
}
if p.Classify("Packages/foo.rpm") != provider.Immutable {
t.Error("rpm should be immutable")
}
if p.ContentType("x.rpm") != "application/x-rpm" {
t.Error("rpm content type")
}
if got := p.UpstreamURL(models.Remote{BaseURL: "https://mirror/"}, "/Packages/x.rpm"); got != "https://mirror/Packages/x.rpm" {
t.Errorf("upstream url %q", got)
}
if out, _ := p.RewriteResponse(nil, models.Remote{}, "http://p"); out != nil {
t.Error("rpm never rewrites")
}
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
if h.Get("Authorization") == "" {
t.Error("auth header")
}
}
func TestRPMValidateUpload(t *testing.T) {
p := &Provider{}
sp, ct, err := p.ValidateUpload("dir/foo-1.0.noarch.rpm")
if err != nil || sp != "Packages/foo-1.0.noarch.rpm" || ct != "application/x-rpm" {
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
}
if _, _, err := p.ValidateUpload("foo.txt"); err == nil {
t.Error("expected error for non-rpm")
}
resp := p.UploadResponse("Packages/foo.rpm", "sha256:abc", 10)
if resp["content_hash"] != "sha256:abc" {
t.Errorf("upload response %v", resp)
}
}
func TestRPMAfterUpload(t *testing.T) {
data := testsupport.MinimalRPM("e2e-testpkg", "1.0", "1", "noarch")
store := &fakeMetaStore{}
(&Provider{}).AfterUpload(context.Background(), "myrepo", "Packages/e2e-testpkg-1.0-1.noarch.rpm",
"sha256:deadbeef", fakeBlobReader{data: data}, store)
m := store.inserted
if m == nil {
t.Fatal("no metadata inserted")
}
if m.Name != "e2e-testpkg" || m.Version != "1.0" || m.Release != "1" || m.Arch != "noarch" {
t.Errorf("unexpected metadata: %+v", m)
}
if m.RPMSize != int64(len(data)) {
t.Errorf("RPMSize = %d, want %d", m.RPMSize, len(data))
}
if len(m.Provides) == 0 {
t.Error("expected the package to provide itself")
}
}
type errBlobReader struct{}
func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
return nil, 0, io.ErrUnexpectedEOF
}
func TestRPMAfterUploadErrors(t *testing.T) {
// Download failure: no metadata inserted, no panic.
store := &fakeMetaStore{}
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", errBlobReader{}, store)
if store.inserted != nil {
t.Error("no metadata should be inserted on download error")
}
// Parse failure: garbage bytes are not a valid RPM.
store2 := &fakeMetaStore{}
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", fakeBlobReader{data: []byte("not an rpm")}, store2)
if store2.inserted != nil {
t.Error("no metadata should be inserted on parse error")
}
}
func TestRPMServeRepodata(t *testing.T) {
p := &Provider{}
reader := fakeRPMReader{metas: []provider.RPMMetadata{{
Name: "e2e-testpkg", Version: "1.0", Release: "1", Arch: "noarch",
Summary: "test & <special>",
ContentHash: "sha256:abc",
Requires: []provider.RPMDep{{Name: "libc", Flags: "GE", Version: "2.0"}},
Provides: []provider.RPMDep{{Name: "e2e-testpkg"}},
Files: []provider.RPMFile{{Path: "/usr/share/e2e/README", Type: "file"}},
Changelogs: []provider.RPMChangelog{{Author: "e2e", Date: 1, Text: "init"}},
}}}
serve := func(path string) *httptest.ResponseRecorder {
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
if !p.ServeLocalIndex(w, r, reader, "myrepo", path) {
t.Fatalf("ServeLocalIndex returned false for %q", path)
}
return w
}
if w := serve("repodata/repomd.xml"); w.Code != 200 || !strings.Contains(w.Body.String(), "<repomd") {
t.Errorf("repomd: code=%d body=%s", w.Code, w.Body.String())
}
for _, name := range []string{"repodata/h-primary.xml.gz", "repodata/h-filelists.xml.gz", "repodata/h-other.xml.gz"} {
w := serve(name)
if w.Code != 200 {
t.Errorf("%s: code %d", name, w.Code)
}
if _, err := gzip.NewReader(bytes.NewReader(w.Body.Bytes())); err != nil {
t.Errorf("%s: not gzip: %v", name, err)
}
}
// Unknown repodata file -> 404.
if w := serve("repodata/bogus"); w.Code != http.StatusNotFound {
t.Errorf("bogus repodata: code %d", w.Code)
}
// Non-repodata path -> not handled.
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/Packages/x.rpm", nil)
if p.ServeLocalIndex(w, r, reader, "myrepo", "Packages/x.rpm") {
t.Error("expected ServeLocalIndex false for non-repodata path")
}
}
type errRPMReader struct{}
func (errRPMReader) ListRPMMetadataEntries(context.Context, string) ([]provider.RPMMetadata, error) {
return nil, io.ErrUnexpectedEOF
}
func (errRPMReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
return nil, nil
}
func (errRPMReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
func TestRPMServeMetadataError(t *testing.T) {
p := &Provider{}
for _, path := range []string{"repodata/repomd.xml", "repodata/h-primary.xml.gz", "repodata/h-filelists.xml.gz", "repodata/h-other.xml.gz"} {
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
p.ServeLocalIndex(w, r, errRPMReader{}, "repo", path)
if w.Code != 500 {
t.Errorf("%s with failing reader = %d, want 500", path, w.Code)
}
}
}
func TestRPMFullMetadataXML(t *testing.T) {
// A fully-populated entry exercises every optional-field branch in the
// primary/filelists/other XML generators.
metas := []provider.RPMMetadata{{
Name: "full", Epoch: 1, Version: "2.0", Release: "3", Arch: "x86_64",
Summary: "s", Description: "d", License: "MIT", Vendor: "acme",
Group: "System", BuildHost: "build.example.com", SourceRPM: "full-2.0.src.rpm",
URL: "https://example.com", Packager: "pkgr", ContentHash: "sha256:abc",
RPMSize: 100, InstalledSize: 200,
Requires: []provider.RPMDep{{Name: "libc", Flags: "GE", Epoch: "0", Version: "2.0", Release: "1"}},
Provides: []provider.RPMDep{{Name: "full", Flags: "EQ", Version: "2.0"}},
Files: []provider.RPMFile{{Path: "/usr/bin/full", Type: "file"}, {Path: "/etc/full", Type: "dir"}},
Changelogs: []provider.RPMChangelog{{Author: "a", Date: 100, Text: "changed"}},
}}
for _, gen := range []func([]provider.RPMMetadata) []byte{generatePrimaryXMLGZ, generateFilelistsXMLGZ, generateOtherXMLGZ} {
zr, err := gzip.NewReader(bytes.NewReader(gen(metas)))
if err != nil {
t.Fatal(err)
}
if _, err := io.ReadAll(zr); err != nil {
t.Error(err)
}
}
}
func TestRPMPrimaryXMLContents(t *testing.T) {
// Exercise xmlEscape and dependency entry writing through the gzip'd XML.
metas := []provider.RPMMetadata{{
Name: "pkg", Version: "1", Release: "1", Arch: "x86_64", Summary: "a & b",
Requires: []provider.RPMDep{{Name: "dep", Flags: "EQ", Version: "1.0", Epoch: "0"}},
}}
gz := generatePrimaryXMLGZ(metas)
zr, err := gzip.NewReader(bytes.NewReader(gz))
if err != nil {
t.Fatal(err)
}
out, _ := io.ReadAll(zr)
s := string(out)
if !strings.Contains(s, "a &amp; b") {
t.Errorf("summary not xml-escaped: %s", s)
}
if !strings.Contains(s, "<name>pkg</name>") {
t.Errorf("package name missing: %s", s)
}
}
func TestRPMContentTypeAndHelpers(t *testing.T) {
p := &Provider{}
for path, want := range map[string]string{
"x.rpm": "application/x-rpm",
"repodata/repomd.xml": "application/xml",
"repodata/h-primary.xml.gz": "application/xml",
"repodata/h-primary.xml.xz": "application/xml",
"Packages/other": "application/octet-stream",
} {
if got := p.ContentType(path); got != want {
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
}
}
for flag, want := range map[int]string{
0x08 | 0x04: "GE",
0x02 | 0x04: "LE",
0x08: "GT",
0x02: "LT",
0x04: "EQ",
0x00: "",
} {
if got := rpmFlagString(flag); got != want {
t.Errorf("rpmFlagString(%d)=%q want %q", flag, got, want)
}
}
if firstGroup(nil) != "Unspecified" {
t.Error("empty groups should be Unspecified")
}
if firstGroup([]string{"System", "Base"}) != "System" {
t.Error("firstGroup should return the first")
}
}
func TestGenerateLocalIndexUnsupported(t *testing.T) {
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeRPMReader{}, "r", "simple/"); err == nil {
t.Error("expected unsupported error")
}
}
-170
View File
@@ -3,7 +3,6 @@ package terraform
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"net/http" "net/http"
"net/url" "net/url"
"regexp" "regexp"
@@ -20,33 +19,6 @@ func init() {
var versionsRe = regexp.MustCompile(`[^/]+/[^/]+/versions$`) var versionsRe = regexp.MustCompile(`[^/]+/[^/]+/versions$`)
var providerZipRe = regexp.MustCompile(
`^terraform-provider-([a-zA-Z0-9_-]+)_([0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`,
)
var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`)
// ParsedProviderZip describes a terraform-provider-{type}_{version}_{os}_{arch}.zip
// filename. Ok is false when the name doesn't match that convention.
type ParsedProviderZip struct {
Type string
Version string
OS string
Arch string
Ok bool
}
// ParseProviderZip extracts the type, version and platform from a provider zip
// filename (the base name, not a full path). It's the canonical parser shared by
// the network-mirror index and the provider registry handler.
func ParseProviderZip(filename string) ParsedProviderZip {
m := providerZipRe.FindStringSubmatch(filename)
if m == nil {
return ParsedProviderZip{}
}
return ParsedProviderZip{Type: m[1], Version: m[2], OS: m[3], Arch: m[4], Ok: true}
}
type Provider struct{} type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageTerraform } func (p *Provider) Type() models.PackageType { return models.PackageTerraform }
@@ -114,145 +86,3 @@ func rewriteDownloadURL(originalURL, releasesRemote, proxyBaseURL string) string
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil return auth.BasicHeaders(remote), nil
} }
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
parts := strings.Split(filePath, "/")
if len(parts) != 3 {
return "", "", fmt.Errorf("path must be {namespace}/{type}/{filename}.zip")
}
namespace, typeName, filename := parts[0], parts[1], parts[2]
m := providerZipRe.FindStringSubmatch(filename)
if m == nil {
return "", "", fmt.Errorf("filename %q does not match terraform-provider-{type}_{version}_{os}_{arch}.zip", filename)
}
if m[1] != typeName {
return "", "", fmt.Errorf("provider type in filename %q does not match path type %q", m[1], typeName)
}
return fmt.Sprintf("%s/%s/%s", namespace, typeName, filename), "application/zip", nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
parts := strings.Split(storagePath, "/")
if len(parts) != 3 {
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
}
m := providerZipRe.FindStringSubmatch(parts[2])
if m == nil {
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
}
return map[string]any{
"namespace": parts[0],
"type": parts[1],
"version": m[2],
"os": m[3],
"arch": m[4],
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
type terraformIndex struct {
Versions map[string]json.RawMessage `json:"versions"`
}
type terraformVersionDoc struct {
Archives map[string]terraformArchive `json:"archives"`
}
type terraformArchive struct {
URL string `json:"url"`
Hashes []string `json:"hashes,omitempty"`
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
parts := strings.Split(path, "/")
if len(parts) < 3 {
return false
}
namespace, typeName := parts[0], parts[1]
tail := parts[2]
if tail == "index.json" {
p.serveIndex(w, r, files, repoName, namespace, typeName)
return true
}
if strings.HasSuffix(tail, ".json") {
version := strings.TrimSuffix(tail, ".json")
if semverRe.MatchString(version) {
p.serveVersionDoc(w, r, files, repoName, namespace, typeName, version)
return true
}
}
return false
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
return nil, fmt.Errorf("terraform local index generation for virtual repos not supported")
}
func (p *Provider) serveIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName string) {
prefix := fmt.Sprintf("%s/%s/", namespace, typeName)
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
versions := map[string]json.RawMessage{}
for _, f := range entries {
filename := strings.TrimPrefix(f.FilePath, prefix)
m := providerZipRe.FindStringSubmatch(filename)
if m == nil {
continue
}
versions[m[2]] = json.RawMessage(`{}`)
}
if len(versions) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(terraformIndex{Versions: versions})
}
func (p *Provider) serveVersionDoc(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName, version string) {
prefix := fmt.Sprintf("%s/%s/terraform-provider-%s_%s_", namespace, typeName, typeName, version)
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
archives := map[string]terraformArchive{}
for _, f := range entries {
filename := strings.TrimPrefix(f.FilePath, fmt.Sprintf("%s/%s/", namespace, typeName))
m := providerZipRe.FindStringSubmatch(filename)
if m == nil || m[2] != version {
continue
}
platform := m[3] + "_" + m[4]
archive := terraformArchive{URL: filename}
if f.ContentHash != "" {
archive.Hashes = []string{"zh:" + strings.TrimPrefix(f.ContentHash, "sha256:")}
}
archives[platform] = archive
}
if len(archives) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(terraformVersionDoc{Archives: archives})
}
@@ -1,171 +0,0 @@
package terraform
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type fakeFileStore struct{ entries []provider.FileEntry }
func (f fakeFileStore) ListFilesByPrefix(_ context.Context, _, prefix string) ([]provider.FileEntry, error) {
var out []provider.FileEntry
for _, e := range f.entries {
if strings.HasPrefix(e.FilePath, prefix) {
out = append(out, e)
}
}
return out, nil
}
func (f fakeFileStore) ListPackages(_ context.Context, _ string) ([]string, error) { return nil, nil }
func TestTFPureFuncs(t *testing.T) {
p := &Provider{}
if p.Classify("hashicorp/aws/versions") != provider.Mutable {
t.Error("versions should be mutable")
}
if p.Classify("hashicorp/aws/terraform-provider-aws_1.0.0_linux_amd64.zip") != provider.Immutable {
t.Error("zip should be immutable")
}
if got := p.UpstreamURL(models.Remote{BaseURL: "https://registry.terraform.io"}, "hashicorp/aws/versions"); got != "https://registry.terraform.io/v1/providers/hashicorp/aws/versions" {
t.Errorf("upstream url %q", got)
}
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
if h.Get("Authorization") == "" {
t.Error("auth header")
}
_ = p.ContentType("x.json")
}
func TestTFValidateUpload(t *testing.T) {
p := &Provider{}
sp, ct, err := p.ValidateUpload("hashicorp/aws/terraform-provider-aws_1.2.3_linux_amd64.zip")
if err != nil || sp != "hashicorp/aws/terraform-provider-aws_1.2.3_linux_amd64.zip" || ct != "application/zip" {
t.Errorf("valid: sp=%q ct=%q err=%v", sp, ct, err)
}
if _, _, err := p.ValidateUpload("too/few"); err == nil {
t.Error("expected error for wrong path depth")
}
if _, _, err := p.ValidateUpload("ns/aws/not-a-provider.zip"); err == nil {
t.Error("expected error for bad filename")
}
if _, _, err := p.ValidateUpload("ns/gcp/terraform-provider-aws_1.0.0_linux_amd64.zip"); err == nil {
t.Error("expected error for type mismatch")
}
}
func TestTFUploadResponse(t *testing.T) {
p := &Provider{}
resp := p.UploadResponse("hashicorp/aws/terraform-provider-aws_1.2.3_linux_amd64.zip", "sha256:abc", 100)
if resp["namespace"] != "hashicorp" || resp["type"] != "aws" || resp["version"] != "1.2.3" || resp["os"] != "linux" || resp["arch"] != "amd64" {
t.Errorf("structured response wrong: %v", resp)
}
fallback := p.UploadResponse("weird/path", "sha256:x", 1)
if fallback["path"] != "weird/path" {
t.Errorf("fallback response wrong: %v", fallback)
}
}
func TestTFRewriteResponse(t *testing.T) {
p := &Provider{}
remote := models.Remote{Name: "tf", ReleasesRemote: "hashicorp-releases"}
if out, _ := p.RewriteResponse([]byte(`{"download_url":"x"}`), models.Remote{}, "http://proxy"); out != nil {
t.Error("no ReleasesRemote should be a no-op")
}
if out, _ := p.RewriteResponse([]byte("not json"), remote, "http://proxy"); out != nil {
t.Error("invalid json should be a no-op")
}
body := []byte(`{"download_url":"https://releases.hashicorp.com/terraform-provider-aws/1.0/aws.zip"}`)
out, err := p.RewriteResponse(body, remote, "http://proxy")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), "http://proxy/api/v1/remote/hashicorp-releases/") {
t.Errorf("download_url not rewritten: %s", out)
}
}
func TestTFServeLocalIndex(t *testing.T) {
p := &Provider{}
fs := fakeFileStore{entries: []provider.FileEntry{
{FilePath: "hashicorp/aws/terraform-provider-aws_1.0.0_linux_amd64.zip", ContentHash: "sha256:deadbeef"},
{FilePath: "hashicorp/aws/terraform-provider-aws_1.0.0_darwin_arm64.zip", ContentHash: "sha256:cafe"},
}}
serve := func(path string) *httptest.ResponseRecorder {
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
p.ServeLocalIndex(w, r, fs, "repo", path)
return w
}
if w := serve("hashicorp/aws/index.json"); w.Code != 200 || !strings.Contains(w.Body.String(), "1.0.0") {
t.Errorf("index.json: code=%d body=%s", w.Code, w.Body.String())
}
if w := serve("hashicorp/aws/1.0.0.json"); w.Code != 200 || !strings.Contains(w.Body.String(), "linux_amd64") {
t.Errorf("version doc: code=%d body=%s", w.Code, w.Body.String())
}
// Not a terraform index path.
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/x", nil)
if p.ServeLocalIndex(w, r, fs, "repo", "hashicorp/aws/other.txt") {
t.Error("non-index path should return false")
}
if p.ServeLocalIndex(httptest.NewRecorder(), r, fs, "repo", "too/short") {
t.Error("short path should return false")
}
}
func TestTFContentTypeAndEmptyIndex(t *testing.T) {
p := &Provider{}
for path, want := range map[string]string{
"x.zip": "application/zip",
"x.sig": "application/octet-stream",
"index.json": "application/json",
} {
if got := p.ContentType(path); got != want {
t.Errorf("ContentType(%q)=%q want %q", path, got, want)
}
}
// index / version doc with no matching files -> 404.
empty := fakeFileStore{}
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/hashicorp/aws/index.json", nil)
p.ServeLocalIndex(w, r, empty, "repo", "hashicorp/aws/index.json")
if w.Code != http.StatusNotFound {
t.Errorf("empty index should be 404, got %d", w.Code)
}
w = httptest.NewRecorder()
p.ServeLocalIndex(w, r, empty, "repo", "hashicorp/aws/1.0.0.json")
if w.Code != http.StatusNotFound {
t.Errorf("empty version doc should be 404, got %d", w.Code)
}
}
func TestRewriteDownloadURL(t *testing.T) {
// Empty proxy base -> unchanged.
if got := rewriteDownloadURL("https://x/a.zip", "rel", ""); got != "https://x/a.zip" {
t.Errorf("empty base: %q", got)
}
// Unparseable URL -> unchanged.
if got := rewriteDownloadURL("://bad", "rel", "http://p"); got != "://bad" {
t.Errorf("bad url: %q", got)
}
// Normal rewrite.
if got := rewriteDownloadURL("https://cdn/path/a.zip", "rel", "http://p"); got != "http://p/api/v1/remote/rel/path/a.zip" {
t.Errorf("rewrite: %q", got)
}
}
func TestTFGenerateLocalIndexUnsupported(t *testing.T) {
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeFileStore{}, "r", "x"); err == nil {
t.Error("expected unsupported error")
}
}
+1 -21
View File
@@ -2,7 +2,6 @@ package proxy
import ( import (
"regexp" "regexp"
"sync"
"git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models" "git.unkin.net/unkin/artifactapi/pkg/models"
@@ -61,29 +60,10 @@ func (c *Classifier) Classify(remote models.Remote, path string) Classification
return ClassImmutable return ClassImmutable
} }
// patternCache memoises regex compilation. Classify runs on every proxied
// request and previously recompiled each remote's pattern lists every time;
// keying by the pattern string lets each distinct pattern compile once and
// then be reused, with no invalidation needed (the pattern text is the key).
// A pattern that fails to compile is cached as a typed nil so we don't retry.
var patternCache sync.Map // map[string]*regexp.Regexp
func compileCached(pattern string) *regexp.Regexp {
if v, ok := patternCache.Load(pattern); ok {
return v.(*regexp.Regexp)
}
re, err := regexp.Compile(pattern)
if err != nil {
re = nil
}
patternCache.Store(pattern, re)
return re
}
func compilePatterns(patterns []string) []*regexp.Regexp { func compilePatterns(patterns []string) []*regexp.Regexp {
compiled := make([]*regexp.Regexp, 0, len(patterns)) compiled := make([]*regexp.Regexp, 0, len(patterns))
for _, p := range patterns { for _, p := range patterns {
if re := compileCached(p); re != nil { if re, err := regexp.Compile(p); err == nil {
compiled = append(compiled, re) compiled = append(compiled, re)
} }
} }
-52
View File
@@ -1,52 +0,0 @@
package proxy
import (
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestClassifierBranches(t *testing.T) {
gp, err := provider.Get(models.PackageGeneric)
if err != nil {
t.Fatal(err)
}
c := NewClassifier(gp)
if c.Classify(models.Remote{Blocklist: []string{`\.exe$`}}, "x.exe") != ClassDenied {
t.Error("blocklist match should be denied")
}
// Allowlist present but path doesn't match -> denied.
allow := models.Remote{Patterns: []string{`^allowed/`}}
if c.Classify(allow, "other/x") != ClassDenied {
t.Error("non-allowlisted path should be denied")
}
if c.Classify(allow, "allowed/x") != ClassImmutable {
t.Error("allowlisted generic path should be immutable")
}
if c.Classify(models.Remote{MutablePatterns: []string{`index$`}}, "a/index") != ClassMutable {
t.Error("mutable pattern override failed")
}
if c.Classify(models.Remote{ImmutablePatterns: []string{`\.bin$`}}, "a.bin") != ClassImmutable {
t.Error("immutable pattern failed")
}
// An invalid regex is skipped (not treated as a match) rather than denying.
if c.Classify(models.Remote{Blocklist: []string{`[invalid`}}, "anything") == ClassDenied {
t.Error("invalid blocklist regex should be skipped, not deny everything")
}
}
func TestClassificationString(t *testing.T) {
for c, want := range map[Classification]string{
ClassImmutable: "immutable",
ClassMutable: "mutable",
ClassDenied: "denied",
Classification(99): "unknown",
} {
if c.String() != want {
t.Errorf("Classification(%d).String() = %q, want %q", c, c.String(), want)
}
}
}
+86 -396
View File
@@ -4,13 +4,10 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"log/slog" "log/slog"
"net/http" "net/http"
"strings"
"time" "time"
"git.unkin.net/unkin/artifactapi/internal/cache" "git.unkin.net/unkin/artifactapi/internal/cache"
@@ -22,65 +19,19 @@ import (
const fetchLockTTL = 30 * time.Second const fetchLockTTL = 30 * time.Second
const (
accessLogBufferSize = 4096
accessLogBatchSize = 128
accessLogFlushEvery = 2 * time.Second
)
type Engine struct { type Engine struct {
db *database.DB db *database.DB
cache *cache.Redis cache *cache.Redis
store *storage.S3 store *storage.S3
cas *storage.CAS cas *storage.CAS
circuit *CircuitBreaker
accessLog chan database.AccessLogEntry
} }
func NewEngine(db *database.DB, c *cache.Redis, s *storage.S3) *Engine { func NewEngine(db *database.DB, c *cache.Redis, s *storage.S3) *Engine {
e := &Engine{ return &Engine{
db: db, db: db,
cache: c, cache: c,
store: s, store: s,
cas: storage.NewCAS(s), cas: storage.NewCAS(s),
circuit: NewCircuitBreaker(c),
accessLog: make(chan database.AccessLogEntry, accessLogBufferSize),
}
go e.runAccessLogWriter()
return e
}
// runAccessLogWriter drains the access-log channel and writes rows in batches,
// replacing a goroutine-per-request insert. It runs for the process lifetime;
// access logs are best-effort telemetry, so a small tail may be lost on abrupt
// shutdown.
func (e *Engine) runAccessLogWriter() {
ticker := time.NewTicker(accessLogFlushEvery)
defer ticker.Stop()
batch := make([]database.AccessLogEntry, 0, accessLogBatchSize)
flush := func() {
if len(batch) == 0 {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
if err := e.db.InsertAccessLogBatch(ctx, batch); err != nil {
slog.Warn("access log batch insert failed", "error", err, "count", len(batch))
}
cancel()
batch = batch[:0]
}
for {
select {
case entry := <-e.accessLog:
batch = append(batch, entry)
if len(batch) >= accessLogBatchSize {
flush()
}
case <-ticker.C:
flush()
}
} }
} }
@@ -91,7 +42,7 @@ type FetchResult struct {
Source string // "cache" or "remote" Source string // "cache" or "remote"
} }
func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, prov provider.Provider, clientHeaders ...http.Header) (*FetchResult, error) { func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*FetchResult, error) {
classifier := NewClassifier(prov) classifier := NewClassifier(prov)
class := classifier.Classify(remote, path) class := classifier.Classify(remote, path)
@@ -110,7 +61,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
result, err := e.serveFromStore(ctx, remote, path) result, err := e.serveFromStore(ctx, remote, path)
if err == nil { if err == nil {
result.Source = "cache" result.Source = "cache"
e.logAccess(remote.Name, path, true, result.Size, 0) go e.logAccess(remote.Name, path, true, result.Size, 0)
return result, nil return result, nil
} }
slog.Warn("cache hit but S3 miss, re-fetching", "remote", remote.Name, "path", path) slog.Warn("cache hit but S3 miss, re-fetching", "remote", remote.Name, "path", path)
@@ -122,12 +73,11 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
} }
if !locked { if !locked {
// Another request holds the fetch lock. Poll the store until the leader time.Sleep(500 * time.Millisecond)
// populates it rather than immediately racing to fetch upstream too; a result, err := e.serveFromStore(ctx, remote, path)
// cold-cache stampede otherwise hits upstream once per waiter. if err == nil {
if result := e.waitForStore(ctx, remote, path); result != nil {
result.Source = "cache" result.Source = "cache"
e.logAccess(remote.Name, path, true, result.Size, 0) go e.logAccess(remote.Name, path, true, result.Size, 0)
return result, nil return result, nil
} }
} }
@@ -146,138 +96,35 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
result, err := e.serveFromStore(ctx, remote, path) result, err := e.serveFromStore(ctx, remote, path)
if err == nil { if err == nil {
result.Source = "cache" result.Source = "cache"
e.logAccess(remote.Name, path, true, result.Size, 0) go e.logAccess(remote.Name, path, true, result.Size, 0)
return result, nil return result, nil
} }
} }
} }
} }
var fwdHeaders http.Header
if len(clientHeaders) > 0 && clientHeaders[0] != nil {
fwdHeaders = clientHeaders[0]
}
// Short-circuit upstream calls when the remote's breaker is open: serve
// stale from the store if we have it, otherwise fail fast rather than
// hammering a known-bad upstream.
if e.circuit.IsOpen(ctx, remote.Name) {
if stale, serr := e.serveFromStore(ctx, remote, path); serr == nil {
slog.Warn("circuit open, serving stale", "remote", remote.Name, "path", path)
stale.Source = "cache"
e.logAccess(remote.Name, path, true, stale.Size, 0)
return stale, nil
}
return nil, &ProxyError{Status: http.StatusServiceUnavailable, Message: "upstream circuit open"}
}
start := time.Now() start := time.Now()
result, err := e.fetchFromUpstream(ctx, remote, path, prov, class, ttl, fwdHeaders) result, err := e.fetchFromUpstream(ctx, remote, path, prov, class, ttl)
upstreamMS := int(time.Since(start).Milliseconds()) upstreamMS := int(time.Since(start).Milliseconds())
if err != nil { if err != nil {
if isNetworkError(err) {
e.circuit.RecordFailure(ctx, remote.Name)
}
if remote.StaleOnError && isNetworkError(err) { if remote.StaleOnError && isNetworkError(err) {
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl) _ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
stale, serr := e.serveFromStore(ctx, remote, path) stale, serr := e.serveFromStore(ctx, remote, path)
if serr == nil { if serr == nil {
slog.Warn("serving stale on upstream error", "remote", remote.Name, "path", path, "error", err) slog.Warn("serving stale on upstream error", "remote", remote.Name, "path", path, "error", err)
stale.Source = "cache" stale.Source = "cache"
e.logAccess(remote.Name, path, true, stale.Size, 0) go e.logAccess(remote.Name, path, true, stale.Size, 0)
return stale, nil return stale, nil
} }
} }
return nil, err return nil, err
} }
e.circuit.RecordSuccess(ctx, remote.Name) go e.logAccess(remote.Name, path, false, result.Size, upstreamMS)
e.logAccess(remote.Name, path, false, result.Size, upstreamMS)
return result, nil return result, nil
} }
// HeadResult carries artifact metadata for a HEAD request. There is no body. func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration) (*FetchResult, error) {
type HeadResult struct {
ContentType string
Size int64
Source string // "cache" or "remote"
}
// Head resolves artifact metadata without fetching or streaming the body.
// Cached artifacts/indexes are answered from the store metadata; on a miss it
// issues an upstream HEAD. It never downloads or caches the body.
func (e *Engine) Head(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*HeadResult, error) {
class := NewClassifier(prov).Classify(remote, path)
if class == ClassDenied {
return nil, &ProxyError{Status: http.StatusForbidden, Message: "access denied"}
}
if artifact, err := e.db.GetArtifact(ctx, remote.Name, path); err == nil && artifact != nil {
return &HeadResult{ContentType: artifact.ContentType, Size: artifact.SizeBytes, Source: "cache"}, nil
}
if info, err := e.store.Stat(ctx, storage.IndexKey(remote.Name, path)); err == nil {
return &HeadResult{ContentType: info.ContentType, Size: info.Size, Source: "cache"}, nil
}
return e.headUpstream(ctx, remote, path, prov)
}
func (e *Engine) headUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*HeadResult, error) {
url := prov.UpstreamURL(remote, path)
authHeaders, err := prov.AuthHeaders(ctx, remote)
if err != nil {
return nil, fmt.Errorf("auth headers: %w", err)
}
doHead := func(extra http.Header) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
for k, vv := range authHeaders {
for _, v := range vv {
req.Header.Add(k, v)
}
}
for k, vv := range extra {
for _, v := range vv {
req.Header.Set(k, v)
}
}
return http.DefaultClient.Do(req)
}
resp, err := doHead(nil)
if err != nil {
return nil, &UpstreamError{Err: err}
}
if resp.StatusCode == http.StatusUnauthorized {
resp.Body.Close()
token, _, terr := fetchBearerToken(ctx, resp.Header.Get("Www-Authenticate"), remote)
if terr == nil && token != "" {
resp, err = doHead(http.Header{"Authorization": []string{"Bearer " + token}})
if err != nil {
return nil, &UpstreamError{Err: err}
}
} else {
return nil, &ProxyError{Status: http.StatusUnauthorized, Message: "upstream returned 401"}
}
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, &ProxyError{Status: resp.StatusCode, Message: fmt.Sprintf("upstream returned %d", resp.StatusCode)}
}
contentType := prov.ContentType(path)
if ct := resp.Header.Get("Content-Type"); ct != "" {
contentType = ct
}
return &HeadResult{ContentType: contentType, Size: resp.ContentLength, Source: "remote"}, nil
}
func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration, clientHeaders http.Header) (*FetchResult, error) {
url := prov.UpstreamURL(remote, path) url := prov.UpstreamURL(remote, path)
authHeaders, err := prov.AuthHeaders(ctx, remote) authHeaders, err := prov.AuthHeaders(ctx, remote)
@@ -294,144 +141,94 @@ func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, pa
req.Header.Add(k, v) req.Header.Add(k, v)
} }
} }
if clientHeaders != nil {
if accept := clientHeaders.Get("Accept"); accept != "" {
req.Header.Set("Accept", accept)
}
}
resp, err := clientForRemote(remote).Do(req) resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return nil, &UpstreamError{Err: err} return nil, &UpstreamError{Err: err}
} }
if resp.StatusCode == http.StatusUnauthorized {
resp.Body.Close()
token, err := e.cachedBearerToken(ctx, resp.Header.Get("Www-Authenticate"), remote)
if err == nil && token != "" {
req2, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
req2.Header.Set("Authorization", "Bearer "+token)
if clientHeaders != nil {
if accept := clientHeaders.Get("Accept"); accept != "" {
req2.Header.Set("Accept", accept)
}
}
resp, err = clientForRemote(remote).Do(req2)
if err != nil {
return nil, &UpstreamError{Err: err}
}
} else {
return nil, &ProxyError{Status: http.StatusUnauthorized, Message: "upstream returned 401"}
}
}
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
resp.Body.Close() resp.Body.Close()
return nil, &ProxyError{Status: resp.StatusCode, Message: fmt.Sprintf("upstream returned %d", resp.StatusCode)} return nil, &ProxyError{Status: resp.StatusCode, Message: fmt.Sprintf("upstream returned %d", resp.StatusCode)}
} }
body, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return nil, fmt.Errorf("read upstream body: %w", err)
}
rewritten, err := prov.RewriteResponse(body, remote, "")
if err != nil {
return nil, fmt.Errorf("rewrite response: %w", err)
}
if rewritten != nil {
body = rewritten
}
contentType := prov.ContentType(path) contentType := prov.ContentType(path)
if ct := resp.Header.Get("Content-Type"); ct != "" { if ct := resp.Header.Get("Content-Type"); ct != "" && contentType == "application/octet-stream" {
contentType = ct contentType = ct
} }
// Mutable indexes are small and may be rewritten, so buffer them in memory.
if class == ClassMutable { if class == ClassMutable {
body, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return nil, fmt.Errorf("read upstream body: %w", err)
}
rewritten, err := prov.RewriteResponse(body, remote, "")
if err != nil {
return nil, fmt.Errorf("rewrite response: %w", err)
}
if rewritten != nil {
body = rewritten
}
s3Key := storage.IndexKey(remote.Name, path) s3Key := storage.IndexKey(remote.Name, path)
if err := e.store.Upload(ctx, s3Key, bytesReader(body), int64(len(body)), contentType); err != nil { if err := e.store.Upload(ctx, s3Key, bytesReader(body), int64(len(body)), contentType); err != nil {
return nil, fmt.Errorf("upload index: %w", err) return nil, fmt.Errorf("upload index: %w", err)
} }
etag := resp.Header.Get("ETag")
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
if etag != "" {
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
}
} else {
hash := sha256Hash(body)
s3Key := storage.BlobKey(hash)
exists, _ := e.store.Exists(ctx, s3Key)
if !exists {
if err := e.store.Upload(ctx, s3Key, bytesReader(body), int64(len(body)), contentType); err != nil {
return nil, fmt.Errorf("upload blob: %w", err)
}
}
contentHash := fmt.Sprintf("sha256:%s", hash)
if err := e.db.UpsertBlob(ctx, contentHash, s3Key, int64(len(body)), contentType); err != nil {
slog.Warn("upsert blob failed", "error", err)
}
if err := e.db.UpsertArtifact(ctx, remote.Name, path, contentHash, resp.Header.Get("ETag")); err != nil {
slog.Warn("upsert artifact failed", "error", err)
}
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl) _ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
if etag := resp.Header.Get("ETag"); etag != "" { if etag := resp.Header.Get("ETag"); etag != "" {
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl) _ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
} }
return &FetchResult{
Reader: io.NopCloser(bytesReader(body)),
ContentType: contentType,
Size: int64(len(body)),
Source: "remote",
}, nil
} }
// Immutable blobs are streamed through the content-addressable store
// (tempfile -> sha256 -> S3) so arbitrarily large artifacts never sit
// fully in memory. Immutable content is never rewritten in the proxy path.
casResult, err := e.cas.Store(ctx, resp.Body, contentType)
resp.Body.Close()
if err != nil {
return nil, fmt.Errorf("store blob: %w", err)
}
if err := e.db.UpsertBlob(ctx, casResult.ContentHash, casResult.S3Key, casResult.SizeBytes, contentType); err != nil {
slog.Warn("upsert blob failed", "error", err)
}
if err := e.db.UpsertArtifact(ctx, remote.Name, path, casResult.ContentHash, resp.Header.Get("ETag")); err != nil {
slog.Warn("upsert artifact failed", "error", err)
}
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
if etag := resp.Header.Get("ETag"); etag != "" {
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
}
reader, info, err := e.store.Download(ctx, casResult.S3Key)
if err != nil {
return nil, fmt.Errorf("serve stored blob: %w", err)
}
return &FetchResult{ return &FetchResult{
Reader: reader, Reader: io.NopCloser(bytesReader(body)),
ContentType: info.ContentType, ContentType: contentType,
Size: casResult.SizeBytes, Size: int64(len(body)),
Source: "remote", Source: "remote",
}, nil }, nil
} }
// waitForStore polls the store for an artifact populated by the request that
// holds the fetch lock, returning it once available or nil if it does not
// appear within the wait budget (after which the caller fetches upstream
// itself). It stops early if the request context is cancelled.
func (e *Engine) waitForStore(ctx context.Context, remote models.Remote, path string) *FetchResult {
const (
pollInterval = 100 * time.Millisecond
maxWait = 5 * time.Second
)
deadline := time.Now().Add(maxWait)
for {
if result, err := e.serveFromStore(ctx, remote, path); err == nil {
return result
}
if time.Now().After(deadline) {
return nil
}
select {
case <-ctx.Done():
return nil
case <-time.After(pollInterval):
}
}
}
func (e *Engine) serveFromStore(ctx context.Context, remote models.Remote, path string) (*FetchResult, error) { func (e *Engine) serveFromStore(ctx context.Context, remote models.Remote, path string) (*FetchResult, error) {
artifact, err := e.db.GetArtifact(ctx, remote.Name, path) artifact, err := e.db.GetArtifact(ctx, remote.Name, path)
if err == nil && artifact != nil { if err == nil && artifact != nil {
reader, info, err := e.store.Download(ctx, artifact.ContentHash[len("sha256:"):])
if err == nil {
_ = e.db.TouchArtifactAccess(ctx, remote.Name, path)
return &FetchResult{
Reader: reader,
ContentType: info.ContentType,
Size: info.Size,
}, nil
}
s3Key := storage.BlobKey(artifact.ContentHash[len("sha256:"):]) s3Key := storage.BlobKey(artifact.ContentHash[len("sha256:"):])
reader, info, err := e.store.Download(ctx, s3Key) reader, info, err = e.store.Download(ctx, s3Key)
if err == nil { if err == nil {
_ = e.db.TouchArtifactAccess(ctx, remote.Name, path) _ = e.db.TouchArtifactAccess(ctx, remote.Name, path)
return &FetchResult{ return &FetchResult{
@@ -473,7 +270,7 @@ func (e *Engine) checkUpstream(ctx context.Context, remote models.Remote, path,
} }
} }
resp, err := clientForRemote(remote).Do(req) resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return false, &UpstreamError{Err: err} return false, &UpstreamError{Err: err}
} }
@@ -494,20 +291,15 @@ func (e *Engine) ttlFor(remote models.Remote, class Classification) time.Duratio
} }
} }
// logAccess enqueues an access-log entry for the batch writer. It never blocks
// the request path: if the buffer is full the entry is dropped.
func (e *Engine) logAccess(remoteName, path string, cacheHit bool, size int64, upstreamMS int) { func (e *Engine) logAccess(remoteName, path string, cacheHit bool, size int64, upstreamMS int) {
select { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
case e.accessLog <- database.AccessLogEntry{ defer cancel()
RemoteName: remoteName, _ = e.db.InsertAccessLog(ctx, remoteName, path, cacheHit, size, upstreamMS, "")
Path: path, }
CacheHit: cacheHit,
SizeBytes: size, func sha256Hash(data []byte) string {
UpstreamMS: upstreamMS, h := sha256.Sum256(data)
}: return hex.EncodeToString(h[:])
default:
slog.Warn("access log buffer full, dropping entry", "remote", remoteName, "path", path)
}
} }
func bytesReader(data []byte) io.Reader { func bytesReader(data []byte) io.Reader {
@@ -527,110 +319,6 @@ func (r readerAt) ReadAt(p []byte, off int64) (n int, err error) {
return return
} }
// bearerTokenTTLDefault/Margin bound how long a token is cached: the default
// is used when the token endpoint omits expires_in, and the margin is
// subtracted so a cached token is refreshed slightly before it actually expires.
const (
bearerTokenTTLDefault = 60 * time.Second
bearerTokenTTLMargin = 10 * time.Second
)
func sha256Hash(data []byte) string {
h := sha256.Sum256(data)
return hex.EncodeToString(h[:])
}
// cachedBearerToken returns a bearer token for the given challenge, reusing a
// Redis-cached token for the same remote+challenge while it is still valid.
func (e *Engine) cachedBearerToken(ctx context.Context, wwwAuth string, remote models.Remote) (string, error) {
key := remote.Name + ":" + sha256Hash([]byte(wwwAuth))
if tok, err := e.cache.GetToken(ctx, key); err == nil && tok != "" {
return tok, nil
}
tok, ttl, err := fetchBearerToken(ctx, wwwAuth, remote)
if err != nil {
return "", err
}
if tok != "" {
if ttl <= 0 {
ttl = bearerTokenTTLDefault
}
if ttl > bearerTokenTTLMargin {
ttl -= bearerTokenTTLMargin
}
_ = e.cache.SetToken(ctx, key, tok, ttl)
}
return tok, nil
}
func fetchBearerToken(ctx context.Context, wwwAuth string, remote models.Remote) (string, time.Duration, error) {
if !strings.HasPrefix(wwwAuth, "Bearer ") {
return "", 0, fmt.Errorf("not a Bearer challenge")
}
params := map[string]string{}
for _, part := range strings.Split(wwwAuth[7:], ",") {
part = strings.TrimSpace(part)
eq := strings.Index(part, "=")
if eq < 0 {
continue
}
key := part[:eq]
val := strings.Trim(part[eq+1:], `"`)
params[key] = val
}
realm := params["realm"]
if realm == "" {
return "", 0, fmt.Errorf("no realm in Bearer challenge")
}
tokenURL := realm
sep := "?"
if s, ok := params["service"]; ok {
tokenURL += sep + "service=" + s
sep = "&"
}
if s, ok := params["scope"]; ok {
tokenURL += sep + "scope=" + s
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL, nil)
if err != nil {
return "", 0, err
}
if remote.Username != "" && remote.Password != "" {
req.SetBasicAuth(remote.Username, remote.Password)
}
resp, err := clientForRemote(remote).Do(req)
if err != nil {
return "", 0, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", 0, fmt.Errorf("token endpoint returned %d", resp.StatusCode)
}
var tokenResp struct {
Token string `json:"token"`
AccessToken string `json:"access_token"`
ExpiresIn int `json:"expires_in"`
}
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
return "", 0, err
}
ttl := time.Duration(tokenResp.ExpiresIn) * time.Second
if tokenResp.Token != "" {
return tokenResp.Token, ttl, nil
}
return tokenResp.AccessToken, ttl, nil
}
type ProxyError struct { type ProxyError struct {
Status int Status int
Message string Message string
@@ -646,6 +334,8 @@ func (e *UpstreamError) Error() string { return fmt.Sprintf("upstream error: %v"
func (e *UpstreamError) Unwrap() error { return e.Err } func (e *UpstreamError) Unwrap() error { return e.Err }
func isNetworkError(err error) bool { func isNetworkError(err error) bool {
var ue *UpstreamError if _, ok := err.(*UpstreamError); ok {
return errors.As(err, &ue) return true
}
return false
} }
-557
View File
@@ -1,557 +0,0 @@
package proxy
import (
"context"
"io"
"net/http"
"net/http/httptest"
"os"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/cache"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
_ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var (
testEngine *Engine
testCache *cache.Redis
testDB *database.DB
upstream *httptest.Server
)
func TestMain(m *testing.M) {
ctx := context.Background()
dsn, termPG, err := testsupport.StartPostgres(ctx)
if err != nil {
os.Exit(m.Run())
}
redisURL, termRedis, err := testsupport.StartRedis(ctx)
if err != nil {
termPG()
os.Exit(m.Run())
}
minio, termMinio, err := testsupport.StartMinio(ctx)
if err != nil {
termPG()
termRedis()
os.Exit(m.Run())
}
db, err := database.New(dsn)
if err != nil {
panic(err)
}
redis, err := cache.NewRedis(redisURL)
if err != nil {
panic(err)
}
var s3 *storage.S3
for i := 0; i < 20; i++ {
if s3, err = storage.NewS3(minio.Endpoint, minio.AccessKey, minio.SecretKey, "proxy-test", false, ""); err == nil {
break
}
time.Sleep(500 * time.Millisecond)
}
if err != nil {
panic(err)
}
testCache = redis
testDB = db
testEngine = NewEngine(db, redis, s3)
upstream = httptest.NewServer(http.HandlerFunc(mockUpstream))
code := m.Run()
upstream.Close()
db.Close()
termMinio()
termRedis()
termPG()
if code != 0 {
os.Exit(code)
}
}
func mockUpstream(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/blob.bin":
w.Header().Set("Content-Type", "application/octet-stream")
w.Write([]byte("immutable blob"))
case "/pkg": // npm metadata: mutable, supports revalidation
if r.Method == http.MethodHead && r.Header.Get("If-None-Match") == `"v1"` {
w.WriteHeader(http.StatusNotModified)
return
}
w.Header().Set("ETag", `"v1"`)
w.Write([]byte(`{"name":"pkg"}`))
case "/protected.bin": // requires a bearer token obtained from /token
if r.Header.Get("Authorization") != "Bearer minted-token" {
w.Header().Set("Www-Authenticate", `Bearer realm="`+upstream.URL+`/token",service="reg",scope="repo:pull"`)
w.WriteHeader(http.StatusUnauthorized)
return
}
w.Write([]byte("protected payload"))
case "/protected2.bin": // same challenge as /protected.bin
if r.Header.Get("Authorization") != "Bearer minted-token" {
w.Header().Set("Www-Authenticate", `Bearer realm="`+upstream.URL+`/token",service="reg",scope="repo:pull"`)
w.WriteHeader(http.StatusUnauthorized)
return
}
w.Write([]byte("protected payload 2"))
case "/token":
w.Write([]byte(`{"token":"minted-token","expires_in":300}`))
case "/token-at":
w.Write([]byte(`{"access_token":"at-token"}`))
case "/token-500":
w.WriteHeader(http.StatusInternalServerError)
case "/err500":
w.WriteHeader(http.StatusInternalServerError)
case "/noauth": // 401 with an unusable challenge (no realm)
w.Header().Set("Www-Authenticate", `Bearer service="reg"`)
w.WriteHeader(http.StatusUnauthorized)
default:
http.NotFound(w, r)
}
}
func requireStack(t *testing.T) {
t.Helper()
if testEngine == nil {
t.Skip("Docker unavailable; skipping proxy engine test")
}
}
func genericRemote(name string) models.Remote {
return models.Remote{Name: name, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, StaleOnError: true}
}
// seed inserts the remote so artifact rows (FK to remotes) can be stored.
func seed(t *testing.T, r models.Remote) models.Remote {
t.Helper()
rr := r
if err := testDB.CreateRemote(context.Background(), &rr); err != nil {
t.Fatalf("seed remote %s: %v", r.Name, err)
}
return r
}
func prov(t *testing.T, pt models.PackageType) provider.Provider {
p, err := provider.Get(pt)
if err != nil {
t.Fatalf("provider %s: %v", pt, err)
}
return p
}
func readAll(t *testing.T, res *FetchResult) string {
t.Helper()
defer res.Reader.Close()
b, _ := io.ReadAll(res.Reader)
return string(b)
}
func TestFetchImmutableMissThenHit(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := seed(t, genericRemote("eng-imm"))
p := prov(t, models.PackageGeneric)
res, err := testEngine.Fetch(ctx, r, "blob.bin", p)
if err != nil {
t.Fatalf("fetch: %v", err)
}
if res.Source != "remote" || readAll(t, res) != "immutable blob" {
t.Errorf("miss: source=%s", res.Source)
}
res, err = testEngine.Fetch(ctx, r, "blob.bin", p)
if err != nil {
t.Fatal(err)
}
if res.Source != "cache" || readAll(t, res) != "immutable blob" {
t.Errorf("hit: source=%s", res.Source)
}
}
func TestFetchDenied(t *testing.T) {
requireStack(t)
r := genericRemote("eng-deny")
r.Blocklist = []string{`\.secret$`}
_, err := testEngine.Fetch(context.Background(), r, "x.secret", prov(t, models.PackageGeneric))
var pe *ProxyError
if err == nil || !asProxyError(err, &pe) || pe.Status != http.StatusForbidden {
t.Errorf("expected 403 ProxyError, got %v", err)
}
}
func TestHead(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := seed(t, genericRemote("eng-head"))
p := prov(t, models.PackageGeneric)
// Uncached HEAD hits upstream.
h, err := testEngine.Head(ctx, r, "blob.bin", p)
if err != nil || h.Source != "remote" {
t.Fatalf("head uncached: %+v %v", h, err)
}
// Populate the cache, then HEAD should be served from metadata.
res, _ := testEngine.Fetch(ctx, r, "blob.bin", p)
res.Reader.Close()
h, err = testEngine.Head(ctx, r, "blob.bin", p)
if err != nil || h.Source != "cache" {
t.Errorf("head cached: %+v %v", h, err)
}
// Denied HEAD.
r.Blocklist = []string{".*"}
if _, err := testEngine.Head(ctx, r, "blob.bin", p); err == nil {
t.Error("expected denied head error")
}
}
func TestStaleOnError(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := seed(t, genericRemote("eng-stale"))
p := prov(t, models.PackageGeneric)
if _, err := testEngine.Fetch(ctx, r, "blob.bin", p); err != nil {
t.Fatal(err)
}
// Drop cache freshness so the next fetch goes upstream, then point at a
// dead upstream: stale-on-error must serve the stored copy.
testCache.FlushRemote(ctx, "eng-stale")
r.BaseURL = "http://127.0.0.1:1"
res, err := testEngine.Fetch(ctx, r, "blob.bin", p)
if err != nil {
t.Fatalf("expected stale serve, got %v", err)
}
if res.Source != "cache" || readAll(t, res) != "immutable blob" {
t.Errorf("stale: source=%s", res.Source)
}
}
func TestCircuitOpenServesStale(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := seed(t, genericRemote("eng-circuit"))
p := prov(t, models.PackageGeneric)
if _, err := testEngine.Fetch(ctx, r, "blob.bin", p); err != nil {
t.Fatal(err)
}
testCache.FlushRemote(ctx, "eng-circuit")
for i := 0; i < 6; i++ {
testEngine.circuit.RecordFailure(ctx, "eng-circuit")
}
res, err := testEngine.Fetch(ctx, r, "blob.bin", p)
if err != nil {
t.Fatalf("circuit-open should serve stale: %v", err)
}
if res.Source != "cache" {
t.Errorf("expected stale from open circuit, got %s", res.Source)
}
res.Reader.Close()
}
func TestMutableRevalidation(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := seed(t, models.Remote{Name: "eng-npm", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600, StaleOnError: true})
p := prov(t, models.PackageNPM)
res, err := testEngine.Fetch(ctx, r, "pkg", p)
if err != nil {
t.Fatalf("initial mutable fetch: %v", err)
}
res.Reader.Close()
// Expire only the freshness marker; the ETag persists, forcing a
// conditional revalidation that the upstream answers with 304.
testCache.SetTTL(ctx, "eng-npm", "pkg", time.Millisecond)
time.Sleep(10 * time.Millisecond)
res, err = testEngine.Fetch(ctx, r, "pkg", p)
if err != nil {
t.Fatalf("revalidation fetch: %v", err)
}
if res.Source != "cache" {
t.Errorf("revalidated response should come from cache, got %s", res.Source)
}
res.Reader.Close()
}
func TestBearerTokenFlow(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := seed(t, genericRemote("eng-bearer"))
p := prov(t, models.PackageGeneric)
// GET: 401 challenge -> token endpoint -> retry with bearer -> 200.
res, err := testEngine.Fetch(ctx, r, "protected.bin", p)
if err != nil {
t.Fatalf("bearer fetch: %v", err)
}
if readAll(t, res) != "protected payload" {
t.Error("bearer-protected content mismatch")
}
// A second protected path with the same challenge reuses the cached token.
res2, err := testEngine.Fetch(ctx, r, "protected2.bin", p)
if err != nil {
t.Fatalf("second bearer fetch: %v", err)
}
if readAll(t, res2) != "protected payload 2" {
t.Error("second bearer content mismatch")
}
// HEAD path also negotiates a bearer token (uncached).
testCache.FlushRemote(ctx, "eng-bearer")
testDB.DeleteArtifact(ctx, "eng-bearer", "protected.bin")
if h, err := testEngine.Head(ctx, r, "protected.bin", p); err != nil || h.Source != "cache" && h.Source != "remote" {
t.Fatalf("bearer head: %+v %v", h, err)
}
}
func TestFetchUpstreamError(t *testing.T) {
requireStack(t)
r := seed(t, genericRemote("eng-404"))
// Upstream 404 (no cached copy, stale-on-error can't help) -> ProxyError.
_, err := testEngine.Fetch(context.Background(), r, "missing", prov(t, models.PackageGeneric))
var pe *ProxyError
if err == nil || !asProxyError(err, &pe) || pe.Status != http.StatusNotFound {
t.Errorf("expected 404 ProxyError, got %v", err)
}
// HEAD of a missing upstream path also errors.
if _, err := testEngine.Head(context.Background(), r, "missing", prov(t, models.PackageGeneric)); err == nil {
t.Error("expected head error for missing path")
}
}
func TestFetchUpstreamStatusErrors(t *testing.T) {
requireStack(t)
ctx := context.Background()
p := prov(t, models.PackageGeneric)
r := seed(t, genericRemote("eng-500"))
_, err := testEngine.Fetch(ctx, r, "err500", p)
var pe *ProxyError
if err == nil || !asProxyError(err, &pe) || pe.Status != http.StatusInternalServerError {
t.Errorf("expected 500 ProxyError, got %v", err)
}
r = seed(t, genericRemote("eng-noauth"))
_, err = testEngine.Fetch(ctx, r, "noauth", p)
if err == nil || !asProxyError(err, &pe) || pe.Status != http.StatusUnauthorized {
t.Errorf("expected 401 ProxyError, got %v", err)
}
}
func TestBearerTokenParsing(t *testing.T) {
// Non-Bearer challenges and missing realms are rejected.
if _, _, err := fetchBearerToken(context.Background(), "Basic realm=x", models.Remote{}); err == nil {
t.Error("expected error for non-Bearer challenge")
}
if _, _, err := fetchBearerToken(context.Background(), `Bearer service="reg"`, models.Remote{}); err == nil {
t.Error("expected error for missing realm")
}
}
func TestWaitForStoreCoalesces(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := seed(t, genericRemote("eng-herd"))
p := prov(t, models.PackageGeneric)
// Fire concurrent cold-cache fetches: only one holds the lock, the others
// wait on the store (waitForStore) and pick up the result.
const n = 4
done := make(chan string, n)
for i := 0; i < n; i++ {
go func() {
res, err := testEngine.Fetch(ctx, r, "blob.bin", p)
if err != nil {
done <- "err:" + err.Error()
return
}
done <- readAll(t, res)
}()
}
for i := 0; i < n; i++ {
if got := <-done; got != "immutable blob" {
t.Errorf("concurrent fetch got %q", got)
}
}
}
func TestRevalidationUpstreamError(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := seed(t, models.Remote{Name: "eng-reval-err", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600, StaleOnError: true})
p := prov(t, models.PackageNPM)
res, err := testEngine.Fetch(ctx, r, "pkg", p)
if err != nil {
t.Fatalf("initial fetch: %v", err)
}
res.Reader.Close()
// Expire freshness but keep the ETag, then break the upstream: the
// conditional HEAD (checkUpstream) errors, and stale-on-error serves the
// stored index.
testCache.SetTTL(ctx, "eng-reval-err", "pkg", time.Millisecond)
time.Sleep(10 * time.Millisecond)
r.BaseURL = "http://127.0.0.1:1"
res, err = testEngine.Fetch(ctx, r, "pkg", p)
if err != nil {
t.Fatalf("expected stale serve on revalidation error, got %v", err)
}
if res.Source != "cache" {
t.Errorf("expected stale cache source, got %s", res.Source)
}
res.Reader.Close()
}
func TestTTLFor(t *testing.T) {
e := &Engine{}
if got := e.ttlFor(models.Remote{ImmutableTTL: 100}, ClassImmutable); got != 100*time.Second {
t.Errorf("immutable ttl = %v", got)
}
if got := e.ttlFor(models.Remote{ImmutableTTL: 0}, ClassImmutable); got != 0 {
t.Errorf("immutable ttl=0 (forever) = %v", got)
}
if got := e.ttlFor(models.Remote{MutableTTL: 50}, ClassMutable); got != 50*time.Second {
t.Errorf("mutable ttl = %v", got)
}
}
func TestHeadUpstreamStatusError(t *testing.T) {
requireStack(t)
r := seed(t, genericRemote("eng-head500"))
if _, err := testEngine.Head(context.Background(), r, "err500", prov(t, models.PackageGeneric)); err == nil {
t.Error("expected error for HEAD of 500 upstream")
}
}
func TestHeadCachedIndex(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := seed(t, models.Remote{Name: "eng-headidx", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600})
p := prov(t, models.PackageNPM)
// Cache the mutable index, then HEAD is answered from the stored index.
res, err := testEngine.Fetch(ctx, r, "pkg", p)
if err != nil {
t.Fatal(err)
}
res.Reader.Close()
h, err := testEngine.Head(ctx, r, "pkg", p)
if err != nil || h.Source != "cache" {
t.Errorf("head of cached index: %+v %v", h, err)
}
}
func TestFetchBearerTokenVariants(t *testing.T) {
requireStack(t)
ctx := context.Background()
// access_token field + service/scope params + basic auth on the token req.
tok, _, err := fetchBearerToken(ctx, `Bearer realm="`+upstream.URL+`/token-at",service="reg",scope="repo:pull"`, models.Remote{Username: "u", Password: "p"})
if err != nil || tok != "at-token" {
t.Errorf("access_token variant: tok=%q err=%v", tok, err)
}
// Token endpoint error status.
if _, _, err := fetchBearerToken(ctx, `Bearer realm="`+upstream.URL+`/token-500"`, models.Remote{}); err == nil {
t.Error("expected error for 500 token endpoint")
}
}
func TestCheckUpstreamChanged(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := genericRemote("eng-check")
// A non-matching ETag yields a normal 200 (not 304): not modified is false.
notModified, err := testEngine.checkUpstream(ctx, r, "pkg", `"stale-etag"`, prov(t, models.PackageNPM))
if err != nil {
t.Fatalf("checkUpstream: %v", err)
}
if notModified {
t.Error("mismatched etag should report modified (notModified=false)")
}
}
func TestUpstreamErrorUnwrap(t *testing.T) {
base := context.DeadlineExceeded
ue := &UpstreamError{Err: base}
if ue.Unwrap() != base {
t.Error("Unwrap should return the wrapped error")
}
if !isNetworkError(ue) {
t.Error("UpstreamError should be a network error")
}
if isNetworkError(context.Canceled) {
t.Error("plain error should not be a network error")
}
}
func TestImmutableBlobDedup(t *testing.T) {
requireStack(t)
ctx := context.Background()
p := prov(t, models.PackageGeneric)
// Two remotes serving identical content: the second store hits the
// already-exists branch (blob content is deduplicated).
for _, name := range []string{"eng-dedup-a", "eng-dedup-b"} {
r := seed(t, genericRemote(name))
res, err := testEngine.Fetch(ctx, r, "blob.bin", p)
if err != nil {
t.Fatalf("%s fetch: %v", name, err)
}
if readAll(t, res) != "immutable blob" {
t.Errorf("%s content mismatch", name)
}
}
}
func TestCircuitBreakerStates(t *testing.T) {
requireStack(t)
ctx := context.Background()
cb := NewCircuitBreaker(testCache)
const key = "cb-states"
testCache.ResetCircuit(ctx, key)
if cb.IsOpen(ctx, key) {
t.Error("fresh breaker should be closed")
}
if cb.Health(ctx, key).Status != "healthy" {
t.Error("fresh breaker should be healthy")
}
cb.RecordFailure(ctx, key)
if s := cb.Health(ctx, key).Status; s != "degraded" {
t.Errorf("one failure should be degraded, got %q", s)
}
for i := 0; i < 6; i++ {
cb.RecordFailure(ctx, key)
}
if !cb.IsOpen(ctx, key) {
t.Error("breaker should be open after threshold failures")
}
if s := cb.Health(ctx, key).Status; s != "down" {
t.Errorf("open breaker should be down, got %q", s)
}
cb.RecordSuccess(ctx, key)
if cb.IsOpen(ctx, key) {
t.Error("breaker should close after success")
}
}
func asProxyError(err error, target **ProxyError) bool {
pe, ok := err.(*ProxyError)
if ok {
*target = pe
}
return ok
}
-83
View File
@@ -1,83 +0,0 @@
package proxy
import (
"net"
"net/http"
"sync"
"time"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// Default upstream timeouts. A remote may override any of these; a zero
// override falls back to the default here. There is deliberately no overall
// Client.Timeout: the proxy streams arbitrarily large artifacts and total time
// is bounded by the request context instead. We only constrain the phases that
// must never hang — connect, TLS handshake, and time-to-first-response-header —
// so a slow or wedged upstream cannot pin a goroutine and connection.
const (
defaultDialTimeout = 10 * time.Second
defaultTLSTimeout = 10 * time.Second
defaultResponseHeaderTimeout = 30 * time.Second
)
type clientKey struct {
dial time.Duration
tls time.Duration
respHeader time.Duration
}
var (
clientCacheMu sync.Mutex
clientCache = map[clientKey]*http.Client{}
)
// upstreamClientFor returns an HTTP client configured with the given timeouts,
// reusing a cached client (and its connection pool) for identical timeout sets.
// Zero values fall back to the defaults.
func upstreamClientFor(dial, tls, respHeader time.Duration) *http.Client {
if dial <= 0 {
dial = defaultDialTimeout
}
if tls <= 0 {
tls = defaultTLSTimeout
}
if respHeader <= 0 {
respHeader = defaultResponseHeaderTimeout
}
key := clientKey{dial: dial, tls: tls, respHeader: respHeader}
clientCacheMu.Lock()
defer clientCacheMu.Unlock()
if c, ok := clientCache[key]; ok {
return c
}
c := &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: dial,
KeepAlive: 30 * time.Second,
}).DialContext,
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: tls,
ExpectContinueTimeout: 1 * time.Second,
ResponseHeaderTimeout: respHeader,
},
}
clientCache[key] = c
return c
}
// clientForRemote returns the upstream client for a remote, applying its
// per-remote timeout overrides (in seconds) on top of the defaults.
func clientForRemote(remote models.Remote) *http.Client {
return upstreamClientFor(
time.Duration(remote.UpstreamDialTimeout)*time.Second,
time.Duration(remote.UpstreamTLSTimeout)*time.Second,
time.Duration(remote.UpstreamResponseHeaderTimeout)*time.Second,
)
}
+22 -67
View File
@@ -12,7 +12,6 @@ import (
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
tfregistry "git.unkin.net/unkin/artifactapi/internal/api/terraform"
v1 "git.unkin.net/unkin/artifactapi/internal/api/v1" v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2" v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
"git.unkin.net/unkin/artifactapi/internal/cache" "git.unkin.net/unkin/artifactapi/internal/cache"
@@ -31,25 +30,21 @@ import (
_ "git.unkin.net/unkin/artifactapi/internal/provider/terraform" _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
"git.unkin.net/unkin/artifactapi/internal/proxy" "git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/tfsign"
"git.unkin.net/unkin/artifactapi/internal/virtual" "git.unkin.net/unkin/artifactapi/internal/virtual"
) )
type Server struct { type Server struct {
cfg *config.Config cfg *config.Config
version string router chi.Router
router chi.Router db *database.DB
db *database.DB cache *cache.Redis
cache *cache.Redis store *storage.S3
store *storage.S3 engine *proxy.Engine
engine *proxy.Engine virtEngine *virtual.Engine
virtEngine *virtual.Engine gc *gc.Collector
localHandler *v2.LocalHandler
tfRegistry *tfregistry.Handler
gc *gc.Collector
} }
func New(cfg *config.Config, version string) (*Server, error) { func New(cfg *config.Config) (*Server, error) {
db, err := database.New(cfg.DatabaseDSN()) db, err := database.New(cfg.DatabaseDSN())
if err != nil { if err != nil {
return nil, fmt.Errorf("database: %w", err) return nil, fmt.Errorf("database: %w", err)
@@ -66,40 +61,17 @@ func New(cfg *config.Config, version string) (*Server, error) {
} }
engine := proxy.NewEngine(db, redis, s3) engine := proxy.NewEngine(db, redis, s3)
localHandler := v2.NewLocalHandler(db, s3)
virtEngine := virtual.NewEngine(db, engine) virtEngine := virtual.NewEngine(db, engine)
collector := gc.New(db, s3, 1*time.Hour) collector := gc.New(db, s3, 1*time.Hour)
// The terraform registry signs with a GPG key. A configured file wins (BYO
// key); otherwise artifactapi generates one on first start and persists it in
// the database so every replica shares it. A failure here must not take the
// server down — the registry just stays disabled.
var signer *tfsign.Signer
if cfg.TFSigningKeyPath != "" {
signer, err = tfsign.Load(cfg.TFSigningKeyPath, cfg.TFSigningKeyPassphrase)
} else {
signer, err = tfsign.LoadOrCreate(context.Background(), db, "terraform-provider")
}
if err != nil {
slog.Warn("terraform provider registry disabled", "error", err)
signer = nil
}
tfRegistry := tfregistry.NewHandler(db, signer, cfg.TFProviderProtocols)
if tfRegistry.Enabled() {
slog.Info("terraform provider registry enabled", "key_id", signer.KeyID())
}
s := &Server{ s := &Server{
cfg: cfg, cfg: cfg,
version: version, db: db,
db: db, cache: redis,
cache: redis, store: s3,
store: s3, engine: engine,
engine: engine, virtEngine: virtEngine,
virtEngine: virtEngine, gc: collector,
localHandler: localHandler,
tfRegistry: tfRegistry,
gc: collector,
} }
s.router = s.routes() s.router = s.routes()
@@ -118,16 +90,11 @@ func (s *Server) routes() chi.Router {
r.Get("/health", s.handleHealth) r.Get("/health", s.handleHealth)
r.Get("/", s.handleRoot) r.Get("/", s.handleRoot)
r.Get("/version", s.handleVersion)
// Terraform provider registry: service discovery at the well-known path, localHandler := v2.NewLocalHandler(s.db, s.store)
// providers.v1 protocol under /terraform/v1/providers.
r.Get("/.well-known/terraform.json", s.tfRegistry.ServiceDiscovery)
r.Mount(tfregistry.MountPath, s.tfRegistry.Routes())
proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler) proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, localHandler)
r.Mount("/api/v1", proxyHandler.Routes()) r.Mount("/api/v1", proxyHandler.Routes())
r.Mount("/v2", proxyHandler.DockerV2Routes())
remotesHandler := v2.NewRemotesHandler(s.db) remotesHandler := v2.NewRemotesHandler(s.db)
virtualsHandler := v2.NewVirtualsHandler(s.db) virtualsHandler := v2.NewVirtualsHandler(s.db)
@@ -150,16 +117,10 @@ func (s *Server) routes() chi.Router {
r.Delete("/*", objHandler.Routes().ServeHTTP) r.Delete("/*", objHandler.Routes().ServeHTTP)
}) })
r.Route("/locals/{name}/objects", func(r chi.Router) {
objHandler := v2.NewObjectsHandler(s.db)
r.Get("/", objHandler.LocalRoutes().ServeHTTP)
r.Delete("/*", objHandler.LocalRoutes().ServeHTTP)
})
r.Route("/remotes/{name}/files", func(r chi.Router) { r.Route("/remotes/{name}/files", func(r chi.Router) {
r.Put("/*", s.localHandler.Routes().ServeHTTP) r.Put("/*", localHandler.Routes().ServeHTTP)
r.Get("/*", s.localHandler.Routes().ServeHTTP) r.Get("/*", localHandler.Routes().ServeHTTP)
r.Delete("/*", s.localHandler.Routes().ServeHTTP) r.Delete("/*", localHandler.Routes().ServeHTTP)
}) })
}) })
@@ -172,16 +133,10 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, `{"status":"ok"}`) fmt.Fprint(w, `{"status":"ok"}`)
} }
// handleRoot sends browsers landing on the bare domain to the web UI, which is
// served under /ui. The service identity that used to live here is at /version.
func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/ui/", http.StatusFound)
}
func (s *Server) handleVersion(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
fmt.Fprintf(w, `{"name":"artifactapi","version":"%s"}`, s.version) fmt.Fprint(w, `{"name":"artifactapi","version":"3.0.0-dev"}`)
} }
func (s *Server) newHTTPServer() *http.Server { func (s *Server) newHTTPServer() *http.Server {
-639
View File
@@ -1,639 +0,0 @@
package server
import (
"bytes"
"context"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strconv"
"strings"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/config"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
var (
testTS *httptest.Server // the artifactapi router
upstream *httptest.Server // mock upstream the proxy fetches from
testSrv *Server
)
func TestMain(m *testing.M) {
ctx := context.Background()
dsn, termPG, err := testsupport.StartPostgres(ctx)
if err != nil {
os.Exit(m.Run())
}
defer termPG()
redisURL, termRedis, err := testsupport.StartRedis(ctx)
if err != nil {
termPG()
os.Exit(m.Run())
}
defer termRedis()
minio, termMinio, err := testsupport.StartMinio(ctx)
if err != nil {
termPG()
termRedis()
os.Exit(m.Run())
}
defer termMinio()
u, _ := url.Parse(dsn)
port, _ := strconv.Atoi(u.Port())
cfg := &config.Config{
ListenAddr: ":0",
DBHost: u.Hostname(),
DBPort: port,
DBUser: "artifacts",
DBPass: "artifacts123",
DBName: "artifacts",
DBSSL: "disable",
RedisURL: redisURL,
S3Endpoint: minio.Endpoint,
S3AccessKey: minio.AccessKey,
S3SecretKey: minio.SecretKey,
S3Bucket: "server-test",
}
var srv *Server
for i := 0; i < 20; i++ { // tolerate MinIO reporting ready before bucket ops succeed
if srv, err = New(cfg, "test-version"); err == nil {
break
}
time.Sleep(500 * time.Millisecond)
}
if err != nil {
panic(err)
}
testSrv = srv
testTS = httptest.NewServer(srv.router)
upstream = httptest.NewServer(http.HandlerFunc(mockUpstream))
code := m.Run()
testTS.Close()
upstream.Close()
termMinio()
termRedis()
termPG()
if code != 0 {
os.Exit(code)
}
}
func mockUpstream(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/data/file.bin":
w.Write([]byte("upstream blob payload"))
case "/helm-a/index.yaml":
w.Write([]byte("apiVersion: v1\nentries:\n alpha:\n - name: alpha\n version: 1.0.0\n urls: [charts/alpha-1.0.0.tgz]\n"))
case "/helm-b/index.yaml":
w.Write([]byte("apiVersion: v1\nentries:\n beta:\n - name: beta\n version: 2.0.0\n urls: [charts/beta-2.0.0.tgz]\n"))
default:
http.NotFound(w, r)
}
}
func requireStack(t *testing.T) {
t.Helper()
if testTS == nil {
t.Skip("Docker unavailable; skipping server integration test")
}
}
func req(t *testing.T, method, path string, body string) (*http.Response, []byte) {
t.Helper()
var r io.Reader
if body != "" {
r = strings.NewReader(body)
}
rq, _ := http.NewRequest(method, testTS.URL+path, r)
if body != "" {
rq.Header.Set("Content-Type", "application/json")
}
resp, err := http.DefaultClient.Do(rq)
if err != nil {
t.Fatalf("%s %s: %v", method, path, err)
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
return resp, b
}
// reqNoRedirect issues a request without following redirects so the response's
// status and Location header can be asserted directly.
func reqNoRedirect(t *testing.T, method, path string) *http.Response {
t.Helper()
rq, _ := http.NewRequest(method, testTS.URL+path, nil)
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
}}
resp, err := client.Do(rq)
if err != nil {
t.Fatalf("%s %s: %v", method, path, err)
}
resp.Body.Close()
return resp
}
func TestServerHealthAndRoot(t *testing.T) {
requireStack(t)
if resp, _ := req(t, "GET", "/health", ""); resp.StatusCode != 200 {
t.Errorf("health: %d", resp.StatusCode)
}
if resp := reqNoRedirect(t, "GET", "/"); resp.StatusCode != http.StatusFound || resp.Header.Get("Location") != "/ui/" {
t.Errorf("root redirect: %d %q", resp.StatusCode, resp.Header.Get("Location"))
}
if resp, b := req(t, "GET", "/version", ""); resp.StatusCode != 200 || !strings.Contains(string(b), "test-version") {
t.Errorf("version: %d %s", resp.StatusCode, b)
}
if resp, _ := req(t, "GET", "/api/v2/health", ""); resp.StatusCode != 200 {
t.Errorf("health v2: %d", resp.StatusCode)
}
}
func TestServerRemoteAndProxy(t *testing.T) {
requireStack(t)
create := fmt.Sprintf(`{"name":"srv-remote","package_type":"generic","repo_type":"remote","base_url":%q,"stale_on_error":true}`, upstream.URL)
if resp, b := req(t, "POST", "/api/v2/remotes", create); resp.StatusCode != 201 {
t.Fatalf("create remote: %d %s", resp.StatusCode, b)
}
defer req(t, "DELETE", "/api/v2/remotes/srv-remote", "")
if resp, _ := req(t, "GET", "/api/v2/remotes/srv-remote", ""); resp.StatusCode != 200 {
t.Errorf("get remote: %d", resp.StatusCode)
}
if resp, _ := req(t, "GET", "/api/v2/remotes", ""); resp.StatusCode != 200 {
t.Errorf("list remotes: %d", resp.StatusCode)
}
// Proxy fetch: miss then hit.
resp, b := req(t, "GET", "/api/v1/remote/srv-remote/data/file.bin", "")
if resp.StatusCode != 200 || string(b) != "upstream blob payload" {
t.Fatalf("proxy miss: %d %s", resp.StatusCode, b)
}
if src := resp.Header.Get("X-Artifact-Source"); src != "remote" {
t.Errorf("expected remote source, got %q", src)
}
resp, _ = req(t, "GET", "/api/v1/remote/srv-remote/data/file.bin", "")
if resp.Header.Get("X-Artifact-Source") != "cache" {
t.Errorf("second fetch should be cache: %q", resp.Header.Get("X-Artifact-Source"))
}
// Objects listing + stats now that we have an artifact.
if resp, _ := req(t, "GET", "/api/v2/remotes/srv-remote/objects", ""); resp.StatusCode != 200 {
t.Errorf("objects: %d", resp.StatusCode)
}
if resp, _ := req(t, "GET", "/api/v2/stats", ""); resp.StatusCode != 200 {
t.Errorf("stats: %d", resp.StatusCode)
}
for _, p := range []string{"/api/v2/stats/top-remotes", "/api/v2/stats/top-files-by-hits", "/api/v2/stats/top-files-by-bandwidth"} {
if resp, _ := req(t, "GET", p, ""); resp.StatusCode != 200 {
t.Errorf("%s: %d", p, resp.StatusCode)
}
}
}
func TestServerLocalUpload(t *testing.T) {
requireStack(t)
if resp, b := req(t, "POST", "/api/v2/remotes", `{"name":"srv-local","package_type":"generic","repo_type":"local"}`); resp.StatusCode != 201 {
t.Fatalf("create local: %d %s", resp.StatusCode, b)
}
defer req(t, "DELETE", "/api/v2/remotes/srv-local", "")
rq, _ := http.NewRequest("PUT", testTS.URL+"/api/v2/remotes/srv-local/files/dir/hello.bin", strings.NewReader("local payload"))
rq.Header.Set("Content-Type", "text/plain") // exercise the content-type branch
resp, err := http.DefaultClient.Do(rq)
if err != nil || resp.StatusCode != 201 {
t.Fatalf("upload: %v %d", err, resp.StatusCode)
}
resp.Body.Close()
resp, b := req(t, "GET", "/api/v1/local/srv-local/dir/hello.bin", "")
if resp.StatusCode != 200 || string(b) != "local payload" {
t.Errorf("download local: %d %s", resp.StatusCode, b)
}
// Also download via the v2 files endpoint.
if resp, b := req(t, "GET", "/api/v2/remotes/srv-local/files/dir/hello.bin", ""); resp.StatusCode != 200 || string(b) != "local payload" {
t.Errorf("v2 download: %d %s", resp.StatusCode, b)
}
}
func TestServerVirtualMerge(t *testing.T) {
requireStack(t)
for _, m := range []string{"a", "b"} {
body := fmt.Sprintf(`{"name":"srv-helm-%s","package_type":"helm","repo_type":"remote","base_url":"%s/helm-%s","stale_on_error":true}`, m, upstream.URL, m)
if resp, b := req(t, "POST", "/api/v2/remotes", body); resp.StatusCode != 201 {
t.Fatalf("create helm-%s: %d %s", m, resp.StatusCode, b)
}
defer req(t, "DELETE", "/api/v2/remotes/srv-helm-"+m, "")
}
if resp, b := req(t, "POST", "/api/v2/virtuals", `{"name":"srv-vh","package_type":"helm","members":["srv-helm-a","srv-helm-b"]}`); resp.StatusCode != 201 {
t.Fatalf("create virtual: %d %s", resp.StatusCode, b)
}
defer req(t, "DELETE", "/api/v2/virtuals/srv-vh", "")
resp, b := req(t, "GET", "/api/v1/virtual/srv-vh/index.yaml", "")
if resp.StatusCode != 200 {
t.Fatalf("virtual fetch: %d %s", resp.StatusCode, b)
}
s := string(b)
if !strings.Contains(s, "alpha") || !strings.Contains(s, "beta") {
t.Errorf("merged index missing charts: %s", s)
}
}
func TestServerProbe(t *testing.T) {
requireStack(t)
create := fmt.Sprintf(`{"name":"srv-probe","package_type":"generic","repo_type":"remote","base_url":%q,"stale_on_error":true}`, upstream.URL)
req(t, "POST", "/api/v2/remotes", create)
defer req(t, "DELETE", "/api/v2/remotes/srv-probe", "")
// Reachable path -> status 200 in the probe body.
if resp, b := req(t, "POST", "/api/v2/probe", `{"remote":"srv-probe","path":"data/file.bin"}`); resp.StatusCode != 200 || !strings.Contains(string(b), `"status":200`) {
t.Errorf("probe reachable: %d %s", resp.StatusCode, b)
}
// Missing upstream path -> upstream error reported (502) in the body.
if resp, b := req(t, "POST", "/api/v2/probe", `{"remote":"srv-probe","path":"missing"}`); resp.StatusCode != 200 || !strings.Contains(string(b), `"status":502`) {
t.Errorf("probe missing: %d %s", resp.StatusCode, b)
}
// Unknown remote -> 404 in the body.
if resp, b := req(t, "POST", "/api/v2/probe", `{"remote":"nope","path":"x"}`); resp.StatusCode != 200 || !strings.Contains(string(b), `"status":404`) {
t.Errorf("probe unknown: %d %s", resp.StatusCode, b)
}
// Bad requests.
if resp, _ := req(t, "POST", "/api/v2/probe", `{}`); resp.StatusCode != 400 {
t.Errorf("probe missing fields: %d", resp.StatusCode)
}
if resp, _ := req(t, "POST", "/api/v2/probe", `not json`); resp.StatusCode != 400 {
t.Errorf("probe invalid json: %d", resp.StatusCode)
}
}
func put(t *testing.T, path string, body []byte) (*http.Response, []byte) {
t.Helper()
rq, _ := http.NewRequest("PUT", testTS.URL+path, bytes.NewReader(body))
resp, err := http.DefaultClient.Do(rq)
if err != nil {
t.Fatalf("PUT %s: %v", path, err)
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
return resp, b
}
func TestServerLocalPyPI(t *testing.T) {
requireStack(t)
if resp, b := req(t, "POST", "/api/v2/remotes", `{"name":"srv-pypi","package_type":"pypi","repo_type":"local"}`); resp.StatusCode != 201 {
t.Fatalf("create pypi local: %d %s", resp.StatusCode, b)
}
defer req(t, "DELETE", "/api/v2/remotes/srv-pypi", "")
if resp, b := put(t, "/api/v2/remotes/srv-pypi/files/foo-1.0-py3-none-any.whl", []byte("wheel bytes")); resp.StatusCode != 201 {
t.Fatalf("upload wheel: %d %s", resp.StatusCode, b)
}
// Re-uploading the same file is rejected.
if resp, _ := put(t, "/api/v2/remotes/srv-pypi/files/foo-1.0-py3-none-any.whl", []byte("again")); resp.StatusCode != 409 {
t.Errorf("expected 409 on overwrite, got %d", resp.StatusCode)
}
// Invalid pypi filename rejected.
if resp, _ := put(t, "/api/v2/remotes/srv-pypi/files/not-a-package.txt", []byte("x")); resp.StatusCode != 400 {
t.Errorf("expected 400 for bad filename, got %d", resp.StatusCode)
}
if resp, b := req(t, "GET", "/api/v1/local/srv-pypi/simple/", ""); resp.StatusCode != 200 || !strings.Contains(string(b), "foo") {
t.Errorf("simple index: %d %s", resp.StatusCode, b)
}
if resp, b := req(t, "GET", "/api/v1/local/srv-pypi/simple/foo/", ""); resp.StatusCode != 200 || !strings.Contains(string(b), "foo-1.0-py3-none-any.whl") {
t.Errorf("package index: %d %s", resp.StatusCode, b)
}
}
func TestServerLocalRPMRepodata(t *testing.T) {
requireStack(t)
rpm := testsupport.MinimalRPM("e2e-testpkg", "1.0", "1", "noarch")
if resp, b := req(t, "POST", "/api/v2/remotes", `{"name":"srv-rpm","package_type":"rpm","repo_type":"local"}`); resp.StatusCode != 201 {
t.Fatalf("create rpm local: %d %s", resp.StatusCode, b)
}
defer req(t, "DELETE", "/api/v2/remotes/srv-rpm", "")
if resp, b := put(t, "/api/v2/remotes/srv-rpm/files/e2e-testpkg-1.0-1.noarch.rpm", rpm); resp.StatusCode != 201 {
t.Fatalf("upload rpm: %d %s", resp.StatusCode, b)
}
// repodata is generated asynchronously; poll for it.
var body []byte
for i := 0; i < 40; i++ {
var resp *http.Response
resp, body = req(t, "GET", "/api/v1/local/srv-rpm/repodata/repomd.xml", "")
if resp.StatusCode == 200 && strings.Contains(string(body), "<repomd") {
return
}
time.Sleep(250 * time.Millisecond)
}
t.Errorf("repomd.xml not generated: %s", body)
}
func TestServerObjectEviction(t *testing.T) {
requireStack(t)
create := fmt.Sprintf(`{"name":"srv-evict","package_type":"generic","repo_type":"remote","base_url":%q,"stale_on_error":true}`, upstream.URL)
req(t, "POST", "/api/v2/remotes", create)
defer req(t, "DELETE", "/api/v2/remotes/srv-evict", "")
resp, _ := req(t, "GET", "/api/v1/remote/srv-evict/data/file.bin", "")
resp.Body.Close()
if resp, _ := req(t, "DELETE", "/api/v2/remotes/srv-evict/objects/data/file.bin", ""); resp.StatusCode >= 400 {
t.Errorf("evict object: %d", resp.StatusCode)
}
}
func TestServerValidationErrors(t *testing.T) {
requireStack(t)
if resp, _ := req(t, "POST", "/api/v2/remotes", `{"name":"bad","package_type":"bogus","base_url":"https://x"}`); resp.StatusCode != 400 {
t.Errorf("invalid package type: %d", resp.StatusCode)
}
if resp, _ := req(t, "POST", "/api/v2/remotes", `{"name":"bad","package_type":"generic","repo_type":"remote"}`); resp.StatusCode != 400 {
t.Errorf("missing base_url: %d", resp.StatusCode)
}
if resp, _ := req(t, "POST", "/api/v2/remotes", `not json`); resp.StatusCode != 400 {
t.Errorf("invalid json: %d", resp.StatusCode)
}
// Invalid regex pattern -> 400 from ValidatePatterns.
if resp, _ := req(t, "POST", "/api/v2/remotes", `{"name":"badre","package_type":"generic","repo_type":"remote","base_url":"https://x","blocklist":["[unterminated"]}`); resp.StatusCode != 400 {
t.Errorf("invalid regex: %d", resp.StatusCode)
}
}
func TestServerDockerAndHead(t *testing.T) {
requireStack(t)
create := fmt.Sprintf(`{"name":"srv-docker","package_type":"generic","repo_type":"remote","base_url":%q,"stale_on_error":true}`, upstream.URL)
req(t, "POST", "/api/v2/remotes", create)
defer req(t, "DELETE", "/api/v2/remotes/srv-docker", "")
// Docker registry ping.
if resp, _ := req(t, "GET", "/v2/", ""); resp.StatusCode != 200 {
t.Errorf("docker ping: %d", resp.StatusCode)
}
// HEAD through the docker route resolves metadata (uncached -> upstream).
rq, _ := http.NewRequest("HEAD", testTS.URL+"/v2/srv-docker/data/file.bin", nil)
resp, err := http.DefaultClient.Do(rq)
if err != nil {
t.Fatalf("head: %v", err)
}
resp.Body.Close()
if resp.StatusCode != 200 {
t.Errorf("head status: %d", resp.StatusCode)
}
}
func TestServerRemoteUpdateAndVirtualCRUD(t *testing.T) {
requireStack(t)
req(t, "POST", "/api/v2/remotes", `{"name":"srv-upd","package_type":"helm","repo_type":"remote","base_url":"https://a.example.com","stale_on_error":true}`)
defer req(t, "DELETE", "/api/v2/remotes/srv-upd", "")
if resp, b := req(t, "PUT", "/api/v2/remotes/srv-upd", `{"package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`); resp.StatusCode != 200 {
t.Errorf("update remote: %d %s", resp.StatusCode, b)
}
req(t, "POST", "/api/v2/virtuals", `{"name":"srv-v2","package_type":"helm","members":["srv-upd"]}`)
defer req(t, "DELETE", "/api/v2/virtuals/srv-v2", "")
if resp, _ := req(t, "GET", "/api/v2/virtuals/srv-v2", ""); resp.StatusCode != 200 {
t.Errorf("get virtual: %d", resp.StatusCode)
}
if resp, _ := req(t, "GET", "/api/v2/virtuals", ""); resp.StatusCode != 200 {
t.Errorf("list virtuals: %d", resp.StatusCode)
}
if resp, b := req(t, "PUT", "/api/v2/virtuals/srv-v2", `{"package_type":"helm","members":["srv-upd"]}`); resp.StatusCode != 200 {
t.Errorf("update virtual: %d %s", resp.StatusCode, b)
}
}
func TestServerLocalRemoveAndMissing(t *testing.T) {
requireStack(t)
req(t, "POST", "/api/v2/remotes", `{"name":"srv-rm","package_type":"generic","repo_type":"local"}`)
defer req(t, "DELETE", "/api/v2/remotes/srv-rm", "")
put(t, "/api/v2/remotes/srv-rm/files/a/b.bin", []byte("payload"))
if resp, _ := req(t, "DELETE", "/api/v2/remotes/srv-rm/files/a/b.bin", ""); resp.StatusCode >= 400 {
t.Errorf("delete local file: %d", resp.StatusCode)
}
if resp, _ := req(t, "GET", "/api/v1/local/srv-rm/a/b.bin", ""); resp.StatusCode != 404 {
t.Errorf("expected 404 for removed file, got %d", resp.StatusCode)
}
}
func TestServerLocalUploadErrors(t *testing.T) {
requireStack(t)
// Uploading to a remote-type repo is rejected.
create := fmt.Sprintf(`{"name":"srv-uerr","package_type":"generic","repo_type":"remote","base_url":%q,"stale_on_error":true}`, upstream.URL)
req(t, "POST", "/api/v2/remotes", create)
defer req(t, "DELETE", "/api/v2/remotes/srv-uerr", "")
if resp, _ := put(t, "/api/v2/remotes/srv-uerr/files/x.bin", []byte("x")); resp.StatusCode != 400 {
t.Errorf("upload to remote repo should be 400, got %d", resp.StatusCode)
}
// Duplicate generic upload is a conflict.
req(t, "POST", "/api/v2/remotes", `{"name":"srv-dup","package_type":"generic","repo_type":"local"}`)
defer req(t, "DELETE", "/api/v2/remotes/srv-dup", "")
put(t, "/api/v2/remotes/srv-dup/files/dup.bin", []byte("one"))
if resp, _ := put(t, "/api/v2/remotes/srv-dup/files/dup.bin", []byte("two")); resp.StatusCode != 409 {
t.Errorf("duplicate upload should be 409, got %d", resp.StatusCode)
}
// Download of a missing local file is 404.
if resp, _ := req(t, "GET", "/api/v1/local/srv-dup/does/not/exist", ""); resp.StatusCode != 404 {
t.Errorf("missing local download should be 404, got %d", resp.StatusCode)
}
// Unknown virtual is 404.
if resp, _ := req(t, "GET", "/api/v1/virtual/nope/index.yaml", ""); resp.StatusCode != 404 {
t.Errorf("unknown virtual should be 404, got %d", resp.StatusCode)
}
}
func TestServerEvents(t *testing.T) {
requireStack(t)
client := &http.Client{Timeout: 800 * time.Millisecond}
resp, err := client.Get(testTS.URL + "/api/v2/events")
if err == nil {
resp.Body.Close()
if resp.StatusCode != 200 {
t.Errorf("events status: %d", resp.StatusCode)
}
}
// A timeout is expected for a streaming endpoint; the handler still ran.
}
func TestRunOnListener(t *testing.T) {
requireStack(t)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
errc := make(chan error, 1)
go func() { errc <- testSrv.RunOnListener(ctx, ln) }()
base := "http://" + ln.Addr().String()
ok := false
for i := 0; i < 50; i++ {
if resp, e := http.Get(base + "/health"); e == nil {
resp.Body.Close()
ok = resp.StatusCode == 200
break
}
time.Sleep(20 * time.Millisecond)
}
if !ok {
t.Error("server did not serve /health")
}
cancel()
select {
case err := <-errc:
if err != nil {
t.Errorf("RunOnListener returned error: %v", err)
}
case <-time.After(12 * time.Second):
t.Fatal("RunOnListener did not shut down")
}
}
func TestRun(t *testing.T) {
requireStack(t)
ctx, cancel := context.WithCancel(context.Background())
errc := make(chan error, 1)
go func() { errc <- testSrv.Run(ctx) }()
time.Sleep(300 * time.Millisecond) // let it bind and start serving
cancel()
select {
case err := <-errc:
if err != nil {
t.Errorf("Run returned error: %v", err)
}
case <-time.After(12 * time.Second):
t.Fatal("Run did not shut down")
}
}
func TestServerVirtualUnreachableMembers(t *testing.T) {
requireStack(t)
// A virtual whose only member does not exist -> no members reachable.
req(t, "POST", "/api/v2/virtuals", `{"name":"srv-vbad","package_type":"helm","members":["nonexistent-member"]}`)
defer req(t, "DELETE", "/api/v2/virtuals/srv-vbad", "")
if resp, _ := req(t, "GET", "/api/v1/virtual/srv-vbad/index.yaml", ""); resp.StatusCode != 502 {
t.Errorf("virtual with dead members = %d, want 502", resp.StatusCode)
}
}
func TestServerVirtualLocalPyPIMerge(t *testing.T) {
requireStack(t)
for _, n := range []string{"a", "b"} {
req(t, "POST", "/api/v2/remotes", `{"name":"srv-pm-`+n+`","package_type":"pypi","repo_type":"local"}`)
defer req(t, "DELETE", "/api/v2/remotes/srv-pm-"+n, "")
}
put(t, "/api/v2/remotes/srv-pm-a/files/foo-1.0-py3-none-any.whl", []byte("foo"))
put(t, "/api/v2/remotes/srv-pm-b/files/bar-2.0-py3-none-any.whl", []byte("bar"))
req(t, "POST", "/api/v2/virtuals", `{"name":"srv-pmv","package_type":"pypi","members":["srv-pm-a","srv-pm-b"]}`)
defer req(t, "DELETE", "/api/v2/virtuals/srv-pmv", "")
resp, b := req(t, "GET", "/api/v1/virtual/srv-pmv/simple/", "")
if resp.StatusCode != 200 {
t.Fatalf("virtual pypi index: %d %s", resp.StatusCode, b)
}
if s := string(b); !strings.Contains(s, "foo") || !strings.Contains(s, "bar") {
t.Errorf("merged local pypi index missing packages: %s", s)
}
}
func TestServerProxyErrors(t *testing.T) {
requireStack(t)
// Blocklisted path -> 403 propagated through handleProxy.
block := fmt.Sprintf(`{"name":"srv-block","package_type":"generic","repo_type":"remote","base_url":%q,"blocklist":["\\.secret$"],"stale_on_error":true}`, upstream.URL)
req(t, "POST", "/api/v2/remotes", block)
defer req(t, "DELETE", "/api/v2/remotes/srv-block", "")
if resp, _ := req(t, "GET", "/api/v1/remote/srv-block/x.secret", ""); resp.StatusCode != 403 {
t.Errorf("blocklisted GET = %d, want 403", resp.StatusCode)
}
rq, _ := http.NewRequest("HEAD", testTS.URL+"/v2/srv-block/x.secret", nil)
if resp, err := http.DefaultClient.Do(rq); err == nil {
resp.Body.Close()
if resp.StatusCode != 403 {
t.Errorf("blocklisted HEAD = %d, want 403", resp.StatusCode)
}
}
// Unreachable upstream, no stale copy -> 502 bad gateway.
dead := `{"name":"srv-dead","package_type":"generic","repo_type":"remote","base_url":"http://127.0.0.1:1","stale_on_error":false}`
req(t, "POST", "/api/v2/remotes", dead)
defer req(t, "DELETE", "/api/v2/remotes/srv-dead", "")
if resp, _ := req(t, "GET", "/api/v1/remote/srv-dead/x", ""); resp.StatusCode != 502 {
t.Errorf("dead upstream GET = %d, want 502", resp.StatusCode)
}
}
func TestServerLocalMissingBlob(t *testing.T) {
requireStack(t)
req(t, "POST", "/api/v2/remotes", `{"name":"srv-ghost","package_type":"generic","repo_type":"local"}`)
defer req(t, "DELETE", "/api/v2/remotes/srv-ghost", "")
ctx := context.Background()
// A local file whose blob object is absent from the store.
testSrv.db.UpsertBlob(ctx, "sha256:ghost", "blobs/sha256/ghost-missing", 5, "text/plain")
if err := testSrv.db.CreateLocalFile(ctx, "srv-ghost", "ghost.bin", "sha256:ghost"); err != nil {
t.Fatalf("create local file: %v", err)
}
if resp, _ := req(t, "GET", "/api/v1/local/srv-ghost/ghost.bin", ""); resp.StatusCode != 500 {
t.Errorf("v1 download missing blob = %d, want 500", resp.StatusCode)
}
if resp, _ := req(t, "GET", "/api/v2/remotes/srv-ghost/files/ghost.bin", ""); resp.StatusCode != 500 {
t.Errorf("v2 download missing blob = %d, want 500", resp.StatusCode)
}
}
func TestServerBogusProviderType(t *testing.T) {
requireStack(t)
// Insert a remote with an unregistered package type directly, bypassing
// validation, to exercise the provider-not-found branches.
_, err := testSrv.db.Pool.Exec(context.Background(),
`INSERT INTO remotes (name, package_type, repo_type, base_url) VALUES ($1,'bogus','remote','https://x')`, "srv-bogus")
if err != nil {
t.Fatalf("insert bogus remote: %v", err)
}
defer testSrv.db.Pool.Exec(context.Background(), `DELETE FROM remotes WHERE name='srv-bogus'`)
if resp, _ := req(t, "GET", "/api/v1/remote/srv-bogus/x", ""); resp.StatusCode != 500 {
t.Errorf("bogus provider GET = %d, want 500", resp.StatusCode)
}
rq, _ := http.NewRequest("HEAD", testTS.URL+"/v2/srv-bogus/x", nil)
if resp, err := http.DefaultClient.Do(rq); err == nil {
resp.Body.Close()
if resp.StatusCode != 500 {
t.Errorf("bogus provider HEAD = %d, want 500", resp.StatusCode)
}
}
if resp, b := req(t, "POST", "/api/v2/probe", `{"remote":"srv-bogus","path":"x"}`); resp.StatusCode != 200 || !strings.Contains(string(b), `"status":500`) {
t.Errorf("bogus provider probe: %d %s", resp.StatusCode, b)
}
}
func TestServerNotFound(t *testing.T) {
requireStack(t)
if resp, _ := req(t, "GET", "/api/v2/remotes/does-not-exist", ""); resp.StatusCode != 404 {
t.Errorf("expected 404, got %d", resp.StatusCode)
}
if resp, _ := req(t, "GET", "/api/v1/remote/nope/x", ""); resp.StatusCode != 404 {
t.Errorf("expected 404 for unknown remote, got %d", resp.StatusCode)
}
// Unknown local repo -> 404 in handleLocal.
if resp, _ := req(t, "GET", "/api/v1/local/nope/x", ""); resp.StatusCode != 404 {
t.Errorf("expected 404 for unknown local repo, got %d", resp.StatusCode)
}
}
-160
View File
@@ -1,160 +0,0 @@
package storage
import (
"bytes"
"context"
"io"
"os"
"strings"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
var (
testS3 *S3
testEndpoint string
)
func TestMain(m *testing.M) {
ctx := context.Background()
conn, terminate, err := testsupport.StartMinio(ctx)
if err != nil {
os.Exit(m.Run())
}
var s3 *S3
for i := 0; i < 20; i++ { // MinIO can report ready before bucket ops succeed
if s3, err = NewS3(conn.Endpoint, conn.AccessKey, conn.SecretKey, "test-bucket", false, ""); err == nil {
break
}
time.Sleep(500 * time.Millisecond)
}
if err != nil {
terminate()
panic(err)
}
testS3 = s3
testEndpoint = conn.Endpoint
code := m.Run()
terminate()
if code != 0 {
os.Exit(code)
}
}
func requireS3(t *testing.T) {
t.Helper()
if testS3 == nil {
t.Skip("Docker unavailable; skipping storage integration test")
}
}
func TestKeys(t *testing.T) {
if BlobKey("abc") != "blobs/sha256/abc" {
t.Error("BlobKey")
}
if IndexKey("remote", "path/to/x") != "indexes/remote/path/to/x" {
t.Error("IndexKey")
}
}
func TestS3RoundTrip(t *testing.T) {
requireS3(t)
ctx := context.Background()
key := "blobs/sha256/test1"
content := []byte("hello storage")
if err := testS3.Upload(ctx, key, bytes.NewReader(content), int64(len(content)), "text/plain"); err != nil {
t.Fatalf("upload: %v", err)
}
exists, err := testS3.Exists(ctx, key)
if err != nil || !exists {
t.Fatalf("exists after upload: %v %v", exists, err)
}
reader, info, err := testS3.Download(ctx, key)
if err != nil {
t.Fatalf("download: %v", err)
}
got, _ := io.ReadAll(reader)
reader.Close()
if !bytes.Equal(got, content) {
t.Errorf("content mismatch: %q", got)
}
if info.Size != int64(len(content)) || info.ContentType != "text/plain" {
t.Errorf("stat info wrong: size=%d ct=%s", info.Size, info.ContentType)
}
if _, err := testS3.Stat(ctx, key); err != nil {
t.Errorf("stat: %v", err)
}
if err := testS3.Delete(ctx, key); err != nil {
t.Fatalf("delete: %v", err)
}
if exists, _ := testS3.Exists(ctx, key); exists {
t.Error("expected object gone after delete")
}
}
func TestNewS3ExistingBucket(t *testing.T) {
requireS3(t)
// The bucket already exists from TestMain, so ensureBucket takes the
// "already present" path.
if _, err := NewS3(testEndpoint, "minioadmin", "minioadmin", "test-bucket", false, ""); err != nil {
t.Fatalf("second NewS3: %v", err)
}
}
func TestS3DownloadMissing(t *testing.T) {
requireS3(t)
if _, _, err := testS3.Download(context.Background(), "does/not/exist"); err == nil {
t.Error("expected error downloading missing key")
}
if _, err := testS3.Stat(context.Background(), "does/not/exist"); err == nil {
t.Error("expected error stat-ing missing key")
}
if exists, err := testS3.Exists(context.Background(), "does/not/exist"); err != nil || exists {
t.Errorf("Exists(missing) = %v, %v; want false, nil", exists, err)
}
}
func TestCASStore(t *testing.T) {
requireS3(t)
ctx := context.Background()
cas := NewCAS(testS3)
content := "content-addressed payload"
res, err := cas.Store(ctx, strings.NewReader(content), "text/plain")
if err != nil {
t.Fatalf("store: %v", err)
}
if res.AlreadyExists {
t.Error("first store should not report AlreadyExists")
}
if res.SizeBytes != int64(len(content)) || !strings.HasPrefix(res.ContentHash, "sha256:") {
t.Errorf("unexpected result: %+v", res)
}
// Storing identical content again is deduplicated.
res2, err := cas.Store(ctx, strings.NewReader(content), "text/plain")
if err != nil {
t.Fatalf("store again: %v", err)
}
if !res2.AlreadyExists || res2.ContentHash != res.ContentHash {
t.Errorf("second store should dedup: %+v", res2)
}
// The stored blob is retrievable.
reader, _, err := testS3.Download(ctx, res.S3Key)
if err != nil {
t.Fatalf("download stored blob: %v", err)
}
got, _ := io.ReadAll(reader)
reader.Close()
if string(got) != content {
t.Errorf("stored content mismatch: %q", got)
}
}
-101
View File
@@ -1,101 +0,0 @@
// Package testsupport starts throwaway backing containers (Postgres, Redis,
// MinIO) for integration-style unit tests. It is only ever imported from
// *_test.go files, so it never reaches the production binary. Each Start*
// function returns a connection detail plus a terminate func; callers wire
// them up in a TestMain and skip the package's tests when Docker is absent.
package testsupport
import (
"context"
"fmt"
"os"
"time"
"github.com/testcontainers/testcontainers-go"
tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres"
tcredis "github.com/testcontainers/testcontainers-go/modules/redis"
"github.com/testcontainers/testcontainers-go/wait"
)
func init() {
// The Ryuk reaper container cannot start in this environment; each Start*
// returns an explicit terminate func for cleanup instead.
if _, ok := os.LookupEnv("TESTCONTAINERS_RYUK_DISABLED"); !ok {
os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true")
}
}
// StartPostgres launches postgres:17-alpine and returns its DSN.
func StartPostgres(ctx context.Context) (dsn string, terminate func(), err error) {
c, err := tcpostgres.Run(ctx,
"postgres:17-alpine",
tcpostgres.WithDatabase("artifacts"),
tcpostgres.WithUsername("artifacts"),
tcpostgres.WithPassword("artifacts123"),
testcontainers.WithWaitStrategy(
// Postgres opens the port, runs init scripts, then restarts, so wait
// for the readiness log to appear twice to avoid connection resets.
wait.ForLog("database system is ready to accept connections").
WithOccurrence(2).
WithStartupTimeout(60*time.Second),
),
)
if err != nil {
return "", nil, err
}
host, _ := c.Host(ctx)
port, _ := c.MappedPort(ctx, "5432/tcp")
dsn = fmt.Sprintf("postgres://artifacts:artifacts123@%s:%s/artifacts?sslmode=disable", host, port.Port())
return dsn, func() { _ = c.Terminate(ctx) }, nil
}
// StartRedis launches redis:7-alpine and returns its URL.
func StartRedis(ctx context.Context) (url string, terminate func(), err error) {
c, err := tcredis.Run(ctx,
"redis:7-alpine",
testcontainers.WithWaitStrategy(
wait.ForListeningPort("6379/tcp").WithStartupTimeout(60*time.Second),
),
)
if err != nil {
return "", nil, err
}
host, _ := c.Host(ctx)
port, _ := c.MappedPort(ctx, "6379/tcp")
url = fmt.Sprintf("redis://%s:%s", host, port.Port())
return url, func() { _ = c.Terminate(ctx) }, nil
}
// MinioConn holds MinIO connection details.
type MinioConn struct {
Endpoint string
AccessKey string
SecretKey string
}
// StartMinio launches minio and returns its connection details.
func StartMinio(ctx context.Context) (conn MinioConn, terminate func(), err error) {
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: testcontainers.ContainerRequest{
Image: "minio/minio:latest",
ExposedPorts: []string{"9000/tcp"},
Cmd: []string{"server", "/data"},
Env: map[string]string{
"MINIO_ROOT_USER": "minioadmin",
"MINIO_ROOT_PASSWORD": "minioadmin",
},
WaitingFor: wait.ForHTTP("/minio/health/ready").WithPort("9000/tcp").WithStartupTimeout(60 * time.Second),
},
Started: true,
})
if err != nil {
return MinioConn{}, nil, err
}
host, _ := c.Host(ctx)
port, _ := c.MappedPort(ctx, "9000/tcp")
return MinioConn{
Endpoint: fmt.Sprintf("%s:%s", host, port.Port()),
AccessKey: "minioadmin",
SecretKey: "minioadmin",
}, func() { _ = c.Terminate(ctx) }, nil
}
-59
View File
@@ -1,59 +0,0 @@
package testsupport
import (
"bytes"
"encoding/binary"
)
// MinimalRPM builds a valid-enough RPM package in pure Go (no committed binary
// fixture, no external rpmbuild). It carries just the header tags the provider
// reads: name/version/release/arch plus a single self Provides entry, which is
// enough for cavaliergopher/rpm to parse and for repodata generation.
func MinimalRPM(name, version, release, arch string) []byte {
type tag struct {
id, typ, count uint32
data []byte
}
cstr := func(s string) []byte { return append([]byte(s), 0) }
tags := []tag{
{1000, 6, 1, cstr(name)}, // RPMTAG_NAME (STRING)
{1001, 6, 1, cstr(version)}, // RPMTAG_VERSION
{1002, 6, 1, cstr(release)}, // RPMTAG_RELEASE
{1022, 6, 1, cstr(arch)}, // RPMTAG_ARCH
{1047, 8, 1, cstr(name)}, // RPMTAG_PROVIDENAME (STRING_ARRAY)
{1112, 4, 1, []byte{0, 0, 0, 0}}, // RPMTAG_PROVIDEFLAGS (INT32)
{1113, 8, 1, cstr(version)}, // RPMTAG_PROVIDEVERSION (STRING_ARRAY)
}
buildHeader := func(entries []tag) []byte {
var index, store bytes.Buffer
for _, e := range entries {
off := uint32(store.Len())
for _, v := range []uint32{e.id, e.typ, off, e.count} {
binary.Write(&index, binary.BigEndian, v)
}
store.Write(e.data)
}
var b bytes.Buffer
b.Write([]byte{0x8e, 0xad, 0xe8, 0x01, 0, 0, 0, 0}) // header magic + reserved
binary.Write(&b, binary.BigEndian, uint32(len(entries)))
binary.Write(&b, binary.BigEndian, uint32(store.Len()))
b.Write(index.Bytes())
b.Write(store.Bytes())
return b.Bytes()
}
lead := make([]byte, 96)
copy(lead[0:4], []byte{0xed, 0xab, 0xee, 0xdb}) // lead magic
lead[4] = 3 // major version
binary.BigEndian.PutUint16(lead[8:10], 1) // archnum
copy(lead[10:76], name) // name (66 bytes, null-padded)
binary.BigEndian.PutUint16(lead[76:78], 1) // osnum
binary.BigEndian.PutUint16(lead[78:80], 5) // signature type
var out bytes.Buffer
out.Write(lead)
out.Write(buildHeader(nil)) // empty signature header (16 bytes, 8-aligned)
out.Write(buildHeader(tags))
return out.Bytes()
}
-173
View File
@@ -1,173 +0,0 @@
// Package tfsign loads a GPG signing key and produces the detached signatures
// the Terraform provider registry protocol requires over SHA256SUMS files.
package tfsign
import (
"bytes"
"context"
"fmt"
"os"
"strings"
"golang.org/x/crypto/openpgp"
"golang.org/x/crypto/openpgp/armor"
)
// KeyStore persists a generated signing key. *database.DB satisfies it.
type KeyStore interface {
GetSigningKey(ctx context.Context, purpose string) (armor, keyID string, found bool, err error)
InsertSigningKeyIfAbsent(ctx context.Context, purpose, armor, keyID string) error
}
// LoadOrCreate returns a signer for purpose, generating and persisting a new key
// the first time it is needed. It is safe across replicas: a lost insert race
// just re-reads whichever key won.
func LoadOrCreate(ctx context.Context, store KeyStore, purpose string) (*Signer, error) {
armored, _, found, err := store.GetSigningKey(ctx, purpose)
if err != nil {
return nil, err
}
if !found {
newArmor, keyID, err := Generate()
if err != nil {
return nil, err
}
if err := store.InsertSigningKeyIfAbsent(ctx, purpose, newArmor, keyID); err != nil {
return nil, err
}
if armored, _, _, err = store.GetSigningKey(ctx, purpose); err != nil {
return nil, err
}
}
return LoadArmored(armored, "")
}
// Signer holds a decrypted GPG entity and exposes what the registry download
// response needs: a detached signature, the armored public key, and the key ID.
type Signer struct {
entity *openpgp.Entity
publicASCII string
keyID string
}
// Load reads an armored private key from path, decrypting it with passphrase if
// the key is protected. A blank path returns (nil, nil): a nil *Signer means the
// caller should fall back to another source (e.g. a DB-stored key).
func Load(path, passphrase string) (*Signer, error) {
if path == "" {
return nil, nil
}
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("open signing key: %w", err)
}
return fromArmor(string(data), passphrase, path)
}
// LoadArmored builds a signer from an in-memory armored private key, e.g. one
// read from the database. A blank key returns (nil, nil).
func LoadArmored(armored, passphrase string) (*Signer, error) {
if armored == "" {
return nil, nil
}
return fromArmor(armored, passphrase, "stored key")
}
// Generate creates a fresh signing keypair and returns the armored private key
// (to persist) and its uppercase key id.
func Generate() (armoredPrivateKey, keyID string, err error) {
entity, err := openpgp.NewEntity("artifactapi terraform registry", "provider signing", "artifactapi@localhost", nil)
if err != nil {
return "", "", err
}
var buf bytes.Buffer
w, err := armor.Encode(&buf, openpgp.PrivateKeyType, nil)
if err != nil {
return "", "", err
}
if err := entity.SerializePrivate(w, nil); err != nil {
return "", "", err
}
if err := w.Close(); err != nil {
return "", "", err
}
return buf.String(), strings.ToUpper(entity.PrimaryKey.KeyIdString()), nil
}
func fromArmor(armored, passphrase, src string) (*Signer, error) {
keyring, err := openpgp.ReadArmoredKeyRing(strings.NewReader(armored))
if err != nil {
return nil, fmt.Errorf("read signing key: %w", err)
}
if len(keyring) == 0 {
return nil, fmt.Errorf("signing key (%s) contains no entities", src)
}
entity := keyring[0]
if entity.PrivateKey == nil {
return nil, fmt.Errorf("signing key (%s) has no private key material", src)
}
if entity.PrivateKey.Encrypted {
if err := decrypt(entity, passphrase); err != nil {
return nil, err
}
}
pub, err := armorPublicKey(entity)
if err != nil {
return nil, err
}
return &Signer{
entity: entity,
publicASCII: pub,
keyID: entity.PrimaryKey.KeyIdString(),
}, nil
}
// decrypt unlocks the entity's private key and all subkeys with the passphrase.
func decrypt(entity *openpgp.Entity, passphrase string) error {
pw := []byte(passphrase)
if err := entity.PrivateKey.Decrypt(pw); err != nil {
return fmt.Errorf("decrypt signing key: %w", err)
}
for _, sub := range entity.Subkeys {
if sub.PrivateKey != nil && sub.PrivateKey.Encrypted {
_ = sub.PrivateKey.Decrypt(pw)
}
}
return nil
}
func armorPublicKey(entity *openpgp.Entity) (string, error) {
var buf bytes.Buffer
w, err := armor.Encode(&buf, openpgp.PublicKeyType, nil)
if err != nil {
return "", err
}
if err := entity.Serialize(w); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
return buf.String(), nil
}
// Sign returns a binary detached signature over message, matching the
// SHA256SUMS.sig format Terraform verifies.
func (s *Signer) Sign(message []byte) ([]byte, error) {
var buf bytes.Buffer
if err := openpgp.DetachSign(&buf, s.entity, bytes.NewReader(message), nil); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// PublicKeyArmor returns the ASCII-armored public key for the registry's
// signing_keys response.
func (s *Signer) PublicKeyArmor() string { return s.publicASCII }
// KeyID returns the 16-hex-char uppercase key ID Terraform matches against the
// signature's issuer.
func (s *Signer) KeyID() string { return strings.ToUpper(s.keyID) }
-155
View File
@@ -1,155 +0,0 @@
package tfsign
import (
"bytes"
"context"
"os"
"path/filepath"
"regexp"
"testing"
"golang.org/x/crypto/openpgp"
"golang.org/x/crypto/openpgp/armor"
)
// armoredPrivateKey generates a throwaway armored private key for tests.
func armoredPrivateKey(t *testing.T) string {
t.Helper()
e, err := openpgp.NewEntity("artifactapi test", "tf registry", "tf@example.com", nil)
if err != nil {
t.Fatal(err)
}
var buf bytes.Buffer
w, err := armor.Encode(&buf, openpgp.PrivateKeyType, nil)
if err != nil {
t.Fatal(err)
}
if err := e.SerializePrivate(w, nil); err != nil {
t.Fatal(err)
}
w.Close()
return buf.String()
}
func writeKey(t *testing.T, contents string) string {
t.Helper()
p := filepath.Join(t.TempDir(), "private-key.asc")
if err := os.WriteFile(p, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
return p
}
func TestLoadSignAndVerify(t *testing.T) {
path := writeKey(t, armoredPrivateKey(t))
s, err := Load(path, "")
if err != nil {
t.Fatal(err)
}
if s == nil {
t.Fatal("expected a signer")
}
if !regexp.MustCompile(`^[0-9A-F]{16}$`).MatchString(s.KeyID()) {
t.Errorf("key id %q is not 16 uppercase hex chars", s.KeyID())
}
msg := []byte("deadbeef terraform-provider-x_1.0.0_linux_amd64.zip\n")
sig, err := s.Sign(msg)
if err != nil {
t.Fatal(err)
}
// The advertised public key must verify the signature over the same bytes.
keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewReader([]byte(s.PublicKeyArmor())))
if err != nil {
t.Fatal(err)
}
if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewReader(msg), bytes.NewReader(sig)); err != nil {
t.Errorf("signature did not verify: %v", err)
}
}
func TestGenerateAndLoadArmored(t *testing.T) {
priv, keyID, err := Generate()
if err != nil {
t.Fatal(err)
}
if !regexp.MustCompile(`^[0-9A-F]{16}$`).MatchString(keyID) {
t.Errorf("generated key id %q malformed", keyID)
}
s, err := LoadArmored(priv, "")
if err != nil {
t.Fatal(err)
}
if s.KeyID() != keyID {
t.Errorf("loaded key id %q != generated %q", s.KeyID(), keyID)
}
msg := []byte("abc terraform-provider-x_1.0.0_linux_amd64.zip\n")
sig, err := s.Sign(msg)
if err != nil {
t.Fatal(err)
}
keyring, _ := openpgp.ReadArmoredKeyRing(bytes.NewReader([]byte(s.PublicKeyArmor())))
if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewReader(msg), bytes.NewReader(sig)); err != nil {
t.Errorf("signature did not verify: %v", err)
}
}
// memStore is an in-memory KeyStore that records how many keys it accepted.
type memStore struct {
armor, keyID string
found bool
inserts int
}
func (m *memStore) GetSigningKey(_ context.Context, _ string) (string, string, bool, error) {
return m.armor, m.keyID, m.found, nil
}
func (m *memStore) InsertSigningKeyIfAbsent(_ context.Context, _, armor, keyID string) error {
if !m.found { // ON CONFLICT DO NOTHING
m.armor, m.keyID, m.found = armor, keyID, true
m.inserts++
}
return nil
}
func TestLoadOrCreateGeneratesOnceThenReuses(t *testing.T) {
store := &memStore{}
first, err := LoadOrCreate(context.Background(), store, "terraform-provider")
if err != nil || first == nil {
t.Fatalf("first LoadOrCreate: signer=%v err=%v", first, err)
}
second, err := LoadOrCreate(context.Background(), store, "terraform-provider")
if err != nil || second == nil {
t.Fatalf("second LoadOrCreate: signer=%v err=%v", second, err)
}
if store.inserts != 1 {
t.Errorf("expected exactly one key generated, got %d", store.inserts)
}
if first.KeyID() != second.KeyID() {
t.Errorf("key id changed between loads: %q vs %q", first.KeyID(), second.KeyID())
}
}
func TestLoadEmptyPathDisabled(t *testing.T) {
s, err := Load("", "")
if err != nil {
t.Fatal(err)
}
if s != nil {
t.Error("empty path should yield a nil (disabled) signer")
}
}
func TestLoadMissingFile(t *testing.T) {
if _, err := Load(filepath.Join(t.TempDir(), "nope.asc"), ""); err == nil {
t.Error("expected an error for a missing key file")
}
}
+1 -25
View File
@@ -73,16 +73,6 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
return return
} }
if remote.RepoType == models.RepoTypeLocal {
body, err := e.fetchLocalIndex(ctx, *remote, path)
if err != nil {
results[idx] = result{err: fmt.Errorf("local index %q: %w", name, err)}
return
}
results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, BaseURL: remote.BaseURL, Body: body}}
return
}
prov, err := provider.Get(remote.PackageType) prov, err := provider.Get(remote.PackageType)
if err != nil { if err != nil {
results[idx] = result{err: fmt.Errorf("provider %q: %w", remote.PackageType, err)} results[idx] = result{err: fmt.Errorf("provider %q: %w", remote.PackageType, err)}
@@ -102,7 +92,7 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
return return
} }
results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, BaseURL: remote.BaseURL, Body: body}} results[idx] = result{index: MemberIndex{RemoteName: name, Body: body}}
}(i, memberName) }(i, memberName)
} }
@@ -119,17 +109,3 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
return members, nil return members, nil
} }
func (e *Engine) fetchLocalIndex(ctx context.Context, remote models.Remote, path string) ([]byte, error) {
prov, err := provider.Get(remote.PackageType)
if err != nil {
return nil, fmt.Errorf("no provider for %q: %w", remote.PackageType, err)
}
indexer, ok := prov.(provider.LocalIndexer)
if !ok {
return nil, fmt.Errorf("provider %q does not support local index generation", remote.PackageType)
}
return indexer.GenerateLocalIndex(ctx, e.db, remote.Name, path)
}
+3 -40
View File
@@ -54,27 +54,15 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
seen[chart][ver.Version] = true seen[chart][ver.Version] = true
if proxyBaseURL != "" { if proxyBaseURL != "" {
routePrefix := "remote"
if member.RepoType == "local" {
routePrefix = "local"
}
baseHost := extractHost(member.BaseURL)
for i, u := range ver.URLs { for i, u := range ver.URLs {
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") { if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
if baseHost != "" && extractHost(u) != baseHost { ver.URLs[i] = fmt.Sprintf("%s/api/v1/remote/%s/%s",
continue
}
relPath := extractPathRelativeToBase(u, member.BaseURL)
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"), strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName, member.RemoteName,
relPath) extractPath(u))
} else { } else {
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s", ver.URLs[i] = fmt.Sprintf("%s/api/v1/remote/%s/%s",
strings.TrimRight(proxyBaseURL, "/"), strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName, member.RemoteName,
u) u)
} }
@@ -90,31 +78,6 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
return yaml.Marshal(merged) return yaml.Marshal(merged)
} }
func extractHost(rawURL string) string {
idx := strings.Index(rawURL, "://")
if idx == -1 {
return ""
}
rest := rawURL[idx+3:]
slashIdx := strings.Index(rest, "/")
if slashIdx == -1 {
return rest
}
return rest[:slashIdx]
}
func extractPathRelativeToBase(rawURL, baseURL string) string {
fullPath := extractPath(rawURL)
basePath := extractPath(baseURL)
if basePath != "" {
basePath = strings.TrimRight(basePath, "/") + "/"
if strings.HasPrefix(fullPath, basePath) {
return fullPath[len(basePath):]
}
}
return fullPath
}
func extractPath(rawURL string) string { func extractPath(rawURL string) string {
idx := strings.Index(rawURL, "://") idx := strings.Index(rawURL, "://")
if idx == -1 { if idx == -1 {
-2
View File
@@ -8,8 +8,6 @@ import (
type MemberIndex struct { type MemberIndex struct {
RemoteName string RemoteName string
RepoType models.RepoType
BaseURL string
Body []byte Body []byte
} }
-155
View File
@@ -1,155 +0,0 @@
package virtual
import (
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestRegisterGetMerger(t *testing.T) {
if _, err := GetMerger(models.PackageHelm); err != nil {
t.Errorf("helm merger should be registered: %v", err)
}
if _, err := GetMerger(models.PackagePyPI); err != nil {
t.Errorf("pypi merger should be registered: %v", err)
}
if _, err := GetMerger(models.PackageType("nope")); err == nil {
t.Error("expected error for unknown merger")
}
}
func TestPyPIMerge(t *testing.T) {
m := &PyPIMerger{}
members := []MemberIndex{
{RemoteName: "a", RepoType: models.RepoTypeRemote, Body: []byte(`<a href="pkg/foo-1.0.whl">foo-1.0.whl</a>`)},
{RemoteName: "b", RepoType: models.RepoTypeLocal, Body: []byte(`<a href="/bar-2.0.whl">bar-2.0.whl</a>`)},
}
out, err := m.MergeIndexes(members, "http://proxy")
if err != nil {
t.Fatal(err)
}
s := string(out)
if !strings.Contains(s, "foo-1.0.whl") || !strings.Contains(s, "bar-2.0.whl") {
t.Errorf("merged index missing entries: %s", s)
}
if !strings.Contains(s, "http://proxy/api/v1/remote/a/pkg/foo-1.0.whl") {
t.Errorf("remote href not rewritten: %s", s)
}
if !strings.Contains(s, "http://proxy/api/v1/local/b/bar-2.0.whl") {
t.Errorf("local href not rewritten: %s", s)
}
// Sorted output: foo before... entries sorted by link text.
if strings.Index(s, "bar-2.0.whl") > strings.Index(s, "foo-1.0.whl") {
t.Error("entries should be sorted by text")
}
// Duplicate link texts across members are de-duplicated.
dup := []MemberIndex{
{RemoteName: "a", Body: []byte(`<a href="x">dup</a>`)},
{RemoteName: "b", Body: []byte(`<a href="y">dup</a>`)},
}
out, _ = m.MergeIndexes(dup, "")
if strings.Count(string(out), ">dup</a>") != 1 {
t.Errorf("duplicate not de-duplicated: %s", out)
}
}
func TestPyPIMergeNoProxyAndBadLinks(t *testing.T) {
m := &PyPIMerger{}
members := []MemberIndex{{
RemoteName: "a",
Body: []byte("<a href=\"foo.whl\">foo.whl</a>\n<a>no href</a>\n<span>not a link</span>"),
}}
// No proxy base URL: hrefs are left as-is.
out, err := m.MergeIndexes(members, "")
if err != nil {
t.Fatal(err)
}
s := string(out)
if !strings.Contains(s, ">foo.whl</a>") {
t.Errorf("missing link: %s", s)
}
}
func TestHelmMerge(t *testing.T) {
m := &HelmMerger{}
memberA := `apiVersion: v1
entries:
alpha:
- name: alpha
version: 1.0.0
urls:
- charts/alpha-1.0.0.tgz
`
memberB := `apiVersion: v1
entries:
beta:
- name: beta
version: 2.0.0
urls:
- https://charts.example.com/beta-2.0.0.tgz
gamma:
- name: gamma
version: 3.0.0
urls:
- https://other-host.example.net/gamma-3.0.0.tgz
`
members := []MemberIndex{
{RemoteName: "a", RepoType: models.RepoTypeLocal, BaseURL: "https://charts.example.com", Body: []byte(memberA)},
{RemoteName: "b", RepoType: models.RepoTypeRemote, BaseURL: "https://charts.example.com", Body: []byte(memberB)},
}
out, err := m.MergeIndexes(members, "http://proxy")
if err != nil {
t.Fatal(err)
}
s := string(out)
for _, chart := range []string{"alpha", "beta", "gamma"} {
if !strings.Contains(s, chart) {
t.Errorf("merged index missing chart %q: %s", chart, s)
}
}
// Relative URL from a local member is rewritten under /local/.
if !strings.Contains(s, "http://proxy/api/v1/local/a/charts/alpha-1.0.0.tgz") {
t.Errorf("relative local url not rewritten: %s", s)
}
// Same-host absolute URL from a remote member is rewritten under /remote/.
if !strings.Contains(s, "http://proxy/api/v1/remote/b/beta-2.0.0.tgz") {
t.Errorf("same-host absolute url not rewritten: %s", s)
}
// Cross-host absolute URL is left untouched.
if !strings.Contains(s, "https://other-host.example.net/gamma-3.0.0.tgz") {
t.Errorf("cross-host url should be preserved: %s", s)
}
}
func TestHelmMergeDedup(t *testing.T) {
m := &HelmMerger{}
body := `apiVersion: v1
entries:
alpha:
- name: alpha
version: 1.0.0
urls: [charts/alpha-1.0.0.tgz]
`
members := []MemberIndex{
{RemoteName: "a", BaseURL: "https://x", Body: []byte(body)},
{RemoteName: "b", BaseURL: "https://x", Body: []byte(body)},
}
out, _ := m.MergeIndexes(members, "")
if strings.Count(string(out), "version: 1.0.0") != 1 {
t.Errorf("duplicate chart version not de-duplicated: %s", out)
}
}
func TestHelmMergeInvalidYAML(t *testing.T) {
m := &HelmMerger{}
out, err := m.MergeIndexes([]MemberIndex{{RemoteName: "a", Body: []byte("::: not yaml :::")}}, "")
if err != nil {
t.Fatalf("invalid member yaml should be skipped, not error: %v", err)
}
if !strings.Contains(string(out), "apiVersion") {
t.Errorf("expected a valid empty merged index: %s", out)
}
}
+1 -6
View File
@@ -36,13 +36,8 @@ func (m *PyPIMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
} }
if proxyBaseURL != "" && href != "" { if proxyBaseURL != "" && href != "" {
routePrefix := "remote" href = fmt.Sprintf("%s/api/v1/remote/%s/%s",
if member.RepoType == "local" {
routePrefix = "local"
}
href = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"), strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName, member.RemoteName,
strings.TrimLeft(href, "/")) strings.TrimLeft(href, "/"))
} }
-145
View File
@@ -1,145 +0,0 @@
package client
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func testServer(t *testing.T, h http.HandlerFunc) *Client {
t.Helper()
srv := httptest.NewServer(h)
t.Cleanup(srv.Close)
return New(srv.URL)
}
func TestRemotesRoundTrip(t *testing.T) {
var gotMethod, gotPath string
c := testServer(t, func(w http.ResponseWriter, r *http.Request) {
gotMethod, gotPath = r.Method, r.URL.Path
switch {
case r.Method == http.MethodGet && r.URL.Path == "/api/v2/remotes":
w.Write([]byte(`[{"name":"a"},{"name":"b"}]`))
case r.Method == http.MethodGet && r.URL.Path == "/api/v2/remotes/a":
w.Write([]byte(`{"name":"a"}`))
case r.Method == http.MethodDelete:
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusCreated)
w.Write([]byte(`{"name":"a"}`))
}
})
ctx := context.Background()
remotes, err := c.ListRemotes(ctx)
if err != nil || len(remotes) != 2 {
t.Fatalf("ListRemotes: %v %v", remotes, err)
}
if r, err := c.GetRemote(ctx, "a"); err != nil || r.Name != "a" {
t.Fatalf("GetRemote: %v %v", r, err)
}
if err := c.CreateRemote(ctx, &models.Remote{Name: "a", PackageType: models.PackageGeneric}); err != nil {
t.Fatalf("CreateRemote: %v", err)
}
if err := c.UpdateRemote(ctx, &models.Remote{Name: "a"}); err != nil {
t.Fatalf("UpdateRemote: %v", err)
}
if err := c.DeleteRemote(ctx, "a"); err != nil {
t.Fatalf("DeleteRemote: %v", err)
}
if gotMethod != http.MethodDelete || gotPath != "/api/v2/remotes/a" {
t.Errorf("last call = %s %s", gotMethod, gotPath)
}
}
func TestVirtualsRoundTrip(t *testing.T) {
c := testServer(t, func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/virtuals"):
w.Write([]byte(`[{"name":"v"}]`))
case r.Method == http.MethodGet:
w.Write([]byte(`{"name":"v"}`))
case r.Method == http.MethodDelete:
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusCreated)
w.Write([]byte(`{"name":"v"}`))
}
})
ctx := context.Background()
if vs, err := c.ListVirtuals(ctx); err != nil || len(vs) != 1 {
t.Fatalf("ListVirtuals: %v %v", vs, err)
}
if v, err := c.GetVirtual(ctx, "v"); err != nil || v.Name != "v" {
t.Fatalf("GetVirtual: %v %v", v, err)
}
if err := c.CreateVirtual(ctx, &models.Virtual{Name: "v"}); err != nil {
t.Fatalf("CreateVirtual: %v", err)
}
if err := c.UpdateVirtual(ctx, &models.Virtual{Name: "v"}); err != nil {
t.Fatalf("UpdateVirtual: %v", err)
}
if err := c.DeleteVirtual(ctx, "v"); err != nil {
t.Fatalf("DeleteVirtual: %v", err)
}
}
func TestStatsHealthObjects(t *testing.T) {
c := testServer(t, func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/stats"):
w.Write([]byte(`{"total_remotes":3}`))
case strings.HasSuffix(r.URL.Path, "/health"):
w.Write([]byte(`{"status":"ok"}`))
case r.Method == http.MethodDelete:
w.WriteHeader(http.StatusNoContent)
default:
w.Write([]byte(`[{"path":"p"}]`))
}
})
ctx := context.Background()
if _, err := c.Stats(ctx); err != nil {
t.Fatalf("Stats: %v", err)
}
if _, err := c.Health(ctx); err != nil {
t.Fatalf("Health: %v", err)
}
if objs, err := c.ListObjects(ctx, "r", 1, 50); err != nil || len(objs) != 1 {
t.Fatalf("ListObjects: %v %v", objs, err)
}
if err := c.EvictObject(ctx, "r", "some/path"); err != nil {
t.Fatalf("EvictObject: %v", err)
}
}
func TestErrorResponses(t *testing.T) {
c := testServer(t, func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "boom", http.StatusInternalServerError)
})
ctx := context.Background()
_, err := c.GetRemote(ctx, "x")
if err == nil || !strings.Contains(err.Error(), "api error 500") {
t.Errorf("expected api error, got %v", err)
}
}
func TestDecodeError(t *testing.T) {
c := testServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`not json`))
})
if _, err := c.ListRemotes(context.Background()); err == nil || !strings.Contains(err.Error(), "decode") {
t.Errorf("expected decode error, got %v", err)
}
}
func TestRequestError(t *testing.T) {
// Invalid base URL triggers request construction failure.
c := New("http://[::1]:namedport")
if err := c.DeleteRemote(context.Background(), "x"); err == nil {
t.Error("expected request error for invalid URL")
}
}
-30
View File
@@ -2,7 +2,6 @@ package models
import ( import (
"fmt" "fmt"
"regexp"
"time" "time"
) )
@@ -47,11 +46,6 @@ type Remote struct {
MutableTTL int `json:"mutable_ttl"` MutableTTL int `json:"mutable_ttl"`
CheckMutable bool `json:"check_mutable"` CheckMutable bool `json:"check_mutable"`
// Upstream HTTP timeouts in seconds. 0 means use the server default.
UpstreamDialTimeout int `json:"upstream_dial_timeout,omitempty"`
UpstreamTLSTimeout int `json:"upstream_tls_timeout,omitempty"`
UpstreamResponseHeaderTimeout int `json:"upstream_response_header_timeout,omitempty"`
Patterns []string `json:"patterns,omitempty"` Patterns []string `json:"patterns,omitempty"`
Blocklist []string `json:"blocklist,omitempty"` Blocklist []string `json:"blocklist,omitempty"`
MutablePatterns []string `json:"mutable_patterns,omitempty"` MutablePatterns []string `json:"mutable_patterns,omitempty"`
@@ -72,30 +66,6 @@ type Remote struct {
UpdatedAt time.Time `json:"updated_at"` UpdatedAt time.Time `json:"updated_at"`
} }
// ValidatePatterns ensures every configured regex compiles. Storing an
// invalid pattern would otherwise be silently dropped at match time, which
// for the blocklist is a fail-open: a mistyped deny rule becomes a no-op.
func (r *Remote) ValidatePatterns() error {
groups := []struct {
field string
patterns []string
}{
{"patterns", r.Patterns},
{"blocklist", r.Blocklist},
{"mutable_patterns", r.MutablePatterns},
{"immutable_patterns", r.ImmutablePatterns},
{"ban_tags", r.BanTags},
}
for _, g := range groups {
for _, p := range g.patterns {
if _, err := regexp.Compile(p); err != nil {
return fmt.Errorf("invalid regex in %s: %q: %w", g.field, p, err)
}
}
}
return nil
}
type RemoteWithStats struct { type RemoteWithStats struct {
Remote Remote
Stats RemoteStats `json:"stats"` Stats RemoteStats `json:"stats"`
-19
View File
@@ -1,19 +0,0 @@
package models
import "testing"
func TestRemote_ValidatePatterns(t *testing.T) {
valid := &Remote{
Patterns: []string{`.*\.tar\.gz$`},
Blocklist: []string{`^secret/`},
ImmutablePatterns: []string{`\.rpm$`},
}
if err := valid.ValidatePatterns(); err != nil {
t.Fatalf("expected valid patterns, got %v", err)
}
bad := &Remote{Blocklist: []string{`[unterminated`}}
if err := bad.ValidatePatterns(); err == nil {
t.Fatal("expected error for invalid blocklist regex, got nil")
}
}
-18
View File
@@ -1,18 +0,0 @@
package models
import "testing"
func TestRepoType(t *testing.T) {
if RepoTypeRemote.String() != "remote" || RepoTypeLocal.String() != "local" {
t.Error("RepoType.String")
}
if !RepoTypeRemote.Valid() || RepoType("bogus").Valid() {
t.Error("RepoType.Valid")
}
if rt, err := ParseRepoType("local"); err != nil || rt != RepoTypeLocal {
t.Errorf("ParseRepoType(local) = %v %v", rt, err)
}
if _, err := ParseRepoType("nope"); err == nil {
t.Error("ParseRepoType should reject unknown")
}
}
-40
View File
@@ -1,40 +0,0 @@
#!/usr/bin/env bash
# Build the artifactapi container, bring up the full stack (postgres, redis,
# minio, artifactapi) plus a static mock upstream, and run the dockerised e2e
# suite against the running product over HTTP. Tears everything down on exit.
set -euo pipefail
cd "$(dirname "$0")/.."
# Publish artifactapi on 8001 to avoid colliding with a local instance on 8000.
export ARTIFACTAPI_PORT="${ARTIFACTAPI_PORT:-8001}"
COMPOSE=(docker compose -f docker-compose.yml -f docker-compose.e2e.yml)
API_URL="${ARTIFACTAPI_URL:-http://localhost:${ARTIFACTAPI_PORT}}"
cleanup() {
echo "==> tearing down stack"
"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
}
trap cleanup EXIT
echo "==> building and starting stack (postgres, redis, minio, mockupstream, artifactapi)"
"${COMPOSE[@]}" up -d --build postgres redis minio mockupstream artifactapi
echo "==> waiting for artifactapi health at ${API_URL}"
for i in $(seq 1 60); do
if curl -fsS "${API_URL}/health" >/dev/null 2>&1; then
echo " healthy after ${i}s"
break
fi
if [ "$i" -eq 60 ]; then
echo "!!! artifactapi did not become healthy in time; recent logs:"
"${COMPOSE[@]}" logs --tail=50 artifactapi
exit 1
fi
sleep 1
done
echo "==> running dockerised e2e suite"
ARTIFACTAPI_URL="${API_URL}" \
MOCK_UPSTREAM_INTERNAL="${MOCK_UPSTREAM_INTERNAL:-http://mockupstream}" \
go test -tags=dockere2e -count=1 -timeout=10m -v ./e2e-docker/...
-7
View File
@@ -6,20 +6,13 @@ COPY package.json package-lock.json* ./
RUN npm ci RUN npm ci
COPY . . COPY . .
ARG BASE_PATH=/
ENV BASE_PATH=${BASE_PATH}
RUN npm run build RUN npm run build
FROM nginx:alpine FROM nginx:alpine
ARG BASE_PATH=/
COPY --from=builder /app/dist /usr/share/nginx/html COPY --from=builder /app/dist /usr/share/nginx/html
COPY nginx.conf /etc/nginx/conf.d/default.conf COPY nginx.conf /etc/nginx/conf.d/default.conf
RUN sed -i "s|\${BASE_PATH}|${BASE_PATH}|g" /etc/nginx/conf.d/default.conf
EXPOSE 80 EXPOSE 80
CMD ["nginx", "-g", "daemon off;"] CMD ["nginx", "-g", "daemon off;"]
+26 -5
View File
@@ -5,12 +5,33 @@ server {
root /usr/share/nginx/html; root /usr/share/nginx/html;
index index.html; index index.html;
location ${BASE_PATH}/ { location /api/ {
rewrite ^${BASE_PATH}(/.*)$ $1 break; proxy_pass http://artifactapi:8000;
try_files $uri $uri/ /index.html; proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off;
} }
location = ${BASE_PATH} { location /v2/ {
return 301 ${BASE_PATH}/; proxy_pass http://artifactapi:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off;
}
location /health {
proxy_pass http://artifactapi:8000;
}
location /metrics {
proxy_pass http://artifactapi:8000;
}
location / {
try_files $uri $uri/ /index.html;
} }
} }
-6
View File
@@ -2,8 +2,6 @@ import { Routes, Route, NavLink } from 'react-router-dom';
import { Dashboard } from './pages/Dashboard'; import { Dashboard } from './pages/Dashboard';
import { Remotes } from './pages/Remotes'; import { Remotes } from './pages/Remotes';
import { RemoteDetail } from './pages/RemoteDetail'; import { RemoteDetail } from './pages/RemoteDetail';
import { Locals } from './pages/Locals';
import { LocalDetail } from './pages/LocalDetail';
import { Virtuals } from './pages/Virtuals'; import { Virtuals } from './pages/Virtuals';
import { Objects } from './pages/Objects'; import { Objects } from './pages/Objects';
import { Probe } from './pages/Probe'; import { Probe } from './pages/Probe';
@@ -20,7 +18,6 @@ export function App() {
<div className="sidebar-nav"> <div className="sidebar-nav">
<NavLink to="/" end>Dashboard</NavLink> <NavLink to="/" end>Dashboard</NavLink>
<NavLink to="/remotes">Remotes</NavLink> <NavLink to="/remotes">Remotes</NavLink>
<NavLink to="/locals">Locals</NavLink>
<NavLink to="/virtuals">Virtuals</NavLink> <NavLink to="/virtuals">Virtuals</NavLink>
<NavLink to="/probe">Test Remote</NavLink> <NavLink to="/probe">Test Remote</NavLink>
</div> </div>
@@ -34,9 +31,6 @@ export function App() {
<Route path="/remotes" element={<Remotes />} /> <Route path="/remotes" element={<Remotes />} />
<Route path="/remotes/:name" element={<RemoteDetail />} /> <Route path="/remotes/:name" element={<RemoteDetail />} />
<Route path="/remotes/:name/objects" element={<Objects />} /> <Route path="/remotes/:name/objects" element={<Objects />} />
<Route path="/locals" element={<Locals />} />
<Route path="/locals/:name" element={<LocalDetail />} />
<Route path="/locals/:name/objects" element={<Objects />} />
<Route path="/virtuals" element={<Virtuals />} /> <Route path="/virtuals" element={<Virtuals />} />
<Route path="/probe" element={<Probe />} /> <Route path="/probe" element={<Probe />} />
</Routes> </Routes>
-6
View File
@@ -34,12 +34,6 @@ export const api = {
evictObject: (remote: string, path: string) => evictObject: (remote: string, path: string) =>
fetchJSON<void>(`/api/v2/remotes/${remote}/objects/${path}`, { method: 'DELETE' }), fetchJSON<void>(`/api/v2/remotes/${remote}/objects/${path}`, { method: 'DELETE' }),
listLocalObjects: (name: string, page = 1, perPage = 50) =>
fetchJSON<Artifact[]>(`/api/v2/locals/${name}/objects?page=${page}&per_page=${perPage}`),
evictLocalObject: (name: string, path: string) =>
fetchJSON<void>(`/api/v2/locals/${name}/objects/${path}`, { method: 'DELETE' }),
flushRemoteCache: (remote: string) => flushRemoteCache: (remote: string) =>
fetchJSON<void>(`/api/v2/remotes/${remote}/cache`, { method: 'DELETE' }), fetchJSON<void>(`/api/v2/remotes/${remote}/cache`, { method: 'DELETE' }),

Some files were not shown because too many files have changed in this diff Show More