Compare commits

..

24 Commits

Author SHA1 Message Date
unkinben 1009bc7c69 test: database closed-db error paths + pure goproxy/generic/helm/repotype gaps
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/test Pipeline failed
2026-07-03 13:55:30 +10:00
unkinben 2ae12d9aef test: local upload store-failure branches (kill minio mid-test) 2026-07-03 13:53:13 +10:00
unkinben 1d97628fbd test: rpm serve errors, terraform rewriteDownloadURL, pypi merger, s3 stat, pattern-validation 400 2026-07-03 13:51:17 +10:00
unkinben 3e590bea2b test: classifier branches, docker provider, pypi index serving, bearer/checkUpstream variants 2026-07-03 13:48:27 +10:00
unkinben 053fbd70e2 test: provider-not-found and missing-blob download error branches 2026-07-03 13:45:19 +10:00
unkinben 03a29f2c0a test: ttlFor, HEAD 500/cached-index, rpm full-metadata xml, generic content-type 2026-07-03 13:42:17 +10:00
unkinben cfd0bae515 test: server Run, upstream 500/401 branches, virtual dead members, local DB accessor 2026-07-03 13:38:49 +10:00
unkinben 6029f19b86 test: proxy HTTP error branches (403/502) + revalidation upstream error 2026-07-03 13:35:40 +10:00
unkinben 2658167346 test: circuit breaker states, bearer cache-hit, blob dedup, terraform/storage branches 2026-07-03 13:33:24 +10:00
unkinben 4ec3d7b59e test: v2 handler error paths (closed db), v1 scheme, rpm content-type/flag helpers 2026-07-03 13:29:23 +10:00
unkinben 1842042241 test: RunOnListener, local pypi virtual merge, v2 download, waitForStore/Unwrap 2026-07-03 13:25:54 +10:00
unkinben 6bcda813fd test: probe branches, local upload errors, upstream errors, rpm/gc error paths 2026-07-03 13:22:24 +10:00
unkinben 3d80837539 test: bearer-token flow (engine) + docker/head/virtual/probe/events (server) 2026-07-03 13:18:51 +10:00
unkinben 915ec910c8 test: expand server test (pypi/rpm local, object evict, validation errors) 2026-07-03 13:15:52 +10:00
unkinben ba4a0e3b43 test: gc sweep integration (delete old orphan, empty no-op) 2026-07-03 13:11:55 +10:00
unkinben 21579c1ec0 test: proxy engine edge cases + storage/server minio retry
Cover denied, HEAD (cached/uncached/denied), stale-on-error, circuit-open,
and mutable revalidation (304) paths. proxy ~8->58%.
2026-07-03 13:11:09 +10:00
unkinben 696a1955b3 test: wait for minio readiness endpoint (avoid init-race flake) 2026-07-03 13:05:40 +10:00
unkinben a6830d6d9f test: wait for postgres readiness log twice (avoid connection-reset flake) 2026-07-03 13:03:40 +10:00
unkinben 6ba78caf4c test: full-stack server integration test
Drives the whole HTTP API against real Postgres/Redis/MinIO + a mock
upstream: health, remote CRUD, proxy miss/hit, local upload/download,
virtual helm merge, objects, stats, probe. Exercises server + api/v1 +
api/v2 + proxy + virtual engines together.
2026-07-03 12:59:39 +10:00
unkinben 5e7500ed04 test: storage (MinIO) and cache (Redis) integration tests
storage 0->79%, cache 0->92%, via the testsupport container harness.
2026-07-03 12:58:00 +10:00
unkinben bf31714e08 test: testcontainers harness + database integration tests
Add internal/testsupport (Postgres/Redis/MinIO container helpers, Ryuk
disabled) and full database coverage: remotes, artifacts/blobs, local
files, virtuals, stats, rpm metadata, orphan/cold cleanup. database
0->83%.
2026-07-03 12:55:51 +10:00
unkinben db663e00d7 test: cover virtual mergers and pkg/client
virtual mergers (helm/pypi merge, dedup, URL rewriting, registry) and the
API client (all endpoints via httptest, error/decode paths). pkg/client
0->96%, virtual 49->57%.
2026-07-03 12:45:49 +10:00
unkinben b6091c859d test: cover rpm provider (metadata parse, repodata generation)
rpm 2.7% -> 83.6%, pure-go via a real RPM testdata fixture and fake
BlobReader/MetadataStore/RPMMetadataReader implementations.
2026-07-03 12:43:21 +10:00
unkinben 428c6d0e97 test: unit tests for pure provider/config/auth packages
First increment toward 90% core-package coverage. Adds Docker-free unit
tests taking these packages to full or near-full coverage:
- provider/npm, provider/alpine, provider/puppet: 100%
- provider/pypi: index generation via a fake FileStore, upload validation,
  name parsing, classification, rewrite
- config: defaults, overrides, DSN, invalid port
- auth: basic header with/without credentials

Infra-backed packages (database, storage, cache, proxy engine, api
handlers, server) still need the testcontainers batch to reach 90%.
2026-07-03 12:35:05 +10:00
32 changed files with 20 additions and 822 deletions
-4
View File
@@ -1,6 +1,2 @@
bin/
terraform/
# e2e-docker fixtures are real package files (.rpm, .tgz, .whl, .zip, ...) that
# are intentionally tracked, overriding any global ignore of those extensions.
!e2e-docker/fixtures/**
+1 -6
View File
@@ -1,4 +1,4 @@
.PHONY: build test lint fmt e2e docker-e2e docker docker-ui compose clean tidy check-go
.PHONY: build test lint fmt e2e docker docker-ui compose clean tidy check-go
BINARY := bin/artifactapi
MODULE := git.unkin.net/unkin/artifactapi
@@ -28,11 +28,6 @@ fmt: check-go
e2e: check-go
TESTCONTAINERS_RYUK_DISABLED=true go test -tags=e2e -race -count=1 -timeout=5m ./e2e/...
# Build the container, bring up the full docker-compose stack + a mock upstream,
# and run the black-box suite against the running product.
docker-e2e: check-go
./scripts/docker-e2e.sh
docker:
docker build -t artifactapi:$(VERSION) .
-18
View File
@@ -1,18 +0,0 @@
# Overlay for the dockerised end-to-end suite (scripts/docker-e2e.sh).
# Adds a static mock upstream that the artifactapi container proxies, so the
# caching tests are hermetic and need no internet access.
services:
mockupstream:
image: nginx:alpine
volumes:
- ./e2e-docker/fixtures:/usr/share/nginx/html:ro,z
# No host port needed: only the artifactapi container talks to it, and the
# tests compare served bytes against the on-disk fixtures.
artifactapi:
# The host port is set via ARTIFACTAPI_PORT (see scripts/docker-e2e.sh),
# defaulting to 8000; the e2e run uses 8001 to avoid colliding with a
# locally-running instance.
depends_on:
mockupstream:
condition: service_started
+1 -1
View File
@@ -2,7 +2,7 @@ services:
artifactapi:
build: .
ports:
- "${ARTIFACTAPI_PORT:-8000}:8000"
- "8000:8000"
environment:
LISTEN_ADDR: ":8000"
DBHOST: postgres
-39
View File
@@ -1,39 +0,0 @@
# Dockerised end-to-end suite
Black-box tests that run against a fully **containerised** artifactapi stack
(built image + Postgres + Redis + MinIO) plus a static mock upstream. Unlike the
in-process `e2e/` suite (testcontainers, server run in-process), these only speak
HTTP to the running product, so they exercise the shipped container image.
## Run
```bash
make docker-e2e # build image, compose up, run suite, compose down
```
`scripts/docker-e2e.sh` builds and starts `docker-compose.yml` +
`docker-compose.e2e.yml`, waits for `/health`, then runs
`go test -tags=dockere2e ./e2e-docker/...` and tears everything down.
The stack publishes artifactapi on host port **8001** (to avoid colliding with a
local instance on 8000). Override with `ARTIFACTAPI_URL` to point the tests at an
already-running stack.
## Coverage
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
- **Caching** — one immutable artifact per remote package type (generic, docker,
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
verified against the origin fixture.
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
index), rpm (real package + **automatic repodata** generation).
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
across two members.
## Fixtures
`fixtures/` is served by the mock upstream at its web root. Paths mirror each
provider's upstream URL layout (e.g. `v2/...` for docker, `v1/providers/...` for
terraform). The RPM under `fixtures/rpmrepo/Packages/` is a real package so the
rpm provider can parse its metadata for repodata generation.
-76
View File
@@ -1,76 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"bytes"
"fmt"
"net/http"
"testing"
)
// TestCachingPerProvider proxies one immutable artifact for every remote
// package type through the mock upstream and asserts: first fetch is served
// from the remote, the second from cache, and the bytes match the origin.
func TestCachingPerProvider(t *testing.T) {
cases := []struct {
pkgType string
// path is the request path under /api/v1/remote/<name>/. The provider
// derives the upstream URL from it (docker prepends /v2/, terraform
// prepends /v1/providers/), and the fixture lives at that resolved path.
path string
fixture string
}{
{"generic", "blobs/hello.bin", "blobs/hello.bin"},
{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"},
{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"},
{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"},
{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"},
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
}
for _, tc := range cases {
t.Run(tc.pkgType, func(t *testing.T) {
name := "cache-" + tc.pkgType
createRepo(t, fmt.Sprintf(`{
"name": %q,
"package_type": %q,
"repo_type": "remote",
"base_url": %q,
"stale_on_error": true
}`, name, tc.pkgType, mockUpstream()))
defer deleteRepo(t, name)
want := fixtureBytes(t, tc.fixture)
url := api("/api/v1/remote/" + name + "/" + tc.path)
// First fetch: from remote.
resp, body := doRequest(t, http.MethodGet, url, nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("first fetch: status %d: %s", resp.StatusCode, body)
}
if src := resp.Header.Get("X-Artifact-Source"); src != "remote" {
t.Fatalf("first fetch source = %q, want remote", src)
}
if !bytes.Equal(body, want) {
t.Fatalf("first fetch body mismatch: got %d bytes, want %d", len(body), len(want))
}
// Second fetch: from cache, identical bytes.
resp, body = doRequest(t, http.MethodGet, url, nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("second fetch: status %d: %s", resp.StatusCode, body)
}
if src := resp.Header.Get("X-Artifact-Source"); src != "cache" {
t.Fatalf("second fetch source = %q, want cache", src)
}
if !bytes.Equal(body, want) {
t.Fatalf("cached body mismatch: got %d bytes, want %d", len(body), len(want))
}
})
}
}
Binary file not shown.
-1
View File
@@ -1 +0,0 @@
hello artifactapi generic blob
Binary file not shown.
-8
View File
@@ -1,8 +0,0 @@
apiVersion: v1
entries:
alpha:
- name: alpha
version: 1.0.0
urls:
- charts/alpha-1.0.0.tgz
generated: "2026-01-01T00:00:00Z"
-8
View File
@@ -1,8 +0,0 @@
apiVersion: v1
entries:
beta:
- name: beta
version: 2.0.0
urls:
- charts/beta-2.0.0.tgz
generated: "2026-01-01T00:00:00Z"
Binary file not shown.
-108
View File
@@ -1,108 +0,0 @@
//go:build dockere2e
// Package e2edocker holds the black-box end-to-end suite that runs against a
// fully dockerised artifactapi stack (see scripts/docker-e2e.sh). Unlike the
// in-process e2e suite, these tests only speak HTTP to the running container.
package e2edocker
import (
"bytes"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func baseURL() string {
if v := os.Getenv("ARTIFACTAPI_URL"); v != "" {
return strings.TrimRight(v, "/")
}
return "http://localhost:8000"
}
// mockUpstream is the base URL the artifactapi *container* uses to reach the
// static mock upstream. It is resolved on the compose network, not the host.
func mockUpstream() string {
if v := os.Getenv("MOCK_UPSTREAM_INTERNAL"); v != "" {
return strings.TrimRight(v, "/")
}
return "http://mockupstream"
}
func api(path string) string { return baseURL() + path }
func fixtureBytes(t *testing.T, rel string) []byte {
t.Helper()
b, err := os.ReadFile(filepath.Join("fixtures", rel))
if err != nil {
t.Fatalf("read fixture %s: %v", rel, err)
}
return b
}
func doRequest(t *testing.T, method, url string, body []byte, contentType string) (*http.Response, []byte) {
t.Helper()
var r io.Reader
if body != nil {
r = bytes.NewReader(body)
}
req, err := http.NewRequest(method, url, r)
if err != nil {
t.Fatalf("%s %s: %v", method, url, err)
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("%s %s: %v", method, url, err)
}
defer resp.Body.Close()
respBody, _ := io.ReadAll(resp.Body)
return resp, respBody
}
func createRepo(t *testing.T, jsonBody string) {
t.Helper()
resp, body := doRequest(t, http.MethodPost, api("/api/v2/remotes"), []byte(jsonBody), "application/json")
if resp.StatusCode != http.StatusCreated {
t.Fatalf("create repo: status %d: %s", resp.StatusCode, body)
}
}
func deleteRepo(t *testing.T, name string) {
t.Helper()
doRequest(t, http.MethodDelete, api("/api/v2/remotes/"+name), nil, "")
}
func createVirtual(t *testing.T, jsonBody string) {
t.Helper()
resp, body := doRequest(t, http.MethodPost, api("/api/v2/virtuals"), []byte(jsonBody), "application/json")
if resp.StatusCode != http.StatusCreated {
t.Fatalf("create virtual: status %d: %s", resp.StatusCode, body)
}
}
func deleteVirtual(t *testing.T, name string) {
t.Helper()
doRequest(t, http.MethodDelete, api("/api/v2/virtuals/"+name), nil, "")
}
// getEventually retries a GET until it returns 200 or the deadline passes. Used
// for asynchronously-generated artifacts (e.g. rpm repodata after upload).
func getEventually(t *testing.T, url string, timeout time.Duration) (*http.Response, []byte) {
t.Helper()
deadline := time.Now().Add(timeout)
var resp *http.Response
var body []byte
for {
resp, body = doRequest(t, http.MethodGet, url, nil, "")
if resp.StatusCode == http.StatusOK || time.Now().After(deadline) {
return resp, body
}
time.Sleep(250 * time.Millisecond)
}
}
-93
View File
@@ -1,93 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"bytes"
"net/http"
"strings"
"testing"
"time"
)
func uploadFile(t *testing.T, repo, filePath string, body []byte, contentType string) {
t.Helper()
url := api("/api/v2/remotes/" + repo + "/files/" + filePath)
resp, respBody := doRequest(t, http.MethodPut, url, body, contentType)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload %s: status %d: %s", filePath, resp.StatusCode, respBody)
}
}
// TestLocalGenericUpload uploads a generic file and downloads it back.
func TestLocalGenericUpload(t *testing.T) {
createRepo(t, `{"name":"local-generic","package_type":"generic","repo_type":"local"}`)
defer deleteRepo(t, "local-generic")
content := []byte("artifactapi local generic upload payload")
uploadFile(t, "local-generic", "data/hello.bin", content, "application/octet-stream")
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-generic/data/hello.bin"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("download: status %d: %s", resp.StatusCode, body)
}
if !bytes.Equal(body, content) {
t.Fatalf("downloaded content mismatch")
}
}
// TestLocalPyPIUpload uploads a wheel and validates the generated simple index.
func TestLocalPyPIUpload(t *testing.T) {
createRepo(t, `{"name":"local-pypi","package_type":"pypi","repo_type":"local"}`)
defer deleteRepo(t, "local-pypi")
wheel := fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl")
uploadFile(t, "local-pypi", "foo-1.0-py3-none-any.whl", wheel, "application/zip")
// Root index lists the package.
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("simple index: status %d: %s", resp.StatusCode, body)
}
if !strings.Contains(string(body), "foo") {
t.Fatalf("simple index missing package 'foo': %s", body)
}
// Per-package index lists the wheel file.
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/foo/"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("package index: status %d: %s", resp.StatusCode, body)
}
if !strings.Contains(string(body), "foo-1.0-py3-none-any.whl") {
t.Fatalf("package index missing wheel: %s", body)
}
// The wheel downloads back byte-identical.
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/foo/foo-1.0-py3-none-any.whl"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("download wheel: status %d: %s", resp.StatusCode, body)
}
if !bytes.Equal(body, wheel) {
t.Fatalf("wheel content mismatch")
}
}
// TestLocalRPMRepodata uploads a real RPM and validates that repodata is
// generated automatically (the special rpm-local feature).
func TestLocalRPMRepodata(t *testing.T) {
createRepo(t, `{"name":"local-rpm","package_type":"rpm","repo_type":"local"}`)
defer deleteRepo(t, "local-rpm")
rpm := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
uploadFile(t, "local-rpm", "e2e-testpkg-1.0-1.noarch.rpm", rpm, "application/x-rpm")
// repodata is generated asynchronously after upload; poll for it.
resp, body := getEventually(t, api("/api/v1/local/local-rpm/repodata/repomd.xml"), 15*time.Second)
if resp.StatusCode != http.StatusOK {
t.Fatalf("repomd.xml: status %d: %s", resp.StatusCode, body)
}
s := string(body)
if !strings.Contains(s, "<repomd") || !strings.Contains(s, "primary") {
t.Fatalf("repomd.xml not a valid repodata document: %s", s)
}
}
-134
View File
@@ -1,134 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"encoding/json"
"net/http"
"testing"
)
func TestHealth(t *testing.T) {
resp, body := doRequest(t, http.MethodGet, api("/health"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("health: status %d: %s", resp.StatusCode, body)
}
}
// TestRemoteLifecycle covers add/change/delete for a remote repository.
func TestRemoteLifecycle(t *testing.T) {
createRepo(t, `{
"name": "crud-remote",
"package_type": "generic",
"repo_type": "remote",
"base_url": "https://example.com",
"mutable_ttl": 600,
"stale_on_error": true
}`)
defer deleteRepo(t, "crud-remote")
got := getRepo(t, "crud-remote")
if got["base_url"] != "https://example.com" || got["mutable_ttl"].(float64) != 600 {
t.Fatalf("unexpected created remote: %v", got)
}
// change
resp, body := doRequest(t, http.MethodPut, api("/api/v2/remotes/crud-remote"), []byte(`{
"package_type": "generic",
"base_url": "https://updated.example.com",
"mutable_ttl": 120,
"stale_on_error": true
}`), "application/json")
if resp.StatusCode != http.StatusOK {
t.Fatalf("update remote: status %d: %s", resp.StatusCode, body)
}
got = getRepo(t, "crud-remote")
if got["base_url"] != "https://updated.example.com" || got["mutable_ttl"].(float64) != 120 {
t.Fatalf("update not applied: %v", got)
}
// delete
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-remote"), nil, "")
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("delete remote: status %d", resp.StatusCode)
}
resp, _ = doRequest(t, http.MethodGet, api("/api/v2/remotes/crud-remote"), nil, "")
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("expected 404 after delete, got %d", resp.StatusCode)
}
}
// TestLocalLifecycle covers add/delete for a local repository.
func TestLocalLifecycle(t *testing.T) {
createRepo(t, `{
"name": "crud-local",
"package_type": "generic",
"repo_type": "local"
}`)
defer deleteRepo(t, "crud-local")
got := getRepo(t, "crud-local")
if got["repo_type"] != "local" {
t.Fatalf("expected repo_type local, got %v", got["repo_type"])
}
resp, _ := doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-local"), nil, "")
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("delete local: status %d", resp.StatusCode)
}
}
// TestVirtualLifecycle covers add/change/delete for a virtual repository.
func TestVirtualLifecycle(t *testing.T) {
createRepo(t, `{"name":"vmem-a","package_type":"helm","repo_type":"remote","base_url":"https://a.example.com","stale_on_error":true}`)
createRepo(t, `{"name":"vmem-b","package_type":"helm","repo_type":"remote","base_url":"https://b.example.com","stale_on_error":true}`)
defer deleteRepo(t, "vmem-a")
defer deleteRepo(t, "vmem-b")
createVirtual(t, `{
"name": "crud-virtual",
"package_type": "helm",
"members": ["vmem-a"]
}`)
defer deleteVirtual(t, "crud-virtual")
// change members
resp, body := doRequest(t, http.MethodPut, api("/api/v2/virtuals/crud-virtual"), []byte(`{
"package_type": "helm",
"members": ["vmem-a", "vmem-b"]
}`), "application/json")
if resp.StatusCode != http.StatusOK {
t.Fatalf("update virtual: status %d: %s", resp.StatusCode, body)
}
resp, body = doRequest(t, http.MethodGet, api("/api/v2/virtuals/crud-virtual"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("get virtual: status %d: %s", resp.StatusCode, body)
}
var v map[string]any
if err := json.Unmarshal(body, &v); err != nil {
t.Fatalf("decode virtual: %v", err)
}
members, _ := v["members"].([]any)
if len(members) != 2 {
t.Fatalf("expected 2 members after update, got %v", v["members"])
}
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/virtuals/crud-virtual"), nil, "")
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("delete virtual: status %d", resp.StatusCode)
}
}
func getRepo(t *testing.T, name string) map[string]any {
t.Helper()
resp, body := doRequest(t, http.MethodGet, api("/api/v2/remotes/"+name), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("get remote %s: status %d: %s", name, resp.StatusCode, body)
}
var m map[string]any
if err := json.Unmarshal(body, &m); err != nil {
t.Fatalf("decode remote %s: %v", name, err)
}
return m
}
-54
View File
@@ -1,54 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"net/http"
"strings"
"testing"
)
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
// checks that a virtual over them serves a merged simple index.
func TestVirtualPyPIMerge(t *testing.T) {
createRepo(t, `{"name":"pmerge-a","package_type":"pypi","repo_type":"local"}`)
createRepo(t, `{"name":"pmerge-b","package_type":"pypi","repo_type":"local"}`)
defer deleteRepo(t, "pmerge-a")
defer deleteRepo(t, "pmerge-b")
uploadFile(t, "pmerge-a", "foo-1.0-py3-none-any.whl", fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl"), "application/zip")
uploadFile(t, "pmerge-b", "bar-2.0-py3-none-any.whl", []byte("bar wheel payload"), "application/zip")
createVirtual(t, `{"name":"pmerge-v","package_type":"pypi","members":["pmerge-a","pmerge-b"]}`)
defer deleteVirtual(t, "pmerge-v")
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/pmerge-v/simple/"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("virtual simple index: status %d: %s", resp.StatusCode, body)
}
s := string(body)
if !strings.Contains(s, "foo") || !strings.Contains(s, "bar") {
t.Fatalf("merged index missing a member package (want foo and bar): %s", s)
}
}
// TestVirtualHelmMerge points two helm remotes at mock index.yaml documents
// with distinct charts and checks the virtual merges both into one index.
func TestVirtualHelmMerge(t *testing.T) {
createRepo(t, `{"name":"hmerge-a","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-a","stale_on_error":true}`)
createRepo(t, `{"name":"hmerge-b","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-b","stale_on_error":true}`)
defer deleteRepo(t, "hmerge-a")
defer deleteRepo(t, "hmerge-b")
createVirtual(t, `{"name":"hmerge-v","package_type":"helm","members":["hmerge-a","hmerge-b"]}`)
defer deleteVirtual(t, "hmerge-v")
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/hmerge-v/index.yaml"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("virtual index.yaml: status %d: %s", resp.StatusCode, body)
}
s := string(body)
if !strings.Contains(s, "alpha") || !strings.Contains(s, "beta") {
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
}
}
-78
View File
@@ -1,78 +0,0 @@
package v2
import (
"context"
"encoding/json"
"net/http/httptest"
"testing"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// TestLocalObjectsListing verifies that files uploaded to a local repo (which
// live in local_files, not artifacts) are listed by the local objects endpoint
// and can be evicted through it.
func TestLocalObjectsListing(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
}
ctx := context.Background()
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
defer db.Close()
const repo = "rpm-local-objs"
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageRPM, RepoType: models.RepoTypeLocal}); err != nil {
t.Fatal(err)
}
const hash = "sha256:aa11"
const path = "Packages/example-0.1.0-1.x86_64.rpm"
if err := db.UpsertBlob(ctx, hash, "blobs/aa/11", 1234, "application/x-rpm"); err != nil {
t.Fatal(err)
}
if err := db.CreateLocalFile(ctx, repo, path, hash); err != nil {
t.Fatal(err)
}
h := NewObjectsHandler(db)
router := chi.NewRouter()
router.Route("/locals/{name}/objects", func(r chi.Router) {
r.Get("/", h.LocalRoutes().ServeHTTP)
r.Delete("/*", h.LocalRoutes().ServeHTTP)
})
// The uploaded package must appear in the listing with its blob size.
req := httptest.NewRequest("GET", "/locals/"+repo+"/objects", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("list = %d, want 200", w.Code)
}
var got []models.Artifact
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
t.Fatalf("decode: %v", err)
}
if len(got) != 1 {
t.Fatalf("got %d objects, want 1", len(got))
}
if got[0].Path != path || got[0].SizeBytes != 1234 || got[0].ContentHash != hash {
t.Fatalf("unexpected object: %+v", got[0])
}
// Eviction removes it from local_files.
del := httptest.NewRequest("DELETE", "/locals/"+repo+"/objects/"+path, nil)
dw := httptest.NewRecorder()
router.ServeHTTP(dw, del)
if dw.Code != 204 {
t.Fatalf("evict = %d, want 204", dw.Code)
}
if f, _ := db.GetLocalFile(ctx, repo, path); f != nil {
t.Fatalf("file still present after evict: %+v", f)
}
}
+4 -41
View File
@@ -25,18 +25,9 @@ func (h *ObjectsHandler) Routes() chi.Router {
return r
}
// LocalRoutes lists and evicts objects for local repos, which live in the
// local_files table rather than the artifacts table used by remotes.
func (h *ObjectsHandler) LocalRoutes() chi.Router {
r := chi.NewRouter()
r.Get("/", h.listLocal)
r.Delete("/*", h.evictLocal)
return r
}
// pageBounds parses the shared page/per_page query params into a SQL limit and offset.
func pageBounds(r *http.Request) (limit, offset int) {
limit, _ = strconv.Atoi(r.URL.Query().Get("per_page"))
func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "name")
limit, _ := strconv.Atoi(r.URL.Query().Get("per_page"))
if limit <= 0 || limit > 5000 {
limit = 50
}
@@ -44,12 +35,7 @@ func pageBounds(r *http.Request) (limit, offset int) {
if page <= 0 {
page = 1
}
return limit, (page - 1) * limit
}
func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "name")
limit, offset := pageBounds(r)
offset := (page - 1) * limit
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
if err != nil {
@@ -59,29 +45,6 @@ func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, artifacts)
}
func (h *ObjectsHandler) listLocal(w http.ResponseWriter, r *http.Request) {
repoName := chi.URLParam(r, "name")
limit, offset := pageBounds(r)
artifacts, err := h.db.ListLocalArtifacts(r.Context(), repoName, limit, offset)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, artifacts)
}
func (h *ObjectsHandler) evictLocal(w http.ResponseWriter, r *http.Request) {
repoName := chi.URLParam(r, "name")
path := chi.URLParam(r, "*")
if err := h.db.DeleteLocalFile(r.Context(), repoName, path); err != nil {
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "name")
path := chi.URLParam(r, "*")
-35
View File
@@ -10,7 +10,6 @@ import (
"github.com/jackc/pgx/v5/pgconn"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type LocalFile struct {
@@ -79,40 +78,6 @@ func (db *DB) ListLocalFiles(ctx context.Context, repoName string, limit, offset
return files, rows.Err()
}
// ListLocalArtifacts returns a repo's local files shaped as models.Artifact so
// the UI's cached-objects view can render them the same way as remote artifacts.
// Local files carry no access/fetch counters, so those are left at zero and the
// timestamps are all derived from created_at.
func (db *DB) ListLocalArtifacts(ctx context.Context, repoName string, limit, offset int) ([]models.Artifact, error) {
rows, err := db.Pool.Query(ctx, `
SELECT lf.id, lf.repo_name, lf.file_path, lf.content_hash,
lf.created_at, b.size_bytes, b.content_type
FROM local_files lf
JOIN blobs b ON lf.content_hash = b.content_hash
WHERE lf.repo_name = $1
ORDER BY lf.file_path
LIMIT $2 OFFSET $3
`, repoName, limit, offset)
if err != nil {
return nil, err
}
defer rows.Close()
var artifacts []models.Artifact
for rows.Next() {
var a models.Artifact
var createdAt time.Time
if err := rows.Scan(&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &createdAt, &a.SizeBytes, &a.ContentType); err != nil {
return nil, err
}
a.FirstSeenAt = createdAt
a.LastFetchedAt = createdAt
a.LastAccessedAt = createdAt
artifacts = append(artifacts, a)
}
return artifacts, rows.Err()
}
func (db *DB) ListLocalFilesByPrefix(ctx context.Context, repoName, prefix string) ([]LocalFile, error) {
rows, err := db.Pool.Query(ctx, `
SELECT id, repo_name, file_path, content_hash, created_at
+7 -2
View File
@@ -7,14 +7,16 @@ import (
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
const fixtureRPM = "testdata/e2e-testpkg-1.0-1.noarch.rpm"
type fakeBlobReader struct{ data []byte }
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
@@ -80,7 +82,10 @@ func TestRPMValidateUpload(t *testing.T) {
}
func TestRPMAfterUpload(t *testing.T) {
data := testsupport.MinimalRPM("e2e-testpkg", "1.0", "1", "noarch")
data, err := os.ReadFile(fixtureRPM)
if err != nil {
t.Fatalf("fixture: %v", err)
}
store := &fakeMetaStore{}
(&Provider{}).AfterUpload(context.Background(), "myrepo", "Packages/e2e-testpkg-1.0-1.noarch.rpm",
"sha256:deadbeef", fakeBlobReader{data: data}, store)
-6
View File
@@ -121,12 +121,6 @@ func (s *Server) routes() chi.Router {
r.Delete("/*", objHandler.Routes().ServeHTTP)
})
r.Route("/locals/{name}/objects", func(r chi.Router) {
objHandler := v2.NewObjectsHandler(s.db)
r.Get("/", objHandler.LocalRoutes().ServeHTTP)
r.Delete("/*", objHandler.LocalRoutes().ServeHTTP)
})
r.Route("/remotes/{name}/files", func(r chi.Router) {
r.Put("/*", s.localHandler.Routes().ServeHTTP)
r.Get("/*", s.localHandler.Routes().ServeHTTP)
+4 -1
View File
@@ -301,7 +301,10 @@ func TestServerLocalPyPI(t *testing.T) {
func TestServerLocalRPMRepodata(t *testing.T) {
requireStack(t)
rpm := testsupport.MinimalRPM("e2e-testpkg", "1.0", "1", "noarch")
rpm, err := os.ReadFile("../provider/rpm/testdata/e2e-testpkg-1.0-1.noarch.rpm")
if err != nil {
t.Skipf("rpm fixture unavailable: %v", err)
}
if resp, b := req(t, "POST", "/api/v2/remotes", `{"name":"srv-rpm","package_type":"rpm","repo_type":"local"}`); resp.StatusCode != 201 {
t.Fatalf("create rpm local: %d %s", resp.StatusCode, b)
}
-59
View File
@@ -1,59 +0,0 @@
package testsupport
import (
"bytes"
"encoding/binary"
)
// MinimalRPM builds a valid-enough RPM package in pure Go (no committed binary
// fixture, no external rpmbuild). It carries just the header tags the provider
// reads: name/version/release/arch plus a single self Provides entry, which is
// enough for cavaliergopher/rpm to parse and for repodata generation.
func MinimalRPM(name, version, release, arch string) []byte {
type tag struct {
id, typ, count uint32
data []byte
}
cstr := func(s string) []byte { return append([]byte(s), 0) }
tags := []tag{
{1000, 6, 1, cstr(name)}, // RPMTAG_NAME (STRING)
{1001, 6, 1, cstr(version)}, // RPMTAG_VERSION
{1002, 6, 1, cstr(release)}, // RPMTAG_RELEASE
{1022, 6, 1, cstr(arch)}, // RPMTAG_ARCH
{1047, 8, 1, cstr(name)}, // RPMTAG_PROVIDENAME (STRING_ARRAY)
{1112, 4, 1, []byte{0, 0, 0, 0}}, // RPMTAG_PROVIDEFLAGS (INT32)
{1113, 8, 1, cstr(version)}, // RPMTAG_PROVIDEVERSION (STRING_ARRAY)
}
buildHeader := func(entries []tag) []byte {
var index, store bytes.Buffer
for _, e := range entries {
off := uint32(store.Len())
for _, v := range []uint32{e.id, e.typ, off, e.count} {
binary.Write(&index, binary.BigEndian, v)
}
store.Write(e.data)
}
var b bytes.Buffer
b.Write([]byte{0x8e, 0xad, 0xe8, 0x01, 0, 0, 0, 0}) // header magic + reserved
binary.Write(&b, binary.BigEndian, uint32(len(entries)))
binary.Write(&b, binary.BigEndian, uint32(store.Len()))
b.Write(index.Bytes())
b.Write(store.Bytes())
return b.Bytes()
}
lead := make([]byte, 96)
copy(lead[0:4], []byte{0xed, 0xab, 0xee, 0xdb}) // lead magic
lead[4] = 3 // major version
binary.BigEndian.PutUint16(lead[8:10], 1) // archnum
copy(lead[10:76], name) // name (66 bytes, null-padded)
binary.BigEndian.PutUint16(lead[76:78], 1) // osnum
binary.BigEndian.PutUint16(lead[78:80], 5) // signature type
var out bytes.Buffer
out.Write(lead)
out.Write(buildHeader(nil)) // empty signature header (16 bytes, 8-aligned)
out.Write(buildHeader(tags))
return out.Bytes()
}
-40
View File
@@ -1,40 +0,0 @@
#!/usr/bin/env bash
# Build the artifactapi container, bring up the full stack (postgres, redis,
# minio, artifactapi) plus a static mock upstream, and run the dockerised e2e
# suite against the running product over HTTP. Tears everything down on exit.
set -euo pipefail
cd "$(dirname "$0")/.."
# Publish artifactapi on 8001 to avoid colliding with a local instance on 8000.
export ARTIFACTAPI_PORT="${ARTIFACTAPI_PORT:-8001}"
COMPOSE=(docker compose -f docker-compose.yml -f docker-compose.e2e.yml)
API_URL="${ARTIFACTAPI_URL:-http://localhost:${ARTIFACTAPI_PORT}}"
cleanup() {
echo "==> tearing down stack"
"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
}
trap cleanup EXIT
echo "==> building and starting stack (postgres, redis, minio, mockupstream, artifactapi)"
"${COMPOSE[@]}" up -d --build postgres redis minio mockupstream artifactapi
echo "==> waiting for artifactapi health at ${API_URL}"
for i in $(seq 1 60); do
if curl -fsS "${API_URL}/health" >/dev/null 2>&1; then
echo " healthy after ${i}s"
break
fi
if [ "$i" -eq 60 ]; then
echo "!!! artifactapi did not become healthy in time; recent logs:"
"${COMPOSE[@]}" logs --tail=50 artifactapi
exit 1
fi
sleep 1
done
echo "==> running dockerised e2e suite"
ARTIFACTAPI_URL="${API_URL}" \
MOCK_UPSTREAM_INTERNAL="${MOCK_UPSTREAM_INTERNAL:-http://mockupstream}" \
go test -tags=dockere2e -count=1 -timeout=10m -v ./e2e-docker/...
-6
View File
@@ -34,12 +34,6 @@ export const api = {
evictObject: (remote: string, path: string) =>
fetchJSON<void>(`/api/v2/remotes/${remote}/objects/${path}`, { method: 'DELETE' }),
listLocalObjects: (name: string, page = 1, perPage = 50) =>
fetchJSON<Artifact[]>(`/api/v2/locals/${name}/objects?page=${page}&per_page=${perPage}`),
evictLocalObject: (name: string, path: string) =>
fetchJSON<void>(`/api/v2/locals/${name}/objects/${path}`, { method: 'DELETE' }),
flushRemoteCache: (remote: string) =>
fetchJSON<void>(`/api/v2/remotes/${remote}/cache`, { method: 'DELETE' }),
+3 -4
View File
@@ -182,17 +182,16 @@ export function Objects() {
const load = useCallback(() => {
if (!name) return;
setLoading(true);
const req = isLocal ? api.listLocalObjects(name, 1, 5000) : api.listObjects(name, 1, 5000);
req
api.listObjects(name, 1, 5000)
.then(a => setArtifacts(a || []))
.finally(() => setLoading(false));
}, [name, isLocal]);
}, [name]);
useEffect(() => { load(); }, [load]);
const handleEvict = async (path: string) => {
if (!name || !confirm(`Evict ${path}?`)) return;
await (isLocal ? api.evictLocalObject(name, path) : api.evictObject(name, path));
await api.evictObject(name, path);
load();
};