Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 unkinben
|
67cedf9bba |
+1
-5
@@ -1,6 +1,2 @@
|
||||
bin/
|
||||
/terraform/
|
||||
|
||||
# e2e-docker fixtures are real package files (.rpm, .tgz, .whl, .zip, ...) that
|
||||
# are intentionally tracked, overriding any global ignore of those extensions.
|
||||
!e2e-docker/fixtures/**
|
||||
terraform/
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
.PHONY: build test lint fmt e2e docker-e2e docker docker-ui compose clean tidy check-go
|
||||
.PHONY: build test lint fmt e2e docker docker-ui compose clean tidy check-go
|
||||
|
||||
BINARY := bin/artifactapi
|
||||
MODULE := git.unkin.net/unkin/artifactapi
|
||||
@@ -28,11 +28,6 @@ fmt: check-go
|
||||
e2e: check-go
|
||||
TESTCONTAINERS_RYUK_DISABLED=true go test -tags=e2e -race -count=1 -timeout=5m ./e2e/...
|
||||
|
||||
# Build the container, bring up the full docker-compose stack + a mock upstream,
|
||||
# and run the black-box suite against the running product.
|
||||
docker-e2e: check-go
|
||||
./scripts/docker-e2e.sh
|
||||
|
||||
docker:
|
||||
docker build -t artifactapi:$(VERSION) .
|
||||
|
||||
|
||||
@@ -32,150 +32,9 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
|
||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
|
||||
|
||||
Providers classify paths automatically. Users only configure what to proxy and TTLs.
|
||||
|
||||
### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache)
|
||||
|
||||
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
|
||||
repository without ever caching the packages. It scans releases for `.rpm`
|
||||
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
|
||||
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
|
||||
metadata comes from a **ranged GET of just the RPM header** (the header sits at
|
||||
the front of the file, so the whole package is never downloaded); the sha256
|
||||
checksum comes from the GitHub asset `digest` when present, else a one-time
|
||||
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
|
||||
is served from primed DB rows, never a cold on-demand derive.
|
||||
|
||||
Each package's `<location>` points back at the remote, which **302-redirects**
|
||||
the download to the `releases_remote` — an existing generic `github.com` remote
|
||||
that streams the actual bytes. `dnf` follows the redirect transparently.
|
||||
|
||||
#### Background syncer
|
||||
|
||||
A single process-wide **background syncer** keeps every `github_rpm` remote's
|
||||
derived metadata current off the client request path:
|
||||
|
||||
- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime
|
||||
scan, so its metadata is derived right away without blocking the create call.
|
||||
The first `dnf` request is served from cache. If a request arrives before the
|
||||
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
|
||||
serving an empty repo or blocking on a multi-minute derive.
|
||||
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
|
||||
new or changed releases no more often than its `mutable_ttl`. New/changed
|
||||
assets are derived incrementally; assets already cached are never re-fetched,
|
||||
and assets that disappear upstream are pruned.
|
||||
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
|
||||
remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
|
||||
and the syncer derives nothing. GitHub does not count `304` conditional
|
||||
responses against the rate limit, so an unchanged repo is nearly free — this is
|
||||
the main lever keeping GitHub traffic low.
|
||||
- **Global rate limit.** Every GitHub call (releases list + each ranged asset
|
||||
header GET) passes through a single token-bucket limiter **shared across all
|
||||
remotes**, so GitHub is never hammered. Configure a token (`password`) on the
|
||||
remote for the higher authenticated rate limit (~5000/hr vs ~60/hr
|
||||
unauthenticated).
|
||||
- **Multi-replica coordination.** State is shared through the database. Before a
|
||||
periodic scan a replica must atomically claim a per-remote lease
|
||||
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
|
||||
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
|
||||
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
|
||||
lets any replica issue the conditional request.
|
||||
|
||||
```hcl
|
||||
# Backend that serves the actual .rpm bytes from github.com.
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
patterns = [
|
||||
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_github_rpm" "acme-tools" {
|
||||
name = "acme-tools"
|
||||
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
|
||||
releases_remote = "github" # backend for downloads
|
||||
mutable_ttl = 3600 # release re-scan interval
|
||||
|
||||
# Optional: restrict which release assets become packages (regex on filename).
|
||||
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
|
||||
|
||||
# Optional: a token for private repos / higher API rate limits.
|
||||
# password = "ghp_..."
|
||||
}
|
||||
```
|
||||
|
||||
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
|
||||
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
|
||||
from the synthesized metadata.
|
||||
|
||||
### GitHub authentication
|
||||
|
||||
Anonymous GitHub is capped at **60 requests/hour** and cannot read private
|
||||
repositories. Configure a **server-level GitHub credential** to raise the ceiling
|
||||
to roughly **5000 requests/hour** and to read private-repo release assets. The
|
||||
credential is a process-wide machine identity applied by default to *every*
|
||||
outbound GitHub request — the releases scan, the ranged asset-header fetches, and
|
||||
the generic-github byte proxy that streams private release assets.
|
||||
|
||||
The credential is read from the environment (deliver it from a Vault or
|
||||
Kubernetes secret). It is **never** stored per-remote in the database, **never**
|
||||
returned by any API, and **never** logged. Configure **exactly one** mode.
|
||||
|
||||
**Precedence.** A remote's own `username`/`password` credential still wins for
|
||||
that remote's requests; the server credential is the default for everything else.
|
||||
With no credential configured at all, requests stay anonymous (current behavior).
|
||||
Partial configuration (e.g. an App id with no private key) is a **startup error**
|
||||
— artifactapi fails closed rather than silently falling back to anonymous.
|
||||
|
||||
Both modes share the syncer's single global rate limiter, so a token simply
|
||||
raises the effective GitHub ceiling; the default limiter settings stay safe.
|
||||
|
||||
#### Mode 1 — Personal Access Token (minimum viable, recommended for free accounts)
|
||||
|
||||
Set `GITHUB_TOKEN`. It is sent as `Authorization: Bearer <token>`.
|
||||
|
||||
Recommended free-account setup — a **fine-grained PAT** scoped to just the target
|
||||
repositories:
|
||||
|
||||
1. GitHub → *Settings → Developer settings → Personal access tokens →
|
||||
Fine-grained tokens → Generate new token*.
|
||||
2. Limit *Repository access* to the specific repo(s) serving releases.
|
||||
3. Grant repository permissions **Contents: Read-only** and **Metadata:
|
||||
Read-only** (Metadata is mandatory and auto-selected).
|
||||
|
||||
A classic PAT with the `repo` scope also works but is broader than necessary.
|
||||
|
||||
```bash
|
||||
GITHUB_TOKEN=github_pat_xxxxxxxx
|
||||
```
|
||||
|
||||
#### Mode 2 — GitHub App installation token (proper machine identity)
|
||||
|
||||
A GitHub App is not tied to a personal account and can be created and installed on
|
||||
free personal repos. artifactapi mints a short-lived RS256 **JWT** from the app
|
||||
private key, exchanges it at `POST /app/installations/{id}/access_tokens` for a
|
||||
~1-hour **installation access token**, caches that token, and refreshes it a few
|
||||
minutes before expiry (thread-safe, single-flighted).
|
||||
|
||||
1. GitHub → *Settings → Developer settings → GitHub Apps → New GitHub App*.
|
||||
2. Under *Permissions → Repository permissions* grant **Contents: Read-only**
|
||||
(Metadata: Read-only is implied).
|
||||
3. Generate a **private key** (downloads a PEM) and note the **App ID**.
|
||||
4. *Install* the App on the account and select the target repositories, then read
|
||||
the **Installation ID** from the installation URL
|
||||
(`.../settings/installations/<installation-id>`).
|
||||
|
||||
```bash
|
||||
GITHUB_APP_ID=123456
|
||||
GITHUB_APP_INSTALLATION_ID=7654321
|
||||
GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem
|
||||
# or inline PEM (e.g. mounted from a secret):
|
||||
# GITHUB_APP_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----"
|
||||
```
|
||||
|
||||
## Terraform
|
||||
|
||||
Remotes and virtuals are managed by Terraform. Each package type has its own resource:
|
||||
@@ -230,54 +89,6 @@ resource "artifactapi_virtual" "helm" {
|
||||
|
||||
Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi)
|
||||
|
||||
### Serving providers as a registry
|
||||
|
||||
A local `terraform` repo is a real provider registry: upload
|
||||
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
|
||||
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
|
||||
no `.terraformrc` mirror config:
|
||||
|
||||
```hcl
|
||||
terraform {
|
||||
required_providers {
|
||||
artifactapi = {
|
||||
source = "artifactapi.k8s.syd1.au.unkin.net/<repo>/<type>"
|
||||
version = "0.1.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The Terraform *namespace* segment is the artifactapi repo name; the provider is
|
||||
matched by *type*. The registry serves service discovery
|
||||
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
|
||||
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
|
||||
|
||||
Signing needs a GPG key. By default artifactapi generates one on first start and
|
||||
stores it in the database (`signing_keys` table), so every replica shares it and
|
||||
there's nothing to provision. To bring your own key instead, point
|
||||
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
|
||||
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
|
||||
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
|
||||
|
||||
### Local docker registry
|
||||
|
||||
A local `docker` repo is a real container registry, not a mirror: it serves the
|
||||
Docker Registry HTTP API V2 for both push and pull, so any client (`docker`,
|
||||
`podman`, `skopeo`, `buildah`) can use it directly.
|
||||
|
||||
```sh
|
||||
docker tag myapp:latest artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
|
||||
docker push artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
|
||||
docker pull artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
|
||||
```
|
||||
|
||||
The first path segment after `/v2/` is the artifactapi repo name; the remainder
|
||||
is the image name. Blobs and manifests are stored through the shared
|
||||
content-addressable store (deduplicated by digest, reaped by GC once
|
||||
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
|
||||
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
|
||||
|
||||
## Access Control
|
||||
|
||||
| Field | Default | Behaviour |
|
||||
@@ -338,15 +149,6 @@ S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go).
|
||||
| `MINIO_BUCKET` | `artifacts` | S3 bucket |
|
||||
| `MINIO_SECURE` | `false` | Use HTTPS for S3 |
|
||||
| `MINIO_REGION` | | S3 region (AWS) |
|
||||
| `GITHUB_SYNC_RATE` | `1` | `github_rpm` syncer global GitHub request rate (req/s), shared across all remotes. `1`/s = 3600/hr, under an authenticated token's ~5000/hr; unauthenticated (~60/hr) relies on ETag/304 |
|
||||
| `GITHUB_SYNC_BURST` | `5` | Token-bucket burst for the shared limiter |
|
||||
| `GITHUB_SYNC_WORKERS` | `3` | Concurrent `github_rpm` scan workers |
|
||||
| `GITHUB_SYNC_POLL_INTERVAL` | `60` | Base scheduler tick in seconds; per-remote cadence is its `mutable_ttl`, enforced by the DB lease |
|
||||
| `GITHUB_TOKEN` | | Server-level GitHub PAT (fine-grained or classic), sent as `Authorization: Bearer`. Applies to every GitHub request; per-remote creds override it. See [GitHub authentication](#github-authentication) |
|
||||
| `GITHUB_APP_ID` | | GitHub App id (App auth mode; mutually exclusive with `GITHUB_TOKEN`) |
|
||||
| `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id |
|
||||
| `GITHUB_APP_PRIVATE_KEY` | | GitHub App private key, inline PEM |
|
||||
| `GITHUB_APP_PRIVATE_KEY_PATH` | | GitHub App private key, file path (alternative to inline PEM) |
|
||||
|
||||
## Development
|
||||
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
# Overlay for the dockerised end-to-end suite (scripts/docker-e2e.sh).
|
||||
# Adds a static mock upstream that the artifactapi container proxies, so the
|
||||
# caching tests are hermetic and need no internet access.
|
||||
services:
|
||||
mockupstream:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./e2e-docker/fixtures:/usr/share/nginx/html:ro,z
|
||||
# No host port needed: only the artifactapi container talks to it, and the
|
||||
# tests compare served bytes against the on-disk fixtures.
|
||||
|
||||
# Two constant-body upstreams for the multi-base_url suite: each returns a
|
||||
# distinct, upstream-identifying body for any path, so round-robin
|
||||
# distribution across a two-mirror remote is directly observable.
|
||||
mockupstreama:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./e2e-docker/mirror-conf/a.conf:/etc/nginx/conf.d/default.conf:ro,z
|
||||
mockupstreamb:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./e2e-docker/mirror-conf/b.conf:/etc/nginx/conf.d/default.conf:ro,z
|
||||
|
||||
artifactapi:
|
||||
# The host port is set via ARTIFACTAPI_PORT (see scripts/docker-e2e.sh),
|
||||
# defaulting to 8000; the e2e run uses 8001 to avoid colliding with a
|
||||
# locally-running instance.
|
||||
depends_on:
|
||||
mockupstream:
|
||||
condition: service_started
|
||||
mockupstreama:
|
||||
condition: service_started
|
||||
mockupstreamb:
|
||||
condition: service_started
|
||||
+1
-1
@@ -2,7 +2,7 @@ services:
|
||||
artifactapi:
|
||||
build: .
|
||||
ports:
|
||||
- "${ARTIFACTAPI_PORT:-8000}:8000"
|
||||
- "8000:8000"
|
||||
environment:
|
||||
LISTEN_ADDR: ":8000"
|
||||
DBHOST: postgres
|
||||
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
# Authentication & Authorization — Design
|
||||
|
||||
Status: **proposed** (tracking issue #79)
|
||||
|
||||
Today ArtifactAPI has no authentication: every proxy and management request is
|
||||
served unconditionally. This document describes an auth/authz system that adds
|
||||
identity and path-scoped authorization **without changing behaviour until an
|
||||
operator turns enforcement on** — the default policy is fully open.
|
||||
|
||||
## Goals
|
||||
|
||||
- Identify callers as one of two principal kinds: **service accounts** and **users**.
|
||||
- Authorize each request against a **path + capability** ACL model.
|
||||
- Let **Vault/OpenBao** mint short-lived tokens so the Terraform provider can get
|
||||
just-in-time credentials to make config changes.
|
||||
- Ship **default-open**: an unconfigured deployment behaves exactly as today.
|
||||
|
||||
## Non-goals (initial phase)
|
||||
|
||||
- Per-object encryption, signing, or content trust.
|
||||
- Rate limiting / quotas (separate concern).
|
||||
- Multi-tenancy beyond what path ACLs express.
|
||||
|
||||
## Principals
|
||||
|
||||
| Kind | Authenticates with | Created by | Lifetime |
|
||||
|---|---|---|---|
|
||||
| Service account — static token | `Authorization: Bearer <token>` | admin via management API | until revoked |
|
||||
| Service account — dynamic token | `Authorization: Bearer <token>` | Vault secrets engine → mint endpoint | lease TTL (auto-revoked) |
|
||||
| User | UI session cookie (OIDC/LDAP login) | external IdP, first-seen on login | session TTL |
|
||||
|
||||
A **service account** is a named identity holding a set of ACL grants. It may
|
||||
have any number of associated tokens (static, or dynamic ones minted by Vault).
|
||||
A **user** is an identity resolved from an external IdP; group membership from
|
||||
the IdP maps to ACL grants.
|
||||
|
||||
## Tokens
|
||||
|
||||
- Format: `aapi_<base62(32 random bytes)>`. The `aapi_` prefix makes tokens
|
||||
greppable and lets us reject obviously-malformed values cheaply.
|
||||
- Storage: only the **SHA-256 of the token** is stored, never the plaintext.
|
||||
Lookup hashes the presented token and matches by hash.
|
||||
- Each token row carries: id, principal (service account) ref, sha256, optional
|
||||
label, `expires_at` (null = non-expiring), `created_at`, `last_used_at`.
|
||||
- Revocation: delete the row (static) or Vault lease revoke → mint endpoint
|
||||
revoke (dynamic).
|
||||
|
||||
## ACL model
|
||||
|
||||
A grant is `(path_pattern, capability)`. A principal is allowed an action iff at
|
||||
least one of its grants matches the request's resource path and capability.
|
||||
|
||||
### Resource paths
|
||||
|
||||
```
|
||||
remote/<remote-name>/<path-in-remote> # proxy + local repo objects
|
||||
virtual/<virtual-name>/<path> # virtual repo reads
|
||||
admin/remotes/<remote-name> # manage a remote definition
|
||||
admin/virtuals/<virtual-name> # manage a virtual definition
|
||||
admin/principals/<name> # manage service accounts / tokens
|
||||
```
|
||||
|
||||
Patterns support a trailing `*` wildcard and `<segment>/*` prefixes, e.g.
|
||||
`remote/dockerhub/*`, `remote/*`, `admin/*`. Matching is longest-prefix by
|
||||
segment; an exact match always wins over a wildcard.
|
||||
|
||||
### Capabilities
|
||||
|
||||
| Capability | Meaning for `remote/...` | Meaning for `admin/...` |
|
||||
|---|---|---|
|
||||
| `read` | GET/HEAD an artifact | GET a definition |
|
||||
| `create` | first upload of a new local file | create a new definition |
|
||||
| `write` | overwrite / re-publish | update an existing definition |
|
||||
| `delete` | remove an object | delete a definition |
|
||||
|
||||
The HTTP layer maps each route to `(resource path, capability)`:
|
||||
|
||||
| Route | Resource | Capability |
|
||||
|---|---|---|
|
||||
| `GET /api/v1/remote/{r}/*`, `/v2/{r}/*` | `remote/{r}/{path}` | `read` |
|
||||
| `GET /api/v1/virtual/{v}/*` | `virtual/{v}/{path}` | `read` |
|
||||
| `PUT /api/v2/remotes/{r}/files/*` (new file) | `remote/{r}/{path}` | `create` |
|
||||
| `PUT ...` (existing file) | `remote/{r}/{path}` | `write` |
|
||||
| `DELETE /api/v2/remotes/{r}/files/*` | `remote/{r}/{path}` | `delete` |
|
||||
| `POST /api/v2/remotes` | `admin/remotes/{name}` | `create` |
|
||||
| `PUT /api/v2/remotes/{r}` | `admin/remotes/{r}` | `write` |
|
||||
| `DELETE /api/v2/remotes/{r}` | `admin/remotes/{r}` | `delete` |
|
||||
|
||||
## Enforcement middleware & default-open
|
||||
|
||||
A single middleware runs before the proxy/management handlers:
|
||||
|
||||
1. Resolve the principal from the request (bearer token → service account, or
|
||||
session cookie → user). No credential → the **anonymous** principal.
|
||||
2. Compute `(resource, capability)` for the route.
|
||||
3. If **enforcement is disabled** (default), allow. Otherwise, evaluate the
|
||||
principal's grants (including the anonymous principal's grants) and allow iff
|
||||
a grant matches; else 401 (no/invalid credential) or 403 (authenticated but
|
||||
unauthorized).
|
||||
|
||||
Enforcement is controlled by a single setting, `AUTH_ENFORCE` (default `false`).
|
||||
While `false`, the middleware still *resolves* the principal (so `last_used_at`
|
||||
and audit logging work) but never denies — making rollout observable before it
|
||||
is enforced. The **anonymous** principal is seeded with `*` → all capabilities,
|
||||
so even flipping `AUTH_ENFORCE=true` with no other config keeps the deployment
|
||||
open until an admin tightens the anonymous grants.
|
||||
|
||||
## Vault integration
|
||||
|
||||
### Mint endpoint (artifactapi side)
|
||||
|
||||
`POST /api/v2/auth/tokens:mint` — restricted to callers Vault trusts. It creates
|
||||
a dynamic token bound to a named service account with a caller-supplied TTL, and
|
||||
returns the plaintext once. `DELETE /api/v2/auth/tokens/{id}` revokes it.
|
||||
|
||||
Trust between Vault and artifactapi: a dedicated **bootstrap service account**
|
||||
whose static token is stored in Vault's engine `config`. The mint endpoint
|
||||
requires `admin/principals/*: write`. (mTLS is a future hardening option.)
|
||||
|
||||
### `vault-plugin-secrets-artifactapi` (new repo)
|
||||
|
||||
Mirrors [`vault-plugin-secrets-litellm`](https://git.unkin.net/unkin/vault-plugin-secrets-litellm):
|
||||
HashiCorp `vault/sdk`, OpenBao-compatible single binary. Paths:
|
||||
|
||||
- `config` — artifactapi base URL + bootstrap token.
|
||||
- `roles/<name>` — target service account + default/max TTL.
|
||||
- `creds/<name>` — mint a dynamic token (calls the mint endpoint); the Vault
|
||||
lease's revoke calls the revoke endpoint.
|
||||
|
||||
E2e (`make e2e`) spins Postgres + MinIO + Redis + artifactapi + Vault + OpenBao
|
||||
in Docker and exercises the full lease lifecycle against both engines. On the
|
||||
Fedora host all bind mounts need `:z` (SELinux).
|
||||
|
||||
## User login (OIDC/LDAP) & UI
|
||||
|
||||
- `GET /api/v2/auth/login` starts an OIDC auth-code flow (or LDAP bind form);
|
||||
`GET /api/v2/auth/callback` establishes a signed session cookie.
|
||||
- IdP groups map to service-account-style grants via configurable group→grant
|
||||
rules. Existing infra: `terraform-authentik`, `terraform-ldap`.
|
||||
- The React UI gains a login state and sends the session cookie; management
|
||||
screens hide actions the principal lacks.
|
||||
|
||||
## Terraform provider
|
||||
|
||||
`terraform-provider-artifactapi` gains a `token` attribute (and
|
||||
`ARTIFACTAPI_TOKEN` env var) sent as `Authorization: Bearer`. In CI the token is
|
||||
sourced from the Vault engine above, so config changes use short-lived creds.
|
||||
|
||||
## Data model (new tables, additive migration)
|
||||
|
||||
```sql
|
||||
service_accounts(name PK, description, disabled, created_at)
|
||||
auth_tokens(id PK, principal TEXT REFERENCES service_accounts(name) ON DELETE CASCADE,
|
||||
token_sha256 TEXT UNIQUE, label, expires_at, created_at, last_used_at)
|
||||
acl_grants(id PK, principal TEXT, path_pattern TEXT, capability TEXT,
|
||||
UNIQUE(principal, path_pattern, capability))
|
||||
-- principal = a service account name, the reserved 'anonymous', or 'user:<sub>'
|
||||
```
|
||||
|
||||
All tables are created with `CREATE TABLE IF NOT EXISTS` alongside the existing
|
||||
inline migrations; adding them changes no current behaviour.
|
||||
|
||||
## Rollout / phased delivery
|
||||
|
||||
Each phase is a separate PR; the system stays open until phase 6 is deliberately
|
||||
enabled.
|
||||
|
||||
1. **Data model + resolution** — tables, token hashing, principal resolution
|
||||
middleware in **observe-only** mode (never denies). Seed anonymous `*`.
|
||||
2. **ACL evaluation** — grant matching + `(resource, capability)` route mapping,
|
||||
still gated by `AUTH_ENFORCE=false`.
|
||||
3. **Management API** — CRUD for service accounts, tokens, grants.
|
||||
4. **Vault mint/revoke endpoints** + bootstrap trust.
|
||||
5. **`vault-plugin-secrets-artifactapi`** (new repo) + `terraform-vault` role,
|
||||
policies; `argocd-apps` deploy.
|
||||
6. **OIDC/LDAP user login + UI**, Terraform provider `token`, and the switch to
|
||||
enable enforcement in an environment.
|
||||
|
||||
## Cross-repo dependencies
|
||||
|
||||
- `terraform-vault` — mount the secrets engine, define `roles/*`, ACL policies,
|
||||
and the K8s auth role the Terraform CI uses.
|
||||
- `argocd-apps` — deploy the plugin sidecar/init and any ServiceAccount.
|
||||
- `terraform-provider-artifactapi` — `token` attribute.
|
||||
- `terraform-authentik` / `terraform-ldap` — IdP client + group mappings.
|
||||
@@ -1,51 +0,0 @@
|
||||
# Dockerised end-to-end suite
|
||||
|
||||
Black-box tests that run against a fully **containerised** artifactapi stack
|
||||
(built image + Postgres + Redis + MinIO) plus a static mock upstream. Unlike the
|
||||
in-process `e2e/` suite (testcontainers, server run in-process), these only speak
|
||||
HTTP to the running product, so they exercise the shipped container image.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
make docker-e2e # build image, compose up, run suite, compose down
|
||||
```
|
||||
|
||||
`scripts/docker-e2e.sh` builds and starts `docker-compose.yml` +
|
||||
`docker-compose.e2e.yml`, waits for `/health`, then runs
|
||||
`go test -tags=dockere2e ./e2e-docker/...` and tears everything down.
|
||||
|
||||
The stack publishes artifactapi on host port **8001** (to avoid colliding with a
|
||||
local instance on 8000). Override with `ARTIFACTAPI_URL` to point the tests at an
|
||||
already-running stack.
|
||||
|
||||
## Coverage
|
||||
|
||||
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
|
||||
- **Caching** — one immutable artifact per remote package type (generic, docker,
|
||||
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
|
||||
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
||||
verified against the origin fixture.
|
||||
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
|
||||
index), rpm (real package + **automatic repodata** generation).
|
||||
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
|
||||
across two members.
|
||||
- **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors
|
||||
(pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors
|
||||
(constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary,
|
||||
no-mirrorlist regression, and a real `dnf` (stock `rockylinux:9` container)
|
||||
`makecache` + `install` through a two-mirror rpm remote whose `base_url` is dead
|
||||
— a dead mirror must not break the client.
|
||||
- **Mirror strategy (`least_conn`)** — a `mirror_strategy: least_conn` rpm remote
|
||||
over the two constant-body mirrors exercises the least-connections selection
|
||||
path end-to-end (both mirrors serve, all requests succeed), plus a real `dnf`
|
||||
install through a `least_conn` remote with a dead primary (failover unchanged).
|
||||
The precise least-loaded pick is asserted deterministically in the proxy unit
|
||||
test, since an in-flight-skew assertion over HTTP is timing-sensitive.
|
||||
|
||||
## Fixtures
|
||||
|
||||
`fixtures/` is served by the mock upstream at its web root. Paths mirror each
|
||||
provider's upstream URL layout (e.g. `v2/...` for docker, `v1/providers/...` for
|
||||
terraform). The RPM under `fixtures/rpmrepo/Packages/` is a real package so the
|
||||
rpm provider can parse its metadata for repodata generation.
|
||||
@@ -1,76 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestCachingPerProvider proxies one immutable artifact for every remote
|
||||
// package type through the mock upstream and asserts: first fetch is served
|
||||
// from the remote, the second from cache, and the bytes match the origin.
|
||||
func TestCachingPerProvider(t *testing.T) {
|
||||
cases := []struct {
|
||||
pkgType string
|
||||
// path is the request path under /api/v1/remote/<name>/. The provider
|
||||
// derives the upstream URL from it (docker prepends /v2/, terraform
|
||||
// prepends /v1/providers/), and the fixture lives at that resolved path.
|
||||
path string
|
||||
fixture string
|
||||
}{
|
||||
{"generic", "blobs/hello.bin", "blobs/hello.bin"},
|
||||
{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"},
|
||||
{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"},
|
||||
{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"},
|
||||
{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"},
|
||||
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
|
||||
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
|
||||
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
|
||||
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
|
||||
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.pkgType, func(t *testing.T) {
|
||||
name := "cache-" + tc.pkgType
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": %q,
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"stale_on_error": true
|
||||
}`, name, tc.pkgType, mockUpstream()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
want := fixtureBytes(t, tc.fixture)
|
||||
url := api("/api/v1/remote/" + name + "/" + tc.path)
|
||||
|
||||
// First fetch: from remote.
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("first fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if src := resp.Header.Get("X-Artifact-Source"); src != "remote" {
|
||||
t.Fatalf("first fetch source = %q, want remote", src)
|
||||
}
|
||||
if !bytes.Equal(body, want) {
|
||||
t.Fatalf("first fetch body mismatch: got %d bytes, want %d", len(body), len(want))
|
||||
}
|
||||
|
||||
// Second fetch: from cache, identical bytes.
|
||||
resp, body = doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("second fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if src := resp.Header.Get("X-Artifact-Source"); src != "cache" {
|
||||
t.Fatalf("second fetch source = %q, want cache", src)
|
||||
}
|
||||
if !bytes.Equal(body, want) {
|
||||
t.Fatalf("cached body mismatch: got %d bytes, want %d", len(body), len(want))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func digestOf(b []byte) string {
|
||||
sum := sha256.Sum256(b)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// pushBlobMonolithic uploads a blob with POST (open session) then PUT?digest
|
||||
// (whole body) — the monolithic-after-POST flow.
|
||||
func pushBlobMonolithic(t *testing.T, repo, image string, blob []byte) {
|
||||
t.Helper()
|
||||
dgst := digestOf(blob)
|
||||
|
||||
resp, body := doRequest(t, http.MethodPost, api("/v2/"+repo+"/"+image+"/blobs/uploads/"), nil, "")
|
||||
if resp.StatusCode != http.StatusAccepted {
|
||||
t.Fatalf("start upload: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
loc := resp.Header.Get("Location")
|
||||
if loc == "" {
|
||||
t.Fatalf("start upload: no Location header")
|
||||
}
|
||||
|
||||
resp, body = doRequest(t, http.MethodPut, baseURL()+loc+"?digest="+dgst, blob, "application/octet-stream")
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("finish upload: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if got := resp.Header.Get("Docker-Content-Digest"); got != dgst {
|
||||
t.Fatalf("finish upload: digest mismatch: got %q want %q", got, dgst)
|
||||
}
|
||||
}
|
||||
|
||||
// pushBlobChunked uploads a blob with POST then PATCH (body) then PUT?digest
|
||||
// (empty) — the chunked flow a real docker daemon uses.
|
||||
func pushBlobChunked(t *testing.T, repo, image string, blob []byte) {
|
||||
t.Helper()
|
||||
dgst := digestOf(blob)
|
||||
|
||||
resp, body := doRequest(t, http.MethodPost, api("/v2/"+repo+"/"+image+"/blobs/uploads/"), nil, "")
|
||||
if resp.StatusCode != http.StatusAccepted {
|
||||
t.Fatalf("start upload: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
loc := resp.Header.Get("Location")
|
||||
|
||||
resp, body = doRequest(t, http.MethodPatch, baseURL()+loc, blob, "application/octet-stream")
|
||||
if resp.StatusCode != http.StatusAccepted {
|
||||
t.Fatalf("patch upload: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if got := resp.Header.Get("Range"); got != fmt.Sprintf("0-%d", len(blob)-1) {
|
||||
t.Fatalf("patch upload: unexpected Range %q", got)
|
||||
}
|
||||
loc = resp.Header.Get("Location")
|
||||
|
||||
resp, body = doRequest(t, http.MethodPut, baseURL()+loc+"?digest="+dgst, nil, "")
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("finish upload: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalDockerPushPull exercises a full container push and pull against a
|
||||
// local docker repo using the Docker Registry HTTP API V2, the way a docker
|
||||
// client would: upload the config and layer blobs, push the manifest under a
|
||||
// tag, then pull the manifest and blobs back byte-identically.
|
||||
func TestLocalDockerPushPull(t *testing.T) {
|
||||
createRepo(t, `{"name":"docker-internal","package_type":"docker","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "docker-internal")
|
||||
|
||||
const image = "team/app"
|
||||
const tag = "v1.0.0"
|
||||
|
||||
// /v2/ version check.
|
||||
resp, _ := doRequest(t, http.MethodGet, api("/v2/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("/v2/ ping: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
config := []byte(`{"architecture":"amd64","os":"linux","config":{},"rootfs":{"type":"layers","diff_ids":["sha256:0000000000000000000000000000000000000000000000000000000000000000"]}}`)
|
||||
layer := bytes.Repeat([]byte("artifactapi-layer-data-"), 4096) // ~90 KB opaque layer
|
||||
|
||||
configDigest := digestOf(config)
|
||||
layerDigest := digestOf(layer)
|
||||
|
||||
// A brand-new blob should be absent (this is the client's mount check).
|
||||
resp, _ = doRequest(t, http.MethodHead, api("/v2/"+"docker-internal/"+image+"/blobs/"+configDigest), nil, "")
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("pre-push blob HEAD: expected 404, got %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
pushBlobMonolithic(t, "docker-internal", image, config)
|
||||
pushBlobChunked(t, "docker-internal", image, layer)
|
||||
|
||||
manifest := []byte(fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":%d,"digest":%q},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":%d,"digest":%q}]}`,
|
||||
len(config), configDigest, len(layer), layerDigest))
|
||||
manifestDigest := digestOf(manifest)
|
||||
manifestType := "application/vnd.docker.distribution.manifest.v2+json"
|
||||
|
||||
resp, body := doRequest(t, http.MethodPut, api("/v2/docker-internal/"+image+"/manifests/"+tag), manifest, manifestType)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("push manifest: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if got := resp.Header.Get("Docker-Content-Digest"); got != manifestDigest {
|
||||
t.Fatalf("push manifest: digest %q want %q", got, manifestDigest)
|
||||
}
|
||||
|
||||
// --- pull back ---
|
||||
|
||||
// Manifest by tag.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/v2/docker-internal/"+image+"/manifests/"+tag), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("pull manifest by tag: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, manifest) {
|
||||
t.Fatalf("pulled manifest bytes differ from pushed")
|
||||
}
|
||||
if ct := resp.Header.Get("Content-Type"); ct != manifestType {
|
||||
t.Fatalf("pulled manifest content-type %q want %q", ct, manifestType)
|
||||
}
|
||||
if got := resp.Header.Get("Docker-Content-Digest"); got != manifestDigest {
|
||||
t.Fatalf("pulled manifest digest %q want %q", got, manifestDigest)
|
||||
}
|
||||
|
||||
// Manifest by digest.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/v2/docker-internal/"+image+"/manifests/"+manifestDigest), nil, "")
|
||||
if resp.StatusCode != http.StatusOK || !bytes.Equal(body, manifest) {
|
||||
t.Fatalf("pull manifest by digest: status %d, equal=%v", resp.StatusCode, bytes.Equal(body, manifest))
|
||||
}
|
||||
|
||||
// Blobs by digest.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
digest string
|
||||
want []byte
|
||||
}{
|
||||
{"config", configDigest, config},
|
||||
{"layer", layerDigest, layer},
|
||||
} {
|
||||
resp, body = doRequest(t, http.MethodGet, api("/v2/docker-internal/"+image+"/blobs/"+tc.digest), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("pull %s blob: status %d", tc.name, resp.StatusCode)
|
||||
}
|
||||
if !bytes.Equal(body, tc.want) {
|
||||
t.Fatalf("pulled %s blob bytes differ", tc.name)
|
||||
}
|
||||
if got := resp.Header.Get("Docker-Content-Digest"); got != tc.digest {
|
||||
t.Fatalf("pulled %s blob digest %q want %q", tc.name, got, tc.digest)
|
||||
}
|
||||
}
|
||||
|
||||
// tags/list reflects the pushed tag.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/v2/docker-internal/"+image+"/tags/list"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("tags/list: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !strings.Contains(string(body), `"`+tag+`"`) {
|
||||
t.Fatalf("tags/list missing tag %q: %s", tag, body)
|
||||
}
|
||||
if !strings.Contains(string(body), `"docker-internal/`+image+`"`) {
|
||||
t.Fatalf("tags/list wrong repository name: %s", body)
|
||||
}
|
||||
|
||||
// A now-present blob HEAD should succeed (client would skip re-upload).
|
||||
resp, _ = doRequest(t, http.MethodHead, api("/v2/docker-internal/"+image+"/blobs/"+layerDigest), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("post-push blob HEAD: expected 200, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
hello artifactapi generic blob
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,8 +0,0 @@
|
||||
apiVersion: v1
|
||||
entries:
|
||||
alpha:
|
||||
- name: alpha
|
||||
version: 1.0.0
|
||||
urls:
|
||||
- charts/alpha-1.0.0.tgz
|
||||
generated: "2026-01-01T00:00:00Z"
|
||||
@@ -1,8 +0,0 @@
|
||||
apiVersion: v1
|
||||
entries:
|
||||
beta:
|
||||
- name: beta
|
||||
version: 2.0.0
|
||||
urls:
|
||||
- charts/beta-2.0.0.tgz
|
||||
generated: "2026-01-01T00:00:00Z"
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -1,55 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
|
||||
<revision>1786573032</revision>
|
||||
<data type="primary">
|
||||
<checksum type="sha256">d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08</checksum>
|
||||
<open-checksum type="sha256">3345bb631380ae6c0620fe2a29cf7dff2ed4e28c5cb06c91e8a1628ba1979bcb</open-checksum>
|
||||
<location href="repodata/d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08-primary.xml.gz"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>631</size>
|
||||
<open-size>1192</open-size>
|
||||
</data>
|
||||
<data type="filelists">
|
||||
<checksum type="sha256">daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be</checksum>
|
||||
<open-checksum type="sha256">648bd0ce00fda09abbc6e9c3ff3278518a76f258576ac24cd10c12e41e0e5bd7</open-checksum>
|
||||
<location href="repodata/daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be-filelists.xml.gz"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>256</size>
|
||||
<open-size>338</open-size>
|
||||
</data>
|
||||
<data type="other">
|
||||
<checksum type="sha256">8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85</checksum>
|
||||
<open-checksum type="sha256">c42cfd3843e9c53a60ad84bded44aa46c65faae7b3da99a099a9ca0b018872a9</open-checksum>
|
||||
<location href="repodata/8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85-other.xml.gz"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>296</size>
|
||||
<open-size>399</open-size>
|
||||
</data>
|
||||
<data type="primary_db">
|
||||
<checksum type="sha256">f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71</checksum>
|
||||
<open-checksum type="sha256">c45c85d12ccb0f8172b7bfae466362c08ac1867559574a9fb9cb2118c04daddb</open-checksum>
|
||||
<location href="repodata/f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71-primary.sqlite.bz2"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>1740</size>
|
||||
<open-size>106496</open-size>
|
||||
<database_version>10</database_version>
|
||||
</data>
|
||||
<data type="filelists_db">
|
||||
<checksum type="sha256">be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57</checksum>
|
||||
<open-checksum type="sha256">1ccfa3dff532d782ce3225aae807506a4ce4534291386f1c47455dcc6b70cfd6</open-checksum>
|
||||
<location href="repodata/be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57-filelists.sqlite.bz2"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>764</size>
|
||||
<open-size>28672</open-size>
|
||||
<database_version>10</database_version>
|
||||
</data>
|
||||
<data type="other_db">
|
||||
<checksum type="sha256">ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4</checksum>
|
||||
<open-checksum type="sha256">5d4d38380f0e359bfc0a50033d4faa84c75c5ae8fe2ef11c1c82d64748e9b8e2</open-checksum>
|
||||
<location href="repodata/ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4-other.sqlite.bz2"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>738</size>
|
||||
<open-size>24576</open-size>
|
||||
<database_version>10</database_version>
|
||||
</data>
|
||||
</repomd>
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,108 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
// Package e2edocker holds the black-box end-to-end suite that runs against a
|
||||
// fully dockerised artifactapi stack (see scripts/docker-e2e.sh). Unlike the
|
||||
// in-process e2e suite, these tests only speak HTTP to the running container.
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func baseURL() string {
|
||||
if v := os.Getenv("ARTIFACTAPI_URL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://localhost:8000"
|
||||
}
|
||||
|
||||
// mockUpstream is the base URL the artifactapi *container* uses to reach the
|
||||
// static mock upstream. It is resolved on the compose network, not the host.
|
||||
func mockUpstream() string {
|
||||
if v := os.Getenv("MOCK_UPSTREAM_INTERNAL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://mockupstream"
|
||||
}
|
||||
|
||||
func api(path string) string { return baseURL() + path }
|
||||
|
||||
func fixtureBytes(t *testing.T, rel string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join("fixtures", rel))
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture %s: %v", rel, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func doRequest(t *testing.T, method, url string, body []byte, contentType string) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
var r io.Reader
|
||||
if body != nil {
|
||||
r = bytes.NewReader(body)
|
||||
}
|
||||
req, err := http.NewRequest(method, url, r)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, url, err)
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return resp, respBody
|
||||
}
|
||||
|
||||
func createRepo(t *testing.T, jsonBody string) {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodPost, api("/api/v2/remotes"), []byte(jsonBody), "application/json")
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create repo: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRepo(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
doRequest(t, http.MethodDelete, api("/api/v2/remotes/"+name), nil, "")
|
||||
}
|
||||
|
||||
func createVirtual(t *testing.T, jsonBody string) {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodPost, api("/api/v2/virtuals"), []byte(jsonBody), "application/json")
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteVirtual(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
doRequest(t, http.MethodDelete, api("/api/v2/virtuals/"+name), nil, "")
|
||||
}
|
||||
|
||||
// getEventually retries a GET until it returns 200 or the deadline passes. Used
|
||||
// for asynchronously-generated artifacts (e.g. rpm repodata after upload).
|
||||
func getEventually(t *testing.T, url string, timeout time.Duration) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
var resp *http.Response
|
||||
var body []byte
|
||||
for {
|
||||
resp, body = doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode == http.StatusOK || time.Now().After(deadline) {
|
||||
return resp, body
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestLeastConnMultiBaseURL configures an rpm remote with mirror_strategy =
|
||||
// least_conn over a two-mirror pool and drives distinct cache-miss paths through
|
||||
// it, asserting every request succeeds and both mirrors serve traffic. This
|
||||
// exercises the least-connections selection path (leastConnOrder + the in-flight
|
||||
// gauge inc/dec around each upstream call) end-to-end through a real HTTP client.
|
||||
// A precise least-loaded assertion is timing-sensitive over HTTP and is covered
|
||||
// deterministically by the proxy unit test (TestLeastConnPicksLeastLoaded);
|
||||
// here, with requests issued serially, in-flight counts return to zero between
|
||||
// them so equal-load mirrors are spread by the round-robin tie-break.
|
||||
func TestLeastConnMultiBaseURL(t *testing.T) {
|
||||
name := "e2e-leastconn"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"mirrorlist": [%q],
|
||||
"mirror_strategy": "least_conn",
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamA(), mockUpstreamB()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
seenA, seenB := false, false
|
||||
const n = 12
|
||||
for i := 0; i < n; i++ {
|
||||
url := api(fmt.Sprintf("/api/v1/remote/%s/lc/%d", name, i))
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("request %d: status %d: %s", i, resp.StatusCode, body)
|
||||
}
|
||||
switch strings.TrimSpace(string(body)) {
|
||||
case "UPSTREAM-A":
|
||||
seenA = true
|
||||
case "UPSTREAM-B":
|
||||
seenB = true
|
||||
default:
|
||||
t.Fatalf("request %d: unexpected body %q", i, body)
|
||||
}
|
||||
}
|
||||
if !seenA || !seenB {
|
||||
t.Fatalf("least_conn remote did not reach both upstreams: A=%v B=%v", seenA, seenB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLeastConnDnfInstall drives a real dnf (stock rockylinux container) at a
|
||||
// two-mirror rpm remote configured with mirror_strategy = least_conn whose
|
||||
// primary base_url is dead: makecache + install must succeed via the live mirror.
|
||||
// This proves a real package-manager client installs correctly through a
|
||||
// least_conn remote and that failover semantics are unchanged under the new
|
||||
// strategy. Requires the compose network exported by scripts/docker-e2e.sh.
|
||||
func TestLeastConnDnfInstall(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
name := "e2e-leastconn-dnf"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": "http://mockupstream-dead:80",
|
||||
"mirrorlist": [%q],
|
||||
"mirror_strategy": "least_conn",
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstream()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
repoURL := strings.TrimRight(internal, "/") + "/api/v1/remote/" + name + "/rpm-mirror"
|
||||
repoConf := fmt.Sprintf("[dnflc]\nname=dnflc\nbaseurl=%s\nenabled=1\ngpgcheck=0\nsslverify=0\nmetadata_expire=0\n", repoURL)
|
||||
script := "set -euo pipefail; " +
|
||||
"printf '%s' \"$REPO\" > /etc/yum.repos.d/dnflc.repo; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=dnflc makecache; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=dnflc install e2e-testpkg; " +
|
||||
"rpm -q e2e-testpkg"
|
||||
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"--network", network,
|
||||
"-e", "REPO="+repoConf,
|
||||
"rockylinux:9", "bash", "-c", script)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("real dnf install through a least_conn remote failed: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(string(out), "e2e-testpkg-1.0-1") {
|
||||
t.Fatalf("dnf did not install the expected package via least_conn remote; output:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
func uploadFile(t *testing.T, repo, filePath string, body []byte, contentType string) {
|
||||
t.Helper()
|
||||
url := api("/api/v2/remotes/" + repo + "/files/" + filePath)
|
||||
resp, respBody := doRequest(t, http.MethodPut, url, body, contentType)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s: status %d: %s", filePath, resp.StatusCode, respBody)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalGenericUpload uploads a generic file and downloads it back.
|
||||
func TestLocalGenericUpload(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-generic","package_type":"generic","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-generic")
|
||||
|
||||
content := []byte("artifactapi local generic upload payload")
|
||||
uploadFile(t, "local-generic", "data/hello.bin", content, "application/octet-stream")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-generic/data/hello.bin"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, content) {
|
||||
t.Fatalf("downloaded content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalPyPIUpload uploads a wheel and validates the generated simple index.
|
||||
func TestLocalPyPIUpload(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-pypi","package_type":"pypi","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-pypi")
|
||||
|
||||
wheel := fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl")
|
||||
uploadFile(t, "local-pypi", "foo-1.0-py3-none-any.whl", wheel, "application/zip")
|
||||
|
||||
// Root index lists the package.
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("simple index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !strings.Contains(string(body), "foo") {
|
||||
t.Fatalf("simple index missing package 'foo': %s", body)
|
||||
}
|
||||
|
||||
// Per-package index lists the wheel file.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/simple/foo/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("package index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !strings.Contains(string(body), "foo-1.0-py3-none-any.whl") {
|
||||
t.Fatalf("package index missing wheel: %s", body)
|
||||
}
|
||||
|
||||
// The wheel downloads back byte-identical.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-pypi/foo/foo-1.0-py3-none-any.whl"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download wheel: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, wheel) {
|
||||
t.Fatalf("wheel content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalRPMRepodata uploads a real RPM and validates that repodata is
|
||||
// generated automatically (the special rpm-local feature).
|
||||
func TestLocalRPMRepodata(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-rpm","package_type":"rpm","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-rpm")
|
||||
|
||||
rpm := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
|
||||
uploadFile(t, "local-rpm", "e2e-testpkg-1.0-1.noarch.rpm", rpm, "application/x-rpm")
|
||||
|
||||
// repodata is generated asynchronously after upload; poll for it.
|
||||
resp, body := getEventually(t, api("/api/v1/local/local-rpm/repodata/repomd.xml"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("repomd.xml: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "<repomd") || !strings.Contains(s, "primary") {
|
||||
t.Fatalf("repomd.xml not a valid repodata document: %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalDebRepo uploads a .deb and validates that the flat apt index
|
||||
// (Packages / Release) is generated automatically from the parsed control
|
||||
// stanza (the deb-local analog of rpm repodata generation).
|
||||
func TestLocalDebRepo(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-deb","package_type":"deb","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-deb")
|
||||
|
||||
deb := testsupport.MinimalDeb("e2e-testpkg", "1.0.0", "amd64")
|
||||
uploadFile(t, "local-deb", "e2e-testpkg_1.0.0_amd64.deb", deb, "application/vnd.debian.binary-package")
|
||||
|
||||
// The index is generated asynchronously after upload; poll for it.
|
||||
resp, body := getEventually(t, api("/api/v1/local/local-deb/Packages"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("Packages: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
pkgs := string(body)
|
||||
for _, want := range []string{"Package: e2e-testpkg", "Version: 1.0.0", "Architecture: amd64", "Filename: pool/e2e-testpkg_1.0.0_amd64.deb", "SHA256:"} {
|
||||
if !strings.Contains(pkgs, want) {
|
||||
t.Fatalf("Packages missing %q:\n%s", want, pkgs)
|
||||
}
|
||||
}
|
||||
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-deb/Release"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("Release: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
rel := string(body)
|
||||
for _, want := range []string{"Architectures: amd64", "SHA256:", "Packages"} {
|
||||
if !strings.Contains(rel, want) {
|
||||
t.Fatalf("Release missing %q:\n%s", want, rel)
|
||||
}
|
||||
}
|
||||
|
||||
// The .deb downloads back byte-identical from its pool path.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-deb/pool/e2e-testpkg_1.0.0_amd64.deb"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download deb: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, deb) {
|
||||
t.Fatalf("deb content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalAlpineIndex uploads an .apk to an alpine local repo and validates
|
||||
// that a per-arch APKINDEX.tar.gz is generated automatically from the parsed
|
||||
// .PKGINFO (the apk-local analog of rpm repodata / deb Packages generation).
|
||||
func TestLocalAlpineIndex(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-alpine","package_type":"alpine","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-alpine")
|
||||
|
||||
apk := testsupport.MinimalApk("e2e-testpkg", "1.0-r0", "x86_64")
|
||||
uploadFile(t, "local-alpine", "x86_64/e2e-testpkg-1.0-r0.apk", apk, "application/vnd.android.package-archive")
|
||||
|
||||
// The index is generated asynchronously after upload; poll for it.
|
||||
resp, body := getEventually(t, api("/api/v1/local/local-alpine/x86_64/APKINDEX.tar.gz"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("APKINDEX: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
zr, err := gzip.NewReader(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not gzip: %v", err)
|
||||
}
|
||||
tarBytes, _ := io.ReadAll(zr)
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
var index string
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not tar: %v", err)
|
||||
}
|
||||
if hdr.Name == "APKINDEX" {
|
||||
b, _ := io.ReadAll(tr)
|
||||
index = string(b)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"P:e2e-testpkg", "V:1.0-r0", "A:x86_64", "C:Q1", "S:", "I:"} {
|
||||
if !strings.Contains(index, want) {
|
||||
t.Fatalf("APKINDEX missing %q:\n%s", want, index)
|
||||
}
|
||||
}
|
||||
|
||||
// The .apk downloads back byte-identical from its arch path.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-alpine/x86_64/e2e-testpkg-1.0-r0.apk"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download apk: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, apk) {
|
||||
t.Fatalf("apk content mismatch")
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
# Mock upstream A for the multi-base_url e2e: any path returns a constant,
|
||||
# upstream-identifying body so round-robin distribution is observable.
|
||||
server {
|
||||
listen 80;
|
||||
location / {
|
||||
default_type text/plain;
|
||||
return 200 "UPSTREAM-A";
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
# Mock upstream B for the multi-base_url e2e (see a.conf).
|
||||
server {
|
||||
listen 80;
|
||||
location / {
|
||||
default_type text/plain;
|
||||
return 200 "UPSTREAM-B";
|
||||
}
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mockUpstreamA/B are the constant-body upstreams (see docker-compose.e2e.yml)
|
||||
// that let the round-robin test observe which mirror served each request.
|
||||
func mockUpstreamA() string {
|
||||
if v := os.Getenv("MOCK_UPSTREAM_A_INTERNAL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://mockupstreama"
|
||||
}
|
||||
|
||||
func mockUpstreamB() string {
|
||||
if v := os.Getenv("MOCK_UPSTREAM_B_INTERNAL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://mockupstreamb"
|
||||
}
|
||||
|
||||
// TestMultiBaseURLRoundRobin configures an rpm remote with base_url = mirror A
|
||||
// and mirrorlist = [mirror B] and drives distinct paths through it, asserting
|
||||
// both mirrors serve traffic. Each path is a cache miss, so every request reaches
|
||||
// upstream and the round-robin cursor alternates mirrors.
|
||||
func TestMultiBaseURLRoundRobin(t *testing.T) {
|
||||
name := "e2e-rr"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"mirrorlist": [%q],
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamA(), mockUpstreamB()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
seenA, seenB := false, false
|
||||
const n = 12
|
||||
for i := 0; i < n; i++ {
|
||||
url := api(fmt.Sprintf("/api/v1/remote/%s/rr/%d", name, i))
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("request %d: status %d: %s", i, resp.StatusCode, body)
|
||||
}
|
||||
switch strings.TrimSpace(string(body)) {
|
||||
case "UPSTREAM-A":
|
||||
seenA = true
|
||||
case "UPSTREAM-B":
|
||||
seenB = true
|
||||
default:
|
||||
t.Fatalf("request %d: unexpected body %q", i, body)
|
||||
}
|
||||
}
|
||||
if !seenA || !seenB {
|
||||
t.Fatalf("round-robin did not reach both upstreams: A=%v B=%v", seenA, seenB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiBaseURLFailover points a two-mirror remote at a dead primary and a
|
||||
// healthy secondary and asserts every request still succeeds via the secondary.
|
||||
func TestMultiBaseURLFailover(t *testing.T) {
|
||||
name := "e2e-failover"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": "http://mockupstream-dead:80",
|
||||
"mirrorlist": [%q],
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamB()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
for i := 0; i < 6; i++ {
|
||||
url := api(fmt.Sprintf("/api/v1/remote/%s/fo/%d", name, i))
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("request %d: dead primary broke fetch: status %d: %s", i, resp.StatusCode, body)
|
||||
}
|
||||
if got := strings.TrimSpace(string(body)); got != "UPSTREAM-B" {
|
||||
t.Fatalf("request %d: body %q, want UPSTREAM-B (served via failover)", i, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSingleBaseURLRegression asserts a remote with no mirrorlist works exactly
|
||||
// as before the mirrorlist change.
|
||||
func TestSingleBaseURLRegression(t *testing.T) {
|
||||
name := "e2e-single"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamA()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/remote/"+name+"/solo/0"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("single-url fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if got := strings.TrimSpace(string(body)); got != "UPSTREAM-A" {
|
||||
t.Fatalf("single-url body %q, want UPSTREAM-A", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiBaseURLDnfFailover drives a real dnf (stock rockylinux container) at
|
||||
// a two-mirror rpm remote whose primary is dead: makecache + install must
|
||||
// succeed via the live secondary mirror, proving a dead mirror does not break a
|
||||
// real package-manager client. Requires the compose network and internal API
|
||||
// URL exported by scripts/docker-e2e.sh; skipped when run standalone.
|
||||
func TestMultiBaseURLDnfFailover(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
name := "e2e-dnf-failover"
|
||||
// Primary base_url is dead; the live mirror serves the real yum repo under
|
||||
// fixtures/rpm-mirror via the shared mock upstream.
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": "http://mockupstream-dead:80",
|
||||
"mirrorlist": [%q],
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstream()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
repoURL := strings.TrimRight(internal, "/") + "/api/v1/remote/" + name + "/rpm-mirror"
|
||||
repoConf := fmt.Sprintf("[dnffo]\nname=dnffo\nbaseurl=%s\nenabled=1\ngpgcheck=0\nsslverify=0\nmetadata_expire=0\n", repoURL)
|
||||
script := "set -euo pipefail; " +
|
||||
"printf '%s' \"$REPO\" > /etc/yum.repos.d/dnffo.repo; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=dnffo makecache; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=dnffo install e2e-testpkg; " +
|
||||
"rpm -q e2e-testpkg"
|
||||
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"--network", network,
|
||||
"-e", "REPO="+repoConf,
|
||||
"rockylinux:9", "bash", "-c", script)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("real dnf install through a dead primary mirror failed: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(string(out), "e2e-testpkg-1.0-1") {
|
||||
t.Fatalf("dnf did not install the expected package via failover; output:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHealth(t *testing.T) {
|
||||
resp, body := doRequest(t, http.MethodGet, api("/health"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("health: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRemoteLifecycle covers add/change/delete for a remote repository.
|
||||
func TestRemoteLifecycle(t *testing.T) {
|
||||
createRepo(t, `{
|
||||
"name": "crud-remote",
|
||||
"package_type": "generic",
|
||||
"repo_type": "remote",
|
||||
"base_url": "https://example.com",
|
||||
"mutable_ttl": 600,
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRepo(t, "crud-remote")
|
||||
|
||||
got := getRepo(t, "crud-remote")
|
||||
if got["base_url"] != "https://example.com" || got["mutable_ttl"].(float64) != 600 {
|
||||
t.Fatalf("unexpected created remote: %v", got)
|
||||
}
|
||||
|
||||
// change
|
||||
resp, body := doRequest(t, http.MethodPut, api("/api/v2/remotes/crud-remote"), []byte(`{
|
||||
"package_type": "generic",
|
||||
"base_url": "https://updated.example.com",
|
||||
"mutable_ttl": 120,
|
||||
"stale_on_error": true
|
||||
}`), "application/json")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update remote: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
got = getRepo(t, "crud-remote")
|
||||
if got["base_url"] != "https://updated.example.com" || got["mutable_ttl"].(float64) != 120 {
|
||||
t.Fatalf("update not applied: %v", got)
|
||||
}
|
||||
|
||||
// delete
|
||||
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-remote"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete remote: status %d", resp.StatusCode)
|
||||
}
|
||||
resp, _ = doRequest(t, http.MethodGet, api("/api/v2/remotes/crud-remote"), nil, "")
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("expected 404 after delete, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalLifecycle covers add/delete for a local repository.
|
||||
func TestLocalLifecycle(t *testing.T) {
|
||||
createRepo(t, `{
|
||||
"name": "crud-local",
|
||||
"package_type": "generic",
|
||||
"repo_type": "local"
|
||||
}`)
|
||||
defer deleteRepo(t, "crud-local")
|
||||
|
||||
got := getRepo(t, "crud-local")
|
||||
if got["repo_type"] != "local" {
|
||||
t.Fatalf("expected repo_type local, got %v", got["repo_type"])
|
||||
}
|
||||
|
||||
resp, _ := doRequest(t, http.MethodDelete, api("/api/v2/remotes/crud-local"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete local: status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVirtualLifecycle covers add/change/delete for a virtual repository.
|
||||
func TestVirtualLifecycle(t *testing.T) {
|
||||
createRepo(t, `{"name":"vmem-a","package_type":"helm","repo_type":"remote","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRepo(t, `{"name":"vmem-b","package_type":"helm","repo_type":"remote","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRepo(t, "vmem-a")
|
||||
defer deleteRepo(t, "vmem-b")
|
||||
|
||||
createVirtual(t, `{
|
||||
"name": "crud-virtual",
|
||||
"package_type": "helm",
|
||||
"members": ["vmem-a"]
|
||||
}`)
|
||||
defer deleteVirtual(t, "crud-virtual")
|
||||
|
||||
// change members
|
||||
resp, body := doRequest(t, http.MethodPut, api("/api/v2/virtuals/crud-virtual"), []byte(`{
|
||||
"package_type": "helm",
|
||||
"members": ["vmem-a", "vmem-b"]
|
||||
}`), "application/json")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v2/virtuals/crud-virtual"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("get virtual: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
var v map[string]any
|
||||
if err := json.Unmarshal(body, &v); err != nil {
|
||||
t.Fatalf("decode virtual: %v", err)
|
||||
}
|
||||
members, _ := v["members"].([]any)
|
||||
if len(members) != 2 {
|
||||
t.Fatalf("expected 2 members after update, got %v", v["members"])
|
||||
}
|
||||
|
||||
resp, _ = doRequest(t, http.MethodDelete, api("/api/v2/virtuals/crud-virtual"), nil, "")
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("delete virtual: status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func getRepo(t *testing.T, name string) map[string]any {
|
||||
t.Helper()
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v2/remotes/"+name), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("get remote %s: status %d: %s", name, resp.StatusCode, body)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
t.Fatalf("decode remote %s: %v", name, err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
|
||||
// checks that a virtual over them serves a merged simple index.
|
||||
func TestVirtualPyPIMerge(t *testing.T) {
|
||||
createRepo(t, `{"name":"pmerge-a","package_type":"pypi","repo_type":"local"}`)
|
||||
createRepo(t, `{"name":"pmerge-b","package_type":"pypi","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "pmerge-a")
|
||||
defer deleteRepo(t, "pmerge-b")
|
||||
|
||||
uploadFile(t, "pmerge-a", "foo-1.0-py3-none-any.whl", fixtureBytes(t, "packages/foo-1.0-py3-none-any.whl"), "application/zip")
|
||||
uploadFile(t, "pmerge-b", "bar-2.0-py3-none-any.whl", []byte("bar wheel payload"), "application/zip")
|
||||
|
||||
createVirtual(t, `{"name":"pmerge-v","package_type":"pypi","members":["pmerge-a","pmerge-b"]}`)
|
||||
defer deleteVirtual(t, "pmerge-v")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/pmerge-v/simple/"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("virtual simple index: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "foo") || !strings.Contains(s, "bar") {
|
||||
t.Fatalf("merged index missing a member package (want foo and bar): %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVirtualHelmMerge points two helm remotes at mock index.yaml documents
|
||||
// with distinct charts and checks the virtual merges both into one index.
|
||||
func TestVirtualHelmMerge(t *testing.T) {
|
||||
createRepo(t, `{"name":"hmerge-a","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-a","stale_on_error":true}`)
|
||||
createRepo(t, `{"name":"hmerge-b","package_type":"helm","repo_type":"remote","base_url":"`+mockUpstream()+`/helm-b","stale_on_error":true}`)
|
||||
defer deleteRepo(t, "hmerge-a")
|
||||
defer deleteRepo(t, "hmerge-b")
|
||||
|
||||
createVirtual(t, `{"name":"hmerge-v","package_type":"helm","members":["hmerge-a","hmerge-b"]}`)
|
||||
defer deleteVirtual(t, "hmerge-v")
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/hmerge-v/index.yaml"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("virtual index.yaml: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
s := string(body)
|
||||
if !strings.Contains(s, "alpha") || !strings.Contains(s, "beta") {
|
||||
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -95,7 +95,7 @@ func TestMain(m *testing.M) {
|
||||
}
|
||||
cfg.ListenAddr = "127.0.0.1:0"
|
||||
|
||||
srv, err := server.New(cfg, "e2e-test")
|
||||
srv, err := server.New(cfg)
|
||||
if err != nil {
|
||||
log.Fatalf("server: %v", err)
|
||||
}
|
||||
|
||||
@@ -24,30 +24,6 @@ func TestRoot(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUpstreamTimeouts(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "timeout-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"stale_on_error": true,
|
||||
"upstream_dial_timeout": 3,
|
||||
"upstream_tls_timeout": 4,
|
||||
"upstream_response_header_timeout": 5
|
||||
}`)
|
||||
defer deleteRemote(t, "timeout-test")
|
||||
|
||||
remote := getJSON(t, apiURL("/api/v2/remotes/timeout-test"))
|
||||
for field, want := range map[string]float64{
|
||||
"upstream_dial_timeout": 3,
|
||||
"upstream_tls_timeout": 4,
|
||||
"upstream_response_header_timeout": 5,
|
||||
} {
|
||||
if got, _ := remote[field].(float64); got != want {
|
||||
t.Errorf("%s: got %v, want %v", field, remote[field], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteCRUD(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "test-generic",
|
||||
|
||||
@@ -24,39 +24,6 @@ func TestProxyBlocklist(t *testing.T) {
|
||||
assertStatus(t, apiURL("/api/v1/remote/blocklist-test/malware.exe"), http.StatusForbidden)
|
||||
}
|
||||
|
||||
func TestProxyHeadBlocklist(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "head-block-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"blocklist": ["\\.exe$"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "head-block-test")
|
||||
|
||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/head-block-test/malware.exe"), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("HEAD blocklisted path: got %d, want 403", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyHeadUnknownRemote(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/nonexistent/some/path"), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("HEAD unknown remote: got %d, want 404", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyPatterns(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "patterns-test",
|
||||
|
||||
@@ -7,17 +7,12 @@ require (
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/go-chi/chi/v5 v5.3.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/klauspost/compress v1.19.2
|
||||
github.com/minio/minio-go/v7 v7.2.0
|
||||
github.com/redis/go-redis/v9 v9.20.0
|
||||
github.com/testcontainers/testcontainers-go v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
|
||||
github.com/ulikunitz/xz v0.5.16
|
||||
golang.org/x/crypto v0.51.0
|
||||
golang.org/x/time v0.15.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
@@ -50,9 +45,11 @@ require (
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
@@ -99,6 +96,7 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.51.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
|
||||
@@ -85,8 +85,8 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
@@ -189,8 +189,6 @@ github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYI
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
||||
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
|
||||
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
@@ -236,8 +234,6 @@ golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
|
||||
@@ -1,301 +0,0 @@
|
||||
// Package terraform serves local terraform repos as a real Terraform provider
|
||||
// registry: service discovery, version listing, and GPG-signed downloads, so
|
||||
// `terraform init` installs from a bare source address with no client config.
|
||||
package terraform
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
tfprov "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
"git.unkin.net/unkin/artifactapi/internal/tfsign"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// ProvidersV1Path is the base the service-discovery document advertises (Terraform
|
||||
// appends "{namespace}/{type}/versions" etc). MountPath is the same prefix without
|
||||
// the trailing slash, for chi.Mount.
|
||||
const (
|
||||
ProvidersV1Path = "/terraform/v1/providers/"
|
||||
MountPath = "/terraform/v1/providers"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
db *database.DB
|
||||
signer *tfsign.Signer
|
||||
protocols []string
|
||||
}
|
||||
|
||||
func NewHandler(db *database.DB, signer *tfsign.Signer, protocols string) *Handler {
|
||||
var protos []string
|
||||
for _, p := range strings.Split(protocols, ",") {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
protos = append(protos, p)
|
||||
}
|
||||
}
|
||||
if len(protos) == 0 {
|
||||
protos = []string{"5.0", "6.0"}
|
||||
}
|
||||
return &Handler{db: db, signer: signer, protocols: protos}
|
||||
}
|
||||
|
||||
// Enabled reports whether a signing key is configured. Without one the registry
|
||||
// cannot produce the signed SHA256SUMS the protocol requires, so it stays off.
|
||||
func (h *Handler) Enabled() bool { return h.signer != nil }
|
||||
|
||||
func (h *Handler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/{namespace}/{type}/versions", h.versions)
|
||||
r.Get("/{namespace}/{type}/{version}/download/{os}/{arch}", h.download)
|
||||
r.Get("/{namespace}/{type}/{version}/sha256sums", h.sha256sums)
|
||||
r.Get("/{namespace}/{type}/{version}/sha256sums.sig", h.sha256sumsSig)
|
||||
return r
|
||||
}
|
||||
|
||||
// ServiceDiscovery answers /.well-known/terraform.json, pointing Terraform at the
|
||||
// providers.v1 protocol base.
|
||||
func (h *Handler) ServiceDiscovery(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.Enabled() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]string{"providers.v1": ProvidersV1Path})
|
||||
}
|
||||
|
||||
// providerFile is one resolved platform artifact within a repo.
|
||||
type providerFile struct {
|
||||
version string
|
||||
os string
|
||||
arch string
|
||||
filePath string // path within the repo, e.g. unkin/artifactapi/...zip
|
||||
sha256 string // hex, no "sha256:" prefix
|
||||
}
|
||||
|
||||
// resolve finds every provider zip of the given type in the repo (namespace).
|
||||
// The Terraform source namespace maps to the artifactapi repo name; the provider
|
||||
// is matched by type across whatever in-repo folder it was uploaded under.
|
||||
func (h *Handler) resolve(r *http.Request, namespace, typeName string) ([]providerFile, error) {
|
||||
remote, err := h.db.GetRemote(r.Context(), namespace)
|
||||
if err != nil || remote.PackageType != models.PackageTerraform {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rows, err := h.db.ListLocalFiles(r.Context(), namespace, 10000, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var out []providerFile
|
||||
for _, row := range rows {
|
||||
parsed := tfprov.ParseProviderZip(path.Base(row.FilePath))
|
||||
if !parsed.Ok || parsed.Type != typeName {
|
||||
continue
|
||||
}
|
||||
out = append(out, providerFile{
|
||||
version: parsed.Version,
|
||||
os: parsed.OS,
|
||||
arch: parsed.Arch,
|
||||
filePath: row.FilePath,
|
||||
sha256: strings.TrimPrefix(row.ContentHash, "sha256:"),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h *Handler) versions(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.Enabled() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
namespace := chi.URLParam(r, "namespace")
|
||||
typeName := chi.URLParam(r, "type")
|
||||
|
||||
files, err := h.resolve(r, namespace, typeName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if len(files) == 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Group platforms by version, de-duplicated and stably ordered.
|
||||
type platform struct {
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
}
|
||||
platforms := map[string]map[string]platform{}
|
||||
for _, f := range files {
|
||||
if platforms[f.version] == nil {
|
||||
platforms[f.version] = map[string]platform{}
|
||||
}
|
||||
platforms[f.version][f.os+"_"+f.arch] = platform{OS: f.os, Arch: f.arch}
|
||||
}
|
||||
|
||||
type versionEntry struct {
|
||||
Version string `json:"version"`
|
||||
Protocols []string `json:"protocols"`
|
||||
Platforms []platform `json:"platforms"`
|
||||
}
|
||||
out := struct {
|
||||
Versions []versionEntry `json:"versions"`
|
||||
}{}
|
||||
for version, plats := range platforms {
|
||||
entry := versionEntry{Version: version, Protocols: h.protocols}
|
||||
for _, p := range plats {
|
||||
entry.Platforms = append(entry.Platforms, p)
|
||||
}
|
||||
sort.Slice(entry.Platforms, func(i, j int) bool {
|
||||
return entry.Platforms[i].OS+entry.Platforms[i].Arch < entry.Platforms[j].OS+entry.Platforms[j].Arch
|
||||
})
|
||||
out.Versions = append(out.Versions, entry)
|
||||
}
|
||||
sort.Slice(out.Versions, func(i, j int) bool { return out.Versions[i].Version < out.Versions[j].Version })
|
||||
|
||||
writeJSON(w, out)
|
||||
}
|
||||
|
||||
func (h *Handler) download(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.Enabled() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
namespace := chi.URLParam(r, "namespace")
|
||||
typeName := chi.URLParam(r, "type")
|
||||
version := chi.URLParam(r, "version")
|
||||
osName := chi.URLParam(r, "os")
|
||||
arch := chi.URLParam(r, "arch")
|
||||
|
||||
files, err := h.resolve(r, namespace, typeName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
var match *providerFile
|
||||
for i := range files {
|
||||
if files[i].version == version && files[i].os == osName && files[i].arch == arch {
|
||||
match = &files[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if match == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
base := baseURL(r)
|
||||
verBase := fmt.Sprintf("%s%s/%s/%s", base+ProvidersV1Path, namespace, typeName, version)
|
||||
|
||||
type gpgKey struct {
|
||||
KeyID string `json:"key_id"`
|
||||
ASCIIArmor string `json:"ascii_armor"`
|
||||
}
|
||||
resp := struct {
|
||||
Protocols []string `json:"protocols"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
Filename string `json:"filename"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
SHASumsURL string `json:"shasums_url"`
|
||||
SHASumsSignatureURL string `json:"shasums_signature_url"`
|
||||
SHASum string `json:"shasum"`
|
||||
SigningKeys struct {
|
||||
GPGPublicKeys []gpgKey `json:"gpg_public_keys"`
|
||||
} `json:"signing_keys"`
|
||||
}{
|
||||
Protocols: h.protocols,
|
||||
OS: match.os,
|
||||
Arch: match.arch,
|
||||
Filename: path.Base(match.filePath),
|
||||
DownloadURL: fmt.Sprintf("%s/api/v1/local/%s/%s", base, namespace, match.filePath),
|
||||
SHASumsURL: verBase + "/sha256sums",
|
||||
SHASumsSignatureURL: verBase + "/sha256sums.sig",
|
||||
SHASum: match.sha256,
|
||||
}
|
||||
resp.SigningKeys.GPGPublicKeys = []gpgKey{{
|
||||
KeyID: h.signer.KeyID(),
|
||||
ASCIIArmor: h.signer.PublicKeyArmor(),
|
||||
}}
|
||||
|
||||
writeJSON(w, resp)
|
||||
}
|
||||
|
||||
func (h *Handler) sha256sums(w http.ResponseWriter, r *http.Request) {
|
||||
sums, ok := h.buildSums(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Write(sums)
|
||||
}
|
||||
|
||||
func (h *Handler) sha256sumsSig(w http.ResponseWriter, r *http.Request) {
|
||||
sums, ok := h.buildSums(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
sig, err := h.signer.Sign(sums)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Write(sig)
|
||||
}
|
||||
|
||||
// buildSums renders the SHA256SUMS body for one version: one "<hex> <filename>"
|
||||
// line per platform zip, sorted by filename so the signed bytes are stable.
|
||||
func (h *Handler) buildSums(w http.ResponseWriter, r *http.Request) ([]byte, bool) {
|
||||
if !h.Enabled() {
|
||||
http.NotFound(w, r)
|
||||
return nil, false
|
||||
}
|
||||
namespace := chi.URLParam(r, "namespace")
|
||||
typeName := chi.URLParam(r, "type")
|
||||
version := chi.URLParam(r, "version")
|
||||
|
||||
files, err := h.resolve(r, namespace, typeName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return nil, false
|
||||
}
|
||||
|
||||
var lines []string
|
||||
for _, f := range files {
|
||||
if f.version != version {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("%s %s", f.sha256, path.Base(f.filePath)))
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
http.NotFound(w, r)
|
||||
return nil, false
|
||||
}
|
||||
sort.Strings(lines)
|
||||
return []byte(strings.Join(lines, "\n") + "\n"), true
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
func baseURL(r *http.Request) string {
|
||||
scheme := "http"
|
||||
if r.TLS != nil {
|
||||
scheme = "https"
|
||||
}
|
||||
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
|
||||
scheme = fwd
|
||||
}
|
||||
return scheme + "://" + r.Host
|
||||
}
|
||||
@@ -1,186 +0,0 @@
|
||||
package terraform
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"golang.org/x/crypto/openpgp"
|
||||
"golang.org/x/crypto/openpgp/armor"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/internal/tfsign"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
var testDSN string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, terminate, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN = dsn
|
||||
code := m.Run()
|
||||
terminate()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// testSigner writes a throwaway armored key and loads it.
|
||||
func testSigner(t *testing.T) *tfsign.Signer {
|
||||
t.Helper()
|
||||
e, err := openpgp.NewEntity("artifactapi test", "tf", "tf@example.com", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
w, _ := armor.Encode(&buf, openpgp.PrivateKeyType, nil)
|
||||
if err := e.SerializePrivate(w, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Close()
|
||||
p := filepath.Join(t.TempDir(), "private-key.asc")
|
||||
if err := os.WriteFile(p, buf.Bytes(), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := tfsign.Load(p, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestProviderRegistryFlow(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const repo = "tf-reg" // Terraform namespace == repo name
|
||||
const filePath = "unkin/artifactapi/terraform-provider-artifactapi_1.2.3_linux_amd64.zip"
|
||||
const hash = "sha256:983cdb25cb7b976538e4334d26e52dee5f44749b9be1500c760cf5cf66be659b"
|
||||
const wantSha = "983cdb25cb7b976538e4334d26e52dee5f44749b9be1500c760cf5cf66be659b"
|
||||
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageTerraform, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.UpsertBlob(ctx, hash, "blobs/98/3c", 6381007, "application/zip"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.CreateLocalFile(ctx, repo, filePath, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
signer := testSigner(t)
|
||||
h := NewHandler(db, signer, "5.0,6.0")
|
||||
router := chi.NewRouter()
|
||||
router.Get("/.well-known/terraform.json", h.ServiceDiscovery)
|
||||
router.Mount(MountPath, h.Routes())
|
||||
|
||||
get := func(p string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest("GET", p, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
return w
|
||||
}
|
||||
|
||||
// Service discovery.
|
||||
w := get("/.well-known/terraform.json")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("discovery = %d", w.Code)
|
||||
}
|
||||
var disc map[string]string
|
||||
json.Unmarshal(w.Body.Bytes(), &disc)
|
||||
if disc["providers.v1"] != ProvidersV1Path {
|
||||
t.Errorf("providers.v1 = %q", disc["providers.v1"])
|
||||
}
|
||||
|
||||
// Versions.
|
||||
w = get("/terraform/v1/providers/tf-reg/artifactapi/versions")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("versions = %d %s", w.Code, w.Body)
|
||||
}
|
||||
var vresp struct {
|
||||
Versions []struct {
|
||||
Version string `json:"version"`
|
||||
Protocols []string `json:"protocols"`
|
||||
Platforms []map[string]string `json:"platforms"`
|
||||
} `json:"versions"`
|
||||
}
|
||||
json.Unmarshal(w.Body.Bytes(), &vresp)
|
||||
if len(vresp.Versions) != 1 || vresp.Versions[0].Version != "1.2.3" {
|
||||
t.Fatalf("unexpected versions: %+v", vresp)
|
||||
}
|
||||
if len(vresp.Versions[0].Platforms) != 1 || vresp.Versions[0].Platforms[0]["os"] != "linux" {
|
||||
t.Fatalf("unexpected platforms: %+v", vresp.Versions[0].Platforms)
|
||||
}
|
||||
|
||||
// Download.
|
||||
w = get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/download/linux/amd64")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("download = %d %s", w.Code, w.Body)
|
||||
}
|
||||
var dl struct {
|
||||
Filename string `json:"filename"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
SHASumsURL string `json:"shasums_url"`
|
||||
SHASumsSignatureURL string `json:"shasums_signature_url"`
|
||||
SHASum string `json:"shasum"`
|
||||
SigningKeys struct {
|
||||
GPGPublicKeys []struct {
|
||||
KeyID string `json:"key_id"`
|
||||
ASCIIArmor string `json:"ascii_armor"`
|
||||
} `json:"gpg_public_keys"`
|
||||
} `json:"signing_keys"`
|
||||
}
|
||||
json.Unmarshal(w.Body.Bytes(), &dl)
|
||||
if dl.SHASum != wantSha {
|
||||
t.Errorf("shasum = %q", dl.SHASum)
|
||||
}
|
||||
wantURL := "http://example.com/api/v1/local/tf-reg/" + filePath
|
||||
if dl.DownloadURL != wantURL {
|
||||
t.Errorf("download_url = %q, want %q", dl.DownloadURL, wantURL)
|
||||
}
|
||||
if len(dl.SigningKeys.GPGPublicKeys) != 1 || dl.SigningKeys.GPGPublicKeys[0].KeyID != signer.KeyID() {
|
||||
t.Errorf("signing key mismatch: %+v", dl.SigningKeys)
|
||||
}
|
||||
|
||||
// SHA256SUMS + signature verify against the advertised key.
|
||||
sums := get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/sha256sums")
|
||||
wantLine := wantSha + " terraform-provider-artifactapi_1.2.3_linux_amd64.zip\n"
|
||||
if sums.Body.String() != wantLine {
|
||||
t.Errorf("sha256sums = %q, want %q", sums.Body.String(), wantLine)
|
||||
}
|
||||
sig := get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/sha256sums.sig")
|
||||
keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewReader([]byte(dl.SigningKeys.GPGPublicKeys[0].ASCIIArmor)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewReader(sums.Body.Bytes()), bytes.NewReader(sig.Body.Bytes())); err != nil {
|
||||
t.Errorf("sha256sums.sig did not verify: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryDisabledWithoutSigner(t *testing.T) {
|
||||
h := NewHandler(nil, nil, "")
|
||||
router := chi.NewRouter()
|
||||
router.Get("/.well-known/terraform.json", h.ServiceDiscovery)
|
||||
req := httptest.NewRequest("GET", "/.well-known/terraform.json", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 404 {
|
||||
t.Errorf("disabled discovery = %d, want 404", w.Code)
|
||||
}
|
||||
}
|
||||
@@ -1,486 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// This file implements the write half of the Docker Registry HTTP API V2 for
|
||||
// *local* docker repositories, so a `docker push` / `docker pull` against
|
||||
// artifactapi treats a local docker repo as a genuine registry (matching the
|
||||
// project's "local repos are the real thing" principle) rather than a mirror.
|
||||
//
|
||||
// Storage reuses the existing content-addressable primitives:
|
||||
// - blob and manifest bytes are stored via the CAS (deduplicated by sha256)
|
||||
// - a local_files row per (repo, "<image>/blobs/<digest>") and
|
||||
// (repo, "<image>/manifests/<ref>") keeps the blob referenced so the GC
|
||||
// does not reap it, and lets pulls resolve a reference back to a blob.
|
||||
// Tags are mutable references (UpsertLocalFile); digests and blobs are
|
||||
// immutable (CreateLocalFile, tolerating an already-exists on re-push).
|
||||
|
||||
const dockerAPIVersionHeader = "registry/2.0"
|
||||
|
||||
// Chunked blob uploads are staged in object storage under uploads/<uuid> rather
|
||||
// than in process memory, so the POST / PATCH / PUT of a single push can each be
|
||||
// served by a different replica (the API runs with minReplicas>1 and no session
|
||||
// affinity). The upload UUID travels in the Location URL handed back to the
|
||||
// client, so any replica reconstructs the staging key with no shared in-process
|
||||
// state. Abandoned stages are dropped by the GC's uploads sweep.
|
||||
func uploadKey(id string) string { return "uploads/" + id }
|
||||
|
||||
var errUploadUnknown = errors.New("unknown upload")
|
||||
|
||||
// appendUpload appends a chunk to the staged upload object and returns the new
|
||||
// total size. The staged bytes live entirely in object storage (download,
|
||||
// append to a per-request temp file, re-upload), which keeps the session state
|
||||
// replica-independent. Docker sends the whole layer in one PATCH, so this is a
|
||||
// single append in the common case.
|
||||
func (h *ProxyHandler) appendUpload(ctx context.Context, id string, chunk io.Reader) (int64, error) {
|
||||
key := uploadKey(id)
|
||||
reader, info, err := h.store.Download(ctx, key)
|
||||
if err != nil {
|
||||
return 0, errUploadUnknown
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp("", "docker-upload-*")
|
||||
if err != nil {
|
||||
reader.Close()
|
||||
return 0, err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
defer tmp.Close()
|
||||
|
||||
if _, err := io.Copy(tmp, reader); err != nil {
|
||||
reader.Close()
|
||||
return 0, err
|
||||
}
|
||||
reader.Close()
|
||||
|
||||
n, err := io.Copy(tmp, chunk)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
size := info.Size + n
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := h.store.Upload(ctx, key, tmp, size, "application/octet-stream"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// dockerReq is a parsed /v2/<remote>/<image>/... request. kind is one of
|
||||
// "manifest", "blob", "upload", "tags".
|
||||
type dockerReq struct {
|
||||
image string
|
||||
kind string
|
||||
ref string // tag, digest, or upload uuid depending on kind
|
||||
}
|
||||
|
||||
// parseDockerPath splits the chi "*" remainder (everything after the repo name)
|
||||
// into the image name and the registry operation. The image name may itself
|
||||
// contain slashes, so operations are located by their well-known infixes.
|
||||
func parseDockerPath(rest string) (dockerReq, bool) {
|
||||
rest = strings.TrimPrefix(rest, "/")
|
||||
switch {
|
||||
case strings.HasSuffix(rest, "/tags/list"):
|
||||
return dockerReq{image: strings.TrimSuffix(rest, "/tags/list"), kind: "tags"}, true
|
||||
case rest == "tags/list":
|
||||
return dockerReq{}, false // no image
|
||||
}
|
||||
if i := strings.Index(rest, "/blobs/uploads"); i >= 0 {
|
||||
image := rest[:i]
|
||||
ref := strings.TrimPrefix(rest[i+len("/blobs/uploads"):], "/")
|
||||
return dockerReq{image: image, kind: "upload", ref: ref}, image != ""
|
||||
}
|
||||
if i := strings.LastIndex(rest, "/manifests/"); i >= 0 {
|
||||
return dockerReq{image: rest[:i], kind: "manifest", ref: rest[i+len("/manifests/"):]}, true
|
||||
}
|
||||
if i := strings.LastIndex(rest, "/blobs/"); i >= 0 {
|
||||
return dockerReq{image: rest[:i], kind: "blob", ref: rest[i+len("/blobs/"):]}, true
|
||||
}
|
||||
return dockerReq{}, false
|
||||
}
|
||||
|
||||
func isDigest(ref string) bool { return strings.HasPrefix(ref, "sha256:") }
|
||||
|
||||
// localDockerRemote returns the repo if name is a local docker repository.
|
||||
func (h *ProxyHandler) localDockerRemote(r *http.Request, name string) (*models.Remote, bool) {
|
||||
remote, err := h.db.GetRemote(r.Context(), name)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageDocker
|
||||
}
|
||||
|
||||
func dockerError(w http.ResponseWriter, status int, code, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Docker-Distribution-Api-Version", dockerAPIVersionHeader)
|
||||
w.WriteHeader(status)
|
||||
fmt.Fprintf(w, `{"errors":[{"code":%q,"message":%q}]}`, code, msg)
|
||||
}
|
||||
|
||||
// dockerGet dispatches a registry GET to the local handler for local docker
|
||||
// repos and falls through to the upstream proxy for everything else.
|
||||
func (h *ProxyHandler) dockerGet(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "remoteName")
|
||||
if remote, ok := h.localDockerRemote(r, name); ok {
|
||||
h.dockerLocalGet(w, r, remote, false)
|
||||
return
|
||||
}
|
||||
h.handleProxy(w, r)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) dockerHead(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "remoteName")
|
||||
if remote, ok := h.localDockerRemote(r, name); ok {
|
||||
h.dockerLocalGet(w, r, remote, true)
|
||||
return
|
||||
}
|
||||
h.handleProxyHead(w, r)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) dockerPost(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "remoteName")
|
||||
remote, ok := h.localDockerRemote(r, name)
|
||||
if !ok {
|
||||
dockerError(w, http.StatusMethodNotAllowed, "UNSUPPORTED", "push is only supported for local docker repositories")
|
||||
return
|
||||
}
|
||||
h.dockerStartUpload(w, r, remote)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) dockerPatch(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "remoteName")
|
||||
remote, ok := h.localDockerRemote(r, name)
|
||||
if !ok {
|
||||
dockerError(w, http.StatusMethodNotAllowed, "UNSUPPORTED", "push is only supported for local docker repositories")
|
||||
return
|
||||
}
|
||||
h.dockerPatchUpload(w, r, remote)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) dockerPut(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "remoteName")
|
||||
remote, ok := h.localDockerRemote(r, name)
|
||||
if !ok {
|
||||
dockerError(w, http.StatusMethodNotAllowed, "UNSUPPORTED", "push is only supported for local docker repositories")
|
||||
return
|
||||
}
|
||||
req, ok := parseDockerPath(chi.URLParam(r, "*"))
|
||||
if !ok {
|
||||
dockerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unrecognised registry path")
|
||||
return
|
||||
}
|
||||
switch req.kind {
|
||||
case "upload":
|
||||
h.dockerFinishUpload(w, r, remote, req)
|
||||
case "manifest":
|
||||
h.dockerPutManifest(w, r, remote, req)
|
||||
default:
|
||||
dockerError(w, http.StatusMethodNotAllowed, "UNSUPPORTED", "PUT not supported for this path")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) dockerDelete(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "remoteName")
|
||||
remote, ok := h.localDockerRemote(r, name)
|
||||
if !ok {
|
||||
dockerError(w, http.StatusMethodNotAllowed, "UNSUPPORTED", "delete is only supported for local docker repositories")
|
||||
return
|
||||
}
|
||||
req, ok := parseDockerPath(chi.URLParam(r, "*"))
|
||||
if !ok {
|
||||
dockerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unrecognised registry path")
|
||||
return
|
||||
}
|
||||
// Cancel an in-progress upload: drop its staging object.
|
||||
if req.kind == "upload" && req.ref != "" {
|
||||
_ = h.store.Delete(r.Context(), uploadKey(req.ref))
|
||||
w.Header().Set("Docker-Distribution-Api-Version", dockerAPIVersionHeader)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
if req.kind != "manifest" && req.kind != "blob" {
|
||||
dockerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unrecognised registry path")
|
||||
return
|
||||
}
|
||||
filePath := req.image + "/" + req.kind + "s/" + req.ref
|
||||
if err := h.db.DeleteLocalFile(r.Context(), remote.Name, filePath); err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Docker-Distribution-Api-Version", dockerAPIVersionHeader)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}
|
||||
|
||||
// dockerLocalGet serves manifest / blob / tags-list reads for a local repo.
|
||||
func (h *ProxyHandler) dockerLocalGet(w http.ResponseWriter, r *http.Request, remote *models.Remote, head bool) {
|
||||
req, ok := parseDockerPath(chi.URLParam(r, "*"))
|
||||
if !ok {
|
||||
dockerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unrecognised registry path")
|
||||
return
|
||||
}
|
||||
switch req.kind {
|
||||
case "tags":
|
||||
h.dockerTagsList(w, r, remote, req.image)
|
||||
case "manifest":
|
||||
h.dockerServeRef(w, r, remote, req.image+"/manifests/"+req.ref, head, true)
|
||||
case "blob":
|
||||
h.dockerServeRef(w, r, remote, req.image+"/blobs/"+req.ref, head, false)
|
||||
default:
|
||||
dockerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unrecognised registry path")
|
||||
}
|
||||
}
|
||||
|
||||
// dockerServeRef streams the blob backing a local_files path. isManifest
|
||||
// controls only the default content type; the stored blob content type wins.
|
||||
func (h *ProxyHandler) dockerServeRef(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string, head, isManifest bool) {
|
||||
file, err := h.db.GetLocalFile(r.Context(), remote.Name, filePath)
|
||||
if err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
if file == nil {
|
||||
code := "BLOB_UNKNOWN"
|
||||
if isManifest {
|
||||
code = "MANIFEST_UNKNOWN"
|
||||
}
|
||||
dockerError(w, http.StatusNotFound, code, "not found")
|
||||
return
|
||||
}
|
||||
|
||||
s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):])
|
||||
reader, info, err := h.store.Download(r.Context(), s3Key)
|
||||
if err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
contentType := info.ContentType
|
||||
if contentType == "" {
|
||||
if isManifest {
|
||||
contentType = "application/vnd.docker.distribution.manifest.v2+json"
|
||||
} else {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.Header().Set("Docker-Content-Digest", file.ContentHash)
|
||||
w.Header().Set("Docker-Distribution-Api-Version", dockerAPIVersionHeader)
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
if head {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) dockerTagsList(w http.ResponseWriter, r *http.Request, remote *models.Remote, image string) {
|
||||
prefix := image + "/manifests/"
|
||||
files, err := h.db.ListLocalFilesByPrefix(r.Context(), remote.Name, prefix)
|
||||
if err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
tags := []string{}
|
||||
for _, f := range files {
|
||||
ref := strings.TrimPrefix(f.FilePath, prefix)
|
||||
if ref == "" || isDigest(ref) {
|
||||
continue
|
||||
}
|
||||
tags = append(tags, ref)
|
||||
}
|
||||
sort.Strings(tags)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Docker-Distribution-Api-Version", dockerAPIVersionHeader)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, `{"name":%q,"tags":`, remote.Name+"/"+image)
|
||||
writeJSONStringList(w, tags)
|
||||
fmt.Fprint(w, "}")
|
||||
}
|
||||
|
||||
func writeJSONStringList(w io.Writer, items []string) {
|
||||
fmt.Fprint(w, "[")
|
||||
for i, s := range items {
|
||||
if i > 0 {
|
||||
fmt.Fprint(w, ",")
|
||||
}
|
||||
fmt.Fprintf(w, "%q", s)
|
||||
}
|
||||
fmt.Fprint(w, "]")
|
||||
}
|
||||
|
||||
// dockerStartUpload begins a blob upload. It honours a monolithic
|
||||
// POST?digest=... (blob in the POST body) and otherwise opens a chunked
|
||||
// session, returning its Location for the client's PATCH/PUT.
|
||||
func (h *ProxyHandler) dockerStartUpload(w http.ResponseWriter, r *http.Request, remote *models.Remote) {
|
||||
req, ok := parseDockerPath(chi.URLParam(r, "*"))
|
||||
if !ok || req.kind != "upload" {
|
||||
dockerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unrecognised registry path")
|
||||
return
|
||||
}
|
||||
|
||||
if digest := r.URL.Query().Get("digest"); digest != "" {
|
||||
h.dockerCommitBlob(w, r, remote, req.image, digest, r.Body)
|
||||
return
|
||||
}
|
||||
|
||||
// Stage an empty object keyed by the upload UUID; PATCH/PUT append to it.
|
||||
id := uuid.NewString()
|
||||
if err := h.store.Upload(r.Context(), uploadKey(id), bytes.NewReader(nil), 0, "application/octet-stream"); err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
loc := fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", remote.Name, req.image, id)
|
||||
w.Header().Set("Location", loc)
|
||||
w.Header().Set("Docker-Upload-UUID", id)
|
||||
w.Header().Set("Range", "0-0")
|
||||
w.Header().Set("Docker-Distribution-Api-Version", dockerAPIVersionHeader)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) dockerPatchUpload(w http.ResponseWriter, r *http.Request, remote *models.Remote) {
|
||||
req, ok := parseDockerPath(chi.URLParam(r, "*"))
|
||||
if !ok || req.kind != "upload" || req.ref == "" {
|
||||
dockerError(w, http.StatusNotFound, "BLOB_UPLOAD_UNKNOWN", "unknown upload")
|
||||
return
|
||||
}
|
||||
size, err := h.appendUpload(r.Context(), req.ref, r.Body)
|
||||
if err != nil {
|
||||
if errors.Is(err, errUploadUnknown) {
|
||||
dockerError(w, http.StatusNotFound, "BLOB_UPLOAD_UNKNOWN", "unknown upload")
|
||||
return
|
||||
}
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
loc := fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", remote.Name, req.image, req.ref)
|
||||
w.Header().Set("Location", loc)
|
||||
w.Header().Set("Docker-Upload-UUID", req.ref)
|
||||
w.Header().Set("Range", fmt.Sprintf("0-%d", size-1))
|
||||
w.Header().Set("Docker-Distribution-Api-Version", dockerAPIVersionHeader)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}
|
||||
|
||||
// dockerFinishUpload completes a chunked upload: appends any final PUT body,
|
||||
// stores the assembled blob, and verifies its digest.
|
||||
func (h *ProxyHandler) dockerFinishUpload(w http.ResponseWriter, r *http.Request, remote *models.Remote, req dockerReq) {
|
||||
digest := r.URL.Query().Get("digest")
|
||||
if digest == "" {
|
||||
dockerError(w, http.StatusBadRequest, "DIGEST_INVALID", "digest query parameter required")
|
||||
return
|
||||
}
|
||||
if req.ref == "" {
|
||||
// Monolithic PUT with no prior session: body is the whole blob.
|
||||
h.dockerCommitBlob(w, r, remote, req.image, digest, r.Body)
|
||||
return
|
||||
}
|
||||
|
||||
key := uploadKey(req.ref)
|
||||
reader, _, err := h.store.Download(r.Context(), key)
|
||||
if err != nil {
|
||||
dockerError(w, http.StatusNotFound, "BLOB_UPLOAD_UNKNOWN", "unknown upload")
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
// Drop the staging object once we're done, regardless of outcome; a fresh
|
||||
// context so cleanup still runs if the client disconnects.
|
||||
defer h.store.Delete(context.Background(), key)
|
||||
|
||||
// Stream the staged bytes plus any trailing PUT body through the CAS in one
|
||||
// pass — no extra round trip to re-assemble.
|
||||
combined := io.MultiReader(reader, r.Body)
|
||||
h.dockerCommitBlob(w, r, remote, req.image, digest, combined)
|
||||
}
|
||||
|
||||
// dockerCommitBlob stores blob bytes through the CAS, verifies the client's
|
||||
// declared digest, and records the per-image local_files reference.
|
||||
func (h *ProxyHandler) dockerCommitBlob(w http.ResponseWriter, r *http.Request, remote *models.Remote, image, digest string, body io.Reader) {
|
||||
result, err := h.cas.Store(r.Context(), body, "application/octet-stream")
|
||||
if err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", fmt.Sprintf("store failed: %v", err))
|
||||
return
|
||||
}
|
||||
if result.ContentHash != digest {
|
||||
dockerError(w, http.StatusBadRequest, "DIGEST_INVALID", fmt.Sprintf("digest mismatch: got %s, declared %s", result.ContentHash, digest))
|
||||
return
|
||||
}
|
||||
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, "application/octet-stream"); err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
if err := h.db.CreateLocalFile(r.Context(), remote.Name, image+"/blobs/"+digest, result.ContentHash); err != nil && !errors.Is(err, database.ErrAlreadyExists) {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", fmt.Sprintf("/v2/%s/%s/blobs/%s", remote.Name, image, digest))
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
w.Header().Set("Docker-Distribution-Api-Version", dockerAPIVersionHeader)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
|
||||
// dockerPutManifest stores a manifest and points its reference (tag or digest)
|
||||
// at it. Tags are mutable so a re-push moves the tag; digests are immutable.
|
||||
func (h *ProxyHandler) dockerPutManifest(w http.ResponseWriter, r *http.Request, remote *models.Remote, req dockerReq) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "application/vnd.docker.distribution.manifest.v2+json"
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
|
||||
result, err := h.cas.Store(r.Context(), strings.NewReader(string(body)), contentType)
|
||||
if err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", fmt.Sprintf("store failed: %v", err))
|
||||
return
|
||||
}
|
||||
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
// Always addressable by digest (immutable).
|
||||
if err := h.db.CreateLocalFile(r.Context(), remote.Name, req.image+"/manifests/"+digest, result.ContentHash); err != nil && !errors.Is(err, database.ErrAlreadyExists) {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
// If pushed under a tag, (re)point the tag at this manifest.
|
||||
if !isDigest(req.ref) {
|
||||
if err := h.db.UpsertLocalFile(r.Context(), remote.Name, req.image+"/manifests/"+req.ref, result.ContentHash); err != nil {
|
||||
dockerError(w, http.StatusInternalServerError, "UNKNOWN", err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
slog.Info("local docker manifest pushed", "repo", remote.Name, "image", req.image, "ref", req.ref, "digest", digest)
|
||||
w.Header().Set("Location", fmt.Sprintf("/v2/%s/%s/manifests/%s", remote.Name, req.image, req.ref))
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
w.Header().Set("Docker-Distribution-Api-Version", dockerAPIVersionHeader)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package v1
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseDockerPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
rest string
|
||||
wantOK bool
|
||||
wantImage string
|
||||
wantKind string
|
||||
wantRef string
|
||||
}{
|
||||
{"start upload trailing slash", "team/app/blobs/uploads/", true, "team/app", "upload", ""},
|
||||
{"start upload no slash", "team/app/blobs/uploads", true, "team/app", "upload", ""},
|
||||
{"patch upload with uuid", "team/app/blobs/uploads/abc-123", true, "team/app", "upload", "abc-123"},
|
||||
{"single-segment image upload", "app/blobs/uploads/", true, "app", "upload", ""},
|
||||
{"blob by digest", "team/app/blobs/sha256:deadbeef", true, "team/app", "blob", "sha256:deadbeef"},
|
||||
{"manifest by tag", "team/app/manifests/v1.0.0", true, "team/app", "manifest", "v1.0.0"},
|
||||
{"manifest by digest", "team/app/manifests/sha256:cafe", true, "team/app", "manifest", "sha256:cafe"},
|
||||
{"tags list", "team/app/tags/list", true, "team/app", "tags", ""},
|
||||
{"leading slash tolerated", "/team/app/manifests/latest", true, "team/app", "manifest", "latest"},
|
||||
{"deep image name", "a/b/c/manifests/latest", true, "a/b/c", "manifest", "latest"},
|
||||
{"unrecognised", "team/app/whatever", false, "", "", ""},
|
||||
{"tags list without image", "tags/list", false, "", "", ""},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, ok := parseDockerPath(tc.rest)
|
||||
if ok != tc.wantOK {
|
||||
t.Fatalf("ok = %v, want %v", ok, tc.wantOK)
|
||||
}
|
||||
if !tc.wantOK {
|
||||
return
|
||||
}
|
||||
if got.image != tc.wantImage || got.kind != tc.wantKind || got.ref != tc.wantRef {
|
||||
t.Fatalf("got %+v, want image=%q kind=%q ref=%q", got, tc.wantImage, tc.wantKind, tc.wantRef)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsDigest(t *testing.T) {
|
||||
if !isDigest("sha256:abc") {
|
||||
t.Fatal("sha256: prefix should be a digest")
|
||||
}
|
||||
if isDigest("v1.0.0") {
|
||||
t.Fatal("a tag is not a digest")
|
||||
}
|
||||
}
|
||||
@@ -23,18 +23,10 @@ type ProxyHandler struct {
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
local *v2.LocalHandler
|
||||
cas *storage.CAS
|
||||
}
|
||||
|
||||
func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *database.DB, store *storage.S3, local *v2.LocalHandler) *ProxyHandler {
|
||||
return &ProxyHandler{
|
||||
engine: engine,
|
||||
virtualEngine: virtualEngine,
|
||||
db: db,
|
||||
store: store,
|
||||
local: local,
|
||||
cas: storage.NewCAS(store),
|
||||
}
|
||||
return &ProxyHandler{engine: engine, virtualEngine: virtualEngine, db: db, store: store, local: local}
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) Routes() chi.Router {
|
||||
@@ -45,20 +37,12 @@ func (h *ProxyHandler) Routes() chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// DockerV2Routes mounts the Docker Registry HTTP API V2. Reads (GET/HEAD)
|
||||
// dispatch to a local registry implementation for local docker repos and fall
|
||||
// through to the upstream proxy otherwise; writes (POST/PATCH/PUT/DELETE) are
|
||||
// only valid for local docker repos and drive push.
|
||||
func (h *ProxyHandler) DockerV2Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.handleDockerPing)
|
||||
r.Head("/", h.handleDockerPing)
|
||||
r.Get("/{remoteName}/*", h.dockerGet)
|
||||
r.Head("/{remoteName}/*", h.dockerHead)
|
||||
r.Post("/{remoteName}/*", h.dockerPost)
|
||||
r.Patch("/{remoteName}/*", h.dockerPatch)
|
||||
r.Put("/{remoteName}/*", h.dockerPut)
|
||||
r.Delete("/{remoteName}/*", h.dockerDelete)
|
||||
r.Get("/{remoteName}/*", h.handleProxy)
|
||||
r.Head("/{remoteName}/*", h.handleProxy)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -83,15 +67,6 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Metadata-only remotes (e.g. github_rpm) synthesize their own responses and
|
||||
// redirect package downloads to a backend remote instead of proxying bytes.
|
||||
if rs, ok := prov.(provider.RemoteServer); ok {
|
||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||
if rs.ServeRemote(w, r, *remote, path, proxyBaseURL, h.db) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
@@ -114,42 +89,6 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
io.Copy(w, result.Reader)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.engine.Head(r.Context(), *remote, path, prov)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
||||
virtualName := chi.URLParam(r, "virtualName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestScheme(t *testing.T) {
|
||||
if got := scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
|
||||
t.Errorf("TLS request scheme = %q, want https", got)
|
||||
}
|
||||
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {"https"}}}
|
||||
if got := scheme(r); got != "https" {
|
||||
t.Errorf("X-Forwarded-Proto scheme = %q, want https", got)
|
||||
}
|
||||
if got := scheme(&http.Request{Header: http.Header{}}); got != "http" {
|
||||
t.Errorf("default scheme = %q, want http", got)
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
var testDSN string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, terminate, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN = dsn
|
||||
code := m.Run()
|
||||
terminate()
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
// closedDB returns a DB whose pool has been closed, so every query fails —
|
||||
// used to drive the handlers' error branches.
|
||||
func closedDB(t *testing.T) *database.DB {
|
||||
t.Helper()
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatalf("new db: %v", err)
|
||||
}
|
||||
db.Close()
|
||||
return db
|
||||
}
|
||||
|
||||
func do(t *testing.T, h http.Handler, method, path, body string) int {
|
||||
t.Helper()
|
||||
var r io.Reader
|
||||
if body != "" {
|
||||
r = strings.NewReader(body)
|
||||
}
|
||||
req := httptest.NewRequest(method, path, r)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
return w.Code
|
||||
}
|
||||
|
||||
func TestRemotesErrorPaths(t *testing.T) {
|
||||
h := NewRemotesHandler(closedDB(t), nil, nil).Routes()
|
||||
if c := do(t, h, "GET", "/", ""); c != 500 {
|
||||
t.Errorf("list with dead db = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "POST", "/", `{"name":"x","package_type":"generic","repo_type":"remote","base_url":"https://x"}`); c != 500 {
|
||||
t.Errorf("create with dead db = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "PUT", "/x", `{"package_type":"generic","base_url":"https://x"}`); c != 500 {
|
||||
t.Errorf("update with dead db = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "GET", "/x", ""); c != 404 {
|
||||
t.Errorf("get missing = %d, want 404", c)
|
||||
}
|
||||
if c := do(t, h, "DELETE", "/x", ""); c != 500 {
|
||||
t.Errorf("delete with dead db = %d, want 500", c)
|
||||
}
|
||||
// Bad request bodies never reach the db.
|
||||
if c := do(t, h, "POST", "/", `not json`); c != 400 {
|
||||
t.Errorf("invalid json = %d, want 400", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualsErrorPaths(t *testing.T) {
|
||||
h := NewVirtualsHandler(closedDB(t)).Routes()
|
||||
if c := do(t, h, "GET", "/", ""); c != 500 {
|
||||
t.Errorf("list = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "GET", "/x", ""); c != 404 {
|
||||
t.Errorf("get missing = %d, want 404", c)
|
||||
}
|
||||
if c := do(t, h, "POST", "/", `{"name":"v","package_type":"helm","members":["a"]}`); c != 500 {
|
||||
t.Errorf("create = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "PUT", "/v", `{"package_type":"helm","members":["a"]}`); c != 500 {
|
||||
t.Errorf("update = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "DELETE", "/v", ""); c != 500 {
|
||||
t.Errorf("delete = %d, want 500", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsErrorPaths(t *testing.T) {
|
||||
h := NewStatsHandler(closedDB(t)).Routes()
|
||||
for _, p := range []string{"/", "/top-remotes", "/top-files-by-hits", "/top-files-by-bandwidth"} {
|
||||
if c := do(t, h, "GET", p, ""); c != 500 {
|
||||
t.Errorf("stats %s = %d, want 500", p, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalErrorPaths(t *testing.T) {
|
||||
h := NewLocalHandler(closedDB(t), nil).Routes()
|
||||
// GetRemote fails on the closed db -> not found.
|
||||
if c := do(t, h, "PUT", "/x/files/a.bin", "data"); c != 404 {
|
||||
t.Errorf("upload unknown repo = %d, want 404", c)
|
||||
}
|
||||
// download / remove hit the db and 500.
|
||||
if c := do(t, h, "GET", "/x/files/a.bin", ""); c != 500 {
|
||||
t.Errorf("download = %d, want 500", c)
|
||||
}
|
||||
if c := do(t, h, "DELETE", "/x/files/a.bin", ""); c != 500 {
|
||||
t.Errorf("remove = %d, want 500", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalHandlerDBAccessor(t *testing.T) {
|
||||
db := closedDB(t)
|
||||
if NewLocalHandler(db, nil).DB() != db {
|
||||
t.Error("DB() should return the handler's database")
|
||||
}
|
||||
}
|
||||
@@ -185,35 +185,13 @@ func (h *LocalHandler) remove(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
if err := deleteLocalFile(r.Context(), h.db, repoName, filePath); err != nil {
|
||||
if err := h.db.DeleteLocalFile(r.Context(), repoName, filePath); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// deleteLocalFile removes a local file and runs the provider's post-delete hook,
|
||||
// so provider-derived state (e.g. RPM metadata that feeds generated repodata)
|
||||
// stops referencing a package that no longer exists.
|
||||
func deleteLocalFile(ctx context.Context, db *database.DB, repoName, filePath string) error {
|
||||
if err := db.DeleteLocalFile(ctx, repoName, filePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
remote, err := db.GetRemote(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil // file is gone; no repo left to resolve a cleanup hook from
|
||||
}
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if hook, ok := prov.(provider.PostDeleteHook); ok {
|
||||
return hook.AfterDelete(ctx, repoName, filePath, db)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *LocalHandler) DB() *database.DB {
|
||||
return h.db
|
||||
}
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/rpm" // register the rpm provider so its PostDeleteHook runs
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestLocalEvictCleansRPMMetadata verifies that evicting an RPM from a local
|
||||
// repo also removes the derived rpm_metadata row, so generated repodata stops
|
||||
// listing the deleted package.
|
||||
func TestLocalEvictCleansRPMMetadata(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const repo = "rpm-evict-cleanup"
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageRPM, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const hash = "sha256:bb22"
|
||||
const path = "Packages/example-0.1.0-1.x86_64.rpm"
|
||||
if err := db.UpsertBlob(ctx, hash, "blobs/bb/22", 2048, "application/x-rpm"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.CreateLocalFile(ctx, repo, path, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.InsertRPMMetadata(ctx, &provider.RPMMetadata{
|
||||
RepoName: repo, FilePath: path, ContentHash: hash,
|
||||
Name: "example", Version: "0.1.0", Release: "1", Arch: "x86_64",
|
||||
Requires: []provider.RPMDep{}, Provides: []provider.RPMDep{},
|
||||
Files: []provider.RPMFile{}, Changelogs: []provider.RPMChangelog{},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewObjectsHandler(db)
|
||||
router := chi.NewRouter()
|
||||
router.Route("/locals/{name}/objects", func(r chi.Router) {
|
||||
r.Delete("/*", h.LocalRoutes().ServeHTTP)
|
||||
})
|
||||
|
||||
del := httptest.NewRequest("DELETE", "/locals/"+repo+"/objects/"+path, nil)
|
||||
dw := httptest.NewRecorder()
|
||||
router.ServeHTTP(dw, del)
|
||||
if dw.Code != 204 {
|
||||
t.Fatalf("evict = %d, want 204", dw.Code)
|
||||
}
|
||||
|
||||
if f, _ := db.GetLocalFile(ctx, repo, path); f != nil {
|
||||
t.Fatalf("local file still present after evict: %+v", f)
|
||||
}
|
||||
entries, err := db.ListRPMMetadataEntries(ctx, repo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("rpm_metadata still present after evict: %+v", entries)
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestLocalUploadStoreFailure covers the upload handlers' store-error branches
|
||||
// by killing the object store after a successful upload.
|
||||
func TestLocalUploadStoreFailure(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
conn, termMinio, err := testsupport.StartMinio(ctx)
|
||||
if err != nil {
|
||||
t.Skip("minio unavailable")
|
||||
}
|
||||
var store *storage.S3
|
||||
for i := 0; i < 20; i++ {
|
||||
if store, err = storage.NewS3(conn.Endpoint, conn.AccessKey, conn.SecretKey, "fault", false, ""); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
termMinio()
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, pt := range []models.PackageType{models.PackageGeneric, models.PackagePyPI} {
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: "fault-" + string(pt), PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
h := NewLocalHandler(db, store)
|
||||
router := chi.NewRouter()
|
||||
router.Route("/remotes/{name}/files", func(r chi.Router) {
|
||||
r.Put("/*", h.Routes().ServeHTTP)
|
||||
})
|
||||
srv := httptest.NewServer(router)
|
||||
defer srv.Close()
|
||||
|
||||
put := func(name, path, body string) int {
|
||||
rq, _ := http.NewRequest("PUT", srv.URL+"/remotes/"+name+"/files/"+path, strings.NewReader(body))
|
||||
resp, err := http.DefaultClient.Do(rq)
|
||||
if err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
// Sanity: uploads succeed while the store is up.
|
||||
if c := put("fault-generic", "ok.bin", "data"); c != 201 {
|
||||
t.Fatalf("generic upload while up = %d", c)
|
||||
}
|
||||
if c := put("fault-pypi", "foo-1.0-py3-none-any.whl", "wheel"); c != 201 {
|
||||
t.Fatalf("pypi upload while up = %d", c)
|
||||
}
|
||||
|
||||
// Kill the store; subsequent CAS.Store calls fail -> 500.
|
||||
termMinio()
|
||||
if c := put("fault-generic", "after.bin", "data"); c != 500 {
|
||||
t.Errorf("generic upload after store down = %d, want 500", c)
|
||||
}
|
||||
if c := put("fault-pypi", "bar-1.0-py3-none-any.whl", "wheel"); c != 500 {
|
||||
t.Errorf("pypi upload after store down = %d, want 500", c)
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestLocalObjectsListing verifies that files uploaded to a local repo (which
|
||||
// live in local_files, not artifacts) are listed by the local objects endpoint
|
||||
// and can be evicted through it.
|
||||
func TestLocalObjectsListing(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const repo = "rpm-local-objs"
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageRPM, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const hash = "sha256:aa11"
|
||||
const path = "Packages/example-0.1.0-1.x86_64.rpm"
|
||||
if err := db.UpsertBlob(ctx, hash, "blobs/aa/11", 1234, "application/x-rpm"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.CreateLocalFile(ctx, repo, path, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewObjectsHandler(db)
|
||||
router := chi.NewRouter()
|
||||
router.Route("/locals/{name}/objects", func(r chi.Router) {
|
||||
r.Get("/", h.LocalRoutes().ServeHTTP)
|
||||
r.Delete("/*", h.LocalRoutes().ServeHTTP)
|
||||
})
|
||||
|
||||
// The uploaded package must appear in the listing with its blob size.
|
||||
req := httptest.NewRequest("GET", "/locals/"+repo+"/objects", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("list = %d, want 200", w.Code)
|
||||
}
|
||||
var got []models.Artifact
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d objects, want 1", len(got))
|
||||
}
|
||||
if got[0].Path != path || got[0].SizeBytes != 1234 || got[0].ContentHash != hash {
|
||||
t.Fatalf("unexpected object: %+v", got[0])
|
||||
}
|
||||
|
||||
// Eviction removes it from local_files.
|
||||
del := httptest.NewRequest("DELETE", "/locals/"+repo+"/objects/"+path, nil)
|
||||
dw := httptest.NewRecorder()
|
||||
router.ServeHTTP(dw, del)
|
||||
if dw.Code != 204 {
|
||||
t.Fatalf("evict = %d, want 204", dw.Code)
|
||||
}
|
||||
if f, _ := db.GetLocalFile(ctx, repo, path); f != nil {
|
||||
t.Fatalf("file still present after evict: %+v", f)
|
||||
}
|
||||
}
|
||||
@@ -25,18 +25,9 @@ func (h *ObjectsHandler) Routes() chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// LocalRoutes lists and evicts objects for local repos, which live in the
|
||||
// local_files table rather than the artifacts table used by remotes.
|
||||
func (h *ObjectsHandler) LocalRoutes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.listLocal)
|
||||
r.Delete("/*", h.evictLocal)
|
||||
return r
|
||||
}
|
||||
|
||||
// pageBounds parses the shared page/per_page query params into a SQL limit and offset.
|
||||
func pageBounds(r *http.Request) (limit, offset int) {
|
||||
limit, _ = strconv.Atoi(r.URL.Query().Get("per_page"))
|
||||
func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("per_page"))
|
||||
if limit <= 0 || limit > 5000 {
|
||||
limit = 50
|
||||
}
|
||||
@@ -44,12 +35,7 @@ func pageBounds(r *http.Request) (limit, offset int) {
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
return limit, (page - 1) * limit
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
limit, offset := pageBounds(r)
|
||||
offset := (page - 1) * limit
|
||||
|
||||
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
|
||||
if err != nil {
|
||||
@@ -59,29 +45,6 @@ func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, artifacts)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) listLocal(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
limit, offset := pageBounds(r)
|
||||
|
||||
artifacts, err := h.db.ListLocalArtifacts(r.Context(), repoName, limit, offset)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, artifacts)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) evictLocal(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
if err := deleteLocalFile(r.Context(), h.db, repoName, path); err != nil {
|
||||
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@@ -13,29 +11,12 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// Primer enqueues a background metadata prime for a newly created remote so the
|
||||
// create call never blocks on a derive. *rpm.Syncer and *deb.Syncer satisfy it.
|
||||
type Primer interface {
|
||||
EnqueuePrime(remote models.Remote)
|
||||
}
|
||||
|
||||
// MetadataFlusher purges a remote's cached mutable metadata (repodata / Release
|
||||
// / APKINDEX freshness keys). *cache.Redis satisfies it.
|
||||
type MetadataFlusher interface {
|
||||
FlushRemote(ctx context.Context, remote string) error
|
||||
}
|
||||
|
||||
type RemotesHandler struct {
|
||||
db *database.DB
|
||||
cache MetadataFlusher
|
||||
primers map[models.PackageType]Primer
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
// NewRemotesHandler wires the handler to the metadata cache and per-type
|
||||
// primers. cache may be nil (flush-on-backend-change is skipped); primers may
|
||||
// be nil (a package type with no registered primer simply skips priming).
|
||||
func NewRemotesHandler(db *database.DB, cache MetadataFlusher, primers map[models.PackageType]Primer) *RemotesHandler {
|
||||
return &RemotesHandler{db: db, cache: cache, primers: primers}
|
||||
func NewRemotesHandler(db *database.DB) *RemotesHandler {
|
||||
return &RemotesHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) Routes() chi.Router {
|
||||
@@ -88,27 +69,10 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidateMirrorlist(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidateMirrorStrategy(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.CreateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Prime a metadata-only remote (github_rpm/github_deb) in the background so
|
||||
// its first index request is served from cache instead of a cold derive.
|
||||
if primer := h.primers[remote.PackageType]; primer != nil {
|
||||
primer.EnqueuePrime(remote)
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, remote)
|
||||
}
|
||||
|
||||
@@ -120,42 +84,10 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
remote.Name = name
|
||||
if err := remote.ValidateMirrorlist(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidateMirrorStrategy(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// Capture the current backend before the update so we can tell whether the
|
||||
// remote's base_url (its upstream) changed. A read failure just means we
|
||||
// skip the freshness flush; it must not block the update.
|
||||
oldBaseURL, oldKnown := "", false
|
||||
if existing, err := h.db.GetRemote(r.Context(), name); err == nil {
|
||||
oldBaseURL, oldKnown = existing.BaseURL, true
|
||||
}
|
||||
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Changing the backend invalidates any cached mutable metadata (repodata /
|
||||
// Release / APKINDEX): purge it so the next request re-fetches from the new
|
||||
// upstream instead of serving stale data until TTL expiry. A flush failure
|
||||
// is logged but does not fail the request — the DB update already landed.
|
||||
if oldKnown && oldBaseURL != remote.BaseURL && h.cache != nil {
|
||||
if err := h.cache.FlushRemote(r.Context(), name); err != nil {
|
||||
slog.Warn("flush cached metadata after base_url change failed",
|
||||
"remote", name, "error", err)
|
||||
} else {
|
||||
slog.Info("flushed cached metadata after base_url change",
|
||||
"remote", name, "old_base_url", oldBaseURL, "new_base_url", remote.BaseURL)
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeFlusher records FlushRemote calls so a test can assert whether — and how
|
||||
// often — a remote's cached metadata was purged.
|
||||
type fakeFlusher struct {
|
||||
calls []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeFlusher) FlushRemote(_ context.Context, remote string) error {
|
||||
f.calls = append(f.calls, remote)
|
||||
return f.err
|
||||
}
|
||||
|
||||
func seedRemote(t *testing.T, db *database.DB, name, baseURL string) {
|
||||
t.Helper()
|
||||
err := db.CreateRemote(context.Background(), &models.Remote{
|
||||
Name: name,
|
||||
PackageType: models.PackageRPM,
|
||||
RepoType: models.RepoTypeRemote,
|
||||
BaseURL: baseURL,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("seed remote: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A base_url change must flush the remote's cached metadata exactly once, while
|
||||
// an update that leaves base_url untouched must not flush at all.
|
||||
func TestUpdateFlushesCacheOnBaseURLChange(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const name = "rpm-flush-change"
|
||||
seedRemote(t, db, name, "https://old.example.com/repo")
|
||||
|
||||
ff := &fakeFlusher{}
|
||||
h := NewRemotesHandler(db, ff, nil).Routes()
|
||||
|
||||
if c := do(t, h, "PUT", "/"+name, `{"package_type":"rpm","repo_type":"remote","base_url":"https://new.example.com/repo"}`); c != 200 {
|
||||
t.Fatalf("update (backend change) = %d, want 200", c)
|
||||
}
|
||||
if len(ff.calls) != 1 || ff.calls[0] != name {
|
||||
t.Fatalf("flush calls = %v, want exactly one flush of %q", ff.calls, name)
|
||||
}
|
||||
|
||||
// Re-updating with the same (now current) base_url must not flush again.
|
||||
ff.calls = nil
|
||||
if c := do(t, h, "PUT", "/"+name, `{"package_type":"rpm","repo_type":"remote","base_url":"https://new.example.com/repo"}`); c != 200 {
|
||||
t.Fatalf("update (no backend change) = %d, want 200", c)
|
||||
}
|
||||
if len(ff.calls) != 0 {
|
||||
t.Fatalf("flush calls = %v, want no flush when base_url is unchanged", ff.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A flush error must be swallowed: the DB update already succeeded, so the
|
||||
// request still returns 200.
|
||||
func TestUpdateFlushFailureStillSucceeds(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const name = "rpm-flush-error"
|
||||
seedRemote(t, db, name, "https://old.example.com/repo")
|
||||
|
||||
ff := &fakeFlusher{err: errors.New("redis down")}
|
||||
h := NewRemotesHandler(db, ff, nil).Routes()
|
||||
|
||||
if c := do(t, h, "PUT", "/"+name, `{"package_type":"rpm","repo_type":"remote","base_url":"https://new.example.com/repo"}`); c != 200 {
|
||||
t.Fatalf("update with failing flush = %d, want 200", c)
|
||||
}
|
||||
if len(ff.calls) != 1 {
|
||||
t.Fatalf("flush calls = %v, want exactly one attempted flush", ff.calls)
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestBasicHeaders(t *testing.T) {
|
||||
h := BasicHeaders(models.Remote{Username: "alice", Password: "secret"})
|
||||
got := h.Get("Authorization")
|
||||
want := "Basic " + base64.StdEncoding.EncodeToString([]byte("alice:secret"))
|
||||
if got != want {
|
||||
t.Errorf("Authorization = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBasicHeadersNoUser(t *testing.T) {
|
||||
if h := BasicHeaders(models.Remote{}); h.Get("Authorization") != "" {
|
||||
t.Error("expected no Authorization header without a username")
|
||||
}
|
||||
}
|
||||
Vendored
-133
@@ -1,133 +0,0 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
var testRedis *Redis
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
url, terminate, err := testsupport.StartRedis(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
r, err := NewRedis(url)
|
||||
if err != nil {
|
||||
terminate()
|
||||
panic(err)
|
||||
}
|
||||
testRedis = r
|
||||
code := m.Run()
|
||||
r.Close()
|
||||
terminate()
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
func requireRedis(t *testing.T) {
|
||||
t.Helper()
|
||||
if testRedis == nil {
|
||||
t.Skip("Docker unavailable; skipping cache integration test")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRedisInvalid(t *testing.T) {
|
||||
if _, err := NewRedis("://bad-url"); err == nil {
|
||||
t.Error("expected error for invalid redis URL")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTTL(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
if fresh, _ := testRedis.CheckTTL(ctx, "r", "missing"); fresh {
|
||||
t.Error("missing key should not be fresh")
|
||||
}
|
||||
if err := testRedis.SetTTL(ctx, "r", "p", time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fresh, err := testRedis.CheckTTL(ctx, "r", "p"); err != nil || !fresh {
|
||||
t.Errorf("expected fresh after SetTTL: %v %v", fresh, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLock(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
ok, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("first acquire should succeed: %v %v", ok, err)
|
||||
}
|
||||
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); ok {
|
||||
t.Error("second acquire should fail while held")
|
||||
}
|
||||
if err := testRedis.ReleaseLock(ctx, "r", "lockpath"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); !ok {
|
||||
t.Error("acquire should succeed after release")
|
||||
}
|
||||
}
|
||||
|
||||
func TestETagAndToken(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
if v, _ := testRedis.GetETag(ctx, "r", "missing"); v != "" {
|
||||
t.Error("missing etag should be empty")
|
||||
}
|
||||
testRedis.SetETag(ctx, "r", "p", `"abc"`, time.Minute)
|
||||
if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` {
|
||||
t.Errorf("etag = %q", v)
|
||||
}
|
||||
|
||||
if v, _ := testRedis.GetToken(ctx, "missing"); v != "" {
|
||||
t.Error("missing token should be empty")
|
||||
}
|
||||
testRedis.SetToken(ctx, "key", "tok", time.Minute)
|
||||
if v, _ := testRedis.GetToken(ctx, "key"); v != "tok" {
|
||||
t.Errorf("token = %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCircuit(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 0 {
|
||||
t.Errorf("initial failures = %d", n)
|
||||
}
|
||||
n1, err := testRedis.IncrCircuitFailure(ctx, "cr", time.Minute)
|
||||
if err != nil || n1 != 1 {
|
||||
t.Fatalf("first incr = %d %v", n1, err)
|
||||
}
|
||||
n2, _ := testRedis.IncrCircuitFailure(ctx, "cr", time.Minute)
|
||||
if n2 != 2 {
|
||||
t.Errorf("second incr = %d", n2)
|
||||
}
|
||||
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 2 {
|
||||
t.Errorf("get failures = %d", n)
|
||||
}
|
||||
testRedis.ResetCircuit(ctx, "cr")
|
||||
if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 0 {
|
||||
t.Errorf("failures after reset = %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushRemote(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
testRedis.SetTTL(ctx, "flushme", "a", time.Hour)
|
||||
testRedis.SetETag(ctx, "flushme", "a", "x", time.Hour)
|
||||
if err := testRedis.FlushRemote(ctx, "flushme"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fresh, _ := testRedis.CheckTTL(ctx, "flushme", "a"); fresh {
|
||||
t.Error("expected keys flushed")
|
||||
}
|
||||
}
|
||||
Vendored
-12
@@ -70,18 +70,6 @@ func (r *Redis) GetETag(ctx context.Context, remote, path string) (string, error
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) GetToken(ctx context.Context, key string) (string, error) {
|
||||
val, err := r.client.Get(ctx, "token:"+key).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) SetToken(ctx context.Context, key, token string, ttl time.Duration) error {
|
||||
return r.client.Set(ctx, "token:"+key, token, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) IncrCircuitFailure(ctx context.Context, remote string, cooldown time.Duration) (int64, error) {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
pipe := r.client.Pipeline()
|
||||
|
||||
+1
-65
@@ -24,38 +24,6 @@ type Config struct {
|
||||
S3Bucket string
|
||||
S3Secure bool
|
||||
S3Region string
|
||||
|
||||
// Terraform provider registry signing. When TFSigningKeyPath points at a
|
||||
// readable armored GPG private key, artifactapi serves local terraform
|
||||
// repos as a real provider registry (service discovery + signed
|
||||
// SHA256SUMS). Left empty, the registry endpoints stay disabled.
|
||||
TFSigningKeyPath string
|
||||
TFSigningKeyPassphrase string
|
||||
TFProviderProtocols string
|
||||
|
||||
// github_rpm background syncer. The syncer keeps derived RPM metadata for
|
||||
// every github_rpm remote fresh off the client request path, sharing a
|
||||
// single global token-bucket limiter across all remotes so GitHub is never
|
||||
// hammered. Defaults are conservative: 1 req/s (3600/hr) sits well under an
|
||||
// authenticated token's 5000/hr. Unauthenticated remotes (60/hr) lean on
|
||||
// ETag/304 — an unchanged repo costs nothing — so keep those repos small or
|
||||
// configure a token.
|
||||
GitHubSyncRatePerSec float64
|
||||
GitHubSyncBurst int
|
||||
GitHubSyncWorkers int
|
||||
GitHubSyncPollInterval int
|
||||
|
||||
// Server-level GitHub machine credential, applied by default to every
|
||||
// outbound GitHub request (releases scan, ranged asset fetches, and the
|
||||
// generic-github byte proxy for private assets). Delivered via env/secret
|
||||
// only — never stored per-remote, never returned by an API, never logged.
|
||||
// Configure exactly one mode: a Personal Access Token, or a GitHub App
|
||||
// (id + installation id + private key). Partial App config fails at startup.
|
||||
GitHubToken string
|
||||
GitHubAppID string
|
||||
GitHubAppInstallationID string
|
||||
GitHubAppPrivateKey string
|
||||
GitHubAppPrivateKeyPath string
|
||||
}
|
||||
|
||||
func (c *Config) DatabaseDSN() string {
|
||||
@@ -73,23 +41,6 @@ func Load() (*Config, error) {
|
||||
|
||||
s3Secure, _ := strconv.ParseBool(getenv("MINIO_SECURE", "false"))
|
||||
|
||||
syncRate, err := strconv.ParseFloat(getenv("GITHUB_SYNC_RATE", "1"), 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid GITHUB_SYNC_RATE: %w", err)
|
||||
}
|
||||
syncBurst, err := strconv.Atoi(getenv("GITHUB_SYNC_BURST", "5"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid GITHUB_SYNC_BURST: %w", err)
|
||||
}
|
||||
syncWorkers, err := strconv.Atoi(getenv("GITHUB_SYNC_WORKERS", "3"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid GITHUB_SYNC_WORKERS: %w", err)
|
||||
}
|
||||
syncPoll, err := strconv.Atoi(getenv("GITHUB_SYNC_POLL_INTERVAL", "60"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid GITHUB_SYNC_POLL_INTERVAL: %w", err)
|
||||
}
|
||||
|
||||
cfg := &Config{
|
||||
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
|
||||
|
||||
@@ -108,28 +59,13 @@ func Load() (*Config, error) {
|
||||
S3Bucket: getenv("MINIO_BUCKET", "artifacts"),
|
||||
S3Secure: s3Secure,
|
||||
S3Region: getenv("MINIO_REGION", ""),
|
||||
|
||||
TFSigningKeyPath: getenv("TF_SIGNING_KEY_PATH", ""),
|
||||
TFSigningKeyPassphrase: getenv("TF_SIGNING_KEY_PASSPHRASE", ""),
|
||||
TFProviderProtocols: getenv("TF_PROVIDER_PROTOCOLS", "5.0,6.0"),
|
||||
|
||||
GitHubSyncRatePerSec: syncRate,
|
||||
GitHubSyncBurst: syncBurst,
|
||||
GitHubSyncWorkers: syncWorkers,
|
||||
GitHubSyncPollInterval: syncPoll,
|
||||
|
||||
GitHubToken: getenv("GITHUB_TOKEN", ""),
|
||||
GitHubAppID: getenv("GITHUB_APP_ID", ""),
|
||||
GitHubAppInstallationID: getenv("GITHUB_APP_INSTALLATION_ID", ""),
|
||||
GitHubAppPrivateKey: getenv("GITHUB_APP_PRIVATE_KEY", ""),
|
||||
GitHubAppPrivateKeyPath: getenv("GITHUB_APP_PRIVATE_KEY_PATH", ""),
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadDefaults(t *testing.T) {
|
||||
// Unset the vars Load reads so the fallback defaults are exercised.
|
||||
for _, k := range []string{
|
||||
"LISTEN_ADDR", "DBHOST", "DBPORT", "DBUSER", "DBPASS", "DBNAME", "DBSSL",
|
||||
"REDIS_URL", "MINIO_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY",
|
||||
"MINIO_BUCKET", "MINIO_SECURE", "MINIO_REGION",
|
||||
} {
|
||||
old, ok := os.LookupEnv(k)
|
||||
os.Unsetenv(k)
|
||||
if ok {
|
||||
t.Cleanup(func() { os.Setenv(k, old) })
|
||||
}
|
||||
}
|
||||
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if cfg.ListenAddr != ":8000" || cfg.DBPort != 5432 || cfg.DBUser != "artifacts" {
|
||||
t.Errorf("unexpected defaults: %+v", cfg)
|
||||
}
|
||||
if cfg.RedisURL != "redis://localhost:6379" || cfg.S3Bucket != "artifacts" || cfg.S3Secure {
|
||||
t.Errorf("unexpected defaults: %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadOverrides(t *testing.T) {
|
||||
t.Setenv("LISTEN_ADDR", ":9999")
|
||||
t.Setenv("DBHOST", "db.example.com")
|
||||
t.Setenv("DBPORT", "6000")
|
||||
t.Setenv("DBUSER", "u")
|
||||
t.Setenv("DBPASS", "pw")
|
||||
t.Setenv("DBNAME", "n")
|
||||
t.Setenv("DBSSL", "require")
|
||||
t.Setenv("MINIO_SECURE", "true")
|
||||
t.Setenv("MINIO_REGION", "us-east-1")
|
||||
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if cfg.ListenAddr != ":9999" || cfg.DBHost != "db.example.com" || cfg.DBPort != 6000 {
|
||||
t.Errorf("overrides not applied: %+v", cfg)
|
||||
}
|
||||
if !cfg.S3Secure {
|
||||
t.Error("MINIO_SECURE=true not parsed")
|
||||
}
|
||||
want := "postgres://u:pw@db.example.com:6000/n?sslmode=require"
|
||||
if got := cfg.DatabaseDSN(); got != want {
|
||||
t.Errorf("DSN = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadInvalidPort(t *testing.T) {
|
||||
t.Setenv("DBPORT", "not-a-number")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Error("expected error for invalid DBPORT")
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// ListGitHubAlpineRemotes returns every github_alpine remote so the syncer can
|
||||
// sweep them on each poll tick.
|
||||
func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes WHERE package_type = $1 ORDER BY name`, models.PackageGitHubAlpine)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var remotes []models.Remote
|
||||
for rows.Next() {
|
||||
var r models.Remote
|
||||
if err := scanRemote(rows, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remotes = append(remotes, r)
|
||||
}
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubAlpineSyncLease atomically claims the per-remote sync lease. It
|
||||
// succeeds only when the remote is due (never synced, or synced longer than
|
||||
// freshness ago) and no live lease is held by another replica. A zero freshness
|
||||
// (prime scans) ignores the recency gate. The returned etag is the stored
|
||||
// releases-list ETag, shared across replicas.
|
||||
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO github_alpine_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
|
||||
var etag string
|
||||
if err := row.Scan(&etag); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, "", nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
return true, etag, nil
|
||||
}
|
||||
|
||||
// ReleaseGitHubAlpineSyncLease records the completed scan and frees the lease.
|
||||
// Only the owning replica may release; last_synced_at advances so the next poll
|
||||
// waits a full freshness window, and etag is persisted for the next conditional
|
||||
// request.
|
||||
func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE github_alpine_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
return err
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertAlpineMetadata(ctx context.Context, meta *provider.AlpineMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO alpine_metadata (
|
||||
repo_name, file_path, content_hash, checksum,
|
||||
name, version, arch, download_size, installed_size,
|
||||
description, url, license, origin, maintainer,
|
||||
build_time, commit_hash, provider_priority,
|
||||
depends, provides, install_if
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash, meta.Checksum,
|
||||
meta.Name, meta.Version, meta.Arch, meta.DownloadSize, meta.InstalledSize,
|
||||
meta.Description, meta.URL, meta.License, meta.Origin, meta.Maintainer,
|
||||
meta.BuildTime, meta.Commit, meta.ProviderPriority,
|
||||
strings.Join(meta.Depends, " "), strings.Join(meta.Provides, " "), strings.Join(meta.InstallIf, " "),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM alpine_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]provider.AlpineMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash, checksum,
|
||||
name, version, arch, download_size, installed_size,
|
||||
description, url, license, origin, maintainer,
|
||||
build_time, commit_hash, provider_priority,
|
||||
depends, provides, install_if
|
||||
FROM alpine_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, version, arch, file_path
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.AlpineMetadata
|
||||
for rows.Next() {
|
||||
var m provider.AlpineMetadata
|
||||
var depends, provides, installIf string
|
||||
if err := rows.Scan(
|
||||
&m.RepoName, &m.FilePath, &m.ContentHash, &m.Checksum,
|
||||
&m.Name, &m.Version, &m.Arch, &m.DownloadSize, &m.InstalledSize,
|
||||
&m.Description, &m.URL, &m.License, &m.Origin, &m.Maintainer,
|
||||
&m.BuildTime, &m.Commit, &m.ProviderPriority,
|
||||
&depends, &provides, &installIf,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Depends = strings.Fields(depends)
|
||||
m.Provides = strings.Fields(provides)
|
||||
m.InstallIf = strings.Fields(installIf)
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -111,49 +109,16 @@ func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cach
|
||||
return err
|
||||
}
|
||||
|
||||
// AccessLogEntry is one buffered access-log record.
|
||||
type AccessLogEntry struct {
|
||||
RemoteName string
|
||||
Path string
|
||||
CacheHit bool
|
||||
SizeBytes int64
|
||||
UpstreamMS int
|
||||
ClientIP string
|
||||
}
|
||||
|
||||
// InsertAccessLogBatch bulk-inserts access-log rows with a single COPY.
|
||||
func (db *DB) InsertAccessLogBatch(ctx context.Context, entries []AccessLogEntry) error {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows := make([][]any, len(entries))
|
||||
for i, e := range entries {
|
||||
rows[i] = []any{e.RemoteName, e.Path, e.CacheHit, e.SizeBytes, e.UpstreamMS, e.ClientIP}
|
||||
}
|
||||
_, err := db.Pool.CopyFrom(ctx,
|
||||
pgx.Identifier{"access_log"},
|
||||
[]string{"remote_name", "path", "cache_hit", "size_bytes", "upstream_ms", "client_ip"},
|
||||
pgx.CopyFromRows(rows),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// FindOrphanedBlobs returns blobs no longer referenced by any artifact or
|
||||
// local file, restricted to those created before now()-minAge. The age cutoff
|
||||
// is a grace period that avoids a TOCTOU race with in-flight dedup uploads,
|
||||
// which insert the blob row before the referencing artifact/local_files row.
|
||||
func (db *DB) FindOrphanedBlobs(ctx context.Context, minAge time.Duration) ([]models.Blob, error) {
|
||||
cutoff := time.Now().Add(-minAge)
|
||||
func (db *DB) FindOrphanedBlobs(ctx context.Context) ([]models.Blob, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT b.content_hash, b.s3_key, b.size_bytes, b.content_type, b.created_at
|
||||
FROM blobs b
|
||||
WHERE b.created_at < $1
|
||||
AND b.content_hash NOT IN (
|
||||
WHERE b.content_hash NOT IN (
|
||||
SELECT content_hash FROM artifacts
|
||||
UNION
|
||||
SELECT content_hash FROM local_files
|
||||
)
|
||||
`, cutoff)
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1,381 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
var (
|
||||
testDB *DB
|
||||
testDSN string
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
c := context.Background()
|
||||
dsn, terminate, err := testsupport.StartPostgres(c)
|
||||
if err != nil {
|
||||
// Docker unavailable: run anyway so tests self-skip via requireDB.
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN = dsn
|
||||
db, err := New(dsn)
|
||||
if err != nil {
|
||||
terminate()
|
||||
panic(err)
|
||||
}
|
||||
testDB = db
|
||||
|
||||
code := m.Run()
|
||||
|
||||
db.Close()
|
||||
terminate()
|
||||
// Return normally on success so the coverage profile is flushed; os.Exit
|
||||
// would truncate it.
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
func requireDB(t *testing.T) {
|
||||
t.Helper()
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable; skipping database integration test")
|
||||
}
|
||||
}
|
||||
|
||||
func ctx() context.Context { return context.Background() }
|
||||
|
||||
func seedRemote(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: name, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://example.com", MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed remote: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// seedBlob inserts a blob and returns its full content hash (sha256:<hash>),
|
||||
// matching the reference convention used by artifacts and local files.
|
||||
func seedBlob(t *testing.T, hash string) string {
|
||||
t.Helper()
|
||||
full := "sha256:" + hash
|
||||
if err := testDB.UpsertBlob(ctx(), full, "blobs/sha256/"+hash, 10, "application/octet-stream"); err != nil {
|
||||
t.Fatalf("seed blob: %v", err)
|
||||
}
|
||||
return full
|
||||
}
|
||||
|
||||
func TestRemotesCRUD(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-crud")
|
||||
got, err := testDB.GetRemote(ctx(), "r-crud")
|
||||
if err != nil || got.BaseURL != "https://example.com" {
|
||||
t.Fatalf("get: %v %v", got, err)
|
||||
}
|
||||
got.BaseURL = "https://updated.example.com"
|
||||
if err := testDB.UpdateRemote(ctx(), got); err != nil {
|
||||
t.Fatalf("update: %v", err)
|
||||
}
|
||||
got, _ = testDB.GetRemote(ctx(), "r-crud")
|
||||
if got.BaseURL != "https://updated.example.com" {
|
||||
t.Errorf("update not applied: %v", got.BaseURL)
|
||||
}
|
||||
list, err := testDB.ListRemotes(ctx())
|
||||
if err != nil || len(list) == 0 {
|
||||
t.Fatalf("list: %v %v", len(list), err)
|
||||
}
|
||||
if err := testDB.DeleteRemote(ctx(), "r-crud"); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetRemote(ctx(), "r-crud"); err == nil {
|
||||
t.Error("expected error after delete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteMirrorlistRoundTrip(t *testing.T) {
|
||||
requireDB(t)
|
||||
mirrors := []string{"https://b.example", "https://c.example"}
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: "r-mirror", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://a.example", Mirrorlist: mirrors, MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatalf("create mirrorlist remote: %v", err)
|
||||
}
|
||||
defer testDB.DeleteRemote(ctx(), "r-mirror")
|
||||
|
||||
got, err := testDB.GetRemote(ctx(), "r-mirror")
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.BaseURL != "https://a.example" {
|
||||
t.Fatalf("BaseURL = %q, want https://a.example", got.BaseURL)
|
||||
}
|
||||
if len(got.Mirrorlist) != 2 || got.Mirrorlist[0] != mirrors[0] || got.Mirrorlist[1] != mirrors[1] {
|
||||
t.Fatalf("Mirrorlist round-trip = %v, want %v", got.Mirrorlist, mirrors)
|
||||
}
|
||||
// An unset strategy is stored as the round_robin default.
|
||||
if got.MirrorStrategy != models.MirrorStrategyRoundRobin {
|
||||
t.Fatalf("MirrorStrategy default = %q, want %q", got.MirrorStrategy, models.MirrorStrategyRoundRobin)
|
||||
}
|
||||
|
||||
// Updating to least_conn round-trips.
|
||||
got.MirrorStrategy = models.MirrorStrategyLeastConn
|
||||
if err := testDB.UpdateRemote(ctx(), got); err != nil {
|
||||
t.Fatalf("update to least_conn: %v", err)
|
||||
}
|
||||
got, _ = testDB.GetRemote(ctx(), "r-mirror")
|
||||
if got.MirrorStrategy != models.MirrorStrategyLeastConn {
|
||||
t.Fatalf("MirrorStrategy after update = %q, want least_conn", got.MirrorStrategy)
|
||||
}
|
||||
|
||||
// Clearing the mirrorlist on update persists an empty list.
|
||||
got.Mirrorlist = nil
|
||||
if err := testDB.UpdateRemote(ctx(), got); err != nil {
|
||||
t.Fatalf("update clearing mirrorlist: %v", err)
|
||||
}
|
||||
got, _ = testDB.GetRemote(ctx(), "r-mirror")
|
||||
if len(got.Mirrorlist) != 0 {
|
||||
t.Fatalf("mirrorlist after clear = %v, want empty", got.Mirrorlist)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactsAndBlobs(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-art")
|
||||
seedBlob(t, "aaaa")
|
||||
hash := "sha256:aaaa"
|
||||
if err := testDB.UpsertBlob(ctx(), hash, "blobs/sha256/aaaa", 10, "text/plain"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Upsert again to exercise the ON CONFLICT update branch.
|
||||
if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
art, err := testDB.GetArtifact(ctx(), "r-art", "path/a.txt")
|
||||
if err != nil || art.ContentHash != hash {
|
||||
t.Fatalf("get artifact: %v %v", art, err)
|
||||
}
|
||||
if err := testDB.TouchArtifactAccess(ctx(), "r-art", "path/a.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arts, err := testDB.ListArtifacts(ctx(), "r-art", 10, 0)
|
||||
if err != nil || len(arts) != 1 {
|
||||
t.Fatalf("list artifacts: %v %v", len(arts), err)
|
||||
}
|
||||
if err := testDB.InsertAccessLog(ctx(), "r-art", "path/a.txt", true, 10, 5, "1.2.3.4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.InsertAccessLogBatch(ctx(), []AccessLogEntry{
|
||||
{RemoteName: "r-art", Path: "b", CacheHit: false, SizeBytes: 20, UpstreamMS: 3},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.InsertAccessLogBatch(ctx(), nil); err != nil {
|
||||
t.Fatalf("empty batch should be a no-op: %v", err)
|
||||
}
|
||||
if err := testDB.DeleteArtifact(ctx(), "r-art", "path/a.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrphanAndColdCleanup(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedBlob(t, "orphanhash")
|
||||
// A blob with no artifact/local_file reference is orphaned, but only past
|
||||
// the grace period.
|
||||
if got, _ := testDB.FindOrphanedBlobs(ctx(), time.Hour); containsHash(got, "sha256:orphanhash") {
|
||||
t.Error("fresh orphan should be excluded by grace period")
|
||||
}
|
||||
orphans, err := testDB.FindOrphanedBlobs(ctx(), -time.Hour) // cutoff in the future => include fresh
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !containsHash(orphans, "sha256:orphanhash") {
|
||||
t.Error("expected orphan to be found with zero grace")
|
||||
}
|
||||
if err := testDB.DeleteBlob(ctx(), "sha256:orphanhash"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
seedRemote(t, "r-cold")
|
||||
seedBlob(t, "coldhash")
|
||||
testDB.UpsertArtifact(ctx(), "r-cold", "cold.txt", "sha256:coldhash", "")
|
||||
n, err := testDB.DeleteColdArtifacts(ctx(), "r-cold", -time.Hour) // negative => everything is "cold"
|
||||
if err != nil || n < 1 {
|
||||
t.Fatalf("delete cold: n=%d err=%v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func containsHash(blobs []models.Blob, hash string) bool {
|
||||
for _, b := range blobs {
|
||||
if b.ContentHash == hash {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestLocalFiles(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-local")
|
||||
seedBlob(t, "localhash")
|
||||
hash := "sha256:localhash"
|
||||
if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Duplicate create must be rejected.
|
||||
if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err == nil {
|
||||
t.Error("expected duplicate local file error")
|
||||
}
|
||||
f, err := testDB.GetLocalFile(ctx(), "r-local", "foo/foo-1.0.whl")
|
||||
if err != nil || f == nil {
|
||||
t.Fatalf("get local file: %v %v", f, err)
|
||||
}
|
||||
if files, err := testDB.ListLocalFiles(ctx(), "r-local", 10, 0); err != nil || len(files) != 1 {
|
||||
t.Fatalf("list: %v %v", len(files), err)
|
||||
}
|
||||
if files, err := testDB.ListLocalFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(files) != 1 {
|
||||
t.Fatalf("list by prefix: %v %v", len(files), err)
|
||||
}
|
||||
if entries, err := testDB.ListFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(entries) != 1 {
|
||||
t.Fatalf("provider list by prefix: %v %v", len(entries), err)
|
||||
}
|
||||
if pkgs, err := testDB.ListLocalFilePackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 {
|
||||
t.Fatalf("list packages: %v %v", pkgs, err)
|
||||
}
|
||||
if pkgs, err := testDB.ListPackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 {
|
||||
t.Fatalf("provider list packages: %v %v", pkgs, err)
|
||||
}
|
||||
if err := testDB.DeleteLocalFile(ctx(), "r-local", "foo/foo-1.0.whl"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualsCRUD(t *testing.T) {
|
||||
requireDB(t)
|
||||
if err := testDB.CreateVirtual(ctx(), &models.Virtual{
|
||||
Name: "v-crud", PackageType: models.PackageHelm, Members: []string{"a", "b"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := testDB.GetVirtual(ctx(), "v-crud")
|
||||
if err != nil || len(v.Members) != 2 {
|
||||
t.Fatalf("get virtual: %v %v", v, err)
|
||||
}
|
||||
v.Members = []string{"a"}
|
||||
if err := testDB.UpdateVirtual(ctx(), v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if vs, err := testDB.ListVirtuals(ctx()); err != nil || len(vs) == 0 {
|
||||
t.Fatalf("list virtuals: %v %v", len(vs), err)
|
||||
}
|
||||
if err := testDB.DeleteVirtual(ctx(), "v-crud"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStats(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-stats")
|
||||
seedBlob(t, "statshash")
|
||||
testDB.UpsertArtifact(ctx(), "r-stats", "s.txt", "sha256:statshash", "")
|
||||
testDB.InsertAccessLog(ctx(), "r-stats", "s.txt", true, 100, 2, "")
|
||||
|
||||
if _, err := testDB.GetOverviewStats(ctx()); err != nil {
|
||||
t.Fatalf("overview: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetTopRemotes(ctx(), 5); err != nil {
|
||||
t.Fatalf("top remotes: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetTopFilesByHits(ctx(), 5); err != nil {
|
||||
t.Fatalf("top files by hits: %v", err)
|
||||
}
|
||||
if _, err := testDB.GetTopFilesByBandwidth(ctx(), 5); err != nil {
|
||||
t.Fatalf("top files by bandwidth: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatabaseErrorPaths(t *testing.T) {
|
||||
requireDB(t)
|
||||
bad, err := New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bad.Close() // every query now fails
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := bad.ListRemotes(ctx); err == nil {
|
||||
t.Error("ListRemotes should error on closed db")
|
||||
}
|
||||
if _, err := bad.ListVirtuals(ctx); err == nil {
|
||||
t.Error("ListVirtuals should error")
|
||||
}
|
||||
if _, err := bad.ListArtifacts(ctx, "r", 10, 0); err == nil {
|
||||
t.Error("ListArtifacts should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFiles(ctx, "r", 10, 0); err == nil {
|
||||
t.Error("ListLocalFiles should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFilesByPrefix(ctx, "r", "p"); err == nil {
|
||||
t.Error("ListLocalFilesByPrefix should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFilePackages(ctx, "r"); err == nil {
|
||||
t.Error("ListLocalFilePackages should error")
|
||||
}
|
||||
if _, err := bad.ListFilesByPrefix(ctx, "r", "p"); err == nil {
|
||||
t.Error("ListFilesByPrefix should error")
|
||||
}
|
||||
if _, err := bad.ListPackages(ctx, "r"); err == nil {
|
||||
t.Error("ListPackages should error")
|
||||
}
|
||||
if _, err := bad.FindOrphanedBlobs(ctx, 0); err == nil {
|
||||
t.Error("FindOrphanedBlobs should error")
|
||||
}
|
||||
if _, err := bad.GetOverviewStats(ctx); err == nil {
|
||||
t.Error("GetOverviewStats should error")
|
||||
}
|
||||
if _, err := bad.GetTopRemotes(ctx, 5); err == nil {
|
||||
t.Error("GetTopRemotes should error")
|
||||
}
|
||||
if _, err := bad.GetTopFilesByHits(ctx, 5); err == nil {
|
||||
t.Error("GetTopFilesByHits should error")
|
||||
}
|
||||
if _, err := bad.GetTopFilesByBandwidth(ctx, 5); err == nil {
|
||||
t.Error("GetTopFilesByBandwidth should error")
|
||||
}
|
||||
if _, err := bad.ListRPMMetadataEntries(ctx, "r"); err == nil {
|
||||
t.Error("ListRPMMetadataEntries should error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMMetadata(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-rpm")
|
||||
meta := &provider.RPMMetadata{
|
||||
RepoName: "r-rpm", FilePath: "Packages/x.rpm", ContentHash: "sha256:rpm",
|
||||
Name: "x", Version: "1.0", Release: "1", Arch: "noarch",
|
||||
Requires: []provider.RPMDep{{Name: "libc"}},
|
||||
Provides: []provider.RPMDep{{Name: "x"}},
|
||||
Files: []provider.RPMFile{},
|
||||
}
|
||||
if err := testDB.InsertRPMMetadata(ctx(), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, err := testDB.ListRPMMetadataEntries(ctx(), "r-rpm")
|
||||
if err != nil || len(entries) != 1 {
|
||||
t.Fatalf("list rpm entries: %v %v", len(entries), err)
|
||||
}
|
||||
if rows, err := testDB.ListRPMMetadata(ctx(), "r-rpm"); err != nil || len(rows) != 1 {
|
||||
t.Fatalf("list rpm rows: %v %v", len(rows), err)
|
||||
}
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// ListGitHubDebRemotes returns every github_deb remote so the syncer can sweep
|
||||
// them on each poll tick.
|
||||
func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes WHERE package_type = $1 ORDER BY name`, models.PackageGitHubDeb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var remotes []models.Remote
|
||||
for rows.Next() {
|
||||
var r models.Remote
|
||||
if err := scanRemote(rows, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remotes = append(remotes, r)
|
||||
}
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubDebSyncLease atomically claims the per-remote sync lease. It
|
||||
// succeeds only when the remote is due (never synced, or synced longer than
|
||||
// freshness ago) and no live lease is held by another replica. A zero freshness
|
||||
// (prime scans) ignores the recency gate. The returned etag is the stored
|
||||
// releases-list ETag, shared across replicas.
|
||||
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO github_deb_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
|
||||
var etag string
|
||||
if err := row.Scan(&etag); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, "", nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
return true, etag, nil
|
||||
}
|
||||
|
||||
// ReleaseGitHubDebSyncLease records the completed scan and frees the lease. Only
|
||||
// the owning replica may release; last_synced_at advances so the next poll waits
|
||||
// a full freshness window, and etag is persisted for the next conditional request.
|
||||
func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE github_deb_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
return err
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func seedGitHubDebRemote(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: name, PackageType: models.PackageGitHubDeb, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed github_deb remote: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGitHubDebSyncLease exercises the real SQL: exactly one replica may hold the
|
||||
// lease, the recency window blocks a too-soon periodic re-claim, and a prime
|
||||
// (freshness 0) bypasses recency but still respects a live lease.
|
||||
func TestGitHubDebSyncLease(t *testing.T) {
|
||||
requireDB(t)
|
||||
name := "ghdeb-lease-" + time.Now().Format("150405.000000")
|
||||
seedGitHubDebRemote(t, name)
|
||||
|
||||
const lease = 15 * time.Minute
|
||||
freshness := time.Hour
|
||||
|
||||
claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
if etag != "" {
|
||||
t.Fatalf("initial etag should be empty, got %q", etag)
|
||||
}
|
||||
|
||||
claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
|
||||
if err != nil {
|
||||
t.Fatalf("replica-2 claim err: %v", err)
|
||||
}
|
||||
if claimed2 {
|
||||
t.Fatal("replica-2 claimed while replica-1 holds the lease")
|
||||
}
|
||||
|
||||
if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
|
||||
claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
|
||||
if err != nil {
|
||||
t.Fatalf("replica-2 recency claim err: %v", err)
|
||||
}
|
||||
if claimed3 {
|
||||
t.Fatal("periodic claim succeeded inside the freshness window")
|
||||
}
|
||||
|
||||
claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease)
|
||||
if err != nil || !claimed4 {
|
||||
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
|
||||
}
|
||||
if etag4 != `"etag-1"` {
|
||||
t.Fatalf("prime claim etag = %q, want persisted \"etag-1\"", etag4)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListGitHubDebRemotes(t *testing.T) {
|
||||
requireDB(t)
|
||||
name := "ghdeb-list-" + time.Now().Format("150405.000000")
|
||||
seedGitHubDebRemote(t, name)
|
||||
seedRemote(t, "generic-"+time.Now().Format("150405.000000"))
|
||||
|
||||
remotes, err := testDB.ListGitHubDebRemotes(ctx())
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range remotes {
|
||||
if r.PackageType != models.PackageGitHubDeb {
|
||||
t.Fatalf("non-github_deb remote returned: %s (%s)", r.Name, r.PackageType)
|
||||
}
|
||||
if r.Name == name {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("seeded remote %q not returned", name)
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertDebMetadata(ctx context.Context, meta *provider.DebMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO deb_metadata (
|
||||
repo_name, file_path, content_hash,
|
||||
name, version, architecture, control,
|
||||
size, md5, sha256
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash,
|
||||
meta.Name, meta.Version, meta.Architecture, meta.Control,
|
||||
meta.Size, meta.MD5, meta.SHA256,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteDebMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM deb_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListDebMetadataEntries(ctx context.Context, repoName string) ([]provider.DebMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash,
|
||||
name, version, architecture, control,
|
||||
size, md5, sha256, created_at
|
||||
FROM deb_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, version, architecture, file_path
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.DebMetadata
|
||||
for rows.Next() {
|
||||
var m provider.DebMetadata
|
||||
if err := rows.Scan(
|
||||
&m.RepoName, &m.FilePath, &m.ContentHash,
|
||||
&m.Name, &m.Version, &m.Architecture, &m.Control,
|
||||
&m.Size, &m.MD5, &m.SHA256, &m.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// ListGitHubRPMRemotes returns every github_rpm remote so the syncer can sweep
|
||||
// them on each poll tick.
|
||||
func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes WHERE package_type = $1 ORDER BY name`, models.PackageGitHubRPM)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var remotes []models.Remote
|
||||
for rows.Next() {
|
||||
var r models.Remote
|
||||
if err := scanRemote(rows, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remotes = append(remotes, r)
|
||||
}
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubSyncLease atomically claims the per-remote sync lease. It succeeds
|
||||
// (claimed=true) only when the remote is due — never synced, or synced longer
|
||||
// than freshness ago — and no live lease is held by another replica. This bounds
|
||||
// total GitHub load to roughly one scan per freshness window regardless of how
|
||||
// many replicas poll. The returned etag is the stored releases-list ETag, shared
|
||||
// across replicas so a conditional request can short-circuit an unchanged repo.
|
||||
// A zero freshness (used for prime scans) ignores the recency gate and claims
|
||||
// whenever no live lease is held.
|
||||
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO github_rpm_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
|
||||
var etag string
|
||||
if err := row.Scan(&etag); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, "", nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
return true, etag, nil
|
||||
}
|
||||
|
||||
// ReleaseGitHubSyncLease records the completed scan and frees the lease. Only the
|
||||
// owning replica may release; last_synced_at advances so the next poll waits a
|
||||
// full freshness window, and etag is persisted for the next conditional request.
|
||||
func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE github_rpm_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
return err
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func seedGitHubRPMRemote(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: name, PackageType: models.PackageGitHubRPM, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed github_rpm remote: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGitHubSyncLease exercises the real SQL: exactly one replica may hold the
|
||||
// lease, the recency window blocks a too-soon periodic re-claim, and a prime
|
||||
// (freshness 0) bypasses recency but still respects a live lease.
|
||||
func TestGitHubSyncLease(t *testing.T) {
|
||||
requireDB(t)
|
||||
name := "gh-lease-" + time.Now().Format("150405.000000")
|
||||
seedGitHubRPMRemote(t, name)
|
||||
|
||||
const lease = 15 * time.Minute
|
||||
freshness := time.Hour
|
||||
|
||||
// First claim on a never-synced remote wins; etag starts empty.
|
||||
claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
if etag != "" {
|
||||
t.Fatalf("initial etag should be empty, got %q", etag)
|
||||
}
|
||||
|
||||
// A second replica cannot claim while the lease is held.
|
||||
claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
|
||||
if err != nil {
|
||||
t.Fatalf("replica-2 claim err: %v", err)
|
||||
}
|
||||
if claimed2 {
|
||||
t.Fatal("replica-2 claimed while replica-1 holds the lease")
|
||||
}
|
||||
|
||||
// Replica 1 finishes: record the sync and persist an etag.
|
||||
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
|
||||
// A periodic re-claim inside the freshness window is blocked by recency.
|
||||
claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
|
||||
if err != nil {
|
||||
t.Fatalf("replica-2 recency claim err: %v", err)
|
||||
}
|
||||
if claimed3 {
|
||||
t.Fatal("periodic claim succeeded inside the freshness window")
|
||||
}
|
||||
|
||||
// A prime (freshness 0) bypasses recency and reads the persisted etag.
|
||||
claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease)
|
||||
if err != nil || !claimed4 {
|
||||
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
|
||||
}
|
||||
if etag4 != `"etag-1"` {
|
||||
t.Fatalf("prime claim etag = %q, want persisted \"etag-1\"", etag4)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListGitHubRPMRemotes(t *testing.T) {
|
||||
requireDB(t)
|
||||
name := "gh-list-" + time.Now().Format("150405.000000")
|
||||
seedGitHubRPMRemote(t, name)
|
||||
seedRemote(t, "generic-"+time.Now().Format("150405.000000"))
|
||||
|
||||
remotes, err := testDB.ListGitHubRPMRemotes(ctx())
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range remotes {
|
||||
if r.PackageType != models.PackageGitHubRPM {
|
||||
t.Fatalf("non-github_rpm remote returned: %s (%s)", r.Name, r.PackageType)
|
||||
}
|
||||
if r.Name == name {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("seeded remote %q not returned", name)
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type LocalFile struct {
|
||||
@@ -38,20 +37,6 @@ func (db *DB) CreateLocalFile(ctx context.Context, repoName, filePath, contentHa
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpsertLocalFile inserts a local file or repoints an existing path at a new
|
||||
// blob. Unlike CreateLocalFile it never errors on a duplicate path — it is for
|
||||
// mutable references such as Docker tags, where re-pushing a tag must move it to
|
||||
// the newly-pushed manifest rather than being rejected as an overwrite.
|
||||
func (db *DB) UpsertLocalFile(ctx context.Context, repoName, filePath, contentHash string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO local_files (repo_name, file_path, content_hash)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (repo_name, file_path)
|
||||
DO UPDATE SET content_hash = EXCLUDED.content_hash, created_at = NOW()
|
||||
`, repoName, filePath, contentHash)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) GetLocalFile(ctx context.Context, repoName, filePath string) (*LocalFile, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
@@ -93,40 +78,6 @@ func (db *DB) ListLocalFiles(ctx context.Context, repoName string, limit, offset
|
||||
return files, rows.Err()
|
||||
}
|
||||
|
||||
// ListLocalArtifacts returns a repo's local files shaped as models.Artifact so
|
||||
// the UI's cached-objects view can render them the same way as remote artifacts.
|
||||
// Local files carry no access/fetch counters, so those are left at zero and the
|
||||
// timestamps are all derived from created_at.
|
||||
func (db *DB) ListLocalArtifacts(ctx context.Context, repoName string, limit, offset int) ([]models.Artifact, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT lf.id, lf.repo_name, lf.file_path, lf.content_hash,
|
||||
lf.created_at, b.size_bytes, b.content_type
|
||||
FROM local_files lf
|
||||
JOIN blobs b ON lf.content_hash = b.content_hash
|
||||
WHERE lf.repo_name = $1
|
||||
ORDER BY lf.file_path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, repoName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var artifacts []models.Artifact
|
||||
for rows.Next() {
|
||||
var a models.Artifact
|
||||
var createdAt time.Time
|
||||
if err := rows.Scan(&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &createdAt, &a.SizeBytes, &a.ContentType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.FirstSeenAt = createdAt
|
||||
a.LastFetchedAt = createdAt
|
||||
a.LastAccessedAt = createdAt
|
||||
artifacts = append(artifacts, a)
|
||||
}
|
||||
return artifacts, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFilesByPrefix(ctx context.Context, repoName, prefix string) ([]LocalFile, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
|
||||
@@ -44,8 +44,6 @@ func (db *DB) migrate() error {
|
||||
package_type TEXT NOT NULL,
|
||||
repo_type TEXT DEFAULT 'remote',
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
mirrorlist TEXT[] DEFAULT '{}',
|
||||
mirror_strategy TEXT NOT NULL DEFAULT 'round_robin',
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
@@ -126,11 +124,6 @@ func (db *DB) migrate() error {
|
||||
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS mirrorlist TEXT[] DEFAULT '{}';
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS mirror_strategy TEXT NOT NULL DEFAULT 'round_robin';
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rpm_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
@@ -155,8 +148,6 @@ func (db *DB) migrate() error {
|
||||
packager TEXT DEFAULT '',
|
||||
requires JSONB DEFAULT '[]',
|
||||
provides JSONB DEFAULT '[]',
|
||||
conflicts JSONB DEFAULT '[]',
|
||||
obsoletes JSONB DEFAULT '[]',
|
||||
files JSONB DEFAULT '[]',
|
||||
changelogs JSONB DEFAULT '[]',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
@@ -164,87 +155,6 @@ func (db *DB) migrate() error {
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
|
||||
|
||||
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
|
||||
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS deb_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
architecture TEXT NOT NULL,
|
||||
control TEXT NOT NULL,
|
||||
size BIGINT DEFAULT 0,
|
||||
md5 TEXT DEFAULT '',
|
||||
sha256 TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alpine_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
checksum TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
download_size BIGINT DEFAULT 0,
|
||||
installed_size BIGINT DEFAULT 0,
|
||||
description TEXT DEFAULT '',
|
||||
url TEXT DEFAULT '',
|
||||
license TEXT DEFAULT '',
|
||||
origin TEXT DEFAULT '',
|
||||
maintainer TEXT DEFAULT '',
|
||||
build_time BIGINT DEFAULT 0,
|
||||
commit_hash TEXT DEFAULT '',
|
||||
provider_priority TEXT DEFAULT '',
|
||||
depends TEXT DEFAULT '',
|
||||
provides TEXT DEFAULT '',
|
||||
install_if TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS github_rpm_sync_state (
|
||||
remote_name TEXT PRIMARY KEY,
|
||||
etag TEXT DEFAULT '',
|
||||
last_synced_at TIMESTAMPTZ,
|
||||
sync_lease_owner TEXT DEFAULT '',
|
||||
sync_lease_expires TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS github_deb_sync_state (
|
||||
remote_name TEXT PRIMARY KEY,
|
||||
etag TEXT DEFAULT '',
|
||||
last_synced_at TIMESTAMPTZ,
|
||||
sync_lease_owner TEXT DEFAULT '',
|
||||
sync_lease_expires TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS github_alpine_sync_state (
|
||||
remote_name TEXT PRIMARY KEY,
|
||||
etag TEXT DEFAULT '',
|
||||
last_synced_at TIMESTAMPTZ,
|
||||
sync_lease_owner TEXT DEFAULT '',
|
||||
sync_lease_expires TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signing_keys (
|
||||
purpose TEXT PRIMARY KEY,
|
||||
private_key_armor TEXT NOT NULL,
|
||||
key_id TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -6,34 +6,21 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, mirror_strategy, description, username, password,
|
||||
const remoteCols = `name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
created_at, updated_at`
|
||||
|
||||
// normalizeMirrorStrategy maps an empty strategy to the round_robin default so
|
||||
// the NOT NULL mirror_strategy column always stores a canonical value.
|
||||
func normalizeMirrorStrategy(s string) string {
|
||||
if s == "" {
|
||||
return models.MirrorStrategyRoundRobin
|
||||
}
|
||||
return s
|
||||
}
|
||||
releases_remote, managed_by, created_at, updated_at`
|
||||
|
||||
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
|
||||
return scanner.Scan(
|
||||
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Mirrorlist, &r.MirrorStrategy, &r.Description, &r.Username, &r.Password,
|
||||
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Description, &r.Username, &r.Password,
|
||||
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
|
||||
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
|
||||
&r.BanTagsEnabled, &r.BanTags,
|
||||
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
|
||||
&r.ReleasesRemote, &r.ManagedBy,
|
||||
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
|
||||
&r.CreatedAt, &r.UpdatedAt,
|
||||
&r.ReleasesRemote, &r.ManagedBy, &r.CreatedAt, &r.UpdatedAt,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -67,24 +54,20 @@ func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO remotes (
|
||||
name, package_type, repo_type, base_url, mirrorlist, description, username, password,
|
||||
name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
mirror_strategy
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26)
|
||||
releases_remote, managed_by
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21)
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Mirrorlist, r.Description, r.Username, r.Password,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
normalizeMirrorStrategy(r.MirrorStrategy),
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -92,15 +75,12 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE remotes SET
|
||||
package_type=$2, repo_type=$3, base_url=$4, mirrorlist=$25, description=$5, username=$6, password=$7,
|
||||
package_type=$2, repo_type=$3, base_url=$4, description=$5, username=$6, password=$7,
|
||||
immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
|
||||
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
|
||||
ban_tags_enabled=$15, ban_tags=$16,
|
||||
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
|
||||
releases_remote=$20, managed_by=$21,
|
||||
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
|
||||
mirror_strategy=$26,
|
||||
updated_at=NOW()
|
||||
releases_remote=$20, managed_by=$21, updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
@@ -109,9 +89,6 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
r.Mirrorlist,
|
||||
normalizeMirrorStrategy(r.MirrorStrategy),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
@@ -11,8 +10,6 @@ import (
|
||||
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
|
||||
requiresJSON, _ := json.Marshal(meta.Requires)
|
||||
providesJSON, _ := json.Marshal(meta.Provides)
|
||||
conflictsJSON, _ := json.Marshal(meta.Conflicts)
|
||||
obsoletesJSON, _ := json.Marshal(meta.Obsoletes)
|
||||
filesJSON, _ := json.Marshal(meta.Files)
|
||||
changelogsJSON, _ := json.Marshal(meta.Changelogs)
|
||||
|
||||
@@ -22,24 +19,19 @@ func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata)
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, conflicts, obsoletes, files, changelogs
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
|
||||
requires, provides, files, changelogs
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash,
|
||||
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
|
||||
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
|
||||
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
|
||||
requiresJSON, providesJSON, conflictsJSON, obsoletesJSON, filesJSON, changelogsJSON,
|
||||
requiresJSON, providesJSON, filesJSON, changelogsJSON,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM rpm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
type RPMMetadataRow struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
@@ -62,11 +54,8 @@ type RPMMetadataRow struct {
|
||||
Packager string
|
||||
Requires json.RawMessage
|
||||
Provides json.RawMessage
|
||||
Conflicts json.RawMessage
|
||||
Obsoletes json.RawMessage
|
||||
Files json.RawMessage
|
||||
Changelogs json.RawMessage
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]provider.RPMMetadata, error) {
|
||||
@@ -96,12 +85,9 @@ func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]pr
|
||||
SourceRPM: r.SourceRPM,
|
||||
URL: r.URL,
|
||||
Packager: r.Packager,
|
||||
CreatedAt: r.CreatedAt,
|
||||
}
|
||||
json.Unmarshal(r.Requires, &meta.Requires)
|
||||
json.Unmarshal(r.Provides, &meta.Provides)
|
||||
json.Unmarshal(r.Conflicts, &meta.Conflicts)
|
||||
json.Unmarshal(r.Obsoletes, &meta.Obsoletes)
|
||||
json.Unmarshal(r.Files, &meta.Files)
|
||||
json.Unmarshal(r.Changelogs, &meta.Changelogs)
|
||||
result[i] = meta
|
||||
@@ -115,11 +101,10 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, conflicts, obsoletes, files, changelogs,
|
||||
created_at
|
||||
requires, provides, files, changelogs
|
||||
FROM rpm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, epoch, version, release, arch, file_path
|
||||
ORDER BY name, epoch, version, release, arch
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -134,8 +119,7 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
|
||||
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
|
||||
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
|
||||
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
|
||||
&r.Requires, &r.Provides, &r.Conflicts, &r.Obsoletes, &r.Files, &r.Changelogs,
|
||||
&r.CreatedAt,
|
||||
&r.Requires, &r.Provides, &r.Files, &r.Changelogs,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// GetSigningKey returns the stored armored private key and key id for a purpose.
|
||||
// found is false when no key has been generated yet.
|
||||
func (db *DB) GetSigningKey(ctx context.Context, purpose string) (armor, keyID string, found bool, err error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT private_key_armor, key_id FROM signing_keys WHERE purpose = $1
|
||||
`, purpose)
|
||||
if err := row.Scan(&armor, &keyID); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", "", false, nil
|
||||
}
|
||||
return "", "", false, err
|
||||
}
|
||||
return armor, keyID, true, nil
|
||||
}
|
||||
|
||||
// InsertSigningKeyIfAbsent stores a freshly generated key, doing nothing if
|
||||
// another replica already inserted one. Callers re-read with GetSigningKey to
|
||||
// pick up whichever key won the race.
|
||||
func (db *DB) InsertSigningKeyIfAbsent(ctx context.Context, purpose, armor, keyID string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO signing_keys (purpose, private_key_armor, key_id)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (purpose) DO NOTHING
|
||||
`, purpose, armor, keyID)
|
||||
return err
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package database
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSigningKeyRoundTripAndIdempotency(t *testing.T) {
|
||||
requireDB(t)
|
||||
|
||||
const purpose = "terraform-provider-test"
|
||||
|
||||
// Absent to start.
|
||||
if _, _, found, err := testDB.GetSigningKey(ctx(), purpose); err != nil || found {
|
||||
t.Fatalf("expected no key, got found=%v err=%v", found, err)
|
||||
}
|
||||
|
||||
if err := testDB.InsertSigningKeyIfAbsent(ctx(), purpose, "ARMOR-1", "KEYID1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A second insert must not overwrite (models the replica race).
|
||||
if err := testDB.InsertSigningKeyIfAbsent(ctx(), purpose, "ARMOR-2", "KEYID2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
armor, keyID, found, err := testDB.GetSigningKey(ctx(), purpose)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("expected key, found=%v err=%v", found, err)
|
||||
}
|
||||
if armor != "ARMOR-1" || keyID != "KEYID1" {
|
||||
t.Errorf("key was overwritten: armor=%q key_id=%q", armor, keyID)
|
||||
}
|
||||
}
|
||||
+1
-34
@@ -9,16 +9,6 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
)
|
||||
|
||||
// blobGracePeriod is how old an orphaned blob must be before GC will delete
|
||||
// it. This avoids racing in-flight dedup uploads that insert the blob row
|
||||
// before the referencing artifact/local_files row exists.
|
||||
const blobGracePeriod = 1 * time.Hour
|
||||
|
||||
// uploadGracePeriod is how long a docker blob-upload staging object
|
||||
// (uploads/<uuid>) may sit idle before GC treats it as an abandoned push and
|
||||
// reaps it. Generous so a slow but live push is never cut off mid-flight.
|
||||
const uploadGracePeriod = 24 * time.Hour
|
||||
|
||||
type Collector struct {
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
@@ -48,9 +38,7 @@ func (c *Collector) Run(ctx context.Context) {
|
||||
func (c *Collector) sweep(ctx context.Context) {
|
||||
start := time.Now()
|
||||
|
||||
c.sweepUploads(ctx)
|
||||
|
||||
orphaned, err := c.db.FindOrphanedBlobs(ctx, blobGracePeriod)
|
||||
orphaned, err := c.db.FindOrphanedBlobs(ctx)
|
||||
if err != nil {
|
||||
slog.Error("gc: find orphaned blobs", "error", err)
|
||||
return
|
||||
@@ -77,24 +65,3 @@ func (c *Collector) sweep(ctx context.Context) {
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// sweepUploads reaps docker blob-upload staging objects abandoned longer than
|
||||
// uploadGracePeriod (cancelled or interrupted pushes that never finalised).
|
||||
func (c *Collector) sweepUploads(ctx context.Context) {
|
||||
stale, err := c.store.ListStaleObjects(ctx, "uploads/", time.Now().Add(-uploadGracePeriod))
|
||||
if err != nil {
|
||||
slog.Error("gc: list stale uploads", "error", err)
|
||||
return
|
||||
}
|
||||
reaped := 0
|
||||
for _, key := range stale {
|
||||
if err := c.store.Delete(ctx, key); err != nil {
|
||||
slog.Warn("gc: delete stale upload", "key", key, "error", err)
|
||||
continue
|
||||
}
|
||||
reaped++
|
||||
}
|
||||
if reaped > 0 {
|
||||
slog.Info("gc: reaped stale docker uploads", "count", reaped)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
var (
|
||||
testDB *database.DB
|
||||
testStore *storage.S3
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, termPG, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
minio, termMinio, err := testsupport.StartMinio(ctx)
|
||||
if err != nil {
|
||||
termPG()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
db, err := database.New(dsn)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var s3 *storage.S3
|
||||
for i := 0; i < 20; i++ {
|
||||
if s3, err = storage.NewS3(minio.Endpoint, minio.AccessKey, minio.SecretKey, "gc-test", false, ""); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
testDB = db
|
||||
testStore = s3
|
||||
|
||||
code := m.Run()
|
||||
db.Close()
|
||||
termMinio()
|
||||
termPG()
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepDeletesOldOrphan(t *testing.T) {
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
hash := "sha256:gcorphan"
|
||||
key := storage.BlobKey("gcorphan")
|
||||
|
||||
if err := testStore.Upload(ctx, key, bytes.NewReader([]byte("orphan")), 6, "application/octet-stream"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.UpsertBlob(ctx, hash, key, 6, "application/octet-stream"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Age the blob past the grace period.
|
||||
if _, err := testDB.Pool.Exec(ctx, `UPDATE blobs SET created_at = now() - interval '2 hours' WHERE content_hash = $1`, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c := New(testDB, testStore, time.Hour)
|
||||
c.sweep(ctx)
|
||||
|
||||
if exists, _ := testStore.Exists(ctx, key); exists {
|
||||
t.Error("expected orphan object deleted from store")
|
||||
}
|
||||
orphans, _ := testDB.FindOrphanedBlobs(ctx, 0)
|
||||
for _, b := range orphans {
|
||||
if b.ContentHash == hash {
|
||||
t.Error("expected orphan blob row deleted")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepNoOrphans(t *testing.T) {
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
// A sweep with nothing to collect should be a clean no-op.
|
||||
New(testDB, testStore, time.Hour).sweep(context.Background())
|
||||
}
|
||||
|
||||
func TestRunStopsOnContextCancel(t *testing.T) {
|
||||
if testDB == nil {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
New(testDB, testStore, time.Hour).Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Run did not return after context cancel")
|
||||
}
|
||||
}
|
||||
@@ -1,199 +0,0 @@
|
||||
package githubauth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultAPIBase = "https://api.github.com"
|
||||
|
||||
// jwtLifetime is how long the app JWT is valid. GitHub caps it at 10 minutes;
|
||||
// 9 leaves headroom for clock skew.
|
||||
jwtLifetime = 9 * time.Minute
|
||||
// jwtBackdate backdates iat to tolerate the app server's clock running behind
|
||||
// GitHub's, which otherwise rejects the JWT.
|
||||
jwtBackdate = 60 * time.Second
|
||||
// refreshSkew refreshes the installation token this long before it expires so
|
||||
// a request never races an expiry.
|
||||
refreshSkew = 5 * time.Minute
|
||||
)
|
||||
|
||||
type httpDoer interface {
|
||||
Do(*http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
// appCredential mints installation access tokens for a GitHub App. It signs a
|
||||
// short-lived RS256 JWT with the app private key, exchanges it for a ~1h
|
||||
// installation token, caches that token, and refreshes it shortly before expiry.
|
||||
// Refreshes are single-flighted by holding the mutex across the exchange, so
|
||||
// concurrent callers coalesce onto one HTTP request and reuse the cached token.
|
||||
type appCredential struct {
|
||||
appID string
|
||||
installationID string
|
||||
key *rsa.PrivateKey
|
||||
apiBase string
|
||||
client httpDoer
|
||||
|
||||
mu sync.Mutex
|
||||
token string
|
||||
expiry time.Time
|
||||
}
|
||||
|
||||
func newAppCredential(opts Options) (*appCredential, error) {
|
||||
if opts.AppID == "" {
|
||||
return nil, errors.New("github app: GITHUB_APP_ID is required")
|
||||
}
|
||||
if opts.InstallationID == "" {
|
||||
return nil, errors.New("github app: GITHUB_APP_INSTALLATION_ID is required")
|
||||
}
|
||||
pemBytes, err := loadPrivateKeyPEM(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := parseRSAPrivateKey(pemBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
apiBase := opts.apiBaseURL
|
||||
if apiBase == "" {
|
||||
apiBase = defaultAPIBase
|
||||
}
|
||||
client := opts.httpClient
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
|
||||
return &appCredential{
|
||||
appID: opts.AppID,
|
||||
installationID: opts.InstallationID,
|
||||
key: key,
|
||||
apiBase: strings.TrimRight(apiBase, "/"),
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Token returns a cached installation token, refreshing it under a single-flight
|
||||
// lock when it is missing or within refreshSkew of expiry.
|
||||
func (a *appCredential) Token(ctx context.Context) (string, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if a.token != "" && time.Now().Before(a.expiry.Add(-refreshSkew)) {
|
||||
return a.token, nil
|
||||
}
|
||||
if err := a.refreshLocked(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return a.token, nil
|
||||
}
|
||||
|
||||
func (a *appCredential) refreshLocked(ctx context.Context) error {
|
||||
jwt, err := mintJWT(a.appID, a.key, time.Now())
|
||||
if err != nil {
|
||||
return fmt.Errorf("github app: mint jwt: %w", err)
|
||||
}
|
||||
|
||||
u := fmt.Sprintf("%s/app/installations/%s/access_tokens", a.apiBase, a.installationID)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+jwt)
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
|
||||
|
||||
resp, err := a.client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("github app: token exchange: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
|
||||
// Never echo the body verbatim — it can contain sensitive material.
|
||||
return fmt.Errorf("github app: token exchange status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var out struct {
|
||||
Token string `json:"token"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
return fmt.Errorf("github app: decode token response: %w", err)
|
||||
}
|
||||
if out.Token == "" {
|
||||
return errors.New("github app: token exchange returned an empty token")
|
||||
}
|
||||
a.token = out.Token
|
||||
a.expiry = out.ExpiresAt
|
||||
if a.expiry.IsZero() {
|
||||
// Defensive: assume the documented ~1h lifetime if GitHub omits it.
|
||||
a.expiry = time.Now().Add(time.Hour)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// mintJWT builds and RS256-signs a GitHub App JWT (iss=app id, backdated iat,
|
||||
// ≤10m exp) using stdlib crypto — no third-party JWT dependency.
|
||||
func mintJWT(appID string, key *rsa.PrivateKey, now time.Time) (string, error) {
|
||||
header := map[string]string{"alg": "RS256", "typ": "JWT"}
|
||||
claims := map[string]any{
|
||||
"iat": now.Add(-jwtBackdate).Unix(),
|
||||
"exp": now.Add(jwtLifetime).Unix(),
|
||||
"iss": appID,
|
||||
}
|
||||
hb, err := json.Marshal(header)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
cb, err := json.Marshal(claims)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
signingInput := b64url(hb) + "." + b64url(cb)
|
||||
digest := sha256.Sum256([]byte(signingInput))
|
||||
sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, digest[:])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return signingInput + "." + b64url(sig), nil
|
||||
}
|
||||
|
||||
func b64url(b []byte) string {
|
||||
return base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
|
||||
// parseRSAPrivateKey accepts PKCS#1 ("RSA PRIVATE KEY") and PKCS#8 ("PRIVATE
|
||||
// KEY") PEM, covering both GitHub App key export formats.
|
||||
func parseRSAPrivateKey(pemBytes []byte) (*rsa.PrivateKey, error) {
|
||||
block, _ := pem.Decode(pemBytes)
|
||||
if block == nil {
|
||||
return nil, errors.New("github app: private key is not valid PEM")
|
||||
}
|
||||
if key, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil {
|
||||
return key, nil
|
||||
}
|
||||
keyAny, err := x509.ParsePKCS8PrivateKey(block.Bytes)
|
||||
if err != nil {
|
||||
return nil, errors.New("github app: private key is not a supported RSA PKCS#1/PKCS#8 key")
|
||||
}
|
||||
rsaKey, ok := keyAny.(*rsa.PrivateKey)
|
||||
if !ok {
|
||||
return nil, errors.New("github app: private key is not an RSA key")
|
||||
}
|
||||
return rsaKey, nil
|
||||
}
|
||||
@@ -1,207 +0,0 @@
|
||||
package githubauth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testRSAKeyPEM(t *testing.T) string {
|
||||
t.Helper()
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("generate key: %v", err)
|
||||
}
|
||||
der := x509.MarshalPKCS1PrivateKey(key)
|
||||
return string(pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: der}))
|
||||
}
|
||||
|
||||
// appFixture serves the installation-token exchange endpoint, records requests,
|
||||
// verifies the presented JWT against the app public key, and returns tokens with
|
||||
// a controllable expiry.
|
||||
type appFixture struct {
|
||||
srv *httptest.Server
|
||||
pub *rsa.PublicKey
|
||||
mu sync.Mutex
|
||||
exchanges int
|
||||
lastJWT string
|
||||
expiresAt func() time.Time
|
||||
tokenSeq int
|
||||
}
|
||||
|
||||
func newAppFixture(t *testing.T, pemKey string) *appFixture {
|
||||
t.Helper()
|
||||
block, _ := pem.Decode([]byte(pemKey))
|
||||
key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
|
||||
if err != nil {
|
||||
t.Fatalf("parse test key: %v", err)
|
||||
}
|
||||
f := &appFixture{
|
||||
pub: &key.PublicKey,
|
||||
expiresAt: func() time.Time { return time.Now().Add(time.Hour) },
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/app/installations/456/access_tokens", func(w http.ResponseWriter, r *http.Request) {
|
||||
auth := r.Header.Get("Authorization")
|
||||
jwt := strings.TrimPrefix(auth, "Bearer ")
|
||||
f.mu.Lock()
|
||||
f.exchanges++
|
||||
f.lastJWT = jwt
|
||||
f.tokenSeq++
|
||||
seq := f.tokenSeq
|
||||
exp := f.expiresAt()
|
||||
f.mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
json.NewEncoder(w).Encode(map[string]any{
|
||||
"token": fmt.Sprintf("ghs_installation_%d", seq),
|
||||
"expires_at": exp.UTC().Format(time.RFC3339),
|
||||
})
|
||||
})
|
||||
f.srv = httptest.NewServer(mux)
|
||||
t.Cleanup(f.srv.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *appFixture) verifyJWT(t *testing.T) {
|
||||
t.Helper()
|
||||
f.mu.Lock()
|
||||
jwt := f.lastJWT
|
||||
f.mu.Unlock()
|
||||
parts := strings.Split(jwt, ".")
|
||||
if len(parts) != 3 {
|
||||
t.Fatalf("jwt not three-part: %q", jwt)
|
||||
}
|
||||
signingInput := parts[0] + "." + parts[1]
|
||||
sig, err := base64.RawURLEncoding.DecodeString(parts[2])
|
||||
if err != nil {
|
||||
t.Fatalf("decode sig: %v", err)
|
||||
}
|
||||
digest := sha256.Sum256([]byte(signingInput))
|
||||
if err := rsa.VerifyPKCS1v15(f.pub, crypto.SHA256, digest[:], sig); err != nil {
|
||||
t.Fatalf("jwt signature invalid: %v", err)
|
||||
}
|
||||
var claims struct {
|
||||
Iss string `json:"iss"`
|
||||
Iat int64 `json:"iat"`
|
||||
Exp int64 `json:"exp"`
|
||||
}
|
||||
cb, _ := base64.RawURLEncoding.DecodeString(parts[1])
|
||||
if err := json.Unmarshal(cb, &claims); err != nil {
|
||||
t.Fatalf("decode claims: %v", err)
|
||||
}
|
||||
if claims.Iss != "123" {
|
||||
t.Fatalf("iss = %q, want 123", claims.Iss)
|
||||
}
|
||||
if claims.Exp-claims.Iat > int64((10*time.Minute)/time.Second) {
|
||||
t.Fatalf("jwt lifetime exceeds 10m: iat=%d exp=%d", claims.Iat, claims.Exp)
|
||||
}
|
||||
if claims.Iat > time.Now().Unix() {
|
||||
t.Fatalf("iat not backdated: %d", claims.Iat)
|
||||
}
|
||||
}
|
||||
|
||||
func newAppCred(t *testing.T, f *appFixture, pemKey string) *appCredential {
|
||||
t.Helper()
|
||||
c, err := newAppCredential(Options{
|
||||
AppID: "123",
|
||||
InstallationID: "456",
|
||||
PrivateKeyPEM: pemKey,
|
||||
apiBaseURL: f.srv.URL,
|
||||
httpClient: f.srv.Client(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("newAppCredential: %v", err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestApp_MintsJWTAndExchangesForInstallationToken(t *testing.T) {
|
||||
pemKey := testRSAKeyPEM(t)
|
||||
f := newAppFixture(t, pemKey)
|
||||
c := newAppCred(t, f, pemKey)
|
||||
|
||||
tok, err := c.Token(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("token: %v", err)
|
||||
}
|
||||
if tok != "ghs_installation_1" {
|
||||
t.Fatalf("token = %q, want ghs_installation_1", tok)
|
||||
}
|
||||
if f.exchanges != 1 {
|
||||
t.Fatalf("exchanges = %d, want 1", f.exchanges)
|
||||
}
|
||||
f.verifyJWT(t)
|
||||
}
|
||||
|
||||
func TestApp_CachesInstallationToken(t *testing.T) {
|
||||
pemKey := testRSAKeyPEM(t)
|
||||
f := newAppFixture(t, pemKey)
|
||||
c := newAppCred(t, f, pemKey)
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
if _, err := c.Token(context.Background()); err != nil {
|
||||
t.Fatalf("token: %v", err)
|
||||
}
|
||||
}
|
||||
if f.exchanges != 1 {
|
||||
t.Fatalf("exchanges = %d, want 1 (token should be cached)", f.exchanges)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApp_RefreshesNearExpiry(t *testing.T) {
|
||||
pemKey := testRSAKeyPEM(t)
|
||||
f := newAppFixture(t, pemKey)
|
||||
// Token expires within refreshSkew, so every call must re-exchange.
|
||||
f.expiresAt = func() time.Time { return time.Now().Add(2 * time.Minute) }
|
||||
c := newAppCred(t, f, pemKey)
|
||||
|
||||
t1, err := c.Token(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("token 1: %v", err)
|
||||
}
|
||||
t2, err := c.Token(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("token 2: %v", err)
|
||||
}
|
||||
if f.exchanges != 2 {
|
||||
t.Fatalf("exchanges = %d, want 2 (near-expiry token must refresh)", f.exchanges)
|
||||
}
|
||||
if t1 == t2 {
|
||||
t.Fatalf("expected a fresh token after refresh, both = %q", t1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApp_ConcurrentTokenSingleFlights(t *testing.T) {
|
||||
pemKey := testRSAKeyPEM(t)
|
||||
f := newAppFixture(t, pemKey)
|
||||
c := newAppCred(t, f, pemKey)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 20; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if _, err := c.Token(context.Background()); err != nil {
|
||||
t.Errorf("token: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if f.exchanges != 1 {
|
||||
t.Fatalf("exchanges = %d, want 1 (concurrent calls must coalesce)", f.exchanges)
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
// Package githubauth provides the process-wide GitHub machine credential used to
|
||||
// authenticate every outbound GitHub request (releases scan, ranged asset header
|
||||
// fetches, and the generic-github byte proxy for private assets). The credential
|
||||
// is delivered via env/secret only — it is never stored per-remote in the DB,
|
||||
// never returned by any API, and never logged.
|
||||
package githubauth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Credential yields a bearer token for GitHub requests. Token may block to mint
|
||||
// or refresh (the GitHub App path); an empty string means "no auth", which only
|
||||
// happens when no credential is configured.
|
||||
type Credential interface {
|
||||
Token(ctx context.Context) (string, error)
|
||||
}
|
||||
|
||||
// Options is the raw, env-sourced auth configuration. Exactly one mode may be
|
||||
// configured: a static token, or a GitHub App (id + installation id + private
|
||||
// key). Partial App configuration is an error (fail closed); no fields at all is
|
||||
// fine and yields a nil credential (anonymous, current behavior).
|
||||
type Options struct {
|
||||
// Token is a Personal Access Token (fine-grained or classic) sent verbatim
|
||||
// as "Authorization: Bearer <token>".
|
||||
Token string
|
||||
|
||||
// GitHub App fields. PrivateKeyPEM and PrivateKeyPath are alternatives; the
|
||||
// inline PEM wins when both are set.
|
||||
AppID string
|
||||
InstallationID string
|
||||
PrivateKeyPEM string
|
||||
PrivateKeyPath string
|
||||
|
||||
// apiBaseURL overrides https://api.github.com for tests. Empty uses the real
|
||||
// endpoint. httpClient likewise overrides the default client for tests.
|
||||
apiBaseURL string
|
||||
httpClient httpDoer
|
||||
}
|
||||
|
||||
// New builds the process credential from options, validating that auth is either
|
||||
// fully configured or fully absent. It returns (nil, nil) when nothing is set.
|
||||
func New(opts Options) (Credential, error) {
|
||||
hasToken := opts.Token != ""
|
||||
hasAppField := opts.AppID != "" || opts.InstallationID != "" ||
|
||||
opts.PrivateKeyPEM != "" || opts.PrivateKeyPath != ""
|
||||
|
||||
switch {
|
||||
case !hasToken && !hasAppField:
|
||||
return nil, nil // no auth configured — anonymous is fine
|
||||
case hasToken && hasAppField:
|
||||
return nil, errors.New("github auth: both a token and GitHub App fields are set; configure exactly one")
|
||||
case hasToken:
|
||||
return staticToken{token: opts.Token}, nil
|
||||
default:
|
||||
return newAppCredential(opts)
|
||||
}
|
||||
}
|
||||
|
||||
// staticToken is a fixed PAT credential.
|
||||
type staticToken struct{ token string }
|
||||
|
||||
func (s staticToken) Token(context.Context) (string, error) { return s.token, nil }
|
||||
|
||||
// server is the process-wide credential set once at startup. A nil value means
|
||||
// no server credential (anonymous). Access is guarded so a late SetServer in a
|
||||
// test is race-free.
|
||||
var (
|
||||
serverMu sync.RWMutex
|
||||
server Credential
|
||||
)
|
||||
|
||||
// SetServer installs the process credential. Call once during startup.
|
||||
func SetServer(c Credential) {
|
||||
serverMu.Lock()
|
||||
server = c
|
||||
serverMu.Unlock()
|
||||
}
|
||||
|
||||
// Server returns the process credential, or nil if none is configured.
|
||||
func Server() Credential {
|
||||
serverMu.RLock()
|
||||
defer serverMu.RUnlock()
|
||||
return server
|
||||
}
|
||||
|
||||
// loadPrivateKeyPEM resolves the App private key bytes from the inline PEM or a
|
||||
// file path, without ever returning the key material in an error message.
|
||||
func loadPrivateKeyPEM(opts Options) ([]byte, error) {
|
||||
if strings.TrimSpace(opts.PrivateKeyPEM) != "" {
|
||||
return []byte(opts.PrivateKeyPEM), nil
|
||||
}
|
||||
if opts.PrivateKeyPath != "" {
|
||||
b, err := os.ReadFile(opts.PrivateKeyPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("github app: read private key file: %w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
return nil, errors.New("github app: no private key configured")
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
package githubauth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNew_NoConfigIsAnonymous(t *testing.T) {
|
||||
c, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if c != nil {
|
||||
t.Fatalf("expected nil credential when nothing configured, got %T", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_TokenMode(t *testing.T) {
|
||||
c, err := New(Options{Token: "ghp_example"})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
tok, err := c.Token(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("token: %v", err)
|
||||
}
|
||||
if tok != "ghp_example" {
|
||||
t.Fatalf("token = %q, want ghp_example", tok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_TokenAndAppConflict(t *testing.T) {
|
||||
_, err := New(Options{Token: "ghp_example", AppID: "123"})
|
||||
if err == nil {
|
||||
t.Fatal("expected error when both token and app fields are set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_PartialAppFailsClosed(t *testing.T) {
|
||||
cases := map[string]Options{
|
||||
"app id without key": {AppID: "123", InstallationID: "456"},
|
||||
"key without app id": {InstallationID: "456", PrivateKeyPEM: testRSAKeyPEM(t)},
|
||||
"app id without inst": {AppID: "123", PrivateKeyPEM: testRSAKeyPEM(t)},
|
||||
}
|
||||
for name, opts := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := New(opts); err == nil {
|
||||
t.Fatalf("expected fail-closed error for %q", name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_AppModeParsesKey(t *testing.T) {
|
||||
c, err := New(Options{
|
||||
AppID: "123",
|
||||
InstallationID: "456",
|
||||
PrivateKeyPEM: testRSAKeyPEM(t),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if _, ok := c.(*appCredential); !ok {
|
||||
t.Fatalf("expected *appCredential, got %T", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_AppModeRejectsBadKey(t *testing.T) {
|
||||
_, err := New(Options{
|
||||
AppID: "123",
|
||||
InstallationID: "456",
|
||||
PrivateKeyPEM: "-----BEGIN RSA PRIVATE KEY-----\nnope\n-----END RSA PRIVATE KEY-----",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for malformed private key")
|
||||
}
|
||||
}
|
||||
@@ -1,27 +1,12 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archive/tar"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -61,349 +46,3 @@ func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte,
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
// --- LocalUploader: hosting real .apk packages -----------------------------
|
||||
|
||||
// ValidateUpload accepts any *.apk and preserves the client-supplied directory
|
||||
// (the arch prefix) as the storage path, since arch cannot be parsed from the
|
||||
// filename alone and the generic uploader hands us only the path. apk clients
|
||||
// fetch packages at <arch>/<file>.apk, so publishers upload to that same path;
|
||||
// AfterUpload records the true arch (from .PKGINFO) for index filtering.
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
clean := strings.TrimPrefix(path.Clean("/"+filePath), "/")
|
||||
filename := clean
|
||||
if i := strings.LastIndex(clean, "/"); i >= 0 {
|
||||
filename = clean[i+1:]
|
||||
}
|
||||
if !strings.HasSuffix(strings.ToLower(filename), ".apk") {
|
||||
return "", "", fmt.Errorf("file must be a .apk package")
|
||||
}
|
||||
return clean, "application/vnd.android.package-archive", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
filename := storagePath
|
||||
if i := strings.LastIndex(storagePath, "/"); i >= 0 {
|
||||
filename = storagePath[i+1:]
|
||||
}
|
||||
return map[string]any{
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
|
||||
|
||||
reader, blobSize, err := blobs.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
slog.Error("alpine metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
raw, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
slog.Error("alpine metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
meta, err := parseApk(raw)
|
||||
if err != nil {
|
||||
slog.Error("alpine metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
meta.RepoName = repoName
|
||||
meta.FilePath = storagePath
|
||||
meta.ContentHash = contentHash
|
||||
meta.DownloadSize = blobSize
|
||||
|
||||
if meta.Name == "" || meta.Arch == "" {
|
||||
slog.Error("alpine metadata: .PKGINFO missing pkgname/arch", "repo", repoName, "path", storagePath)
|
||||
return
|
||||
}
|
||||
|
||||
store, ok := db.(provider.AlpineMetadataStore)
|
||||
if !ok {
|
||||
slog.Error("alpine metadata: store does not support alpine metadata", "repo", repoName)
|
||||
return
|
||||
}
|
||||
if err := store.InsertAlpineMetadata(ctx, meta); err != nil {
|
||||
slog.Error("alpine metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
slog.Info("alpine metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
}
|
||||
|
||||
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
|
||||
deleter, ok := db.(provider.AlpineMetadataDeleter)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if err := deleter.DeleteAlpineMetadata(ctx, repoName, storagePath); err != nil {
|
||||
slog.Error("alpine metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return err
|
||||
}
|
||||
slog.Info("alpine metadata: deleted", "repo", repoName, "path", storagePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- LocalIndexer: generating a per-arch APKINDEX.tar.gz -------------------
|
||||
|
||||
// normalizeIndexPath collapses apk's dot-segment prefix: an /etc/apk/repositories
|
||||
// line of "<url>/api/v1/local/<name>" makes apk request "./<arch>/APKINDEX.tar.gz".
|
||||
// Mirrors deb's flat-repo normalization.
|
||||
func normalizeIndexPath(p string) string {
|
||||
return strings.TrimPrefix(path.Clean("/"+p), "/")
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
|
||||
clean := normalizeIndexPath(reqPath)
|
||||
if !strings.HasSuffix(clean, "APKINDEX.tar.gz") {
|
||||
return false
|
||||
}
|
||||
arch := strings.TrimSuffix(clean, "APKINDEX.tar.gz")
|
||||
arch = strings.Trim(arch, "/")
|
||||
if arch == "" || strings.Contains(arch, "/") {
|
||||
http.Error(w, "APKINDEX must be requested per-arch: <arch>/APKINDEX.tar.gz", http.StatusNotFound)
|
||||
return true
|
||||
}
|
||||
|
||||
reader, ok := files.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "alpine metadata not available", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
metas, err := reader.ListAlpineMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
slog.Warn("alpine: metadata read canceled", "repo", repoName, "error", err)
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return true
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
var filtered []provider.AlpineMetadata
|
||||
for _, m := range metas {
|
||||
if m.Arch == arch {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateAPKIndex(filtered))
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("alpine local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
// --- pure-Go .apk parsing --------------------------------------------------
|
||||
|
||||
// parseApk reads an .apk (up to three concatenated, independently gzipped tar
|
||||
// streams: optional signature, control, data). It locates the control stream by
|
||||
// its .PKGINFO member, computes the apk pull checksum C: = "Q1" +
|
||||
// base64(sha1(<control gzip stream bytes>)), and reads the .PKGINFO fields.
|
||||
func parseApk(raw []byte) (*provider.AlpineMetadata, error) {
|
||||
members, err := gzipMembers(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, m := range members {
|
||||
pkginfo, ok := pkginfoFromTar(m.tar)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
meta := parsePkginfo(pkginfo)
|
||||
sum := sha1.Sum(m.raw)
|
||||
meta.Checksum = "Q1" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
return meta, nil
|
||||
}
|
||||
return nil, errors.New("no .PKGINFO found in any .apk gzip stream")
|
||||
}
|
||||
|
||||
type gzMember struct {
|
||||
raw []byte // the raw bytes of this gzip stream (for the Q1 checksum)
|
||||
tar []byte // the decompressed tar payload
|
||||
}
|
||||
|
||||
// gzipMembers splits the concatenated gzip streams, returning each stream's raw
|
||||
// bytes alongside its decompressed tar. It relies on bytes.Reader being an
|
||||
// io.ByteReader (so compress/gzip does not over-read past a member's trailer)
|
||||
// to recover exact stream boundaries via Multistream(false)+Reset.
|
||||
func gzipMembers(data []byte) ([]gzMember, error) {
|
||||
br := bytes.NewReader(data)
|
||||
zr, err := gzip.NewReader(br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var members []gzMember
|
||||
prev := 0
|
||||
for {
|
||||
zr.Multistream(false)
|
||||
out, err := io.ReadAll(zr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
end := len(data) - br.Len()
|
||||
members = append(members, gzMember{raw: data[prev:end], tar: out})
|
||||
prev = end
|
||||
if err := zr.Reset(br); err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return members, nil
|
||||
}
|
||||
|
||||
func pkginfoFromTar(tarBytes []byte) (string, bool) {
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
if strings.TrimPrefix(hdr.Name, "./") == ".PKGINFO" {
|
||||
b, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return string(b), true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parsePkginfo reads the "key = value" .PKGINFO text, collecting the repeated
|
||||
// depend/provides/install_if keys into slices.
|
||||
func parsePkginfo(text string) *provider.AlpineMetadata {
|
||||
m := &provider.AlpineMetadata{}
|
||||
sc := bufio.NewScanner(strings.NewReader(text))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
idx := strings.Index(line, "=")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(line[:idx])
|
||||
val := strings.TrimSpace(line[idx+1:])
|
||||
switch key {
|
||||
case "pkgname":
|
||||
m.Name = val
|
||||
case "pkgver":
|
||||
m.Version = val
|
||||
case "arch":
|
||||
m.Arch = val
|
||||
case "pkgdesc":
|
||||
m.Description = val
|
||||
case "url":
|
||||
m.URL = val
|
||||
case "license":
|
||||
m.License = val
|
||||
case "origin":
|
||||
m.Origin = val
|
||||
case "maintainer":
|
||||
m.Maintainer = val
|
||||
case "builddate":
|
||||
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
m.BuildTime = n
|
||||
}
|
||||
case "commit":
|
||||
m.Commit = val
|
||||
case "size":
|
||||
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
m.InstalledSize = n
|
||||
}
|
||||
case "provider_priority":
|
||||
m.ProviderPriority = val
|
||||
case "depend":
|
||||
if val != "" {
|
||||
m.Depends = append(m.Depends, val)
|
||||
}
|
||||
case "provides":
|
||||
if val != "" {
|
||||
m.Provides = append(m.Provides, val)
|
||||
}
|
||||
case "install_if":
|
||||
if val != "" {
|
||||
m.InstallIf = append(m.InstallIf, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// generateAPKIndex builds the APKINDEX.tar.gz = gzip(tar(APKINDEX)) for the
|
||||
// given (already arch-filtered) rows. Records are blank-line separated; fields
|
||||
// follow the canonical C/P/V/A/S/I/T/U/L/o/m/t/c/k/D/p/i order and empties are
|
||||
// omitted. Unsigned (clients use --allow-untrusted), matching rpm gpgcheck=0.
|
||||
func generateAPKIndex(metas []provider.AlpineMetadata) []byte {
|
||||
var idx bytes.Buffer
|
||||
for i, m := range metas {
|
||||
if i > 0 {
|
||||
idx.WriteString("\n")
|
||||
}
|
||||
writeField(&idx, "C", m.Checksum)
|
||||
writeField(&idx, "P", m.Name)
|
||||
writeField(&idx, "V", m.Version)
|
||||
writeField(&idx, "A", m.Arch)
|
||||
writeField(&idx, "S", intField(m.DownloadSize))
|
||||
writeField(&idx, "I", intField(m.InstalledSize))
|
||||
writeField(&idx, "T", m.Description)
|
||||
writeField(&idx, "U", m.URL)
|
||||
writeField(&idx, "L", m.License)
|
||||
writeField(&idx, "o", m.Origin)
|
||||
writeField(&idx, "m", m.Maintainer)
|
||||
writeField(&idx, "t", intField(m.BuildTime))
|
||||
writeField(&idx, "c", m.Commit)
|
||||
writeField(&idx, "k", m.ProviderPriority)
|
||||
writeField(&idx, "D", strings.Join(m.Depends, " "))
|
||||
writeField(&idx, "p", strings.Join(m.Provides, " "))
|
||||
writeField(&idx, "i", strings.Join(m.InstallIf, " "))
|
||||
}
|
||||
|
||||
var tarBuf bytes.Buffer
|
||||
tw := tar.NewWriter(&tarBuf)
|
||||
body := idx.Bytes()
|
||||
// ModTime is pinned to the Unix epoch (never wall clock) so APKINDEX.tar.gz
|
||||
// is byte-identical across replicas and regenerations (issue #117); apk
|
||||
// clients ignore the tar mtime.
|
||||
tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg, ModTime: time.Unix(0, 0)})
|
||||
tw.Write(body)
|
||||
tw.Close()
|
||||
|
||||
var gzBuf bytes.Buffer
|
||||
gz := gzip.NewWriter(&gzBuf)
|
||||
gz.Write(tarBuf.Bytes())
|
||||
gz.Close()
|
||||
return gzBuf.Bytes()
|
||||
}
|
||||
|
||||
func writeField(b *bytes.Buffer, key, val string) {
|
||||
if val == "" {
|
||||
return
|
||||
}
|
||||
b.WriteString(key)
|
||||
b.WriteString(":")
|
||||
b.WriteString(val)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
func intField(n int64) string {
|
||||
if n == 0 {
|
||||
return ""
|
||||
}
|
||||
return strconv.FormatInt(n, 10)
|
||||
}
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func apkFixture() []provider.AlpineMetadata {
|
||||
return []provider.AlpineMetadata{
|
||||
{
|
||||
RepoName: "r", FilePath: "x86_64/aaa-1.0-r0.apk", Checksum: "Q1aaa",
|
||||
Name: "aaa", Version: "1.0-r0", Arch: "x86_64", DownloadSize: 100, InstalledSize: 10,
|
||||
Description: "pkg aaa", URL: "https://a", License: "MIT",
|
||||
Depends: []string{"so:libc"}, Provides: []string{"cmd:aaa"}, BuildTime: 1710000000,
|
||||
},
|
||||
{
|
||||
RepoName: "r", FilePath: "x86_64/bbb-2.0-r0.apk", Checksum: "Q1bbb",
|
||||
Name: "bbb", Version: "2.0-r0", Arch: "x86_64", DownloadSize: 200, InstalledSize: 20,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPKIndexDeterministic asserts APKINDEX.tar.gz is byte-identical across two
|
||||
// generations separated by wall-clock time, so the two no-affinity replicas and
|
||||
// every regeneration serve the same bytes (issue #117).
|
||||
func TestAPKIndexDeterministic(t *testing.T) {
|
||||
metas := apkFixture()
|
||||
|
||||
first := generateAPKIndex(metas)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
second := generateAPKIndex(metas)
|
||||
|
||||
if !bytes.Equal(first, second) {
|
||||
t.Error("APKINDEX.tar.gz differs across generations")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPKIndexTarModTimePinned guards the tar header: its ModTime must be the
|
||||
// pinned Unix epoch, never wall clock. Fails if a future edit stamps time.Now().
|
||||
func TestAPKIndexTarModTimePinned(t *testing.T) {
|
||||
metas := apkFixture()
|
||||
|
||||
zr, err := gzip.NewReader(bytes.NewReader(generateAPKIndex(metas)))
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
if !zr.ModTime.IsZero() && zr.ModTime.Unix() != 0 {
|
||||
t.Errorf("gzip header ModTime = %v, want zero/epoch", zr.ModTime)
|
||||
}
|
||||
|
||||
tarBytes, err := io.ReadAll(zr)
|
||||
if err != nil {
|
||||
t.Fatalf("gunzip: %v", err)
|
||||
}
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
hdr, err := tr.Next()
|
||||
if err != nil {
|
||||
t.Fatalf("tar: %v", err)
|
||||
}
|
||||
if hdr.Name != "APKINDEX" {
|
||||
t.Fatalf("tar entry = %q, want APKINDEX", hdr.Name)
|
||||
}
|
||||
if hdr.ModTime.Unix() != 0 {
|
||||
t.Errorf("APKINDEX tar ModTime = %v (unix %d), want epoch (0)", hdr.ModTime, hdr.ModTime.Unix())
|
||||
}
|
||||
}
|
||||
@@ -1,324 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
type fakeBlobReader struct{ data []byte }
|
||||
|
||||
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
|
||||
}
|
||||
|
||||
type errBlobReader struct{}
|
||||
|
||||
func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return nil, 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
// fakeAlpineStore satisfies provider.MetadataStore (shared) and
|
||||
// provider.AlpineMetadataStore, recording the row AfterUpload writes.
|
||||
type fakeAlpineStore struct{ inserted *provider.AlpineMetadata }
|
||||
|
||||
func (f *fakeAlpineStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeAlpineStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
|
||||
func (f *fakeAlpineStore) InsertAlpineMetadata(_ context.Context, m *provider.AlpineMetadata) error {
|
||||
f.inserted = m
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeAlpineDeleter satisfies provider.MetadataDeleter and AlpineMetadataDeleter.
|
||||
type fakeAlpineDeleter struct{ deleted bool }
|
||||
|
||||
func (f *fakeAlpineDeleter) DeleteRPMMetadata(context.Context, string, string) error { return nil }
|
||||
func (f *fakeAlpineDeleter) DeleteDebMetadata(context.Context, string, string) error { return nil }
|
||||
func (f *fakeAlpineDeleter) DeleteAlpineMetadata(context.Context, string, string) error {
|
||||
f.deleted = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeAlpineReader is a FileStore that also serves alpine metadata rows.
|
||||
type fakeAlpineReader struct{ metas []provider.AlpineMetadata }
|
||||
|
||||
func (f fakeAlpineReader) ListAlpineMetadataEntries(context.Context, string) ([]provider.AlpineMetadata, error) {
|
||||
return f.metas, nil
|
||||
}
|
||||
func (f fakeAlpineReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f fakeAlpineReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
type errAlpineReader struct{}
|
||||
|
||||
func (errAlpineReader) ListAlpineMetadataEntries(context.Context, string) ([]provider.AlpineMetadata, error) {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
func (errAlpineReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (errAlpineReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestAlpineValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("x86_64/foo-1.0-r0.apk")
|
||||
if err != nil || sp != "x86_64/foo-1.0-r0.apk" || ct != "application/vnd.android.package-archive" {
|
||||
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
// Dot-segment prefix is normalized away.
|
||||
if sp, _, err := p.ValidateUpload("./aarch64/bar-2.0-r1.apk"); err != nil || sp != "aarch64/bar-2.0-r1.apk" {
|
||||
t.Errorf("dot-seg: sp=%q err=%v", sp, err)
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("foo.rpm"); err == nil {
|
||||
t.Error("expected error for non-apk")
|
||||
}
|
||||
resp := p.UploadResponse("x86_64/foo-1.0-r0.apk", "sha256:abc", 42)
|
||||
if resp["filename"] != "foo-1.0-r0.apk" || resp["content_hash"] != "sha256:abc" || resp["size_bytes"] != int64(42) {
|
||||
t.Errorf("upload response %v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineAfterUpload(t *testing.T) {
|
||||
data := testsupport.MinimalApk("hello", "1.0-r0", "x86_64")
|
||||
store := &fakeAlpineStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "myrepo", "x86_64/hello-1.0-r0.apk",
|
||||
"sha256:deadbeef", fakeBlobReader{data: data}, store)
|
||||
|
||||
m := store.inserted
|
||||
if m == nil {
|
||||
t.Fatal("no metadata inserted")
|
||||
}
|
||||
if m.Name != "hello" || m.Version != "1.0-r0" || m.Arch != "x86_64" {
|
||||
t.Errorf("unexpected metadata: %+v", m)
|
||||
}
|
||||
if m.DownloadSize != int64(len(data)) {
|
||||
t.Errorf("DownloadSize = %d, want %d", m.DownloadSize, len(data))
|
||||
}
|
||||
if m.InstalledSize != 4 {
|
||||
t.Errorf("InstalledSize = %d, want 4", m.InstalledSize)
|
||||
}
|
||||
if m.License != "MIT" || m.Origin != "hello" || !strings.HasPrefix(m.Maintainer, "e2e") {
|
||||
t.Errorf("scalar fields not parsed: %+v", m)
|
||||
}
|
||||
if len(m.Depends) != 1 || m.Depends[0] != "so:libc.musl-x86_64.so.1" {
|
||||
t.Errorf("Depends = %v", m.Depends)
|
||||
}
|
||||
if len(m.Provides) != 1 || m.Provides[0] != "cmd:hello=1.0-r0" {
|
||||
t.Errorf("Provides = %v", m.Provides)
|
||||
}
|
||||
|
||||
// The Q1 checksum is the sha1 of the CONTROL gzip stream (the member whose
|
||||
// tar carries .PKGINFO), not of the whole file.
|
||||
controlRaw := controlStreamBytes(t, data)
|
||||
sum := sha1.Sum(controlRaw)
|
||||
want := "Q1" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
if m.Checksum != want {
|
||||
t.Errorf("Checksum = %q, want %q (sha1 of control stream)", m.Checksum, want)
|
||||
}
|
||||
// And explicitly NOT the sha1 of the whole apk.
|
||||
whole := sha1.Sum(data)
|
||||
if m.Checksum == "Q1"+base64.StdEncoding.EncodeToString(whole[:]) {
|
||||
t.Error("Checksum was computed over the whole file, not the control stream")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineAfterUploadErrors(t *testing.T) {
|
||||
store := &fakeAlpineStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "x86_64/p.apk", "sha256:x", errBlobReader{}, store)
|
||||
if store.inserted != nil {
|
||||
t.Error("no metadata should be inserted on download error")
|
||||
}
|
||||
store2 := &fakeAlpineStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "x86_64/p.apk", "sha256:x", fakeBlobReader{data: []byte("not an apk")}, store2)
|
||||
if store2.inserted != nil {
|
||||
t.Error("no metadata should be inserted on parse error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineAfterDelete(t *testing.T) {
|
||||
d := &fakeAlpineDeleter{}
|
||||
if err := (&Provider{}).AfterDelete(context.Background(), "r", "x86_64/p.apk", d); err != nil {
|
||||
t.Fatalf("AfterDelete: %v", err)
|
||||
}
|
||||
if !d.deleted {
|
||||
t.Error("DeleteAlpineMetadata not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineServeLocalIndex(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
|
||||
{Name: "aaa", Version: "1.0-r0", Arch: "x86_64", Checksum: "Q1aaa", DownloadSize: 100, InstalledSize: 10,
|
||||
Description: "pkg aaa", URL: "https://a", License: "MIT", Depends: []string{"so:libc"}, Provides: []string{"cmd:aaa"}},
|
||||
{Name: "bbb", Version: "2.0-r0", Arch: "aarch64", Checksum: "Q1bbb", DownloadSize: 200, InstalledSize: 20},
|
||||
}}
|
||||
|
||||
// x86_64 index contains only aaa, with its fields, and not bbb.
|
||||
w := serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("code %d", w.Code)
|
||||
}
|
||||
idx := untarIndex(t, w.Body.Bytes())
|
||||
for _, want := range []string{"C:Q1aaa", "P:aaa", "V:1.0-r0", "A:x86_64", "S:100", "I:10", "T:pkg aaa", "U:https://a", "L:MIT", "D:so:libc", "p:cmd:aaa"} {
|
||||
if !strings.Contains(idx, want) {
|
||||
t.Errorf("x86_64 APKINDEX missing %q:\n%s", want, idx)
|
||||
}
|
||||
}
|
||||
if strings.Contains(idx, "P:bbb") {
|
||||
t.Errorf("x86_64 APKINDEX leaked aarch64 package:\n%s", idx)
|
||||
}
|
||||
|
||||
// aarch64 index contains only bbb.
|
||||
w = serveIndex(t, p, reader, "aarch64/APKINDEX.tar.gz")
|
||||
idx = untarIndex(t, w.Body.Bytes())
|
||||
if !strings.Contains(idx, "P:bbb") || strings.Contains(idx, "P:aaa") {
|
||||
t.Errorf("aarch64 filtering wrong:\n%s", idx)
|
||||
}
|
||||
|
||||
// Non-index and .apk paths are not owned by the indexer.
|
||||
for _, path := range []string{"x86_64/foo-1.0-r0.apk", "x86_64/", "README"} {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if p.ServeLocalIndex(w, r, reader, "repo", path) {
|
||||
t.Errorf("ServeLocalIndex should return false for %q", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Empty fields are omitted from the record (bbb has no description/url).
|
||||
func TestAlpineIndexOmitsEmptyFields(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
|
||||
{Name: "bbb", Version: "2.0-r0", Arch: "x86_64", Checksum: "Q1bbb", DownloadSize: 200, InstalledSize: 20},
|
||||
}}
|
||||
idx := untarIndex(t, serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz").Body.Bytes())
|
||||
for _, absent := range []string{"T:", "U:", "L:", "D:", "p:", "i:", "o:", "m:", "c:", "k:"} {
|
||||
if strings.Contains(idx, absent) {
|
||||
t.Errorf("empty field %q should be omitted:\n%s", absent, idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// apk requests "./<arch>/APKINDEX.tar.gz" for a bare repo base URL; the
|
||||
// dot-segment must be collapsed and yield the same bytes as the plain path.
|
||||
func TestAlpineServeLocalIndexDotSegment(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
|
||||
{Name: "aaa", Version: "1.0-r0", Arch: "x86_64", Checksum: "Q1aaa", DownloadSize: 100, InstalledSize: 10},
|
||||
}}
|
||||
plain := untarIndex(t, serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz").Body.Bytes())
|
||||
dotted := untarIndex(t, serveIndex(t, p, reader, "./x86_64/APKINDEX.tar.gz").Body.Bytes())
|
||||
if plain != dotted {
|
||||
t.Errorf("dot-segment path differs:\nplain=%q\ndotted=%q", plain, dotted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineServeLocalIndexArchRequired(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{}
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/APKINDEX.tar.gz", nil)
|
||||
if !p.ServeLocalIndex(w, r, reader, "repo", "APKINDEX.tar.gz") {
|
||||
t.Fatal("bare APKINDEX should be owned (and rejected) by the indexer")
|
||||
}
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("bare APKINDEX code = %d, want 404", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineServeMetadataError(t *testing.T) {
|
||||
p := &Provider{}
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/x86_64/APKINDEX.tar.gz", nil)
|
||||
p.ServeLocalIndex(w, r, errAlpineReader{}, "repo", "x86_64/APKINDEX.tar.gz")
|
||||
if w.Code != 500 {
|
||||
t.Errorf("failing reader code = %d, want 500", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineGenerateLocalIndexUnsupported(t *testing.T) {
|
||||
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeAlpineReader{}, "r", "x86_64/APKINDEX.tar.gz"); err == nil {
|
||||
t.Error("expected unsupported error")
|
||||
}
|
||||
}
|
||||
|
||||
func serveIndex(t *testing.T, p *Provider, files provider.FileStore, path string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if !p.ServeLocalIndex(w, r, files, "repo", path) {
|
||||
t.Fatalf("ServeLocalIndex returned false for %q", path)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// untarIndex un-gzips and un-tars an APKINDEX.tar.gz and returns the APKINDEX text.
|
||||
func untarIndex(t *testing.T, gzTar []byte) string {
|
||||
t.Helper()
|
||||
zr, err := gzip.NewReader(bytes.NewReader(gzTar))
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not gzip: %v", err)
|
||||
}
|
||||
tarBytes, _ := io.ReadAll(zr)
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not tar: %v", err)
|
||||
}
|
||||
if hdr.Name == "APKINDEX" {
|
||||
b, _ := io.ReadAll(tr)
|
||||
return string(b)
|
||||
}
|
||||
}
|
||||
t.Fatal("no APKINDEX member in tarball")
|
||||
return ""
|
||||
}
|
||||
|
||||
// controlStreamBytes returns the raw bytes of the gzip stream whose tar carries
|
||||
// .PKGINFO, so the test can independently compute the expected Q1 checksum.
|
||||
func controlStreamBytes(t *testing.T, apk []byte) []byte {
|
||||
t.Helper()
|
||||
br := bytes.NewReader(apk)
|
||||
zr, err := gzip.NewReader(br)
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
prev := 0
|
||||
for {
|
||||
zr.Multistream(false)
|
||||
out, _ := io.ReadAll(zr)
|
||||
end := len(apk) - br.Len()
|
||||
tr := tar.NewReader(bytes.NewReader(out))
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if strings.TrimPrefix(h.Name, "./") == ".PKGINFO" {
|
||||
return apk[prev:end]
|
||||
}
|
||||
}
|
||||
prev = end
|
||||
if err := zr.Reset(br); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
t.Fatal("no control stream found")
|
||||
return nil
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestType(t *testing.T) {
|
||||
if (&Provider{}).Type() != models.PackageAlpine {
|
||||
t.Fatal("wrong type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("v3.19/main/x86_64/APKINDEX.tar.gz") != provider.Mutable {
|
||||
t.Error("APKINDEX should be mutable")
|
||||
}
|
||||
if p.Classify("v3.19/main/x86_64/curl-8.0-r0.apk") != provider.Immutable {
|
||||
t.Error("apk should be immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
cases := map[string]string{
|
||||
"pkg.apk": "application/vnd.android.package-archive",
|
||||
"APKINDEX.tar.gz": "application/gzip",
|
||||
"something.random": "application/octet-stream",
|
||||
}
|
||||
for path, want := range cases {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
p := &Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://dl-cdn.alpinelinux.org/alpine/"}, "/v3.19/main/x86_64/curl.apk")
|
||||
if got != "https://dl-cdn.alpinelinux.org/alpine/v3.19/main/x86_64/curl.apk" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
if out, err := (&Provider{}).RewriteResponse([]byte("x"), models.Remote{}, "http://proxy"); out != nil || err != nil {
|
||||
t.Error("alpine never rewrites")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthHeaders(t *testing.T) {
|
||||
h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("expected auth header")
|
||||
}
|
||||
}
|
||||
@@ -1,714 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// gitHubProvider is the process-wide singleton for github_alpine. The background
|
||||
// Syncer binds its shared rate limiter and work queue onto this instance so the
|
||||
// request path and the syncer drive the same derive machinery.
|
||||
var gitHubProvider = newGitHubProvider()
|
||||
|
||||
func init() {
|
||||
provider.Register(gitHubProvider)
|
||||
}
|
||||
|
||||
// Tuning knobs for the no-precache control fetch. An .apk is up to three
|
||||
// concatenated gzip streams (optional signature, control, data); the control
|
||||
// stream carrying .PKGINFO sits near the front, so a small prefix reliably
|
||||
// covers it.
|
||||
const (
|
||||
defaultHeaderRangeInitial = 32 << 10 // 32 KiB — covers the control stream of almost every .apk
|
||||
defaultHeaderRangeMax = 16 << 20 // 16 MiB — give up past this and skip the asset
|
||||
defaultReleasePageCap = 10 // 100 releases/page * 10 pages
|
||||
|
||||
defaultScanTimeout = 10 * time.Minute
|
||||
defaultServeTimeout = 30 * time.Second
|
||||
defaultColdWait = 8 * time.Second
|
||||
)
|
||||
|
||||
// GitHubProvider is a metadata-only remote: it scans a GitHub repo's releases
|
||||
// for .apk assets, derives per-asset .PKGINFO metadata via a ranged prefix fetch
|
||||
// (never downloading whole packages), synthesizes a per-arch APKINDEX from that
|
||||
// cached metadata, and redirects package downloads to a backend "releases_remote"
|
||||
// (the generic github.com remote) that serves the actual bytes.
|
||||
type GitHubProvider struct {
|
||||
client *http.Client
|
||||
|
||||
headerInitial int64
|
||||
headerMax int64
|
||||
pageCap int
|
||||
scanTimeout time.Duration
|
||||
serveTimeout time.Duration
|
||||
coldWait time.Duration
|
||||
|
||||
limiter *rate.Limiter
|
||||
syncer *Syncer
|
||||
|
||||
serverCred githubauth.Credential
|
||||
|
||||
mu sync.Mutex
|
||||
scanning map[string]bool
|
||||
lastScan map[string]time.Time
|
||||
}
|
||||
|
||||
func newGitHubProvider() *GitHubProvider {
|
||||
return &GitHubProvider{
|
||||
client: &http.Client{},
|
||||
headerInitial: defaultHeaderRangeInitial,
|
||||
headerMax: defaultHeaderRangeMax,
|
||||
pageCap: defaultReleasePageCap,
|
||||
scanTimeout: defaultScanTimeout,
|
||||
serveTimeout: defaultServeTimeout,
|
||||
coldWait: defaultColdWait,
|
||||
scanning: map[string]bool{},
|
||||
lastScan: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) limiterWait(ctx context.Context) error {
|
||||
if p.limiter == nil {
|
||||
return nil
|
||||
}
|
||||
return p.limiter.Wait(ctx)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) Type() models.PackageType { return models.PackageGitHubAlpine }
|
||||
|
||||
func (p *GitHubProvider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "APKINDEX.tar.gz") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) ContentType(path string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".apk"):
|
||||
return "application/vnd.android.package-archive"
|
||||
case strings.HasSuffix(path, ".tar.gz"):
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error) {
|
||||
return p.githubHeaders(ctx, remote, false)
|
||||
}
|
||||
|
||||
// ServeRemote answers a request against a github_alpine remote. It refreshes the
|
||||
// derived metadata (bounded by mutable_ttl), serves a synthesized per-arch
|
||||
// APKINDEX.tar.gz, and 302-redirects .apk downloads to the backend
|
||||
// releases_remote. Returns false only for paths it does not own.
|
||||
func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, reqPath, proxyBaseURL string, store provider.RemoteMetadataStore) bool {
|
||||
p.onRequest(remote, store)
|
||||
|
||||
// apk requests the index at "./<arch>/APKINDEX.tar.gz"; collapse the
|
||||
// dot-segment before matching, mirroring the local indexer.
|
||||
path := normalizeIndexPath(reqPath)
|
||||
|
||||
if strings.HasSuffix(path, "APKINDEX.tar.gz") {
|
||||
p.serveIndex(w, r, remote, path, store)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasSuffix(path, ".apk") {
|
||||
if remote.ReleasesRemote == "" {
|
||||
http.Error(w, "github_alpine remote has no releases_remote configured for downloads", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
p.serveApkRedirect(w, r, remote, path, proxyBaseURL, store)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// serveApkRedirect resolves an apk-reconstructed download path — apk builds
|
||||
// "<arch>/<name>-<version>.apk" itself because APKINDEX carries no filename — to
|
||||
// the real github-relative asset path stored on the metadata row, then redirects
|
||||
// to the backend releases_remote. Passing the inbound path through verbatim would
|
||||
// point at a nonexistent, allowlist-denied github.com path.
|
||||
func (p *GitHubProvider) serveApkRedirect(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store provider.RemoteMetadataStore) {
|
||||
arch := strings.TrimSuffix(path[:strings.LastIndex(path, "/")+1], "/")
|
||||
basename := path[strings.LastIndex(path, "/")+1:]
|
||||
if arch == "" || strings.Contains(arch, "/") {
|
||||
http.Error(w, "apk download must be requested per-arch: <arch>/<name>-<version>.apk", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
reader, ok := store.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "alpine metadata not available", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), p.serveTimeout)
|
||||
defer cancel()
|
||||
rows, err := reader.ListAlpineMetadataEntries(sctx, remote.Name)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
for _, row := range rows {
|
||||
if row.Arch == arch && row.Name+"-"+row.Version+".apk" == basename {
|
||||
loc := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.ReleasesRemote + "/" + strings.TrimLeft(row.FilePath, "/")
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
http.Error(w, "package not found", http.StatusNotFound)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) serveIndex(w http.ResponseWriter, r *http.Request, remote models.Remote, path string, store provider.RemoteMetadataStore) {
|
||||
arch := strings.TrimSuffix(path, "APKINDEX.tar.gz")
|
||||
arch = strings.Trim(arch, "/")
|
||||
if arch == "" || strings.Contains(arch, "/") {
|
||||
http.Error(w, "APKINDEX must be requested per-arch: <arch>/APKINDEX.tar.gz", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Serve on a context detached from the inbound request so a client disconnect
|
||||
// never cancels the metadata DB read and surfaces as a 500.
|
||||
sctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), p.serveTimeout)
|
||||
defer cancel()
|
||||
|
||||
if p.syncer != nil && !p.ensurePrimed(sctx, remote, store) {
|
||||
w.Header().Set("Retry-After", "5")
|
||||
http.Error(w, "metadata is being prepared, retry shortly", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
reader, ok := store.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "alpine metadata not available", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
metas, err := reader.ListAlpineMetadataEntries(sctx, remote.Name)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
var filtered []provider.AlpineMetadata
|
||||
for _, m := range metas {
|
||||
if m.Arch == arch {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateAPKIndex(filtered))
|
||||
}
|
||||
|
||||
// onRequest keeps a remote's derived metadata fresh off the request path.
|
||||
func (p *GitHubProvider) onRequest(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
if p.syncer != nil {
|
||||
p.syncer.enqueue(remote, false)
|
||||
return
|
||||
}
|
||||
p.refresh(remote, store)
|
||||
}
|
||||
|
||||
// ensurePrimed returns true once the remote has at least one cached row. On an
|
||||
// empty cache it enqueues a prime and polls briefly for it to land.
|
||||
func (p *GitHubProvider) ensurePrimed(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) bool {
|
||||
if !p.cacheEmpty(ctx, store, remote.Name) {
|
||||
return true
|
||||
}
|
||||
if p.syncer != nil {
|
||||
p.syncer.enqueue(remote, true)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(p.coldWait)
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-time.After(400 * time.Millisecond):
|
||||
}
|
||||
if !p.cacheEmpty(ctx, store, remote.Name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) cacheEmpty(ctx context.Context, store provider.RemoteMetadataStore, name string) bool {
|
||||
reader, ok := store.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
rows, err := reader.ListAlpineMetadataEntries(ctx, name)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return len(rows) == 0
|
||||
}
|
||||
|
||||
// refresh brings the derived metadata up to date without coupling the scan to
|
||||
// the inbound request (legacy inline path used without a syncer / in unit tests).
|
||||
func (p *GitHubProvider) refresh(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
ttl := time.Duration(remote.MutableTTL) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = 5 * time.Minute
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
last, ok := p.lastScan[remote.Name]
|
||||
fresh := ok && time.Since(last) < ttl
|
||||
if fresh || p.scanning[remote.Name] {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.scanning[remote.Name] = true
|
||||
p.mu.Unlock()
|
||||
|
||||
if p.cacheEmpty(context.Background(), store, remote.Name) {
|
||||
p.runScan(remote, store)
|
||||
return
|
||||
}
|
||||
go p.runScan(remote, store)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) runScan(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
delete(p.scanning, remote.Name)
|
||||
p.mu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), p.scanTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := p.scan(ctx, remote, store); err != nil {
|
||||
slog.Error("github_alpine: release scan failed", "remote", remote.Name, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.lastScan[remote.Name] = time.Now()
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// scan runs a full unconditional derive. Retained for the legacy inline refresh
|
||||
// path and existing tests; the syncer uses scanWithState.
|
||||
func (p *GitHubProvider) scan(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) error {
|
||||
_, _, err := p.scanWithState(ctx, remote, store, "")
|
||||
return err
|
||||
}
|
||||
|
||||
// scanWithState derives metadata incrementally. It sends the prior releases-list
|
||||
// ETag as a conditional request: a 304 means nothing changed. On a 200 it diffs
|
||||
// the release assets against the cache, derives only new/changed assets, prunes
|
||||
// assets that disappeared, and returns the new ETag.
|
||||
func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore, etag string) (newEtag string, changed bool, err error) {
|
||||
inserter, ok := store.(provider.AlpineMetadataStore)
|
||||
if !ok {
|
||||
return etag, false, errors.New("store does not support alpine metadata writes")
|
||||
}
|
||||
deleter, ok := store.(provider.AlpineMetadataDeleter)
|
||||
if !ok {
|
||||
return etag, false, errors.New("store does not support alpine metadata deletes")
|
||||
}
|
||||
reader, ok := store.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
return etag, false, errors.New("store does not support alpine metadata reads")
|
||||
}
|
||||
|
||||
releases, newEtag, notModified, err := p.fetchReleases(ctx, remote, etag)
|
||||
if err != nil {
|
||||
return etag, false, err
|
||||
}
|
||||
if notModified {
|
||||
return etag, false, nil
|
||||
}
|
||||
|
||||
existing, err := reader.ListAlpineMetadataEntries(ctx, remote.Name)
|
||||
if err != nil {
|
||||
return newEtag, false, err
|
||||
}
|
||||
existingByPath := make(map[string]provider.AlpineMetadata, len(existing))
|
||||
for _, m := range existing {
|
||||
existingByPath[m.FilePath] = m
|
||||
}
|
||||
|
||||
allow, err := compilePatterns(remote.Patterns)
|
||||
if err != nil {
|
||||
return newEtag, false, err
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
for _, rel := range releases {
|
||||
if rel.Draft {
|
||||
continue
|
||||
}
|
||||
for _, asset := range rel.Assets {
|
||||
if !strings.HasSuffix(strings.ToLower(asset.Name), ".apk") {
|
||||
continue
|
||||
}
|
||||
if !matchesAny(allow, asset.Name) {
|
||||
continue
|
||||
}
|
||||
fp := assetPath(asset)
|
||||
if fp == "" {
|
||||
continue
|
||||
}
|
||||
seen[fp] = true
|
||||
|
||||
if cur, ok := existingByPath[fp]; ok {
|
||||
if asset.Digest == "" || cur.ContentHash == asset.Digest {
|
||||
continue
|
||||
}
|
||||
_ = deleter.DeleteAlpineMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
|
||||
meta, err := p.deriveAsset(ctx, remote, asset, fp)
|
||||
if err != nil {
|
||||
slog.Warn("github_alpine: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
if err := inserter.InsertAlpineMetadata(ctx, meta); err != nil {
|
||||
slog.Error("github_alpine: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
slog.Info("github_alpine: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
}
|
||||
}
|
||||
|
||||
for fp := range existingByPath {
|
||||
if !seen[fp] {
|
||||
_ = deleter.DeleteAlpineMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
}
|
||||
return newEtag, true, nil
|
||||
}
|
||||
|
||||
type ghRelease struct {
|
||||
TagName string `json:"tag_name"`
|
||||
Draft bool `json:"draft"`
|
||||
Assets []ghAsset `json:"assets"`
|
||||
}
|
||||
|
||||
type ghAsset struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
BrowserDownloadURL string `json:"browser_download_url"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// fetchReleases lists a repo's releases, sending the prior ETag as If-None-Match
|
||||
// on page 1 so an unchanged repo short-circuits to notModified. Every call waits
|
||||
// on the shared limiter first.
|
||||
func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote, etag string) (all []ghRelease, newEtag string, notModified bool, err error) {
|
||||
base := strings.TrimRight(remote.BaseURL, "/") + "/releases"
|
||||
for page := 1; page <= p.pageCap; page++ {
|
||||
u := fmt.Sprintf("%s?per_page=100&page=%d", base, page)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
hdr, err := p.githubHeaders(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
copyHeaders(req, hdr)
|
||||
if page == 1 && etag != "" {
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
|
||||
if err := p.limiterWait(ctx); err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
if page == 1 && resp.StatusCode == http.StatusNotModified {
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, etag, true, nil
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
respEtag := resp.Header.Get("ETag")
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
|
||||
}
|
||||
if page == 1 {
|
||||
newEtag = respEtag
|
||||
}
|
||||
var releases []ghRelease
|
||||
if err := json.Unmarshal(body, &releases); err != nil {
|
||||
return nil, "", false, fmt.Errorf("decode releases: %w", err)
|
||||
}
|
||||
if len(releases) == 0 {
|
||||
break
|
||||
}
|
||||
all = append(all, releases...)
|
||||
if len(releases) < 100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return all, newEtag, false, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote, asset ghAsset, fp string) (*provider.AlpineMetadata, error) {
|
||||
meta, err := p.fetchPkginfo(ctx, remote, asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if meta.Name == "" || meta.Arch == "" {
|
||||
return nil, errors.New(".PKGINFO missing pkgname/arch")
|
||||
}
|
||||
|
||||
meta.RepoName = remote.Name
|
||||
meta.FilePath = fp
|
||||
// S: the on-disk .apk size comes straight from the releases API, so we never
|
||||
// download the body just to size it.
|
||||
meta.DownloadSize = asset.Size
|
||||
// ContentHash records the GitHub asset digest (when present) purely so the
|
||||
// next scan can detect a changed asset; unlike deb it is not the index
|
||||
// checksum (that is the Q1 control-stream sum already set in fetchPkginfo).
|
||||
if asset.Digest != "" {
|
||||
meta.ContentHash = asset.Digest
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
// fetchPkginfo pulls only the front of the .apk with a ranged GET and derives the
|
||||
// .PKGINFO fields plus the apk pull checksum (C: = Q1 + base64(sha1(control gzip
|
||||
// stream))). The control stream sits near the front, so a small prefix suffices;
|
||||
// a prefix that truncates it doubles the range and retries.
|
||||
func (p *GitHubProvider) fetchPkginfo(ctx context.Context, remote models.Remote, downloadURL string) (*provider.AlpineMetadata, error) {
|
||||
n := p.headerInitial
|
||||
for {
|
||||
body, full, err := p.rangeGet(ctx, remote, downloadURL, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meta, complete, perr := pkginfoFromPrefix(body)
|
||||
if perr != nil {
|
||||
return nil, fmt.Errorf("parse apk .PKGINFO: %w", perr)
|
||||
}
|
||||
if complete {
|
||||
return meta, nil
|
||||
}
|
||||
if full || n >= p.headerMax {
|
||||
return nil, fmt.Errorf(".PKGINFO not found within %d bytes of %s", n, downloadURL)
|
||||
}
|
||||
n *= 2
|
||||
if n > p.headerMax {
|
||||
n = p.headerMax
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pkginfoFromPrefix parses the concatenated gzip streams present in a front
|
||||
// prefix of an .apk. It walks each fully-covered gzip member until it finds the
|
||||
// control stream (the one whose tar carries .PKGINFO), computes the Q1 pull
|
||||
// checksum from that stream's raw bytes, and reads the .PKGINFO fields. A prefix
|
||||
// too short to fully cover the control stream returns complete=false so the
|
||||
// caller can widen the range.
|
||||
func pkginfoFromPrefix(prefix []byte) (meta *provider.AlpineMetadata, complete bool, err error) {
|
||||
br := bytes.NewReader(prefix)
|
||||
zr, zerr := gzip.NewReader(br)
|
||||
if zerr != nil {
|
||||
if zerr == io.EOF || zerr == io.ErrUnexpectedEOF {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, zerr
|
||||
}
|
||||
prev := 0
|
||||
for {
|
||||
zr.Multistream(false)
|
||||
out, rerr := io.ReadAll(zr)
|
||||
if rerr != nil {
|
||||
// A member truncated by the range boundary is not an error — widen.
|
||||
if rerr == io.ErrUnexpectedEOF || rerr == io.EOF {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, rerr
|
||||
}
|
||||
end := len(prefix) - br.Len()
|
||||
raw := prefix[prev:end]
|
||||
|
||||
if pkginfo, ok := pkginfoFromTar(out); ok {
|
||||
m := parsePkginfo(pkginfo)
|
||||
sum := sha1.Sum(raw)
|
||||
m.Checksum = "Q1" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
return m, true, nil
|
||||
}
|
||||
|
||||
prev = end
|
||||
if rsterr := zr.Reset(br); rsterr != nil {
|
||||
if rsterr == io.EOF {
|
||||
// No more complete members in the prefix; the control stream is
|
||||
// either not covered yet or genuinely absent — let the caller
|
||||
// decide by widening (or hitting the full-object guard).
|
||||
return nil, false, nil
|
||||
}
|
||||
if rsterr == io.ErrUnexpectedEOF {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, rsterr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rangeGet returns the first n bytes of downloadURL. full is true when the
|
||||
// response body was shorter than n (i.e. we already have the whole object).
|
||||
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
hdr, err := p.githubHeaders(ctx, remote, false)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
copyHeaders(req, hdr)
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", n-1))
|
||||
|
||||
if err := p.limiterWait(ctx); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
full := int64(len(body)) < n
|
||||
return body, full, nil
|
||||
}
|
||||
|
||||
// assetPath is the package's location relative to github.com — the path the
|
||||
// backend releases_remote (base https://github.com) proxies. It doubles as the
|
||||
// alpine_metadata key and the redirect target, so an .apk download resolves back
|
||||
// to this remote and redirects to the backend.
|
||||
func assetPath(asset ghAsset) string {
|
||||
u, err := url.Parse(asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimPrefix(u.Path, "/")
|
||||
}
|
||||
|
||||
// githubHeaders builds the outbound headers for a GitHub request, attaching a
|
||||
// bearer credential when one is available. A per-remote credential wins; absent
|
||||
// that, the process-wide server credential is used; absent both, the request is
|
||||
// unauthenticated.
|
||||
func (p *GitHubProvider) githubHeaders(ctx context.Context, remote models.Remote, api bool) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if api {
|
||||
h.Set("Accept", "application/vnd.github+json")
|
||||
h.Set("X-GitHub-Api-Version", "2022-11-28")
|
||||
}
|
||||
tok, err := p.githubToken(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tok != "" {
|
||||
h.Set("Authorization", "Bearer "+tok)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// githubToken resolves the bearer token for a remote. Precedence: a per-remote
|
||||
// credential (password, then username) overrides the server credential.
|
||||
func (p *GitHubProvider) githubToken(ctx context.Context, remote models.Remote) (string, error) {
|
||||
if remote.Password != "" {
|
||||
return remote.Password, nil
|
||||
}
|
||||
if remote.Username != "" {
|
||||
return remote.Username, nil
|
||||
}
|
||||
if c := p.serverCredential(); c != nil {
|
||||
return c.Token(ctx)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) serverCredential() githubauth.Credential {
|
||||
if p.serverCred != nil {
|
||||
return p.serverCred
|
||||
}
|
||||
return githubauth.Server()
|
||||
}
|
||||
|
||||
func copyHeaders(req *http.Request, h http.Header) {
|
||||
for k, vals := range h {
|
||||
for _, v := range vals {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func compilePatterns(patterns []string) ([]*regexp.Regexp, error) {
|
||||
var out []*regexp.Regexp
|
||||
for _, p := range patterns {
|
||||
re, err := regexp.Compile(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid pattern %q: %w", p, err)
|
||||
}
|
||||
out = append(out, re)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func matchesAny(res []*regexp.Regexp, s string) bool {
|
||||
if len(res) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, re := range res {
|
||||
if re.MatchString(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,497 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeStore is an in-memory provider.RemoteMetadataStore + AlpineMetadata
|
||||
// store/reader/deleter keyed by file_path, mirroring the (repo_name, file_path)
|
||||
// uniqueness of the real alpine_metadata table.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
rows map[string]provider.AlpineMetadata
|
||||
}
|
||||
|
||||
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.AlpineMetadata{}} }
|
||||
|
||||
func (f *fakeStore) InsertAlpineMetadata(_ context.Context, m *provider.AlpineMetadata) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.rows[m.FilePath]; ok {
|
||||
return nil // ON CONFLICT DO NOTHING
|
||||
}
|
||||
f.rows[m.FilePath] = *m
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) DeleteAlpineMetadata(_ context.Context, _, filePath string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.rows, filePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) ListAlpineMetadataEntries(ctx context.Context, _ string) ([]provider.AlpineMetadata, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := make([]provider.AlpineMetadata, 0, len(f.rows))
|
||||
for _, m := range f.rows {
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The generic RemoteMetadataStore surface (rpm/deb) is unused by the alpine
|
||||
// github provider but required to satisfy the interface passed to ServeRemote.
|
||||
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeStore) DeleteRPMMetadata(context.Context, string, string) error { return nil }
|
||||
func (f *fakeStore) ListRPMMetadataEntries(context.Context, string) ([]provider.RPMMetadata, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
|
||||
func (f *fakeStore) DeleteDebMetadata(context.Context, string, string) error { return nil }
|
||||
|
||||
var _ provider.RemoteMetadataStore = (*fakeStore)(nil)
|
||||
|
||||
// githubFixture serves the releases API and the .apk asset downloads (with Range
|
||||
// support) for a set of packages. digest controls whether the asset carries a
|
||||
// sha256 digest (change-detection path) or not.
|
||||
type githubFixture struct {
|
||||
srv *httptest.Server
|
||||
apkBytes map[string][]byte
|
||||
rangeHit map[string]int
|
||||
fullHit map[string]int
|
||||
etag string
|
||||
releasesHit int
|
||||
notModHit int
|
||||
releaseAuth string
|
||||
assetAuth string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
t.Helper()
|
||||
f := &githubFixture{
|
||||
apkBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
}
|
||||
f.apkBytes["demo-1.2.3-r0.apk"] = testsupport.MinimalApk("demo", "1.2.3-r0", "x86_64")
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) {
|
||||
page := r.URL.Query().Get("page")
|
||||
if page != "" && page != "1" {
|
||||
w.Write([]byte("[]"))
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.releasesHit++
|
||||
f.releaseAuth = r.Header.Get("Authorization")
|
||||
etag := f.etag
|
||||
if etag != "" && r.Header.Get("If-None-Match") == etag {
|
||||
f.notModHit++
|
||||
f.mu.Unlock()
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if etag != "" {
|
||||
w.Header().Set("ETag", etag)
|
||||
}
|
||||
var assets []map[string]any
|
||||
for name := range f.apkBytes {
|
||||
a := map[string]any{
|
||||
"name": name,
|
||||
"size": len(f.apkBytes[name]),
|
||||
"browser_download_url": f.srv.URL + "/acme/tools/releases/download/v1.2.3/" + name,
|
||||
}
|
||||
if withDigest {
|
||||
sum := sha256.Sum256(f.apkBytes[name])
|
||||
a["digest"] = "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
assets = append(assets, a)
|
||||
}
|
||||
rel := []map[string]any{{"tag_name": "v1.2.3", "draft": false, "assets": assets}}
|
||||
json.NewEncoder(w).Encode(rel)
|
||||
})
|
||||
mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
||||
body, ok := f.apkBytes[name]
|
||||
if !ok {
|
||||
http.Error(w, "not found", 404)
|
||||
return
|
||||
}
|
||||
rng := r.Header.Get("Range")
|
||||
f.mu.Lock()
|
||||
f.assetAuth = r.Header.Get("Authorization")
|
||||
if rng != "" {
|
||||
f.rangeHit[name]++
|
||||
} else {
|
||||
f.fullHit[name]++
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
if rng == "" {
|
||||
w.WriteHeader(200)
|
||||
w.Write(body)
|
||||
return
|
||||
}
|
||||
var end int
|
||||
fmt.Sscanf(rng, "bytes=0-%d", &end)
|
||||
if end >= len(body)-1 {
|
||||
end = len(body) - 1
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
|
||||
w.Header().Set("Content-Length", strconv.Itoa(end+1))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write(body[:end+1])
|
||||
})
|
||||
f.srv = httptest.NewServer(mux)
|
||||
t.Cleanup(f.srv.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *githubFixture) remote() models.Remote {
|
||||
return models.Remote{
|
||||
Name: "acme-apk",
|
||||
PackageType: models.PackageGitHubAlpine,
|
||||
BaseURL: f.srv.URL + "/repos/acme/tools",
|
||||
ReleasesRemote: "github",
|
||||
MutableTTL: 3600,
|
||||
}
|
||||
}
|
||||
|
||||
func newTestProvider() *GitHubProvider {
|
||||
p := newGitHubProvider()
|
||||
p.headerInitial = 32 // force the ranged-fetch retry loop against the tiny fixture
|
||||
p.headerMax = 1 << 20
|
||||
return p
|
||||
}
|
||||
|
||||
const demoPath = "acme/tools/releases/download/v1.2.3/demo-1.2.3-r0.apk"
|
||||
|
||||
func TestGitHubScanDerivesPkginfoFromPrefix(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
metas, _ := store.ListAlpineMetadataEntries(context.Background(), "acme-apk")
|
||||
if len(metas) != 1 {
|
||||
t.Fatalf("want 1 metadata row, got %d", len(metas))
|
||||
}
|
||||
m := metas[0]
|
||||
if m.Name != "demo" || m.Version != "1.2.3-r0" || m.Arch != "x86_64" {
|
||||
t.Fatalf("bad .PKGINFO fields: %+v", m)
|
||||
}
|
||||
if m.FilePath != demoPath {
|
||||
t.Fatalf("FilePath = %q, want %q", m.FilePath, demoPath)
|
||||
}
|
||||
if int(m.DownloadSize) != len(fx.apkBytes["demo-1.2.3-r0.apk"]) {
|
||||
t.Fatalf("DownloadSize = %d, want %d", m.DownloadSize, len(fx.apkBytes["demo-1.2.3-r0.apk"]))
|
||||
}
|
||||
if !strings.HasPrefix(m.Checksum, "Q1") {
|
||||
t.Fatalf("Checksum not a Q1 pull checksum: %q", m.Checksum)
|
||||
}
|
||||
// The C: checksum must equal Q1 over the raw control gzip stream, matching the
|
||||
// local-upload parser applied to the same bytes.
|
||||
want, err := parseApk(fx.apkBytes["demo-1.2.3-r0.apk"])
|
||||
if err != nil {
|
||||
t.Fatalf("reference parseApk: %v", err)
|
||||
}
|
||||
if m.Checksum != want.Checksum {
|
||||
t.Fatalf("Checksum = %q, want %q (Q1 of control stream)", m.Checksum, want.Checksum)
|
||||
}
|
||||
if fx.fullHit["demo-1.2.3-r0.apk"] != 0 {
|
||||
t.Fatalf("expected no full download, got %d", fx.fullHit["demo-1.2.3-r0.apk"])
|
||||
}
|
||||
if fx.rangeHit["demo-1.2.3-r0.apk"] == 0 {
|
||||
t.Fatalf("expected ranged .PKGINFO fetch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteIndexAndRedirect(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://artifactapi.example"
|
||||
|
||||
// The per-arch index is served and triggers the initial scan.
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/APKINDEX.tar.gz", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/APKINDEX.tar.gz", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle APKINDEX")
|
||||
}
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("APKINDEX bad: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
idx := readAPKIndex(t, rec.Body.Bytes())
|
||||
if !strings.Contains(idx, "P:demo") || !strings.Contains(idx, "A:x86_64") {
|
||||
t.Fatalf("APKINDEX missing package record: %s", idx)
|
||||
}
|
||||
if !strings.Contains(idx, "C:Q1") {
|
||||
t.Fatalf("APKINDEX missing pull checksum: %s", idx)
|
||||
}
|
||||
|
||||
// A different arch yields an empty (but valid) index.
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "aarch64/APKINDEX.tar.gz", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle aarch64 APKINDEX")
|
||||
}
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("empty-arch index bad: %d", rec.Code)
|
||||
}
|
||||
if got := readAPKIndex(t, rec.Body.Bytes()); strings.Contains(got, "P:demo") {
|
||||
t.Fatalf("aarch64 index should not carry the x86_64 package: %s", got)
|
||||
}
|
||||
|
||||
// An .apk request arrives in apk's reconstructed shape
|
||||
// "<arch>/<name>-<version>.apk" (APKINDEX carries no filename), NOT as the
|
||||
// github-relative FilePath. ServeRemote must resolve it back to the stored
|
||||
// FilePath before redirecting to the backend releases_remote.
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/demo-1.2.3-r0.apk", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/demo-1.2.3-r0.apk", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle .apk")
|
||||
}
|
||||
if rec.Code != http.StatusFound {
|
||||
t.Fatalf("want 302, got %d", rec.Code)
|
||||
}
|
||||
wantLoc := proxyBase + "/api/v1/remote/github/" + demoPath
|
||||
if got := rec.Header().Get("Location"); got != wantLoc {
|
||||
t.Fatalf("Location = %q, want %q (must be the stored FilePath, not the inbound path)", got, wantLoc)
|
||||
}
|
||||
}
|
||||
|
||||
// An apk download whose reconstructed "<arch>/<name>-<version>.apk" matches no
|
||||
// cached row must 404, never redirect to a bad path.
|
||||
func TestGitHubServeRemoteApkRedirectNotFound(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
// Warm the cache so the store is populated but lacks the requested package.
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/nope-9.9.9.apk", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/nope-9.9.9.apk", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle .apk")
|
||||
}
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("want 404 for unknown package, got %d (Location=%q)", rec.Code, rec.Header().Get("Location"))
|
||||
}
|
||||
}
|
||||
|
||||
// apk requests the index at "./<arch>/APKINDEX.tar.gz"; ServeRemote must collapse
|
||||
// the dot-segment and synthesize the same index as the un-prefixed request.
|
||||
func TestGitHubServeRemoteApkDotSegment(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://artifactapi.example"
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/"+path, nil)
|
||||
if !p.ServeRemote(rec, req, remote, path, proxyBase, store) {
|
||||
t.Fatalf("ServeRemote did not handle %q", path)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
plain, dotted := serve("x86_64/APKINDEX.tar.gz"), serve("./x86_64/APKINDEX.tar.gz")
|
||||
if plain.Code != 200 || dotted.Code != 200 {
|
||||
t.Fatalf("index: plain=%d dotted=%d, want 200/200", plain.Code, dotted.Code)
|
||||
}
|
||||
if !bytes.Equal(plain.Body.Bytes(), dotted.Body.Bytes()) {
|
||||
t.Error("./<arch>/APKINDEX.tar.gz body differs from the un-prefixed body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteRejectsNonPerArchIndex(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, fx.remote(), "APKINDEX.tar.gz", "https://x", store) {
|
||||
t.Fatal("expected handled")
|
||||
}
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("bare APKINDEX must 404 (per-arch required), got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A canceled inbound request must still serve the warm cache (detached context),
|
||||
// not turn the metadata read into a 500.
|
||||
func TestGitHubServeRemoteCanceledRequestServesCache(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.lastScan[remote.Name] = time.Now()
|
||||
p.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/APKINDEX.tar.gz", nil).WithContext(ctx)
|
||||
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/APKINDEX.tar.gz", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle APKINDEX")
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("canceled request must serve cache, not error; got code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := readAPKIndex(t, rec.Body.Bytes()); !strings.Contains(got, "P:demo") {
|
||||
t.Fatalf("expected index served from cache, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteRedirectRequiresReleasesRemote(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
remote.ReleasesRemote = ""
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, demoPath, "https://x", store) {
|
||||
t.Fatal("expected handled")
|
||||
}
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("want 500 when releases_remote unset, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubScanPrunesRemovedAssets(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if rows, _ := store.ListAlpineMetadataEntries(context.Background(), "acme-apk"); len(rows) != 1 {
|
||||
t.Fatalf("want 1 row after first scan, got %d", len(rows))
|
||||
}
|
||||
|
||||
delete(fx.apkBytes, "demo-1.2.3-r0.apk")
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("rescan: %v", err)
|
||||
}
|
||||
if rows, _ := store.ListAlpineMetadataEntries(context.Background(), "acme-apk"); len(rows) != 0 {
|
||||
t.Fatalf("want 0 rows after prune, got %d", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubAssetPatternFilter(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.apkBytes["other-9-r0.apk"] = testsupport.MinimalApk("other", "9-r0", "aarch64")
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
remote.Patterns = []string{`^demo-.*\.apk$`}
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
rows, _ := store.ListAlpineMetadataEntries(context.Background(), "acme-apk")
|
||||
if len(rows) != 1 || rows[0].Name != "demo" {
|
||||
t.Fatalf("pattern filter failed, rows=%+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-arch: each asset's index record lands under its own arch bucket.
|
||||
func TestGitHubServeRemotePerArchGrouping(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.apkBytes["demo-1.2.3-r0-aarch64.apk"] = testsupport.MinimalApk("demo", "1.2.3-r0", "aarch64")
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://x"
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
serve := func(arch string) string {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, arch+"/APKINDEX.tar.gz", proxyBase, store) {
|
||||
t.Fatalf("ServeRemote did not handle %s", arch)
|
||||
}
|
||||
return readAPKIndex(t, rec.Body.Bytes())
|
||||
}
|
||||
|
||||
x86 := serve("x86_64")
|
||||
if !strings.Contains(x86, "A:x86_64") || strings.Contains(x86, "A:aarch64") {
|
||||
t.Fatalf("x86_64 index leaked another arch: %s", x86)
|
||||
}
|
||||
arm := serve("aarch64")
|
||||
if !strings.Contains(arm, "A:aarch64") || strings.Contains(arm, "A:x86_64") {
|
||||
t.Fatalf("aarch64 index leaked another arch: %s", arm)
|
||||
}
|
||||
}
|
||||
|
||||
func readAPKIndex(t *testing.T, gzBytes []byte) string {
|
||||
t.Helper()
|
||||
gz, err := gzip.NewReader(bytes.NewReader(gzBytes))
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err != nil {
|
||||
t.Fatal("APKINDEX member missing from tar.gz")
|
||||
}
|
||||
if strings.TrimPrefix(hdr.Name, "./") == "APKINDEX" {
|
||||
body, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
t.Fatalf("read APKINDEX: %v", err)
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,238 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const (
|
||||
syncLeaseDuration = 15 * time.Minute
|
||||
defaultSyncFreshness = 5 * time.Minute
|
||||
jobQueueDepth = 256
|
||||
)
|
||||
|
||||
// SyncStore is the persistence surface the alpine syncer needs: the metadata
|
||||
// cache it primes plus the shared sync-state coordination (remote enumeration
|
||||
// and the per-remote lease). *database.DB satisfies it.
|
||||
type SyncStore interface {
|
||||
provider.RemoteMetadataStore
|
||||
ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
|
||||
ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
|
||||
}
|
||||
|
||||
// SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
|
||||
type SyncConfig struct {
|
||||
RatePerSec float64
|
||||
Burst int
|
||||
Workers int
|
||||
PollInterval time.Duration
|
||||
}
|
||||
|
||||
type syncJob struct {
|
||||
remote models.Remote
|
||||
prime bool
|
||||
}
|
||||
|
||||
// Syncer is the single per-process background worker that keeps every
|
||||
// github_alpine remote's derived metadata fresh. It owns a deduped work queue, a
|
||||
// pool of workers, and a global token-bucket rate limiter shared across all
|
||||
// remotes and bound onto the github_alpine provider. Periodic checks are gated by
|
||||
// a shared DB lease so, across replicas, only one performs each scan.
|
||||
type Syncer struct {
|
||||
store SyncStore
|
||||
prov *GitHubProvider
|
||||
limiter *rate.Limiter
|
||||
cfg SyncConfig
|
||||
owner string
|
||||
|
||||
jobs chan syncJob
|
||||
mu sync.Mutex
|
||||
active map[string]bool
|
||||
}
|
||||
|
||||
// NewSyncer builds the syncer bound to the process-wide github_alpine provider
|
||||
// singleton. Call Run to start it.
|
||||
func NewSyncer(store SyncStore, cfg SyncConfig) *Syncer {
|
||||
return newSyncer(store, gitHubProvider, cfg)
|
||||
}
|
||||
|
||||
func newSyncer(store SyncStore, prov *GitHubProvider, cfg SyncConfig) *Syncer {
|
||||
if cfg.RatePerSec <= 0 {
|
||||
cfg.RatePerSec = 1
|
||||
}
|
||||
if cfg.Burst <= 0 {
|
||||
cfg.Burst = 5
|
||||
}
|
||||
if cfg.Workers <= 0 {
|
||||
cfg.Workers = 3
|
||||
}
|
||||
if cfg.PollInterval <= 0 {
|
||||
cfg.PollInterval = 60 * time.Second
|
||||
}
|
||||
|
||||
lim := rate.NewLimiter(rate.Limit(cfg.RatePerSec), cfg.Burst)
|
||||
s := &Syncer{
|
||||
store: store,
|
||||
prov: prov,
|
||||
limiter: lim,
|
||||
cfg: cfg,
|
||||
owner: leaseOwner(),
|
||||
jobs: make(chan syncJob, jobQueueDepth),
|
||||
active: map[string]bool{},
|
||||
}
|
||||
prov.limiter = lim
|
||||
prov.syncer = s
|
||||
return s
|
||||
}
|
||||
|
||||
// Run starts the worker pool and the periodic scheduler and blocks until ctx is
|
||||
// canceled, at which point it drains in-flight scans and returns.
|
||||
func (s *Syncer) Run(ctx context.Context) {
|
||||
slog.Info("github_alpine syncer started",
|
||||
"rate_per_sec", s.cfg.RatePerSec, "burst", s.cfg.Burst,
|
||||
"workers", s.cfg.Workers, "poll_interval", s.cfg.PollInterval, "owner", s.owner)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < s.cfg.Workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
s.worker(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(s.cfg.PollInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
s.schedule(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
wg.Wait()
|
||||
slog.Info("github_alpine syncer stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.schedule(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// schedule enqueues a periodic check for every github_alpine remote. The DB lease
|
||||
// enforces the per-remote mutable_ttl cadence and cross-replica coordination.
|
||||
func (s *Syncer) schedule(ctx context.Context) {
|
||||
remotes, err := s.store.ListGitHubAlpineRemotes(ctx)
|
||||
if err != nil {
|
||||
slog.Error("github_alpine syncer: list remotes", "error", err)
|
||||
return
|
||||
}
|
||||
for _, r := range remotes {
|
||||
s.enqueue(r, false)
|
||||
}
|
||||
}
|
||||
|
||||
// EnqueuePrime queues an immediate background prime for a freshly created remote.
|
||||
func (s *Syncer) EnqueuePrime(remote models.Remote) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.enqueue(remote, true)
|
||||
}
|
||||
|
||||
// enqueue adds a job unless the remote is already queued or in-flight, coalescing
|
||||
// duplicate requests down to one scan. It never blocks.
|
||||
func (s *Syncer) enqueue(remote models.Remote, prime bool) {
|
||||
s.mu.Lock()
|
||||
if s.active[remote.Name] {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.active[remote.Name] = true
|
||||
s.mu.Unlock()
|
||||
|
||||
select {
|
||||
case s.jobs <- syncJob{remote: remote, prime: prime}:
|
||||
default:
|
||||
s.mu.Lock()
|
||||
delete(s.active, remote.Name)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Syncer) worker(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case job := <-s.jobs:
|
||||
s.process(ctx, job)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// process claims the shared lease and, if won, runs an incremental scan. Losing
|
||||
// the claim (another replica scanning, or not yet due) is a no-op.
|
||||
func (s *Syncer) process(ctx context.Context, job syncJob) {
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
delete(s.active, job.remote.Name)
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
freshness := time.Duration(job.remote.MutableTTL) * time.Second
|
||||
if freshness <= 0 {
|
||||
freshness = defaultSyncFreshness
|
||||
}
|
||||
if job.prime {
|
||||
freshness = 0
|
||||
}
|
||||
|
||||
claimed, etag, err := s.store.ClaimGitHubAlpineSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration)
|
||||
if err != nil {
|
||||
slog.Error("github_alpine syncer: claim lease", "remote", job.remote.Name, "error", err)
|
||||
return
|
||||
}
|
||||
if !claimed {
|
||||
return
|
||||
}
|
||||
|
||||
scanCtx, cancel := context.WithTimeout(ctx, s.prov.scanTimeout)
|
||||
defer cancel()
|
||||
|
||||
newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
|
||||
releaseEtag := etag
|
||||
if scanErr == nil {
|
||||
releaseEtag = newEtag
|
||||
} else {
|
||||
slog.Error("github_alpine syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
|
||||
}
|
||||
|
||||
relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer relCancel()
|
||||
if err := s.store.ReleaseGitHubAlpineSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
|
||||
slog.Warn("github_alpine syncer: release lease", "remote", job.remote.Name, "error", err)
|
||||
}
|
||||
|
||||
if scanErr == nil && changed {
|
||||
slog.Info("github_alpine syncer: refreshed", "remote", job.remote.Name, "prime", job.prime)
|
||||
}
|
||||
}
|
||||
|
||||
// leaseOwner is a per-replica identity for the lease: hostname plus a random
|
||||
// suffix so restarts and colocated replicas never collide.
|
||||
func leaseOwner() string {
|
||||
host, _ := os.Hostname()
|
||||
var b [6]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return host + "-" + hex.EncodeToString(b[:])
|
||||
}
|
||||
@@ -1,300 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeSyncStore is an in-memory SyncStore: the metadata cache (via the embedded
|
||||
// fakeStore) plus the shared sync-state lease, whose claim mirrors the atomic
|
||||
// semantics of the real SQL (recency gate AND no live lease).
|
||||
type fakeSyncStore struct {
|
||||
*fakeStore
|
||||
|
||||
mu sync.Mutex
|
||||
remotes []models.Remote
|
||||
leaseOwner map[string]string
|
||||
leaseExp map[string]time.Time
|
||||
lastSynced map[string]time.Time
|
||||
etags map[string]string
|
||||
}
|
||||
|
||||
func newFakeSyncStore() *fakeSyncStore {
|
||||
return &fakeSyncStore{
|
||||
fakeStore: newFakeStore(),
|
||||
leaseOwner: map[string]string{},
|
||||
leaseExp: map[string]time.Time{},
|
||||
lastSynced: map[string]time.Time{},
|
||||
etags: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ListGitHubAlpineRemotes(_ context.Context) ([]models.Remote, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]models.Remote(nil), f.remotes...), nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
now := time.Now()
|
||||
ls, hasLS := f.lastSynced[name]
|
||||
exp, hasExp := f.leaseExp[name]
|
||||
freshOK := !hasLS || now.Sub(ls) >= freshness
|
||||
leaseOK := !hasExp || exp.Before(now)
|
||||
if freshOK && leaseOK {
|
||||
f.leaseOwner[name] = owner
|
||||
f.leaseExp[name] = now.Add(lease)
|
||||
return true, f.etags[name], nil
|
||||
}
|
||||
return false, "", nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ReleaseGitHubAlpineSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.leaseOwner[name] != owner {
|
||||
return nil
|
||||
}
|
||||
f.lastSynced[name] = syncedAt
|
||||
f.etags[name] = etag
|
||||
delete(f.leaseOwner, name)
|
||||
delete(f.leaseExp, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func testSyncConfig() SyncConfig {
|
||||
return SyncConfig{RatePerSec: 1000, Burst: 100, Workers: 1, PollInterval: time.Hour}
|
||||
}
|
||||
|
||||
// (a) A 304 conditional response must derive nothing: no asset fetches and
|
||||
// changed=false, so an unchanged repo is nearly free.
|
||||
func TestSyncerConditionalNotModifiedSkipsDerive(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
etag1, changed, err := p.scanWithState(context.Background(), remote, store, "")
|
||||
if err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
if !changed || etag1 != `"v1"` {
|
||||
t.Fatalf("first scan changed=%v etag=%q, want true and \"v1\"", changed, etag1)
|
||||
}
|
||||
priorRange := fx.rangeHit["demo-1.2.3-r0.apk"]
|
||||
if priorRange == 0 {
|
||||
t.Fatal("first scan should have fetched the asset .PKGINFO")
|
||||
}
|
||||
|
||||
etag2, changed2, err := p.scanWithState(context.Background(), remote, store, etag1)
|
||||
if err != nil {
|
||||
t.Fatalf("second scan: %v", err)
|
||||
}
|
||||
if changed2 {
|
||||
t.Fatal("304 scan must report changed=false")
|
||||
}
|
||||
if etag2 != etag1 {
|
||||
t.Fatalf("etag changed across 304: %q -> %q", etag1, etag2)
|
||||
}
|
||||
if fx.notModHit != 1 {
|
||||
t.Fatalf("want exactly one 304 releases response, got %d", fx.notModHit)
|
||||
}
|
||||
if got := fx.rangeHit["demo-1.2.3-r0.apk"]; got != priorRange {
|
||||
t.Fatalf("304 scan re-fetched asset .PKGINFO: %d -> %d", priorRange, got)
|
||||
}
|
||||
}
|
||||
|
||||
// (b) On a real change, only the newly added asset is derived.
|
||||
func TestSyncerIncrementalDerivesOnlyNewAsset(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
if _, _, err := p.scanWithState(context.Background(), remote, store, ""); err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
demoRange := fx.rangeHit["demo-1.2.3-r0.apk"]
|
||||
|
||||
fx.apkBytes["other-9-r0.apk"] = testsupport.MinimalApk("other", "9-r0", "aarch64")
|
||||
fx.etag = `"v2"`
|
||||
|
||||
if _, changed, err := p.scanWithState(context.Background(), remote, store, `"v1"`); err != nil || !changed {
|
||||
t.Fatalf("second scan changed=%v err=%v", changed, err)
|
||||
}
|
||||
|
||||
rows, _ := store.ListAlpineMetadataEntries(context.Background(), remote.Name)
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("want 2 cached rows after incremental derive, got %d", len(rows))
|
||||
}
|
||||
if got := fx.rangeHit["demo-1.2.3-r0.apk"]; got != demoRange {
|
||||
t.Fatalf("already-cached asset was re-fetched: %d -> %d", demoRange, got)
|
||||
}
|
||||
if fx.rangeHit["other-9-r0.apk"] == 0 {
|
||||
t.Fatal("newly added asset was not derived")
|
||||
}
|
||||
}
|
||||
|
||||
// (c) The shared limiter caps the request rate.
|
||||
func TestRateLimiterCapsRequestRate(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
p.limiter = rate.NewLimiter(rate.Every(120*time.Millisecond), 1)
|
||||
remote := fx.remote()
|
||||
|
||||
start := time.Now()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, _, _, err := p.fetchReleases(context.Background(), remote, ""); err != nil {
|
||||
t.Fatalf("fetchReleases %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
|
||||
t.Fatalf("rate limiter did not throttle: 3 calls took %v, want >= 200ms", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// (d) Concurrent enqueues for the same remote coalesce to a single queued job.
|
||||
func TestSyncerEnqueueDedup(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := models.Remote{Name: "acme-apk", PackageType: models.PackageGitHubAlpine, MutableTTL: 3600}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); s.enqueue(remote, false) }()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if got := len(s.jobs); got != 1 {
|
||||
t.Fatalf("want exactly 1 coalesced job, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// (e) Prime-on-create enqueues a prime job.
|
||||
func TestSyncerEnqueuePrime(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := models.Remote{Name: "acme-apk", PackageType: models.PackageGitHubAlpine, MutableTTL: 3600}
|
||||
|
||||
s.EnqueuePrime(remote)
|
||||
select {
|
||||
case job := <-s.jobs:
|
||||
if !job.prime || job.remote.Name != "acme-apk" {
|
||||
t.Fatalf("bad prime job: %+v", job)
|
||||
}
|
||||
default:
|
||||
t.Fatal("EnqueuePrime did not enqueue a job")
|
||||
}
|
||||
}
|
||||
|
||||
// (f) A held lease prevents a second replica from scanning.
|
||||
func TestSyncerLeasePreventsSecondReplica(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
claimed, _, err := store.ClaimGitHubAlpineSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote})
|
||||
|
||||
if fx.releasesHit != 0 {
|
||||
t.Fatalf("second replica scanned while lease held: %d releases calls", fx.releasesHit)
|
||||
}
|
||||
if rows, _ := store.ListAlpineMetadataEntries(context.Background(), remote.Name); len(rows) != 0 {
|
||||
t.Fatalf("second replica derived metadata while lease held: %d rows", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
// With the syncer wired and the cache empty, an index request enqueues a prime
|
||||
// and returns a retryable 503 when it has not landed within the cold wait.
|
||||
func TestServeRemoteColdStartReturns503(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
p.coldWait = 300 * time.Millisecond
|
||||
_ = newSyncer(store, p, testSyncConfig()) // binds p.syncer, but no workers running
|
||||
remote := fx.remote()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/APKINDEX.tar.gz", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/APKINDEX.tar.gz", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle APKINDEX")
|
||||
}
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("cold empty cache must return 503, got %d", rec.Code)
|
||||
}
|
||||
if rec.Header().Get("Retry-After") == "" {
|
||||
t.Fatal("503 should carry Retry-After")
|
||||
}
|
||||
if got := len(p.syncer.jobs); got != 1 {
|
||||
t.Fatalf("cold start did not enqueue a prime, jobs=%d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With the cache warm, the same request serves the index immediately (no 503).
|
||||
func TestServeRemoteWarmCacheServesImmediately(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
_ = newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/APKINDEX.tar.gz", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/APKINDEX.tar.gz", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle APKINDEX")
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("warm cache must serve 200, got %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A prime job (freshness 0) runs even right after a sync; a periodic job at the
|
||||
// same moment is gated by the recency window.
|
||||
func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
var _ provider.RemoteMetadataStore = store
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: true})
|
||||
if rows, _ := store.ListAlpineMetadataEntries(context.Background(), remote.Name); len(rows) != 1 {
|
||||
t.Fatalf("prime did not derive: %d rows", len(rows))
|
||||
}
|
||||
releasesAfterPrime := fx.releasesHit
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: false})
|
||||
if fx.releasesHit != releasesAfterPrime {
|
||||
t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
|
||||
}
|
||||
}
|
||||
@@ -1,462 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
// mutableRe marks the apt index surface (both the flat local repo and a proxied
|
||||
// Debian/Ubuntu mirror's dists/ tree) so the caching engine revalidates it
|
||||
// instead of freezing it like an immutable .deb.
|
||||
var mutableRe = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(^|/)Packages(\.gz|\.xz|\.bz2)?$`),
|
||||
regexp.MustCompile(`(^|/)Sources(\.gz|\.xz|\.bz2)?$`),
|
||||
regexp.MustCompile(`(^|/)Release$`),
|
||||
regexp.MustCompile(`(^|/)InRelease$`),
|
||||
regexp.MustCompile(`(^|/)Release\.gpg$`),
|
||||
regexp.MustCompile(`(^|/)Contents-`),
|
||||
regexp.MustCompile(`^dists/`),
|
||||
regexp.MustCompile(`/by-hash/`),
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageDeb }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
for _, re := range mutableRe {
|
||||
if re.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".deb"):
|
||||
return "application/vnd.debian.binary-package"
|
||||
case strings.HasSuffix(path, ".gz"):
|
||||
return "application/gzip"
|
||||
case strings.HasSuffix(path, ".xz"):
|
||||
return "application/x-xz"
|
||||
case strings.HasSuffix(path, "Packages"), strings.HasSuffix(path, "Release"),
|
||||
strings.HasSuffix(path, "InRelease"), strings.HasSuffix(path, "Sources"):
|
||||
return "text/plain"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(strings.ToLower(filename), ".deb") {
|
||||
return "", "", fmt.Errorf("file must be a .deb package")
|
||||
}
|
||||
|
||||
return "pool/" + filename, "application/vnd.debian.binary-package", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
filename := strings.TrimPrefix(storagePath, "pool/")
|
||||
return map[string]any{
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
|
||||
|
||||
reader, blobSize, err := blobs.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
slog.Error("deb metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
raw, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
slog.Error("deb metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
control, err := extractControl(raw)
|
||||
if err != nil {
|
||||
slog.Error("deb metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
fields := parseControlFields(control)
|
||||
|
||||
sum := md5.Sum(raw)
|
||||
meta := &provider.DebMetadata{
|
||||
RepoName: repoName,
|
||||
FilePath: storagePath,
|
||||
ContentHash: contentHash,
|
||||
Name: fields["Package"],
|
||||
Version: fields["Version"],
|
||||
Architecture: fields["Architecture"],
|
||||
Control: strings.TrimRight(control, "\n"),
|
||||
Size: blobSize,
|
||||
MD5: hex.EncodeToString(sum[:]),
|
||||
SHA256: strings.TrimPrefix(contentHash, "sha256:"),
|
||||
}
|
||||
|
||||
if meta.Name == "" {
|
||||
slog.Error("deb metadata: control missing Package field", "repo", repoName, "path", storagePath)
|
||||
return
|
||||
}
|
||||
|
||||
if err := db.InsertDebMetadata(ctx, meta); err != nil {
|
||||
slog.Error("deb metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("deb metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
|
||||
}
|
||||
|
||||
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
|
||||
if err := db.DeleteDebMetadata(ctx, repoName, storagePath); err != nil {
|
||||
slog.Error("deb metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return err
|
||||
}
|
||||
slog.Info("deb metadata: deleted", "repo", repoName, "path", storagePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractControl reads a .deb (an ar archive), locates the control.tar.* member,
|
||||
// decompresses it, and returns the raw ./control paragraph. Pure Go: no dpkg.
|
||||
func extractControl(deb []byte) (string, error) {
|
||||
members, err := readAr(deb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var name string
|
||||
var data []byte
|
||||
for _, m := range members {
|
||||
if strings.HasPrefix(m.name, "control.tar") {
|
||||
name = m.name
|
||||
data = m.data
|
||||
break
|
||||
}
|
||||
}
|
||||
if data == nil {
|
||||
return "", errors.New("no control.tar member in .deb")
|
||||
}
|
||||
|
||||
tarBytes, err := decompress(name, data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return readControlParagraph(tarBytes)
|
||||
}
|
||||
|
||||
// readControlParagraph scans a decompressed control.tar and returns the raw
|
||||
// ./control paragraph. Shared by the local upload path (extractControl) and the
|
||||
// github_deb ranged-prefix parser.
|
||||
func readControlParagraph(controlTar []byte) (string, error) {
|
||||
tr := tar.NewReader(bytes.NewReader(controlTar))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read control.tar: %w", err)
|
||||
}
|
||||
clean := strings.TrimPrefix(hdr.Name, "./")
|
||||
if clean == "control" {
|
||||
b, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read control file: %w", err)
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("no ./control in control.tar")
|
||||
}
|
||||
|
||||
func decompress(name string, data []byte) ([]byte, error) {
|
||||
switch {
|
||||
case strings.HasSuffix(name, ".gz"):
|
||||
zr, err := gzip.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer zr.Close()
|
||||
return io.ReadAll(zr)
|
||||
case strings.HasSuffix(name, ".xz"):
|
||||
xr, err := xz.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(xr)
|
||||
case strings.HasSuffix(name, ".zst"):
|
||||
zr, err := zstd.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer zr.Close()
|
||||
return io.ReadAll(zr)
|
||||
case strings.HasSuffix(name, ".tar"):
|
||||
return data, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported control.tar compression: %s", name)
|
||||
}
|
||||
|
||||
type arMember struct {
|
||||
name string
|
||||
data []byte
|
||||
}
|
||||
|
||||
// readAr parses the (trivial) Unix ar archive that wraps a .deb. Each member has
|
||||
// a 60-byte header; the size field is decimal ASCII and data is padded to an
|
||||
// even offset.
|
||||
func readAr(data []byte) ([]arMember, error) {
|
||||
const magic = "!<arch>\n"
|
||||
if len(data) < len(magic) || string(data[:len(magic)]) != magic {
|
||||
return nil, errors.New("not an ar archive")
|
||||
}
|
||||
off := len(magic)
|
||||
|
||||
var members []arMember
|
||||
for off+60 <= len(data) {
|
||||
hdr := data[off : off+60]
|
||||
off += 60
|
||||
|
||||
name := strings.TrimRight(string(hdr[0:16]), " ")
|
||||
name = strings.TrimSuffix(name, "/")
|
||||
size, err := strconv.ParseInt(strings.TrimSpace(string(hdr[48:58])), 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad ar size for %q: %w", name, err)
|
||||
}
|
||||
if off+int(size) > len(data) {
|
||||
return nil, fmt.Errorf("truncated ar member %q", name)
|
||||
}
|
||||
members = append(members, arMember{name: name, data: data[off : off+int(size)]})
|
||||
off += int(size)
|
||||
if size%2 == 1 {
|
||||
off++
|
||||
}
|
||||
}
|
||||
return members, nil
|
||||
}
|
||||
|
||||
// parseControlFields reads the single-line fields of an RFC822-style control
|
||||
// paragraph. Continuation lines (leading whitespace) belong to the previous
|
||||
// field and are ignored here since only Package/Version/Architecture are read.
|
||||
func parseControlFields(control string) map[string]string {
|
||||
fields := map[string]string{}
|
||||
sc := bufio.NewScanner(strings.NewReader(control))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
if line == "" || line[0] == ' ' || line[0] == '\t' {
|
||||
continue
|
||||
}
|
||||
idx := strings.IndexByte(line, ':')
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(line[:idx])
|
||||
if _, seen := fields[key]; seen {
|
||||
continue
|
||||
}
|
||||
fields[key] = strings.TrimSpace(line[idx+1:])
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
// normalizeIndexPath collapses apt's verbatim dist prefix from a flat-repo
|
||||
// request. For `deb ... <repo>/ ./`, apt appends the "./" dist literally and asks
|
||||
// for "./Packages" (and "./Release", "./InRelease"); dot-segments must be
|
||||
// collapsed so the index matcher sees "Packages". A no-op for pool/*.deb paths.
|
||||
func normalizeIndexPath(p string) string {
|
||||
return strings.TrimPrefix(path.Clean("/"+p), "/")
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
|
||||
path := normalizeIndexPath(reqPath)
|
||||
switch path {
|
||||
case "Packages", "Packages.gz", "Release":
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
reader, ok := files.(provider.DebMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "deb metadata not available", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
metas, err := reader.ListDebMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
slog.Warn("deb: metadata read canceled", "repo", repoName, "error", err)
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return true
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
switch path {
|
||||
case "Packages":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generatePackages(metas))
|
||||
case "Packages.gz":
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(gzipBytes(generatePackages(metas)))
|
||||
case "Release":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateRelease(metas))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("deb local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
// generatePackages emits the flat-repo Packages file: each package's raw control
|
||||
// stanza followed by the apt-required Filename/Size/MD5sum/SHA256 fields, blank
|
||||
// line separated.
|
||||
func generatePackages(metas []provider.DebMetadata) []byte {
|
||||
var b bytes.Buffer
|
||||
for _, m := range metas {
|
||||
b.WriteString(strings.TrimRight(m.Control, "\n"))
|
||||
b.WriteString("\n")
|
||||
fmt.Fprintf(&b, "Filename: %s\n", m.FilePath)
|
||||
fmt.Fprintf(&b, "Size: %d\n", m.Size)
|
||||
if m.MD5 != "" {
|
||||
fmt.Fprintf(&b, "MD5sum: %s\n", m.MD5)
|
||||
}
|
||||
if m.SHA256 != "" {
|
||||
fmt.Fprintf(&b, "SHA256: %s\n", m.SHA256)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func generateRelease(metas []provider.DebMetadata) []byte {
|
||||
packages := generatePackages(metas)
|
||||
packagesGz := gzipBytes(packages)
|
||||
|
||||
arches := uniqueArches(metas)
|
||||
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "Date: %s\n", releaseDate(metas).Format(time.RFC1123Z))
|
||||
fmt.Fprintf(&b, "Architectures: %s\n", strings.Join(arches, " "))
|
||||
b.WriteString("Acquire-By-Hash: no\n")
|
||||
|
||||
b.WriteString("MD5Sum:\n")
|
||||
writeReleaseEntry(&b, md5Hex(packages), len(packages), "Packages")
|
||||
writeReleaseEntry(&b, md5Hex(packagesGz), len(packagesGz), "Packages.gz")
|
||||
|
||||
b.WriteString("SHA256:\n")
|
||||
writeReleaseEntry(&b, sha256Hex(packages), len(packages), "Packages")
|
||||
writeReleaseEntry(&b, sha256Hex(packagesGz), len(packagesGz), "Packages.gz")
|
||||
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
// releaseDate derives the Release Date: from the newest package's persisted
|
||||
// created_at (in UTC) so the file is byte-identical across the no-affinity
|
||||
// replicas and across regenerations (issue #117); an empty repo falls back to
|
||||
// the Unix epoch. This never uses wall clock, which also keeps Date: from
|
||||
// running ahead of any Valid-Until logic.
|
||||
func releaseDate(metas []provider.DebMetadata) time.Time {
|
||||
newest := time.Unix(0, 0)
|
||||
for _, m := range metas {
|
||||
if m.CreatedAt.After(newest) {
|
||||
newest = m.CreatedAt
|
||||
}
|
||||
}
|
||||
return newest.UTC()
|
||||
}
|
||||
|
||||
func writeReleaseEntry(b *bytes.Buffer, hash string, size int, name string) {
|
||||
fmt.Fprintf(b, " %s %d %s\n", hash, size, name)
|
||||
}
|
||||
|
||||
func uniqueArches(metas []provider.DebMetadata) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, m := range metas {
|
||||
a := m.Architecture
|
||||
if a == "" || seen[a] {
|
||||
continue
|
||||
}
|
||||
seen[a] = true
|
||||
out = append(out, a)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func gzipBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
gz.Write(data)
|
||||
gz.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func md5Hex(data []byte) string {
|
||||
h := md5.Sum(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
func sha256Hex(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
// debFixture returns a fixed set of rows with persisted created_at values, in
|
||||
// the total order ListDebMetadataEntries produces (name, version, arch,
|
||||
// file_path), so the generators are exercised on a stable input.
|
||||
func debFixture() []provider.DebMetadata {
|
||||
t1 := time.Date(2026, 3, 1, 8, 30, 0, 0, time.UTC)
|
||||
t2 := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC) // newest
|
||||
return []provider.DebMetadata{
|
||||
{
|
||||
RepoName: "r", FilePath: "pool/aaa_1.0_amd64.deb", ContentHash: "sha256:aa",
|
||||
Name: "aaa", Version: "1.0", Architecture: "amd64",
|
||||
Control: "Package: aaa\nVersion: 1.0\nArchitecture: amd64",
|
||||
Size: 100, MD5: "d41d8cd98f00b204e9800998ecf8427e", SHA256: "aa", CreatedAt: t1,
|
||||
},
|
||||
{
|
||||
RepoName: "r", FilePath: "pool/bbb_2.0_arm64.deb", ContentHash: "sha256:bb",
|
||||
Name: "bbb", Version: "2.0", Architecture: "arm64",
|
||||
Control: "Package: bbb\nVersion: 2.0\nArchitecture: arm64",
|
||||
Size: 200, MD5: "0cc175b9c0f1b6a831c399e269772661", SHA256: "bb", CreatedAt: t2,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestDebGeneratorsDeterministic asserts the served bytes are a pure function of
|
||||
// DB state: Packages, Packages.gz and Release are byte-identical across two
|
||||
// generations separated by wall-clock time. Fails against the old
|
||||
// time.Now()-stamped Release Date:.
|
||||
func TestDebGeneratorsDeterministic(t *testing.T) {
|
||||
metas := debFixture()
|
||||
|
||||
pkgs1 := generatePackages(metas)
|
||||
rel1 := generateRelease(metas)
|
||||
gz1 := gzipBytes(pkgs1)
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
pkgs2 := generatePackages(metas)
|
||||
rel2 := generateRelease(metas)
|
||||
gz2 := gzipBytes(pkgs2)
|
||||
|
||||
if !bytes.Equal(pkgs1, pkgs2) {
|
||||
t.Error("Packages differs across generations")
|
||||
}
|
||||
if !bytes.Equal(gz1, gz2) {
|
||||
t.Error("Packages.gz differs across generations")
|
||||
}
|
||||
if !bytes.Equal(rel1, rel2) {
|
||||
t.Errorf("Release differs across generations:\n--- first ---\n%s\n--- second ---\n%s", rel1, rel2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDebReleaseDateUsesPersistedCreatedAt pins the Release Date: to the newest
|
||||
// persisted created_at (RFC1123Z, UTC), not wall clock. Fails against the old
|
||||
// time.Now() code.
|
||||
func TestDebReleaseDateUsesPersistedCreatedAt(t *testing.T) {
|
||||
metas := debFixture()
|
||||
want := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC).Format(time.RFC1123Z)
|
||||
|
||||
rel := string(generateRelease(metas))
|
||||
var got string
|
||||
for _, line := range strings.Split(rel, "\n") {
|
||||
if strings.HasPrefix(line, "Date:") {
|
||||
got = strings.TrimSpace(strings.TrimPrefix(line, "Date:"))
|
||||
break
|
||||
}
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("Release Date: = %q, want %q (newest created_at)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDebReleaseDateEmptyRepoIsEpoch guards the fallback: an empty repo yields a
|
||||
// deterministic epoch Date: rather than wall clock.
|
||||
func TestDebReleaseDateEmptyRepoIsEpoch(t *testing.T) {
|
||||
want := time.Unix(0, 0).UTC().Format(time.RFC1123Z)
|
||||
rel := string(generateRelease(nil))
|
||||
if !strings.Contains(rel, "Date: "+want+"\n") {
|
||||
t.Errorf("empty-repo Release missing epoch Date: %q\n%s", want, rel)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDebReleaseChecksumsMatchServedBytes is the exact apt invariant: the
|
||||
// sha256/size (and md5/size) advertised for Packages and Packages.gz in Release
|
||||
// equal the sha256/size of the actual bytes ServeLocalIndex serves. apt rejects
|
||||
// any mismatch.
|
||||
func TestDebReleaseChecksumsMatchServedBytes(t *testing.T) {
|
||||
metas := debFixture()
|
||||
|
||||
packages := generatePackages(metas)
|
||||
packagesGz := gzipBytes(packages)
|
||||
rel := string(generateRelease(metas))
|
||||
|
||||
wantSHA := map[string]struct {
|
||||
hash string
|
||||
size int
|
||||
}{
|
||||
"Packages": {sha256Hex(packages), len(packages)},
|
||||
"Packages.gz": {sha256Hex(packagesGz), len(packagesGz)},
|
||||
}
|
||||
wantMD5 := map[string]struct {
|
||||
hash string
|
||||
size int
|
||||
}{
|
||||
"Packages": {md5Hex(packages), len(packages)},
|
||||
"Packages.gz": {md5Hex(packagesGz), len(packagesGz)},
|
||||
}
|
||||
|
||||
sha := parseReleaseSection(rel, "SHA256:")
|
||||
md5s := parseReleaseSection(rel, "MD5Sum:")
|
||||
|
||||
for name, w := range wantSHA {
|
||||
got, ok := sha[name]
|
||||
if !ok {
|
||||
t.Fatalf("Release SHA256 section missing %q", name)
|
||||
}
|
||||
if got.hash != w.hash || got.size != w.size {
|
||||
t.Errorf("Release SHA256 %s = (%s, %d), served bytes are (%s, %d)", name, got.hash, got.size, w.hash, w.size)
|
||||
}
|
||||
}
|
||||
for name, w := range wantMD5 {
|
||||
got, ok := md5s[name]
|
||||
if !ok {
|
||||
t.Fatalf("Release MD5Sum section missing %q", name)
|
||||
}
|
||||
if got.hash != w.hash || got.size != w.size {
|
||||
t.Errorf("Release MD5Sum %s = (%s, %d), served bytes are (%s, %d)", name, got.hash, got.size, w.hash, w.size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type releaseEntry struct {
|
||||
hash string
|
||||
size int
|
||||
}
|
||||
|
||||
// parseReleaseSection reads the indented " <hash> <size> <name>" lines that
|
||||
// follow a "SHA256:" / "MD5Sum:" header until the next non-indented line.
|
||||
func parseReleaseSection(release, header string) map[string]releaseEntry {
|
||||
out := map[string]releaseEntry{}
|
||||
lines := strings.Split(release, "\n")
|
||||
in := false
|
||||
for _, line := range lines {
|
||||
if line == header {
|
||||
in = true
|
||||
continue
|
||||
}
|
||||
if !in {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(line, " ") {
|
||||
break
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 3 {
|
||||
continue
|
||||
}
|
||||
size, _ := strconv.Atoi(fields[1])
|
||||
out[fields[2]] = releaseEntry{hash: fields[0], size: size}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,408 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type fakeBlobReader struct{ data []byte }
|
||||
|
||||
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
|
||||
}
|
||||
|
||||
type errBlobReader struct{}
|
||||
|
||||
func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return nil, 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
// fakeDebStore satisfies provider.MetadataStore (both insert methods) and
|
||||
// records the deb row that AfterUpload writes.
|
||||
type fakeDebStore struct{ inserted *provider.DebMetadata }
|
||||
|
||||
func (f *fakeDebStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeDebStore) InsertDebMetadata(_ context.Context, m *provider.DebMetadata) error {
|
||||
f.inserted = m
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeDebReader struct{ metas []provider.DebMetadata }
|
||||
|
||||
func (f fakeDebReader) ListDebMetadataEntries(context.Context, string) ([]provider.DebMetadata, error) {
|
||||
return f.metas, nil
|
||||
}
|
||||
func (f fakeDebReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f fakeDebReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
type errDebReader struct{}
|
||||
|
||||
func (errDebReader) ListDebMetadataEntries(context.Context, string) ([]provider.DebMetadata, error) {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
func (errDebReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (errDebReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestDebPureFuncs(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Type() != models.PackageDeb {
|
||||
t.Errorf("type = %q", p.Type())
|
||||
}
|
||||
if out, _ := p.RewriteResponse(nil, models.Remote{}, "http://p"); out != nil {
|
||||
t.Error("deb never rewrites")
|
||||
}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://mirror/"}, "/dists/bookworm/Release"); got != "https://mirror/dists/bookworm/Release" {
|
||||
t.Errorf("upstream url %q", got)
|
||||
}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("auth header")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"pool/foo_1.0_amd64.deb", provider.Immutable},
|
||||
{"Packages", provider.Mutable},
|
||||
{"Packages.gz", provider.Mutable},
|
||||
{"Release", provider.Mutable},
|
||||
{"InRelease", provider.Mutable},
|
||||
{"Release.gpg", provider.Mutable},
|
||||
{"dists/bookworm/main/binary-amd64/Packages", provider.Mutable},
|
||||
{"dists/bookworm/Release", provider.Mutable},
|
||||
{"dists/bookworm/main/by-hash/SHA256/abc", provider.Mutable},
|
||||
{"dists/bookworm/main/Contents-amd64.gz", provider.Mutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for path, want := range map[string]string{
|
||||
"pool/foo_1.0_amd64.deb": "application/vnd.debian.binary-package",
|
||||
"dists/bookworm/main/bin/Packages.gz": "application/gzip",
|
||||
"dists/bookworm/main/bin/Packages.xz": "application/x-xz",
|
||||
"Packages": "text/plain",
|
||||
"Release": "text/plain",
|
||||
"InRelease": "text/plain",
|
||||
"pool/other": "application/octet-stream",
|
||||
} {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("dir/foo_1.0_amd64.deb")
|
||||
if err != nil || sp != "pool/foo_1.0_amd64.deb" || ct != "application/vnd.debian.binary-package" {
|
||||
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("foo.rpm"); err == nil {
|
||||
t.Error("expected error for non-deb")
|
||||
}
|
||||
resp := p.UploadResponse("pool/foo_1.0_amd64.deb", "sha256:abc", 42)
|
||||
if resp["filename"] != "foo_1.0_amd64.deb" || resp["content_hash"] != "sha256:abc" || resp["size_bytes"] != int64(42) {
|
||||
t.Errorf("upload response %v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebAfterUpload(t *testing.T) {
|
||||
data := testsupport.MinimalDeb("e2e-testpkg", "1.2.3", "amd64")
|
||||
store := &fakeDebStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "myrepo", "pool/e2e-testpkg_1.2.3_amd64.deb",
|
||||
"sha256:deadbeef", fakeBlobReader{data: data}, store)
|
||||
|
||||
m := store.inserted
|
||||
if m == nil {
|
||||
t.Fatal("no metadata inserted")
|
||||
}
|
||||
if m.Name != "e2e-testpkg" || m.Version != "1.2.3" || m.Architecture != "amd64" {
|
||||
t.Errorf("unexpected metadata: %+v", m)
|
||||
}
|
||||
if m.Size != int64(len(data)) {
|
||||
t.Errorf("Size = %d, want %d", m.Size, len(data))
|
||||
}
|
||||
if m.SHA256 != "deadbeef" {
|
||||
t.Errorf("SHA256 = %q, want deadbeef", m.SHA256)
|
||||
}
|
||||
if m.MD5 == "" {
|
||||
t.Error("MD5 not computed")
|
||||
}
|
||||
if !strings.Contains(m.Control, "Package: e2e-testpkg") {
|
||||
t.Errorf("raw control not stored: %q", m.Control)
|
||||
}
|
||||
// The raw stanza is stored verbatim (no trailing newline) so Packages can
|
||||
// reproduce it faithfully.
|
||||
if strings.HasSuffix(m.Control, "\n") {
|
||||
t.Error("control should be trimmed of trailing newline")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebAfterUploadErrors(t *testing.T) {
|
||||
// Download failure: no insert, no panic.
|
||||
store := &fakeDebStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", errBlobReader{}, store)
|
||||
if store.inserted != nil {
|
||||
t.Error("no metadata should be inserted on download error")
|
||||
}
|
||||
// Not a .deb (ar) archive.
|
||||
store2 := &fakeDebStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", fakeBlobReader{data: []byte("not a deb")}, store2)
|
||||
if store2.inserted != nil {
|
||||
t.Error("no metadata should be inserted on parse error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebControlDecompression(t *testing.T) {
|
||||
// The control tarball may be gzip, xz, or zstd (goreleaser/nfpm emit gzip or
|
||||
// xz); each must round-trip to the same control stanza.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
member string
|
||||
comp func([]byte) []byte
|
||||
}{
|
||||
{"gzip", "control.tar.gz", gzipBytes},
|
||||
{"xz", "control.tar.xz", xzBytes},
|
||||
{"zstd", "control.tar.zst", zstdBytes},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
deb := buildDeb("pkg", "9.9", "arm64", tc.member, tc.comp)
|
||||
control, err := extractControl(deb)
|
||||
if err != nil {
|
||||
t.Fatalf("extractControl: %v", err)
|
||||
}
|
||||
fields := parseControlFields(control)
|
||||
if fields["Package"] != "pkg" || fields["Version"] != "9.9" || fields["Architecture"] != "arm64" {
|
||||
t.Errorf("fields = %v", fields)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebParseControlContinuationLines(t *testing.T) {
|
||||
control := "Package: p\nVersion: 1\n" +
|
||||
"Description: short\n very long\n .\n more\n" +
|
||||
"Architecture: all\n"
|
||||
f := parseControlFields(control)
|
||||
if f["Package"] != "p" || f["Version"] != "1" || f["Architecture"] != "all" {
|
||||
t.Errorf("continuation lines corrupted parse: %v", f)
|
||||
}
|
||||
if f["Description"] != "short" {
|
||||
t.Errorf("Description folded continuation into value: %q", f["Description"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebServeLocalIndex(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeDebReader{metas: []provider.DebMetadata{
|
||||
{Name: "aaa", Version: "1.0", Architecture: "amd64", FilePath: "pool/aaa_1.0_amd64.deb",
|
||||
Control: "Package: aaa\nVersion: 1.0\nArchitecture: amd64", Size: 100, MD5: "md5aaa", SHA256: "sha256aaa"},
|
||||
{Name: "bbb", Version: "2.0", Architecture: "arm64", FilePath: "pool/bbb_2.0_arm64.deb",
|
||||
Control: "Package: bbb\nVersion: 2.0\nArchitecture: arm64", Size: 200, MD5: "md5bbb", SHA256: "sha256bbb"},
|
||||
}}
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if !p.ServeLocalIndex(w, r, reader, "myrepo", path) {
|
||||
t.Fatalf("ServeLocalIndex returned false for %q", path)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// Packages lists both packages with their apt fields.
|
||||
w := serve("Packages")
|
||||
body := w.Body.String()
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("Packages code %d", w.Code)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"Package: aaa", "Package: bbb",
|
||||
"Filename: pool/aaa_1.0_amd64.deb", "Size: 100", "MD5sum: md5aaa", "SHA256: sha256aaa",
|
||||
"Filename: pool/bbb_2.0_arm64.deb", "Size: 200",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("Packages missing %q:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
// Stanzas are blank-line separated.
|
||||
if !strings.Contains(body, "SHA256: sha256aaa\n\n") {
|
||||
t.Errorf("stanzas not blank-line separated:\n%s", body)
|
||||
}
|
||||
|
||||
// Packages.gz decompresses to exactly the plain Packages bytes.
|
||||
w = serve("Packages.gz")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("Packages.gz code %d", w.Code)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(w.Body.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("Packages.gz not gzip: %v", err)
|
||||
}
|
||||
plain, _ := io.ReadAll(zr)
|
||||
if !bytes.Equal(plain, []byte(body)) {
|
||||
t.Error("Packages.gz does not decompress to Packages")
|
||||
}
|
||||
|
||||
// Release lists arches and both index files under MD5Sum/SHA256.
|
||||
w = serve("Release")
|
||||
rel := w.Body.String()
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("Release code %d", w.Code)
|
||||
}
|
||||
for _, want := range []string{"Date:", "Architectures: amd64 arm64", "Acquire-By-Hash: no", "MD5Sum:", "SHA256:", " Packages\n", " Packages.gz\n"} {
|
||||
if !strings.Contains(rel, want) {
|
||||
t.Errorf("Release missing %q:\n%s", want, rel)
|
||||
}
|
||||
}
|
||||
|
||||
// Unsigned trust model: no InRelease / Release.gpg served here.
|
||||
for _, path := range []string{"InRelease", "Release.gpg", "pool/aaa_1.0_amd64.deb"} {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if p.ServeLocalIndex(w, r, reader, "myrepo", path) {
|
||||
t.Errorf("ServeLocalIndex should return false for %q", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Real apt appends the flat-repo dist "./" verbatim, so it requests "./Packages"
|
||||
// / "./Release" (curl pre-normalizes /./ which masks this). The handler must
|
||||
// collapse the dot-segment and return the same bytes as the un-prefixed request.
|
||||
func TestDebServeLocalIndexAptDotSegment(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeDebReader{metas: []provider.DebMetadata{
|
||||
{Name: "aaa", Version: "1.0", Architecture: "amd64", FilePath: "pool/aaa_1.0_amd64.deb",
|
||||
Control: "Package: aaa\nVersion: 1.0\nArchitecture: amd64", Size: 100, MD5: "md5aaa", SHA256: "sha256aaa"},
|
||||
}}
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if !p.ServeLocalIndex(w, r, reader, "myrepo", path) {
|
||||
t.Fatalf("ServeLocalIndex returned false for %q", path)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// Packages is deterministic: require exact byte identity.
|
||||
if plain, dotted := serve("Packages"), serve("./Packages"); plain.Code != 200 || dotted.Code != 200 {
|
||||
t.Fatalf("Packages: plain=%d dotted=%d, want 200/200", plain.Code, dotted.Code)
|
||||
} else if !bytes.Equal(plain.Body.Bytes(), dotted.Body.Bytes()) {
|
||||
t.Error("./Packages body differs from Packages body")
|
||||
}
|
||||
|
||||
// Release carries a Date: header stamped from time.Now(); compare the rest.
|
||||
plain, dotted := serve("Release"), serve("./Release")
|
||||
if plain.Code != 200 || dotted.Code != 200 {
|
||||
t.Fatalf("Release: plain=%d dotted=%d, want 200/200", plain.Code, dotted.Code)
|
||||
}
|
||||
if stripDate(plain.Body.String()) != stripDate(dotted.Body.String()) {
|
||||
t.Error("./Release body differs from Release body (ignoring Date)")
|
||||
}
|
||||
}
|
||||
|
||||
func stripDate(s string) string {
|
||||
var out []string
|
||||
for _, line := range strings.Split(s, "\n") {
|
||||
if strings.HasPrefix(line, "Date:") {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
func TestDebServeMetadataError(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for _, path := range []string{"Packages", "Packages.gz", "Release"} {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
p.ServeLocalIndex(w, r, errDebReader{}, "repo", path)
|
||||
if w.Code != 500 {
|
||||
t.Errorf("%s with failing reader = %d, want 500", path, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebGenerateLocalIndexUnsupported(t *testing.T) {
|
||||
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeDebReader{}, "r", "Packages"); err == nil {
|
||||
t.Error("expected unsupported error")
|
||||
}
|
||||
}
|
||||
|
||||
// buildDeb assembles an ar .deb whose control member uses the given name and
|
||||
// compressor, so the decompression branches can be exercised directly.
|
||||
func buildDeb(name, version, arch, member string, comp func([]byte) []byte) []byte {
|
||||
control := "Package: " + name + "\nVersion: " + version + "\nArchitecture: " + arch + "\n"
|
||||
controlTar := comp(tarSingle("./control", []byte(control)))
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.WriteString("!<arch>\n")
|
||||
arWrite(&buf, "debian-binary", []byte("2.0\n"))
|
||||
arWrite(&buf, member, controlTar)
|
||||
arWrite(&buf, "data.tar.gz", gzipBytes(tarSingle("./x", []byte("x"))))
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func tarSingle(name string, data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), Typeflag: tar.TypeReg})
|
||||
tw.Write(data)
|
||||
tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func arWrite(buf *bytes.Buffer, name string, data []byte) {
|
||||
fmt.Fprintf(buf, "%-16s%-12s%-6s%-6s%-8s%-10d`\n", name, "0", "0", "0", "100644", len(data))
|
||||
buf.Write(data)
|
||||
if len(data)%2 == 1 {
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
|
||||
func xzBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
w, _ := xz.NewWriter(&buf)
|
||||
w.Write(data)
|
||||
w.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func zstdBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
w, _ := zstd.NewWriter(&buf)
|
||||
w.Write(data)
|
||||
w.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
@@ -1,724 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// gitHubProvider is the process-wide singleton for github_deb. The background
|
||||
// Syncer binds its shared rate limiter and work queue onto this instance so the
|
||||
// request path and the syncer drive the same derive machinery.
|
||||
var gitHubProvider = newGitHubProvider()
|
||||
|
||||
func init() {
|
||||
provider.Register(gitHubProvider)
|
||||
}
|
||||
|
||||
// Tuning knobs for the no-precache control fetch. A .deb is an ar archive whose
|
||||
// control.tar member sits right after the tiny debian-binary member, so a small
|
||||
// front prefix reliably covers it.
|
||||
const (
|
||||
defaultHeaderRangeInitial = 32 << 10 // 32 KiB — covers control.tar of almost every .deb
|
||||
defaultHeaderRangeMax = 16 << 20 // 16 MiB — give up past this and skip the asset
|
||||
defaultReleasePageCap = 10 // 100 releases/page * 10 pages
|
||||
|
||||
defaultScanTimeout = 10 * time.Minute
|
||||
defaultServeTimeout = 30 * time.Second
|
||||
defaultColdWait = 8 * time.Second
|
||||
)
|
||||
|
||||
// GitHubProvider is a metadata-only remote: it scans a GitHub repo's releases
|
||||
// for .deb assets, derives per-asset control metadata via a ranged prefix fetch
|
||||
// (never downloading whole packages), synthesizes a flat apt repository from that
|
||||
// cached metadata, and redirects package downloads to a backend "releases_remote"
|
||||
// (the generic github.com remote) that serves the actual bytes.
|
||||
type GitHubProvider struct {
|
||||
client *http.Client
|
||||
|
||||
headerInitial int64
|
||||
headerMax int64
|
||||
pageCap int
|
||||
scanTimeout time.Duration
|
||||
serveTimeout time.Duration
|
||||
coldWait time.Duration
|
||||
|
||||
limiter *rate.Limiter
|
||||
syncer *Syncer
|
||||
|
||||
serverCred githubauth.Credential
|
||||
|
||||
mu sync.Mutex
|
||||
scanning map[string]bool
|
||||
lastScan map[string]time.Time
|
||||
}
|
||||
|
||||
func newGitHubProvider() *GitHubProvider {
|
||||
return &GitHubProvider{
|
||||
client: &http.Client{},
|
||||
headerInitial: defaultHeaderRangeInitial,
|
||||
headerMax: defaultHeaderRangeMax,
|
||||
pageCap: defaultReleasePageCap,
|
||||
scanTimeout: defaultScanTimeout,
|
||||
serveTimeout: defaultServeTimeout,
|
||||
coldWait: defaultColdWait,
|
||||
scanning: map[string]bool{},
|
||||
lastScan: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) limiterWait(ctx context.Context) error {
|
||||
if p.limiter == nil {
|
||||
return nil
|
||||
}
|
||||
return p.limiter.Wait(ctx)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) Type() models.PackageType { return models.PackageGitHubDeb }
|
||||
|
||||
func (p *GitHubProvider) Classify(path string) provider.Mutability {
|
||||
switch path {
|
||||
case "Packages", "Packages.gz", "Release", "InRelease", "Release.gpg":
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) ContentType(path string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".deb"):
|
||||
return "application/vnd.debian.binary-package"
|
||||
case strings.HasSuffix(path, ".gz"):
|
||||
return "application/gzip"
|
||||
case path == "Packages" || path == "Release" || path == "InRelease":
|
||||
return "text/plain"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error) {
|
||||
return p.githubHeaders(ctx, remote, false)
|
||||
}
|
||||
|
||||
// ServeRemote answers a request against a github_deb remote. It refreshes the
|
||||
// derived metadata (bounded by mutable_ttl), serves a synthesized flat apt repo
|
||||
// (Packages/Packages.gz/Release), 404s the signed index variants (the repo is
|
||||
// consumed via [trusted=yes]), and 302-redirects .deb downloads to the backend
|
||||
// releases_remote. Returns false only for paths it does not own.
|
||||
func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, reqPath, proxyBaseURL string, store provider.RemoteMetadataStore) bool {
|
||||
p.onRequest(remote, store)
|
||||
|
||||
// apt appends the flat-repo dist "./" verbatim, so it asks for "./Packages"
|
||||
// etc.; collapse the dot-segment before matching the synthesized index.
|
||||
path := normalizeIndexPath(reqPath)
|
||||
|
||||
switch path {
|
||||
case "Packages", "Packages.gz", "Release":
|
||||
p.serveIndex(w, r, remote, path, store)
|
||||
return true
|
||||
case "InRelease", "Release.gpg":
|
||||
// Unsigned flat repo: apt consumes it with [trusted=yes]. Signal absence
|
||||
// so apt falls back to the plain Release without waiting on a signature.
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasSuffix(path, ".deb") {
|
||||
if remote.ReleasesRemote == "" {
|
||||
http.Error(w, "github_deb remote has no releases_remote configured for downloads", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
loc := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.ReleasesRemote + "/" + strings.TrimLeft(path, "/")
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) serveIndex(w http.ResponseWriter, r *http.Request, remote models.Remote, path string, store provider.RemoteMetadataStore) {
|
||||
// Serve on a context detached from the inbound request so a client disconnect
|
||||
// never cancels the metadata DB read and surfaces as a 500.
|
||||
sctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), p.serveTimeout)
|
||||
defer cancel()
|
||||
|
||||
if p.syncer != nil && !p.ensurePrimed(sctx, remote, store) {
|
||||
w.Header().Set("Retry-After", "5")
|
||||
http.Error(w, "metadata is being prepared, retry shortly", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
reader, ok := store.(provider.DebMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "deb metadata not available", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
metas, err := reader.ListDebMetadataEntries(sctx, remote.Name)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
switch path {
|
||||
case "Packages":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generatePackages(metas))
|
||||
case "Packages.gz":
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(gzipBytes(generatePackages(metas)))
|
||||
case "Release":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateRelease(metas))
|
||||
}
|
||||
}
|
||||
|
||||
// onRequest keeps a remote's derived metadata fresh off the request path.
|
||||
func (p *GitHubProvider) onRequest(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
if p.syncer != nil {
|
||||
p.syncer.enqueue(remote, false)
|
||||
return
|
||||
}
|
||||
p.refresh(remote, store)
|
||||
}
|
||||
|
||||
// ensurePrimed returns true once the remote has at least one cached row. On an
|
||||
// empty cache it enqueues a prime and polls briefly for it to land.
|
||||
func (p *GitHubProvider) ensurePrimed(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) bool {
|
||||
if !p.cacheEmpty(ctx, store, remote.Name) {
|
||||
return true
|
||||
}
|
||||
if p.syncer != nil {
|
||||
p.syncer.enqueue(remote, true)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(p.coldWait)
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-time.After(400 * time.Millisecond):
|
||||
}
|
||||
if !p.cacheEmpty(ctx, store, remote.Name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) cacheEmpty(ctx context.Context, store provider.RemoteMetadataStore, name string) bool {
|
||||
reader, ok := store.(provider.DebMetadataReader)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
rows, err := reader.ListDebMetadataEntries(ctx, name)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return len(rows) == 0
|
||||
}
|
||||
|
||||
// refresh brings the derived metadata up to date without coupling the scan to
|
||||
// the inbound request (legacy inline path used without a syncer / in unit tests).
|
||||
func (p *GitHubProvider) refresh(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
ttl := time.Duration(remote.MutableTTL) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = 5 * time.Minute
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
last, ok := p.lastScan[remote.Name]
|
||||
fresh := ok && time.Since(last) < ttl
|
||||
if fresh || p.scanning[remote.Name] {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.scanning[remote.Name] = true
|
||||
p.mu.Unlock()
|
||||
|
||||
if p.cacheEmpty(context.Background(), store, remote.Name) {
|
||||
p.runScan(remote, store)
|
||||
return
|
||||
}
|
||||
go p.runScan(remote, store)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) runScan(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
delete(p.scanning, remote.Name)
|
||||
p.mu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), p.scanTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := p.scan(ctx, remote, store); err != nil {
|
||||
slog.Error("github_deb: release scan failed", "remote", remote.Name, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.lastScan[remote.Name] = time.Now()
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// scan runs a full unconditional derive. Retained for the legacy inline refresh
|
||||
// path and existing tests; the syncer uses scanWithState.
|
||||
func (p *GitHubProvider) scan(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) error {
|
||||
_, _, err := p.scanWithState(ctx, remote, store, "")
|
||||
return err
|
||||
}
|
||||
|
||||
// scanWithState derives metadata incrementally. It sends the prior releases-list
|
||||
// ETag as a conditional request: a 304 means nothing changed. On a 200 it diffs
|
||||
// the release assets against the cache, derives only new/changed assets, prunes
|
||||
// assets that disappeared, and returns the new ETag.
|
||||
func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore, etag string) (newEtag string, changed bool, err error) {
|
||||
releases, newEtag, notModified, err := p.fetchReleases(ctx, remote, etag)
|
||||
if err != nil {
|
||||
return etag, false, err
|
||||
}
|
||||
if notModified {
|
||||
return etag, false, nil
|
||||
}
|
||||
|
||||
reader, ok := store.(provider.DebMetadataReader)
|
||||
if !ok {
|
||||
return newEtag, false, errors.New("store does not support deb metadata reads")
|
||||
}
|
||||
existing, err := reader.ListDebMetadataEntries(ctx, remote.Name)
|
||||
if err != nil {
|
||||
return newEtag, false, err
|
||||
}
|
||||
existingByPath := make(map[string]provider.DebMetadata, len(existing))
|
||||
for _, m := range existing {
|
||||
existingByPath[m.FilePath] = m
|
||||
}
|
||||
|
||||
allow, err := compilePatterns(remote.Patterns)
|
||||
if err != nil {
|
||||
return newEtag, false, err
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
for _, rel := range releases {
|
||||
if rel.Draft {
|
||||
continue
|
||||
}
|
||||
for _, asset := range rel.Assets {
|
||||
if !strings.HasSuffix(strings.ToLower(asset.Name), ".deb") {
|
||||
continue
|
||||
}
|
||||
if !matchesAny(allow, asset.Name) {
|
||||
continue
|
||||
}
|
||||
fp := assetPath(asset)
|
||||
if fp == "" {
|
||||
continue
|
||||
}
|
||||
seen[fp] = true
|
||||
|
||||
if cur, ok := existingByPath[fp]; ok {
|
||||
if asset.Digest == "" || cur.ContentHash == asset.Digest {
|
||||
continue
|
||||
}
|
||||
_ = store.DeleteDebMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
|
||||
meta, err := p.deriveAsset(ctx, remote, asset, fp)
|
||||
if err != nil {
|
||||
slog.Warn("github_deb: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
if err := store.InsertDebMetadata(ctx, meta); err != nil {
|
||||
slog.Error("github_deb: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
slog.Info("github_deb: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
|
||||
}
|
||||
}
|
||||
|
||||
for fp := range existingByPath {
|
||||
if !seen[fp] {
|
||||
_ = store.DeleteDebMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
}
|
||||
return newEtag, true, nil
|
||||
}
|
||||
|
||||
type ghRelease struct {
|
||||
TagName string `json:"tag_name"`
|
||||
Draft bool `json:"draft"`
|
||||
Assets []ghAsset `json:"assets"`
|
||||
}
|
||||
|
||||
type ghAsset struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
BrowserDownloadURL string `json:"browser_download_url"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// fetchReleases lists a repo's releases, sending the prior ETag as If-None-Match
|
||||
// on page 1 so an unchanged repo short-circuits to notModified. Every call waits
|
||||
// on the shared limiter first.
|
||||
func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote, etag string) (all []ghRelease, newEtag string, notModified bool, err error) {
|
||||
base := strings.TrimRight(remote.BaseURL, "/") + "/releases"
|
||||
for page := 1; page <= p.pageCap; page++ {
|
||||
u := fmt.Sprintf("%s?per_page=100&page=%d", base, page)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
hdr, err := p.githubHeaders(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
copyHeaders(req, hdr)
|
||||
if page == 1 && etag != "" {
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
|
||||
if err := p.limiterWait(ctx); err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
if page == 1 && resp.StatusCode == http.StatusNotModified {
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, etag, true, nil
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
respEtag := resp.Header.Get("ETag")
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
|
||||
}
|
||||
if page == 1 {
|
||||
newEtag = respEtag
|
||||
}
|
||||
var releases []ghRelease
|
||||
if err := json.Unmarshal(body, &releases); err != nil {
|
||||
return nil, "", false, fmt.Errorf("decode releases: %w", err)
|
||||
}
|
||||
if len(releases) == 0 {
|
||||
break
|
||||
}
|
||||
all = append(all, releases...)
|
||||
if len(releases) < 100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return all, newEtag, false, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote, asset ghAsset, fp string) (*provider.DebMetadata, error) {
|
||||
control, err := p.fetchControl(ctx, remote, asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fields := parseControlFields(control)
|
||||
|
||||
meta := &provider.DebMetadata{
|
||||
RepoName: remote.Name,
|
||||
FilePath: fp,
|
||||
Name: fields["Package"],
|
||||
Version: fields["Version"],
|
||||
Architecture: fields["Architecture"],
|
||||
Control: strings.TrimRight(control, "\n"),
|
||||
Size: asset.Size,
|
||||
}
|
||||
if meta.Name == "" {
|
||||
return nil, errors.New("control missing Package field")
|
||||
}
|
||||
|
||||
// The Packages SHA256 must be the sha256 of the whole .deb. Prefer GitHub's
|
||||
// asset digest so we never download the body; only when it is absent (or not
|
||||
// sha256) do we stream the asset once. MD5sum is left unset — apt verifies the
|
||||
// download against SHA256 alone under [trusted=yes].
|
||||
if h, ok := sha256FromDigest(asset.Digest); ok {
|
||||
meta.ContentHash = "sha256:" + h
|
||||
meta.SHA256 = h
|
||||
} else {
|
||||
h, err := p.computeSHA256(ctx, remote, asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("compute sha256: %w", err)
|
||||
}
|
||||
meta.ContentHash = "sha256:" + h
|
||||
meta.SHA256 = h
|
||||
}
|
||||
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
// fetchControl pulls only the front of the .deb with a ranged GET and extracts
|
||||
// the control paragraph from it. control.tar sits right after the tiny
|
||||
// debian-binary member, so a small prefix suffices; a prefix that truncates the
|
||||
// control member doubles the range and retries.
|
||||
func (p *GitHubProvider) fetchControl(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
|
||||
n := p.headerInitial
|
||||
for {
|
||||
body, full, err := p.rangeGet(ctx, remote, downloadURL, n)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
control, complete, perr := controlFromPrefix(body)
|
||||
if perr != nil {
|
||||
return "", fmt.Errorf("parse deb control: %w", perr)
|
||||
}
|
||||
if complete {
|
||||
return control, nil
|
||||
}
|
||||
if full || n >= p.headerMax {
|
||||
return "", fmt.Errorf("control.tar not found within %d bytes of %s", n, downloadURL)
|
||||
}
|
||||
n *= 2
|
||||
if n > p.headerMax {
|
||||
n = p.headerMax
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// controlFromPrefix parses the ar members present in a front prefix of a .deb.
|
||||
// It returns the ./control paragraph once control.tar.* is fully covered
|
||||
// (complete=true); a prefix too short to cover it returns complete=false so the
|
||||
// caller can widen the range. Later members (data.tar.*) are ignored.
|
||||
func controlFromPrefix(prefix []byte) (control string, complete bool, err error) {
|
||||
const magic = "!<arch>\n"
|
||||
if len(prefix) < len(magic) {
|
||||
return "", false, nil
|
||||
}
|
||||
if string(prefix[:len(magic)]) != magic {
|
||||
return "", false, errors.New("not an ar archive")
|
||||
}
|
||||
off := len(magic)
|
||||
for {
|
||||
if off+60 > len(prefix) {
|
||||
return "", false, nil
|
||||
}
|
||||
hdr := prefix[off : off+60]
|
||||
off += 60
|
||||
name := strings.TrimSuffix(strings.TrimRight(string(hdr[0:16]), " "), "/")
|
||||
size, err := strconv.ParseInt(strings.TrimSpace(string(hdr[48:58])), 10, 64)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("bad ar size for %q: %w", name, err)
|
||||
}
|
||||
if strings.HasPrefix(name, "control.tar") {
|
||||
if off+int(size) > len(prefix) {
|
||||
return "", false, nil
|
||||
}
|
||||
tarBytes, err := decompress(name, prefix[off:off+int(size)])
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
c, err := readControlParagraph(tarBytes)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
if off+int(size) > len(prefix) {
|
||||
return "", false, nil
|
||||
}
|
||||
off += int(size)
|
||||
if size%2 == 1 {
|
||||
off++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rangeGet returns the first n bytes of downloadURL. full is true when the
|
||||
// response body was shorter than n (i.e. we already have the whole object).
|
||||
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
hdr, err := p.githubHeaders(ctx, remote, false)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
copyHeaders(req, hdr)
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", n-1))
|
||||
|
||||
if err := p.limiterWait(ctx); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
full := int64(len(body)) < n
|
||||
return body, full, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
hdr, err := p.githubHeaders(ctx, remote, false)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
copyHeaders(req, hdr)
|
||||
|
||||
if err := p.limiterWait(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, resp.Body); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// assetPath is the package's location relative to github.com — the path the
|
||||
// backend releases_remote (base https://github.com) proxies. It doubles as the
|
||||
// deb_metadata key and the Filename field in the Packages index, so a .deb
|
||||
// download resolves back to this remote and redirects to the backend.
|
||||
func assetPath(asset ghAsset) string {
|
||||
u, err := url.Parse(asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimPrefix(u.Path, "/")
|
||||
}
|
||||
|
||||
func sha256FromDigest(digest string) (string, bool) {
|
||||
if strings.HasPrefix(digest, "sha256:") {
|
||||
return strings.TrimPrefix(digest, "sha256:"), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// githubHeaders builds the outbound headers for a GitHub request, attaching a
|
||||
// bearer credential when one is available. A per-remote credential wins; absent
|
||||
// that, the process-wide server credential is used; absent both, the request is
|
||||
// unauthenticated.
|
||||
func (p *GitHubProvider) githubHeaders(ctx context.Context, remote models.Remote, api bool) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if api {
|
||||
h.Set("Accept", "application/vnd.github+json")
|
||||
h.Set("X-GitHub-Api-Version", "2022-11-28")
|
||||
}
|
||||
tok, err := p.githubToken(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tok != "" {
|
||||
h.Set("Authorization", "Bearer "+tok)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// githubToken resolves the bearer token for a remote. Precedence: a per-remote
|
||||
// credential (password, then username) overrides the server credential.
|
||||
func (p *GitHubProvider) githubToken(ctx context.Context, remote models.Remote) (string, error) {
|
||||
if remote.Password != "" {
|
||||
return remote.Password, nil
|
||||
}
|
||||
if remote.Username != "" {
|
||||
return remote.Username, nil
|
||||
}
|
||||
if c := p.serverCredential(); c != nil {
|
||||
return c.Token(ctx)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) serverCredential() githubauth.Credential {
|
||||
if p.serverCred != nil {
|
||||
return p.serverCred
|
||||
}
|
||||
return githubauth.Server()
|
||||
}
|
||||
|
||||
func copyHeaders(req *http.Request, h http.Header) {
|
||||
for k, vals := range h {
|
||||
for _, v := range vals {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func compilePatterns(patterns []string) ([]*regexp.Regexp, error) {
|
||||
var out []*regexp.Regexp
|
||||
for _, p := range patterns {
|
||||
re, err := regexp.Compile(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid pattern %q: %w", p, err)
|
||||
}
|
||||
out = append(out, re)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func matchesAny(res []*regexp.Regexp, s string) bool {
|
||||
if len(res) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, re := range res {
|
||||
if re.MatchString(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,462 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeStore is an in-memory provider.RemoteMetadataStore + DebMetadataReader
|
||||
// keyed by file_path, mirroring the (repo_name, file_path) uniqueness of the
|
||||
// real deb_metadata table.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
rows map[string]provider.DebMetadata
|
||||
}
|
||||
|
||||
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.DebMetadata{}} }
|
||||
|
||||
func (f *fakeStore) InsertDebMetadata(_ context.Context, m *provider.DebMetadata) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.rows[m.FilePath]; ok {
|
||||
return nil // ON CONFLICT DO NOTHING
|
||||
}
|
||||
f.rows[m.FilePath] = *m
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) DeleteDebMetadata(_ context.Context, _, filePath string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.rows, filePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeStore) DeleteRPMMetadata(context.Context, string, string) error { return nil }
|
||||
func (f *fakeStore) ListRPMMetadataEntries(context.Context, string) ([]provider.RPMMetadata, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) ListDebMetadataEntries(ctx context.Context, _ string) ([]provider.DebMetadata, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := make([]provider.DebMetadata, 0, len(f.rows))
|
||||
for _, m := range f.rows {
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// githubFixture serves the releases API and the .deb asset downloads (with Range
|
||||
// support) for a set of packages. digest controls whether the asset carries a
|
||||
// sha256 digest (no-download path) or not (compute path).
|
||||
type githubFixture struct {
|
||||
srv *httptest.Server
|
||||
debBytes map[string][]byte
|
||||
rangeHit map[string]int
|
||||
fullHit map[string]int
|
||||
etag string
|
||||
releasesHit int
|
||||
notModHit int
|
||||
releaseAuth string
|
||||
assetAuth string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
t.Helper()
|
||||
f := &githubFixture{
|
||||
debBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
}
|
||||
f.debBytes["demo_1.2-3_amd64.deb"] = testsupport.MinimalDeb("demo", "1.2-3", "amd64")
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) {
|
||||
page := r.URL.Query().Get("page")
|
||||
if page != "" && page != "1" {
|
||||
w.Write([]byte("[]"))
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.releasesHit++
|
||||
f.releaseAuth = r.Header.Get("Authorization")
|
||||
etag := f.etag
|
||||
if etag != "" && r.Header.Get("If-None-Match") == etag {
|
||||
f.notModHit++
|
||||
f.mu.Unlock()
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if etag != "" {
|
||||
w.Header().Set("ETag", etag)
|
||||
}
|
||||
var assets []map[string]any
|
||||
for name := range f.debBytes {
|
||||
a := map[string]any{
|
||||
"name": name,
|
||||
"size": len(f.debBytes[name]),
|
||||
"browser_download_url": f.srv.URL + "/acme/tools/releases/download/v1.2-3/" + name,
|
||||
}
|
||||
if withDigest {
|
||||
sum := sha256.Sum256(f.debBytes[name])
|
||||
a["digest"] = "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
assets = append(assets, a)
|
||||
}
|
||||
rel := []map[string]any{{"tag_name": "v1.2-3", "draft": false, "assets": assets}}
|
||||
json.NewEncoder(w).Encode(rel)
|
||||
})
|
||||
mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
||||
body, ok := f.debBytes[name]
|
||||
if !ok {
|
||||
http.Error(w, "not found", 404)
|
||||
return
|
||||
}
|
||||
rng := r.Header.Get("Range")
|
||||
f.mu.Lock()
|
||||
f.assetAuth = r.Header.Get("Authorization")
|
||||
if rng != "" {
|
||||
f.rangeHit[name]++
|
||||
} else {
|
||||
f.fullHit[name]++
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
if rng == "" {
|
||||
w.WriteHeader(200)
|
||||
w.Write(body)
|
||||
return
|
||||
}
|
||||
var end int
|
||||
fmt.Sscanf(rng, "bytes=0-%d", &end)
|
||||
if end >= len(body)-1 {
|
||||
end = len(body) - 1
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
|
||||
w.Header().Set("Content-Length", strconv.Itoa(end+1))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write(body[:end+1])
|
||||
})
|
||||
f.srv = httptest.NewServer(mux)
|
||||
t.Cleanup(f.srv.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *githubFixture) remote() models.Remote {
|
||||
return models.Remote{
|
||||
Name: "acme-deb",
|
||||
PackageType: models.PackageGitHubDeb,
|
||||
BaseURL: f.srv.URL + "/repos/acme/tools",
|
||||
ReleasesRemote: "github",
|
||||
MutableTTL: 3600,
|
||||
}
|
||||
}
|
||||
|
||||
func newTestProvider() *GitHubProvider {
|
||||
p := newGitHubProvider()
|
||||
p.headerInitial = 32 // force the ranged-fetch retry loop against the tiny fixture
|
||||
p.headerMax = 1 << 20
|
||||
return p
|
||||
}
|
||||
|
||||
const demoPath = "acme/tools/releases/download/v1.2-3/demo_1.2-3_amd64.deb"
|
||||
|
||||
func TestGitHubScanDerivesControlFromPrefixAndDigest(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
metas, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
|
||||
if len(metas) != 1 {
|
||||
t.Fatalf("want 1 metadata row, got %d", len(metas))
|
||||
}
|
||||
m := metas[0]
|
||||
if m.Name != "demo" || m.Version != "1.2-3" || m.Architecture != "amd64" {
|
||||
t.Fatalf("bad control fields: %+v", m)
|
||||
}
|
||||
if m.FilePath != demoPath {
|
||||
t.Fatalf("FilePath = %q, want %q", m.FilePath, demoPath)
|
||||
}
|
||||
if int(m.Size) != len(fx.debBytes["demo_1.2-3_amd64.deb"]) {
|
||||
t.Fatalf("Size = %d, want %d", m.Size, len(fx.debBytes["demo_1.2-3_amd64.deb"]))
|
||||
}
|
||||
sum := sha256.Sum256(fx.debBytes["demo_1.2-3_amd64.deb"])
|
||||
if m.SHA256 != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("SHA256 = %q, want digest", m.SHA256)
|
||||
}
|
||||
if m.ContentHash != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("ContentHash = %q", m.ContentHash)
|
||||
}
|
||||
if m.MD5 != "" {
|
||||
t.Fatalf("MD5 should be unset for metadata-only derive, got %q", m.MD5)
|
||||
}
|
||||
if fx.fullHit["demo_1.2-3_amd64.deb"] != 0 {
|
||||
t.Fatalf("expected no full download when digest present, got %d", fx.fullHit["demo_1.2-3_amd64.deb"])
|
||||
}
|
||||
if fx.rangeHit["demo_1.2-3_amd64.deb"] == 0 {
|
||||
t.Fatalf("expected ranged control fetch")
|
||||
}
|
||||
if !strings.Contains(m.Control, "Package: demo") {
|
||||
t.Fatalf("raw control not captured: %q", m.Control)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubChecksumComputedWhenDigestAbsent(t *testing.T) {
|
||||
fx := newGitHubFixture(t, false)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
metas, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
|
||||
if len(metas) != 1 {
|
||||
t.Fatalf("want 1 row, got %d", len(metas))
|
||||
}
|
||||
sum := sha256.Sum256(fx.debBytes["demo_1.2-3_amd64.deb"])
|
||||
if metas[0].SHA256 != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("computed checksum mismatch: %q", metas[0].SHA256)
|
||||
}
|
||||
if fx.fullHit["demo_1.2-3_amd64.deb"] == 0 {
|
||||
t.Fatalf("expected a full download to compute sha256 when digest absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteIndexAndRedirect(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://artifactapi.example"
|
||||
|
||||
// Release is served and triggers the initial scan.
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Release", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "Release", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle Release")
|
||||
}
|
||||
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "Architectures:") {
|
||||
t.Fatalf("Release bad: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "amd64") {
|
||||
t.Fatalf("Release missing arch: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
// Packages carries the package with a Filename that is the github-relative
|
||||
// download path (so it resolves back to this remote and redirects).
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "Packages", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle Packages")
|
||||
}
|
||||
pkgs := rec.Body.String()
|
||||
if !strings.Contains(pkgs, "Package: demo") {
|
||||
t.Fatalf("Packages missing package: %s", pkgs)
|
||||
}
|
||||
if !strings.Contains(pkgs, "Filename: "+demoPath) {
|
||||
t.Fatalf("Packages missing/incorrect Filename: %s", pkgs)
|
||||
}
|
||||
if !strings.Contains(pkgs, "SHA256: ") {
|
||||
t.Fatalf("Packages missing SHA256: %s", pkgs)
|
||||
}
|
||||
if strings.Contains(pkgs, "MD5sum:") {
|
||||
t.Fatalf("Packages should omit empty MD5sum: %s", pkgs)
|
||||
}
|
||||
|
||||
// Packages.gz decompresses to the same content.
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "Packages.gz", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle Packages.gz")
|
||||
}
|
||||
gz, err := gzip.NewReader(rec.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
unz, _ := io.ReadAll(gz)
|
||||
if !strings.Contains(string(unz), "Package: demo") {
|
||||
t.Fatalf("Packages.gz missing package: %s", unz)
|
||||
}
|
||||
|
||||
// InRelease/Release.gpg 404 (unsigned, consumed via [trusted=yes]).
|
||||
for _, sp := range []string{"InRelease", "Release.gpg"} {
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, sp, proxyBase, store) {
|
||||
t.Fatalf("ServeRemote did not handle %s", sp)
|
||||
}
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("%s want 404, got %d", sp, rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A .deb request redirects to the backend releases_remote.
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/"+demoPath, nil)
|
||||
if !p.ServeRemote(rec, req, remote, demoPath, proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle .deb")
|
||||
}
|
||||
if rec.Code != http.StatusFound {
|
||||
t.Fatalf("want 302, got %d", rec.Code)
|
||||
}
|
||||
wantLoc := proxyBase + "/api/v1/remote/github/" + demoPath
|
||||
if got := rec.Header().Get("Location"); got != wantLoc {
|
||||
t.Fatalf("Location = %q, want %q", got, wantLoc)
|
||||
}
|
||||
}
|
||||
|
||||
// Real apt appends the flat-repo dist "./" verbatim, so the metadata-only remote
|
||||
// receives "./Packages" / "./Release"; ServeRemote must collapse the dot-segment
|
||||
// and synthesize the same index as the un-prefixed request.
|
||||
func TestGitHubServeRemoteAptDotSegment(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://artifactapi.example"
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/"+path, nil)
|
||||
if !p.ServeRemote(rec, req, remote, path, proxyBase, store) {
|
||||
t.Fatalf("ServeRemote did not handle %q", path)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
// Packages is deterministic: byte-identical to the un-prefixed request.
|
||||
plain, dotted := serve("Packages"), serve("./Packages")
|
||||
if plain.Code != 200 || dotted.Code != 200 {
|
||||
t.Fatalf("Packages: plain=%d dotted=%d, want 200/200", plain.Code, dotted.Code)
|
||||
}
|
||||
if !strings.Contains(dotted.Body.String(), "Package: demo") {
|
||||
t.Fatalf("./Packages missing synthesized body: %s", dotted.Body.String())
|
||||
}
|
||||
if !bytes.Equal(plain.Body.Bytes(), dotted.Body.Bytes()) {
|
||||
t.Error("./Packages body differs from Packages body")
|
||||
}
|
||||
|
||||
// Release carries a time.Now() Date: header; compare the rest.
|
||||
rPlain, rDotted := serve("Release"), serve("./Release")
|
||||
if rPlain.Code != 200 || rDotted.Code != 200 {
|
||||
t.Fatalf("Release: plain=%d dotted=%d, want 200/200", rPlain.Code, rDotted.Code)
|
||||
}
|
||||
if stripDate(rPlain.Body.String()) != stripDate(rDotted.Body.String()) {
|
||||
t.Error("./Release body differs from Release body (ignoring Date)")
|
||||
}
|
||||
}
|
||||
|
||||
// A canceled inbound request must still serve the warm cache (detached context),
|
||||
// not turn the metadata read into a 500.
|
||||
func TestGitHubServeRemoteCanceledRequestServesCache(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.lastScan[remote.Name] = time.Now()
|
||||
p.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Packages", nil).WithContext(ctx)
|
||||
|
||||
if !p.ServeRemote(rec, req, remote, "Packages", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle Packages")
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("canceled request must serve cache, not error; got code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "Package: demo") {
|
||||
t.Fatalf("expected Packages served from cache, got %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteRedirectRequiresReleasesRemote(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
remote.ReleasesRemote = ""
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, demoPath, "https://x", store) {
|
||||
t.Fatal("expected handled")
|
||||
}
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("want 500 when releases_remote unset, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubScanPrunesRemovedAssets(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb"); len(rows) != 1 {
|
||||
t.Fatalf("want 1 row after first scan, got %d", len(rows))
|
||||
}
|
||||
|
||||
delete(fx.debBytes, "demo_1.2-3_amd64.deb")
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("rescan: %v", err)
|
||||
}
|
||||
if rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb"); len(rows) != 0 {
|
||||
t.Fatalf("want 0 rows after prune, got %d", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubAssetPatternFilter(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.debBytes["other_9_arm64.deb"] = testsupport.MinimalDeb("other", "9", "arm64")
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
remote.Patterns = []string{`^demo_.*_amd64\.deb$`}
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
|
||||
if len(rows) != 1 || rows[0].Name != "demo" {
|
||||
t.Fatalf("pattern filter failed, rows=%+v", rows)
|
||||
}
|
||||
}
|
||||
@@ -1,238 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const (
|
||||
syncLeaseDuration = 15 * time.Minute
|
||||
defaultSyncFreshness = 5 * time.Minute
|
||||
jobQueueDepth = 256
|
||||
)
|
||||
|
||||
// SyncStore is the persistence surface the deb syncer needs: the metadata cache
|
||||
// it primes plus the shared sync-state coordination (remote enumeration and the
|
||||
// per-remote lease). *database.DB satisfies it.
|
||||
type SyncStore interface {
|
||||
provider.RemoteMetadataStore
|
||||
ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
|
||||
ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
|
||||
}
|
||||
|
||||
// SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
|
||||
type SyncConfig struct {
|
||||
RatePerSec float64
|
||||
Burst int
|
||||
Workers int
|
||||
PollInterval time.Duration
|
||||
}
|
||||
|
||||
type syncJob struct {
|
||||
remote models.Remote
|
||||
prime bool
|
||||
}
|
||||
|
||||
// Syncer is the single per-process background worker that keeps every github_deb
|
||||
// remote's derived metadata fresh. It owns a deduped work queue, a pool of
|
||||
// workers, and a global token-bucket rate limiter shared across all remotes and
|
||||
// bound onto the github_deb provider. Periodic checks are gated by a shared DB
|
||||
// lease so, across replicas, only one performs each scan.
|
||||
type Syncer struct {
|
||||
store SyncStore
|
||||
prov *GitHubProvider
|
||||
limiter *rate.Limiter
|
||||
cfg SyncConfig
|
||||
owner string
|
||||
|
||||
jobs chan syncJob
|
||||
mu sync.Mutex
|
||||
active map[string]bool
|
||||
}
|
||||
|
||||
// NewSyncer builds the syncer bound to the process-wide github_deb provider
|
||||
// singleton. Call Run to start it.
|
||||
func NewSyncer(store SyncStore, cfg SyncConfig) *Syncer {
|
||||
return newSyncer(store, gitHubProvider, cfg)
|
||||
}
|
||||
|
||||
func newSyncer(store SyncStore, prov *GitHubProvider, cfg SyncConfig) *Syncer {
|
||||
if cfg.RatePerSec <= 0 {
|
||||
cfg.RatePerSec = 1
|
||||
}
|
||||
if cfg.Burst <= 0 {
|
||||
cfg.Burst = 5
|
||||
}
|
||||
if cfg.Workers <= 0 {
|
||||
cfg.Workers = 3
|
||||
}
|
||||
if cfg.PollInterval <= 0 {
|
||||
cfg.PollInterval = 60 * time.Second
|
||||
}
|
||||
|
||||
lim := rate.NewLimiter(rate.Limit(cfg.RatePerSec), cfg.Burst)
|
||||
s := &Syncer{
|
||||
store: store,
|
||||
prov: prov,
|
||||
limiter: lim,
|
||||
cfg: cfg,
|
||||
owner: leaseOwner(),
|
||||
jobs: make(chan syncJob, jobQueueDepth),
|
||||
active: map[string]bool{},
|
||||
}
|
||||
prov.limiter = lim
|
||||
prov.syncer = s
|
||||
return s
|
||||
}
|
||||
|
||||
// Run starts the worker pool and the periodic scheduler and blocks until ctx is
|
||||
// canceled, at which point it drains in-flight scans and returns.
|
||||
func (s *Syncer) Run(ctx context.Context) {
|
||||
slog.Info("github_deb syncer started",
|
||||
"rate_per_sec", s.cfg.RatePerSec, "burst", s.cfg.Burst,
|
||||
"workers", s.cfg.Workers, "poll_interval", s.cfg.PollInterval, "owner", s.owner)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < s.cfg.Workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
s.worker(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(s.cfg.PollInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
s.schedule(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
wg.Wait()
|
||||
slog.Info("github_deb syncer stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.schedule(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// schedule enqueues a periodic check for every github_deb remote. The DB lease
|
||||
// enforces the per-remote mutable_ttl cadence and cross-replica coordination.
|
||||
func (s *Syncer) schedule(ctx context.Context) {
|
||||
remotes, err := s.store.ListGitHubDebRemotes(ctx)
|
||||
if err != nil {
|
||||
slog.Error("github_deb syncer: list remotes", "error", err)
|
||||
return
|
||||
}
|
||||
for _, r := range remotes {
|
||||
s.enqueue(r, false)
|
||||
}
|
||||
}
|
||||
|
||||
// EnqueuePrime queues an immediate background prime for a freshly created remote.
|
||||
func (s *Syncer) EnqueuePrime(remote models.Remote) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.enqueue(remote, true)
|
||||
}
|
||||
|
||||
// enqueue adds a job unless the remote is already queued or in-flight, coalescing
|
||||
// duplicate requests down to one scan. It never blocks.
|
||||
func (s *Syncer) enqueue(remote models.Remote, prime bool) {
|
||||
s.mu.Lock()
|
||||
if s.active[remote.Name] {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.active[remote.Name] = true
|
||||
s.mu.Unlock()
|
||||
|
||||
select {
|
||||
case s.jobs <- syncJob{remote: remote, prime: prime}:
|
||||
default:
|
||||
s.mu.Lock()
|
||||
delete(s.active, remote.Name)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Syncer) worker(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case job := <-s.jobs:
|
||||
s.process(ctx, job)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// process claims the shared lease and, if won, runs an incremental scan. Losing
|
||||
// the claim (another replica scanning, or not yet due) is a no-op.
|
||||
func (s *Syncer) process(ctx context.Context, job syncJob) {
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
delete(s.active, job.remote.Name)
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
freshness := time.Duration(job.remote.MutableTTL) * time.Second
|
||||
if freshness <= 0 {
|
||||
freshness = defaultSyncFreshness
|
||||
}
|
||||
if job.prime {
|
||||
freshness = 0
|
||||
}
|
||||
|
||||
claimed, etag, err := s.store.ClaimGitHubDebSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration)
|
||||
if err != nil {
|
||||
slog.Error("github_deb syncer: claim lease", "remote", job.remote.Name, "error", err)
|
||||
return
|
||||
}
|
||||
if !claimed {
|
||||
return
|
||||
}
|
||||
|
||||
scanCtx, cancel := context.WithTimeout(ctx, s.prov.scanTimeout)
|
||||
defer cancel()
|
||||
|
||||
newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
|
||||
releaseEtag := etag
|
||||
if scanErr == nil {
|
||||
releaseEtag = newEtag
|
||||
} else {
|
||||
slog.Error("github_deb syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
|
||||
}
|
||||
|
||||
relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer relCancel()
|
||||
if err := s.store.ReleaseGitHubDebSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
|
||||
slog.Warn("github_deb syncer: release lease", "remote", job.remote.Name, "error", err)
|
||||
}
|
||||
|
||||
if scanErr == nil && changed {
|
||||
slog.Info("github_deb syncer: refreshed", "remote", job.remote.Name, "prime", job.prime)
|
||||
}
|
||||
}
|
||||
|
||||
// leaseOwner is a per-replica identity for the lease: hostname plus a random
|
||||
// suffix so restarts and colocated replicas never collide.
|
||||
func leaseOwner() string {
|
||||
host, _ := os.Hostname()
|
||||
var b [6]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return host + "-" + hex.EncodeToString(b[:])
|
||||
}
|
||||
@@ -1,300 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeSyncStore is an in-memory SyncStore: the metadata cache (via the embedded
|
||||
// fakeStore) plus the shared sync-state lease, whose claim mirrors the atomic
|
||||
// semantics of the real SQL (recency gate AND no live lease).
|
||||
type fakeSyncStore struct {
|
||||
*fakeStore
|
||||
|
||||
mu sync.Mutex
|
||||
remotes []models.Remote
|
||||
leaseOwner map[string]string
|
||||
leaseExp map[string]time.Time
|
||||
lastSynced map[string]time.Time
|
||||
etags map[string]string
|
||||
}
|
||||
|
||||
func newFakeSyncStore() *fakeSyncStore {
|
||||
return &fakeSyncStore{
|
||||
fakeStore: newFakeStore(),
|
||||
leaseOwner: map[string]string{},
|
||||
leaseExp: map[string]time.Time{},
|
||||
lastSynced: map[string]time.Time{},
|
||||
etags: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ListGitHubDebRemotes(_ context.Context) ([]models.Remote, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]models.Remote(nil), f.remotes...), nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ClaimGitHubDebSyncLease(_ context.Context, name, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
now := time.Now()
|
||||
ls, hasLS := f.lastSynced[name]
|
||||
exp, hasExp := f.leaseExp[name]
|
||||
freshOK := !hasLS || now.Sub(ls) >= freshness
|
||||
leaseOK := !hasExp || exp.Before(now)
|
||||
if freshOK && leaseOK {
|
||||
f.leaseOwner[name] = owner
|
||||
f.leaseExp[name] = now.Add(lease)
|
||||
return true, f.etags[name], nil
|
||||
}
|
||||
return false, "", nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ReleaseGitHubDebSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.leaseOwner[name] != owner {
|
||||
return nil
|
||||
}
|
||||
f.lastSynced[name] = syncedAt
|
||||
f.etags[name] = etag
|
||||
delete(f.leaseOwner, name)
|
||||
delete(f.leaseExp, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func testSyncConfig() SyncConfig {
|
||||
return SyncConfig{RatePerSec: 1000, Burst: 100, Workers: 1, PollInterval: time.Hour}
|
||||
}
|
||||
|
||||
// (a) A 304 conditional response must derive nothing: no asset fetches and
|
||||
// changed=false, so an unchanged repo is nearly free.
|
||||
func TestSyncerConditionalNotModifiedSkipsDerive(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
etag1, changed, err := p.scanWithState(context.Background(), remote, store, "")
|
||||
if err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
if !changed || etag1 != `"v1"` {
|
||||
t.Fatalf("first scan changed=%v etag=%q, want true and \"v1\"", changed, etag1)
|
||||
}
|
||||
priorRange := fx.rangeHit["demo_1.2-3_amd64.deb"]
|
||||
if priorRange == 0 {
|
||||
t.Fatal("first scan should have fetched the asset control")
|
||||
}
|
||||
|
||||
etag2, changed2, err := p.scanWithState(context.Background(), remote, store, etag1)
|
||||
if err != nil {
|
||||
t.Fatalf("second scan: %v", err)
|
||||
}
|
||||
if changed2 {
|
||||
t.Fatal("304 scan must report changed=false")
|
||||
}
|
||||
if etag2 != etag1 {
|
||||
t.Fatalf("etag changed across 304: %q -> %q", etag1, etag2)
|
||||
}
|
||||
if fx.notModHit != 1 {
|
||||
t.Fatalf("want exactly one 304 releases response, got %d", fx.notModHit)
|
||||
}
|
||||
if got := fx.rangeHit["demo_1.2-3_amd64.deb"]; got != priorRange {
|
||||
t.Fatalf("304 scan re-fetched asset control: %d -> %d", priorRange, got)
|
||||
}
|
||||
}
|
||||
|
||||
// (b) On a real change, only the newly added asset is derived.
|
||||
func TestSyncerIncrementalDerivesOnlyNewAsset(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
if _, _, err := p.scanWithState(context.Background(), remote, store, ""); err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
demoRange := fx.rangeHit["demo_1.2-3_amd64.deb"]
|
||||
|
||||
fx.debBytes["other_9_arm64.deb"] = testsupport.MinimalDeb("other", "9", "arm64")
|
||||
fx.etag = `"v2"`
|
||||
|
||||
if _, changed, err := p.scanWithState(context.Background(), remote, store, `"v1"`); err != nil || !changed {
|
||||
t.Fatalf("second scan changed=%v err=%v", changed, err)
|
||||
}
|
||||
|
||||
rows, _ := store.ListDebMetadataEntries(context.Background(), remote.Name)
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("want 2 cached rows after incremental derive, got %d", len(rows))
|
||||
}
|
||||
if got := fx.rangeHit["demo_1.2-3_amd64.deb"]; got != demoRange {
|
||||
t.Fatalf("already-cached asset was re-fetched: %d -> %d", demoRange, got)
|
||||
}
|
||||
if fx.rangeHit["other_9_arm64.deb"] == 0 {
|
||||
t.Fatal("newly added asset was not derived")
|
||||
}
|
||||
}
|
||||
|
||||
// (c) The shared limiter caps the request rate.
|
||||
func TestRateLimiterCapsRequestRate(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
p.limiter = rate.NewLimiter(rate.Every(120*time.Millisecond), 1)
|
||||
remote := fx.remote()
|
||||
|
||||
start := time.Now()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, _, _, err := p.fetchReleases(context.Background(), remote, ""); err != nil {
|
||||
t.Fatalf("fetchReleases %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
|
||||
t.Fatalf("rate limiter did not throttle: 3 calls took %v, want >= 200ms", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// (d) Concurrent enqueues for the same remote coalesce to a single queued job.
|
||||
func TestSyncerEnqueueDedup(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := models.Remote{Name: "acme-deb", PackageType: models.PackageGitHubDeb, MutableTTL: 3600}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); s.enqueue(remote, false) }()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if got := len(s.jobs); got != 1 {
|
||||
t.Fatalf("want exactly 1 coalesced job, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// (e) Prime-on-create enqueues a prime job.
|
||||
func TestSyncerEnqueuePrime(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := models.Remote{Name: "acme-deb", PackageType: models.PackageGitHubDeb, MutableTTL: 3600}
|
||||
|
||||
s.EnqueuePrime(remote)
|
||||
select {
|
||||
case job := <-s.jobs:
|
||||
if !job.prime || job.remote.Name != "acme-deb" {
|
||||
t.Fatalf("bad prime job: %+v", job)
|
||||
}
|
||||
default:
|
||||
t.Fatal("EnqueuePrime did not enqueue a job")
|
||||
}
|
||||
}
|
||||
|
||||
// (f) A held lease prevents a second replica from scanning.
|
||||
func TestSyncerLeasePreventsSecondReplica(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
claimed, _, err := store.ClaimGitHubDebSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote})
|
||||
|
||||
if fx.releasesHit != 0 {
|
||||
t.Fatalf("second replica scanned while lease held: %d releases calls", fx.releasesHit)
|
||||
}
|
||||
if rows, _ := store.ListDebMetadataEntries(context.Background(), remote.Name); len(rows) != 0 {
|
||||
t.Fatalf("second replica derived metadata while lease held: %d rows", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
// With the syncer wired and the cache empty, an index request enqueues a prime
|
||||
// and returns a retryable 503 when it has not landed within the cold wait.
|
||||
func TestServeRemoteColdStartReturns503(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
p.coldWait = 300 * time.Millisecond
|
||||
_ = newSyncer(store, p, testSyncConfig()) // binds p.syncer, but no workers running
|
||||
remote := fx.remote()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Packages", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "Packages", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle Packages")
|
||||
}
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("cold empty cache must return 503, got %d", rec.Code)
|
||||
}
|
||||
if rec.Header().Get("Retry-After") == "" {
|
||||
t.Fatal("503 should carry Retry-After")
|
||||
}
|
||||
if got := len(p.syncer.jobs); got != 1 {
|
||||
t.Fatalf("cold start did not enqueue a prime, jobs=%d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With the cache warm, the same request serves the index immediately (no 503).
|
||||
func TestServeRemoteWarmCacheServesImmediately(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
_ = newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Packages", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "Packages", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle Packages")
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("warm cache must serve 200, got %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A prime job (freshness 0) runs even right after a sync; a periodic job at the
|
||||
// same moment is gated by the recency window.
|
||||
func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
var _ provider.RemoteMetadataStore = store
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: true})
|
||||
if rows, _ := store.ListDebMetadataEntries(context.Background(), remote.Name); len(rows) != 1 {
|
||||
t.Fatalf("prime did not derive: %d rows", len(rows))
|
||||
}
|
||||
releasesAfterPrime := fx.releasesHit
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: false})
|
||||
if fx.releasesHit != releasesAfterPrime {
|
||||
t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestDockerClassifyBranches(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Classify("library/nginx/tags/list") != provider.Mutable {
|
||||
t.Error("tags/list should be mutable")
|
||||
}
|
||||
if p.Classify("library/nginx/manifests/latest") != provider.Mutable {
|
||||
t.Error("tag manifest should be mutable")
|
||||
}
|
||||
if p.Classify("library/nginx/manifests/sha256:abcdef") != provider.Immutable {
|
||||
t.Error("digest manifest should be immutable")
|
||||
}
|
||||
if p.Classify("library/nginx/blobs/sha256:abc") != provider.Immutable {
|
||||
t.Error("blob should be immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.ContentType("x/blobs/sha256:abc") != "application/octet-stream" {
|
||||
t.Error("blob content type")
|
||||
}
|
||||
if p.ContentType("x/manifests/latest") != "application/vnd.docker.distribution.manifest.v2+json" {
|
||||
t.Error("manifest content type")
|
||||
}
|
||||
if p.ContentType("x/tags/list") != "application/json" {
|
||||
t.Error("default content type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerRewriteAndAuth(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if out, err := p.RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
|
||||
t.Error("docker never rewrites")
|
||||
}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("expected basic auth header")
|
||||
}
|
||||
h, _ = p.AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("no creds, no header")
|
||||
}
|
||||
}
|
||||
@@ -4,11 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
@@ -61,42 +59,10 @@ func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte,
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// AuthHeaders authenticates outbound requests. A per-remote username/password
|
||||
// (Basic auth) takes precedence. Otherwise, when the remote points at a GitHub
|
||||
// host (e.g. a releases_remote proxying private release assets), the process-wide
|
||||
// GitHub credential is attached as a bearer token so private downloads work.
|
||||
func (p *Provider) AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error) {
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
|
||||
return h, nil
|
||||
}
|
||||
if isGitHubHost(remote.BaseURL) {
|
||||
if c := githubauth.Server(); c != nil {
|
||||
tok, err := c.Token(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tok != "" {
|
||||
h.Set("Authorization", "Bearer "+tok)
|
||||
}
|
||||
}
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// isGitHubHost reports whether rawURL targets a GitHub API/download host that
|
||||
// accepts the server credential. objects.githubusercontent.com is deliberately
|
||||
// excluded: release-asset downloads 302-redirect there with a pre-signed URL
|
||||
// that must not carry an Authorization header.
|
||||
func isGitHubHost(rawURL string) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(u.Hostname()) {
|
||||
case "github.com", "www.github.com", "api.github.com", "codeload.github.com", "uploads.github.com":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
package generic
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestGenericRewriteResponse(t *testing.T) {
|
||||
if out, err := (&Provider{}).RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil {
|
||||
t.Error("generic never rewrites")
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user