Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 unkinben
|
1cca9fef00 |
+47
-2
@@ -1,2 +1,47 @@
|
||||
bin/
|
||||
terraform/
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# Virtual environment
|
||||
.venv/
|
||||
venv/
|
||||
ENV/
|
||||
env/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
remotes.yaml
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# uv
|
||||
uv.lock
|
||||
|
||||
# Docker volumes
|
||||
minio_data/
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v5.0.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
- id: check-added-large-files
|
||||
- id: check-merge-conflict
|
||||
|
||||
- repo: https://github.com/dnephin/pre-commit-golang
|
||||
rev: v0.5.1
|
||||
hooks:
|
||||
- id: go-fmt
|
||||
- id: go-mod-tidy
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: go-vet
|
||||
name: go vet
|
||||
entry: go vet ./...
|
||||
language: system
|
||||
types: [go]
|
||||
pass_filenames: false
|
||||
@@ -1,9 +0,0 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: docker-build
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
repo: git.unkin.net/unkin/artifactapi
|
||||
dry_run: true
|
||||
@@ -1,34 +0,0 @@
|
||||
when:
|
||||
- event: tag
|
||||
ref: refs/tags/v*
|
||||
|
||||
steps:
|
||||
- name: docker-api
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi
|
||||
build_args:
|
||||
VERSION: ${CI_COMMIT_TAG}
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
|
||||
- name: docker-web
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi-ui
|
||||
dockerfile: ui/Dockerfile.ui
|
||||
context: ui
|
||||
build_args:
|
||||
BASE_PATH: /ui
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
@@ -1,17 +0,0 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: pre-commit
|
||||
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
||||
commands:
|
||||
- uvx pre-commit run --all-files
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
@@ -1,8 +0,0 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- go test -race -count=1 ./pkg/... ./internal/...
|
||||
+43
-11
@@ -1,21 +1,53 @@
|
||||
FROM golang:1.25-alpine AS builder
|
||||
# Use Alpine Linux as base image
|
||||
FROM python:3.11-alpine
|
||||
|
||||
RUN apk add --no-cache git
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
WORKDIR /build
|
||||
# Install system dependencies
|
||||
RUN apk add --no-cache \
|
||||
gcc \
|
||||
musl-dev \
|
||||
libffi-dev \
|
||||
postgresql-dev \
|
||||
curl \
|
||||
wget \
|
||||
tar
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
# Install uv
|
||||
ARG PACKAGE_VERSION=0.9.21
|
||||
RUN wget -O /app/uv-x86_64-unknown-linux-musl.tar.gz https://github.com/astral-sh/uv/releases/download/${PACKAGE_VERSION}/uv-x86_64-unknown-linux-musl.tar.gz && \
|
||||
tar xf /app/uv-x86_64-unknown-linux-musl.tar.gz -C /app && \
|
||||
mv /app/uv-x86_64-unknown-linux-musl/uv /usr/local/bin/uv && \
|
||||
rm -rf /app/uv-x86_64-unknown-linux-musl* && \
|
||||
chmod +x /usr/local/bin/uv && \
|
||||
uv --version
|
||||
|
||||
COPY . .
|
||||
# Copy CA bundle from host
|
||||
COPY ca-bundle.pem /app/ca-bundle.pem
|
||||
RUN chmod 644 /app/ca-bundle.pem
|
||||
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=${VERSION}" -o artifactapi ./cmd/artifactapi
|
||||
# Create non-root user first
|
||||
RUN adduser -D -s /bin/sh appuser && \
|
||||
chown -R appuser:appuser /app
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:nonroot
|
||||
# Copy dependency files and change ownership
|
||||
COPY --chown=appuser:appuser pyproject.toml uv.lock README.md ./
|
||||
|
||||
COPY --from=builder /build/artifactapi /usr/local/bin/artifactapi
|
||||
# Switch to appuser and install Python dependencies
|
||||
USER appuser
|
||||
RUN uv sync --frozen
|
||||
|
||||
# Copy application source
|
||||
COPY --chown=appuser:appuser src/ ./src/
|
||||
COPY --chown=appuser:appuser remotes.yaml ./
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8000
|
||||
|
||||
ENTRYPOINT ["artifactapi"]
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD curl -f http://localhost:8000/health || exit 1
|
||||
|
||||
# Run the application
|
||||
CMD ["uv", "run", "python", "-m", "src.artifactapi.main"]
|
||||
@@ -1,66 +1,49 @@
|
||||
.PHONY: build test lint fmt e2e docker docker-ui compose clean tidy check-go
|
||||
.PHONY: build install dev clean test lint format docker-build docker-up docker-down docker-logs docker-rebuild docker-clean docker-restart
|
||||
|
||||
BINARY := bin/artifactapi
|
||||
MODULE := git.unkin.net/unkin/artifactapi
|
||||
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0.0-dev")
|
||||
GO_VERSION_REQUIRED := 1.23
|
||||
GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/')
|
||||
build:
|
||||
docker build --no-cache -t artifactapi:latest .
|
||||
|
||||
check-go:
|
||||
@if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \
|
||||
echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \
|
||||
fi
|
||||
install: build
|
||||
|
||||
build: check-go tidy
|
||||
go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(BINARY) ./cmd/artifactapi
|
||||
docker-build: build
|
||||
|
||||
test: check-go
|
||||
go test -race -count=1 ./pkg/... ./internal/...
|
||||
|
||||
lint: check-go
|
||||
golangci-lint run ./...
|
||||
go vet ./...
|
||||
|
||||
fmt: check-go
|
||||
gofmt -w .
|
||||
goimports -w .
|
||||
|
||||
e2e: check-go
|
||||
TESTCONTAINERS_RYUK_DISABLED=true go test -tags=e2e -race -count=1 -timeout=5m ./e2e/...
|
||||
|
||||
docker:
|
||||
docker build -t artifactapi:$(VERSION) .
|
||||
|
||||
docker-ui:
|
||||
docker build -t artifactapi-ui:$(VERSION) -f ui/Dockerfile.ui ui/
|
||||
|
||||
compose:
|
||||
docker compose up -d
|
||||
dev: build
|
||||
uv sync --dev
|
||||
|
||||
clean:
|
||||
rm -rf bin/
|
||||
rm -rf .venv
|
||||
rm -rf build/
|
||||
rm -rf dist/
|
||||
rm -rf *.egg-info/
|
||||
|
||||
tidy:
|
||||
go mod tidy
|
||||
test:
|
||||
uv run pytest
|
||||
|
||||
# Bump helpers — reads the latest semver tag and creates the next one.
|
||||
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
||||
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
||||
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
||||
_MIN := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f2)
|
||||
_PAT := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f3)
|
||||
lint:
|
||||
uv run ruff check --fix .
|
||||
|
||||
patch:
|
||||
@NEW=v$(_MAJ).$(_MIN).$(shell expr $(_PAT) + 1); \
|
||||
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||
format:
|
||||
uv run ruff format .
|
||||
|
||||
minor:
|
||||
@NEW=v$(_MAJ).$(shell expr $(_MIN) + 1).0; \
|
||||
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||
run:
|
||||
uv venv --python 3.11 && \
|
||||
source .venv/bin/activate && \
|
||||
uv run python -m src.artifactapi.main
|
||||
|
||||
major:
|
||||
@NEW=v$(shell expr $(_MAJ) + 1).0.0; \
|
||||
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||
docker-up:
|
||||
docker-compose up --build --force-recreate -d
|
||||
|
||||
_tag:
|
||||
git push origin $(TAG)
|
||||
docker-down:
|
||||
docker-compose down
|
||||
|
||||
docker-logs:
|
||||
docker-compose logs -f
|
||||
|
||||
docker-rebuild:
|
||||
docker-compose build --no-cache
|
||||
|
||||
docker-clean:
|
||||
docker-compose down -v --remove-orphans
|
||||
docker system prune -f
|
||||
|
||||
docker-restart: docker-down docker-up
|
||||
|
||||
@@ -1,880 +0,0 @@
|
||||
# ArtifactAPI v3 — Go Rewrite Plan
|
||||
|
||||
## Context
|
||||
|
||||
ArtifactAPI is a production artifact proxy/cache serving ~42 remotes (Docker registries, Helm repos, RPM/Alpine repos, GitHub releases, PyPI, npm, Puppet Forge, Terraform registries, Go module proxies) across a Kubernetes cluster. The current Python (FastAPI) implementation works but has architectural debt: opaque hashed S3 paths, no UI for visibility, YAML config files that drift, no garbage collection, no access logging, and virtual repos limited to Helm only.
|
||||
|
||||
The v3 rewrite targets: a single Go binary (API + TUI), a separate React frontend (own Dockerfile), a Terraform provider (separate repo), content-addressable storage, and a cleaner data model that makes the cache inspectable and manageable.
|
||||
|
||||
**Repo**: Same repo (`git.unkin.net/unkin/artifactapi`), new branch.
|
||||
**Module**: `git.unkin.net/unkin/artifactapi`
|
||||
**Frontend**: React + Vite, separate Dockerfile, talks to API
|
||||
**Terraform provider**: Separate repo (`terraform-provider-artifactapi`)
|
||||
|
||||
---
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
┌───────────────────────────────────┐ ┌──────────────────────┐
|
||||
│ Go Binary (API + TUI) │ │ Frontend Container │
|
||||
│ │ │ │
|
||||
│ ┌──────────┐ ┌───────────────┐ │ │ React + Vite SPA │
|
||||
│ │ REST API │ │ Proxy Engine │ │◄───│ nginx / node serve │
|
||||
│ │ /api/v2 │ │ /api/v1/... │ │ │ Dockerfile.ui │
|
||||
│ │ │ │ /v2/... (OCI) │ │ └──────────────────────┘
|
||||
│ └────┬─────┘ └──────┬────────┘ │
|
||||
│ │ │ │ ┌──────────────────────┐
|
||||
│ ┌────┴───────────────┴────────┐ │ │ Terraform Provider │
|
||||
│ │ Data Layer │ │◄───│ (separate repo) │
|
||||
│ │ PostgreSQL · Redis · S3 │ │ └──────────────────────┘
|
||||
│ └─────────────────────────────┘ │
|
||||
│ │ ┌──────────────────────┐
|
||||
│ ┌─────────────────────────────┐ │ │ TUI (subcommand) │
|
||||
│ │ artifactapi tui │──│───►│ artifactapi tui │
|
||||
│ └─────────────────────────────┘ │ │ --endpoint <url> │
|
||||
└───────────────────────────────────┘ └──────────────────────┘
|
||||
```
|
||||
|
||||
Three independent deployment units:
|
||||
1. **Go binary** — API server + TUI subcommand (single `Dockerfile`)
|
||||
2. **React frontend** — SPA served by nginx (`Dockerfile.ui`), talks to `/api/v2`
|
||||
3. **Terraform provider** — separate repo, calls `/api/v2` CRUD
|
||||
|
||||
---
|
||||
|
||||
## Project Structure (Modular)
|
||||
|
||||
```
|
||||
artifactapi/
|
||||
├── cmd/
|
||||
│ └── artifactapi/
|
||||
│ └── main.go # entrypoint: serve / tui subcommands
|
||||
│
|
||||
├── pkg/ # PUBLIC — importable by terraform provider, CLI tools
|
||||
│ ├── models/ # shared domain types
|
||||
│ │ ├── remote.go # Remote, RemoteConfig, PackageType enum
|
||||
│ │ ├── virtual.go # Virtual, VirtualConfig
|
||||
│ │ ├── artifact.go # Artifact, Blob, AccessLogEntry
|
||||
│ │ ├── local.go # LocalFile, LocalRepo
|
||||
│ │ └── stats.go # RemoteStats, OverviewStats
|
||||
│ └── client/ # typed Go API client (used by TUI + Terraform provider)
|
||||
│ ├── client.go # Client struct, base HTTP
|
||||
│ ├── remotes.go # remote CRUD methods
|
||||
│ ├── virtuals.go # virtual CRUD methods
|
||||
│ ├── objects.go # object browse/evict methods
|
||||
│ └── stats.go # stats methods
|
||||
│
|
||||
├── internal/ # PRIVATE — server internals
|
||||
│ ├── server/
|
||||
│ │ ├── server.go # HTTP server setup, router
|
||||
│ │ └── middleware.go # logging, recovery, request-id, access logging
|
||||
│ │
|
||||
│ ├── api/
|
||||
│ │ ├── v1/ # proxy endpoints (v1 compat)
|
||||
│ │ │ ├── proxy.go # GET /api/v1/remote/{name}/{path}
|
||||
│ │ │ ├── docker.go # /v2/{name}/{path}
|
||||
│ │ │ ├── virtual.go # GET /api/v1/virtual/{name}/{path}
|
||||
│ │ │ └── local.go # CRUD /api/v1/local/{name}/{path}
|
||||
│ │ └── v2/ # management API
|
||||
│ │ ├── remotes.go # CRUD + stats
|
||||
│ │ ├── virtuals.go # CRUD
|
||||
│ │ ├── objects.go # browse/evict cached objects
|
||||
│ │ ├── stats.go # overview, top-remotes
|
||||
│ │ ├── events.go # SSE stream
|
||||
│ │ └── health.go # health, metrics
|
||||
│ │
|
||||
│ ├── provider/ # package-type providers (registry protocol handlers)
|
||||
│ │ ├── provider.go # Provider interface + registry
|
||||
│ │ ├── generic/
|
||||
│ │ │ ├── generic.go
|
||||
│ │ │ └── generic_test.go
|
||||
│ │ ├── docker/
|
||||
│ │ │ ├── docker.go # OCI Distribution v2 via go-containerregistry
|
||||
│ │ │ ├── auth.go # Bearer token fetch + cache
|
||||
│ │ │ └── docker_test.go
|
||||
│ │ ├── helm/
|
||||
│ │ │ ├── helm.go # index rewriting via helm.sh/helm/v3/pkg/repo
|
||||
│ │ │ ├── merger.go # virtual index merge
|
||||
│ │ │ └── helm_test.go
|
||||
│ │ ├── pypi/
|
||||
│ │ │ ├── pypi.go # simple index HTML rewriting
|
||||
│ │ │ ├── merger.go # virtual simple index merge
|
||||
│ │ │ └── pypi_test.go
|
||||
│ │ ├── npm/
|
||||
│ │ │ ├── npm.go # metadata JSON rewriting
|
||||
│ │ │ └── npm_test.go
|
||||
│ │ ├── rpm/
|
||||
│ │ │ ├── rpm.go # repodata patterns
|
||||
│ │ │ └── rpm_test.go
|
||||
│ │ ├── alpine/
|
||||
│ │ │ ├── alpine.go # APKINDEX patterns
|
||||
│ │ │ └── alpine_test.go
|
||||
│ │ ├── puppet/
|
||||
│ │ │ ├── puppet.go # file_uri JSON rewriting
|
||||
│ │ │ └── puppet_test.go
|
||||
│ │ ├── terraform/
|
||||
│ │ │ ├── terraform.go # registry protocol, download URL rewriting
|
||||
│ │ │ └── terraform_test.go
|
||||
│ │ └── goproxy/
|
||||
│ │ ├── goproxy.go # Go module proxy protocol (GOPROXY)
|
||||
│ │ └── goproxy_test.go
|
||||
│ │
|
||||
│ ├── proxy/
|
||||
│ │ ├── engine.go # core fetch-or-cache logic
|
||||
│ │ ├── engine_test.go
|
||||
│ │ ├── classifier.go # immutable vs mutable classification
|
||||
│ │ ├── classifier_test.go
|
||||
│ │ ├── revalidator.go # conditional HEAD requests (ETag/Last-Modified)
|
||||
│ │ └── circuit.go # per-remote circuit breaker
|
||||
│ │
|
||||
│ ├── storage/
|
||||
│ │ ├── s3.go # S3 client (minio-go — works with MinIO, Ceph, AWS)
|
||||
│ │ ├── s3_test.go
|
||||
│ │ ├── cas.go # content-addressable store logic
|
||||
│ │ └── cas_test.go
|
||||
│ │
|
||||
│ ├── cache/
|
||||
│ │ ├── redis.go # TTL management, fetch locks
|
||||
│ │ ├── redis_test.go
|
||||
│ │ └── lock.go # distributed lock abstraction
|
||||
│ │
|
||||
│ ├── database/
|
||||
│ │ ├── postgres.go # connection pool, migration runner
|
||||
│ │ ├── queries/ # SQL query files or sqlc-generated code
|
||||
│ │ │ ├── remotes.sql.go
|
||||
│ │ │ ├── virtuals.sql.go
|
||||
│ │ │ ├── artifacts.sql.go
|
||||
│ │ │ └── access_log.sql.go
|
||||
│ │ └── migrations/ # golang-migrate SQL files
|
||||
│ │ ├── 001_initial.up.sql
|
||||
│ │ └── 001_initial.down.sql
|
||||
│ │
|
||||
│ ├── metrics/
|
||||
│ │ └── prometheus.go # counters, gauges, histograms
|
||||
│ │
|
||||
│ ├── gc/
|
||||
│ │ ├── gc.go # background garbage collection goroutine
|
||||
│ │ └── gc_test.go
|
||||
│ │
|
||||
│ ├── tui/
|
||||
│ │ ├── app.go # Bubble Tea main model
|
||||
│ │ ├── views/
|
||||
│ │ │ ├── dashboard.go
|
||||
│ │ │ ├── remotes.go
|
||||
│ │ │ ├── objects.go
|
||||
│ │ │ └── virtuals.go
|
||||
│ │ └── components/
|
||||
│ │ ├── table.go
|
||||
│ │ └── statusbar.go
|
||||
│ │
|
||||
│ └── config/
|
||||
│ └── env.go # environment variable parsing + validation
|
||||
│
|
||||
├── ui/ # React frontend — SEPARATE DOCKERFILE
|
||||
│ ├── src/
|
||||
│ │ ├── App.tsx
|
||||
│ │ ├── pages/
|
||||
│ │ │ ├── Dashboard.tsx
|
||||
│ │ │ ├── Remotes.tsx
|
||||
│ │ │ ├── RemoteDetail.tsx
|
||||
│ │ │ ├── Virtuals.tsx
|
||||
│ │ │ └── Objects.tsx
|
||||
│ │ ├── components/
|
||||
│ │ │ ├── RemoteTable.tsx
|
||||
│ │ │ ├── ObjectBrowser.tsx
|
||||
│ │ │ ├── StatsCard.tsx
|
||||
│ │ │ └── EventFeed.tsx
|
||||
│ │ └── api/
|
||||
│ │ └── client.ts # typed API client
|
||||
│ ├── package.json
|
||||
│ ├── vite.config.ts
|
||||
│ ├── tsconfig.json
|
||||
│ ├── Dockerfile.ui # multi-stage: node build → nginx
|
||||
│ └── nginx.conf # proxy /api/* to backend, serve SPA
|
||||
│
|
||||
├── e2e/ # end-to-end integration tests
|
||||
│ ├── e2e_test.go # TestMain spins up docker-compose stack
|
||||
│ ├── proxy_test.go # proxy through real remotes
|
||||
│ ├── docker_test.go # Docker v2 protocol e2e
|
||||
│ ├── management_test.go # v2 API CRUD
|
||||
│ ├── virtual_test.go # virtual repo merge e2e
|
||||
│ └── docker-compose.e2e.yml # postgres + redis + minio for tests
|
||||
│
|
||||
├── go.mod
|
||||
├── go.sum
|
||||
├── Makefile
|
||||
├── Dockerfile # Go binary (API server + TUI)
|
||||
├── Dockerfile.ui # symlink or copy → ui/Dockerfile.ui
|
||||
└── docker-compose.yml
|
||||
```
|
||||
|
||||
### Key Modularisation Decisions
|
||||
|
||||
- **`pkg/models/`** — Shared domain types importable by the Terraform provider and any external tooling. No dependencies on internal packages
|
||||
- **`pkg/client/`** — Typed Go API client used by both the TUI and the Terraform provider. Depends only on `pkg/models/` and stdlib
|
||||
- **`internal/provider/`** — Each package type is its own subpackage with isolated tests. A provider registry maps `PackageType → Provider`
|
||||
- **`internal/database/queries/`** — Use [sqlc](https://sqlc.dev/) to generate type-safe query functions from SQL, or hand-written query files
|
||||
- **`e2e/`** — Separate test binary that spins up a real docker-compose stack
|
||||
|
||||
---
|
||||
|
||||
## Go Ecosystem Libraries
|
||||
|
||||
Prefer existing, maintained Go modules over writing protocol handlers from scratch.
|
||||
|
||||
### Package-Type Libraries
|
||||
|
||||
| Package Type | Go Module | What It Gives Us |
|
||||
|---|---|---|
|
||||
| **Docker/OCI** | `github.com/google/go-containerregistry` | Full Registry v2/OCI client: manifest parsing, auth challenges, blob operations. `pkg/registry` can implement a v2 server. Reference: `github.com/regclient/regclient` |
|
||||
| **Helm** | `helm.sh/helm/v3/pkg/repo` | Parse/generate `index.yaml`, `IndexFile`/`ChartVersion` types, URL entries. Used directly for merge |
|
||||
| **Terraform** | `github.com/hashicorp/terraform-registry-address` | Provider/module address parsing, `ForRegistryProtocol()` URL generation. Protocol spec: provider registry protocol v1 |
|
||||
| **Go Modules** | `github.com/goproxy/goproxy` | Minimalist GOPROXY protocol handler, implements full spec as `http.Handler`. Handles `/@v/list`, `/@v/{v}.info`, `/@v/{v}.mod`, `/@v/{v}.zip`, `/@latest` |
|
||||
| **RPM** | `rs3.io/go/rpm/repomd` | Parse `repomd.xml`, `primary.xml` with proper XML namespace handling |
|
||||
| **Alpine** | `gitlab.alpinelinux.org/alpine/go` | Official Alpine library: parse APKINDEX, `.apk` files |
|
||||
| **PyPI** | stdlib `golang.org/x/net/html` | No dedicated Go PyPI library exists. Parse simple index HTML with `x/net/html`, extract `<a>` tags. Minimal — the rewriting is just href replacement |
|
||||
| **npm** | stdlib `encoding/json` | npm metadata is JSON — parse with stdlib, rewrite `dist.tarball` URLs. No special library needed |
|
||||
| **Puppet Forge** | stdlib `encoding/json` | Forge API is JSON — parse and rewrite `file_uri` fields. Community lib `github.com/johnmccabe/go-puppetforge` exists but is thin; stdlib suffices |
|
||||
|
||||
### Infrastructure Libraries
|
||||
|
||||
| Purpose | Go Module | Why This One |
|
||||
|---|---|---|
|
||||
| **HTTP router** | `github.com/go-chi/chi/v5` | Lightweight, stdlib `http.Handler` compatible, middleware chain |
|
||||
| **PostgreSQL** | `github.com/jackc/pgx/v5` | Pure Go, connection pooling, COPY support, prepared statements |
|
||||
| **SQL generation** | `github.com/sqlc-dev/sqlc` | Generate type-safe Go from SQL queries — no ORM, no reflection |
|
||||
| **Redis** | `github.com/redis/go-redis/v9` | Full Redis client, pipelining, pub/sub |
|
||||
| **S3 (MinIO/Ceph/AWS)** | `github.com/minio/minio-go/v7` | Native S3-compatible client. Works with MinIO, Ceph RGW, AWS S3, any S3-compatible backend out of the box. Lighter than aws-sdk-go-v2, purpose-built for S3 compat |
|
||||
| **DB migrations** | `github.com/golang-migrate/migrate/v4` | SQL file-based migrations, CLI + library |
|
||||
| **Prometheus** | `github.com/prometheus/client_golang` | Counters, gauges, histograms |
|
||||
| **TUI** | `github.com/charmbracelet/bubbletea` | Elm-architecture TUI framework |
|
||||
| **TUI styling** | `github.com/charmbracelet/lipgloss` | Terminal styling |
|
||||
| **TUI components** | `github.com/charmbracelet/bubbles` | Table, text input, spinner, etc. |
|
||||
| **Structured logging** | `log/slog` (stdlib) | Go 1.21+ structured logging, zero dependencies |
|
||||
| **Testing** | `github.com/stretchr/testify` | Assertions + require for unit tests |
|
||||
| **Test containers** | `github.com/testcontainers/testcontainers-go` | Spin up Postgres/Redis/MinIO in e2e tests |
|
||||
|
||||
### S3 Client: Multi-Backend Support
|
||||
|
||||
Using `minio-go/v7` as the S3 client because it natively supports:
|
||||
- **MinIO** — primary development/production target
|
||||
- **Ceph RGW** — S3-compatible via endpoint config
|
||||
- **AWS S3** — via region + credential config
|
||||
- **Any S3-compatible** — GCS (interop mode), Wasabi, DigitalOcean Spaces, etc.
|
||||
|
||||
No abstraction layer needed — `minio-go` handles endpoint differences internally. Config:
|
||||
```go
|
||||
client, _ := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: useTLS,
|
||||
Region: region, // optional, for AWS
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Data Layer
|
||||
|
||||
### PostgreSQL Schema
|
||||
|
||||
```sql
|
||||
-- Remotes: managed exclusively by Terraform
|
||||
CREATE TABLE remotes (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL, -- generic, docker, helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy
|
||||
base_url TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
immutable_ttl INTEGER DEFAULT 0,
|
||||
mutable_ttl INTEGER DEFAULT 3600,
|
||||
check_mutable BOOLEAN DEFAULT TRUE,
|
||||
immutable_patterns TEXT[] DEFAULT '{}', -- user-defined immutable patterns
|
||||
mutable_patterns TEXT[] DEFAULT '{}', -- user-defined mutable patterns (merged with provider built-ins)
|
||||
allowlist TEXT[] DEFAULT '{}', -- if empty, allow all paths; if non-empty, only matching paths proxied
|
||||
blocklist TEXT[] DEFAULT '{}', -- always denied, checked before allowlist
|
||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
||||
ban_tags TEXT[] DEFAULT '{}',
|
||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
||||
quarantine_days INTEGER DEFAULT 3,
|
||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
||||
releases_remote TEXT DEFAULT '', -- terraform type: name of CDN remote for download URL rewriting
|
||||
managed_by TEXT DEFAULT '', -- 'terraform' or empty
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Virtual repositories
|
||||
CREATE TABLE virtuals (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
members TEXT[] NOT NULL,
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Content-addressable blob storage tracking
|
||||
CREATE TABLE blobs (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
s3_key TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
content_type TEXT DEFAULT 'application/octet-stream',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Artifact metadata: maps (remote, path) → content blob
|
||||
CREATE TABLE artifacts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
upstream_etag TEXT DEFAULT '',
|
||||
upstream_last_modified TIMESTAMPTZ,
|
||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
fetch_count BIGINT DEFAULT 1,
|
||||
access_count BIGINT DEFAULT 1,
|
||||
UNIQUE(remote_name, path)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_artifacts_remote ON artifacts(remote_name);
|
||||
CREATE INDEX idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
||||
|
||||
-- Local file uploads
|
||||
CREATE TABLE local_files (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
-- Access log (append-only, powers dashboards)
|
||||
CREATE TABLE access_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
cache_hit BOOLEAN NOT NULL,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
upstream_ms INTEGER DEFAULT 0,
|
||||
client_ip TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
```
|
||||
|
||||
### Redis Usage (Ephemeral Only)
|
||||
|
||||
| Key pattern | Type | TTL | Purpose |
|
||||
|---|---|---|---|
|
||||
| `ttl:{remote}:{path}` | STRING | remote's immutable/mutable TTL | Artifact freshness — existence = still fresh |
|
||||
| `lock:{remote}:{path}` | STRING (NX) | 30s | Fetch lock — prevents thundering herd |
|
||||
| `etag:{remote}:{path}` | STRING | same as TTL key | Cached ETag for conditional revalidation |
|
||||
| `circuit:{remote}` | STRING | configurable | Circuit breaker — consecutive failure count |
|
||||
|
||||
Losing Redis = all TTLs expire = next request re-validates upstream. No data loss.
|
||||
|
||||
### S3 Layout (Content-Addressable)
|
||||
|
||||
```
|
||||
artifacts-bucket/
|
||||
├── blobs/sha256/{content_hash} # immutable CAS blobs
|
||||
├── indexes/{remote}/{path} # mutable index files (helm, pypi, rpm, etc.)
|
||||
├── indexes/{virtual}/{path} # merged virtual indexes
|
||||
└── local/{repo}/{path} # user uploads (CAS-backed via blobs table)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Terraform Remote Type (New in v2)
|
||||
|
||||
The `terraform` package type proxies the Terraform Provider Registry Protocol:
|
||||
|
||||
- **URL construction**: prepends `/v1/providers/` to request paths
|
||||
- **Built-in mutable pattern**: `[^/]+/[^/]+/versions$` (version listings change over time)
|
||||
- **Built-in immutable pattern**: `[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$` (per-version download info is fixed)
|
||||
- **Response rewriting**: download info JSON — rewrites `download_url`, `shasums_url`, `shasums_signature_url` to route through a companion `releases_remote` (e.g., `hashicorp-releases` generic remote)
|
||||
- **Config**: requires `releases_remote` field pointing to the CDN remote that serves the actual binaries
|
||||
|
||||
Uses `github.com/hashicorp/terraform-registry-address` for address parsing and protocol-compliant URL generation.
|
||||
|
||||
---
|
||||
|
||||
## Go Module Proxy Remote Type (New)
|
||||
|
||||
The `goproxy` package type implements the GOPROXY protocol (Go module proxy):
|
||||
|
||||
| Endpoint | Mutability | Description |
|
||||
|---|---|---|
|
||||
| `{module}/@v/list` | Mutable | Plain text list of known versions |
|
||||
| `{module}/@latest` | Mutable | JSON metadata for latest version |
|
||||
| `{module}/@v/{version}.info` | Immutable | JSON version metadata (`Version`, `Time`) |
|
||||
| `{module}/@v/{version}.mod` | Immutable | `go.mod` file for that version |
|
||||
| `{module}/@v/{version}.zip` | Immutable | Source archive for that version |
|
||||
|
||||
- **No URL rewriting needed** — responses are self-contained (no embedded URLs)
|
||||
- **Config**: `base_url` points to upstream proxy (e.g., `https://proxy.golang.org`)
|
||||
- **Client usage**: set `GOPROXY=https://artifactapi.example.com/api/v1/remote/goproxy`
|
||||
- Uses `github.com/goproxy/goproxy` for protocol handling
|
||||
|
||||
---
|
||||
|
||||
## Allowlist / Blocklist / Automatic Mutable Patterns
|
||||
|
||||
### Access Control (Per-Remote)
|
||||
|
||||
| Field | Default | Behavior |
|
||||
|---|---|---|
|
||||
| `blocklist` | `[]` (empty) | If a path matches any blocklist pattern → **403 Forbidden**. Checked first |
|
||||
| `allowlist` | `[]` (empty) | If empty → **allow everything**. If non-empty → only matching paths are proxied; everything else → **403** |
|
||||
|
||||
Evaluation order: blocklist → allowlist → proxy. No allowlist + no blocklist = open proxy (default).
|
||||
|
||||
### Automatic Mutable Patterns (Per-Provider Built-ins)
|
||||
|
||||
Each provider declares built-in mutable patterns that are **always merged** with user-defined `mutable_patterns`. Users never need to configure these — the provider knows which paths change over time.
|
||||
|
||||
| Provider | Built-in Mutable Patterns | Rationale |
|
||||
|---|---|---|
|
||||
| **generic** | *(none)* | No convention for what's mutable |
|
||||
| **docker** | `/manifests/(?!sha256:)[^/]+$`, `/tags/list$` | Tag manifests change; digest manifests don't |
|
||||
| **helm** | `index\.yaml$` | Chart index changes when new charts are published |
|
||||
| **pypi** | `simple/` | Package index pages change with new releases |
|
||||
| **npm** | `^[^/]+$` (package metadata, not `.tgz`) | Package metadata changes; tarballs are immutable |
|
||||
| **rpm** | `repomd\.xml$`, `repodata/.*`, `Packages\.gz$` | Repo metadata rebuilt on every publish |
|
||||
| **alpine** | `APKINDEX\.tar\.gz$` | Package index rebuilt on every publish |
|
||||
| **puppet** | `^v3/modules/`, `^v3/releases` | Module metadata changes with new releases |
|
||||
| **terraform** | `[^/]+/[^/]+/versions$` | Provider version listings grow over time |
|
||||
| **goproxy** | `@v/list$`, `@latest$` | Version list and latest pointer change |
|
||||
|
||||
These are returned by `Provider.BuiltinMutablePatterns()` and merged at classification time:
|
||||
```
|
||||
effective_mutable = provider.BuiltinMutablePatterns() ∪ remote.mutable_patterns
|
||||
```
|
||||
|
||||
If a path matches `effective_mutable` → use `mutable_ttl`. If it matches `remote.immutable_patterns` → use `immutable_ttl`. Immutable patterns take precedence over mutable when both match.
|
||||
|
||||
---
|
||||
|
||||
## API Design
|
||||
|
||||
### v1 Proxy Endpoints (Backwards Compatible)
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| `GET` | `/api/v1/remote/{name}/{path}` | Proxy/cache artifact |
|
||||
| `GET` | `/api/v1/virtual/{name}/{path}` | Virtual repo proxy |
|
||||
| `GET/HEAD` | `/v2/{name}/{path}` | Docker Registry v2 |
|
||||
| `GET` | `/v2/` | Docker v2 ping |
|
||||
| `GET/PUT/HEAD/DELETE` | `/api/v1/local/{name}/{path}` | Local repo CRUD |
|
||||
|
||||
### v2 Management API (New)
|
||||
|
||||
```
|
||||
GET /api/v2/remotes → [{name, package_type, base_url, description, stats}]
|
||||
GET /api/v2/remotes/{name} → {full config + stats + health}
|
||||
POST /api/v2/remotes → create remote (Terraform provider)
|
||||
PUT /api/v2/remotes/{name} → update remote (Terraform provider)
|
||||
DELETE /api/v2/remotes/{name} → delete remote — cascades artifacts, GC cleans S3
|
||||
|
||||
GET /api/v2/virtuals → [{name, package_type, members, stats}]
|
||||
GET /api/v2/virtuals/{name} → {full config + member details}
|
||||
POST /api/v2/virtuals → create virtual
|
||||
PUT /api/v2/virtuals/{name} → update virtual
|
||||
DELETE /api/v2/virtuals/{name} → delete virtual
|
||||
|
||||
GET /api/v2/remotes/{name}/objects → paginated objects
|
||||
?q=pattern&sort=size|accessed|age&page=1&per_page=50
|
||||
DELETE /api/v2/remotes/{name}/objects/{path} → evict specific cached object
|
||||
DELETE /api/v2/remotes/{name}/cache → flush cache
|
||||
?type=all|indexes|blobs
|
||||
|
||||
GET /api/v2/stats → overview stats
|
||||
GET /api/v2/stats/top-remotes → top remotes by size/requests/hit-rate
|
||||
|
||||
GET /api/v2/health → {status, postgres, redis, s3, uptime}
|
||||
GET /metrics → Prometheus format
|
||||
GET /api/v2/events → SSE stream
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Proxy Engine
|
||||
|
||||
### Request Flow
|
||||
|
||||
```
|
||||
Client Request
|
||||
│
|
||||
▼
|
||||
Classify (immutable/mutable/denied)
|
||||
│
|
||||
├── blocklist match → 403
|
||||
├── allowlist non-empty + no match → 403
|
||||
│
|
||||
▼
|
||||
Check Redis TTL key
|
||||
│
|
||||
├── exists (fresh) → serve from S3, log access
|
||||
│
|
||||
├── missing (expired or uncached)
|
||||
│ │
|
||||
│ ▼
|
||||
│ Acquire fetch lock (Redis SETNX, 30s TTL)
|
||||
│ │
|
||||
│ ├── lock acquired
|
||||
│ │ ├── mutable + check_mutable + have ETag → HEAD upstream
|
||||
│ │ │ ├── 304 → refresh TTL, serve from S3
|
||||
│ │ │ └── changed → full fetch
|
||||
│ │ └── full fetch from upstream
|
||||
│ │ → provider.RewriteResponse() if needed
|
||||
│ │ → CAS store (hash → check blobs → upload if new)
|
||||
│ │ → upsert artifact in Postgres
|
||||
│ │ → set Redis TTL + release lock
|
||||
│ │ → on upstream error + stale_on_error → refresh TTL, serve stale
|
||||
│ │
|
||||
│ └── lock not acquired → poll S3 briefly, serve if another pod fetched it
|
||||
│
|
||||
▼
|
||||
Stream response from S3, log access
|
||||
```
|
||||
|
||||
### Circuit Breaker
|
||||
|
||||
Per-remote, tracked in Redis. Closed → Open (after N failures) → Half-open (after cooldown). Exposed via `GET /api/v2/remotes/{name}` health field.
|
||||
|
||||
### Content-Addressable Storage
|
||||
|
||||
1. Stream upstream → temp file, compute SHA256 inline
|
||||
2. Check `blobs` table for hash
|
||||
3. Exists → skip S3 upload, upsert `artifacts` row only
|
||||
4. New → upload to `blobs/sha256/{hash}`, insert both rows
|
||||
|
||||
### Garbage Collection
|
||||
|
||||
Background goroutine (configurable interval, default 1h):
|
||||
1. Orphaned blobs: delete S3 objects whose `content_hash` has no referencing `artifacts` or `local_files` rows
|
||||
2. Cold artifacts: optional per-remote, delete artifacts not accessed in N days
|
||||
3. Remote deletion: `ON DELETE CASCADE` handles Postgres; GC sweeps orphaned blobs
|
||||
|
||||
---
|
||||
|
||||
## Package Providers
|
||||
|
||||
### Provider Interface
|
||||
|
||||
```go
|
||||
type Provider interface {
|
||||
Type() models.PackageType
|
||||
BuiltinMutablePatterns() []*regexp.Regexp
|
||||
BuiltinImmutablePatterns() []*regexp.Regexp
|
||||
ContentType(path string) string
|
||||
UpstreamURL(remote models.Remote, path string) string
|
||||
RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error)
|
||||
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
```
|
||||
|
||||
### Provider Registry
|
||||
|
||||
```go
|
||||
var registry = map[models.PackageType]Provider{
|
||||
models.PackageGeneric: &generic.Provider{},
|
||||
models.PackageDocker: &docker.Provider{},
|
||||
models.PackageHelm: &helm.Provider{},
|
||||
models.PackagePyPI: &pypi.Provider{},
|
||||
models.PackageNPM: &npm.Provider{},
|
||||
models.PackageRPM: &rpm.Provider{},
|
||||
models.PackageAlpine: &alpine.Provider{},
|
||||
models.PackagePuppet: &puppet.Provider{},
|
||||
models.PackageTerraform: &terraform.Provider{},
|
||||
models.PackageGoProxy: &goproxy.Provider{},
|
||||
}
|
||||
|
||||
func Get(t models.PackageType) (Provider, error) { ... }
|
||||
```
|
||||
|
||||
Each provider lives in its own subpackage under `internal/provider/` with its own `_test.go`.
|
||||
|
||||
---
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
Every package gets `_test.go` files alongside the source. Run with `go test ./...`.
|
||||
|
||||
| Package | What's Tested |
|
||||
|---|---|
|
||||
| `internal/provider/docker/` | Auth token parsing/caching, manifest classification, tag banning, URL construction, blob key generation |
|
||||
| `internal/provider/helm/` | `index.yaml` parsing (using `helm.sh/helm/v3/pkg/repo`), URL rewriting, index merging |
|
||||
| `internal/provider/pypi/` | Simple index HTML parsing, URL rewriting, index merging |
|
||||
| `internal/provider/npm/` | Metadata JSON rewriting (`dist.tarball` URLs) |
|
||||
| `internal/provider/terraform/` | Registry URL construction, download info JSON rewriting, `releases_remote` URL extraction |
|
||||
| `internal/provider/rpm/` | Mutable pattern matching (repodata) |
|
||||
| `internal/provider/alpine/` | Mutable pattern matching (APKINDEX) |
|
||||
| `internal/provider/puppet/` | `file_uri` JSON rewriting |
|
||||
| `internal/proxy/` | Classifier (immutable vs mutable vs denied), circuit breaker state transitions, revalidator logic |
|
||||
| `internal/storage/` | CAS key generation, dedup detection, S3 operation mocking |
|
||||
| `internal/cache/` | Redis TTL set/check, fetch lock acquire/release/contention |
|
||||
| `internal/gc/` | Orphan detection queries, cold artifact selection |
|
||||
| `pkg/models/` | Model validation, PackageType enum |
|
||||
| `pkg/client/` | API client request/response serialization |
|
||||
|
||||
### End-to-End Tests
|
||||
|
||||
Located in `e2e/`. Use `testcontainers-go` to spin up real Postgres, Redis, and MinIO containers. The test binary starts the actual `artifactapi` server against these backends.
|
||||
|
||||
```go
|
||||
// e2e/e2e_test.go
|
||||
func TestMain(m *testing.M) {
|
||||
// Start postgres, redis, minio via testcontainers-go
|
||||
// Run migrations
|
||||
// Start artifactapi server on random port
|
||||
// Run tests
|
||||
// Tear down
|
||||
}
|
||||
```
|
||||
|
||||
| Test File | What's Tested |
|
||||
|---|---|
|
||||
| `e2e/proxy_test.go` | Proxy a real GitHub release through generic remote, verify S3 storage, verify Redis TTL, verify Postgres artifact row, verify cache hit on second request |
|
||||
| `e2e/docker_test.go` | Pull a real image manifest + blob through Docker v2 proxy, verify blob deduplication, tag banning |
|
||||
| `e2e/management_test.go` | Full CRUD lifecycle: create remote via v2 API, proxy through it, list objects, evict object, flush cache, delete remote |
|
||||
| `e2e/virtual_test.go` | Create two helm remotes + virtual, fetch merged index, verify priority ordering |
|
||||
| `e2e/terraform_test.go` | Proxy terraform provider version listing + download info, verify URL rewriting to releases_remote |
|
||||
| `e2e/goproxy_test.go` | Proxy Go module `@v/list`, `.info`, `.mod`, `.zip` through GOPROXY remote, verify mutable vs immutable classification |
|
||||
| `e2e/gc_test.go` | Create artifact, delete remote, trigger GC, verify S3 blob cleaned up |
|
||||
|
||||
### Code Quality
|
||||
|
||||
- `gofmt` / `goimports` — enforced in CI, run on save
|
||||
- `golangci-lint` — comprehensive linter suite (staticcheck, errcheck, govet, etc.)
|
||||
- `go vet ./...` — run in CI
|
||||
- Makefile targets: `make test`, `make lint`, `make e2e`, `make fmt`
|
||||
|
||||
---
|
||||
|
||||
## Terraform Provider (Separate Repo)
|
||||
|
||||
**Repo**: `terraform-provider-artifactapi`
|
||||
**Uses**: `pkg/client/` and `pkg/models/` from the main module
|
||||
|
||||
```hcl
|
||||
provider "artifactapi" {
|
||||
endpoint = "https://artifactapi.k8s.syd1.au.unkin.net"
|
||||
}
|
||||
|
||||
resource "artifactapi_remote" "terraform_registry" {
|
||||
name = "terraform-registry"
|
||||
package_type = "terraform"
|
||||
base_url = "https://registry.terraform.io"
|
||||
description = "Terraform provider registry"
|
||||
releases_remote = artifactapi_remote.hashicorp_releases.name
|
||||
|
||||
immutable_patterns = [
|
||||
"[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$",
|
||||
]
|
||||
|
||||
cache {
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
}
|
||||
}
|
||||
|
||||
resource "artifactapi_remote" "hashicorp_releases" {
|
||||
name = "hashicorp-releases"
|
||||
package_type = "generic"
|
||||
base_url = "https://releases.hashicorp.com"
|
||||
|
||||
immutable_patterns = [
|
||||
".*\\.zip$",
|
||||
".*SHA256SUMS(\\.sig)?$",
|
||||
]
|
||||
|
||||
cache {
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 0
|
||||
}
|
||||
}
|
||||
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
description = "All helm repos merged"
|
||||
members = [
|
||||
artifactapi_remote.jetstack.name,
|
||||
artifactapi_remote.hashicorp_helm.name,
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Web UI (React + Vite — Separate Container)
|
||||
|
||||
### Deployment
|
||||
|
||||
Separate `Dockerfile.ui`: multi-stage build (node → nginx). Served as its own container/pod. nginx proxies `/api/*` to the Go backend.
|
||||
|
||||
### Pages
|
||||
|
||||
| Route | Content |
|
||||
|---|---|
|
||||
| `/` | Dashboard: total objects, storage used, dedup savings, bandwidth saved, top remotes chart, live SSE event feed, health indicators |
|
||||
| `/remotes` | Remote table: name, type, description, object count, size, hit rate, health. Filter by type, sort any column |
|
||||
| `/remotes/:name` | Config (read-only, "Managed by Terraform" badge), stats, object browser with search/sort/evict, flush actions |
|
||||
| `/virtuals` | Virtual table: name, type, members, merged object count |
|
||||
| `/virtuals/:name` | Member list with individual stats |
|
||||
|
||||
All config is read-only — managed by Terraform.
|
||||
|
||||
---
|
||||
|
||||
## TUI (Bubble Tea — Subcommand)
|
||||
|
||||
`artifactapi tui --endpoint http://localhost:8000` or via `ARTIFACTAPI_ENDPOINT` env.
|
||||
|
||||
Uses `pkg/client/` for all API calls (same client as Terraform provider).
|
||||
|
||||
| View | Key bindings |
|
||||
|---|---|
|
||||
| Dashboard | summary stats, top remotes |
|
||||
| Remotes list | `j`/`k` navigate, `/` filter, `Enter` detail |
|
||||
| Remote detail | config + stats, `Enter` → object browser |
|
||||
| Object browser | `/` search, `d` evict, `f` flush |
|
||||
| Virtuals | `j`/`k`, `Enter` detail |
|
||||
|
||||
---
|
||||
|
||||
## Improvements Over v2
|
||||
|
||||
| Area | v2 (Python) | v3 (Go) |
|
||||
|---|---|---|
|
||||
| S3 paths | Hashed, opaque | Content-addressed CAS |
|
||||
| Config | YAML files, mtime reload | Terraform via API |
|
||||
| Package types | 8 types | 10 types (+ terraform, goproxy) |
|
||||
| Virtual repos | Helm only | Helm + PyPI, extensible |
|
||||
| Deduplication | Docker blobs only | All types via CAS |
|
||||
| Revalidation | Opt-in flag | Default for all mutable |
|
||||
| Access logging | None | Per-artifact in Postgres |
|
||||
| GC | None | Background goroutine |
|
||||
| Upstream health | Per-request | Circuit breaker |
|
||||
| S3 backends | MinIO only | MinIO, Ceph, AWS (minio-go) |
|
||||
| UI | None | Web dashboard + TUI |
|
||||
| Binary | Python + venv | Static Go binary |
|
||||
| Frontend | N/A | Separate container (React) |
|
||||
| Testing | Mocked unit tests | Unit + e2e with real backends |
|
||||
|
||||
---
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Core Engine + Models
|
||||
- Go module, Makefile (`make build test lint fmt e2e`), Dockerfile, docker-compose
|
||||
- `pkg/models/` — all domain types
|
||||
- PostgreSQL schema + migrations
|
||||
- S3 storage layer with CAS (`minio-go/v7`)
|
||||
- Redis cache layer (TTL, locks)
|
||||
- Proxy engine: fetch-or-cache, classifier, revalidator
|
||||
- Generic + Docker providers (most complexity: OCI auth, CAS, tag banning)
|
||||
- Health + metrics endpoints
|
||||
- Unit tests for all packages
|
||||
- **Milestone**: proxy Docker + generic, cache in S3, track in Postgres
|
||||
|
||||
### Phase 2: All Providers
|
||||
- Helm (using `helm.sh/helm/v3/pkg/repo`)
|
||||
- PyPI (stdlib `x/net/html`)
|
||||
- npm (stdlib `encoding/json`)
|
||||
- RPM (using `rs3.io/go/rpm/repomd`)
|
||||
- Alpine (using `gitlab.alpinelinux.org/alpine/go`)
|
||||
- Puppet Forge (stdlib `encoding/json`)
|
||||
- Terraform (using `hashicorp/terraform-registry-address`)
|
||||
- Go Modules / GOPROXY (using `github.com/goproxy/goproxy`)
|
||||
- Unit tests per provider
|
||||
- **Milestone**: feature parity with v2 + goproxy
|
||||
|
||||
### Phase 3: Management API + Virtual Repos + GC
|
||||
- `pkg/client/` — shared Go API client
|
||||
- v2 CRUD endpoints
|
||||
- Virtual repo engine: `IndexMerger` for Helm + PyPI
|
||||
- Circuit breaker
|
||||
- Access logging middleware
|
||||
- GC goroutine
|
||||
- **Milestone**: full API, virtuals, GC
|
||||
|
||||
### Phase 4: End-to-End Tests
|
||||
- `e2e/` test suite with `testcontainers-go`
|
||||
- Proxy, Docker, management, virtual, terraform, GC tests
|
||||
- CI pipeline: `make e2e`
|
||||
- **Milestone**: comprehensive e2e coverage
|
||||
|
||||
### Phase 5: Terraform Provider
|
||||
- Separate repo: `terraform-provider-artifactapi`
|
||||
- Imports `pkg/client/` and `pkg/models/`
|
||||
- `artifactapi_remote` + `artifactapi_virtual` resources + data sources
|
||||
- Import support
|
||||
- **Milestone**: manage all config via Terraform
|
||||
|
||||
### Phase 6: Web UI
|
||||
- React + Vite in `ui/`
|
||||
- `Dockerfile.ui` (multi-stage → nginx)
|
||||
- Dashboard, remotes, objects, virtuals pages
|
||||
- SSE event feed
|
||||
- **Milestone**: full web UI in separate container
|
||||
|
||||
### Phase 7: TUI
|
||||
- Bubble Tea in `internal/tui/`
|
||||
- Uses `pkg/client/`
|
||||
- Dashboard, remotes, objects, virtuals views
|
||||
- **Milestone**: TUI feature parity with web UI
|
||||
|
||||
### Phase 8: Migration + Cutover
|
||||
- Migration tool: v2 YAML → Terraform HCL + `terraform import` commands
|
||||
- S3 rehash script: `{remote}/{hash16}/{file}` → `blobs/sha256/{content_hash}`
|
||||
- Parallel run, response comparison
|
||||
- Cutover
|
||||
|
||||
---
|
||||
|
||||
## Makefile Targets
|
||||
|
||||
```makefile
|
||||
.PHONY: build test lint fmt e2e docker docker-ui
|
||||
|
||||
build: ## Build Go binary
|
||||
go build -o bin/artifactapi ./cmd/artifactapi
|
||||
|
||||
test: ## Run unit tests
|
||||
go test ./...
|
||||
|
||||
lint: ## Run golangci-lint + go vet
|
||||
golangci-lint run ./...
|
||||
go vet ./...
|
||||
|
||||
fmt: ## Format code (gofmt + goimports)
|
||||
gofmt -w .
|
||||
goimports -w .
|
||||
|
||||
e2e: ## Run end-to-end tests (requires Docker)
|
||||
go test -tags=e2e -count=1 -timeout=5m ./e2e/...
|
||||
|
||||
docker: ## Build API server Docker image
|
||||
docker build -t artifactapi .
|
||||
|
||||
docker-ui: ## Build frontend Docker image
|
||||
docker build -t artifactapi-ui -f ui/Dockerfile.ui ui/
|
||||
|
||||
compose: ## Start full stack (API + UI + Postgres + Redis + MinIO)
|
||||
docker compose up -d
|
||||
```
|
||||
@@ -1,167 +1,664 @@
|
||||
# ArtifactAPI
|
||||
# Artifact Storage System
|
||||
|
||||
Caching proxy for package repositories. Single Go binary, 10 package types, content-addressable storage, managed by Terraform.
|
||||
A generic FastAPI-based artifact caching system that downloads and stores files from remote sources (GitHub, Gitea, HashiCorp, etc.) in S3-compatible storage with configuration-based access control.
|
||||
|
||||
## Quick Start
|
||||
## Features
|
||||
|
||||
```bash
|
||||
# Start backing services
|
||||
docker compose up -d postgres redis minio
|
||||
|
||||
# Build and run
|
||||
make build
|
||||
./bin/artifactapi
|
||||
|
||||
# Frontend (separate container or dev server)
|
||||
cd ui && npm install && npm run dev
|
||||
```
|
||||
|
||||
API: `http://localhost:8000` | Frontend: `http://localhost:5173`
|
||||
|
||||
## Package Types
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `generic` | nothing | everything |
|
||||
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
|
||||
| `helm` | `index.yaml` | `.tgz` charts |
|
||||
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
|
||||
| `npm` | package metadata | `.tgz` tarballs |
|
||||
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
|
||||
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
|
||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||
|
||||
Providers classify paths automatically. Users only configure what to proxy and TTLs.
|
||||
|
||||
## Terraform
|
||||
|
||||
Remotes and virtuals are managed by Terraform. Each package type has its own resource:
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 7200
|
||||
|
||||
patterns = [
|
||||
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
|
||||
"mikefarah/yq/.*/yq_linux_amd64$",
|
||||
]
|
||||
|
||||
mutable_patterns = [
|
||||
".*/archive/refs/heads/.*\\.tar\\.gz$",
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_docker" "dockerhub" {
|
||||
name = "dockerhub"
|
||||
base_url = "https://registry-1.docker.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
ban_tags_enabled = true
|
||||
ban_tags = ["latest"]
|
||||
|
||||
patterns = [
|
||||
"^library/postgres",
|
||||
"^library/redis",
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_helm" "jetstack" {
|
||||
name = "jetstack"
|
||||
base_url = "https://charts.jetstack.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 3600
|
||||
}
|
||||
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
members = [artifactapi_remote_helm.jetstack.name]
|
||||
}
|
||||
```
|
||||
|
||||
Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi)
|
||||
|
||||
## Access Control
|
||||
|
||||
| Field | Default | Behaviour |
|
||||
|---|---|---|
|
||||
| `patterns` | empty (proxy all) | If set, only matching paths are proxied. Acts as allowlist. |
|
||||
| `blocklist` | empty | Matching paths always denied. Checked first. |
|
||||
| `mutable_patterns` | empty | Override: force paths to mutable TTL. |
|
||||
| `immutable_patterns` | empty | Override: force paths to immutable TTL. |
|
||||
|
||||
No patterns + no blocklist = open proxy. Provider handles mutability classification automatically.
|
||||
|
||||
## API
|
||||
|
||||
### Proxy (v1)
|
||||
|
||||
```
|
||||
GET /api/v1/remote/{name}/{path} Proxy/cache artifact
|
||||
GET /api/v1/virtual/{name}/{path} Virtual repo (merged index)
|
||||
GET /v2/{name}/{path} Docker Registry v2
|
||||
```
|
||||
|
||||
### Management (v2)
|
||||
|
||||
```
|
||||
GET/POST /api/v2/remotes List / create remotes
|
||||
GET/PUT/DELETE /api/v2/remotes/{name} Read / update / delete remote
|
||||
GET/DELETE /api/v2/remotes/{name}/objects Browse / evict cached objects
|
||||
GET /api/v2/stats Overview stats
|
||||
GET /api/v2/health Service health
|
||||
POST /api/v2/probe Test a remote (fetch without streaming to client)
|
||||
GET /api/v2/events SSE event stream
|
||||
```
|
||||
- **Generic Remote Support**: Works with any HTTP-based file server (GitHub, Gitea, HashiCorp, custom servers)
|
||||
- **Configuration-Based**: YAML configuration for remotes, patterns, and access control
|
||||
- **Direct URL API**: Access cached files via clean URLs like `/api/github/owner/repo/path/file.tar.gz`
|
||||
- **Pattern Filtering**: Regex-based inclusion patterns for security and organization
|
||||
- **Smart Caching**: Automatic download and cache on first access, serve from cache afterward
|
||||
- **S3 Storage**: MinIO/S3 backend with predictable paths
|
||||
- **Content-Type Detection**: Automatic MIME type detection for downloads
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
PostgreSQL ─── config (remotes, virtuals), artifact metadata, access log
|
||||
Redis ─── TTL keys, fetch locks, circuit breaker state
|
||||
S3/MinIO ─── content-addressable blob storage (blobs/sha256/{hash})
|
||||
```
|
||||
The system acts as a caching proxy that:
|
||||
1. Receives requests via the `/api/{remote}/{path}` endpoint
|
||||
2. Checks if the file is already cached
|
||||
3. If not cached, downloads from the configured remote and caches it
|
||||
4. Serves the file with appropriate headers and content types
|
||||
5. Enforces access control via configurable regex patterns
|
||||
|
||||
S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go).
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `LISTEN_ADDR` | `:8000` | Server listen address |
|
||||
| `DBHOST` | `localhost` | PostgreSQL host |
|
||||
| `DBPORT` | `5432` | PostgreSQL port |
|
||||
| `DBUSER` | `artifacts` | PostgreSQL user |
|
||||
| `DBPASS` | | PostgreSQL password |
|
||||
| `DBNAME` | `artifacts` | PostgreSQL database |
|
||||
| `REDIS_URL` | `redis://localhost:6379` | Redis URL |
|
||||
| `MINIO_ENDPOINT` | `localhost:9000` | S3 endpoint |
|
||||
| `MINIO_ACCESS_KEY` | | S3 access key |
|
||||
| `MINIO_SECRET_KEY` | | S3 secret key |
|
||||
| `MINIO_BUCKET` | `artifacts` | S3 bucket |
|
||||
| `MINIO_SECURE` | `false` | Use HTTPS for S3 |
|
||||
| `MINIO_REGION` | | S3 region (AWS) |
|
||||
|
||||
## Development
|
||||
## Quick Start
|
||||
|
||||
1. Start MinIO container:
|
||||
```bash
|
||||
make build # Build binary
|
||||
make test # Unit tests
|
||||
make e2e # E2E tests (needs Docker)
|
||||
make lint # golangci-lint + go vet
|
||||
make fmt # gofmt + goimports
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
### TUI
|
||||
|
||||
2. Create virtual environment and install dependencies:
|
||||
```bash
|
||||
./bin/artifactapi tui --endpoint http://localhost:8000
|
||||
uv venv
|
||||
source .venv/bin/activate
|
||||
uv pip install -r requirements.txt
|
||||
```
|
||||
|
||||
3. Start the API:
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
|
||||
4. Access artifacts directly via URL:
|
||||
```bash
|
||||
# This will download and cache the file on first access
|
||||
xh GET localhost:8000/api/github/gruntwork-io/terragrunt/releases/download/v0.96.1/terragrunt_linux_amd64.tar.gz
|
||||
|
||||
# Subsequent requests serve from cache (see X-Artifact-Source: cache header)
|
||||
curl -I localhost:8000/api/github/gruntwork-io/terragrunt/releases/download/v0.96.1/terragrunt_linux_amd64.tar.gz
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Direct Access
|
||||
- `GET /api/{remote}/{path}` - Direct access to artifacts with auto-caching
|
||||
|
||||
### Management
|
||||
- `GET /` - API info and available remotes
|
||||
- `GET /health` - Health check
|
||||
- `GET /config` - View current configuration
|
||||
- `POST /cache-artifact` - Batch cache artifacts matching pattern
|
||||
- `GET /artifacts/{remote}` - List cached artifacts
|
||||
|
||||
## Configuration
|
||||
|
||||
The system uses `remotes.yaml` to define remote repositories and access patterns. All other configuration is provided via environment variables.
|
||||
|
||||
### remotes.yaml Structure
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
remote-name:
|
||||
base_url: "https://example.com" # Base URL for the remote
|
||||
type: "remote" # Type: "remote" or "local"
|
||||
package: "generic" # Package type: "generic", "alpine", "rpm"
|
||||
description: "Human readable description"
|
||||
include_patterns: # Regex patterns for allowed files
|
||||
- "pattern1"
|
||||
- "pattern2"
|
||||
cache: # Cache configuration (optional)
|
||||
file_ttl: 0 # File cache TTL (0 = indefinite)
|
||||
index_ttl: 300 # Index file TTL in seconds
|
||||
```
|
||||
|
||||
### Remote Types
|
||||
|
||||
#### Generic Remotes
|
||||
For general file hosting (GitHub releases, custom servers):
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
github:
|
||||
base_url: "https://github.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "GitHub releases and files"
|
||||
include_patterns:
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
- "lxc/incus/.*\\.tar\\.gz$"
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
cache:
|
||||
file_ttl: 0 # Cache files indefinitely
|
||||
index_ttl: 0 # No index files for generic remotes
|
||||
|
||||
hashicorp-releases:
|
||||
base_url: "https://releases.hashicorp.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "HashiCorp product releases"
|
||||
include_patterns:
|
||||
- "terraform/.*terraform_.*_linux_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_linux_amd64\\.zip$"
|
||||
- "consul/.*/consul_.*_linux_amd64\\.zip$"
|
||||
cache:
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
```
|
||||
|
||||
#### Package Repository Remotes
|
||||
For Linux package repositories with index files:
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
alpine:
|
||||
base_url: "https://dl-cdn.alpinelinux.org"
|
||||
type: "remote"
|
||||
package: "alpine"
|
||||
description: "Alpine Linux APK package repository"
|
||||
include_patterns:
|
||||
- ".*/x86_64/.*\\.apk$" # Only x86_64 packages
|
||||
cache:
|
||||
file_ttl: 0 # Cache packages indefinitely
|
||||
index_ttl: 7200 # Cache APKINDEX.tar.gz for 2 hours
|
||||
|
||||
almalinux:
|
||||
base_url: "http://mirror.aarnet.edu.au/pub/almalinux"
|
||||
type: "remote"
|
||||
package: "rpm"
|
||||
description: "AlmaLinux RPM package repository"
|
||||
include_patterns:
|
||||
- ".*/x86_64/.*\\.rpm$"
|
||||
- ".*/noarch/.*\\.rpm$"
|
||||
cache:
|
||||
file_ttl: 0
|
||||
index_ttl: 7200 # Cache metadata files for 2 hours
|
||||
```
|
||||
|
||||
#### Local Repositories
|
||||
For storing custom artifacts:
|
||||
|
||||
```yaml
|
||||
remotes:
|
||||
local-generic:
|
||||
type: "local"
|
||||
package: "generic"
|
||||
description: "Local generic file repository"
|
||||
cache:
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
```
|
||||
|
||||
### Include Patterns
|
||||
|
||||
Include patterns are regular expressions that control which files can be accessed:
|
||||
|
||||
```yaml
|
||||
include_patterns:
|
||||
# Specific project patterns
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
|
||||
# File extension patterns
|
||||
- ".*\\.tar\\.gz$"
|
||||
- ".*\\.zip$"
|
||||
- ".*\\.rpm$"
|
||||
|
||||
# Architecture-specific patterns
|
||||
- ".*/x86_64/.*"
|
||||
- ".*/linux-amd64/.*"
|
||||
|
||||
# Version-specific patterns
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
```
|
||||
|
||||
**Security Note**: Only files matching at least one include pattern are accessible. Files not matching any pattern return HTTP 403.
|
||||
|
||||
### Cache Configuration
|
||||
|
||||
Control how long different file types are cached:
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
file_ttl: 0 # Regular files (0 = cache indefinitely)
|
||||
index_ttl: 300 # Index files like APKINDEX.tar.gz (seconds)
|
||||
```
|
||||
|
||||
**Index Files**: Repository metadata files that change frequently:
|
||||
- Alpine: `APKINDEX.tar.gz`
|
||||
- RPM: `repomd.xml`, `*-primary.xml.gz`, etc.
|
||||
- These are automatically detected and use `index_ttl`
|
||||
|
||||
### Environment Variables
|
||||
|
||||
All runtime configuration comes from environment variables:
|
||||
|
||||
**Database Configuration:**
|
||||
- `DBHOST` - PostgreSQL host
|
||||
- `DBPORT` - PostgreSQL port
|
||||
- `DBUSER` - PostgreSQL username
|
||||
- `DBPASS` - PostgreSQL password
|
||||
- `DBNAME` - PostgreSQL database name
|
||||
|
||||
**Redis Configuration:**
|
||||
- `REDIS_URL` - Redis connection URL (e.g., `redis://localhost:6379`)
|
||||
|
||||
**S3/MinIO Configuration:**
|
||||
- `MINIO_ENDPOINT` - MinIO/S3 endpoint
|
||||
- `MINIO_ACCESS_KEY` - S3 access key
|
||||
- `MINIO_SECRET_KEY` - S3 secret key
|
||||
- `MINIO_BUCKET` - S3 bucket name
|
||||
- `MINIO_SECURE` - Use HTTPS (`true`/`false`)
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Direct File Access
|
||||
```bash
|
||||
# Access GitHub releases
|
||||
curl localhost:8000/api/github/gruntwork-io/terragrunt/releases/download/v0.96.1/terragrunt_linux_amd64.tar.gz
|
||||
|
||||
# Access HashiCorp releases (when configured)
|
||||
curl localhost:8000/api/hashicorp/terraform/1.6.0/terraform_1.6.0_linux_amd64.zip
|
||||
|
||||
# Access custom remotes
|
||||
curl localhost:8000/api/custom/path/to/file.tar.gz
|
||||
```
|
||||
|
||||
### Response Headers
|
||||
- `X-Artifact-Source: cache|remote` - Indicates if served from cache or freshly downloaded
|
||||
- `Content-Type` - Automatically detected (application/gzip, application/zip, etc.)
|
||||
- `Content-Disposition` - Download filename
|
||||
- `Content-Length` - File size
|
||||
|
||||
### Pattern Enforcement
|
||||
Access is controlled by regex patterns in the configuration. Requests for files not matching any pattern return HTTP 403.
|
||||
|
||||
## Storage Path Format
|
||||
|
||||
Files are stored with keys like:
|
||||
- `{remote_name}/{path_hash}/{filename}` for direct API access
|
||||
- `{hostname}/{url_hash}/{filename}` for legacy batch operations
|
||||
|
||||
Example: `github/a1b2c3d4e5f6g7h8/terragrunt_linux_amd64.tar.gz`
|
||||
|
||||
## Kubernetes Deployment
|
||||
|
||||
Deploy the artifact storage system to Kubernetes using the following manifests:
|
||||
|
||||
### 1. Namespace
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: artifact-storage
|
||||
```
|
||||
|
||||
### 2. ConfigMap for remotes.yaml
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: artifactapi-config
|
||||
namespace: artifact-storage
|
||||
data:
|
||||
remotes.yaml: |
|
||||
remotes:
|
||||
github:
|
||||
base_url: "https://github.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "GitHub releases and files"
|
||||
include_patterns:
|
||||
- "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
|
||||
- "lxc/incus/.*\\.tar\\.gz$"
|
||||
- "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
|
||||
cache:
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
|
||||
hashicorp-releases:
|
||||
base_url: "https://releases.hashicorp.com"
|
||||
type: "remote"
|
||||
package: "generic"
|
||||
description: "HashiCorp product releases"
|
||||
include_patterns:
|
||||
- "terraform/.*terraform_.*_linux_amd64\\.zip$"
|
||||
- "vault/.*vault_.*_linux_amd64\\.zip$"
|
||||
- "consul/.*/consul_.*_linux_amd64\\.zip$"
|
||||
cache:
|
||||
file_ttl: 0
|
||||
index_ttl: 0
|
||||
```
|
||||
|
||||
### 3. Secret for Environment Variables
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: artifactapi-secret
|
||||
namespace: artifact-storage
|
||||
type: Opaque
|
||||
stringData:
|
||||
DBHOST: "postgres-service"
|
||||
DBPORT: "5432"
|
||||
DBUSER: "artifacts"
|
||||
DBPASS: "artifacts123"
|
||||
DBNAME: "artifacts"
|
||||
REDIS_URL: "redis://redis-service:6379"
|
||||
MINIO_ENDPOINT: "minio-service:9000"
|
||||
MINIO_ACCESS_KEY: "minioadmin"
|
||||
MINIO_SECRET_KEY: "minioadmin"
|
||||
MINIO_BUCKET: "artifacts"
|
||||
MINIO_SECURE: "false"
|
||||
```
|
||||
|
||||
### 4. PostgreSQL Deployment
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: postgres
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: postgres
|
||||
spec:
|
||||
containers:
|
||||
- name: postgres
|
||||
image: postgres:15-alpine
|
||||
env:
|
||||
- name: POSTGRES_DB
|
||||
value: artifacts
|
||||
- name: POSTGRES_USER
|
||||
value: artifacts
|
||||
- name: POSTGRES_PASSWORD
|
||||
value: artifacts123
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
volumeMounts:
|
||||
- name: postgres-storage
|
||||
mountPath: /var/lib/postgresql/data
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["pg_isready", "-U", "artifacts", "-d", "artifacts"]
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: postgres-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: postgres-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: postgres-service
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
selector:
|
||||
app: postgres
|
||||
ports:
|
||||
- port: 5432
|
||||
targetPort: 5432
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: postgres-pvc
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
```
|
||||
|
||||
### 5. Redis Deployment
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: redis
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:7-alpine
|
||||
command: ["redis-server", "--save", "20", "1"]
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
volumeMounts:
|
||||
- name: redis-storage
|
||||
mountPath: /data
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: redis-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: redis-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redis-service
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
selector:
|
||||
app: redis
|
||||
ports:
|
||||
- port: 6379
|
||||
targetPort: 6379
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: redis-pvc
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
```
|
||||
|
||||
### 6. MinIO Deployment
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: minio
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: minio
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: minio
|
||||
spec:
|
||||
containers:
|
||||
- name: minio
|
||||
image: minio/minio:latest
|
||||
command: ["minio", "server", "/data", "--console-address", ":9001"]
|
||||
env:
|
||||
- name: MINIO_ROOT_USER
|
||||
value: minioadmin
|
||||
- name: MINIO_ROOT_PASSWORD
|
||||
value: minioadmin
|
||||
ports:
|
||||
- containerPort: 9000
|
||||
- containerPort: 9001
|
||||
volumeMounts:
|
||||
- name: minio-storage
|
||||
mountPath: /data
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /minio/health/live
|
||||
port: 9000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: minio-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: minio-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: minio-service
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
selector:
|
||||
app: minio
|
||||
ports:
|
||||
- name: api
|
||||
port: 9000
|
||||
targetPort: 9000
|
||||
- name: console
|
||||
port: 9001
|
||||
targetPort: 9001
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: minio-pvc
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
```
|
||||
|
||||
### 7. Artifact API Deployment
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: artifactapi
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: artifactapi
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: artifactapi
|
||||
spec:
|
||||
containers:
|
||||
- name: artifactapi
|
||||
image: artifactapi:latest
|
||||
ports:
|
||||
- containerPort: 8000
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: artifactapi-secret
|
||||
volumeMounts:
|
||||
- name: config-volume
|
||||
mountPath: /app/remotes.yaml
|
||||
subPath: remotes.yaml
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
volumes:
|
||||
- name: config-volume
|
||||
configMap:
|
||||
name: artifactapi-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: artifactapi-service
|
||||
namespace: artifact-storage
|
||||
spec:
|
||||
selector:
|
||||
app: artifactapi
|
||||
ports:
|
||||
- port: 8000
|
||||
targetPort: 8000
|
||||
type: ClusterIP
|
||||
```
|
||||
|
||||
### 8. Ingress (Optional)
|
||||
```yaml
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: artifactapi-ingress
|
||||
namespace: artifact-storage
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/rewrite-target: /
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "10g"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||
spec:
|
||||
rules:
|
||||
- host: artifacts.example.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: artifactapi-service
|
||||
port:
|
||||
number: 8000
|
||||
```
|
||||
|
||||
### Deployment Commands
|
||||
```bash
|
||||
# Create namespace
|
||||
kubectl apply -f namespace.yaml
|
||||
|
||||
# Deploy PostgreSQL, Redis, and MinIO
|
||||
kubectl apply -f postgres.yaml
|
||||
kubectl apply -f redis.yaml
|
||||
kubectl apply -f minio.yaml
|
||||
|
||||
# Wait for databases to be ready
|
||||
kubectl wait --for=condition=ready pod -l app=postgres -n artifact-storage --timeout=300s
|
||||
kubectl wait --for=condition=ready pod -l app=redis -n artifact-storage --timeout=300s
|
||||
kubectl wait --for=condition=ready pod -l app=minio -n artifact-storage --timeout=300s
|
||||
|
||||
# Deploy configuration and application
|
||||
kubectl apply -f configmap.yaml
|
||||
kubectl apply -f secret.yaml
|
||||
kubectl apply -f artifactapi.yaml
|
||||
|
||||
# Optional: Deploy ingress
|
||||
kubectl apply -f ingress.yaml
|
||||
|
||||
# Check deployment status
|
||||
kubectl get pods -n artifact-storage
|
||||
kubectl logs -f deployment/artifactapi -n artifact-storage
|
||||
```
|
||||
|
||||
### Access the API
|
||||
```bash
|
||||
# Port-forward to access locally
|
||||
kubectl port-forward service/artifactapi-service 8000:8000 -n artifact-storage
|
||||
|
||||
# Test the API
|
||||
curl http://localhost:8000/health
|
||||
curl http://localhost:8000/
|
||||
|
||||
# Access artifacts
|
||||
curl "http://localhost:8000/api/github/gruntwork-io/terragrunt/releases/download/v0.96.1/terragrunt_linux_amd64"
|
||||
```
|
||||
|
||||
### Notes for Production
|
||||
- Use proper secrets management (e.g., Vault, Sealed Secrets)
|
||||
- Configure resource limits and requests appropriately
|
||||
- Set up monitoring and alerting
|
||||
- Use external managed databases for production workloads
|
||||
- Configure backup strategies for persistent volumes
|
||||
- Set up proper TLS certificates for ingress
|
||||
- Consider using StatefulSets for databases with persistent storage
|
||||
@@ -1,57 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
||||
"git.unkin.net/unkin/artifactapi/internal/server"
|
||||
"git.unkin.net/unkin/artifactapi/internal/tui"
|
||||
)
|
||||
|
||||
var version = "dev"
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == "tui" {
|
||||
endpoint := os.Getenv("ARTIFACTAPI_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
endpoint = "http://localhost:8000"
|
||||
}
|
||||
for i, arg := range os.Args {
|
||||
if arg == "--endpoint" && i+1 < len(os.Args) {
|
||||
endpoint = os.Args[i+1]
|
||||
}
|
||||
}
|
||||
|
||||
app := tui.New(endpoint)
|
||||
if err := app.Run(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "tui error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
slog.Error("failed to load config", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
srv, err := server.New(cfg, version)
|
||||
if err != nil {
|
||||
slog.Error("failed to create server", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := srv.Run(ctx); err != nil {
|
||||
slog.Error("server exited with error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
+50
-55
@@ -1,22 +1,26 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
artifactapi:
|
||||
build: .
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
no_cache: true
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
LISTEN_ADDR: ":8000"
|
||||
DBHOST: postgres
|
||||
DBPORT: "5432"
|
||||
DBUSER: artifacts
|
||||
DBPASS: artifacts123
|
||||
DBNAME: artifacts
|
||||
DBSSL: disable
|
||||
REDIS_URL: redis://redis:6379
|
||||
MINIO_ENDPOINT: minio:9000
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
MINIO_BUCKET: artifacts
|
||||
MINIO_SECURE: "false"
|
||||
- CONFIG_PATH=/app/remotes.yaml
|
||||
- DBHOST=postgres
|
||||
- DBPORT=5432
|
||||
- DBUSER=artifacts
|
||||
- DBPASS=artifacts123
|
||||
- DBNAME=artifacts
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- MINIO_ENDPOINT=minio:9000
|
||||
- MINIO_ACCESS_KEY=minioadmin
|
||||
- MINIO_SECRET_KEY=minioadmin
|
||||
- MINIO_BUCKET=artifacts
|
||||
- MINIO_SECURE=false
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -25,35 +29,27 @@ services:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8000/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
ui:
|
||||
build:
|
||||
context: ui
|
||||
dockerfile: Dockerfile.ui
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
ports:
|
||||
- "8080:80"
|
||||
depends_on:
|
||||
- artifactapi
|
||||
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
ports:
|
||||
- "5432:5432"
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
environment:
|
||||
POSTGRES_USER: artifacts
|
||||
POSTGRES_PASSWORD: artifacts123
|
||||
POSTGRES_DB: artifacts
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
command: server /data --console-address ":9001"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U artifacts -d artifacts"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
@@ -64,28 +60,27 @@ services:
|
||||
command: redis-server --save 20 1
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
- "5432:5432"
|
||||
environment:
|
||||
POSTGRES_DB: artifacts
|
||||
POSTGRES_USER: artifacts
|
||||
POSTGRES_PASSWORD: artifacts123
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
test: ["CMD-SHELL", "pg_isready -U artifacts -d artifacts"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
redis_data:
|
||||
postgres_data:
|
||||
|
||||
-137
@@ -1,137 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres"
|
||||
tcredis "github.com/testcontainers/testcontainers-go/modules/redis"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
||||
"git.unkin.net/unkin/artifactapi/internal/server"
|
||||
)
|
||||
|
||||
var baseURL string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
|
||||
pgContainer, err := tcpostgres.Run(ctx,
|
||||
"postgres:17-alpine",
|
||||
tcpostgres.WithDatabase("artifacts"),
|
||||
tcpostgres.WithUsername("artifacts"),
|
||||
tcpostgres.WithPassword("artifacts123"),
|
||||
testcontainers.WithWaitStrategy(
|
||||
wait.ForListeningPort("5432/tcp").WithStartupTimeout(30*time.Second),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer pgContainer.Terminate(ctx)
|
||||
|
||||
redisContainer, err := tcredis.Run(ctx,
|
||||
"redis:7-alpine",
|
||||
testcontainers.WithWaitStrategy(
|
||||
wait.ForListeningPort("6379/tcp").WithStartupTimeout(30*time.Second),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("redis: %v", err)
|
||||
}
|
||||
defer redisContainer.Terminate(ctx)
|
||||
|
||||
minioContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
ContainerRequest: testcontainers.ContainerRequest{
|
||||
Image: "minio/minio:latest",
|
||||
ExposedPorts: []string{"9000/tcp"},
|
||||
Cmd: []string{"server", "/data"},
|
||||
Env: map[string]string{
|
||||
"MINIO_ROOT_USER": "minioadmin",
|
||||
"MINIO_ROOT_PASSWORD": "minioadmin",
|
||||
},
|
||||
WaitingFor: wait.ForHTTP("/minio/health/live").WithPort("9000/tcp").WithStartupTimeout(30 * time.Second),
|
||||
},
|
||||
Started: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("minio: %v", err)
|
||||
}
|
||||
defer minioContainer.Terminate(ctx)
|
||||
|
||||
pgHost, _ := pgContainer.Host(ctx)
|
||||
pgPort, _ := pgContainer.MappedPort(ctx, "5432/tcp")
|
||||
redisHost, _ := redisContainer.Host(ctx)
|
||||
redisPort, _ := redisContainer.MappedPort(ctx, "6379/tcp")
|
||||
minioHost, _ := minioContainer.Host(ctx)
|
||||
minioPort, _ := minioContainer.MappedPort(ctx, "9000/tcp")
|
||||
|
||||
os.Setenv("DBHOST", pgHost)
|
||||
os.Setenv("DBPORT", pgPort.Port())
|
||||
os.Setenv("DBUSER", "artifacts")
|
||||
os.Setenv("DBPASS", "artifacts123")
|
||||
os.Setenv("DBNAME", "artifacts")
|
||||
os.Setenv("DBSSL", "disable")
|
||||
os.Setenv("REDIS_URL", fmt.Sprintf("redis://%s:%s", redisHost, redisPort.Port()))
|
||||
os.Setenv("MINIO_ENDPOINT", fmt.Sprintf("%s:%s", minioHost, minioPort.Port()))
|
||||
os.Setenv("MINIO_ACCESS_KEY", "minioadmin")
|
||||
os.Setenv("MINIO_SECRET_KEY", "minioadmin")
|
||||
os.Setenv("MINIO_BUCKET", "artifacts-test")
|
||||
os.Setenv("MINIO_SECURE", "false")
|
||||
os.Setenv("LISTEN_ADDR", ":0")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
cfg.ListenAddr = "127.0.0.1:0"
|
||||
|
||||
srv, err := server.New(cfg, "e2e-test")
|
||||
if err != nil {
|
||||
log.Fatalf("server: %v", err)
|
||||
}
|
||||
|
||||
srvCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
addr := startServer(srvCtx, srv)
|
||||
baseURL = "http://" + addr
|
||||
|
||||
code := m.Run()
|
||||
cancel()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func startServer(ctx context.Context, srv *server.Server) string {
|
||||
ln, err := findListener()
|
||||
if err != nil {
|
||||
log.Fatalf("listener: %v", err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
|
||||
go srv.RunOnListener(ctx, ln)
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := http.Get("http://" + addr + "/health")
|
||||
if err == nil && resp.StatusCode == 200 {
|
||||
resp.Body.Close()
|
||||
return addr
|
||||
}
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
log.Fatalf("server did not start in time at %s", addr)
|
||||
return ""
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func findListener() (net.Listener, error) {
|
||||
return net.Listen("tcp", "127.0.0.1:0")
|
||||
}
|
||||
|
||||
func apiURL(path string) string {
|
||||
return baseURL + path
|
||||
}
|
||||
|
||||
func createRemote(t *testing.T, body string) {
|
||||
t.Helper()
|
||||
resp, err := http.Post(apiURL("/api/v2/remotes"), "application/json", strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("create remote: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("create remote: status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRemote(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(http.MethodDelete, apiURL("/api/v2/remotes/"+name), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("delete remote: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
func getJSON(t *testing.T, url string) map[string]any {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("GET %s: status %d: %s", url, resp.StatusCode, b)
|
||||
}
|
||||
var result map[string]any
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func getBody(t *testing.T, url string) ([]byte, int) {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return b, resp.StatusCode
|
||||
}
|
||||
|
||||
func getString(t *testing.T, url string) string {
|
||||
t.Helper()
|
||||
b, status := getBody(t, url)
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("GET %s: status %d: %s", url, status, b)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func assertStatus(t *testing.T, url string, wantStatus int) {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != wantStatus {
|
||||
t.Errorf("GET %s: got %d, want %d", url, resp.StatusCode, wantStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRequest(t *testing.T, url string) int {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(http.MethodDelete, url, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("DELETE %s: %v", url, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
func mustFormat(format string, args ...any) string {
|
||||
return fmt.Sprintf(format, args...)
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHealth(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/health"))
|
||||
if result["status"] != "ok" {
|
||||
t.Errorf("expected ok, got %v", result["status"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoot(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/"))
|
||||
if result["name"] != "artifactapi" {
|
||||
t.Errorf("expected artifactapi, got %v", result["name"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUpstreamTimeouts(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "timeout-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"stale_on_error": true,
|
||||
"upstream_dial_timeout": 3,
|
||||
"upstream_tls_timeout": 4,
|
||||
"upstream_response_header_timeout": 5
|
||||
}`)
|
||||
defer deleteRemote(t, "timeout-test")
|
||||
|
||||
remote := getJSON(t, apiURL("/api/v2/remotes/timeout-test"))
|
||||
for field, want := range map[string]float64{
|
||||
"upstream_dial_timeout": 3,
|
||||
"upstream_tls_timeout": 4,
|
||||
"upstream_response_header_timeout": 5,
|
||||
} {
|
||||
if got, _ := remote[field].(float64); got != want {
|
||||
t.Errorf("%s: got %v, want %v", field, remote[field], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteCRUD(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "test-generic",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"description": "test remote",
|
||||
"mutable_ttl": 600,
|
||||
"check_mutable": true,
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "test-generic")
|
||||
|
||||
remote := getJSON(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if remote["name"] != "test-generic" {
|
||||
t.Errorf("expected test-generic, got %v", remote["name"])
|
||||
}
|
||||
if remote["package_type"] != "generic" {
|
||||
t.Errorf("expected generic, got %v", remote["package_type"])
|
||||
}
|
||||
|
||||
req, _ := http.NewRequest(http.MethodPut, apiURL("/api/v2/remotes/test-generic"),
|
||||
strings.NewReader(`{
|
||||
"package_type": "generic",
|
||||
"base_url": "https://updated.example.com",
|
||||
"description": "updated",
|
||||
"mutable_ttl": 300,
|
||||
"check_mutable": true,
|
||||
"stale_on_error": true
|
||||
}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
updated := getJSON(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if updated["base_url"] != "https://updated.example.com" {
|
||||
t.Errorf("expected updated URL, got %v", updated["base_url"])
|
||||
}
|
||||
|
||||
status := deleteRequest(t, apiURL("/api/v2/remotes/test-generic"))
|
||||
if status != http.StatusNoContent {
|
||||
t.Errorf("delete: got %d, want 204", status)
|
||||
}
|
||||
|
||||
assertStatus(t, apiURL("/api/v2/remotes/test-generic"), http.StatusNotFound)
|
||||
}
|
||||
|
||||
func TestRemoteList(t *testing.T) {
|
||||
createRemote(t, `{"name":"list-a","package_type":"generic","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRemote(t, `{"name":"list-b","package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRemote(t, "list-a")
|
||||
defer deleteRemote(t, "list-b")
|
||||
|
||||
resp, err := http.Get(apiURL("/api/v2/remotes"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
|
||||
var remotes []map[string]any
|
||||
json.Unmarshal(body, &remotes)
|
||||
|
||||
if len(remotes) < 2 {
|
||||
t.Fatalf("expected at least 2 remotes, got %d", len(remotes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualCRUD(t *testing.T) {
|
||||
createRemote(t, `{"name":"virt-member-a","package_type":"helm","base_url":"https://a.example.com","stale_on_error":true}`)
|
||||
createRemote(t, `{"name":"virt-member-b","package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`)
|
||||
defer deleteRemote(t, "virt-member-a")
|
||||
defer deleteRemote(t, "virt-member-b")
|
||||
|
||||
resp, err := http.Post(apiURL("/api/v2/virtuals"), "application/json",
|
||||
strings.NewReader(`{
|
||||
"name": "test-virtual",
|
||||
"package_type": "helm",
|
||||
"description": "test virtual",
|
||||
"members": ["virt-member-a", "virt-member-b"]
|
||||
}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("create virtual: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
virt := getJSON(t, apiURL("/api/v2/virtuals/test-virtual"))
|
||||
if virt["name"] != "test-virtual" {
|
||||
t.Errorf("expected test-virtual, got %v", virt["name"])
|
||||
}
|
||||
|
||||
status := deleteRequest(t, apiURL("/api/v2/virtuals/test-virtual"))
|
||||
if status != http.StatusNoContent {
|
||||
t.Errorf("delete virtual: got %d, want 204", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsEndpoint(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/api/v2/stats"))
|
||||
if _, ok := result["total_remotes"]; !ok {
|
||||
t.Error("expected total_remotes in stats")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthV2(t *testing.T) {
|
||||
result := getJSON(t, apiURL("/api/v2/health"))
|
||||
if result["status"] != "ok" {
|
||||
t.Errorf("expected ok, got %v", result["status"])
|
||||
}
|
||||
if result["postgres"] != "ok" {
|
||||
t.Errorf("expected postgres ok, got %v", result["postgres"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidPackageType(t *testing.T) {
|
||||
resp, err := http.Post(apiURL("/api/v2/remotes"), "application/json",
|
||||
strings.NewReader(`{"name":"bad","package_type":"bogus","base_url":"https://x.com"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("expected 400 for invalid package type, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProxyUnknownRemote(t *testing.T) {
|
||||
assertStatus(t, apiURL("/api/v1/remote/nonexistent/some/path"), http.StatusNotFound)
|
||||
}
|
||||
|
||||
func TestProxyBlocklist(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "blocklist-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"blocklist": ["\\.exe$"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "blocklist-test")
|
||||
|
||||
assertStatus(t, apiURL("/api/v1/remote/blocklist-test/malware.exe"), http.StatusForbidden)
|
||||
}
|
||||
|
||||
func TestProxyHeadBlocklist(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "head-block-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"blocklist": ["\\.exe$"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "head-block-test")
|
||||
|
||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/head-block-test/malware.exe"), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("HEAD blocklisted path: got %d, want 403", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyHeadUnknownRemote(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodHead, apiURL("/v2/nonexistent/some/path"), nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("HEAD unknown remote: got %d, want 404", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyPatterns(t *testing.T) {
|
||||
createRemote(t, `{
|
||||
"name": "patterns-test",
|
||||
"package_type": "generic",
|
||||
"base_url": "https://example.com",
|
||||
"patterns": ["^releases/"],
|
||||
"stale_on_error": true
|
||||
}`)
|
||||
defer deleteRemote(t, "patterns-test")
|
||||
|
||||
assertStatus(t, apiURL("/api/v1/remote/patterns-test/uploads/file.tar.gz"), http.StatusForbidden)
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
module git.unkin.net/unkin/artifactapi
|
||||
|
||||
go 1.25.9
|
||||
|
||||
require (
|
||||
github.com/cavaliergopher/rpm v1.3.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/go-chi/chi/v5 v5.3.0
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/minio/minio-go/v7 v7.2.0
|
||||
github.com/redis/go-redis/v9 v9.20.0
|
||||
github.com/testcontainers/testcontainers-go v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.4.3 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.11.7 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
|
||||
github.com/charmbracelet/x/term v0.2.2 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.11.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v0.2.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.23 // indirect
|
||||
github.com/mdelapenya/tlscert v0.2.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/moby/api v1.54.1 // indirect
|
||||
github.com/moby/moby/client v0.4.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/stretchr/testify v1.11.1 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.51.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.2 // indirect
|
||||
)
|
||||
@@ -1,249 +0,0 @@
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cavaliergopher/rpm v1.3.0 h1:UHX46sasX8MesUXXQ+UbkFLUX4eUWTlEcX8jcnRBIgI=
|
||||
github.com/cavaliergopher/rpm v1.3.0/go.mod h1:vEumo1vvtrHM1Ov86f6+k8j7zNKOxQfHDCAIcR/36ZI=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
|
||||
github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
|
||||
github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=
|
||||
github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q=
|
||||
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
|
||||
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
|
||||
github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI=
|
||||
github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
|
||||
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
|
||||
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
|
||||
github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8=
|
||||
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
|
||||
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw=
|
||||
github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
|
||||
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs=
|
||||
github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
||||
github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4=
|
||||
github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||
github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw=
|
||||
github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g=
|
||||
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
|
||||
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
|
||||
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
|
||||
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0=
|
||||
github.com/redis/go-redis/v9 v9.20.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
||||
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
|
||||
github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 h1:id/6LH8ZeDrtAUVSuNvZUAJ1kVpb82y1pr9yweAWsRg=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0/go.mod h1:uF0jI8FITagQpBNOgweGBmPf6rP4K0SeL1XFPbsZSSY=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
||||
go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
|
||||
go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
|
||||
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss=
|
||||
gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||
pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
|
||||
pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||
@@ -1,209 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
type ProxyHandler struct {
|
||||
engine *proxy.Engine
|
||||
virtualEngine *virtual.Engine
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
local *v2.LocalHandler
|
||||
}
|
||||
|
||||
func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *database.DB, store *storage.S3, local *v2.LocalHandler) *ProxyHandler {
|
||||
return &ProxyHandler{engine: engine, virtualEngine: virtualEngine, db: db, store: store, local: local}
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/remote/{remoteName}/*", h.handleProxy)
|
||||
r.Get("/local/{localName}/*", h.handleLocal)
|
||||
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) DockerV2Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.handleDockerPing)
|
||||
r.Head("/", h.handleDockerPing)
|
||||
r.Get("/{remoteName}/*", h.handleProxy)
|
||||
r.Head("/{remoteName}/*", h.handleProxyHead)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Docker-Distribution-Api-Version", "registry/2.0")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy fetch failed", "remote", remoteName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer result.Reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, result.Reader)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.engine.Head(r.Context(), *remote, path, prov)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
||||
virtualName := chi.URLParam(r, "virtualName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
virt, err := h.db.GetVirtual(r.Context(), virtualName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("virtual %q not found", virtualName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||
|
||||
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
||||
if err != nil {
|
||||
slog.Error("virtual fetch failed", "virtual", virtualName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("X-Artifact-Source", "virtual")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
|
||||
localName := chi.URLParam(r, "localName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), localName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("local %q not found", localName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
if indexer, ok := prov.(provider.LocalIndexer); ok {
|
||||
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
h.serveLocalFile(w, r, localName, path)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, repoName, path string) {
|
||||
file, err := h.db.GetLocalFile(r.Context(), repoName, path)
|
||||
if err != nil {
|
||||
slog.Error("local file lookup failed", "repo", repoName, "path", path, "error", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if file == nil {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):])
|
||||
reader, info, err := h.store.Download(r.Context(), s3Key)
|
||||
if err != nil {
|
||||
slog.Error("local file download failed", "repo", repoName, "path", path, "error", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", info.ContentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func scheme(r *http.Request) string {
|
||||
if r.TLS != nil {
|
||||
return "https"
|
||||
}
|
||||
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
|
||||
return fwd
|
||||
}
|
||||
return "http"
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
type EventsHandler struct{}
|
||||
|
||||
func NewEventsHandler() *EventsHandler {
|
||||
return &EventsHandler{}
|
||||
}
|
||||
|
||||
func (h *EventsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.stream)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *EventsHandler) stream(w http.ResponseWriter, r *http.Request) {
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
http.Error(w, "streaming not supported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("X-Accel-Buffering", "no")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
flusher.Flush()
|
||||
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
fmt.Fprintf(w, ": keepalive\n\n")
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
)
|
||||
|
||||
type HealthHandler struct {
|
||||
db *database.DB
|
||||
cache *cache.Redis
|
||||
store *storage.S3
|
||||
}
|
||||
|
||||
func NewHealthHandler(db *database.DB, c *cache.Redis, s *storage.S3) *HealthHandler {
|
||||
return &HealthHandler{db: db, cache: c, store: s}
|
||||
}
|
||||
|
||||
func (h *HealthHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.health)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *HealthHandler) health(w http.ResponseWriter, r *http.Request) {
|
||||
status := map[string]string{
|
||||
"status": "ok",
|
||||
"postgres": "ok",
|
||||
"redis": "ok",
|
||||
"s3": "ok",
|
||||
}
|
||||
|
||||
if err := h.db.Pool.Ping(r.Context()); err != nil {
|
||||
status["postgres"] = "error"
|
||||
status["status"] = "degraded"
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type LocalHandler struct {
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
cas *storage.CAS
|
||||
}
|
||||
|
||||
func NewLocalHandler(db *database.DB, store *storage.S3) *LocalHandler {
|
||||
return &LocalHandler{
|
||||
db: db,
|
||||
store: store,
|
||||
cas: storage.NewCAS(store),
|
||||
}
|
||||
}
|
||||
|
||||
func (h *LocalHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Put("/*", h.upload)
|
||||
r.Get("/*", h.download)
|
||||
r.Delete("/*", h.remove)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *LocalHandler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
if filePath == "" {
|
||||
http.Error(w, "file path required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", repoName), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if remote.RepoType != models.RepoTypeLocal {
|
||||
http.Error(w, "upload only allowed for local repository types", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
|
||||
if uploader, ok := prov.(provider.LocalUploader); ok {
|
||||
h.uploadValidated(w, r, remote, filePath, prov, uploader)
|
||||
return
|
||||
}
|
||||
|
||||
h.uploadGeneric(w, r, remote, filePath)
|
||||
}
|
||||
|
||||
func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string, prov provider.Provider, uploader provider.LocalUploader) {
|
||||
storagePath, contentType, err := uploader.ValidateUpload(filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, storagePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if existing != nil {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.cas.Store(r.Context(), r.Body, contentType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.CreateLocalFile(r.Context(), remote.Name, storagePath, result.ContentHash); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if hook, ok := prov.(provider.PostUploadHook); ok {
|
||||
go hook.AfterUpload(context.Background(), remote.Name, storagePath, result.ContentHash, h, h.db)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, uploader.UploadResponse(storagePath, result.ContentHash, result.SizeBytes))
|
||||
}
|
||||
|
||||
func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) {
|
||||
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if existing != nil {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", filePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := "application/octet-stream"
|
||||
if ct := r.Header.Get("Content-Type"); ct != "" && ct != "application/octet-stream" {
|
||||
contentType = ct
|
||||
}
|
||||
|
||||
result, err := h.cas.Store(r.Context(), r.Body, contentType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.db.CreateLocalFile(r.Context(), remote.Name, filePath, result.ContentHash); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", filePath), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, map[string]any{
|
||||
"path": filePath,
|
||||
"content_hash": result.ContentHash,
|
||||
"size_bytes": result.SizeBytes,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *LocalHandler) download(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
file, err := h.db.GetLocalFile(r.Context(), repoName, filePath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if file == nil {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):])
|
||||
reader, info, err := h.store.Download(r.Context(), s3Key)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("download failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
w.Header().Set("Content-Type", info.ContentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func (h *LocalHandler) remove(w http.ResponseWriter, r *http.Request) {
|
||||
repoName := chi.URLParam(r, "name")
|
||||
filePath := chi.URLParam(r, "*")
|
||||
|
||||
if err := h.db.DeleteLocalFile(r.Context(), repoName, filePath); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *LocalHandler) DB() *database.DB {
|
||||
return h.db
|
||||
}
|
||||
|
||||
func (h *LocalHandler) Download(ctx context.Context, key string) (io.ReadCloser, int64, error) {
|
||||
reader, info, err := h.store.Download(ctx, key)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return reader, info.Size, nil
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
type ObjectsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewObjectsHandler(db *database.DB) *ObjectsHandler {
|
||||
return &ObjectsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Delete("/*", h.evict)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("per_page"))
|
||||
if limit <= 0 || limit > 5000 {
|
||||
limit = 50
|
||||
}
|
||||
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
offset := (page - 1) * limit
|
||||
|
||||
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, artifacts)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
if err := h.db.DeleteArtifact(r.Context(), remoteName, path); err != nil {
|
||||
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
)
|
||||
|
||||
type ProbeHandler struct {
|
||||
engine *proxy.Engine
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewProbeHandler(engine *proxy.Engine, db *database.DB) *ProbeHandler {
|
||||
return &ProbeHandler{engine: engine, db: db}
|
||||
}
|
||||
|
||||
func (h *ProbeHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Post("/", h.probe)
|
||||
return r
|
||||
}
|
||||
|
||||
type probeRequest struct {
|
||||
Remote string `json:"remote"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
type probeResponse struct {
|
||||
Status int `json:"status"`
|
||||
Source string `json:"source,omitempty"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (h *ProbeHandler) probe(w http.ResponseWriter, r *http.Request) {
|
||||
var req probeRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if req.Remote == "" || req.Path == "" {
|
||||
http.Error(w, "remote and path are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), req.Remote)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 404,
|
||||
Error: fmt.Sprintf("remote %q not found", req.Remote),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 500,
|
||||
Error: fmt.Sprintf("no provider for %q", remote.PackageType),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
result, err := h.engine.Fetch(ctx, *remote, req.Path, prov)
|
||||
duration := time.Since(start).Milliseconds()
|
||||
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 502,
|
||||
DurationMS: duration,
|
||||
Error: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
io.Copy(io.Discard, result.Reader)
|
||||
result.Reader.Close()
|
||||
|
||||
writeJSON(w, http.StatusOK, probeResponse{
|
||||
Status: 200,
|
||||
Source: result.Source,
|
||||
ContentType: result.ContentType,
|
||||
SizeBytes: result.Size,
|
||||
DurationMS: duration,
|
||||
Headers: map[string]string{
|
||||
"X-Artifact-Source": result.Source,
|
||||
"X-Artifact-Size": fmt.Sprintf("%d", result.Size),
|
||||
"Content-Type": result.ContentType,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type RemotesHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewRemotesHandler(db *database.DB) *RemotesHandler {
|
||||
return &RemotesHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Post("/", h.create)
|
||||
r.Get("/{name}", h.get)
|
||||
r.Put("/{name}", h.update)
|
||||
r.Delete("/{name}", h.del)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remotes, err := h.db.ListRemotes(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remotes)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
remote, err := h.db.GetRemote(r.Context(), name)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", name), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
var remote models.Remote
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !remote.PackageType.Valid() {
|
||||
http.Error(w, fmt.Sprintf("invalid package type: %q", remote.PackageType), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if remote.RepoType == "" {
|
||||
remote.RepoType = models.RepoTypeRemote
|
||||
}
|
||||
if !remote.RepoType.Valid() {
|
||||
http.Error(w, fmt.Sprintf("invalid repo type: %q", remote.RepoType), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if remote.RepoType == models.RepoTypeRemote && remote.BaseURL == "" {
|
||||
http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.CreateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
var remote models.Remote
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
remote.Name = name
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
if err := h.db.DeleteRemote(r.Context(), name); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
type StatsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewStatsHandler(db *database.DB) *StatsHandler {
|
||||
return &StatsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *StatsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.overview)
|
||||
r.Get("/top-remotes", h.topRemotes)
|
||||
r.Get("/top-files-by-hits", h.topFilesByHits)
|
||||
r.Get("/top-files-by-bandwidth", h.topFilesByBandwidth)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *StatsHandler) overview(w http.ResponseWriter, r *http.Request) {
|
||||
stats, err := h.db.GetOverviewStats(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, stats)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topRemotes(w http.ResponseWriter, r *http.Request) {
|
||||
remotes, err := h.db.GetTopRemotes(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remotes)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topFilesByHits(w http.ResponseWriter, r *http.Request) {
|
||||
files, err := h.db.GetTopFilesByHits(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, files)
|
||||
}
|
||||
|
||||
func (h *StatsHandler) topFilesByBandwidth(w http.ResponseWriter, r *http.Request) {
|
||||
files, err := h.db.GetTopFilesByBandwidth(r.Context(), 10)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, files)
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type VirtualsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
func NewVirtualsHandler(db *database.DB) *VirtualsHandler {
|
||||
return &VirtualsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Post("/", h.create)
|
||||
r.Get("/{name}", h.get)
|
||||
r.Put("/{name}", h.update)
|
||||
r.Delete("/{name}", h.del)
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
virtuals, err := h.db.ListVirtuals(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virtuals)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
virt, err := h.db.GetVirtual(r.Context(), name)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("virtual %q not found", name), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
var virt models.Virtual
|
||||
if err := json.NewDecoder(r.Body).Decode(&virt); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.db.CreateVirtual(r.Context(), &virt); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
var virt models.Virtual
|
||||
if err := json.NewDecoder(r.Body).Decode(&virt); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
virt.Name = name
|
||||
if err := h.db.UpdateVirtual(r.Context(), &virt); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, virt)
|
||||
}
|
||||
|
||||
func (h *VirtualsHandler) del(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
if err := h.db.DeleteVirtual(r.Context(), name); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func BasicHeaders(remote models.Remote) http.Header {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString(
|
||||
[]byte(remote.Username+":"+remote.Password),
|
||||
))
|
||||
}
|
||||
return h
|
||||
}
|
||||
Vendored
-117
@@ -1,117 +0,0 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type Redis struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
func NewRedis(url string) (*Redis, error) {
|
||||
opts, err := redis.ParseURL(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse redis url: %w", err)
|
||||
}
|
||||
|
||||
client := redis.NewClient(opts)
|
||||
|
||||
if err := client.Ping(context.Background()).Err(); err != nil {
|
||||
return nil, fmt.Errorf("ping redis: %w", err)
|
||||
}
|
||||
|
||||
return &Redis{client: client}, nil
|
||||
}
|
||||
|
||||
func (r *Redis) Close() error {
|
||||
return r.client.Close()
|
||||
}
|
||||
|
||||
func (r *Redis) SetTTL(ctx context.Context, remote, path string, ttl time.Duration) error {
|
||||
key := fmt.Sprintf("ttl:%s:%s", remote, path)
|
||||
return r.client.Set(ctx, key, "1", ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) CheckTTL(ctx context.Context, remote, path string) (bool, error) {
|
||||
key := fmt.Sprintf("ttl:%s:%s", remote, path)
|
||||
exists, err := r.client.Exists(ctx, key).Result()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exists > 0, nil
|
||||
}
|
||||
|
||||
func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (bool, error) {
|
||||
key := fmt.Sprintf("lock:%s:%s", remote, path)
|
||||
ok, err := r.client.SetNX(ctx, key, "1", ttl).Result()
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error {
|
||||
key := fmt.Sprintf("lock:%s:%s", remote, path)
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error {
|
||||
key := fmt.Sprintf("etag:%s:%s", remote, path)
|
||||
return r.client.Set(ctx, key, etag, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) GetETag(ctx context.Context, remote, path string) (string, error) {
|
||||
key := fmt.Sprintf("etag:%s:%s", remote, path)
|
||||
val, err := r.client.Get(ctx, key).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) GetToken(ctx context.Context, key string) (string, error) {
|
||||
val, err := r.client.Get(ctx, "token:"+key).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) SetToken(ctx context.Context, key, token string, ttl time.Duration) error {
|
||||
return r.client.Set(ctx, "token:"+key, token, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) IncrCircuitFailure(ctx context.Context, remote string, cooldown time.Duration) (int64, error) {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
pipe := r.client.Pipeline()
|
||||
incr := pipe.Incr(ctx, key)
|
||||
pipe.Expire(ctx, key, cooldown)
|
||||
_, err := pipe.Exec(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return incr.Val(), nil
|
||||
}
|
||||
|
||||
func (r *Redis) ResetCircuit(ctx context.Context, remote string) error {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) GetCircuitFailures(ctx context.Context, remote string) (int64, error) {
|
||||
key := fmt.Sprintf("circuit:%s", remote)
|
||||
val, err := r.client.Get(ctx, key).Int64()
|
||||
if err == redis.Nil {
|
||||
return 0, nil
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (r *Redis) FlushRemote(ctx context.Context, remote string) error {
|
||||
iter := r.client.Scan(ctx, 0, fmt.Sprintf("*:%s:*", remote), 100).Iterator()
|
||||
for iter.Next(ctx) {
|
||||
r.client.Del(ctx, iter.Val())
|
||||
}
|
||||
return iter.Err()
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ListenAddr string
|
||||
|
||||
DBHost string
|
||||
DBPort int
|
||||
DBUser string
|
||||
DBPass string
|
||||
DBName string
|
||||
DBSSL string
|
||||
|
||||
RedisURL string
|
||||
|
||||
S3Endpoint string
|
||||
S3AccessKey string
|
||||
S3SecretKey string
|
||||
S3Bucket string
|
||||
S3Secure bool
|
||||
S3Region string
|
||||
}
|
||||
|
||||
func (c *Config) DatabaseDSN() string {
|
||||
return fmt.Sprintf(
|
||||
"postgres://%s:%s@%s:%d/%s?sslmode=%s",
|
||||
c.DBUser, c.DBPass, c.DBHost, c.DBPort, c.DBName, c.DBSSL,
|
||||
)
|
||||
}
|
||||
|
||||
func Load() (*Config, error) {
|
||||
dbPort, err := strconv.Atoi(getenv("DBPORT", "5432"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid DBPORT: %w", err)
|
||||
}
|
||||
|
||||
s3Secure, _ := strconv.ParseBool(getenv("MINIO_SECURE", "false"))
|
||||
|
||||
cfg := &Config{
|
||||
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
|
||||
|
||||
DBHost: getenv("DBHOST", "localhost"),
|
||||
DBPort: dbPort,
|
||||
DBUser: getenv("DBUSER", "artifacts"),
|
||||
DBPass: getenv("DBPASS", ""),
|
||||
DBName: getenv("DBNAME", "artifacts"),
|
||||
DBSSL: getenv("DBSSL", "disable"),
|
||||
|
||||
RedisURL: getenv("REDIS_URL", "redis://localhost:6379"),
|
||||
|
||||
S3Endpoint: getenv("MINIO_ENDPOINT", "localhost:9000"),
|
||||
S3AccessKey: getenv("MINIO_ACCESS_KEY", ""),
|
||||
S3SecretKey: getenv("MINIO_SECRET_KEY", ""),
|
||||
S3Bucket: getenv("MINIO_BUCKET", "artifacts"),
|
||||
S3Secure: s3Secure,
|
||||
S3Region: getenv("MINIO_REGION", ""),
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) UpsertBlob(ctx context.Context, contentHash, s3Key string, sizeBytes int64, contentType string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO blobs (content_hash, s3_key, size_bytes, content_type)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (content_hash) DO NOTHING
|
||||
`, contentHash, s3Key, sizeBytes, contentType)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpsertArtifact(ctx context.Context, remoteName, path, contentHash, upstreamETag string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO artifacts (remote_name, path, content_hash, upstream_etag)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (remote_name, path) DO UPDATE SET
|
||||
content_hash = EXCLUDED.content_hash,
|
||||
upstream_etag = EXCLUDED.upstream_etag,
|
||||
last_fetched_at = NOW(),
|
||||
fetch_count = artifacts.fetch_count + 1
|
||||
`, remoteName, path, contentHash, upstreamETag)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) GetArtifact(ctx context.Context, remoteName, path string) (*models.Artifact, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||
a.last_accessed_at, a.fetch_count, a.access_count,
|
||||
b.size_bytes, b.content_type
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
WHERE a.remote_name = $1 AND a.path = $2
|
||||
`, remoteName, path)
|
||||
|
||||
var a models.Artifact
|
||||
err := row.Scan(
|
||||
&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &a.UpstreamETag,
|
||||
&a.UpstreamLastModified, &a.FirstSeenAt, &a.LastFetchedAt,
|
||||
&a.LastAccessedAt, &a.FetchCount, &a.AccessCount,
|
||||
&a.SizeBytes, &a.ContentType,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func (db *DB) TouchArtifactAccess(ctx context.Context, remoteName, path string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE artifacts SET
|
||||
last_accessed_at = NOW(),
|
||||
access_count = access_count + 1
|
||||
WHERE remote_name = $1 AND path = $2
|
||||
`, remoteName, path)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListArtifacts(ctx context.Context, remoteName string, limit, offset int) ([]models.Artifact, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||
a.last_accessed_at, a.fetch_count, a.access_count,
|
||||
b.size_bytes, b.content_type
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
WHERE a.remote_name = $1
|
||||
ORDER BY a.path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, remoteName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var artifacts []models.Artifact
|
||||
for rows.Next() {
|
||||
var a models.Artifact
|
||||
if err := rows.Scan(
|
||||
&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &a.UpstreamETag,
|
||||
&a.UpstreamLastModified, &a.FirstSeenAt, &a.LastFetchedAt,
|
||||
&a.LastAccessedAt, &a.FetchCount, &a.AccessCount,
|
||||
&a.SizeBytes, &a.ContentType,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
artifacts = append(artifacts, a)
|
||||
}
|
||||
return artifacts, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) DeleteArtifact(ctx context.Context, remoteName, path string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM artifacts WHERE remote_name = $1 AND path = $2`, remoteName, path)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cacheHit bool, sizeBytes int64, upstreamMS int, clientIP string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO access_log (remote_name, path, cache_hit, size_bytes, upstream_ms, client_ip)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
`, remoteName, path, cacheHit, sizeBytes, upstreamMS, clientIP)
|
||||
return err
|
||||
}
|
||||
|
||||
// AccessLogEntry is one buffered access-log record.
|
||||
type AccessLogEntry struct {
|
||||
RemoteName string
|
||||
Path string
|
||||
CacheHit bool
|
||||
SizeBytes int64
|
||||
UpstreamMS int
|
||||
ClientIP string
|
||||
}
|
||||
|
||||
// InsertAccessLogBatch bulk-inserts access-log rows with a single COPY.
|
||||
func (db *DB) InsertAccessLogBatch(ctx context.Context, entries []AccessLogEntry) error {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows := make([][]any, len(entries))
|
||||
for i, e := range entries {
|
||||
rows[i] = []any{e.RemoteName, e.Path, e.CacheHit, e.SizeBytes, e.UpstreamMS, e.ClientIP}
|
||||
}
|
||||
_, err := db.Pool.CopyFrom(ctx,
|
||||
pgx.Identifier{"access_log"},
|
||||
[]string{"remote_name", "path", "cache_hit", "size_bytes", "upstream_ms", "client_ip"},
|
||||
pgx.CopyFromRows(rows),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// FindOrphanedBlobs returns blobs no longer referenced by any artifact or
|
||||
// local file, restricted to those created before now()-minAge. The age cutoff
|
||||
// is a grace period that avoids a TOCTOU race with in-flight dedup uploads,
|
||||
// which insert the blob row before the referencing artifact/local_files row.
|
||||
func (db *DB) FindOrphanedBlobs(ctx context.Context, minAge time.Duration) ([]models.Blob, error) {
|
||||
cutoff := time.Now().Add(-minAge)
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT b.content_hash, b.s3_key, b.size_bytes, b.content_type, b.created_at
|
||||
FROM blobs b
|
||||
WHERE b.created_at < $1
|
||||
AND b.content_hash NOT IN (
|
||||
SELECT content_hash FROM artifacts
|
||||
UNION
|
||||
SELECT content_hash FROM local_files
|
||||
)
|
||||
`, cutoff)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var blobs []models.Blob
|
||||
for rows.Next() {
|
||||
var b models.Blob
|
||||
if err := rows.Scan(&b.ContentHash, &b.S3Key, &b.SizeBytes, &b.ContentType, &b.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blobs = append(blobs, b)
|
||||
}
|
||||
return blobs, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) DeleteBlob(ctx context.Context, contentHash string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM blobs WHERE content_hash = $1`, contentHash)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteColdArtifacts(ctx context.Context, remoteName string, olderThan time.Duration) (int64, error) {
|
||||
cutoff := time.Now().Add(-olderThan)
|
||||
tag, err := db.Pool.Exec(ctx, `
|
||||
DELETE FROM artifacts
|
||||
WHERE remote_name = $1 AND last_accessed_at < $2
|
||||
`, remoteName, cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
type LocalFile struct {
|
||||
ID int64 `json:"id"`
|
||||
RepoName string `json:"repo_name"`
|
||||
FilePath string `json:"file_path"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
var ErrAlreadyExists = fmt.Errorf("file already exists")
|
||||
|
||||
func (db *DB) CreateLocalFile(ctx context.Context, repoName, filePath, contentHash string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO local_files (repo_name, file_path, content_hash)
|
||||
VALUES ($1, $2, $3)
|
||||
`, repoName, filePath, contentHash)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return ErrAlreadyExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) GetLocalFile(ctx context.Context, repoName, filePath string) (*LocalFile, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1 AND file_path = $2
|
||||
`, repoName, filePath)
|
||||
|
||||
var f LocalFile
|
||||
if err := row.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &f, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFiles(ctx context.Context, repoName string, limit, offset int) ([]LocalFile, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1
|
||||
ORDER BY file_path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, repoName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var files []LocalFile
|
||||
for rows.Next() {
|
||||
var f LocalFile
|
||||
if err := rows.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, f)
|
||||
}
|
||||
return files, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFilesByPrefix(ctx context.Context, repoName, prefix string) ([]LocalFile, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT id, repo_name, file_path, content_hash, created_at
|
||||
FROM local_files
|
||||
WHERE repo_name = $1 AND file_path LIKE $2
|
||||
ORDER BY file_path
|
||||
`, repoName, prefix+"%")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var files []LocalFile
|
||||
for rows.Next() {
|
||||
var f LocalFile
|
||||
if err := rows.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, f)
|
||||
}
|
||||
return files, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListLocalFilePackages(ctx context.Context, repoName string) ([]string, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT DISTINCT split_part(file_path, '/', 1)
|
||||
FROM local_files
|
||||
WHERE repo_name = $1
|
||||
ORDER BY 1
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var packages []string
|
||||
for rows.Next() {
|
||||
var pkg string
|
||||
if err := rows.Scan(&pkg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
return packages, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]provider.FileEntry, error) {
|
||||
files, err := db.ListLocalFilesByPrefix(ctx, repoName, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]provider.FileEntry, len(files))
|
||||
for i, f := range files {
|
||||
result[i] = provider.FileEntry{FilePath: f.FilePath, ContentHash: f.ContentHash}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListPackages(ctx context.Context, repoName string) ([]string, error) {
|
||||
return db.ListLocalFilePackages(ctx, repoName)
|
||||
}
|
||||
|
||||
func (db *DB) DeleteLocalFile(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM local_files WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type DB struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func New(dsn string) (*DB, error) {
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect to postgres: %w", err)
|
||||
}
|
||||
|
||||
if err := pool.Ping(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("ping postgres: %w", err)
|
||||
}
|
||||
|
||||
db := &DB{Pool: pool}
|
||||
if err := db.migrate(); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("run migrations: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (db *DB) Close() {
|
||||
db.Pool.Close()
|
||||
}
|
||||
|
||||
func (db *DB) migrate() error {
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS remotes (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
repo_type TEXT DEFAULT 'remote',
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
immutable_ttl INTEGER DEFAULT 0,
|
||||
mutable_ttl INTEGER DEFAULT 3600,
|
||||
check_mutable BOOLEAN DEFAULT TRUE,
|
||||
patterns TEXT[] DEFAULT '{}',
|
||||
blocklist TEXT[] DEFAULT '{}',
|
||||
mutable_patterns TEXT[] DEFAULT '{}',
|
||||
immutable_patterns TEXT[] DEFAULT '{}',
|
||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
||||
ban_tags TEXT[] DEFAULT '{}',
|
||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
||||
quarantine_days INTEGER DEFAULT 3,
|
||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
||||
releases_remote TEXT DEFAULT '',
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS virtuals (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
members TEXT[] NOT NULL,
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS blobs (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
s3_key TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
content_type TEXT DEFAULT 'application/octet-stream',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artifacts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
upstream_etag TEXT DEFAULT '',
|
||||
upstream_last_modified TIMESTAMPTZ,
|
||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
fetch_count BIGINT DEFAULT 1,
|
||||
access_count BIGINT DEFAULT 1,
|
||||
UNIQUE(remote_name, path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS local_files (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS access_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
cache_hit BOOLEAN NOT NULL,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
upstream_ms INTEGER DEFAULT 0,
|
||||
client_ip TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rpm_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
epoch INTEGER DEFAULT 0,
|
||||
version TEXT NOT NULL,
|
||||
release TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
summary TEXT DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
rpm_size BIGINT DEFAULT 0,
|
||||
installed_size BIGINT DEFAULT 0,
|
||||
license TEXT DEFAULT '',
|
||||
vendor TEXT DEFAULT '',
|
||||
build_group TEXT DEFAULT '',
|
||||
build_host TEXT DEFAULT '',
|
||||
source_rpm TEXT DEFAULT '',
|
||||
url TEXT DEFAULT '',
|
||||
packager TEXT DEFAULT '',
|
||||
requires JSONB DEFAULT '[]',
|
||||
provides JSONB DEFAULT '[]',
|
||||
files JSONB DEFAULT '[]',
|
||||
changelogs JSONB DEFAULT '[]',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
|
||||
`)
|
||||
return err
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const remoteCols = `name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
created_at, updated_at`
|
||||
|
||||
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
|
||||
return scanner.Scan(
|
||||
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Description, &r.Username, &r.Password,
|
||||
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
|
||||
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
|
||||
&r.BanTagsEnabled, &r.BanTags,
|
||||
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
|
||||
&r.ReleasesRemote, &r.ManagedBy,
|
||||
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
|
||||
&r.CreatedAt, &r.UpdatedAt,
|
||||
)
|
||||
}
|
||||
|
||||
func (db *DB) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
|
||||
row := db.Pool.QueryRow(ctx, `SELECT `+remoteCols+` FROM remotes WHERE name = $1`, name)
|
||||
var r models.Remote
|
||||
if err := scanRemote(row, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes ORDER BY name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var remotes []models.Remote
|
||||
for rows.Next() {
|
||||
var r models.Remote
|
||||
if err := scanRemote(rows, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remotes = append(remotes, r)
|
||||
}
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO remotes (
|
||||
name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24)
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE remotes SET
|
||||
package_type=$2, repo_type=$3, base_url=$4, description=$5, username=$6, password=$7,
|
||||
immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
|
||||
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
|
||||
ban_tags_enabled=$15, ban_tags=$16,
|
||||
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
|
||||
releases_remote=$20, managed_by=$21,
|
||||
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
|
||||
updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteRemote(ctx context.Context, name string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM remotes WHERE name = $1`, name)
|
||||
return err
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
|
||||
requiresJSON, _ := json.Marshal(meta.Requires)
|
||||
providesJSON, _ := json.Marshal(meta.Provides)
|
||||
filesJSON, _ := json.Marshal(meta.Files)
|
||||
changelogsJSON, _ := json.Marshal(meta.Changelogs)
|
||||
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO rpm_metadata (
|
||||
repo_name, file_path, content_hash,
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, files, changelogs
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash,
|
||||
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
|
||||
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
|
||||
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
|
||||
requiresJSON, providesJSON, filesJSON, changelogsJSON,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
type RPMMetadataRow struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Epoch int
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
Summary string
|
||||
Description string
|
||||
RPMSize int64
|
||||
InstalledSize int64
|
||||
License string
|
||||
Vendor string
|
||||
Group string
|
||||
BuildHost string
|
||||
SourceRPM string
|
||||
URL string
|
||||
Packager string
|
||||
Requires json.RawMessage
|
||||
Provides json.RawMessage
|
||||
Files json.RawMessage
|
||||
Changelogs json.RawMessage
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]provider.RPMMetadata, error) {
|
||||
rows, err := db.ListRPMMetadata(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]provider.RPMMetadata, len(rows))
|
||||
for i, r := range rows {
|
||||
meta := provider.RPMMetadata{
|
||||
RepoName: r.RepoName,
|
||||
FilePath: r.FilePath,
|
||||
ContentHash: r.ContentHash,
|
||||
Name: r.Name,
|
||||
Epoch: r.Epoch,
|
||||
Version: r.Version,
|
||||
Release: r.Release,
|
||||
Arch: r.Arch,
|
||||
Summary: r.Summary,
|
||||
Description: r.Description,
|
||||
RPMSize: r.RPMSize,
|
||||
InstalledSize: r.InstalledSize,
|
||||
License: r.License,
|
||||
Vendor: r.Vendor,
|
||||
Group: r.Group,
|
||||
BuildHost: r.BuildHost,
|
||||
SourceRPM: r.SourceRPM,
|
||||
URL: r.URL,
|
||||
Packager: r.Packager,
|
||||
}
|
||||
json.Unmarshal(r.Requires, &meta.Requires)
|
||||
json.Unmarshal(r.Provides, &meta.Provides)
|
||||
json.Unmarshal(r.Files, &meta.Files)
|
||||
json.Unmarshal(r.Changelogs, &meta.Changelogs)
|
||||
result[i] = meta
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetadataRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash,
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, files, changelogs
|
||||
FROM rpm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, epoch, version, release, arch
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []RPMMetadataRow
|
||||
for rows.Next() {
|
||||
var r RPMMetadataRow
|
||||
if err := rows.Scan(
|
||||
&r.RepoName, &r.FilePath, &r.ContentHash,
|
||||
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
|
||||
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
|
||||
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
|
||||
&r.Requires, &r.Provides, &r.Files, &r.Changelogs,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) GetOverviewStats(ctx context.Context) (*models.OverviewStats, error) {
|
||||
var stats models.OverviewStats
|
||||
|
||||
err := db.Pool.QueryRow(ctx, `SELECT COUNT(*) FROM remotes`).Scan(&stats.TotalRemotes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `SELECT COALESCE(COUNT(*), 0), COALESCE(SUM(b.size_bytes), 0)
|
||||
FROM artifacts a JOIN blobs b ON a.content_hash = b.content_hash`).
|
||||
Scan(&stats.TotalObjects, &stats.TotalBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(
|
||||
(SELECT COUNT(*) FROM artifacts) - (SELECT COUNT(DISTINCT content_hash) FROM artifacts),
|
||||
0
|
||||
)`).Scan(&stats.TotalBlobsDeduped)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(size_bytes), 0)
|
||||
FROM access_log
|
||||
WHERE cache_hit = TRUE AND created_at > NOW() - INTERVAL '30 days'
|
||||
`).Scan(&stats.BandwidthSaved30d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
type RemoteStatRow struct {
|
||||
Name string `json:"name"`
|
||||
ObjectCount int64 `json:"object_count"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
Requests30d int64 `json:"requests_30d"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopRemotes(ctx context.Context, limit int) ([]RemoteStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT r.name,
|
||||
COALESCE(a.cnt, 0) AS object_count,
|
||||
COALESCE(a.total_bytes, 0) AS total_bytes,
|
||||
COALESCE(l.req_count, 0) AS requests_30d
|
||||
FROM remotes r
|
||||
LEFT JOIN (
|
||||
SELECT remote_name, COUNT(*) AS cnt, SUM(b.size_bytes) AS total_bytes
|
||||
FROM artifacts a JOIN blobs b ON a.content_hash = b.content_hash
|
||||
GROUP BY remote_name
|
||||
) a ON r.name = a.remote_name
|
||||
LEFT JOIN (
|
||||
SELECT remote_name, COUNT(*) AS req_count
|
||||
FROM access_log
|
||||
WHERE created_at > NOW() - INTERVAL '30 days'
|
||||
GROUP BY remote_name
|
||||
) l ON r.name = l.remote_name
|
||||
ORDER BY COALESCE(a.total_bytes, 0) DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []RemoteStatRow
|
||||
for rows.Next() {
|
||||
var r RemoteStatRow
|
||||
if err := rows.Scan(&r.Name, &r.ObjectCount, &r.TotalBytes, &r.Requests30d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
type FileStatRow struct {
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
AccessCount int64 `json:"access_count"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopFilesByHits(ctx context.Context, limit int) ([]FileStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT a.remote_name, a.path, a.access_count, b.size_bytes
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
ORDER BY a.access_count DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []FileStatRow
|
||||
for rows.Next() {
|
||||
var r FileStatRow
|
||||
if err := rows.Scan(&r.RemoteName, &r.Path, &r.AccessCount, &r.SizeBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
type BandwidthStatRow struct {
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
Bandwidth int64 `json:"bandwidth"`
|
||||
Requests int64 `json:"requests"`
|
||||
}
|
||||
|
||||
func (db *DB) GetTopFilesByBandwidth(ctx context.Context, limit int) ([]BandwidthStatRow, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT remote_name, path,
|
||||
COALESCE(SUM(size_bytes), 0) AS bandwidth,
|
||||
COUNT(*) AS requests
|
||||
FROM access_log
|
||||
WHERE created_at > NOW() - INTERVAL '30 days'
|
||||
GROUP BY remote_name, path
|
||||
ORDER BY bandwidth DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []BandwidthStatRow
|
||||
for rows.Next() {
|
||||
var r BandwidthStatRow
|
||||
if err := rows.Scan(&r.RemoteName, &r.Path, &r.Bandwidth, &r.Requests); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, r)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (db *DB) GetVirtual(ctx context.Context, name string) (*models.Virtual, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
SELECT name, package_type, description, members, managed_by, created_at, updated_at
|
||||
FROM virtuals WHERE name = $1
|
||||
`, name)
|
||||
|
||||
var v models.Virtual
|
||||
err := row.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
func (db *DB) ListVirtuals(ctx context.Context) ([]models.Virtual, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT name, package_type, description, members, managed_by, created_at, updated_at
|
||||
FROM virtuals ORDER BY name
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var virtuals []models.Virtual
|
||||
for rows.Next() {
|
||||
var v models.Virtual
|
||||
if err := rows.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
virtuals = append(virtuals, v)
|
||||
}
|
||||
return virtuals, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) CreateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO virtuals (name, package_type, description, members, managed_by)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpdateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE virtuals SET
|
||||
package_type=$2, description=$3, members=$4, managed_by=$5, updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteVirtual(ctx context.Context, name string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM virtuals WHERE name = $1`, name)
|
||||
return err
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
)
|
||||
|
||||
// blobGracePeriod is how old an orphaned blob must be before GC will delete
|
||||
// it. This avoids racing in-flight dedup uploads that insert the blob row
|
||||
// before the referencing artifact/local_files row exists.
|
||||
const blobGracePeriod = 1 * time.Hour
|
||||
|
||||
type Collector struct {
|
||||
db *database.DB
|
||||
store *storage.S3
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func New(db *database.DB, store *storage.S3, interval time.Duration) *Collector {
|
||||
return &Collector{db: db, store: store, interval: interval}
|
||||
}
|
||||
|
||||
func (c *Collector) Run(ctx context.Context) {
|
||||
slog.Info("gc started", "interval", c.interval)
|
||||
ticker := time.NewTicker(c.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("gc stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
c.sweep(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Collector) sweep(ctx context.Context) {
|
||||
start := time.Now()
|
||||
|
||||
orphaned, err := c.db.FindOrphanedBlobs(ctx, blobGracePeriod)
|
||||
if err != nil {
|
||||
slog.Error("gc: find orphaned blobs", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for _, blob := range orphaned {
|
||||
if err := c.store.Delete(ctx, blob.S3Key); err != nil {
|
||||
slog.Warn("gc: delete s3 object", "key", blob.S3Key, "error", err)
|
||||
continue
|
||||
}
|
||||
if err := c.db.DeleteBlob(ctx, blob.ContentHash); err != nil {
|
||||
slog.Warn("gc: delete blob row", "hash", blob.ContentHash, "error", err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
|
||||
if deleted > 0 || len(orphaned) > 0 {
|
||||
slog.Info("gc sweep complete",
|
||||
"orphaned_found", len(orphaned),
|
||||
"deleted", deleted,
|
||||
"duration_ms", time.Since(start).Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package gc_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/gc"
|
||||
)
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
c := gc.New(nil, nil, 1*time.Hour)
|
||||
if c == nil {
|
||||
t.Fatal("expected non-nil collector")
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageAlpine }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "APKINDEX.tar.gz") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".apk") {
|
||||
return "application/vnd.android.package-archive"
|
||||
}
|
||||
if strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var (
|
||||
tagManifestRe = regexp.MustCompile(`/manifests/[^/]+$`)
|
||||
digestManifestRe = regexp.MustCompile(`/manifests/sha256:[0-9a-fA-F]+$`)
|
||||
tagsListRe = regexp.MustCompile(`/tags/list$`)
|
||||
)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageDocker }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if tagsListRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
if tagManifestRe.MatchString(path) && !digestManifestRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.Contains(path, "/blobs/") {
|
||||
return "application/octet-stream"
|
||||
}
|
||||
if strings.Contains(path, "/manifests/") {
|
||||
return "application/vnd.docker.distribution.manifest.v2+json"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/v2/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" && remote.Password != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package docker_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
if p.Type() != models.PackageDocker {
|
||||
t.Errorf("expected docker, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"library/nginx/manifests/latest", provider.Mutable},
|
||||
{"library/nginx/manifests/v1.25", provider.Mutable},
|
||||
{"library/nginx/manifests/sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", provider.Immutable},
|
||||
{"library/nginx/tags/list", provider.Mutable},
|
||||
{"library/nginx/blobs/sha256:abc123", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_UpstreamURL(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://registry-1.docker.io"}, "library/nginx/manifests/latest")
|
||||
want := "https://registry-1.docker.io/v2/library/nginx/manifests/latest"
|
||||
if got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &docker.Provider{}
|
||||
if p.ContentType("x/blobs/sha256:abc") != "application/octet-stream" {
|
||||
t.Error("blobs should be octet-stream")
|
||||
}
|
||||
if p.ContentType("x/manifests/latest") != "application/vnd.docker.distribution.manifest.v2+json" {
|
||||
t.Error("manifests should be manifest type")
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package generic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageGeneric }
|
||||
|
||||
func (p *Provider) Classify(_ string) provider.Mutability {
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
var contentTypeMap = map[string]string{
|
||||
".tar.gz": "application/gzip",
|
||||
".tgz": "application/gzip",
|
||||
".gz": "application/gzip",
|
||||
".zip": "application/zip",
|
||||
".whl": "application/zip",
|
||||
".exe": "application/x-msdownload",
|
||||
".rpm": "application/x-rpm",
|
||||
".xml": "application/xml",
|
||||
".yaml": "text/yaml",
|
||||
".yml": "text/yaml",
|
||||
".json": "application/json",
|
||||
".sig": "application/octet-stream",
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(filePath string) string {
|
||||
lower := strings.ToLower(filePath)
|
||||
if strings.HasSuffix(lower, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
ext := path.Ext(lower)
|
||||
if ct, ok := contentTypeMap[ext]; ok {
|
||||
return ct
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, reqPath string) string {
|
||||
base := strings.TrimRight(remote.BaseURL, "/")
|
||||
return base + "/" + strings.TrimLeft(reqPath, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if remote.Username != "" {
|
||||
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package generic_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
if p.Type() != models.PackageGeneric {
|
||||
t.Errorf("expected generic, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify_AllImmutable(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
paths := []string{"file.tar.gz", "path/to/binary", "index.html", "data.json"}
|
||||
for _, path := range paths {
|
||||
if p.Classify(path) != provider.Immutable {
|
||||
t.Errorf("generic should classify %q as immutable", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
tests := []struct{ path, want string }{
|
||||
{"file.tar.gz", "application/gzip"},
|
||||
{"file.tgz", "application/gzip"},
|
||||
{"file.zip", "application/zip"},
|
||||
{"file.rpm", "application/x-rpm"},
|
||||
{"file.json", "application/json"},
|
||||
{"file.unknown", "application/octet-stream"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.ContentType(tt.path); got != tt.want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_UpstreamURL(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
got := p.UpstreamURL(models.Remote{BaseURL: "https://example.com/repo"}, "path/to/file.tar.gz")
|
||||
want := "https://example.com/repo/path/to/file.tar.gz"
|
||||
if got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_AuthHeaders_BasicAuth(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "user", Password: "pass"})
|
||||
if h.Get("Authorization") != "Basic dXNlcjpwYXNz" {
|
||||
t.Errorf("unexpected auth header: %q", h.Get("Authorization"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_AuthHeaders_NoAuth(t *testing.T) {
|
||||
p := &generic.Provider{}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{})
|
||||
if h.Get("Authorization") != "" {
|
||||
t.Error("expected no auth header")
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package goproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageGoProxy }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "/@v/list") || strings.HasSuffix(path, "/@latest") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(path, ".mod") {
|
||||
return "text/plain"
|
||||
}
|
||||
if strings.HasSuffix(path, ".info") {
|
||||
return "application/json"
|
||||
}
|
||||
if strings.HasSuffix(path, "/list") {
|
||||
return "text/plain"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package goproxy_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
if p.Type() != models.PackageGoProxy {
|
||||
t.Errorf("expected goproxy, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"golang.org/x/net/@v/list", provider.Mutable},
|
||||
{"golang.org/x/net/@latest", provider.Mutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.info", provider.Immutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.mod", provider.Immutable},
|
||||
{"golang.org/x/net/@v/v0.1.0.zip", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_ContentType(t *testing.T) {
|
||||
p := &goproxy.Provider{}
|
||||
tests := []struct{ path, want string }{
|
||||
{"m/@v/v1.0.0.zip", "application/zip"},
|
||||
{"m/@v/v1.0.0.mod", "text/plain"},
|
||||
{"m/@v/v1.0.0.info", "application/json"},
|
||||
{"m/@v/list", "text/plain"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.ContentType(tt.path); got != tt.want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package helm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageHelm }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "index.yaml") || strings.HasSuffix(path, "index.yml") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tgz") || strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") {
|
||||
return "text/yaml"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
rewritten := strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
if rewritten == content {
|
||||
return nil, nil
|
||||
}
|
||||
return []byte(rewritten), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package helm_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/helm"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
if p.Type() != models.PackageHelm {
|
||||
t.Errorf("expected helm, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"index.yaml", provider.Mutable},
|
||||
{"index.yml", provider.Mutable},
|
||||
{"chart-1.0.tgz", provider.Immutable},
|
||||
{"charts/nginx-1.0.tgz", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_RewriteResponse(t *testing.T) {
|
||||
p := &helm.Provider{}
|
||||
body := []byte("urls:\n- https://charts.example.com/chart-1.0.tgz")
|
||||
remote := models.Remote{Name: "helm-test", BaseURL: "https://charts.example.com"}
|
||||
rewritten, err := p.RewriteResponse(body, remote, "https://proxy.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rewritten == nil {
|
||||
t.Fatal("expected rewrite")
|
||||
}
|
||||
if !strings.Contains(string(rewritten), "proxy.example.com/api/v1/remote/helm-test") {
|
||||
t.Errorf("expected proxy URL in body: %s", rewritten)
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package npm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageNPM }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, ".tgz") {
|
||||
return provider.Immutable
|
||||
}
|
||||
return provider.Mutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tgz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" || !json.Valid(body) {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
rewritten := strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
if rewritten == content {
|
||||
return nil, nil
|
||||
}
|
||||
return []byte(rewritten), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type Mutability int
|
||||
|
||||
const (
|
||||
Immutable Mutability = iota
|
||||
Mutable
|
||||
)
|
||||
|
||||
type Provider interface {
|
||||
Type() models.PackageType
|
||||
Classify(path string) Mutability
|
||||
ContentType(path string) string
|
||||
UpstreamURL(remote models.Remote, path string) string
|
||||
RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error)
|
||||
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
|
||||
}
|
||||
|
||||
type FileEntry struct {
|
||||
FilePath string
|
||||
ContentHash string
|
||||
}
|
||||
|
||||
type FileStore interface {
|
||||
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
|
||||
ListPackages(ctx context.Context, repoName string) ([]string, error)
|
||||
}
|
||||
|
||||
type LocalUploader interface {
|
||||
ValidateUpload(filePath string) (storagePath, contentType string, err error)
|
||||
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
|
||||
}
|
||||
|
||||
type LocalIndexer interface {
|
||||
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
|
||||
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
|
||||
}
|
||||
|
||||
type BlobReader interface {
|
||||
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
||||
}
|
||||
|
||||
type PostUploadHook interface {
|
||||
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
|
||||
}
|
||||
|
||||
type MetadataStore interface {
|
||||
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
|
||||
}
|
||||
|
||||
type RPMMetadataReader interface {
|
||||
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
|
||||
}
|
||||
|
||||
type RPMMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Epoch int
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
Summary string
|
||||
Description string
|
||||
RPMSize int64
|
||||
InstalledSize int64
|
||||
License string
|
||||
Vendor string
|
||||
Group string
|
||||
BuildHost string
|
||||
SourceRPM string
|
||||
URL string
|
||||
Packager string
|
||||
Requires []RPMDep
|
||||
Provides []RPMDep
|
||||
Files []RPMFile
|
||||
Changelogs []RPMChangelog
|
||||
}
|
||||
|
||||
type RPMDep struct {
|
||||
Name string `json:"name"`
|
||||
Flags string `json:"flags,omitempty"`
|
||||
Epoch string `json:"epoch,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Release string `json:"release,omitempty"`
|
||||
}
|
||||
|
||||
type RPMFile struct {
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
type RPMChangelog struct {
|
||||
Author string `json:"author"`
|
||||
Date int64 `json:"date"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
|
||||
type MemberIndex struct {
|
||||
RemoteName string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
var registry = map[models.PackageType]Provider{}
|
||||
|
||||
func Register(p Provider) {
|
||||
registry[p.Type()] = p
|
||||
}
|
||||
|
||||
func Get(t models.PackageType) (Provider, error) {
|
||||
p, ok := registry[t]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no provider registered for package type %q", t)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func All() map[models.PackageType]Provider {
|
||||
return registry
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package puppet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackagePuppet }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.HasPrefix(path, "v3/modules/") || strings.HasPrefix(path, "v3/releases") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.HasPrefix(path, "v3/") {
|
||||
return "application/json"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
content = strings.ReplaceAll(content, `"/v3/files/`, `"`+proxyURL+`/v3/files/`)
|
||||
baseURL := strings.TrimRight(remote.BaseURL, "/")
|
||||
content = strings.ReplaceAll(content, baseURL, proxyURL)
|
||||
return []byte(content), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
package pypi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var fileRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*\.(whl|tar\.gz|zip)$`)
|
||||
var normalizeRe = regexp.MustCompile(`[-_.]+`)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackagePyPI }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if strings.Contains(path, "simple/") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
lower := strings.ToLower(path)
|
||||
if strings.HasSuffix(lower, ".whl") || strings.HasSuffix(lower, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.gz") {
|
||||
return "application/gzip"
|
||||
}
|
||||
if strings.Contains(path, "simple/") {
|
||||
return "text/html"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
if strings.HasPrefix(path, "simple/") {
|
||||
return "https://pypi.org/" + path
|
||||
}
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
content := string(body)
|
||||
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + "/"
|
||||
content = strings.ReplaceAll(content, "https://files.pythonhosted.org/", proxyURL)
|
||||
content = strings.ReplaceAll(content, "../../", proxyURL)
|
||||
return []byte(content), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func normalize(name string) string {
|
||||
return strings.ToLower(normalizeRe.ReplaceAllString(name, "-"))
|
||||
}
|
||||
|
||||
func packageFromWheel(filename string) string {
|
||||
parts := strings.SplitN(filename, "-", 3)
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return normalize(parts[0])
|
||||
}
|
||||
|
||||
func packageFromSdist(filename string) string {
|
||||
name := filename
|
||||
for _, suffix := range []string{".tar.gz", ".zip"} {
|
||||
if strings.HasSuffix(name, suffix) {
|
||||
name = strings.TrimSuffix(name, suffix)
|
||||
break
|
||||
}
|
||||
}
|
||||
idx := strings.LastIndex(name, "-")
|
||||
if idx <= 0 {
|
||||
return ""
|
||||
}
|
||||
return normalize(name[:idx])
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !fileRe.MatchString(filename) {
|
||||
return "", "", fmt.Errorf("filename %q must be a .whl, .tar.gz, or .zip file", filename)
|
||||
}
|
||||
|
||||
var pkgName string
|
||||
if strings.HasSuffix(filename, ".whl") {
|
||||
pkgName = packageFromWheel(filename)
|
||||
} else {
|
||||
pkgName = packageFromSdist(filename)
|
||||
}
|
||||
if pkgName == "" {
|
||||
return "", "", fmt.Errorf("cannot parse package name from %q", filename)
|
||||
}
|
||||
|
||||
ct := "application/zip"
|
||||
if strings.HasSuffix(filename, ".tar.gz") {
|
||||
ct = "application/gzip"
|
||||
}
|
||||
|
||||
return pkgName + "/" + filename, ct, nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
parts := strings.SplitN(storagePath, "/", 2)
|
||||
filename := storagePath
|
||||
if len(parts) == 2 {
|
||||
filename = parts[1]
|
||||
}
|
||||
return map[string]any{
|
||||
"package": parts[0],
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
if path == "simple" || path == "simple/" {
|
||||
p.servePackageList(w, r, files, repoName)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(path, "simple/") {
|
||||
pkg := strings.TrimPrefix(path, "simple/")
|
||||
pkg = strings.TrimSuffix(pkg, "/")
|
||||
if pkg != "" && !strings.Contains(pkg, "/") {
|
||||
p.servePackageFiles(w, r, files, repoName, pkg)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
if !strings.HasPrefix(path, "simple/") {
|
||||
return nil, fmt.Errorf("unsupported index path: %q", path)
|
||||
}
|
||||
|
||||
pkg := strings.TrimPrefix(path, "simple/")
|
||||
pkg = strings.TrimSuffix(pkg, "/")
|
||||
if pkg == "" {
|
||||
return p.generatePackageListHTML(ctx, files, repoName)
|
||||
}
|
||||
return p.generatePackageFilesHTML(ctx, files, repoName, pkg)
|
||||
}
|
||||
|
||||
func (p *Provider) servePackageList(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName string) {
|
||||
body, err := p.generatePackageListHTML(r.Context(), files, repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
func (p *Provider) servePackageFiles(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, packageName string) {
|
||||
normalized := normalize(packageName)
|
||||
prefix := normalized + "/"
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
|
||||
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n",
|
||||
normalized, filename, hash, filename)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.WriteString(w, b.String())
|
||||
}
|
||||
|
||||
func (p *Provider) generatePackageListHTML(ctx context.Context, files provider.FileStore, repoName string) ([]byte, error) {
|
||||
packages, err := files.ListPackages(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, pkg := range packages {
|
||||
fmt.Fprintf(&b, "<a href=\"%s/\">%s</a>\n", pkg, pkg)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
|
||||
func (p *Provider) generatePackageFilesHTML(ctx context.Context, files provider.FileStore, repoName, packageName string) ([]byte, error) {
|
||||
normalized := normalize(packageName)
|
||||
prefix := normalized + "/"
|
||||
entries, err := files.ListFilesByPrefix(ctx, repoName, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
|
||||
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&b, "<a href=\"%s/%s#sha256=%s\">%s</a>\n",
|
||||
normalized, filename, hash, filename)
|
||||
}
|
||||
b.WriteString("</body></html>\n")
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
@@ -1,444 +0,0 @@
|
||||
package rpm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
rpmlib "github.com/cavaliergopher/rpm"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var mutableRe = []*regexp.Regexp{
|
||||
regexp.MustCompile(`repomd\.xml$`),
|
||||
regexp.MustCompile(`repodata/`),
|
||||
regexp.MustCompile(`Packages\.gz$`),
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageRPM }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
for _, re := range mutableRe {
|
||||
if re.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if strings.HasSuffix(path, ".rpm") {
|
||||
return "application/x-rpm"
|
||||
}
|
||||
if strings.HasSuffix(path, ".xml") || strings.HasSuffix(path, ".xml.gz") || strings.HasSuffix(path, ".xml.xz") {
|
||||
return "application/xml"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(strings.ToLower(filename), ".rpm") {
|
||||
return "", "", fmt.Errorf("file must be an .rpm package")
|
||||
}
|
||||
|
||||
return "Packages/" + filename, "application/x-rpm", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
filename := strings.TrimPrefix(storagePath, "Packages/")
|
||||
return map[string]any{
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
|
||||
|
||||
reader, blobSize, err := blobs.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
slog.Error("rpm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
pkg, err := rpmlib.Read(reader)
|
||||
if err != nil {
|
||||
slog.Error("rpm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
meta := &provider.RPMMetadata{
|
||||
RepoName: repoName,
|
||||
FilePath: storagePath,
|
||||
ContentHash: contentHash,
|
||||
Name: pkg.Name(),
|
||||
Epoch: pkg.Epoch(),
|
||||
Version: pkg.Version(),
|
||||
Release: pkg.Release(),
|
||||
Arch: pkg.Architecture(),
|
||||
Summary: pkg.Summary(),
|
||||
Description: pkg.Description(),
|
||||
RPMSize: blobSize,
|
||||
InstalledSize: int64(pkg.Size()),
|
||||
License: pkg.License(),
|
||||
Vendor: pkg.Vendor(),
|
||||
Group: firstGroup(pkg.Groups()),
|
||||
BuildHost: pkg.BuildHost(),
|
||||
SourceRPM: pkg.SourceRPM(),
|
||||
URL: pkg.URL(),
|
||||
Packager: pkg.Packager(),
|
||||
}
|
||||
|
||||
for _, req := range pkg.Requires() {
|
||||
meta.Requires = append(meta.Requires, rpmDepFromEntry(req))
|
||||
}
|
||||
for _, prov := range pkg.Provides() {
|
||||
meta.Provides = append(meta.Provides, rpmDepFromEntry(prov))
|
||||
}
|
||||
|
||||
if meta.Requires == nil {
|
||||
meta.Requires = []provider.RPMDep{}
|
||||
}
|
||||
if meta.Provides == nil {
|
||||
meta.Provides = []provider.RPMDep{}
|
||||
}
|
||||
meta.Files = []provider.RPMFile{}
|
||||
meta.Changelogs = []provider.RPMChangelog{}
|
||||
|
||||
if err := db.InsertRPMMetadata(ctx, meta); err != nil {
|
||||
slog.Error("rpm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("rpm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
}
|
||||
|
||||
func rpmDepFromEntry(e rpmlib.Dependency) provider.RPMDep {
|
||||
dep := provider.RPMDep{Name: e.Name()}
|
||||
if e.Flags() != 0 {
|
||||
dep.Flags = rpmFlagString(e.Flags())
|
||||
dep.Version = e.Version()
|
||||
dep.Release = e.Release()
|
||||
if e.Epoch() > 0 {
|
||||
dep.Epoch = fmt.Sprintf("%d", e.Epoch())
|
||||
}
|
||||
}
|
||||
return dep
|
||||
}
|
||||
|
||||
func rpmFlagString(f int) string {
|
||||
switch {
|
||||
case f&0x08 != 0 && f&0x04 != 0:
|
||||
return "GE"
|
||||
case f&0x02 != 0 && f&0x04 != 0:
|
||||
return "LE"
|
||||
case f&0x08 != 0:
|
||||
return "GT"
|
||||
case f&0x02 != 0:
|
||||
return "LT"
|
||||
case f&0x04 != 0:
|
||||
return "EQ"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func firstGroup(groups []string) string {
|
||||
if len(groups) > 0 {
|
||||
return groups[0]
|
||||
}
|
||||
return "Unspecified"
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
if !strings.HasPrefix(path, "repodata/") {
|
||||
return false
|
||||
}
|
||||
|
||||
rpmReader, ok := files.(provider.RPMMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "rpm metadata not available", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
tail := strings.TrimPrefix(path, "repodata/")
|
||||
|
||||
switch {
|
||||
case tail == "repomd.xml":
|
||||
p.serveRepomd(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-primary.xml.gz"):
|
||||
p.servePrimary(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-filelists.xml.gz"):
|
||||
p.serveFilelists(w, r, rpmReader, repoName)
|
||||
case strings.HasSuffix(tail, "-other.xml.gz"):
|
||||
p.serveOther(w, r, rpmReader, repoName)
|
||||
default:
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("rpm local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
primary := generatePrimaryXMLGZ(metas)
|
||||
filelists := generateFilelistsXMLGZ(metas)
|
||||
other := generateOtherXMLGZ(metas)
|
||||
|
||||
primaryHash := sha256Hex(primary)
|
||||
filelistsHash := sha256Hex(filelists)
|
||||
otherHash := sha256Hex(other)
|
||||
|
||||
repomd := generateRepomd(primaryHash, len(primary), filelistsHash, len(filelists), otherHash, len(other))
|
||||
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(repomd)
|
||||
}
|
||||
|
||||
func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generatePrimaryXMLGZ(metas))
|
||||
}
|
||||
|
||||
func (p *Provider) serveFilelists(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateFilelistsXMLGZ(metas))
|
||||
}
|
||||
|
||||
func (p *Provider) serveOther(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateOtherXMLGZ(metas))
|
||||
}
|
||||
|
||||
func generateRepomd(primaryHash string, primarySize int, filelistsHash string, filelistsSize int, otherHash string, otherSize int) []byte {
|
||||
ts := fmt.Sprintf("%d", time.Now().Unix())
|
||||
var b bytes.Buffer
|
||||
b.WriteString(xml.Header)
|
||||
b.WriteString(`<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">` + "\n")
|
||||
fmt.Fprintf(&b, " <revision>%s</revision>\n", ts)
|
||||
|
||||
writeRepomdData(&b, "primary", primaryHash, primarySize, ts)
|
||||
writeRepomdData(&b, "filelists", filelistsHash, filelistsSize, ts)
|
||||
writeRepomdData(&b, "other", otherHash, otherSize, ts)
|
||||
|
||||
b.WriteString("</repomd>\n")
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func writeRepomdData(b *bytes.Buffer, dtype, hash string, size int, ts string) {
|
||||
fmt.Fprintf(b, " <data type=\"%s\">\n", dtype)
|
||||
fmt.Fprintf(b, " <checksum type=\"sha256\">%s</checksum>\n", hash)
|
||||
fmt.Fprintf(b, " <location href=\"repodata/%s-%s.xml.gz\"/>\n", hash, dtype)
|
||||
fmt.Fprintf(b, " <timestamp>%s</timestamp>\n", ts)
|
||||
fmt.Fprintf(b, " <size>%d</size>\n", size)
|
||||
fmt.Fprintf(b, " </data>\n")
|
||||
}
|
||||
|
||||
func generatePrimaryXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package type=\"rpm\">\n")
|
||||
fmt.Fprintf(&xmlBuf, " <name>%s</name>\n", xmlEscape(m.Name))
|
||||
fmt.Fprintf(&xmlBuf, " <arch>%s</arch>\n", xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
fmt.Fprintf(&xmlBuf, " <checksum type=\"sha256\" pkgid=\"YES\">%s</checksum>\n", pkgHash)
|
||||
fmt.Fprintf(&xmlBuf, " <summary>%s</summary>\n", xmlEscape(m.Summary))
|
||||
fmt.Fprintf(&xmlBuf, " <description>%s</description>\n", xmlEscape(m.Description))
|
||||
if m.Packager != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <packager>%s</packager>\n", xmlEscape(m.Packager))
|
||||
}
|
||||
if m.URL != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <url>%s</url>\n", xmlEscape(m.URL))
|
||||
}
|
||||
fmt.Fprintf(&xmlBuf, " <time file=\"%d\" build=\"0\"/>\n", time.Now().Unix())
|
||||
fmt.Fprintf(&xmlBuf, " <size package=\"%d\" installed=\"%d\" archive=\"0\"/>\n", m.RPMSize, m.InstalledSize)
|
||||
fmt.Fprintf(&xmlBuf, " <location href=\"%s\"/>\n", xmlEscape(m.FilePath))
|
||||
fmt.Fprintf(&xmlBuf, " <format>\n")
|
||||
if m.License != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:license>%s</rpm:license>\n", xmlEscape(m.License))
|
||||
}
|
||||
if m.Vendor != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:vendor>%s</rpm:vendor>\n", xmlEscape(m.Vendor))
|
||||
}
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:group>%s</rpm:group>\n", xmlEscape(m.Group))
|
||||
if m.BuildHost != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:buildhost>%s</rpm:buildhost>\n", xmlEscape(m.BuildHost))
|
||||
}
|
||||
if m.SourceRPM != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <rpm:sourcerpm>%s</rpm:sourcerpm>\n", xmlEscape(m.SourceRPM))
|
||||
}
|
||||
|
||||
if len(m.Provides) > 0 {
|
||||
xmlBuf.WriteString(" <rpm:provides>\n")
|
||||
for _, d := range m.Provides {
|
||||
writeRPMEntry(&xmlBuf, d)
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:provides>\n")
|
||||
}
|
||||
if len(m.Requires) > 0 {
|
||||
xmlBuf.WriteString(" <rpm:requires>\n")
|
||||
for _, d := range m.Requires {
|
||||
writeRPMEntry(&xmlBuf, d)
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:requires>\n")
|
||||
}
|
||||
|
||||
fmt.Fprintf(&xmlBuf, " </format>\n")
|
||||
fmt.Fprintf(&xmlBuf, "</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</metadata>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func generateFilelistsXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<filelists xmlns=\"http://linux.duke.edu/metadata/filelists\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
for _, f := range m.Files {
|
||||
if f.Type != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <file type=\"%s\">%s</file>\n", f.Type, xmlEscape(f.Path))
|
||||
} else {
|
||||
fmt.Fprintf(&xmlBuf, " <file>%s</file>\n", xmlEscape(f.Path))
|
||||
}
|
||||
}
|
||||
xmlBuf.WriteString("</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</filelists>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func generateOtherXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
var xmlBuf bytes.Buffer
|
||||
xmlBuf.WriteString(xml.Header)
|
||||
fmt.Fprintf(&xmlBuf, "<otherdata xmlns=\"http://linux.duke.edu/metadata/other\" packages=\"%d\">\n", len(metas))
|
||||
|
||||
for _, m := range metas {
|
||||
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
|
||||
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
|
||||
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
|
||||
for _, cl := range m.Changelogs {
|
||||
fmt.Fprintf(&xmlBuf, " <changelog author=\"%s\" date=\"%d\">%s</changelog>\n",
|
||||
xmlEscape(cl.Author), cl.Date, xmlEscape(cl.Text))
|
||||
}
|
||||
xmlBuf.WriteString("</package>\n")
|
||||
}
|
||||
xmlBuf.WriteString("</otherdata>\n")
|
||||
|
||||
return gzipBytes(xmlBuf.Bytes())
|
||||
}
|
||||
|
||||
func writeRPMEntry(b *bytes.Buffer, d provider.RPMDep) {
|
||||
if d.Flags != "" {
|
||||
fmt.Fprintf(b, " <rpm:entry name=\"%s\" flags=\"%s\"", xmlEscape(d.Name), d.Flags)
|
||||
if d.Epoch != "" {
|
||||
fmt.Fprintf(b, " epoch=\"%s\"", d.Epoch)
|
||||
}
|
||||
if d.Version != "" {
|
||||
fmt.Fprintf(b, " ver=\"%s\"", xmlEscape(d.Version))
|
||||
}
|
||||
if d.Release != "" {
|
||||
fmt.Fprintf(b, " rel=\"%s\"", xmlEscape(d.Release))
|
||||
}
|
||||
b.WriteString("/>\n")
|
||||
} else {
|
||||
fmt.Fprintf(b, " <rpm:entry name=\"%s\"/>\n", xmlEscape(d.Name))
|
||||
}
|
||||
}
|
||||
|
||||
func xmlEscape(s string) string {
|
||||
var b bytes.Buffer
|
||||
xml.EscapeText(&b, []byte(s))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func gzipBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
gz.Write(data)
|
||||
gz.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func sha256Hex(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package rpm_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &rpm.Provider{}
|
||||
if p.Type() != models.PackageRPM {
|
||||
t.Errorf("expected rpm, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &rpm.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"repomd.xml", provider.Mutable},
|
||||
{"repodata/primary.xml.gz", provider.Mutable},
|
||||
{"Packages.gz", provider.Mutable},
|
||||
{"package-1.0.rpm", provider.Immutable},
|
||||
{"RPM-GPG-KEY-almalinux", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
package terraform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
var versionsRe = regexp.MustCompile(`[^/]+/[^/]+/versions$`)
|
||||
|
||||
var providerZipRe = regexp.MustCompile(
|
||||
`^terraform-provider-([a-zA-Z0-9_-]+)_([0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`,
|
||||
)
|
||||
|
||||
var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageTerraform }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if versionsRe.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
lower := strings.ToLower(path)
|
||||
if strings.HasSuffix(lower, ".zip") {
|
||||
return "application/zip"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".sig") {
|
||||
return "application/octet-stream"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/v1/providers/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if remote.ReleasesRemote == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if !json.Valid(body) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var data map[string]any
|
||||
if err := json.Unmarshal(body, &data); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
changed := false
|
||||
for _, field := range []string{"download_url", "shasums_url", "shasums_signature_url"} {
|
||||
if val, ok := data[field].(string); ok && val != "" {
|
||||
rewritten := rewriteDownloadURL(val, remote.ReleasesRemote, proxyBaseURL)
|
||||
if rewritten != val {
|
||||
data[field] = rewritten
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return nil, nil
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
|
||||
func rewriteDownloadURL(originalURL, releasesRemote, proxyBaseURL string) string {
|
||||
parsed, err := url.Parse(originalURL)
|
||||
if err != nil || proxyBaseURL == "" {
|
||||
return originalURL
|
||||
}
|
||||
return strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + releasesRemote + parsed.Path
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
parts := strings.Split(filePath, "/")
|
||||
if len(parts) != 3 {
|
||||
return "", "", fmt.Errorf("path must be {namespace}/{type}/{filename}.zip")
|
||||
}
|
||||
namespace, typeName, filename := parts[0], parts[1], parts[2]
|
||||
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil {
|
||||
return "", "", fmt.Errorf("filename %q does not match terraform-provider-{type}_{version}_{os}_{arch}.zip", filename)
|
||||
}
|
||||
|
||||
if m[1] != typeName {
|
||||
return "", "", fmt.Errorf("provider type in filename %q does not match path type %q", m[1], typeName)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s/%s/%s", namespace, typeName, filename), "application/zip", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
parts := strings.Split(storagePath, "/")
|
||||
if len(parts) != 3 {
|
||||
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
|
||||
}
|
||||
|
||||
m := providerZipRe.FindStringSubmatch(parts[2])
|
||||
if m == nil {
|
||||
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"namespace": parts[0],
|
||||
"type": parts[1],
|
||||
"version": m[2],
|
||||
"os": m[3],
|
||||
"arch": m[4],
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
type terraformIndex struct {
|
||||
Versions map[string]json.RawMessage `json:"versions"`
|
||||
}
|
||||
|
||||
type terraformVersionDoc struct {
|
||||
Archives map[string]terraformArchive `json:"archives"`
|
||||
}
|
||||
|
||||
type terraformArchive struct {
|
||||
URL string `json:"url"`
|
||||
Hashes []string `json:"hashes,omitempty"`
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) < 3 {
|
||||
return false
|
||||
}
|
||||
|
||||
namespace, typeName := parts[0], parts[1]
|
||||
tail := parts[2]
|
||||
|
||||
if tail == "index.json" {
|
||||
p.serveIndex(w, r, files, repoName, namespace, typeName)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasSuffix(tail, ".json") {
|
||||
version := strings.TrimSuffix(tail, ".json")
|
||||
if semverRe.MatchString(version) {
|
||||
p.serveVersionDoc(w, r, files, repoName, namespace, typeName, version)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("terraform local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
func (p *Provider) serveIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName string) {
|
||||
prefix := fmt.Sprintf("%s/%s/", namespace, typeName)
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
versions := map[string]json.RawMessage{}
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, prefix)
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
versions[m[2]] = json.RawMessage(`{}`)
|
||||
}
|
||||
|
||||
if len(versions) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(terraformIndex{Versions: versions})
|
||||
}
|
||||
|
||||
func (p *Provider) serveVersionDoc(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName, version string) {
|
||||
prefix := fmt.Sprintf("%s/%s/terraform-provider-%s_%s_", namespace, typeName, typeName, version)
|
||||
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
archives := map[string]terraformArchive{}
|
||||
for _, f := range entries {
|
||||
filename := strings.TrimPrefix(f.FilePath, fmt.Sprintf("%s/%s/", namespace, typeName))
|
||||
m := providerZipRe.FindStringSubmatch(filename)
|
||||
if m == nil || m[2] != version {
|
||||
continue
|
||||
}
|
||||
platform := m[3] + "_" + m[4]
|
||||
archive := terraformArchive{URL: filename}
|
||||
if f.ContentHash != "" {
|
||||
archive.Hashes = []string{"zh:" + strings.TrimPrefix(f.ContentHash, "sha256:")}
|
||||
}
|
||||
archives[platform] = archive
|
||||
}
|
||||
|
||||
if len(archives) == 0 {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(terraformVersionDoc{Archives: archives})
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package terraform_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestProvider_Type(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
if p.Type() != models.PackageTerraform {
|
||||
t.Errorf("expected terraform, got %q", p.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_Classify(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"hashicorp/vault/versions", provider.Mutable},
|
||||
{"hashicorp/vault/0.28.0/download/linux/amd64", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_RewriteResponse_DownloadInfo(t *testing.T) {
|
||||
p := &terraform.Provider{}
|
||||
remote := models.Remote{Name: "tf", ReleasesRemote: "hashicorp-releases"}
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"download_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/file.zip",
|
||||
"shasums_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/SHA256SUMS",
|
||||
})
|
||||
rewritten, err := p.RewriteResponse(body, remote, "https://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rewritten == nil {
|
||||
t.Fatal("expected rewrite")
|
||||
}
|
||||
var result map[string]any
|
||||
json.Unmarshal(rewritten, &result)
|
||||
if !strings.Contains(result["download_url"].(string), "proxy/api/v1/remote/hashicorp-releases") {
|
||||
t.Errorf("download_url not rewritten: %s", result["download_url"])
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultCircuitThreshold = 5
|
||||
defaultCircuitCooldown = 60 * time.Second
|
||||
)
|
||||
|
||||
type CircuitBreaker struct {
|
||||
cache *cache.Redis
|
||||
threshold int64
|
||||
cooldown time.Duration
|
||||
}
|
||||
|
||||
func NewCircuitBreaker(c *cache.Redis) *CircuitBreaker {
|
||||
return &CircuitBreaker{
|
||||
cache: c,
|
||||
threshold: defaultCircuitThreshold,
|
||||
cooldown: defaultCircuitCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) IsOpen(ctx context.Context, remote string) bool {
|
||||
failures, err := cb.cache.GetCircuitFailures(ctx, remote)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return failures >= cb.threshold
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) RecordFailure(ctx context.Context, remote string) {
|
||||
cb.cache.IncrCircuitFailure(ctx, remote, cb.cooldown)
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) RecordSuccess(ctx context.Context, remote string) {
|
||||
cb.cache.ResetCircuit(ctx, remote)
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) Health(ctx context.Context, remote string) models.RemoteHealth {
|
||||
failures, err := cb.cache.GetCircuitFailures(ctx, remote)
|
||||
if err != nil {
|
||||
return models.RemoteHealth{Status: "unknown"}
|
||||
}
|
||||
|
||||
switch {
|
||||
case failures == 0:
|
||||
return models.RemoteHealth{Status: "healthy", ConsecutiveFailures: int(failures)}
|
||||
case failures < cb.threshold:
|
||||
return models.RemoteHealth{Status: "degraded", ConsecutiveFailures: int(failures)}
|
||||
default:
|
||||
return models.RemoteHealth{Status: "down", ConsecutiveFailures: int(failures)}
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package proxy_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
)
|
||||
|
||||
func TestCircuitBreaker_New(t *testing.T) {
|
||||
cb := proxy.NewCircuitBreaker(nil)
|
||||
if cb == nil {
|
||||
t.Fatal("expected non-nil circuit breaker")
|
||||
}
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sync"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type Classification int
|
||||
|
||||
const (
|
||||
ClassImmutable Classification = iota
|
||||
ClassMutable
|
||||
ClassDenied
|
||||
)
|
||||
|
||||
func (c Classification) String() string {
|
||||
switch c {
|
||||
case ClassImmutable:
|
||||
return "immutable"
|
||||
case ClassMutable:
|
||||
return "mutable"
|
||||
case ClassDenied:
|
||||
return "denied"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
type Classifier struct {
|
||||
provider provider.Provider
|
||||
}
|
||||
|
||||
func NewClassifier(p provider.Provider) *Classifier {
|
||||
return &Classifier{provider: p}
|
||||
}
|
||||
|
||||
func (c *Classifier) Classify(remote models.Remote, path string) Classification {
|
||||
if matchesAny(path, compilePatterns(remote.Blocklist)) {
|
||||
return ClassDenied
|
||||
}
|
||||
|
||||
if len(remote.Patterns) > 0 && !matchesAny(path, compilePatterns(remote.Patterns)) {
|
||||
return ClassDenied
|
||||
}
|
||||
|
||||
if matchesAny(path, compilePatterns(remote.ImmutablePatterns)) {
|
||||
return ClassImmutable
|
||||
}
|
||||
|
||||
if matchesAny(path, compilePatterns(remote.MutablePatterns)) {
|
||||
return ClassMutable
|
||||
}
|
||||
|
||||
if c.provider.Classify(path) == provider.Mutable {
|
||||
return ClassMutable
|
||||
}
|
||||
|
||||
return ClassImmutable
|
||||
}
|
||||
|
||||
// patternCache memoises regex compilation. Classify runs on every proxied
|
||||
// request and previously recompiled each remote's pattern lists every time;
|
||||
// keying by the pattern string lets each distinct pattern compile once and
|
||||
// then be reused, with no invalidation needed (the pattern text is the key).
|
||||
// A pattern that fails to compile is cached as a typed nil so we don't retry.
|
||||
var patternCache sync.Map // map[string]*regexp.Regexp
|
||||
|
||||
func compileCached(pattern string) *regexp.Regexp {
|
||||
if v, ok := patternCache.Load(pattern); ok {
|
||||
return v.(*regexp.Regexp)
|
||||
}
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
re = nil
|
||||
}
|
||||
patternCache.Store(pattern, re)
|
||||
return re
|
||||
}
|
||||
|
||||
func compilePatterns(patterns []string) []*regexp.Regexp {
|
||||
compiled := make([]*regexp.Regexp, 0, len(patterns))
|
||||
for _, p := range patterns {
|
||||
if re := compileCached(p); re != nil {
|
||||
compiled = append(compiled, re)
|
||||
}
|
||||
}
|
||||
return compiled
|
||||
}
|
||||
|
||||
func matchesAny(path string, patterns []*regexp.Regexp) bool {
|
||||
for _, re := range patterns {
|
||||
if re.MatchString(path) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package proxy_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/helm"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestClassifier_EmptyPatternsAllowsAll(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{Name: "test"}
|
||||
if c.Classify(remote, "any/path") == proxy.ClassDenied {
|
||||
t.Error("empty patterns should allow all paths")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_PatternsActAsAllowlist(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{
|
||||
Name: "test",
|
||||
Patterns: []string{`^releases/`},
|
||||
}
|
||||
if c.Classify(remote, "releases/v1.0/app.tar.gz") == proxy.ClassDenied {
|
||||
t.Error("path matching patterns should be allowed")
|
||||
}
|
||||
if c.Classify(remote, "uploads/other.tar.gz") != proxy.ClassDenied {
|
||||
t.Error("path not matching patterns should be denied")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_BlocklistDenies(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{
|
||||
Name: "test",
|
||||
Blocklist: []string{`\.exe$`},
|
||||
}
|
||||
if c.Classify(remote, "malware.exe") != proxy.ClassDenied {
|
||||
t.Error("blocklist match should deny")
|
||||
}
|
||||
if c.Classify(remote, "legit.tar.gz") == proxy.ClassDenied {
|
||||
t.Error("non-blocked path should be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_BlocklistBeforePatterns(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{
|
||||
Name: "test",
|
||||
Patterns: []string{`^releases/`},
|
||||
Blocklist: []string{`releases/v0\.1/`},
|
||||
}
|
||||
if c.Classify(remote, "releases/v0.1/app.tar.gz") != proxy.ClassDenied {
|
||||
t.Error("blocklist should take priority")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_GenericAllImmutable(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{Name: "test"}
|
||||
if c.Classify(remote, "any/file.tar.gz") != proxy.ClassImmutable {
|
||||
t.Error("generic provider should classify everything as immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_GenericMutableOverride(t *testing.T) {
|
||||
c := proxy.NewClassifier(&generic.Provider{})
|
||||
remote := models.Remote{
|
||||
Name: "test",
|
||||
MutablePatterns: []string{`/archive/refs/heads/`},
|
||||
}
|
||||
if c.Classify(remote, "repo/archive/refs/heads/main.tar.gz") != proxy.ClassMutable {
|
||||
t.Error("mutable_patterns should override provider default")
|
||||
}
|
||||
if c.Classify(remote, "repo/releases/v1.0.tar.gz") != proxy.ClassImmutable {
|
||||
t.Error("non-mutable path should stay immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_ImmutableOverride(t *testing.T) {
|
||||
c := proxy.NewClassifier(&helm.Provider{})
|
||||
remote := models.Remote{
|
||||
Name: "test",
|
||||
ImmutablePatterns: []string{`special-index\.yaml$`},
|
||||
}
|
||||
if c.Classify(remote, "special-index.yaml") != proxy.ClassImmutable {
|
||||
t.Error("immutable_patterns should force immutable even for normally mutable paths")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_HelmAutoClassifies(t *testing.T) {
|
||||
c := proxy.NewClassifier(&helm.Provider{})
|
||||
remote := models.Remote{Name: "test"}
|
||||
if c.Classify(remote, "index.yaml") != proxy.ClassMutable {
|
||||
t.Error("helm should auto-classify index.yaml as mutable")
|
||||
}
|
||||
if c.Classify(remote, "chart-1.0.tgz") != proxy.ClassImmutable {
|
||||
t.Error("helm should auto-classify .tgz as immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_DockerAutoClassifies(t *testing.T) {
|
||||
c := proxy.NewClassifier(&docker.Provider{})
|
||||
remote := models.Remote{Name: "test"}
|
||||
if c.Classify(remote, "library/nginx/manifests/latest") != proxy.ClassMutable {
|
||||
t.Error("docker should classify tag manifest as mutable")
|
||||
}
|
||||
if c.Classify(remote, "library/nginx/manifests/sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890") != proxy.ClassImmutable {
|
||||
t.Error("docker should classify digest manifest as immutable")
|
||||
}
|
||||
if c.Classify(remote, "library/nginx/blobs/sha256:abc") != proxy.ClassImmutable {
|
||||
t.Error("docker should classify blobs as immutable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifier_RPMAutoClassifies(t *testing.T) {
|
||||
c := proxy.NewClassifier(&rpm.Provider{})
|
||||
remote := models.Remote{Name: "test"}
|
||||
if c.Classify(remote, "repodata/primary.xml.gz") != proxy.ClassMutable {
|
||||
t.Error("rpm should classify repodata as mutable")
|
||||
}
|
||||
if c.Classify(remote, "packages/foo-1.0.rpm") != proxy.ClassImmutable {
|
||||
t.Error("rpm should classify .rpm as immutable")
|
||||
}
|
||||
}
|
||||
@@ -1,651 +0,0 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const fetchLockTTL = 30 * time.Second
|
||||
|
||||
const (
|
||||
accessLogBufferSize = 4096
|
||||
accessLogBatchSize = 128
|
||||
accessLogFlushEvery = 2 * time.Second
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
db *database.DB
|
||||
cache *cache.Redis
|
||||
store *storage.S3
|
||||
cas *storage.CAS
|
||||
circuit *CircuitBreaker
|
||||
accessLog chan database.AccessLogEntry
|
||||
}
|
||||
|
||||
func NewEngine(db *database.DB, c *cache.Redis, s *storage.S3) *Engine {
|
||||
e := &Engine{
|
||||
db: db,
|
||||
cache: c,
|
||||
store: s,
|
||||
cas: storage.NewCAS(s),
|
||||
circuit: NewCircuitBreaker(c),
|
||||
accessLog: make(chan database.AccessLogEntry, accessLogBufferSize),
|
||||
}
|
||||
go e.runAccessLogWriter()
|
||||
return e
|
||||
}
|
||||
|
||||
// runAccessLogWriter drains the access-log channel and writes rows in batches,
|
||||
// replacing a goroutine-per-request insert. It runs for the process lifetime;
|
||||
// access logs are best-effort telemetry, so a small tail may be lost on abrupt
|
||||
// shutdown.
|
||||
func (e *Engine) runAccessLogWriter() {
|
||||
ticker := time.NewTicker(accessLogFlushEvery)
|
||||
defer ticker.Stop()
|
||||
|
||||
batch := make([]database.AccessLogEntry, 0, accessLogBatchSize)
|
||||
flush := func() {
|
||||
if len(batch) == 0 {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if err := e.db.InsertAccessLogBatch(ctx, batch); err != nil {
|
||||
slog.Warn("access log batch insert failed", "error", err, "count", len(batch))
|
||||
}
|
||||
cancel()
|
||||
batch = batch[:0]
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case entry := <-e.accessLog:
|
||||
batch = append(batch, entry)
|
||||
if len(batch) >= accessLogBatchSize {
|
||||
flush()
|
||||
}
|
||||
case <-ticker.C:
|
||||
flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type FetchResult struct {
|
||||
Reader io.ReadCloser
|
||||
ContentType string
|
||||
Size int64
|
||||
Source string // "cache" or "remote"
|
||||
}
|
||||
|
||||
func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, prov provider.Provider, clientHeaders ...http.Header) (*FetchResult, error) {
|
||||
classifier := NewClassifier(prov)
|
||||
class := classifier.Classify(remote, path)
|
||||
|
||||
if class == ClassDenied {
|
||||
return nil, &ProxyError{Status: http.StatusForbidden, Message: "access denied"}
|
||||
}
|
||||
|
||||
ttl := e.ttlFor(remote, class)
|
||||
|
||||
fresh, err := e.cache.CheckTTL(ctx, remote.Name, path)
|
||||
if err != nil {
|
||||
slog.Warn("redis check failed, treating as miss", "error", err)
|
||||
}
|
||||
|
||||
if fresh {
|
||||
result, err := e.serveFromStore(ctx, remote, path)
|
||||
if err == nil {
|
||||
result.Source = "cache"
|
||||
e.logAccess(remote.Name, path, true, result.Size, 0)
|
||||
return result, nil
|
||||
}
|
||||
slog.Warn("cache hit but S3 miss, re-fetching", "remote", remote.Name, "path", path)
|
||||
}
|
||||
|
||||
locked, err := e.cache.AcquireLock(ctx, remote.Name, path, fetchLockTTL)
|
||||
if err != nil {
|
||||
slog.Warn("lock acquire failed", "error", err)
|
||||
}
|
||||
|
||||
if !locked {
|
||||
// Another request holds the fetch lock. Poll the store until the leader
|
||||
// populates it rather than immediately racing to fetch upstream too; a
|
||||
// cold-cache stampede otherwise hits upstream once per waiter.
|
||||
if result := e.waitForStore(ctx, remote, path); result != nil {
|
||||
result.Source = "cache"
|
||||
e.logAccess(remote.Name, path, true, result.Size, 0)
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
if locked {
|
||||
defer e.cache.ReleaseLock(ctx, remote.Name, path)
|
||||
}
|
||||
|
||||
if class == ClassMutable && remote.CheckMutable {
|
||||
etag, _ := e.cache.GetETag(ctx, remote.Name, path)
|
||||
if etag != "" {
|
||||
notModified, err := e.checkUpstream(ctx, remote, path, etag, prov)
|
||||
if err == nil && notModified {
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
|
||||
result, err := e.serveFromStore(ctx, remote, path)
|
||||
if err == nil {
|
||||
result.Source = "cache"
|
||||
e.logAccess(remote.Name, path, true, result.Size, 0)
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var fwdHeaders http.Header
|
||||
if len(clientHeaders) > 0 && clientHeaders[0] != nil {
|
||||
fwdHeaders = clientHeaders[0]
|
||||
}
|
||||
|
||||
// Short-circuit upstream calls when the remote's breaker is open: serve
|
||||
// stale from the store if we have it, otherwise fail fast rather than
|
||||
// hammering a known-bad upstream.
|
||||
if e.circuit.IsOpen(ctx, remote.Name) {
|
||||
if stale, serr := e.serveFromStore(ctx, remote, path); serr == nil {
|
||||
slog.Warn("circuit open, serving stale", "remote", remote.Name, "path", path)
|
||||
stale.Source = "cache"
|
||||
e.logAccess(remote.Name, path, true, stale.Size, 0)
|
||||
return stale, nil
|
||||
}
|
||||
return nil, &ProxyError{Status: http.StatusServiceUnavailable, Message: "upstream circuit open"}
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
result, err := e.fetchFromUpstream(ctx, remote, path, prov, class, ttl, fwdHeaders)
|
||||
upstreamMS := int(time.Since(start).Milliseconds())
|
||||
if err != nil {
|
||||
if isNetworkError(err) {
|
||||
e.circuit.RecordFailure(ctx, remote.Name)
|
||||
}
|
||||
if remote.StaleOnError && isNetworkError(err) {
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
stale, serr := e.serveFromStore(ctx, remote, path)
|
||||
if serr == nil {
|
||||
slog.Warn("serving stale on upstream error", "remote", remote.Name, "path", path, "error", err)
|
||||
stale.Source = "cache"
|
||||
e.logAccess(remote.Name, path, true, stale.Size, 0)
|
||||
return stale, nil
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
e.circuit.RecordSuccess(ctx, remote.Name)
|
||||
e.logAccess(remote.Name, path, false, result.Size, upstreamMS)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// HeadResult carries artifact metadata for a HEAD request. There is no body.
|
||||
type HeadResult struct {
|
||||
ContentType string
|
||||
Size int64
|
||||
Source string // "cache" or "remote"
|
||||
}
|
||||
|
||||
// Head resolves artifact metadata without fetching or streaming the body.
|
||||
// Cached artifacts/indexes are answered from the store metadata; on a miss it
|
||||
// issues an upstream HEAD. It never downloads or caches the body.
|
||||
func (e *Engine) Head(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*HeadResult, error) {
|
||||
class := NewClassifier(prov).Classify(remote, path)
|
||||
if class == ClassDenied {
|
||||
return nil, &ProxyError{Status: http.StatusForbidden, Message: "access denied"}
|
||||
}
|
||||
|
||||
if artifact, err := e.db.GetArtifact(ctx, remote.Name, path); err == nil && artifact != nil {
|
||||
return &HeadResult{ContentType: artifact.ContentType, Size: artifact.SizeBytes, Source: "cache"}, nil
|
||||
}
|
||||
if info, err := e.store.Stat(ctx, storage.IndexKey(remote.Name, path)); err == nil {
|
||||
return &HeadResult{ContentType: info.ContentType, Size: info.Size, Source: "cache"}, nil
|
||||
}
|
||||
|
||||
return e.headUpstream(ctx, remote, path, prov)
|
||||
}
|
||||
|
||||
func (e *Engine) headUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*HeadResult, error) {
|
||||
url := prov.UpstreamURL(remote, path)
|
||||
|
||||
authHeaders, err := prov.AuthHeaders(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("auth headers: %w", err)
|
||||
}
|
||||
|
||||
doHead := func(extra http.Header) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
for k, vv := range authHeaders {
|
||||
for _, v := range vv {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
for k, vv := range extra {
|
||||
for _, v := range vv {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
}
|
||||
return http.DefaultClient.Do(req)
|
||||
}
|
||||
|
||||
resp, err := doHead(nil)
|
||||
if err != nil {
|
||||
return nil, &UpstreamError{Err: err}
|
||||
}
|
||||
if resp.StatusCode == http.StatusUnauthorized {
|
||||
resp.Body.Close()
|
||||
token, _, terr := fetchBearerToken(ctx, resp.Header.Get("Www-Authenticate"), remote)
|
||||
if terr == nil && token != "" {
|
||||
resp, err = doHead(http.Header{"Authorization": []string{"Bearer " + token}})
|
||||
if err != nil {
|
||||
return nil, &UpstreamError{Err: err}
|
||||
}
|
||||
} else {
|
||||
return nil, &ProxyError{Status: http.StatusUnauthorized, Message: "upstream returned 401"}
|
||||
}
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, &ProxyError{Status: resp.StatusCode, Message: fmt.Sprintf("upstream returned %d", resp.StatusCode)}
|
||||
}
|
||||
|
||||
contentType := prov.ContentType(path)
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "" {
|
||||
contentType = ct
|
||||
}
|
||||
return &HeadResult{ContentType: contentType, Size: resp.ContentLength, Source: "remote"}, nil
|
||||
}
|
||||
|
||||
func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration, clientHeaders http.Header) (*FetchResult, error) {
|
||||
url := prov.UpstreamURL(remote, path)
|
||||
|
||||
authHeaders, err := prov.AuthHeaders(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("auth headers: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
for k, vv := range authHeaders {
|
||||
for _, v := range vv {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
if clientHeaders != nil {
|
||||
if accept := clientHeaders.Get("Accept"); accept != "" {
|
||||
req.Header.Set("Accept", accept)
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := clientForRemote(remote).Do(req)
|
||||
if err != nil {
|
||||
return nil, &UpstreamError{Err: err}
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusUnauthorized {
|
||||
resp.Body.Close()
|
||||
token, err := e.cachedBearerToken(ctx, resp.Header.Get("Www-Authenticate"), remote)
|
||||
if err == nil && token != "" {
|
||||
req2, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
req2.Header.Set("Authorization", "Bearer "+token)
|
||||
if clientHeaders != nil {
|
||||
if accept := clientHeaders.Get("Accept"); accept != "" {
|
||||
req2.Header.Set("Accept", accept)
|
||||
}
|
||||
}
|
||||
resp, err = clientForRemote(remote).Do(req2)
|
||||
if err != nil {
|
||||
return nil, &UpstreamError{Err: err}
|
||||
}
|
||||
} else {
|
||||
return nil, &ProxyError{Status: http.StatusUnauthorized, Message: "upstream returned 401"}
|
||||
}
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
return nil, &ProxyError{Status: resp.StatusCode, Message: fmt.Sprintf("upstream returned %d", resp.StatusCode)}
|
||||
}
|
||||
|
||||
contentType := prov.ContentType(path)
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "" {
|
||||
contentType = ct
|
||||
}
|
||||
|
||||
// Mutable indexes are small and may be rewritten, so buffer them in memory.
|
||||
if class == ClassMutable {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read upstream body: %w", err)
|
||||
}
|
||||
|
||||
rewritten, err := prov.RewriteResponse(body, remote, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rewrite response: %w", err)
|
||||
}
|
||||
if rewritten != nil {
|
||||
body = rewritten
|
||||
}
|
||||
|
||||
s3Key := storage.IndexKey(remote.Name, path)
|
||||
if err := e.store.Upload(ctx, s3Key, bytesReader(body), int64(len(body)), contentType); err != nil {
|
||||
return nil, fmt.Errorf("upload index: %w", err)
|
||||
}
|
||||
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
if etag := resp.Header.Get("ETag"); etag != "" {
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
|
||||
}
|
||||
|
||||
return &FetchResult{
|
||||
Reader: io.NopCloser(bytesReader(body)),
|
||||
ContentType: contentType,
|
||||
Size: int64(len(body)),
|
||||
Source: "remote",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Immutable blobs are streamed through the content-addressable store
|
||||
// (tempfile -> sha256 -> S3) so arbitrarily large artifacts never sit
|
||||
// fully in memory. Immutable content is never rewritten in the proxy path.
|
||||
casResult, err := e.cas.Store(ctx, resp.Body, contentType)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("store blob: %w", err)
|
||||
}
|
||||
|
||||
if err := e.db.UpsertBlob(ctx, casResult.ContentHash, casResult.S3Key, casResult.SizeBytes, contentType); err != nil {
|
||||
slog.Warn("upsert blob failed", "error", err)
|
||||
}
|
||||
if err := e.db.UpsertArtifact(ctx, remote.Name, path, casResult.ContentHash, resp.Header.Get("ETag")); err != nil {
|
||||
slog.Warn("upsert artifact failed", "error", err)
|
||||
}
|
||||
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
if etag := resp.Header.Get("ETag"); etag != "" {
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
|
||||
}
|
||||
|
||||
reader, info, err := e.store.Download(ctx, casResult.S3Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serve stored blob: %w", err)
|
||||
}
|
||||
return &FetchResult{
|
||||
Reader: reader,
|
||||
ContentType: info.ContentType,
|
||||
Size: casResult.SizeBytes,
|
||||
Source: "remote",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// waitForStore polls the store for an artifact populated by the request that
|
||||
// holds the fetch lock, returning it once available or nil if it does not
|
||||
// appear within the wait budget (after which the caller fetches upstream
|
||||
// itself). It stops early if the request context is cancelled.
|
||||
func (e *Engine) waitForStore(ctx context.Context, remote models.Remote, path string) *FetchResult {
|
||||
const (
|
||||
pollInterval = 100 * time.Millisecond
|
||||
maxWait = 5 * time.Second
|
||||
)
|
||||
deadline := time.Now().Add(maxWait)
|
||||
for {
|
||||
if result, err := e.serveFromStore(ctx, remote, path); err == nil {
|
||||
return result
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(pollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) serveFromStore(ctx context.Context, remote models.Remote, path string) (*FetchResult, error) {
|
||||
artifact, err := e.db.GetArtifact(ctx, remote.Name, path)
|
||||
if err == nil && artifact != nil {
|
||||
s3Key := storage.BlobKey(artifact.ContentHash[len("sha256:"):])
|
||||
reader, info, err := e.store.Download(ctx, s3Key)
|
||||
if err == nil {
|
||||
_ = e.db.TouchArtifactAccess(ctx, remote.Name, path)
|
||||
return &FetchResult{
|
||||
Reader: reader,
|
||||
ContentType: info.ContentType,
|
||||
Size: info.Size,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
s3Key := storage.IndexKey(remote.Name, path)
|
||||
reader, info, err := e.store.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("not in store: %w", err)
|
||||
}
|
||||
return &FetchResult{
|
||||
Reader: reader,
|
||||
ContentType: info.ContentType,
|
||||
Size: info.Size,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *Engine) checkUpstream(ctx context.Context, remote models.Remote, path, etag string, prov provider.Provider) (bool, error) {
|
||||
url := prov.UpstreamURL(remote, path)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
|
||||
authHeaders, err := prov.AuthHeaders(ctx, remote)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for k, vv := range authHeaders {
|
||||
for _, v := range vv {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := clientForRemote(remote).Do(req)
|
||||
if err != nil {
|
||||
return false, &UpstreamError{Err: err}
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
return resp.StatusCode == http.StatusNotModified, nil
|
||||
}
|
||||
|
||||
func (e *Engine) ttlFor(remote models.Remote, class Classification) time.Duration {
|
||||
switch class {
|
||||
case ClassImmutable:
|
||||
if remote.ImmutableTTL == 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(remote.ImmutableTTL) * time.Second
|
||||
default:
|
||||
return time.Duration(remote.MutableTTL) * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
// logAccess enqueues an access-log entry for the batch writer. It never blocks
|
||||
// the request path: if the buffer is full the entry is dropped.
|
||||
func (e *Engine) logAccess(remoteName, path string, cacheHit bool, size int64, upstreamMS int) {
|
||||
select {
|
||||
case e.accessLog <- database.AccessLogEntry{
|
||||
RemoteName: remoteName,
|
||||
Path: path,
|
||||
CacheHit: cacheHit,
|
||||
SizeBytes: size,
|
||||
UpstreamMS: upstreamMS,
|
||||
}:
|
||||
default:
|
||||
slog.Warn("access log buffer full, dropping entry", "remote", remoteName, "path", path)
|
||||
}
|
||||
}
|
||||
|
||||
func bytesReader(data []byte) io.Reader {
|
||||
return io.NewSectionReader(readerAt(data), 0, int64(len(data)))
|
||||
}
|
||||
|
||||
type readerAt []byte
|
||||
|
||||
func (r readerAt) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
if off >= int64(len(r)) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n = copy(p, r[off:])
|
||||
if off+int64(n) >= int64(len(r)) {
|
||||
err = io.EOF
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// bearerTokenTTLDefault/Margin bound how long a token is cached: the default
|
||||
// is used when the token endpoint omits expires_in, and the margin is
|
||||
// subtracted so a cached token is refreshed slightly before it actually expires.
|
||||
const (
|
||||
bearerTokenTTLDefault = 60 * time.Second
|
||||
bearerTokenTTLMargin = 10 * time.Second
|
||||
)
|
||||
|
||||
func sha256Hash(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// cachedBearerToken returns a bearer token for the given challenge, reusing a
|
||||
// Redis-cached token for the same remote+challenge while it is still valid.
|
||||
func (e *Engine) cachedBearerToken(ctx context.Context, wwwAuth string, remote models.Remote) (string, error) {
|
||||
key := remote.Name + ":" + sha256Hash([]byte(wwwAuth))
|
||||
if tok, err := e.cache.GetToken(ctx, key); err == nil && tok != "" {
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
tok, ttl, err := fetchBearerToken(ctx, wwwAuth, remote)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if tok != "" {
|
||||
if ttl <= 0 {
|
||||
ttl = bearerTokenTTLDefault
|
||||
}
|
||||
if ttl > bearerTokenTTLMargin {
|
||||
ttl -= bearerTokenTTLMargin
|
||||
}
|
||||
_ = e.cache.SetToken(ctx, key, tok, ttl)
|
||||
}
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
func fetchBearerToken(ctx context.Context, wwwAuth string, remote models.Remote) (string, time.Duration, error) {
|
||||
if !strings.HasPrefix(wwwAuth, "Bearer ") {
|
||||
return "", 0, fmt.Errorf("not a Bearer challenge")
|
||||
}
|
||||
|
||||
params := map[string]string{}
|
||||
for _, part := range strings.Split(wwwAuth[7:], ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
eq := strings.Index(part, "=")
|
||||
if eq < 0 {
|
||||
continue
|
||||
}
|
||||
key := part[:eq]
|
||||
val := strings.Trim(part[eq+1:], `"`)
|
||||
params[key] = val
|
||||
}
|
||||
|
||||
realm := params["realm"]
|
||||
if realm == "" {
|
||||
return "", 0, fmt.Errorf("no realm in Bearer challenge")
|
||||
}
|
||||
|
||||
tokenURL := realm
|
||||
sep := "?"
|
||||
if s, ok := params["service"]; ok {
|
||||
tokenURL += sep + "service=" + s
|
||||
sep = "&"
|
||||
}
|
||||
if s, ok := params["scope"]; ok {
|
||||
tokenURL += sep + "scope=" + s
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL, nil)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
if remote.Username != "" && remote.Password != "" {
|
||||
req.SetBasicAuth(remote.Username, remote.Password)
|
||||
}
|
||||
|
||||
resp, err := clientForRemote(remote).Do(req)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", 0, fmt.Errorf("token endpoint returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var tokenResp struct {
|
||||
Token string `json:"token"`
|
||||
AccessToken string `json:"access_token"`
|
||||
ExpiresIn int `json:"expires_in"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
ttl := time.Duration(tokenResp.ExpiresIn) * time.Second
|
||||
if tokenResp.Token != "" {
|
||||
return tokenResp.Token, ttl, nil
|
||||
}
|
||||
return tokenResp.AccessToken, ttl, nil
|
||||
}
|
||||
|
||||
type ProxyError struct {
|
||||
Status int
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *ProxyError) Error() string { return e.Message }
|
||||
|
||||
type UpstreamError struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *UpstreamError) Error() string { return fmt.Sprintf("upstream error: %v", e.Err) }
|
||||
func (e *UpstreamError) Unwrap() error { return e.Err }
|
||||
|
||||
func isNetworkError(err error) bool {
|
||||
var ue *UpstreamError
|
||||
return errors.As(err, &ue)
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// Default upstream timeouts. A remote may override any of these; a zero
|
||||
// override falls back to the default here. There is deliberately no overall
|
||||
// Client.Timeout: the proxy streams arbitrarily large artifacts and total time
|
||||
// is bounded by the request context instead. We only constrain the phases that
|
||||
// must never hang — connect, TLS handshake, and time-to-first-response-header —
|
||||
// so a slow or wedged upstream cannot pin a goroutine and connection.
|
||||
const (
|
||||
defaultDialTimeout = 10 * time.Second
|
||||
defaultTLSTimeout = 10 * time.Second
|
||||
defaultResponseHeaderTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
type clientKey struct {
|
||||
dial time.Duration
|
||||
tls time.Duration
|
||||
respHeader time.Duration
|
||||
}
|
||||
|
||||
var (
|
||||
clientCacheMu sync.Mutex
|
||||
clientCache = map[clientKey]*http.Client{}
|
||||
)
|
||||
|
||||
// upstreamClientFor returns an HTTP client configured with the given timeouts,
|
||||
// reusing a cached client (and its connection pool) for identical timeout sets.
|
||||
// Zero values fall back to the defaults.
|
||||
func upstreamClientFor(dial, tls, respHeader time.Duration) *http.Client {
|
||||
if dial <= 0 {
|
||||
dial = defaultDialTimeout
|
||||
}
|
||||
if tls <= 0 {
|
||||
tls = defaultTLSTimeout
|
||||
}
|
||||
if respHeader <= 0 {
|
||||
respHeader = defaultResponseHeaderTimeout
|
||||
}
|
||||
key := clientKey{dial: dial, tls: tls, respHeader: respHeader}
|
||||
|
||||
clientCacheMu.Lock()
|
||||
defer clientCacheMu.Unlock()
|
||||
if c, ok := clientCache[key]; ok {
|
||||
return c
|
||||
}
|
||||
|
||||
c := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: dial,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: tls,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
ResponseHeaderTimeout: respHeader,
|
||||
},
|
||||
}
|
||||
clientCache[key] = c
|
||||
return c
|
||||
}
|
||||
|
||||
// clientForRemote returns the upstream client for a remote, applying its
|
||||
// per-remote timeout overrides (in seconds) on top of the defaults.
|
||||
func clientForRemote(remote models.Remote) *http.Client {
|
||||
return upstreamClientFor(
|
||||
time.Duration(remote.UpstreamDialTimeout)*time.Second,
|
||||
time.Duration(remote.UpstreamTLSTimeout)*time.Second,
|
||||
time.Duration(remote.UpstreamResponseHeaderTimeout)*time.Second,
|
||||
)
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
)
|
||||
|
||||
func cors(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func NewStructuredLogger() func(next http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
|
||||
|
||||
defer func() {
|
||||
slog.Info("request",
|
||||
"method", r.Method,
|
||||
"path", r.URL.Path,
|
||||
"status", ww.Status(),
|
||||
"bytes", ww.BytesWritten(),
|
||||
"duration_ms", time.Since(start).Milliseconds(),
|
||||
"remote", r.RemoteAddr,
|
||||
"request_id", middleware.GetReqID(r.Context()),
|
||||
)
|
||||
}()
|
||||
|
||||
next.ServeHTTP(ww, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
|
||||
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/internal/config"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/gc"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
cfg *config.Config
|
||||
version string
|
||||
router chi.Router
|
||||
db *database.DB
|
||||
cache *cache.Redis
|
||||
store *storage.S3
|
||||
engine *proxy.Engine
|
||||
virtEngine *virtual.Engine
|
||||
localHandler *v2.LocalHandler
|
||||
gc *gc.Collector
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, version string) (*Server, error) {
|
||||
db, err := database.New(cfg.DatabaseDSN())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("database: %w", err)
|
||||
}
|
||||
|
||||
redis, err := cache.NewRedis(cfg.RedisURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("redis: %w", err)
|
||||
}
|
||||
|
||||
s3, err := storage.NewS3(cfg.S3Endpoint, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3Bucket, cfg.S3Secure, cfg.S3Region)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("s3: %w", err)
|
||||
}
|
||||
|
||||
engine := proxy.NewEngine(db, redis, s3)
|
||||
localHandler := v2.NewLocalHandler(db, s3)
|
||||
virtEngine := virtual.NewEngine(db, engine)
|
||||
collector := gc.New(db, s3, 1*time.Hour)
|
||||
|
||||
s := &Server{
|
||||
cfg: cfg,
|
||||
version: version,
|
||||
db: db,
|
||||
cache: redis,
|
||||
store: s3,
|
||||
engine: engine,
|
||||
virtEngine: virtEngine,
|
||||
localHandler: localHandler,
|
||||
gc: collector,
|
||||
}
|
||||
|
||||
s.router = s.routes()
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Server) routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
|
||||
r.Use(middleware.RequestID)
|
||||
r.Use(middleware.RealIP)
|
||||
r.Use(NewStructuredLogger())
|
||||
r.Use(middleware.Recoverer)
|
||||
|
||||
r.Use(cors)
|
||||
|
||||
r.Get("/health", s.handleHealth)
|
||||
r.Get("/", s.handleRoot)
|
||||
|
||||
proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler)
|
||||
r.Mount("/api/v1", proxyHandler.Routes())
|
||||
r.Mount("/v2", proxyHandler.DockerV2Routes())
|
||||
|
||||
remotesHandler := v2.NewRemotesHandler(s.db)
|
||||
virtualsHandler := v2.NewVirtualsHandler(s.db)
|
||||
healthHandler := v2.NewHealthHandler(s.db, s.cache, s.store)
|
||||
statsHandler := v2.NewStatsHandler(s.db)
|
||||
eventsHandler := v2.NewEventsHandler()
|
||||
probeHandler := v2.NewProbeHandler(s.engine, s.db)
|
||||
|
||||
r.Route("/api/v2", func(r chi.Router) {
|
||||
r.Mount("/remotes", remotesHandler.Routes())
|
||||
r.Mount("/virtuals", virtualsHandler.Routes())
|
||||
r.Mount("/health", healthHandler.Routes())
|
||||
r.Mount("/stats", statsHandler.Routes())
|
||||
r.Mount("/events", eventsHandler.Routes())
|
||||
r.Mount("/probe", probeHandler.Routes())
|
||||
|
||||
r.Route("/remotes/{name}/objects", func(r chi.Router) {
|
||||
objHandler := v2.NewObjectsHandler(s.db)
|
||||
r.Get("/", objHandler.Routes().ServeHTTP)
|
||||
r.Delete("/*", objHandler.Routes().ServeHTTP)
|
||||
})
|
||||
|
||||
r.Route("/remotes/{name}/files", func(r chi.Router) {
|
||||
r.Put("/*", s.localHandler.Routes().ServeHTTP)
|
||||
r.Get("/*", s.localHandler.Routes().ServeHTTP)
|
||||
r.Delete("/*", s.localHandler.Routes().ServeHTTP)
|
||||
})
|
||||
})
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprint(w, `{"status":"ok"}`)
|
||||
}
|
||||
|
||||
func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, `{"name":"artifactapi","version":"%s"}`, s.version)
|
||||
}
|
||||
|
||||
func (s *Server) newHTTPServer() *http.Server {
|
||||
return &http.Server{
|
||||
Addr: s.cfg.ListenAddr,
|
||||
Handler: s.router,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 300 * time.Second,
|
||||
IdleTimeout: 120 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) Run(ctx context.Context) error {
|
||||
go s.gc.Run(ctx)
|
||||
|
||||
httpServer := s.newHTTPServer()
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
slog.Info("shutting down server")
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = httpServer.Shutdown(shutdownCtx)
|
||||
}()
|
||||
|
||||
slog.Info("starting server", "addr", s.cfg.ListenAddr)
|
||||
if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) RunOnListener(ctx context.Context, ln net.Listener) error {
|
||||
go s.gc.Run(ctx)
|
||||
|
||||
httpServer := s.newHTTPServer()
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = httpServer.Shutdown(shutdownCtx)
|
||||
}()
|
||||
|
||||
slog.Info("starting server", "addr", ln.Addr().String())
|
||||
if err := httpServer.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type CAS struct {
|
||||
s3 *S3
|
||||
}
|
||||
|
||||
func NewCAS(s3 *S3) *CAS {
|
||||
return &CAS{s3: s3}
|
||||
}
|
||||
|
||||
type CASResult struct {
|
||||
ContentHash string
|
||||
S3Key string
|
||||
SizeBytes int64
|
||||
AlreadyExists bool
|
||||
}
|
||||
|
||||
func (c *CAS) Store(ctx context.Context, reader io.Reader, contentType string) (*CASResult, error) {
|
||||
tmp, err := os.CreateTemp("", "artifact-*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
defer tmp.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
size, err := io.Copy(io.MultiWriter(tmp, hasher), reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("write temp file: %w", err)
|
||||
}
|
||||
|
||||
hash := hex.EncodeToString(hasher.Sum(nil))
|
||||
s3Key := BlobKey(hash)
|
||||
|
||||
exists, err := c.s3.Exists(ctx, s3Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("check blob exists: %w", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, fmt.Errorf("seek temp file: %w", err)
|
||||
}
|
||||
if err := c.s3.Upload(ctx, s3Key, tmp, size, contentType); err != nil {
|
||||
return nil, fmt.Errorf("upload blob: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &CASResult{
|
||||
ContentHash: fmt.Sprintf("sha256:%s", hash),
|
||||
S3Key: s3Key,
|
||||
SizeBytes: size,
|
||||
AlreadyExists: exists,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func BlobKey(hash string) string {
|
||||
return fmt.Sprintf("blobs/sha256/%s", hash)
|
||||
}
|
||||
|
||||
func IndexKey(remote, path string) string {
|
||||
return fmt.Sprintf("indexes/%s/%s", remote, path)
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type S3 struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
func NewS3(endpoint, accessKey, secretKey, bucket string, secure bool, region string) (*S3, error) {
|
||||
opts := &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: secure,
|
||||
}
|
||||
if region != "" {
|
||||
opts.Region = region
|
||||
}
|
||||
|
||||
client, err := minio.New(endpoint, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create s3 client: %w", err)
|
||||
}
|
||||
|
||||
s := &S3{client: client, bucket: bucket}
|
||||
|
||||
if err := s.ensureBucket(context.Background()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *S3) ensureBucket(ctx context.Context) error {
|
||||
exists, err := s.client.BucketExists(ctx, s.bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
return fmt.Errorf("create bucket: %w", err)
|
||||
}
|
||||
slog.Info("created bucket", "bucket", s.bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3) Upload(ctx context.Context, key string, reader io.Reader, size int64, contentType string) error {
|
||||
_, err := s.client.PutObject(ctx, s.bucket, key, reader, size, minio.PutObjectOptions{
|
||||
ContentType: contentType,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3) Download(ctx context.Context, key string) (io.ReadCloser, *minio.ObjectInfo, error) {
|
||||
obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
info, err := obj.Stat()
|
||||
if err != nil {
|
||||
obj.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return obj, &info, nil
|
||||
}
|
||||
|
||||
func (s *S3) Exists(ctx context.Context, key string) (bool, error) {
|
||||
_, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
resp := minio.ToErrorResponse(err)
|
||||
if resp.Code == "NoSuchKey" {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *S3) Delete(ctx context.Context, key string) error {
|
||||
return s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (s *S3) Stat(ctx context.Context, key string) (*minio.ObjectInfo, error) {
|
||||
info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &info, nil
|
||||
}
|
||||
@@ -1,305 +0,0 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/client"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type view int
|
||||
|
||||
const (
|
||||
viewDashboard view = iota
|
||||
viewRemotes
|
||||
viewRemoteDetail
|
||||
viewObjects
|
||||
viewVirtuals
|
||||
)
|
||||
|
||||
type model struct {
|
||||
client *client.Client
|
||||
view view
|
||||
width int
|
||||
height int
|
||||
err error
|
||||
loading bool
|
||||
|
||||
stats *models.OverviewStats
|
||||
remotes []models.Remote
|
||||
virtuals []models.Virtual
|
||||
objects []models.Artifact
|
||||
|
||||
selectedRemote string
|
||||
cursor int
|
||||
page int
|
||||
}
|
||||
|
||||
func New(endpoint string) *model {
|
||||
return &model{
|
||||
client: client.New(endpoint),
|
||||
view: viewDashboard,
|
||||
loading: true,
|
||||
page: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) Run() error {
|
||||
p := tea.NewProgram(m, tea.WithAltScreen())
|
||||
_, err := p.Run()
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *model) Init() tea.Cmd {
|
||||
return m.loadDashboard()
|
||||
}
|
||||
|
||||
type dashboardLoaded struct {
|
||||
stats *models.OverviewStats
|
||||
remotes []models.Remote
|
||||
virtuals []models.Virtual
|
||||
}
|
||||
|
||||
type remotesLoaded struct{ remotes []models.Remote }
|
||||
type virtualsLoaded struct{ virtuals []models.Virtual }
|
||||
type objectsLoaded struct{ objects []models.Artifact }
|
||||
type errMsg struct{ err error }
|
||||
|
||||
func (m *model) loadDashboard() tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
ctx := context.Background()
|
||||
stats, err := m.client.Stats(ctx)
|
||||
if err != nil {
|
||||
return errMsg{err}
|
||||
}
|
||||
remotes, _ := m.client.ListRemotes(ctx)
|
||||
virtuals, _ := m.client.ListVirtuals(ctx)
|
||||
return dashboardLoaded{stats: stats, remotes: remotes, virtuals: virtuals}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) loadRemotes() tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
remotes, err := m.client.ListRemotes(context.Background())
|
||||
if err != nil {
|
||||
return errMsg{err}
|
||||
}
|
||||
return remotesLoaded{remotes}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) loadVirtuals() tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
virtuals, err := m.client.ListVirtuals(context.Background())
|
||||
if err != nil {
|
||||
return errMsg{err}
|
||||
}
|
||||
return virtualsLoaded{virtuals}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) loadObjects() tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
objects, err := m.client.ListObjects(context.Background(), m.selectedRemote, m.page, 30)
|
||||
if err != nil {
|
||||
return errMsg{err}
|
||||
}
|
||||
return objectsLoaded{objects}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.WindowSizeMsg:
|
||||
m.width = msg.Width
|
||||
m.height = msg.Height
|
||||
return m, nil
|
||||
|
||||
case tea.KeyMsg:
|
||||
return m.handleKey(msg)
|
||||
|
||||
case dashboardLoaded:
|
||||
m.loading = false
|
||||
m.stats = msg.stats
|
||||
m.remotes = msg.remotes
|
||||
m.virtuals = msg.virtuals
|
||||
return m, nil
|
||||
|
||||
case remotesLoaded:
|
||||
m.loading = false
|
||||
m.remotes = msg.remotes
|
||||
m.cursor = 0
|
||||
return m, nil
|
||||
|
||||
case virtualsLoaded:
|
||||
m.loading = false
|
||||
m.virtuals = msg.virtuals
|
||||
m.cursor = 0
|
||||
return m, nil
|
||||
|
||||
case objectsLoaded:
|
||||
m.loading = false
|
||||
m.objects = msg.objects
|
||||
m.cursor = 0
|
||||
return m, nil
|
||||
|
||||
case errMsg:
|
||||
m.loading = false
|
||||
m.err = msg.err
|
||||
return m, nil
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *model) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.String() {
|
||||
case "q", "ctrl+c":
|
||||
if m.view == viewDashboard {
|
||||
return m, tea.Quit
|
||||
}
|
||||
m.view = viewDashboard
|
||||
m.cursor = 0
|
||||
m.loading = true
|
||||
return m, m.loadDashboard()
|
||||
|
||||
case "esc":
|
||||
switch m.view {
|
||||
case viewRemoteDetail, viewObjects:
|
||||
m.view = viewRemotes
|
||||
m.cursor = 0
|
||||
m.loading = true
|
||||
return m, m.loadRemotes()
|
||||
case viewRemotes, viewVirtuals:
|
||||
m.view = viewDashboard
|
||||
m.cursor = 0
|
||||
m.loading = true
|
||||
return m, m.loadDashboard()
|
||||
default:
|
||||
return m, tea.Quit
|
||||
}
|
||||
|
||||
case "1":
|
||||
m.view = viewDashboard
|
||||
m.loading = true
|
||||
return m, m.loadDashboard()
|
||||
|
||||
case "2":
|
||||
m.view = viewRemotes
|
||||
m.loading = true
|
||||
return m, m.loadRemotes()
|
||||
|
||||
case "3":
|
||||
m.view = viewVirtuals
|
||||
m.loading = true
|
||||
return m, m.loadVirtuals()
|
||||
|
||||
case "j", "down":
|
||||
m.cursor++
|
||||
m.clampCursor()
|
||||
return m, nil
|
||||
|
||||
case "k", "up":
|
||||
if m.cursor > 0 {
|
||||
m.cursor--
|
||||
}
|
||||
return m, nil
|
||||
|
||||
case "enter":
|
||||
return m.handleEnter()
|
||||
|
||||
case "r":
|
||||
m.loading = true
|
||||
switch m.view {
|
||||
case viewDashboard:
|
||||
return m, m.loadDashboard()
|
||||
case viewRemotes:
|
||||
return m, m.loadRemotes()
|
||||
case viewVirtuals:
|
||||
return m, m.loadVirtuals()
|
||||
case viewObjects:
|
||||
return m, m.loadObjects()
|
||||
}
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *model) handleEnter() (tea.Model, tea.Cmd) {
|
||||
switch m.view {
|
||||
case viewRemotes:
|
||||
if m.cursor < len(m.remotes) {
|
||||
m.selectedRemote = m.remotes[m.cursor].Name
|
||||
m.view = viewRemoteDetail
|
||||
return m, nil
|
||||
}
|
||||
case viewRemoteDetail:
|
||||
m.view = viewObjects
|
||||
m.page = 1
|
||||
m.loading = true
|
||||
return m, m.loadObjects()
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *model) clampCursor() {
|
||||
max := 0
|
||||
switch m.view {
|
||||
case viewRemotes:
|
||||
max = len(m.remotes) - 1
|
||||
case viewVirtuals:
|
||||
max = len(m.virtuals) - 1
|
||||
case viewObjects:
|
||||
max = len(m.objects) - 1
|
||||
}
|
||||
if m.cursor > max {
|
||||
m.cursor = max
|
||||
}
|
||||
if m.cursor < 0 {
|
||||
m.cursor = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) View() string {
|
||||
if m.loading {
|
||||
return m.chrome("Loading...")
|
||||
}
|
||||
if m.err != nil {
|
||||
return m.chrome(errStyle.Render(fmt.Sprintf("Error: %v", m.err)))
|
||||
}
|
||||
|
||||
var body string
|
||||
switch m.view {
|
||||
case viewDashboard:
|
||||
body = m.viewDashboard()
|
||||
case viewRemotes:
|
||||
body = m.viewRemotesList()
|
||||
case viewRemoteDetail:
|
||||
body = m.viewRemoteDetail()
|
||||
case viewObjects:
|
||||
body = m.viewObjectsList()
|
||||
case viewVirtuals:
|
||||
body = m.viewVirtualsList()
|
||||
}
|
||||
|
||||
return m.chrome(body)
|
||||
}
|
||||
|
||||
func (m *model) chrome(body string) string {
|
||||
nav := navStyle.Render(
|
||||
"[1] Dashboard [2] Remotes [3] Virtuals │ [r] Refresh [q] Quit",
|
||||
)
|
||||
return lipgloss.JoinVertical(lipgloss.Left, body, "", nav)
|
||||
}
|
||||
|
||||
var (
|
||||
titleStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("12"))
|
||||
navStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8"))
|
||||
errStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("9"))
|
||||
mutedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8"))
|
||||
selStyle = lipgloss.NewStyle().Background(lipgloss.Color("4")).Foreground(lipgloss.Color("15"))
|
||||
)
|
||||
@@ -1,140 +0,0 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/tui/views"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (m *model) viewDashboard() string {
|
||||
return titleStyle.Render("ArtifactAPI Dashboard") + "\n\n" +
|
||||
views.RenderDashboard(m.stats, len(m.remotes), len(m.virtuals)) +
|
||||
"\n\n" + mutedStyle.Render("Press [2] for remotes, [3] for virtuals")
|
||||
}
|
||||
|
||||
func (m *model) viewRemotesList() string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(titleStyle.Render("Remotes") + "\n\n")
|
||||
|
||||
if len(m.remotes) == 0 {
|
||||
sb.WriteString(mutedStyle.Render("No remotes configured"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
for i, r := range m.remotes {
|
||||
line := fmt.Sprintf(" %-25s %-12s %s", r.Name, r.PackageType, r.Description)
|
||||
if i == m.cursor {
|
||||
sb.WriteString(selStyle.Render(line))
|
||||
} else {
|
||||
sb.WriteString(line)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
sb.WriteString("\n" + mutedStyle.Render("j/k navigate · enter detail · esc back"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func (m *model) viewRemoteDetail() string {
|
||||
var r *remoteView
|
||||
for i := range m.remotes {
|
||||
if m.remotes[i].Name == m.selectedRemote {
|
||||
r = &remoteView{m.remotes[i]}
|
||||
break
|
||||
}
|
||||
}
|
||||
if r == nil {
|
||||
return mutedStyle.Render("Remote not found")
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString(titleStyle.Render(r.Name) + "\n\n")
|
||||
sb.WriteString(fmt.Sprintf(" Type: %s\n", r.PackageType))
|
||||
sb.WriteString(fmt.Sprintf(" Base URL: %s\n", r.BaseURL))
|
||||
sb.WriteString(fmt.Sprintf(" Description: %s\n", r.Description))
|
||||
sb.WriteString(fmt.Sprintf(" Immutable TTL: %s\n", ttlStr(r.ImmutableTTL)))
|
||||
sb.WriteString(fmt.Sprintf(" Mutable TTL: %ds\n", r.MutableTTL))
|
||||
sb.WriteString(fmt.Sprintf(" Revalidation: %v\n", r.CheckMutable))
|
||||
sb.WriteString(fmt.Sprintf(" Stale on Error: %v\n", r.StaleOnError))
|
||||
|
||||
if len(r.Patterns) > 0 {
|
||||
sb.WriteString(fmt.Sprintf(" Patterns: %s\n", strings.Join(r.Patterns, ", ")))
|
||||
}
|
||||
if len(r.Blocklist) > 0 {
|
||||
sb.WriteString(fmt.Sprintf(" Blocklist: %s\n", strings.Join(r.Blocklist, ", ")))
|
||||
}
|
||||
if r.ManagedBy != "" {
|
||||
sb.WriteString(fmt.Sprintf(" Managed by: %s\n", r.ManagedBy))
|
||||
}
|
||||
|
||||
sb.WriteString("\n" + mutedStyle.Render("enter → browse objects · esc back"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func (m *model) viewObjectsList() string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(titleStyle.Render(fmt.Sprintf("Objects: %s (page %d)", m.selectedRemote, m.page)) + "\n\n")
|
||||
|
||||
if len(m.objects) == 0 {
|
||||
sb.WriteString(mutedStyle.Render("No cached objects"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
for i, a := range m.objects {
|
||||
size := views.FormatBytes(a.SizeBytes)
|
||||
line := fmt.Sprintf(" %-50s %10s %5d hits", truncate(a.Path, 50), size, a.AccessCount)
|
||||
if i == m.cursor {
|
||||
sb.WriteString(selStyle.Render(line))
|
||||
} else {
|
||||
sb.WriteString(line)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
sb.WriteString("\n" + mutedStyle.Render("j/k navigate · esc back"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func (m *model) viewVirtualsList() string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(titleStyle.Render("Virtual Repositories") + "\n\n")
|
||||
|
||||
if len(m.virtuals) == 0 {
|
||||
sb.WriteString(mutedStyle.Render("No virtual repositories configured"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
for i, v := range m.virtuals {
|
||||
line := fmt.Sprintf(" %-25s %-12s %d members %s",
|
||||
v.Name, v.PackageType, len(v.Members), v.Description)
|
||||
if i == m.cursor {
|
||||
sb.WriteString(selStyle.Render(line))
|
||||
} else {
|
||||
sb.WriteString(line)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
sb.WriteString("\n" + mutedStyle.Render("j/k navigate · esc back"))
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
type remoteView struct {
|
||||
models.Remote
|
||||
}
|
||||
|
||||
func ttlStr(ttl int) string {
|
||||
if ttl == 0 {
|
||||
return "forever"
|
||||
}
|
||||
return fmt.Sprintf("%ds", ttl)
|
||||
}
|
||||
|
||||
func truncate(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max-3] + "..."
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package views
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func FormatBytes(bytes int64) string {
|
||||
if bytes == 0 {
|
||||
return "0 B"
|
||||
}
|
||||
units := []string{"B", "KB", "MB", "GB", "TB"}
|
||||
i := 0
|
||||
b := float64(bytes)
|
||||
for b >= 1024 && i < len(units)-1 {
|
||||
b /= 1024
|
||||
i++
|
||||
}
|
||||
if i == 0 {
|
||||
return fmt.Sprintf("%.0f %s", b, units[i])
|
||||
}
|
||||
return fmt.Sprintf("%.1f %s", b, units[i])
|
||||
}
|
||||
|
||||
func RenderDashboard(stats *models.OverviewStats, remoteCount, virtualCount int) string {
|
||||
if stats == nil {
|
||||
return "No stats available"
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"╭─ Dashboard ──────────────────────────────╮\n"+
|
||||
"│ Remotes: %-24d│\n"+
|
||||
"│ Cached Objects: %-24d│\n"+
|
||||
"│ Storage Used: %-24s│\n"+
|
||||
"│ Dedup Savings: %-20d blobs │\n"+
|
||||
"│ Virtuals: %-24d│\n"+
|
||||
"╰──────────────────────────────────────────╯",
|
||||
stats.TotalRemotes,
|
||||
stats.TotalObjects,
|
||||
FormatBytes(stats.TotalBytes),
|
||||
stats.TotalBlobsDeduped,
|
||||
virtualCount,
|
||||
)
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
db *database.DB
|
||||
proxyEngine *proxy.Engine
|
||||
}
|
||||
|
||||
func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
|
||||
return &Engine{db: db, proxyEngine: proxyEngine}
|
||||
}
|
||||
|
||||
func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) {
|
||||
merger, err := GetMerger(virt.PackageType)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("unsupported virtual type %q: %w", virt.PackageType, err)
|
||||
}
|
||||
|
||||
members, err := e.fetchMemberIndexes(ctx, virt, path)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
if len(members) == 0 {
|
||||
return nil, "", fmt.Errorf("no members reachable for virtual %q", virt.Name)
|
||||
}
|
||||
|
||||
merged, err := merger.MergeIndexes(members, proxyBaseURL)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("merge indexes: %w", err)
|
||||
}
|
||||
|
||||
contentType := "application/octet-stream"
|
||||
switch virt.PackageType {
|
||||
case models.PackageHelm:
|
||||
contentType = "text/yaml"
|
||||
case models.PackagePyPI:
|
||||
contentType = "text/html"
|
||||
}
|
||||
|
||||
return merged, contentType, nil
|
||||
}
|
||||
|
||||
func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, path string) ([]MemberIndex, error) {
|
||||
type result struct {
|
||||
index MemberIndex
|
||||
err error
|
||||
}
|
||||
|
||||
results := make([]result, len(virt.Members))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i, memberName := range virt.Members {
|
||||
wg.Add(1)
|
||||
go func(idx int, name string) {
|
||||
defer wg.Done()
|
||||
|
||||
remote, err := e.db.GetRemote(ctx, name)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("remote %q: %w", name, err)}
|
||||
return
|
||||
}
|
||||
|
||||
if remote.RepoType == models.RepoTypeLocal {
|
||||
body, err := e.fetchLocalIndex(ctx, *remote, path)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("local index %q: %w", name, err)}
|
||||
return
|
||||
}
|
||||
results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, BaseURL: remote.BaseURL, Body: body}}
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("provider %q: %w", remote.PackageType, err)}
|
||||
return
|
||||
}
|
||||
|
||||
fetchResult, err := e.proxyEngine.Fetch(ctx, *remote, path, prov)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("fetch %q/%s: %w", name, path, err)}
|
||||
return
|
||||
}
|
||||
defer fetchResult.Reader.Close()
|
||||
|
||||
body, err := io.ReadAll(fetchResult.Reader)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("read %q: %w", name, err)}
|
||||
return
|
||||
}
|
||||
|
||||
results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, BaseURL: remote.BaseURL, Body: body}}
|
||||
}(i, memberName)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
var members []MemberIndex
|
||||
for _, r := range results {
|
||||
if r.err != nil {
|
||||
slog.Warn("virtual member fetch failed", "error", r.err)
|
||||
continue
|
||||
}
|
||||
members = append(members, r.index)
|
||||
}
|
||||
|
||||
return members, nil
|
||||
}
|
||||
|
||||
func (e *Engine) fetchLocalIndex(ctx context.Context, remote models.Remote, path string) ([]byte, error) {
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no provider for %q: %w", remote.PackageType, err)
|
||||
}
|
||||
|
||||
indexer, ok := prov.(provider.LocalIndexer)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("provider %q does not support local index generation", remote.PackageType)
|
||||
}
|
||||
|
||||
return indexer.GenerateLocalIndex(ctx, e.db, remote.Name, path)
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterMerger(models.PackageHelm, &HelmMerger{})
|
||||
}
|
||||
|
||||
type HelmMerger struct{}
|
||||
|
||||
type helmIndex struct {
|
||||
APIVersion string `yaml:"apiVersion"`
|
||||
Entries map[string][]helmChartVersion `yaml:"entries"`
|
||||
Generated string `yaml:"generated,omitempty"`
|
||||
}
|
||||
|
||||
type helmChartVersion struct {
|
||||
Name string `yaml:"name"`
|
||||
Version string `yaml:"version"`
|
||||
URLs []string `yaml:"urls"`
|
||||
rest map[string]any
|
||||
}
|
||||
|
||||
func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
|
||||
merged := &helmIndex{
|
||||
APIVersion: "v1",
|
||||
Entries: make(map[string][]helmChartVersion),
|
||||
}
|
||||
|
||||
seen := map[string]map[string]bool{}
|
||||
|
||||
for _, member := range members {
|
||||
var idx helmIndex
|
||||
if err := yaml.Unmarshal(member.Body, &idx); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for chart, versions := range idx.Entries {
|
||||
if seen[chart] == nil {
|
||||
seen[chart] = map[string]bool{}
|
||||
}
|
||||
for _, ver := range versions {
|
||||
key := chart + ":" + ver.Version
|
||||
if seen[chart][ver.Version] {
|
||||
continue
|
||||
}
|
||||
seen[chart][ver.Version] = true
|
||||
|
||||
if proxyBaseURL != "" {
|
||||
routePrefix := "remote"
|
||||
if member.RepoType == "local" {
|
||||
routePrefix = "local"
|
||||
}
|
||||
baseHost := extractHost(member.BaseURL)
|
||||
|
||||
for i, u := range ver.URLs {
|
||||
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
|
||||
if baseHost != "" && extractHost(u) != baseHost {
|
||||
continue
|
||||
}
|
||||
relPath := extractPathRelativeToBase(u, member.BaseURL)
|
||||
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
|
||||
strings.TrimRight(proxyBaseURL, "/"),
|
||||
routePrefix,
|
||||
member.RemoteName,
|
||||
relPath)
|
||||
} else {
|
||||
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
|
||||
strings.TrimRight(proxyBaseURL, "/"),
|
||||
routePrefix,
|
||||
member.RemoteName,
|
||||
u)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
merged.Entries[chart] = append(merged.Entries[chart], ver)
|
||||
_ = key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return yaml.Marshal(merged)
|
||||
}
|
||||
|
||||
func extractHost(rawURL string) string {
|
||||
idx := strings.Index(rawURL, "://")
|
||||
if idx == -1 {
|
||||
return ""
|
||||
}
|
||||
rest := rawURL[idx+3:]
|
||||
slashIdx := strings.Index(rest, "/")
|
||||
if slashIdx == -1 {
|
||||
return rest
|
||||
}
|
||||
return rest[:slashIdx]
|
||||
}
|
||||
|
||||
func extractPathRelativeToBase(rawURL, baseURL string) string {
|
||||
fullPath := extractPath(rawURL)
|
||||
basePath := extractPath(baseURL)
|
||||
if basePath != "" {
|
||||
basePath = strings.TrimRight(basePath, "/") + "/"
|
||||
if strings.HasPrefix(fullPath, basePath) {
|
||||
return fullPath[len(basePath):]
|
||||
}
|
||||
}
|
||||
return fullPath
|
||||
}
|
||||
|
||||
func extractPath(rawURL string) string {
|
||||
idx := strings.Index(rawURL, "://")
|
||||
if idx == -1 {
|
||||
return rawURL
|
||||
}
|
||||
rest := rawURL[idx+3:]
|
||||
slashIdx := strings.Index(rest, "/")
|
||||
if slashIdx == -1 {
|
||||
return ""
|
||||
}
|
||||
return rest[slashIdx+1:]
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
package virtual_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
func TestHelmMerger_BasicMerge(t *testing.T) {
|
||||
m := &virtual.HelmMerger{}
|
||||
|
||||
member1 := virtual.MemberIndex{
|
||||
RemoteName: "repo-a",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
nginx:
|
||||
- name: nginx
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- https://charts-a.example.com/nginx-1.0.0.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
member2 := virtual.MemberIndex{
|
||||
RemoteName: "repo-b",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
redis:
|
||||
- name: redis
|
||||
version: "2.0.0"
|
||||
urls:
|
||||
- https://charts-b.example.com/redis-2.0.0.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
if !strings.Contains(body, "nginx") {
|
||||
t.Error("expected nginx in merged index")
|
||||
}
|
||||
if !strings.Contains(body, "redis") {
|
||||
t.Error("expected redis in merged index")
|
||||
}
|
||||
if !strings.Contains(body, "proxy.example.com/api/v1/remote/repo-a") {
|
||||
t.Error("expected proxy URL for repo-a")
|
||||
}
|
||||
if !strings.Contains(body, "proxy.example.com/api/v1/remote/repo-b") {
|
||||
t.Error("expected proxy URL for repo-b")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmMerger_Dedup(t *testing.T) {
|
||||
m := &virtual.HelmMerger{}
|
||||
|
||||
idx := []byte(`apiVersion: v1
|
||||
entries:
|
||||
nginx:
|
||||
- name: nginx
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- nginx-1.0.0.tgz
|
||||
`)
|
||||
|
||||
members := []virtual.MemberIndex{
|
||||
{RemoteName: "repo-a", Body: idx},
|
||||
{RemoteName: "repo-b", Body: idx},
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes(members, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
count := strings.Count(string(result), "name: nginx")
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 entry for nginx, got %d\n%s", count, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmMerger_PriorityOrder(t *testing.T) {
|
||||
m := &virtual.HelmMerger{}
|
||||
|
||||
member1 := virtual.MemberIndex{
|
||||
RemoteName: "priority-repo",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
chart:
|
||||
- name: chart
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- chart-from-priority.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
member2 := virtual.MemberIndex{
|
||||
RemoteName: "fallback-repo",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
chart:
|
||||
- name: chart
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- chart-from-fallback.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
if !strings.Contains(body, "priority-repo") {
|
||||
t.Error("expected priority repo URL to win")
|
||||
}
|
||||
if strings.Contains(body, "fallback-repo") {
|
||||
t.Error("expected fallback repo to be excluded for duplicate")
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type MemberIndex struct {
|
||||
RemoteName string
|
||||
RepoType models.RepoType
|
||||
BaseURL string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
|
||||
var mergers = map[models.PackageType]IndexMerger{}
|
||||
|
||||
func RegisterMerger(pt models.PackageType, m IndexMerger) {
|
||||
mergers[pt] = m
|
||||
}
|
||||
|
||||
func GetMerger(pt models.PackageType) (IndexMerger, error) {
|
||||
m, ok := mergers[pt]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no merger registered for package type %q", pt)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterMerger(models.PackagePyPI, &PyPIMerger{})
|
||||
}
|
||||
|
||||
type PyPIMerger struct{}
|
||||
|
||||
func (m *PyPIMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
|
||||
links := map[string]string{}
|
||||
|
||||
for _, member := range members {
|
||||
body := string(member.Body)
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "<a ") {
|
||||
continue
|
||||
}
|
||||
|
||||
href := extractHref(line)
|
||||
text := extractLinkText(line)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, exists := links[text]; exists {
|
||||
continue
|
||||
}
|
||||
|
||||
if proxyBaseURL != "" && href != "" {
|
||||
routePrefix := "remote"
|
||||
if member.RepoType == "local" {
|
||||
routePrefix = "local"
|
||||
}
|
||||
href = fmt.Sprintf("%s/api/v1/%s/%s/%s",
|
||||
strings.TrimRight(proxyBaseURL, "/"),
|
||||
routePrefix,
|
||||
member.RemoteName,
|
||||
strings.TrimLeft(href, "/"))
|
||||
}
|
||||
|
||||
links[text] = href
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(links))
|
||||
for k := range links {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, name := range keys {
|
||||
sb.WriteString(fmt.Sprintf(" <a href=\"%s\">%s</a>\n", links[name], name))
|
||||
}
|
||||
sb.WriteString("</body></html>\n")
|
||||
|
||||
return []byte(sb.String()), nil
|
||||
}
|
||||
|
||||
func extractHref(tag string) string {
|
||||
idx := strings.Index(tag, `href="`)
|
||||
if idx == -1 {
|
||||
return ""
|
||||
}
|
||||
rest := tag[idx+6:]
|
||||
end := strings.Index(rest, `"`)
|
||||
if end == -1 {
|
||||
return rest
|
||||
}
|
||||
return rest[:end]
|
||||
}
|
||||
|
||||
func extractLinkText(tag string) string {
|
||||
start := strings.Index(tag, ">")
|
||||
if start == -1 {
|
||||
return ""
|
||||
}
|
||||
rest := tag[start+1:]
|
||||
end := strings.Index(rest, "</a>")
|
||||
if end == -1 {
|
||||
return strings.TrimSpace(rest)
|
||||
}
|
||||
return strings.TrimSpace(rest[:end])
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
package virtual_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
func TestPyPIMerger_BasicMerge(t *testing.T) {
|
||||
m := &virtual.PyPIMerger{}
|
||||
|
||||
member1 := virtual.MemberIndex{
|
||||
RemoteName: "pypi-a",
|
||||
Body: []byte(`<!DOCTYPE html>
|
||||
<html><body>
|
||||
<a href="/simple/requests/">requests</a>
|
||||
<a href="/simple/flask/">flask</a>
|
||||
</body></html>`),
|
||||
}
|
||||
|
||||
member2 := virtual.MemberIndex{
|
||||
RemoteName: "pypi-b",
|
||||
Body: []byte(`<!DOCTYPE html>
|
||||
<html><body>
|
||||
<a href="/simple/django/">django</a>
|
||||
</body></html>`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
if !strings.Contains(body, "requests") {
|
||||
t.Error("expected requests")
|
||||
}
|
||||
if !strings.Contains(body, "flask") {
|
||||
t.Error("expected flask")
|
||||
}
|
||||
if !strings.Contains(body, "django") {
|
||||
t.Error("expected django")
|
||||
}
|
||||
if !strings.Contains(body, "proxy.example.com/api/v1/remote/pypi-a") {
|
||||
t.Error("expected proxy URL for pypi-a")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyPIMerger_Dedup(t *testing.T) {
|
||||
m := &virtual.PyPIMerger{}
|
||||
|
||||
idx := []byte(`<html><body>
|
||||
<a href="/simple/requests/">requests</a>
|
||||
</body></html>`)
|
||||
|
||||
members := []virtual.MemberIndex{
|
||||
{RemoteName: "a", Body: idx},
|
||||
{RemoteName: "b", Body: idx},
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes(members, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
count := strings.Count(string(result), "<a ")
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 <a> tag for deduplicated requests, got %d\n%s", count, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyPIMerger_Sorted(t *testing.T) {
|
||||
m := &virtual.PyPIMerger{}
|
||||
|
||||
member := virtual.MemberIndex{
|
||||
RemoteName: "pypi",
|
||||
Body: []byte(`<html><body>
|
||||
<a href="/z/">zebra</a>
|
||||
<a href="/a/">alpha</a>
|
||||
<a href="/m/">middle</a>
|
||||
</body></html>`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
alphaIdx := strings.Index(body, "alpha")
|
||||
middleIdx := strings.Index(body, "middle")
|
||||
zebraIdx := strings.Index(body, "zebra")
|
||||
|
||||
if alphaIdx > middleIdx || middleIdx > zebraIdx {
|
||||
t.Error("expected sorted output")
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
baseURL string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func New(baseURL string) *Client {
|
||||
return &Client{
|
||||
baseURL: baseURL,
|
||||
httpClient: http.DefaultClient,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) get(ctx context.Context, path string, out any) error {
|
||||
return c.do(ctx, http.MethodGet, path, nil, out)
|
||||
}
|
||||
|
||||
func (c *Client) post(ctx context.Context, path string, body any, out any) error {
|
||||
return c.do(ctx, http.MethodPost, path, body, out)
|
||||
}
|
||||
|
||||
func (c *Client) put(ctx context.Context, path string, body any, out any) error {
|
||||
return c.do(ctx, http.MethodPut, path, body, out)
|
||||
}
|
||||
|
||||
func (c *Client) delete(ctx context.Context, path string) error {
|
||||
return c.do(ctx, http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *Client) do(ctx context.Context, method, path string, body any, out any) error {
|
||||
var bodyReader io.Reader
|
||||
if body != nil {
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal: %w", err)
|
||||
}
|
||||
bodyReader = bytes.NewReader(b)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, bodyReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request: %w", err)
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("do: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("api error %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
|
||||
if out != nil && resp.StatusCode != http.StatusNoContent {
|
||||
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
|
||||
return fmt.Errorf("decode: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (c *Client) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
var remotes []models.Remote
|
||||
err := c.get(ctx, "/api/v2/remotes", &remotes)
|
||||
return remotes, err
|
||||
}
|
||||
|
||||
func (c *Client) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
|
||||
var remote models.Remote
|
||||
err := c.get(ctx, fmt.Sprintf("/api/v2/remotes/%s", name), &remote)
|
||||
return &remote, err
|
||||
}
|
||||
|
||||
func (c *Client) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
return c.post(ctx, "/api/v2/remotes", r, r)
|
||||
}
|
||||
|
||||
func (c *Client) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
return c.put(ctx, fmt.Sprintf("/api/v2/remotes/%s", r.Name), r, r)
|
||||
}
|
||||
|
||||
func (c *Client) DeleteRemote(ctx context.Context, name string) error {
|
||||
return c.delete(ctx, fmt.Sprintf("/api/v2/remotes/%s", name))
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (c *Client) Stats(ctx context.Context) (*models.OverviewStats, error) {
|
||||
var stats models.OverviewStats
|
||||
err := c.get(ctx, "/api/v2/stats", &stats)
|
||||
return &stats, err
|
||||
}
|
||||
|
||||
func (c *Client) Health(ctx context.Context) (*models.RemoteHealth, error) {
|
||||
var health models.RemoteHealth
|
||||
err := c.get(ctx, "/api/v2/health", &health)
|
||||
return &health, err
|
||||
}
|
||||
|
||||
func (c *Client) ListObjects(ctx context.Context, remote string, page, perPage int) ([]models.Artifact, error) {
|
||||
var artifacts []models.Artifact
|
||||
err := c.get(ctx, fmt.Sprintf("/api/v2/remotes/%s/objects?page=%d&per_page=%d", remote, page, perPage), &artifacts)
|
||||
return artifacts, err
|
||||
}
|
||||
|
||||
func (c *Client) EvictObject(ctx context.Context, remote, path string) error {
|
||||
return c.delete(ctx, fmt.Sprintf("/api/v2/remotes/%s/objects/%s", remote, path))
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func (c *Client) ListVirtuals(ctx context.Context) ([]models.Virtual, error) {
|
||||
var virtuals []models.Virtual
|
||||
err := c.get(ctx, "/api/v2/virtuals", &virtuals)
|
||||
return virtuals, err
|
||||
}
|
||||
|
||||
func (c *Client) GetVirtual(ctx context.Context, name string) (*models.Virtual, error) {
|
||||
var virt models.Virtual
|
||||
err := c.get(ctx, fmt.Sprintf("/api/v2/virtuals/%s", name), &virt)
|
||||
return &virt, err
|
||||
}
|
||||
|
||||
func (c *Client) CreateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
return c.post(ctx, "/api/v2/virtuals", v, v)
|
||||
}
|
||||
|
||||
func (c *Client) UpdateVirtual(ctx context.Context, v *models.Virtual) error {
|
||||
return c.put(ctx, fmt.Sprintf("/api/v2/virtuals/%s", v.Name), v, v)
|
||||
}
|
||||
|
||||
func (c *Client) DeleteVirtual(ctx context.Context, name string) error {
|
||||
return c.delete(ctx, fmt.Sprintf("/api/v2/virtuals/%s", name))
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Blob struct {
|
||||
ContentHash string `json:"content_hash"`
|
||||
S3Key string `json:"s3_key"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ContentType string `json:"content_type"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type Artifact struct {
|
||||
ID int64 `json:"id"`
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
UpstreamETag string `json:"upstream_etag,omitempty"`
|
||||
UpstreamLastModified *time.Time `json:"upstream_last_modified,omitempty"`
|
||||
FirstSeenAt time.Time `json:"first_seen_at"`
|
||||
LastFetchedAt time.Time `json:"last_fetched_at"`
|
||||
LastAccessedAt time.Time `json:"last_accessed_at"`
|
||||
FetchCount int64 `json:"fetch_count"`
|
||||
AccessCount int64 `json:"access_count"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
}
|
||||
|
||||
type AccessLogEntry struct {
|
||||
ID int64 `json:"id"`
|
||||
RemoteName string `json:"remote_name"`
|
||||
Path string `json:"path"`
|
||||
CacheHit bool `json:"cache_hit"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
UpstreamMS int `json:"upstream_ms"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type LocalFile struct {
|
||||
ID int64 `json:"id"`
|
||||
RepoName string `json:"repo_name"`
|
||||
FilePath string `json:"file_path"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package models
|
||||
|
||||
import "fmt"
|
||||
|
||||
type PackageType string
|
||||
|
||||
const (
|
||||
PackageGeneric PackageType = "generic"
|
||||
PackageDocker PackageType = "docker"
|
||||
PackageHelm PackageType = "helm"
|
||||
PackagePyPI PackageType = "pypi"
|
||||
PackageNPM PackageType = "npm"
|
||||
PackageRPM PackageType = "rpm"
|
||||
PackageAlpine PackageType = "alpine"
|
||||
PackagePuppet PackageType = "puppet"
|
||||
PackageTerraform PackageType = "terraform"
|
||||
PackageGoProxy PackageType = "goproxy"
|
||||
)
|
||||
|
||||
var validPackageTypes = map[PackageType]bool{
|
||||
PackageGeneric: true,
|
||||
PackageDocker: true,
|
||||
PackageHelm: true,
|
||||
PackagePyPI: true,
|
||||
PackageNPM: true,
|
||||
PackageRPM: true,
|
||||
PackageAlpine: true,
|
||||
PackagePuppet: true,
|
||||
PackageTerraform: true,
|
||||
PackageGoProxy: true,
|
||||
}
|
||||
|
||||
func (p PackageType) Valid() bool {
|
||||
return validPackageTypes[p]
|
||||
}
|
||||
|
||||
func (p PackageType) String() string {
|
||||
return string(p)
|
||||
}
|
||||
|
||||
func ParsePackageType(s string) (PackageType, error) {
|
||||
pt := PackageType(s)
|
||||
if !pt.Valid() {
|
||||
return "", fmt.Errorf("unknown package type: %q", s)
|
||||
}
|
||||
return pt, nil
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestPackageTypeValid(t *testing.T) {
|
||||
valid := []models.PackageType{
|
||||
models.PackageGeneric,
|
||||
models.PackageDocker,
|
||||
models.PackageHelm,
|
||||
models.PackagePyPI,
|
||||
models.PackageNPM,
|
||||
models.PackageRPM,
|
||||
models.PackageAlpine,
|
||||
models.PackagePuppet,
|
||||
models.PackageTerraform,
|
||||
models.PackageGoProxy,
|
||||
}
|
||||
for _, pt := range valid {
|
||||
if !pt.Valid() {
|
||||
t.Errorf("expected %q to be valid", pt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackageTypeInvalid(t *testing.T) {
|
||||
invalid := []string{"", "bogus", "Docker", "HELM"}
|
||||
for _, s := range invalid {
|
||||
pt := models.PackageType(s)
|
||||
if pt.Valid() {
|
||||
t.Errorf("expected %q to be invalid", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePackageType(t *testing.T) {
|
||||
pt, err := models.ParsePackageType("docker")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if pt != models.PackageDocker {
|
||||
t.Errorf("expected docker, got %q", pt)
|
||||
}
|
||||
|
||||
_, err = models.ParsePackageType("nope")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for unknown type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackageTypeString(t *testing.T) {
|
||||
if models.PackageGoProxy.String() != "goproxy" {
|
||||
t.Errorf("expected 'goproxy', got %q", models.PackageGoProxy.String())
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RepoType string
|
||||
|
||||
const (
|
||||
RepoTypeRemote RepoType = "remote"
|
||||
RepoTypeLocal RepoType = "local"
|
||||
)
|
||||
|
||||
var validRepoTypes = map[RepoType]bool{
|
||||
RepoTypeRemote: true,
|
||||
RepoTypeLocal: true,
|
||||
}
|
||||
|
||||
func (r RepoType) Valid() bool {
|
||||
return validRepoTypes[r]
|
||||
}
|
||||
|
||||
func (r RepoType) String() string {
|
||||
return string(r)
|
||||
}
|
||||
|
||||
func ParseRepoType(s string) (RepoType, error) {
|
||||
rt := RepoType(s)
|
||||
if !rt.Valid() {
|
||||
return "", fmt.Errorf("unknown repo type: %q", s)
|
||||
}
|
||||
return rt, nil
|
||||
}
|
||||
|
||||
type Remote struct {
|
||||
Name string `json:"name"`
|
||||
PackageType PackageType `json:"package_type"`
|
||||
RepoType RepoType `json:"repo_type"`
|
||||
BaseURL string `json:"base_url"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Username string `json:"-"`
|
||||
Password string `json:"-"`
|
||||
|
||||
ImmutableTTL int `json:"immutable_ttl"`
|
||||
MutableTTL int `json:"mutable_ttl"`
|
||||
CheckMutable bool `json:"check_mutable"`
|
||||
|
||||
// Upstream HTTP timeouts in seconds. 0 means use the server default.
|
||||
UpstreamDialTimeout int `json:"upstream_dial_timeout,omitempty"`
|
||||
UpstreamTLSTimeout int `json:"upstream_tls_timeout,omitempty"`
|
||||
UpstreamResponseHeaderTimeout int `json:"upstream_response_header_timeout,omitempty"`
|
||||
|
||||
Patterns []string `json:"patterns,omitempty"`
|
||||
Blocklist []string `json:"blocklist,omitempty"`
|
||||
MutablePatterns []string `json:"mutable_patterns,omitempty"`
|
||||
ImmutablePatterns []string `json:"immutable_patterns,omitempty"`
|
||||
|
||||
BanTagsEnabled bool `json:"ban_tags_enabled,omitempty"`
|
||||
BanTags []string `json:"ban_tags,omitempty"`
|
||||
|
||||
QuarantineEnabled bool `json:"quarantine_enabled,omitempty"`
|
||||
QuarantineDays int `json:"quarantine_days,omitempty"`
|
||||
|
||||
StaleOnError bool `json:"stale_on_error"`
|
||||
|
||||
ReleasesRemote string `json:"releases_remote,omitempty"`
|
||||
ManagedBy string `json:"managed_by,omitempty"`
|
||||
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// ValidatePatterns ensures every configured regex compiles. Storing an
|
||||
// invalid pattern would otherwise be silently dropped at match time, which
|
||||
// for the blocklist is a fail-open: a mistyped deny rule becomes a no-op.
|
||||
func (r *Remote) ValidatePatterns() error {
|
||||
groups := []struct {
|
||||
field string
|
||||
patterns []string
|
||||
}{
|
||||
{"patterns", r.Patterns},
|
||||
{"blocklist", r.Blocklist},
|
||||
{"mutable_patterns", r.MutablePatterns},
|
||||
{"immutable_patterns", r.ImmutablePatterns},
|
||||
{"ban_tags", r.BanTags},
|
||||
}
|
||||
for _, g := range groups {
|
||||
for _, p := range g.patterns {
|
||||
if _, err := regexp.Compile(p); err != nil {
|
||||
return fmt.Errorf("invalid regex in %s: %q: %w", g.field, p, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type RemoteWithStats struct {
|
||||
Remote
|
||||
Stats RemoteStats `json:"stats"`
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
package models
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRemote_ValidatePatterns(t *testing.T) {
|
||||
valid := &Remote{
|
||||
Patterns: []string{`.*\.tar\.gz$`},
|
||||
Blocklist: []string{`^secret/`},
|
||||
ImmutablePatterns: []string{`\.rpm$`},
|
||||
}
|
||||
if err := valid.ValidatePatterns(); err != nil {
|
||||
t.Fatalf("expected valid patterns, got %v", err)
|
||||
}
|
||||
|
||||
bad := &Remote{Blocklist: []string{`[unterminated`}}
|
||||
if err := bad.ValidatePatterns(); err == nil {
|
||||
t.Fatal("expected error for invalid blocklist regex, got nil")
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package models
|
||||
|
||||
type RemoteStats struct {
|
||||
ObjectCount int64 `json:"object_count"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
HitRate30d float64 `json:"hit_rate_30d"`
|
||||
Requests30d int64 `json:"requests_30d"`
|
||||
BandwidthSaved int64 `json:"bandwidth_saved_30d"`
|
||||
}
|
||||
|
||||
type OverviewStats struct {
|
||||
TotalRemotes int `json:"total_remotes"`
|
||||
TotalObjects int64 `json:"total_objects"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
TotalBlobsDeduped int64 `json:"total_blobs_deduped"`
|
||||
BandwidthSaved30d int64 `json:"bandwidth_saved_30d"`
|
||||
}
|
||||
|
||||
type RemoteHealth struct {
|
||||
Status string `json:"status"` // healthy, degraded, down
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
ConsecutiveFailures int `json:"consecutive_failures"`
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Virtual struct {
|
||||
Name string `json:"name"`
|
||||
PackageType PackageType `json:"package_type"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Members []string `json:"members"`
|
||||
ManagedBy string `json:"managed_by,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
[project]
|
||||
name = "artifactapi"
|
||||
version = "2.0.0"
|
||||
description = "Generic artifact caching system with support for various package managers"
|
||||
|
||||
dependencies = [
|
||||
"fastapi>=0.104.0",
|
||||
"uvicorn[standard]>=0.24.0",
|
||||
"httpx>=0.25.0",
|
||||
"redis>=5.0.0",
|
||||
"boto3>=1.29.0",
|
||||
"psycopg2-binary>=2.9.0",
|
||||
"pyyaml>=6.0",
|
||||
"lxml>=4.9.0",
|
||||
"prometheus-client>=0.19.0",
|
||||
"python-multipart>=0.0.6",
|
||||
]
|
||||
requires-python = ">=3.11"
|
||||
readme = "README.md"
|
||||
license = {text = "MIT"}
|
||||
|
||||
[project.scripts]
|
||||
artifactapi = "artifactapi.main:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.metadata]
|
||||
allow-direct-references = true
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/artifactapi"]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=7.4.0",
|
||||
"pytest-asyncio>=0.21.0",
|
||||
"black>=23.9.0",
|
||||
"isort>=5.12.0",
|
||||
"mypy>=1.6.0",
|
||||
"ruff>=0.1.0",
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
# Artifact API package
|
||||
@@ -0,0 +1,69 @@
|
||||
import time
|
||||
import hashlib
|
||||
import redis
|
||||
|
||||
|
||||
class RedisCache:
|
||||
def __init__(self, redis_url: str):
|
||||
self.redis_url = redis_url
|
||||
|
||||
try:
|
||||
self.client = redis.from_url(self.redis_url, decode_responses=True)
|
||||
# Test connection
|
||||
self.client.ping()
|
||||
self.available = True
|
||||
except Exception as e:
|
||||
print(f"Redis not available: {e}")
|
||||
self.client = None
|
||||
self.available = False
|
||||
|
||||
def is_index_file(self, file_path: str) -> bool:
|
||||
"""Check if the file is an index file that should have TTL"""
|
||||
return (
|
||||
file_path.endswith("APKINDEX.tar.gz")
|
||||
or file_path.endswith("Packages.gz")
|
||||
or file_path.endswith("repomd.xml")
|
||||
or "repodata/" in file_path
|
||||
and file_path.endswith((".xml", ".xml.gz", ".xml.bz2", ".xml.xz"))
|
||||
)
|
||||
|
||||
def get_index_cache_key(self, remote_name: str, path: str) -> str:
|
||||
"""Generate cache key for index files"""
|
||||
return f"index:{remote_name}:{hashlib.sha256(path.encode()).hexdigest()[:16]}"
|
||||
|
||||
def is_index_valid(
|
||||
self, remote_name: str, path: str, ttl_override: int = None
|
||||
) -> bool:
|
||||
"""Check if index file is still valid (not expired)"""
|
||||
if not self.available:
|
||||
return False
|
||||
|
||||
try:
|
||||
key = self.get_index_cache_key(remote_name, path)
|
||||
return self.client.exists(key) > 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def mark_index_cached(self, remote_name: str, path: str, ttl: int = 300) -> None:
|
||||
"""Mark index file as cached with TTL"""
|
||||
if not self.available:
|
||||
return
|
||||
|
||||
try:
|
||||
key = self.get_index_cache_key(remote_name, path)
|
||||
self.client.setex(key, ttl, str(int(time.time())))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def cleanup_expired_index(self, storage, remote_name: str, path: str) -> None:
|
||||
"""Remove expired index from S3 storage"""
|
||||
if not self.available:
|
||||
return
|
||||
|
||||
try:
|
||||
# Get the S3 key and remove it
|
||||
s3_key = storage.get_object_key_from_path(remote_name, path)
|
||||
if storage.exists(s3_key):
|
||||
storage.client.delete_object(Bucket=storage.bucket, Key=s3_key)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,120 @@
|
||||
import os
|
||||
import json
|
||||
import yaml
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
def __init__(self, config_file: str = "remotes.yaml"):
|
||||
self.config_file = config_file
|
||||
self._last_modified = 0
|
||||
self.config = self._load_config()
|
||||
|
||||
def _load_config(self) -> dict:
|
||||
try:
|
||||
with open(self.config_file, "r") as f:
|
||||
if self.config_file.endswith(".yaml") or self.config_file.endswith(
|
||||
".yml"
|
||||
):
|
||||
return yaml.safe_load(f)
|
||||
else:
|
||||
return json.load(f)
|
||||
except FileNotFoundError:
|
||||
return {"remotes": {}}
|
||||
|
||||
def _check_reload(self) -> None:
|
||||
"""Check if config file has been modified and reload if needed"""
|
||||
try:
|
||||
import os
|
||||
|
||||
current_modified = os.path.getmtime(self.config_file)
|
||||
if current_modified > self._last_modified:
|
||||
self._last_modified = current_modified
|
||||
self.config = self._load_config()
|
||||
print(f"Config reloaded from {self.config_file}")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def get_remote_config(self, remote_name: str) -> Optional[dict]:
|
||||
self._check_reload()
|
||||
return self.config.get("remotes", {}).get(remote_name)
|
||||
|
||||
def get_repository_patterns(self, remote_name: str, repo_path: str) -> list:
|
||||
remote_config = self.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
return []
|
||||
|
||||
repositories = remote_config.get("repositories", {})
|
||||
|
||||
# Handle both dict (GitHub style) and list (Alpine style) repositories
|
||||
if isinstance(repositories, dict):
|
||||
repo_config = repositories.get(repo_path)
|
||||
if repo_config:
|
||||
patterns = repo_config.get("include_patterns", [])
|
||||
else:
|
||||
patterns = remote_config.get("include_patterns", [])
|
||||
elif isinstance(repositories, list):
|
||||
# For Alpine, repositories is just a list of allowed repo names
|
||||
# Pattern matching is handled by the main include_patterns
|
||||
patterns = remote_config.get("include_patterns", [])
|
||||
else:
|
||||
patterns = remote_config.get("include_patterns", [])
|
||||
|
||||
return patterns
|
||||
|
||||
def get_s3_config(self) -> dict:
|
||||
"""Get S3 configuration from environment variables"""
|
||||
endpoint = os.getenv("MINIO_ENDPOINT")
|
||||
access_key = os.getenv("MINIO_ACCESS_KEY")
|
||||
secret_key = os.getenv("MINIO_SECRET_KEY")
|
||||
bucket = os.getenv("MINIO_BUCKET")
|
||||
|
||||
if not endpoint:
|
||||
raise ValueError("MINIO_ENDPOINT environment variable is required")
|
||||
if not access_key:
|
||||
raise ValueError("MINIO_ACCESS_KEY environment variable is required")
|
||||
if not secret_key:
|
||||
raise ValueError("MINIO_SECRET_KEY environment variable is required")
|
||||
if not bucket:
|
||||
raise ValueError("MINIO_BUCKET environment variable is required")
|
||||
|
||||
return {
|
||||
"endpoint": endpoint,
|
||||
"access_key": access_key,
|
||||
"secret_key": secret_key,
|
||||
"bucket": bucket,
|
||||
"secure": os.getenv("MINIO_SECURE", "false").lower() == "true",
|
||||
}
|
||||
|
||||
def get_redis_config(self) -> dict:
|
||||
"""Get Redis configuration from environment variables"""
|
||||
redis_url = os.getenv("REDIS_URL")
|
||||
if not redis_url:
|
||||
raise ValueError("REDIS_URL environment variable is required")
|
||||
|
||||
return {
|
||||
"url": redis_url
|
||||
}
|
||||
|
||||
def get_database_config(self) -> dict:
|
||||
"""Get database configuration from environment variables"""
|
||||
db_host = os.getenv("DBHOST")
|
||||
db_port = os.getenv("DBPORT")
|
||||
db_user = os.getenv("DBUSER")
|
||||
db_pass = os.getenv("DBPASS")
|
||||
db_name = os.getenv("DBNAME")
|
||||
|
||||
if not all([db_host, db_port, db_user, db_pass, db_name]):
|
||||
missing = [var for var, val in [("DBHOST", db_host), ("DBPORT", db_port), ("DBUSER", db_user), ("DBPASS", db_pass), ("DBNAME", db_name)] if not val]
|
||||
raise ValueError(f"All database environment variables are required: {', '.join(missing)}")
|
||||
|
||||
db_url = f"postgresql://{db_user}:{db_pass}@{db_host}:{db_port}/{db_name}"
|
||||
return {"url": db_url}
|
||||
|
||||
def get_cache_config(self, remote_name: str) -> dict:
|
||||
"""Get cache configuration for a specific remote"""
|
||||
remote_config = self.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
return {}
|
||||
|
||||
return remote_config.get("cache", {})
|
||||
@@ -0,0 +1,282 @@
|
||||
import os
|
||||
from typing import Optional
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
|
||||
class DatabaseManager:
|
||||
def __init__(self, db_url: str):
|
||||
self.db_url = db_url
|
||||
self.available = False
|
||||
self._init_database()
|
||||
|
||||
def _init_database(self):
|
||||
"""Initialize database connection and create schema if needed"""
|
||||
try:
|
||||
self.connection = psycopg2.connect(self.db_url)
|
||||
self.connection.autocommit = True
|
||||
self._create_schema()
|
||||
self.available = True
|
||||
print("Database connection established")
|
||||
except Exception as e:
|
||||
print(f"Database not available: {e}")
|
||||
self.available = False
|
||||
|
||||
def _create_schema(self):
|
||||
"""Create tables if they don't exist"""
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
# Create table to map S3 keys to remote names
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS artifact_mappings (
|
||||
id SERIAL PRIMARY KEY,
|
||||
s3_key VARCHAR(255) UNIQUE NOT NULL,
|
||||
remote_name VARCHAR(100) NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
size_bytes BIGINT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS local_files (
|
||||
id SERIAL PRIMARY KEY,
|
||||
repository_name VARCHAR(100) NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
s3_key VARCHAR(255) UNIQUE NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256_sum VARCHAR(64) NOT NULL,
|
||||
content_type VARCHAR(100),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
uploaded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(repository_name, file_path)
|
||||
)
|
||||
""")
|
||||
|
||||
# Create indexes separately
|
||||
cursor.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_s3_key ON artifact_mappings (s3_key)"
|
||||
)
|
||||
cursor.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_remote_name ON artifact_mappings (remote_name)"
|
||||
)
|
||||
cursor.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_local_repo_path ON local_files (repository_name, file_path)"
|
||||
)
|
||||
cursor.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_local_s3_key ON local_files (s3_key)"
|
||||
)
|
||||
print("Database schema initialized")
|
||||
except Exception as e:
|
||||
print(f"Error creating schema: {e}")
|
||||
|
||||
def record_artifact_mapping(
|
||||
self, s3_key: str, remote_name: str, file_path: str, size_bytes: int
|
||||
):
|
||||
"""Record mapping between S3 key and remote"""
|
||||
if not self.available:
|
||||
return
|
||||
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO artifact_mappings (s3_key, remote_name, file_path, size_bytes)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
ON CONFLICT (s3_key)
|
||||
DO UPDATE SET
|
||||
remote_name = EXCLUDED.remote_name,
|
||||
file_path = EXCLUDED.file_path,
|
||||
size_bytes = EXCLUDED.size_bytes
|
||||
""",
|
||||
(s3_key, remote_name, file_path, size_bytes),
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error recording artifact mapping: {e}")
|
||||
|
||||
def get_storage_by_remote(self) -> dict[str, int]:
|
||||
"""Get storage size breakdown by remote from database"""
|
||||
if not self.available:
|
||||
return {}
|
||||
|
||||
try:
|
||||
with self.connection.cursor(cursor_factory=RealDictCursor) as cursor:
|
||||
cursor.execute("""
|
||||
SELECT remote_name, SUM(size_bytes) as total_size
|
||||
FROM artifact_mappings
|
||||
GROUP BY remote_name
|
||||
""")
|
||||
results = cursor.fetchall()
|
||||
return {row["remote_name"]: row["total_size"] or 0 for row in results}
|
||||
except Exception as e:
|
||||
print(f"Error getting storage by remote: {e}")
|
||||
return {}
|
||||
|
||||
def get_remote_for_s3_key(self, s3_key: str) -> Optional[str]:
|
||||
"""Get remote name for given S3 key"""
|
||||
if not self.available:
|
||||
return None
|
||||
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT remote_name FROM artifact_mappings WHERE s3_key = %s",
|
||||
(s3_key,),
|
||||
)
|
||||
result = cursor.fetchone()
|
||||
return result[0] if result else None
|
||||
except Exception as e:
|
||||
print(f"Error getting remote for S3 key: {e}")
|
||||
return None
|
||||
|
||||
def add_local_file(
|
||||
self,
|
||||
repository_name: str,
|
||||
file_path: str,
|
||||
s3_key: str,
|
||||
size_bytes: int,
|
||||
sha256_sum: str,
|
||||
content_type: str = None,
|
||||
):
|
||||
"""Add a file to local repository"""
|
||||
if not self.available:
|
||||
return False
|
||||
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO local_files (repository_name, file_path, s3_key, size_bytes, sha256_sum, content_type)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(
|
||||
repository_name,
|
||||
file_path,
|
||||
s3_key,
|
||||
size_bytes,
|
||||
sha256_sum,
|
||||
content_type,
|
||||
),
|
||||
)
|
||||
self.connection.commit()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error adding local file: {e}")
|
||||
return False
|
||||
|
||||
def get_local_file_metadata(self, repository_name: str, file_path: str):
|
||||
"""Get metadata for a local file"""
|
||||
if not self.available:
|
||||
return None
|
||||
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT repository_name, file_path, s3_key, size_bytes, sha256_sum, content_type, created_at, uploaded_at
|
||||
FROM local_files
|
||||
WHERE repository_name = %s AND file_path = %s
|
||||
""",
|
||||
(repository_name, file_path),
|
||||
)
|
||||
result = cursor.fetchone()
|
||||
if result:
|
||||
return {
|
||||
"repository_name": result[0],
|
||||
"file_path": result[1],
|
||||
"s3_key": result[2],
|
||||
"size_bytes": result[3],
|
||||
"sha256_sum": result[4],
|
||||
"content_type": result[5],
|
||||
"created_at": result[6],
|
||||
"uploaded_at": result[7],
|
||||
}
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"Error getting local file metadata: {e}")
|
||||
return None
|
||||
|
||||
def list_local_files(self, repository_name: str, prefix: str = ""):
|
||||
"""List files in local repository with optional path prefix"""
|
||||
if not self.available:
|
||||
return []
|
||||
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
if prefix:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT file_path, size_bytes, sha256_sum, content_type, created_at, uploaded_at
|
||||
FROM local_files
|
||||
WHERE repository_name = %s AND file_path LIKE %s
|
||||
ORDER BY file_path
|
||||
""",
|
||||
(repository_name, f"{prefix}%"),
|
||||
)
|
||||
else:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT file_path, size_bytes, sha256_sum, content_type, created_at, uploaded_at
|
||||
FROM local_files
|
||||
WHERE repository_name = %s
|
||||
ORDER BY file_path
|
||||
""",
|
||||
(repository_name,),
|
||||
)
|
||||
|
||||
results = cursor.fetchall()
|
||||
return [
|
||||
{
|
||||
"file_path": result[0],
|
||||
"size_bytes": result[1],
|
||||
"sha256_sum": result[2],
|
||||
"content_type": result[3],
|
||||
"created_at": result[4],
|
||||
"uploaded_at": result[5],
|
||||
}
|
||||
for result in results
|
||||
]
|
||||
except Exception as e:
|
||||
print(f"Error listing local files: {e}")
|
||||
return []
|
||||
|
||||
def delete_local_file(self, repository_name: str, file_path: str):
|
||||
"""Delete a file from local repository"""
|
||||
if not self.available:
|
||||
return False
|
||||
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
DELETE FROM local_files
|
||||
WHERE repository_name = %s AND file_path = %s
|
||||
RETURNING s3_key
|
||||
""",
|
||||
(repository_name, file_path),
|
||||
)
|
||||
result = cursor.fetchone()
|
||||
self.connection.commit()
|
||||
return result[0] if result else None
|
||||
except Exception as e:
|
||||
print(f"Error deleting local file: {e}")
|
||||
return None
|
||||
|
||||
def file_exists(self, repository_name: str, file_path: str):
|
||||
"""Check if file exists in local repository"""
|
||||
if not self.available:
|
||||
return False
|
||||
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT 1 FROM local_files
|
||||
WHERE repository_name = %s AND file_path = %s
|
||||
""",
|
||||
(repository_name, file_path),
|
||||
)
|
||||
return cursor.fetchone() is not None
|
||||
except Exception as e:
|
||||
print(f"Error checking file existence: {e}")
|
||||
return False
|
||||
@@ -0,0 +1,583 @@
|
||||
import os
|
||||
import re
|
||||
import hashlib
|
||||
from typing import Dict, Any, Optional
|
||||
import httpx
|
||||
from fastapi import FastAPI, HTTPException, Response, Query, File, UploadFile
|
||||
from fastapi.responses import PlainTextResponse, JSONResponse
|
||||
from pydantic import BaseModel
|
||||
from prometheus_client import generate_latest, CONTENT_TYPE_LATEST
|
||||
|
||||
from .config import ConfigManager
|
||||
from .database import DatabaseManager
|
||||
from .storage import S3Storage
|
||||
from .cache import RedisCache
|
||||
from .metrics import MetricsManager
|
||||
|
||||
|
||||
class ArtifactRequest(BaseModel):
|
||||
remote: str
|
||||
include_pattern: str
|
||||
|
||||
|
||||
app = FastAPI(title="Artifact Storage API", version="2.0.0")
|
||||
|
||||
# Initialize components using config
|
||||
config_path = os.environ.get("CONFIG_PATH")
|
||||
if not config_path:
|
||||
raise ValueError("CONFIG_PATH environment variable is required")
|
||||
config = ConfigManager(config_path)
|
||||
|
||||
# Get configurations
|
||||
s3_config = config.get_s3_config()
|
||||
redis_config = config.get_redis_config()
|
||||
db_config = config.get_database_config()
|
||||
|
||||
# Initialize services
|
||||
storage = S3Storage(**s3_config)
|
||||
cache = RedisCache(redis_config["url"])
|
||||
database = DatabaseManager(db_config["url"])
|
||||
metrics = MetricsManager(cache, database)
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def read_root():
|
||||
config._check_reload()
|
||||
return {
|
||||
"message": "Artifact Storage API",
|
||||
"version": "2.0.0",
|
||||
"remotes": list(config.config.get("remotes", {}).keys()),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health_check():
|
||||
return {"status": "healthy"}
|
||||
|
||||
|
||||
async def construct_remote_url(remote_name: str, path: str) -> str:
|
||||
remote_config = config.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Remote '{remote_name}' not configured"
|
||||
)
|
||||
|
||||
base_url = remote_config.get("base_url")
|
||||
if not base_url:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"No base_url configured for remote '{remote_name}'"
|
||||
)
|
||||
|
||||
return f"{base_url}/{path}"
|
||||
|
||||
|
||||
async def check_artifact_patterns(
|
||||
remote_name: str, repo_path: str, file_path: str, full_path: str
|
||||
) -> bool:
|
||||
# First check if this is an index file - always allow index files
|
||||
if cache.is_index_file(file_path) or cache.is_index_file(full_path):
|
||||
return True
|
||||
|
||||
# Then check basic include patterns
|
||||
patterns = config.get_repository_patterns(remote_name, repo_path)
|
||||
if not patterns:
|
||||
return True # Allow all if no patterns configured
|
||||
|
||||
pattern_matched = False
|
||||
for pattern in patterns:
|
||||
# Check both file_path and full_path to handle different pattern types
|
||||
if re.search(pattern, file_path) or re.search(pattern, full_path):
|
||||
pattern_matched = True
|
||||
break
|
||||
|
||||
if not pattern_matched:
|
||||
return False
|
||||
|
||||
# All remotes now use pattern-based filtering only - no additional checks needed
|
||||
return True
|
||||
|
||||
|
||||
async def cache_single_artifact(url: str, remote_name: str, path: str) -> dict:
|
||||
# Check if using URL-based key or path-based key
|
||||
if url.startswith("http"):
|
||||
key = storage.get_object_key(url)
|
||||
else:
|
||||
key = storage.get_object_key_from_path(remote_name, path)
|
||||
|
||||
if storage.exists(key):
|
||||
return {
|
||||
"url": url,
|
||||
"cached_url": storage.get_url(key),
|
||||
"status": "already_cached",
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
storage_path = storage.upload(key, response.content)
|
||||
|
||||
return {
|
||||
"url": url,
|
||||
"cached_url": storage.get_url(key),
|
||||
"storage_path": storage_path,
|
||||
"size": len(response.content),
|
||||
"status": "cached",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {"url": url, "status": "error", "error": str(e)}
|
||||
|
||||
|
||||
@app.get("/api/v1/remote/{remote_name}/{path:path}")
|
||||
async def get_artifact(remote_name: str, path: str):
|
||||
# Check if remote is configured
|
||||
remote_config = config.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Remote '{remote_name}' not configured"
|
||||
)
|
||||
|
||||
# Check if this is a local repository
|
||||
if remote_config.get("type") == "local":
|
||||
# Handle local repository download
|
||||
metadata = database.get_local_file_metadata(remote_name, path)
|
||||
if not metadata:
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
|
||||
# Get file from S3
|
||||
content = storage.download_object(metadata["s3_key"])
|
||||
if content is None:
|
||||
raise HTTPException(status_code=500, detail="File not accessible")
|
||||
|
||||
# Determine content type
|
||||
content_type = metadata.get("content_type", "application/octet-stream")
|
||||
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=content_type,
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename={os.path.basename(path)}"
|
||||
},
|
||||
)
|
||||
|
||||
# Extract repository path for pattern checking
|
||||
path_parts = path.split("/")
|
||||
if len(path_parts) >= 2:
|
||||
repo_path = f"{path_parts[0]}/{path_parts[1]}"
|
||||
file_path = "/".join(path_parts[2:])
|
||||
else:
|
||||
repo_path = path
|
||||
file_path = path
|
||||
|
||||
# Check if artifact matches configured patterns
|
||||
if not await check_artifact_patterns(remote_name, repo_path, file_path, path):
|
||||
raise HTTPException(
|
||||
status_code=403, detail="Artifact not allowed by configuration patterns"
|
||||
)
|
||||
|
||||
# Construct the remote URL
|
||||
remote_url = await construct_remote_url(remote_name, path)
|
||||
|
||||
# Check if artifact is already cached (try both URL and path-based keys)
|
||||
url_key = storage.get_object_key(remote_url)
|
||||
path_key = storage.get_object_key_from_path(remote_name, path)
|
||||
|
||||
cached_key = None
|
||||
if storage.exists(url_key):
|
||||
cached_key = url_key
|
||||
elif storage.exists(path_key):
|
||||
cached_key = path_key
|
||||
|
||||
# For index files, check Redis TTL validity
|
||||
filename = os.path.basename(path)
|
||||
is_index = cache.is_index_file(filename)
|
||||
|
||||
if cached_key and is_index:
|
||||
# Index file exists, but check if it's still valid
|
||||
if not cache.is_index_valid(remote_name, path):
|
||||
# Index has expired, remove it from S3
|
||||
cache.cleanup_expired_index(storage, remote_name, path)
|
||||
cached_key = None # Force re-download
|
||||
|
||||
if cached_key:
|
||||
# Return cached artifact
|
||||
try:
|
||||
artifact_data = storage.download_object(cached_key)
|
||||
filename = os.path.basename(path)
|
||||
|
||||
# Determine content type based on file extension
|
||||
content_type = "application/octet-stream"
|
||||
if filename.endswith(".tar.gz"):
|
||||
content_type = "application/gzip"
|
||||
elif filename.endswith(".zip"):
|
||||
content_type = "application/zip"
|
||||
elif filename.endswith(".exe"):
|
||||
content_type = "application/x-msdownload"
|
||||
elif filename.endswith(".rpm"):
|
||||
content_type = "application/x-rpm"
|
||||
elif filename.endswith(".xml"):
|
||||
content_type = "application/xml"
|
||||
elif filename.endswith((".xml.gz", ".xml.bz2", ".xml.xz")):
|
||||
content_type = "application/gzip"
|
||||
|
||||
# Record cache hit metrics
|
||||
metrics.record_cache_hit(remote_name, len(artifact_data))
|
||||
|
||||
# Record artifact mapping in database if not already recorded
|
||||
database.record_artifact_mapping(
|
||||
cached_key, remote_name, path, len(artifact_data)
|
||||
)
|
||||
|
||||
return Response(
|
||||
content=artifact_data,
|
||||
media_type=content_type,
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename={filename}",
|
||||
"X-Artifact-Source": "cache",
|
||||
"X-Artifact-Size": str(len(artifact_data)),
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Error retrieving cached artifact: {str(e)}"
|
||||
)
|
||||
|
||||
# Artifact not cached, cache it first
|
||||
result = await cache_single_artifact(remote_url, remote_name, path)
|
||||
|
||||
if result["status"] == "error":
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"Failed to fetch artifact: {result['error']}"
|
||||
)
|
||||
|
||||
# Mark index files as cached in Redis if this was a new download
|
||||
if result["status"] == "cached" and is_index:
|
||||
# Get TTL from remote config
|
||||
cache_config = config.get_cache_config(remote_name)
|
||||
index_ttl = cache_config.get("index_ttl", 300) # Default 5 minutes
|
||||
cache.mark_index_cached(remote_name, path, index_ttl)
|
||||
|
||||
# Now return the cached artifact
|
||||
try:
|
||||
cache_key = storage.get_object_key(remote_url)
|
||||
artifact_data = storage.download_object(cache_key)
|
||||
filename = os.path.basename(path)
|
||||
|
||||
content_type = "application/octet-stream"
|
||||
if filename.endswith(".tar.gz"):
|
||||
content_type = "application/gzip"
|
||||
elif filename.endswith(".zip"):
|
||||
content_type = "application/zip"
|
||||
elif filename.endswith(".exe"):
|
||||
content_type = "application/x-msdownload"
|
||||
elif filename.endswith(".rpm"):
|
||||
content_type = "application/x-rpm"
|
||||
elif filename.endswith(".xml"):
|
||||
content_type = "application/xml"
|
||||
elif filename.endswith((".xml.gz", ".xml.bz2", ".xml.xz")):
|
||||
content_type = "application/gzip"
|
||||
|
||||
# Record cache miss metrics
|
||||
metrics.record_cache_miss(remote_name, len(artifact_data))
|
||||
|
||||
# Record artifact mapping in database
|
||||
cache_key = storage.get_object_key(remote_url)
|
||||
database.record_artifact_mapping(
|
||||
cache_key, remote_name, path, len(artifact_data)
|
||||
)
|
||||
|
||||
return Response(
|
||||
content=artifact_data,
|
||||
media_type=content_type,
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename={filename}",
|
||||
"X-Artifact-Source": "remote",
|
||||
"X-Artifact-Size": str(len(artifact_data)),
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Error serving artifact: {str(e)}")
|
||||
|
||||
|
||||
async def discover_artifacts(remote: str, include_pattern: str) -> list[str]:
|
||||
if "github.com" in remote:
|
||||
return await discover_github_releases(remote, include_pattern)
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported remote: {remote}")
|
||||
|
||||
|
||||
async def discover_github_releases(remote: str, include_pattern: str) -> list[str]:
|
||||
match = re.match(r"github\.com/([^/]+)/([^/]+)", remote)
|
||||
if not match:
|
||||
raise HTTPException(status_code=400, detail="Invalid GitHub remote format")
|
||||
|
||||
owner, repo = match.groups()
|
||||
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
response = await client.get(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/releases"
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise HTTPException(
|
||||
status_code=response.status_code,
|
||||
detail=f"Failed to fetch releases: {response.text}",
|
||||
)
|
||||
|
||||
releases = response.json()
|
||||
|
||||
matching_urls = []
|
||||
pattern = include_pattern.replace("*", ".*")
|
||||
regex = re.compile(pattern)
|
||||
|
||||
for release in releases:
|
||||
for asset in release.get("assets", []):
|
||||
download_url = asset["browser_download_url"]
|
||||
if regex.search(download_url):
|
||||
matching_urls.append(download_url)
|
||||
|
||||
return matching_urls
|
||||
|
||||
|
||||
@app.put("/api/v1/remote/{remote_name}/{path:path}")
|
||||
async def upload_file(remote_name: str, path: str, file: UploadFile = File(...)):
|
||||
"""Upload a file to local repository"""
|
||||
# Check if remote is configured and is local
|
||||
remote_config = config.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Remote '{remote_name}' not configured"
|
||||
)
|
||||
|
||||
if remote_config.get("type") != "local":
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Upload only supported for local repositories"
|
||||
)
|
||||
|
||||
try:
|
||||
# Read file content
|
||||
content = await file.read()
|
||||
|
||||
# Calculate SHA256
|
||||
sha256_sum = hashlib.sha256(content).hexdigest()
|
||||
|
||||
# Check if file already exists (prevent overwrite)
|
||||
if database.file_exists(remote_name, path):
|
||||
raise HTTPException(status_code=409, detail="File already exists")
|
||||
|
||||
# Generate S3 key
|
||||
s3_key = f"local/{remote_name}/{path}"
|
||||
|
||||
# Determine content type
|
||||
content_type = file.content_type or "application/octet-stream"
|
||||
|
||||
# Upload to S3
|
||||
try:
|
||||
storage.upload(s3_key, content)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Upload failed: {e}")
|
||||
|
||||
# Add to database
|
||||
success = database.add_local_file(
|
||||
repository_name=remote_name,
|
||||
file_path=path,
|
||||
s3_key=s3_key,
|
||||
size_bytes=len(content),
|
||||
sha256_sum=sha256_sum,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
if not success:
|
||||
# Clean up S3 if database insert failed
|
||||
storage.delete_object(s3_key)
|
||||
raise HTTPException(status_code=500, detail="Failed to save file metadata")
|
||||
|
||||
return JSONResponse(
|
||||
{
|
||||
"message": "File uploaded successfully",
|
||||
"file_path": path,
|
||||
"size_bytes": len(content),
|
||||
"sha256_sum": sha256_sum,
|
||||
"content_type": content_type,
|
||||
}
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
|
||||
|
||||
|
||||
@app.head("/api/v1/remote/{remote_name}/{path:path}")
|
||||
def check_file_exists(remote_name: str, path: str):
|
||||
"""Check if file exists (for CI jobs) - supports local repositories only"""
|
||||
# Check if remote is configured
|
||||
remote_config = config.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Remote '{remote_name}' not configured"
|
||||
)
|
||||
|
||||
# Handle local repository
|
||||
if remote_config.get("type") == "local":
|
||||
try:
|
||||
metadata = database.get_local_file_metadata(remote_name, path)
|
||||
if not metadata:
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
|
||||
return Response(
|
||||
headers={
|
||||
"Content-Length": str(metadata["size_bytes"]),
|
||||
"Content-Type": metadata.get(
|
||||
"content_type", "application/octet-stream"
|
||||
),
|
||||
"X-SHA256": metadata["sha256_sum"],
|
||||
"X-Created-At": metadata["created_at"].isoformat()
|
||||
if metadata["created_at"]
|
||||
else "",
|
||||
"X-Uploaded-At": metadata["uploaded_at"].isoformat()
|
||||
if metadata["uploaded_at"]
|
||||
else "",
|
||||
}
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Check failed: {str(e)}")
|
||||
else:
|
||||
# For remote repositories, just return 405 Method Not Allowed
|
||||
raise HTTPException(
|
||||
status_code=405, detail="HEAD method only supported for local repositories"
|
||||
)
|
||||
|
||||
|
||||
@app.delete("/api/v1/remote/{remote_name}/{path:path}")
|
||||
def delete_file(remote_name: str, path: str):
|
||||
"""Delete a file from local repository"""
|
||||
# Check if remote is configured and is local
|
||||
remote_config = config.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Remote '{remote_name}' not configured"
|
||||
)
|
||||
|
||||
if remote_config.get("type") != "local":
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Delete only supported for local repositories"
|
||||
)
|
||||
|
||||
try:
|
||||
# Get S3 key before deleting from database
|
||||
s3_key = database.delete_local_file(remote_name, path)
|
||||
if not s3_key:
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
|
||||
# Delete from S3
|
||||
if not storage.delete_object(s3_key):
|
||||
# File was deleted from database but not from S3 - log warning but continue
|
||||
print(f"Warning: Failed to delete S3 object {s3_key}")
|
||||
|
||||
return JSONResponse({"message": "File deleted successfully"})
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Delete failed: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/api/v1/artifacts/cache")
|
||||
async def cache_artifact(request: ArtifactRequest) -> Dict[str, Any]:
|
||||
try:
|
||||
matching_urls = await discover_artifacts(
|
||||
request.remote, request.include_pattern
|
||||
)
|
||||
|
||||
if not matching_urls:
|
||||
return {
|
||||
"message": "No matching artifacts found",
|
||||
"cached_count": 0,
|
||||
"artifacts": [],
|
||||
}
|
||||
|
||||
cached_artifacts = []
|
||||
|
||||
for url in matching_urls:
|
||||
result = await cache_single_artifact(url, "", "")
|
||||
cached_artifacts.append(result)
|
||||
|
||||
cached_count = sum(
|
||||
1
|
||||
for artifact in cached_artifacts
|
||||
if artifact["status"] in ["cached", "already_cached"]
|
||||
)
|
||||
|
||||
return {
|
||||
"message": f"Processed {len(matching_urls)} artifacts, {cached_count} successfully cached",
|
||||
"cached_count": cached_count,
|
||||
"artifacts": cached_artifacts,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@app.get("/api/v1/artifacts/{remote:path}")
|
||||
async def list_cached_artifacts(
|
||||
remote: str, include_pattern: str = ".*"
|
||||
) -> Dict[str, Any]:
|
||||
try:
|
||||
matching_urls = await discover_artifacts(remote, include_pattern)
|
||||
|
||||
cached_artifacts = []
|
||||
for url in matching_urls:
|
||||
key = storage.get_object_key(url)
|
||||
if storage.exists(key):
|
||||
cached_artifacts.append(
|
||||
{"url": url, "cached_url": storage.get_url(key), "key": key}
|
||||
)
|
||||
|
||||
return {
|
||||
"remote": remote,
|
||||
"pattern": include_pattern,
|
||||
"total_found": len(matching_urls),
|
||||
"cached_count": len(cached_artifacts),
|
||||
"artifacts": cached_artifacts,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@app.get("/metrics")
|
||||
def get_metrics(
|
||||
json: Optional[bool] = Query(
|
||||
False, description="Return JSON format instead of Prometheus"
|
||||
),
|
||||
):
|
||||
"""Get comprehensive metrics about the artifact storage system"""
|
||||
config._check_reload()
|
||||
|
||||
if json:
|
||||
# Return JSON format
|
||||
return metrics.get_metrics(storage, config)
|
||||
else:
|
||||
# Return Prometheus format
|
||||
metrics.get_metrics(storage, config) # Update gauges
|
||||
prometheus_data = generate_latest().decode("utf-8")
|
||||
return PlainTextResponse(prometheus_data, media_type=CONTENT_TYPE_LATEST)
|
||||
|
||||
|
||||
@app.get("/config")
|
||||
def get_config():
|
||||
return config.config
|
||||
|
||||
|
||||
def main():
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,229 @@
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any
|
||||
from prometheus_client import Counter, Gauge
|
||||
|
||||
|
||||
# Prometheus metrics
|
||||
request_counter = Counter(
|
||||
"artifact_requests_total", "Total artifact requests", ["remote", "status"]
|
||||
)
|
||||
cache_hit_counter = Counter("artifact_cache_hits_total", "Total cache hits", ["remote"])
|
||||
cache_miss_counter = Counter(
|
||||
"artifact_cache_misses_total", "Total cache misses", ["remote"]
|
||||
)
|
||||
bandwidth_saved_counter = Counter(
|
||||
"artifact_bandwidth_saved_bytes_total", "Total bandwidth saved", ["remote"]
|
||||
)
|
||||
storage_size_gauge = Gauge(
|
||||
"artifact_storage_size_bytes", "Storage size by remote", ["remote"]
|
||||
)
|
||||
redis_keys_gauge = Gauge("artifact_redis_keys_total", "Total Redis keys")
|
||||
|
||||
|
||||
class MetricsManager:
|
||||
def __init__(self, redis_client=None, database_manager=None):
|
||||
self.redis_client = redis_client
|
||||
self.database_manager = database_manager
|
||||
self.start_time = datetime.now()
|
||||
|
||||
def record_cache_hit(self, remote_name: str, size_bytes: int):
|
||||
"""Record a cache hit with size for bandwidth calculation"""
|
||||
# Update Prometheus metrics
|
||||
request_counter.labels(remote=remote_name, status="cache_hit").inc()
|
||||
cache_hit_counter.labels(remote=remote_name).inc()
|
||||
bandwidth_saved_counter.labels(remote=remote_name).inc(size_bytes)
|
||||
|
||||
# Update Redis for persistence across instances
|
||||
if self.redis_client and self.redis_client.available:
|
||||
try:
|
||||
# Increment global counters
|
||||
self.redis_client.client.incr("metrics:cache_hits")
|
||||
self.redis_client.client.incr("metrics:total_requests")
|
||||
self.redis_client.client.incrby("metrics:bandwidth_saved", size_bytes)
|
||||
|
||||
# Increment per-remote counters
|
||||
self.redis_client.client.incr(f"metrics:cache_hits:{remote_name}")
|
||||
self.redis_client.client.incr(f"metrics:total_requests:{remote_name}")
|
||||
self.redis_client.client.incrby(
|
||||
f"metrics:bandwidth_saved:{remote_name}", size_bytes
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def record_cache_miss(self, remote_name: str, size_bytes: int):
|
||||
"""Record a cache miss (new download)"""
|
||||
# Update Prometheus metrics
|
||||
request_counter.labels(remote=remote_name, status="cache_miss").inc()
|
||||
cache_miss_counter.labels(remote=remote_name).inc()
|
||||
|
||||
# Update Redis for persistence across instances
|
||||
if self.redis_client and self.redis_client.available:
|
||||
try:
|
||||
# Increment global counters
|
||||
self.redis_client.client.incr("metrics:cache_misses")
|
||||
self.redis_client.client.incr("metrics:total_requests")
|
||||
|
||||
# Increment per-remote counters
|
||||
self.redis_client.client.incr(f"metrics:cache_misses:{remote_name}")
|
||||
self.redis_client.client.incr(f"metrics:total_requests:{remote_name}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_redis_key_count(self) -> int:
|
||||
"""Get total number of keys in Redis"""
|
||||
if self.redis_client and self.redis_client.available:
|
||||
try:
|
||||
return self.redis_client.client.dbsize()
|
||||
except Exception:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
def get_s3_total_size(self, storage) -> int:
|
||||
"""Get total size of all objects in S3 bucket"""
|
||||
try:
|
||||
total_size = 0
|
||||
paginator = storage.client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=storage.bucket):
|
||||
if "Contents" in page:
|
||||
for obj in page["Contents"]:
|
||||
total_size += obj["Size"]
|
||||
return total_size
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def get_s3_size_by_remote(self, storage, config_manager) -> Dict[str, int]:
|
||||
"""Get size of stored data per remote using database mappings"""
|
||||
if self.database_manager and self.database_manager.available:
|
||||
# Get from database if available
|
||||
db_sizes = self.database_manager.get_storage_by_remote()
|
||||
if db_sizes:
|
||||
# Initialize all configured remotes to 0
|
||||
remote_sizes = {}
|
||||
for remote in config_manager.config.get("remotes", {}).keys():
|
||||
remote_sizes[remote] = db_sizes.get(remote, 0)
|
||||
|
||||
# Update Prometheus gauges
|
||||
for remote, size in remote_sizes.items():
|
||||
storage_size_gauge.labels(remote=remote).set(size)
|
||||
|
||||
return remote_sizes
|
||||
|
||||
# Fallback to S3 scanning if database not available
|
||||
try:
|
||||
remote_sizes = {}
|
||||
remotes = config_manager.config.get("remotes", {}).keys()
|
||||
|
||||
# Initialize all remotes to 0
|
||||
for remote in remotes:
|
||||
remote_sizes[remote] = 0
|
||||
|
||||
paginator = storage.client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=storage.bucket):
|
||||
if "Contents" in page:
|
||||
for obj in page["Contents"]:
|
||||
key = obj["Key"]
|
||||
# Try to map from database first
|
||||
remote = None
|
||||
if self.database_manager:
|
||||
remote = self.database_manager.get_remote_for_s3_key(key)
|
||||
|
||||
# Fallback to key parsing
|
||||
if not remote:
|
||||
remote = key.split("/")[0] if "/" in key else "unknown"
|
||||
|
||||
if remote in remote_sizes:
|
||||
remote_sizes[remote] += obj["Size"]
|
||||
else:
|
||||
remote_sizes.setdefault("unknown", 0)
|
||||
remote_sizes["unknown"] += obj["Size"]
|
||||
|
||||
# Update Prometheus gauges
|
||||
for remote, size in remote_sizes.items():
|
||||
if remote != "unknown": # Don't set gauge for unknown
|
||||
storage_size_gauge.labels(remote=remote).set(size)
|
||||
|
||||
return remote_sizes
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def get_metrics(self, storage, config_manager) -> Dict[str, Any]:
|
||||
"""Get comprehensive metrics"""
|
||||
# Update Redis keys gauge
|
||||
redis_key_count = self.get_redis_key_count()
|
||||
redis_keys_gauge.set(redis_key_count)
|
||||
|
||||
metrics = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"uptime_seconds": int((datetime.now() - self.start_time).total_seconds()),
|
||||
"redis": {"total_keys": redis_key_count},
|
||||
"storage": {
|
||||
"total_size_bytes": self.get_s3_total_size(storage),
|
||||
"size_by_remote": self.get_s3_size_by_remote(storage, config_manager),
|
||||
},
|
||||
"requests": {
|
||||
"cache_hits": 0,
|
||||
"cache_misses": 0,
|
||||
"total_requests": 0,
|
||||
"cache_hit_ratio": 0.0,
|
||||
},
|
||||
"bandwidth": {"saved_bytes": 0},
|
||||
"per_remote": {},
|
||||
}
|
||||
|
||||
if self.redis_client and self.redis_client.available:
|
||||
try:
|
||||
# Get global metrics
|
||||
cache_hits = int(
|
||||
self.redis_client.client.get("metrics:cache_hits") or 0
|
||||
)
|
||||
cache_misses = int(
|
||||
self.redis_client.client.get("metrics:cache_misses") or 0
|
||||
)
|
||||
total_requests = cache_hits + cache_misses
|
||||
bandwidth_saved = int(
|
||||
self.redis_client.client.get("metrics:bandwidth_saved") or 0
|
||||
)
|
||||
|
||||
metrics["requests"]["cache_hits"] = cache_hits
|
||||
metrics["requests"]["cache_misses"] = cache_misses
|
||||
metrics["requests"]["total_requests"] = total_requests
|
||||
metrics["requests"]["cache_hit_ratio"] = (
|
||||
cache_hits / total_requests if total_requests > 0 else 0.0
|
||||
)
|
||||
metrics["bandwidth"]["saved_bytes"] = bandwidth_saved
|
||||
|
||||
# Get per-remote metrics
|
||||
for remote in config_manager.config.get("remotes", {}).keys():
|
||||
remote_cache_hits = int(
|
||||
self.redis_client.client.get(f"metrics:cache_hits:{remote}")
|
||||
or 0
|
||||
)
|
||||
remote_cache_misses = int(
|
||||
self.redis_client.client.get(f"metrics:cache_misses:{remote}")
|
||||
or 0
|
||||
)
|
||||
remote_total = remote_cache_hits + remote_cache_misses
|
||||
remote_bandwidth_saved = int(
|
||||
self.redis_client.client.get(
|
||||
f"metrics:bandwidth_saved:{remote}"
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
metrics["per_remote"][remote] = {
|
||||
"cache_hits": remote_cache_hits,
|
||||
"cache_misses": remote_cache_misses,
|
||||
"total_requests": remote_total,
|
||||
"cache_hit_ratio": remote_cache_hits / remote_total
|
||||
if remote_total > 0
|
||||
else 0.0,
|
||||
"bandwidth_saved_bytes": remote_bandwidth_saved,
|
||||
"storage_size_bytes": metrics["storage"]["size_by_remote"].get(
|
||||
remote, 0
|
||||
),
|
||||
}
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return metrics
|
||||
@@ -0,0 +1,110 @@
|
||||
import os
|
||||
import hashlib
|
||||
from urllib.parse import urlparse
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
class S3Storage:
|
||||
def __init__(
|
||||
self,
|
||||
endpoint: str,
|
||||
access_key: str,
|
||||
secret_key: str,
|
||||
bucket: str,
|
||||
secure: bool = False,
|
||||
):
|
||||
self.endpoint = endpoint
|
||||
self.access_key = access_key
|
||||
self.secret_key = secret_key
|
||||
self.bucket = bucket
|
||||
self.secure = secure
|
||||
|
||||
ca_bundle = os.environ.get('REQUESTS_CA_BUNDLE') or os.environ.get('SSL_CERT_FILE')
|
||||
config_kwargs = {
|
||||
"request_checksum_calculation": "when_required",
|
||||
"response_checksum_validation": "when_required"
|
||||
}
|
||||
client_kwargs = {
|
||||
"endpoint_url": f"http{'s' if self.secure else ''}://{self.endpoint}",
|
||||
"aws_access_key_id": self.access_key,
|
||||
"aws_secret_access_key": self.secret_key,
|
||||
"config": Config(**config_kwargs)
|
||||
}
|
||||
|
||||
if ca_bundle and os.path.exists(ca_bundle):
|
||||
client_kwargs["verify"] = ca_bundle
|
||||
print(f"Debug: Using CA bundle: {ca_bundle}")
|
||||
else:
|
||||
print(f"Debug: No CA bundle found. REQUESTS_CA_BUNDLE={os.environ.get('REQUESTS_CA_BUNDLE')}, SSL_CERT_FILE={os.environ.get('SSL_CERT_FILE')}")
|
||||
|
||||
self.client = boto3.client("s3", **client_kwargs)
|
||||
|
||||
# Try to ensure bucket exists, but don't fail if MinIO isn't ready yet
|
||||
try:
|
||||
self._ensure_bucket_exists()
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not ensure bucket exists during initialization: {e}")
|
||||
print("Bucket creation will be attempted on first use")
|
||||
|
||||
def _ensure_bucket_exists(self):
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
except ClientError:
|
||||
self.client.create_bucket(Bucket=self.bucket)
|
||||
|
||||
def get_object_key(self, url: str) -> str:
|
||||
url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
|
||||
parsed = urlparse(url)
|
||||
filename = os.path.basename(parsed.path)
|
||||
return f"{parsed.netloc}/{url_hash}/{filename}"
|
||||
|
||||
def get_object_key_from_path(self, remote_name: str, path: str) -> str:
|
||||
# Create a key based on the API path for direct access
|
||||
path_hash = hashlib.sha256(path.encode()).hexdigest()[:16]
|
||||
filename = os.path.basename(path)
|
||||
return f"{remote_name}/{path_hash}/{filename}"
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
try:
|
||||
self._ensure_bucket_exists()
|
||||
self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except ClientError:
|
||||
return False
|
||||
|
||||
def upload(self, key: str, data: bytes) -> str:
|
||||
self._ensure_bucket_exists()
|
||||
self.client.put_object(Bucket=self.bucket, Key=key, Body=data)
|
||||
return f"s3://{self.bucket}/{key}"
|
||||
|
||||
def get_url(self, key: str) -> str:
|
||||
return f"http://{self.endpoint}/{self.bucket}/{key}"
|
||||
|
||||
def get_presigned_url(self, key: str, expiration: int = 3600) -> str:
|
||||
try:
|
||||
return self.client.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": self.bucket, "Key": key},
|
||||
ExpiresIn=expiration,
|
||||
)
|
||||
except Exception:
|
||||
return self.get_url(key)
|
||||
|
||||
def download_object(self, key: str) -> bytes:
|
||||
try:
|
||||
self._ensure_bucket_exists()
|
||||
response = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
return response["Body"].read()
|
||||
except ClientError:
|
||||
raise HTTPException(status_code=404, detail="Artifact not found")
|
||||
|
||||
def delete_object(self, key: str) -> bool:
|
||||
try:
|
||||
self._ensure_bucket_exists()
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except ClientError:
|
||||
return False
|
||||
@@ -1,2 +0,0 @@
|
||||
node_modules/
|
||||
dist/
|
||||
@@ -1,25 +0,0 @@
|
||||
FROM node:22-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package-lock.json* ./
|
||||
RUN npm ci
|
||||
|
||||
COPY . .
|
||||
|
||||
ARG BASE_PATH=/
|
||||
ENV BASE_PATH=${BASE_PATH}
|
||||
RUN npm run build
|
||||
|
||||
FROM nginx:alpine
|
||||
|
||||
ARG BASE_PATH=/
|
||||
|
||||
COPY --from=builder /app/dist /usr/share/nginx/html
|
||||
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||
|
||||
RUN sed -i "s|\${BASE_PATH}|${BASE_PATH}|g" /etc/nginx/conf.d/default.conf
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
CMD ["nginx", "-g", "daemon off;"]
|
||||
@@ -1,12 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>ArtifactAPI</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,16 +0,0 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
location ${BASE_PATH}/ {
|
||||
rewrite ^${BASE_PATH}(/.*)$ $1 break;
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
location = ${BASE_PATH} {
|
||||
return 301 ${BASE_PATH}/;
|
||||
}
|
||||
}
|
||||
Generated
-2758
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user