From 82bb5708c8ee50f098b78566fbf591a32bf65ba8 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Wed, 2 Sep 2026 00:18:46 +1000 Subject: [PATCH] Adopt golib/pg for migrations and pool construction artifactapi's schema was a ~180-line inline DDL blob re-executed on every start, growing an ALTER TABLE ... IF NOT EXISTS line per change with nothing recording what had run. golib/pg already owns that mechanic for the estate, so move the SQL into a versioned, embedded set and let the library apply it. - Depend on git.unkin.net/unkin/golib v0.1.0. - Move the DDL verbatim to migrations/0001_init.sql, embedded via the new migrations package, and build the pool with pg.NewMigrated (LockName "artifactapi-migrations"). The runner adds a cluster-wide advisory lock the old blob never took, so replicas starting together queue instead of racing each other through the DDL. - The live database has the schema but no schema_migrations, so its first start on this build re-runs 0001. Every statement is IF NOT EXISTS-guarded, so that run is a no-op landing only the tracking row; a container-backed test drops the row from a migrated database and asserts exactly that, and a static guard keeps future migrations additive and idempotent. - Keep config.DatabaseDSN as the DSN source rather than pg.DSNFromEnv: the variable names match, but golib has no default user or database name, and artifactapi documents and ships DBUSER/DBNAME defaults of "artifacts". The deployed env var contract is unchanged. - Guard the embedded set against migrations/ and pin the derived advisory key, so neither can drift unnoticed. - Plumb GOPRIVATE=git.unkin.net for the first cross-repo Go dependency: exported by the Makefile, set in the Dockerfile and the woodpecker Go steps, documented in the README. --- .woodpecker/pre-commit.yaml | 3 + .woodpecker/test.yaml | 3 + Dockerfile | 4 + Makefile | 5 + README.md | 39 +++++ go.mod | 45 ++--- go.sum | 104 ++++++------ internal/database/migrations_test.go | 162 ++++++++++++++++++ internal/database/postgres.go | 245 ++------------------------- migrations/0001_init.sql | 206 ++++++++++++++++++++++ migrations/embed.go | 11 ++ 11 files changed, 524 insertions(+), 303 deletions(-) create mode 100644 internal/database/migrations_test.go create mode 100644 migrations/0001_init.sql create mode 100644 migrations/embed.go diff --git a/.woodpecker/pre-commit.yaml b/.woodpecker/pre-commit.yaml index c250c97..fdafb10 100644 --- a/.woodpecker/pre-commit.yaml +++ b/.woodpecker/pre-commit.yaml @@ -6,6 +6,9 @@ steps: image: git.unkin.net/unkin/almalinux9-gobuilder:20260606 commands: - uvx pre-commit run --all-files + environment: + # golib lives on Gitea; skip the public proxy/sum db. + GOPRIVATE: git.unkin.net backend_options: kubernetes: resources: diff --git a/.woodpecker/test.yaml b/.woodpecker/test.yaml index 7bffc3e..5f75c37 100644 --- a/.woodpecker/test.yaml +++ b/.woodpecker/test.yaml @@ -6,3 +6,6 @@ steps: image: golang:1.25 commands: - go test -race -count=1 ./pkg/... ./internal/... + environment: + # golib lives on Gitea; skip the public proxy/sum db. + GOPRIVATE: git.unkin.net diff --git a/Dockerfile b/Dockerfile index c6d9db9..aca988a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,10 @@ RUN apk add --no-cache git WORKDIR /build +# golib is fetched straight from Gitea; the public proxy and sum db have no +# view of git.unkin.net modules. +ENV GOPRIVATE=git.unkin.net + COPY go.mod go.sum ./ RUN go mod download diff --git a/Makefile b/Makefile index 62b99aa..24b6a74 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,11 @@ VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0 GO_VERSION_REQUIRED := 1.23 GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/') +# git.unkin.net modules (golib) are fetched straight from Gitea, never via the +# public proxy or sum db, which have no view of them. Exported here rather than +# written with `go env -w`, so a fresh checkout needs no machine-local setup. +export GOPRIVATE := git.unkin.net + check-go: @if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \ echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \ diff --git a/README.md b/README.md index 734b71a..214850c 100644 --- a/README.md +++ b/README.md @@ -321,6 +321,28 @@ S3/MinIO ─── content-addressable blob storage (blobs/sha256/{hash}) S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go). +### Schema migrations + +The SQL schema lives in `migrations/` as numbered `.sql` files, embedded into +the binary and applied at startup before the server listens, so there is no +mirrored copy of the schema in the deployment to drift out of sync. + +The runner is [`golib/pg`](https://git.unkin.net/unkin/golib)'s `pg.NewMigrated` +— artifactapi owns the SQL, the shared library owns the mechanics. + +Each start takes `pg_advisory_lock` on a fixed key (FNV-1a/64 of the lock name +`artifactapi-migrations`), creates `schema_migrations` (`version`, `applied_at`) +if missing, and applies every embedded file whose filename is not yet recorded — +in lexical (version) order, each file's SQL and its tracking row in one +transaction — then unlocks. Replicas starting together queue on the lock and +then find nothing to do. + +A file absent from `schema_migrations` is re-run even when the database already +has the schema, which is how a database migrated by the old untracked inline DDL +picks `0001_init.sql` up: the statements are `IF NOT EXISTS`-guarded, so the +re-run is a no-op that only lands the tracking row. New migrations must stay +additive and idempotent for the same reason; a test enforces it. + ## Environment Variables | Variable | Default | Description | @@ -331,6 +353,7 @@ S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go). | `DBUSER` | `artifacts` | PostgreSQL user | | `DBPASS` | | PostgreSQL password | | `DBNAME` | `artifacts` | PostgreSQL database | +| `DBSSL` | `disable` | PostgreSQL `sslmode` | | `REDIS_URL` | `redis://localhost:6379` | Redis URL | | `MINIO_ENDPOINT` | `localhost:9000` | S3 endpoint | | `MINIO_ACCESS_KEY` | | S3 access key | @@ -358,6 +381,22 @@ make lint # golangci-lint + go vet make fmt # gofmt + goimports ``` +### `GOPRIVATE` + +artifactapi depends on `git.unkin.net/unkin/golib`, which is served by Gitea and +is unknown to `proxy.golang.org` / `sum.golang.org`. Module resolution therefore +needs: + +``` +export GOPRIVATE=git.unkin.net +``` + +The `Makefile` exports it for every target, and the `Dockerfile` and the +woodpecker Go steps set it themselves, so `make build|test|lint` and CI work on +a clean checkout. Only bare `go` commands run outside `make` need it in your +shell — set it there (or in your shell profile) rather than with `go env -w`, +which is machine state this repo cannot carry. + ### TUI ```bash diff --git a/go.mod b/go.mod index 57d1236..6333d15 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module git.unkin.net/unkin/artifactapi go 1.25.9 require ( + git.unkin.net/unkin/golib v0.1.0 github.com/cavaliergopher/rpm v1.3.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 @@ -12,11 +13,11 @@ require ( github.com/klauspost/compress v1.19.2 github.com/minio/minio-go/v7 v7.2.0 github.com/redis/go-redis/v9 v9.20.0 - github.com/testcontainers/testcontainers-go v0.42.0 - github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 + github.com/testcontainers/testcontainers-go v0.44.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 github.com/ulikunitz/xz v0.5.16 - golang.org/x/crypto v0.51.0 + golang.org/x/crypto v0.54.0 golang.org/x/time v0.15.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -41,22 +42,22 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ebitengine/purego v0.10.0 // indirect + github.com/ebitengine/purego v0.10.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect @@ -66,10 +67,10 @@ require ( github.com/minio/md5-simd v1.1.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect - github.com/moby/moby/api v1.54.1 // indirect - github.com/moby/moby/client v0.4.0 // indirect + github.com/moby/moby/api v1.55.0 // indirect + github.com/moby/moby/client v0.5.0 // indirect github.com/moby/patternmatcher v0.6.1 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect @@ -83,25 +84,25 @@ require ( github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/xid v1.6.0 // indirect - github.com/shirou/gopsutil/v4 v4.26.3 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/stretchr/testify v1.11.1 // indirect github.com/tinylib/msgp v1.6.1 // indirect - github.com/tklauser/go-sysconf v0.3.16 // indirect - github.com/tklauser/numcpus v0.11.0 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.53.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.44.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect gopkg.in/ini.v1 v1.67.2 // indirect ) diff --git a/go.sum b/go.sum index 7d07007..b77ce4b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +git.unkin.net/unkin/golib v0.1.0 h1:OjKT5TO7PuXiYQ/1A+I4S2VZ/IsAxXY1reGWwasDMqE= +git.unkin.net/unkin/golib v0.1.0/go.mod h1:1e3PpMLEfa03Re/6tlk+I2XPWzBQMSpw2MB+Aw2lkX4= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= @@ -51,18 +53,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= -github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= -github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM= github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -70,9 +72,9 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -100,8 +102,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -122,14 +124,14 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= -github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4= -github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= -github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= -github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= +github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= @@ -160,8 +162,8 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= -github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -177,18 +179,18 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= -github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= -github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo= -github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs= +github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI= +github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE= +github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 h1:8fdv/9y3JMxjQ+ULAcOG8RtgeNu5t9XF9LolSXDuTwM= +github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0/go.mod h1:CFr2LncGYokw+OKjXcr8ARCKG1SaC2UEnGxFBovE86g= github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 h1:id/6LH8ZeDrtAUVSuNvZUAJ1kVpb82y1pr9yweAWsRg= github.com/testcontainers/testcontainers-go/modules/redis v0.42.0/go.mod h1:uF0jI8FITagQpBNOgweGBmPf6rP4K0SeL1XFPbsZSSY= github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= -github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= -github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= -github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= -github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= @@ -201,44 +203,44 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= -golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/database/migrations_test.go b/internal/database/migrations_test.go new file mode 100644 index 0000000..5e75c01 --- /dev/null +++ b/internal/database/migrations_test.go @@ -0,0 +1,162 @@ +package database + +import ( + "context" + "io/fs" + "maps" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + + "git.unkin.net/unkin/golib/pg" + + "git.unkin.net/unkin/artifactapi/migrations" +) + +// migrationsDir is the repo's migrations/ directory, relative to this package. +const migrationsDir = "../../migrations" + +func readMigrationsFromDisk(t *testing.T) map[string]string { + t.Helper() + entries, err := os.ReadDir(migrationsDir) + if err != nil { + t.Fatalf("read %s: %v", migrationsDir, err) + } + files := map[string]string{} + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") { + continue + } + b, err := os.ReadFile(filepath.Join(migrationsDir, e.Name())) + if err != nil { + t.Fatalf("read %s: %v", e.Name(), err) + } + files[e.Name()] = string(b) + } + if len(files) == 0 { + t.Fatalf("no .sql files in %s", migrationsDir) + } + return files +} + +// The embedded set is the shipped schema, so it must match the migrations/ +// directory exactly — a file added on disk but not embedded would never run. +func TestEmbeddedMigrationsMatchDirectory(t *testing.T) { + onDisk := readMigrationsFromDisk(t) + entries, err := fs.ReadDir(migrations.FS, ".") + if err != nil { + t.Fatalf("read embedded migrations: %v", err) + } + embedded := map[string]string{} + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") { + continue + } + b, err := migrations.FS.ReadFile(e.Name()) + if err != nil { + t.Fatalf("read embedded %s: %v", e.Name(), err) + } + embedded[e.Name()] = string(b) + } + if len(embedded) != len(onDisk) { + t.Fatalf("embedded %d files, migrations/ has %d", len(embedded), len(onDisk)) + } + for name, body := range embedded { + want, ok := onDisk[name] + if !ok { + t.Errorf("%s is embedded but not in migrations/", name) + continue + } + if body != want { + t.Errorf("%s: embedded body differs from migrations/%s", name, name) + } + } +} + +var ( + createTableRe = regexp.MustCompile(`(?i)\bCREATE\s+TABLE\b(\s+IF\s+NOT\s+EXISTS\b)?`) + createIndexRe = regexp.MustCompile(`(?i)\bCREATE\s+(UNIQUE\s+)?INDEX\b(\s+IF\s+NOT\s+EXISTS\b)?`) + addColumnRe = regexp.MustCompile(`(?i)\bADD\s+COLUMN\b(\s+IF\s+NOT\s+EXISTS\b)?`) + destructiveRe = regexp.MustCompile(`(?i)\b(DROP\s+(TABLE|COLUMN|INDEX)|TRUNCATE|DELETE\s+FROM)\b`) +) + +// A migration absent from schema_migrations is re-run even when the live +// database already has the schema, which is exactly how the deployed database — +// migrated for years by an untracked inline DDL blob — picks 0001 up. Every +// statement must therefore be idempotent, or that first tracked run would fail +// against production. +func TestMigrationsAreIdempotent(t *testing.T) { + for name, body := range readMigrationsFromDisk(t) { + for _, m := range createTableRe.FindAllStringSubmatch(body, -1) { + if m[1] == "" { + t.Errorf("%s: %q is not IF NOT EXISTS-guarded", name, strings.Join(strings.Fields(m[0]), " ")) + } + } + for _, m := range createIndexRe.FindAllStringSubmatch(body, -1) { + if m[2] == "" { + t.Errorf("%s: %q is not IF NOT EXISTS-guarded", name, strings.Join(strings.Fields(m[0]), " ")) + } + } + for _, m := range addColumnRe.FindAllStringSubmatch(body, -1) { + if m[1] == "" { + t.Errorf("%s: %q is not IF NOT EXISTS-guarded", name, strings.Join(strings.Fields(m[0]), " ")) + } + } + if loc := destructiveRe.FindString(body); loc != "" { + t.Errorf("%s: destructive statement %q; migrations are additive", name, loc) + } + } +} + +// The deployed database was built by the untracked inline DDL this runner +// replaces, so its very first tracked start runs 0001 against a schema that +// already exists. Reproduce that by dropping the tracking row from an +// already-migrated database and starting again: it must succeed and re-record +// the version, changing nothing else. +func TestMigratingAnAlreadyPopulatedSchemaIsANoOp(t *testing.T) { + requireDB(t) + c := context.Background() + + if _, err := testDB.Pool.Exec(c, "DELETE FROM schema_migrations"); err != nil { + t.Fatalf("clear schema_migrations: %v", err) + } + db, err := New(testDSN) + if err != nil { + t.Fatalf("migrate over an existing schema: %v", err) + } + defer db.Close() + + var versions []string + rows, err := db.Pool.Query(c, "SELECT version FROM schema_migrations ORDER BY version") + if err != nil { + t.Fatalf("read schema_migrations: %v", err) + } + defer rows.Close() + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + t.Fatalf("scan version: %v", err) + } + versions = append(versions, v) + } + if err := rows.Err(); err != nil { + t.Fatalf("read schema_migrations: %v", err) + } + + want := slices.Sorted(maps.Keys(readMigrationsFromDisk(t))) + if !slices.Equal(versions, want) { + t.Fatalf("schema_migrations = %v, want %v", versions, want) + } +} + +// The advisory lock key is derived from migrationLockName by golib. Pin it so a +// rename cannot silently let two builds migrate the same cluster at once. +func TestMigrationLockKeyIsPinned(t *testing.T) { + const wantKey int64 = -6981019939451326383 + if got := pg.LockKey(migrationLockName); got != wantKey { + t.Fatalf("LockKey(%q) = %d, want %d", migrationLockName, got, wantKey) + } +} diff --git a/internal/database/postgres.go b/internal/database/postgres.go index 4e5839a..fca5e5f 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -2,249 +2,34 @@ package database import ( "context" - "fmt" "github.com/jackc/pgx/v5/pgxpool" + + "git.unkin.net/unkin/golib/pg" + + "git.unkin.net/unkin/artifactapi/migrations" ) +// migrationLockName names the cluster-wide advisory lock the migration run +// contends for; golib derives the key as FNV-1a/64 of it. Replicas starting +// together queue on it instead of racing each other through the DDL. +const migrationLockName = "artifactapi-migrations" + type DB struct { Pool *pgxpool.Pool } func New(dsn string) (*DB, error) { - pool, err := pgxpool.New(context.Background(), dsn) + ctx := context.Background() + pool, err := pg.NewMigrated(ctx, dsn, migrations.FS, pg.MigrateOptions{ + LockName: migrationLockName, + }) if err != nil { - return nil, fmt.Errorf("connect to postgres: %w", err) + return nil, err } - - if err := pool.Ping(context.Background()); err != nil { - pool.Close() - return nil, fmt.Errorf("ping postgres: %w", err) - } - - db := &DB{Pool: pool} - if err := db.migrate(); err != nil { - pool.Close() - return nil, fmt.Errorf("run migrations: %w", err) - } - - return db, nil + return &DB{Pool: pool}, nil } func (db *DB) Close() { db.Pool.Close() } - -func (db *DB) migrate() error { - ctx := context.Background() - - _, err := db.Pool.Exec(ctx, ` - CREATE TABLE IF NOT EXISTS remotes ( - name TEXT PRIMARY KEY, - package_type TEXT NOT NULL, - repo_type TEXT DEFAULT 'remote', - base_url TEXT NOT NULL DEFAULT '', - mirrorlist TEXT[] DEFAULT '{}', - mirror_strategy TEXT NOT NULL DEFAULT 'round_robin', - description TEXT DEFAULT '', - username TEXT DEFAULT '', - password TEXT DEFAULT '', - immutable_ttl INTEGER DEFAULT 0, - mutable_ttl INTEGER DEFAULT 3600, - check_mutable BOOLEAN DEFAULT TRUE, - patterns TEXT[] DEFAULT '{}', - blocklist TEXT[] DEFAULT '{}', - mutable_patterns TEXT[] DEFAULT '{}', - immutable_patterns TEXT[] DEFAULT '{}', - ban_tags_enabled BOOLEAN DEFAULT FALSE, - ban_tags TEXT[] DEFAULT '{}', - quarantine_enabled BOOLEAN DEFAULT FALSE, - quarantine_days INTEGER DEFAULT 3, - stale_on_error BOOLEAN DEFAULT TRUE, - releases_remote TEXT DEFAULT '', - managed_by TEXT DEFAULT '', - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW() - ); - - CREATE TABLE IF NOT EXISTS virtuals ( - name TEXT PRIMARY KEY, - package_type TEXT NOT NULL, - description TEXT DEFAULT '', - members TEXT[] NOT NULL, - managed_by TEXT DEFAULT '', - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW() - ); - - CREATE TABLE IF NOT EXISTS blobs ( - content_hash TEXT PRIMARY KEY, - s3_key TEXT NOT NULL, - size_bytes BIGINT NOT NULL, - content_type TEXT DEFAULT 'application/octet-stream', - created_at TIMESTAMPTZ DEFAULT NOW() - ); - - CREATE TABLE IF NOT EXISTS artifacts ( - id BIGSERIAL PRIMARY KEY, - remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE, - path TEXT NOT NULL, - content_hash TEXT NOT NULL REFERENCES blobs(content_hash), - upstream_etag TEXT DEFAULT '', - upstream_last_modified TIMESTAMPTZ, - first_seen_at TIMESTAMPTZ DEFAULT NOW(), - last_fetched_at TIMESTAMPTZ DEFAULT NOW(), - last_accessed_at TIMESTAMPTZ DEFAULT NOW(), - fetch_count BIGINT DEFAULT 1, - access_count BIGINT DEFAULT 1, - UNIQUE(remote_name, path) - ); - - CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name); - CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); - - CREATE TABLE IF NOT EXISTS local_files ( - id BIGSERIAL PRIMARY KEY, - repo_name TEXT NOT NULL, - file_path TEXT NOT NULL, - content_hash TEXT NOT NULL REFERENCES blobs(content_hash), - created_at TIMESTAMPTZ DEFAULT NOW(), - UNIQUE(repo_name, file_path) - ); - - CREATE TABLE IF NOT EXISTS access_log ( - id BIGSERIAL PRIMARY KEY, - remote_name TEXT NOT NULL, - path TEXT NOT NULL, - cache_hit BOOLEAN NOT NULL, - size_bytes BIGINT DEFAULT 0, - upstream_ms INTEGER DEFAULT 0, - client_ip TEXT DEFAULT '', - created_at TIMESTAMPTZ DEFAULT NOW() - ); - - CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at); - - ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote'; - ALTER TABLE remotes ADD COLUMN IF NOT EXISTS mirrorlist TEXT[] DEFAULT '{}'; - ALTER TABLE remotes ADD COLUMN IF NOT EXISTS mirror_strategy TEXT NOT NULL DEFAULT 'round_robin'; - ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0; - ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0; - ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0; - - CREATE TABLE IF NOT EXISTS rpm_metadata ( - id BIGSERIAL PRIMARY KEY, - repo_name TEXT NOT NULL, - file_path TEXT NOT NULL, - content_hash TEXT NOT NULL, - name TEXT NOT NULL, - epoch INTEGER DEFAULT 0, - version TEXT NOT NULL, - release TEXT NOT NULL, - arch TEXT NOT NULL, - summary TEXT DEFAULT '', - description TEXT DEFAULT '', - rpm_size BIGINT DEFAULT 0, - installed_size BIGINT DEFAULT 0, - license TEXT DEFAULT '', - vendor TEXT DEFAULT '', - build_group TEXT DEFAULT '', - build_host TEXT DEFAULT '', - source_rpm TEXT DEFAULT '', - url TEXT DEFAULT '', - packager TEXT DEFAULT '', - requires JSONB DEFAULT '[]', - provides JSONB DEFAULT '[]', - conflicts JSONB DEFAULT '[]', - obsoletes JSONB DEFAULT '[]', - files JSONB DEFAULT '[]', - changelogs JSONB DEFAULT '[]', - created_at TIMESTAMPTZ DEFAULT NOW(), - UNIQUE(repo_name, file_path) - ); - - CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name); - - ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]'; - ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]'; - - CREATE TABLE IF NOT EXISTS deb_metadata ( - id BIGSERIAL PRIMARY KEY, - repo_name TEXT NOT NULL, - file_path TEXT NOT NULL, - content_hash TEXT NOT NULL, - name TEXT NOT NULL, - version TEXT NOT NULL, - architecture TEXT NOT NULL, - control TEXT NOT NULL, - size BIGINT DEFAULT 0, - md5 TEXT DEFAULT '', - sha256 TEXT DEFAULT '', - created_at TIMESTAMPTZ DEFAULT NOW(), - UNIQUE(repo_name, file_path) - ); - - CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name); - - CREATE TABLE IF NOT EXISTS alpine_metadata ( - id BIGSERIAL PRIMARY KEY, - repo_name TEXT NOT NULL, - file_path TEXT NOT NULL, - content_hash TEXT NOT NULL, - checksum TEXT NOT NULL, - name TEXT NOT NULL, - version TEXT NOT NULL, - arch TEXT NOT NULL, - download_size BIGINT DEFAULT 0, - installed_size BIGINT DEFAULT 0, - description TEXT DEFAULT '', - url TEXT DEFAULT '', - license TEXT DEFAULT '', - origin TEXT DEFAULT '', - maintainer TEXT DEFAULT '', - build_time BIGINT DEFAULT 0, - commit_hash TEXT DEFAULT '', - provider_priority TEXT DEFAULT '', - depends TEXT DEFAULT '', - provides TEXT DEFAULT '', - install_if TEXT DEFAULT '', - created_at TIMESTAMPTZ DEFAULT NOW(), - UNIQUE(repo_name, file_path) - ); - - CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name); - CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch); - - CREATE TABLE IF NOT EXISTS github_rpm_sync_state ( - remote_name TEXT PRIMARY KEY, - etag TEXT DEFAULT '', - last_synced_at TIMESTAMPTZ, - sync_lease_owner TEXT DEFAULT '', - sync_lease_expires TIMESTAMPTZ - ); - - CREATE TABLE IF NOT EXISTS github_deb_sync_state ( - remote_name TEXT PRIMARY KEY, - etag TEXT DEFAULT '', - last_synced_at TIMESTAMPTZ, - sync_lease_owner TEXT DEFAULT '', - sync_lease_expires TIMESTAMPTZ - ); - - CREATE TABLE IF NOT EXISTS github_alpine_sync_state ( - remote_name TEXT PRIMARY KEY, - etag TEXT DEFAULT '', - last_synced_at TIMESTAMPTZ, - sync_lease_owner TEXT DEFAULT '', - sync_lease_expires TIMESTAMPTZ - ); - - CREATE TABLE IF NOT EXISTS signing_keys ( - purpose TEXT PRIMARY KEY, - private_key_armor TEXT NOT NULL, - key_id TEXT NOT NULL, - created_at TIMESTAMPTZ DEFAULT NOW() - ); - `) - return err -} diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..5c93257 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,206 @@ +CREATE TABLE IF NOT EXISTS remotes ( + name TEXT PRIMARY KEY, + package_type TEXT NOT NULL, + repo_type TEXT DEFAULT 'remote', + base_url TEXT NOT NULL DEFAULT '', + mirrorlist TEXT[] DEFAULT '{}', + mirror_strategy TEXT NOT NULL DEFAULT 'round_robin', + description TEXT DEFAULT '', + username TEXT DEFAULT '', + password TEXT DEFAULT '', + immutable_ttl INTEGER DEFAULT 0, + mutable_ttl INTEGER DEFAULT 3600, + check_mutable BOOLEAN DEFAULT TRUE, + patterns TEXT[] DEFAULT '{}', + blocklist TEXT[] DEFAULT '{}', + mutable_patterns TEXT[] DEFAULT '{}', + immutable_patterns TEXT[] DEFAULT '{}', + ban_tags_enabled BOOLEAN DEFAULT FALSE, + ban_tags TEXT[] DEFAULT '{}', + quarantine_enabled BOOLEAN DEFAULT FALSE, + quarantine_days INTEGER DEFAULT 3, + stale_on_error BOOLEAN DEFAULT TRUE, + releases_remote TEXT DEFAULT '', + managed_by TEXT DEFAULT '', + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS virtuals ( + name TEXT PRIMARY KEY, + package_type TEXT NOT NULL, + description TEXT DEFAULT '', + members TEXT[] NOT NULL, + managed_by TEXT DEFAULT '', + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS blobs ( + content_hash TEXT PRIMARY KEY, + s3_key TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + content_type TEXT DEFAULT 'application/octet-stream', + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS artifacts ( + id BIGSERIAL PRIMARY KEY, + remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE, + path TEXT NOT NULL, + content_hash TEXT NOT NULL REFERENCES blobs(content_hash), + upstream_etag TEXT DEFAULT '', + upstream_last_modified TIMESTAMPTZ, + first_seen_at TIMESTAMPTZ DEFAULT NOW(), + last_fetched_at TIMESTAMPTZ DEFAULT NOW(), + last_accessed_at TIMESTAMPTZ DEFAULT NOW(), + fetch_count BIGINT DEFAULT 1, + access_count BIGINT DEFAULT 1, + UNIQUE(remote_name, path) +); + +CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name); +CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); + +CREATE TABLE IF NOT EXISTS local_files ( + id BIGSERIAL PRIMARY KEY, + repo_name TEXT NOT NULL, + file_path TEXT NOT NULL, + content_hash TEXT NOT NULL REFERENCES blobs(content_hash), + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(repo_name, file_path) +); + +CREATE TABLE IF NOT EXISTS access_log ( + id BIGSERIAL PRIMARY KEY, + remote_name TEXT NOT NULL, + path TEXT NOT NULL, + cache_hit BOOLEAN NOT NULL, + size_bytes BIGINT DEFAULT 0, + upstream_ms INTEGER DEFAULT 0, + client_ip TEXT DEFAULT '', + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at); + +ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote'; +ALTER TABLE remotes ADD COLUMN IF NOT EXISTS mirrorlist TEXT[] DEFAULT '{}'; +ALTER TABLE remotes ADD COLUMN IF NOT EXISTS mirror_strategy TEXT NOT NULL DEFAULT 'round_robin'; +ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0; +ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0; +ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0; + +CREATE TABLE IF NOT EXISTS rpm_metadata ( + id BIGSERIAL PRIMARY KEY, + repo_name TEXT NOT NULL, + file_path TEXT NOT NULL, + content_hash TEXT NOT NULL, + name TEXT NOT NULL, + epoch INTEGER DEFAULT 0, + version TEXT NOT NULL, + release TEXT NOT NULL, + arch TEXT NOT NULL, + summary TEXT DEFAULT '', + description TEXT DEFAULT '', + rpm_size BIGINT DEFAULT 0, + installed_size BIGINT DEFAULT 0, + license TEXT DEFAULT '', + vendor TEXT DEFAULT '', + build_group TEXT DEFAULT '', + build_host TEXT DEFAULT '', + source_rpm TEXT DEFAULT '', + url TEXT DEFAULT '', + packager TEXT DEFAULT '', + requires JSONB DEFAULT '[]', + provides JSONB DEFAULT '[]', + conflicts JSONB DEFAULT '[]', + obsoletes JSONB DEFAULT '[]', + files JSONB DEFAULT '[]', + changelogs JSONB DEFAULT '[]', + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(repo_name, file_path) +); + +CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name); + +ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]'; +ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]'; + +CREATE TABLE IF NOT EXISTS deb_metadata ( + id BIGSERIAL PRIMARY KEY, + repo_name TEXT NOT NULL, + file_path TEXT NOT NULL, + content_hash TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + architecture TEXT NOT NULL, + control TEXT NOT NULL, + size BIGINT DEFAULT 0, + md5 TEXT DEFAULT '', + sha256 TEXT DEFAULT '', + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(repo_name, file_path) +); + +CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name); + +CREATE TABLE IF NOT EXISTS alpine_metadata ( + id BIGSERIAL PRIMARY KEY, + repo_name TEXT NOT NULL, + file_path TEXT NOT NULL, + content_hash TEXT NOT NULL, + checksum TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + arch TEXT NOT NULL, + download_size BIGINT DEFAULT 0, + installed_size BIGINT DEFAULT 0, + description TEXT DEFAULT '', + url TEXT DEFAULT '', + license TEXT DEFAULT '', + origin TEXT DEFAULT '', + maintainer TEXT DEFAULT '', + build_time BIGINT DEFAULT 0, + commit_hash TEXT DEFAULT '', + provider_priority TEXT DEFAULT '', + depends TEXT DEFAULT '', + provides TEXT DEFAULT '', + install_if TEXT DEFAULT '', + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(repo_name, file_path) +); + +CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name); +CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch); + +CREATE TABLE IF NOT EXISTS github_rpm_sync_state ( + remote_name TEXT PRIMARY KEY, + etag TEXT DEFAULT '', + last_synced_at TIMESTAMPTZ, + sync_lease_owner TEXT DEFAULT '', + sync_lease_expires TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS github_deb_sync_state ( + remote_name TEXT PRIMARY KEY, + etag TEXT DEFAULT '', + last_synced_at TIMESTAMPTZ, + sync_lease_owner TEXT DEFAULT '', + sync_lease_expires TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS github_alpine_sync_state ( + remote_name TEXT PRIMARY KEY, + etag TEXT DEFAULT '', + last_synced_at TIMESTAMPTZ, + sync_lease_owner TEXT DEFAULT '', + sync_lease_expires TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS signing_keys ( + purpose TEXT PRIMARY KEY, + private_key_armor TEXT NOT NULL, + key_id TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW() +); diff --git a/migrations/embed.go b/migrations/embed.go new file mode 100644 index 0000000..a30d8cd --- /dev/null +++ b/migrations/embed.go @@ -0,0 +1,11 @@ +// Package migrations embeds the SQL schema files so artifactapi carries its own +// schema and applies it at startup, with no externally mirrored copy to drift +// out of sync. +package migrations + +import "embed" + +// FS holds every numbered migration; lexical filename order is version order. +// +//go:embed *.sql +var FS embed.FS