From bf31714e08d4e0c16bbb4718d58a3f5b6001cb1f Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Fri, 3 Jul 2026 12:55:51 +1000 Subject: [PATCH] test: testcontainers harness + database integration tests Add internal/testsupport (Postgres/Redis/MinIO container helpers, Ryuk disabled) and full database coverage: remotes, artifacts/blobs, local files, virtuals, stats, rpm metadata, orphan/cold cleanup. database 0->83%. --- internal/database/database_test.go | 277 +++++++++++++++++++++++++++++ internal/testsupport/containers.go | 97 ++++++++++ 2 files changed, 374 insertions(+) create mode 100644 internal/database/database_test.go create mode 100644 internal/testsupport/containers.go diff --git a/internal/database/database_test.go b/internal/database/database_test.go new file mode 100644 index 0000000..b1e6aba --- /dev/null +++ b/internal/database/database_test.go @@ -0,0 +1,277 @@ +package database + +import ( + "context" + "os" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/testsupport" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +var testDB *DB + +func TestMain(m *testing.M) { + c := context.Background() + dsn, terminate, err := testsupport.StartPostgres(c) + if err != nil { + // Docker unavailable: run anyway so tests self-skip via requireDB. + os.Exit(m.Run()) + } + db, err := New(dsn) + if err != nil { + terminate() + panic(err) + } + testDB = db + + code := m.Run() + + db.Close() + terminate() + // Return normally on success so the coverage profile is flushed; os.Exit + // would truncate it. + if code != 0 { + os.Exit(code) + } +} + +func requireDB(t *testing.T) { + t.Helper() + if testDB == nil { + t.Skip("Docker unavailable; skipping database integration test") + } +} + +func ctx() context.Context { return context.Background() } + +func seedRemote(t *testing.T, name string) { + t.Helper() + if err := testDB.CreateRemote(ctx(), &models.Remote{ + Name: name, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote, + BaseURL: "https://example.com", MutableTTL: 3600, + }); err != nil { + t.Fatalf("seed remote: %v", err) + } +} + +// seedBlob inserts a blob and returns its full content hash (sha256:), +// matching the reference convention used by artifacts and local files. +func seedBlob(t *testing.T, hash string) string { + t.Helper() + full := "sha256:" + hash + if err := testDB.UpsertBlob(ctx(), full, "blobs/sha256/"+hash, 10, "application/octet-stream"); err != nil { + t.Fatalf("seed blob: %v", err) + } + return full +} + +func TestRemotesCRUD(t *testing.T) { + requireDB(t) + seedRemote(t, "r-crud") + got, err := testDB.GetRemote(ctx(), "r-crud") + if err != nil || got.BaseURL != "https://example.com" { + t.Fatalf("get: %v %v", got, err) + } + got.BaseURL = "https://updated.example.com" + if err := testDB.UpdateRemote(ctx(), got); err != nil { + t.Fatalf("update: %v", err) + } + got, _ = testDB.GetRemote(ctx(), "r-crud") + if got.BaseURL != "https://updated.example.com" { + t.Errorf("update not applied: %v", got.BaseURL) + } + list, err := testDB.ListRemotes(ctx()) + if err != nil || len(list) == 0 { + t.Fatalf("list: %v %v", len(list), err) + } + if err := testDB.DeleteRemote(ctx(), "r-crud"); err != nil { + t.Fatalf("delete: %v", err) + } + if _, err := testDB.GetRemote(ctx(), "r-crud"); err == nil { + t.Error("expected error after delete") + } +} + +func TestArtifactsAndBlobs(t *testing.T) { + requireDB(t) + seedRemote(t, "r-art") + seedBlob(t, "aaaa") + hash := "sha256:aaaa" + if err := testDB.UpsertBlob(ctx(), hash, "blobs/sha256/aaaa", 10, "text/plain"); err != nil { + t.Fatal(err) + } + if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag1"); err != nil { + t.Fatal(err) + } + // Upsert again to exercise the ON CONFLICT update branch. + if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag2"); err != nil { + t.Fatal(err) + } + art, err := testDB.GetArtifact(ctx(), "r-art", "path/a.txt") + if err != nil || art.ContentHash != hash { + t.Fatalf("get artifact: %v %v", art, err) + } + if err := testDB.TouchArtifactAccess(ctx(), "r-art", "path/a.txt"); err != nil { + t.Fatal(err) + } + arts, err := testDB.ListArtifacts(ctx(), "r-art", 10, 0) + if err != nil || len(arts) != 1 { + t.Fatalf("list artifacts: %v %v", len(arts), err) + } + if err := testDB.InsertAccessLog(ctx(), "r-art", "path/a.txt", true, 10, 5, "1.2.3.4"); err != nil { + t.Fatal(err) + } + if err := testDB.InsertAccessLogBatch(ctx(), []AccessLogEntry{ + {RemoteName: "r-art", Path: "b", CacheHit: false, SizeBytes: 20, UpstreamMS: 3}, + }); err != nil { + t.Fatal(err) + } + if err := testDB.InsertAccessLogBatch(ctx(), nil); err != nil { + t.Fatalf("empty batch should be a no-op: %v", err) + } + if err := testDB.DeleteArtifact(ctx(), "r-art", "path/a.txt"); err != nil { + t.Fatal(err) + } +} + +func TestOrphanAndColdCleanup(t *testing.T) { + requireDB(t) + seedBlob(t, "orphanhash") + // A blob with no artifact/local_file reference is orphaned, but only past + // the grace period. + if got, _ := testDB.FindOrphanedBlobs(ctx(), time.Hour); containsHash(got, "sha256:orphanhash") { + t.Error("fresh orphan should be excluded by grace period") + } + orphans, err := testDB.FindOrphanedBlobs(ctx(), -time.Hour) // cutoff in the future => include fresh + if err != nil { + t.Fatal(err) + } + if !containsHash(orphans, "sha256:orphanhash") { + t.Error("expected orphan to be found with zero grace") + } + if err := testDB.DeleteBlob(ctx(), "sha256:orphanhash"); err != nil { + t.Fatal(err) + } + + seedRemote(t, "r-cold") + seedBlob(t, "coldhash") + testDB.UpsertArtifact(ctx(), "r-cold", "cold.txt", "sha256:coldhash", "") + n, err := testDB.DeleteColdArtifacts(ctx(), "r-cold", -time.Hour) // negative => everything is "cold" + if err != nil || n < 1 { + t.Fatalf("delete cold: n=%d err=%v", n, err) + } +} + +func containsHash(blobs []models.Blob, hash string) bool { + for _, b := range blobs { + if b.ContentHash == hash { + return true + } + } + return false +} + +func TestLocalFiles(t *testing.T) { + requireDB(t) + seedRemote(t, "r-local") + seedBlob(t, "localhash") + hash := "sha256:localhash" + if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err != nil { + t.Fatal(err) + } + // Duplicate create must be rejected. + if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err == nil { + t.Error("expected duplicate local file error") + } + f, err := testDB.GetLocalFile(ctx(), "r-local", "foo/foo-1.0.whl") + if err != nil || f == nil { + t.Fatalf("get local file: %v %v", f, err) + } + if files, err := testDB.ListLocalFiles(ctx(), "r-local", 10, 0); err != nil || len(files) != 1 { + t.Fatalf("list: %v %v", len(files), err) + } + if files, err := testDB.ListLocalFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(files) != 1 { + t.Fatalf("list by prefix: %v %v", len(files), err) + } + if entries, err := testDB.ListFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(entries) != 1 { + t.Fatalf("provider list by prefix: %v %v", len(entries), err) + } + if pkgs, err := testDB.ListLocalFilePackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 { + t.Fatalf("list packages: %v %v", pkgs, err) + } + if pkgs, err := testDB.ListPackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 { + t.Fatalf("provider list packages: %v %v", pkgs, err) + } + if err := testDB.DeleteLocalFile(ctx(), "r-local", "foo/foo-1.0.whl"); err != nil { + t.Fatal(err) + } +} + +func TestVirtualsCRUD(t *testing.T) { + requireDB(t) + if err := testDB.CreateVirtual(ctx(), &models.Virtual{ + Name: "v-crud", PackageType: models.PackageHelm, Members: []string{"a", "b"}, + }); err != nil { + t.Fatal(err) + } + v, err := testDB.GetVirtual(ctx(), "v-crud") + if err != nil || len(v.Members) != 2 { + t.Fatalf("get virtual: %v %v", v, err) + } + v.Members = []string{"a"} + if err := testDB.UpdateVirtual(ctx(), v); err != nil { + t.Fatal(err) + } + if vs, err := testDB.ListVirtuals(ctx()); err != nil || len(vs) == 0 { + t.Fatalf("list virtuals: %v %v", len(vs), err) + } + if err := testDB.DeleteVirtual(ctx(), "v-crud"); err != nil { + t.Fatal(err) + } +} + +func TestStats(t *testing.T) { + requireDB(t) + seedRemote(t, "r-stats") + seedBlob(t, "statshash") + testDB.UpsertArtifact(ctx(), "r-stats", "s.txt", "sha256:statshash", "") + testDB.InsertAccessLog(ctx(), "r-stats", "s.txt", true, 100, 2, "") + + if _, err := testDB.GetOverviewStats(ctx()); err != nil { + t.Fatalf("overview: %v", err) + } + if _, err := testDB.GetTopRemotes(ctx(), 5); err != nil { + t.Fatalf("top remotes: %v", err) + } + if _, err := testDB.GetTopFilesByHits(ctx(), 5); err != nil { + t.Fatalf("top files by hits: %v", err) + } + if _, err := testDB.GetTopFilesByBandwidth(ctx(), 5); err != nil { + t.Fatalf("top files by bandwidth: %v", err) + } +} + +func TestRPMMetadata(t *testing.T) { + requireDB(t) + seedRemote(t, "r-rpm") + meta := &provider.RPMMetadata{ + RepoName: "r-rpm", FilePath: "Packages/x.rpm", ContentHash: "sha256:rpm", + Name: "x", Version: "1.0", Release: "1", Arch: "noarch", + Requires: []provider.RPMDep{{Name: "libc"}}, + Provides: []provider.RPMDep{{Name: "x"}}, + Files: []provider.RPMFile{}, + } + if err := testDB.InsertRPMMetadata(ctx(), meta); err != nil { + t.Fatal(err) + } + entries, err := testDB.ListRPMMetadataEntries(ctx(), "r-rpm") + if err != nil || len(entries) != 1 { + t.Fatalf("list rpm entries: %v %v", len(entries), err) + } + if rows, err := testDB.ListRPMMetadata(ctx(), "r-rpm"); err != nil || len(rows) != 1 { + t.Fatalf("list rpm rows: %v %v", len(rows), err) + } +} diff --git a/internal/testsupport/containers.go b/internal/testsupport/containers.go new file mode 100644 index 0000000..d4a1dfe --- /dev/null +++ b/internal/testsupport/containers.go @@ -0,0 +1,97 @@ +// Package testsupport starts throwaway backing containers (Postgres, Redis, +// MinIO) for integration-style unit tests. It is only ever imported from +// *_test.go files, so it never reaches the production binary. Each Start* +// function returns a connection detail plus a terminate func; callers wire +// them up in a TestMain and skip the package's tests when Docker is absent. +package testsupport + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + tcredis "github.com/testcontainers/testcontainers-go/modules/redis" + "github.com/testcontainers/testcontainers-go/wait" +) + +func init() { + // The Ryuk reaper container cannot start in this environment; each Start* + // returns an explicit terminate func for cleanup instead. + if _, ok := os.LookupEnv("TESTCONTAINERS_RYUK_DISABLED"); !ok { + os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true") + } +} + +// StartPostgres launches postgres:17-alpine and returns its DSN. +func StartPostgres(ctx context.Context) (dsn string, terminate func(), err error) { + c, err := tcpostgres.Run(ctx, + "postgres:17-alpine", + tcpostgres.WithDatabase("artifacts"), + tcpostgres.WithUsername("artifacts"), + tcpostgres.WithPassword("artifacts123"), + testcontainers.WithWaitStrategy( + wait.ForListeningPort("5432/tcp").WithStartupTimeout(60*time.Second), + ), + ) + if err != nil { + return "", nil, err + } + host, _ := c.Host(ctx) + port, _ := c.MappedPort(ctx, "5432/tcp") + dsn = fmt.Sprintf("postgres://artifacts:artifacts123@%s:%s/artifacts?sslmode=disable", host, port.Port()) + return dsn, func() { _ = c.Terminate(ctx) }, nil +} + +// StartRedis launches redis:7-alpine and returns its URL. +func StartRedis(ctx context.Context) (url string, terminate func(), err error) { + c, err := tcredis.Run(ctx, + "redis:7-alpine", + testcontainers.WithWaitStrategy( + wait.ForListeningPort("6379/tcp").WithStartupTimeout(60*time.Second), + ), + ) + if err != nil { + return "", nil, err + } + host, _ := c.Host(ctx) + port, _ := c.MappedPort(ctx, "6379/tcp") + url = fmt.Sprintf("redis://%s:%s", host, port.Port()) + return url, func() { _ = c.Terminate(ctx) }, nil +} + +// MinioConn holds MinIO connection details. +type MinioConn struct { + Endpoint string + AccessKey string + SecretKey string +} + +// StartMinio launches minio and returns its connection details. +func StartMinio(ctx context.Context) (conn MinioConn, terminate func(), err error) { + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "minio/minio:latest", + ExposedPorts: []string{"9000/tcp"}, + Cmd: []string{"server", "/data"}, + Env: map[string]string{ + "MINIO_ROOT_USER": "minioadmin", + "MINIO_ROOT_PASSWORD": "minioadmin", + }, + WaitingFor: wait.ForHTTP("/minio/health/live").WithPort("9000/tcp").WithStartupTimeout(60 * time.Second), + }, + Started: true, + }) + if err != nil { + return MinioConn{}, nil, err + } + host, _ := c.Host(ctx) + port, _ := c.MappedPort(ctx, "9000/tcp") + return MinioConn{ + Endpoint: fmt.Sprintf("%s:%s", host, port.Port()), + AccessKey: "minioadmin", + SecretKey: "minioadmin", + }, func() { _ = c.Terminate(ctx) }, nil +}