diff --git a/internal/api/v1/scheme_test.go b/internal/api/v1/scheme_test.go new file mode 100644 index 0000000..e74ce67 --- /dev/null +++ b/internal/api/v1/scheme_test.go @@ -0,0 +1,20 @@ +package v1 + +import ( + "crypto/tls" + "net/http" + "testing" +) + +func TestScheme(t *testing.T) { + if got := scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" { + t.Errorf("TLS request scheme = %q, want https", got) + } + r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {"https"}}} + if got := scheme(r); got != "https" { + t.Errorf("X-Forwarded-Proto scheme = %q, want https", got) + } + if got := scheme(&http.Request{Header: http.Header{}}); got != "http" { + t.Errorf("default scheme = %q, want http", got) + } +} diff --git a/internal/api/v2/errorpaths_test.go b/internal/api/v2/errorpaths_test.go new file mode 100644 index 0000000..abedb70 --- /dev/null +++ b/internal/api/v2/errorpaths_test.go @@ -0,0 +1,130 @@ +package v2 + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/testsupport" +) + +var testDSN string + +func TestMain(m *testing.M) { + ctx := context.Background() + dsn, terminate, err := testsupport.StartPostgres(ctx) + if err != nil { + os.Exit(m.Run()) + } + testDSN = dsn + code := m.Run() + terminate() + if code != 0 { + os.Exit(code) + } +} + +// closedDB returns a DB whose pool has been closed, so every query fails — +// used to drive the handlers' error branches. +func closedDB(t *testing.T) *database.DB { + t.Helper() + if testDSN == "" { + t.Skip("Docker unavailable") + } + db, err := database.New(testDSN) + if err != nil { + t.Fatalf("new db: %v", err) + } + db.Close() + return db +} + +func do(t *testing.T, h http.Handler, method, path, body string) int { + t.Helper() + var r io.Reader + if body != "" { + r = strings.NewReader(body) + } + req := httptest.NewRequest(method, path, r) + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + return w.Code +} + +func TestRemotesErrorPaths(t *testing.T) { + h := NewRemotesHandler(closedDB(t)).Routes() + if c := do(t, h, "GET", "/", ""); c != 500 { + t.Errorf("list with dead db = %d, want 500", c) + } + if c := do(t, h, "POST", "/", `{"name":"x","package_type":"generic","repo_type":"remote","base_url":"https://x"}`); c != 500 { + t.Errorf("create with dead db = %d, want 500", c) + } + if c := do(t, h, "PUT", "/x", `{"package_type":"generic","base_url":"https://x"}`); c != 500 { + t.Errorf("update with dead db = %d, want 500", c) + } + if c := do(t, h, "GET", "/x", ""); c != 404 { + t.Errorf("get missing = %d, want 404", c) + } + if c := do(t, h, "DELETE", "/x", ""); c != 500 { + t.Errorf("delete with dead db = %d, want 500", c) + } + // Bad request bodies never reach the db. + if c := do(t, h, "POST", "/", `not json`); c != 400 { + t.Errorf("invalid json = %d, want 400", c) + } +} + +func TestVirtualsErrorPaths(t *testing.T) { + h := NewVirtualsHandler(closedDB(t)).Routes() + if c := do(t, h, "GET", "/", ""); c != 500 { + t.Errorf("list = %d, want 500", c) + } + if c := do(t, h, "GET", "/x", ""); c != 404 { + t.Errorf("get missing = %d, want 404", c) + } + if c := do(t, h, "POST", "/", `{"name":"v","package_type":"helm","members":["a"]}`); c != 500 { + t.Errorf("create = %d, want 500", c) + } + if c := do(t, h, "PUT", "/v", `{"package_type":"helm","members":["a"]}`); c != 500 { + t.Errorf("update = %d, want 500", c) + } + if c := do(t, h, "DELETE", "/v", ""); c != 500 { + t.Errorf("delete = %d, want 500", c) + } +} + +func TestStatsErrorPaths(t *testing.T) { + h := NewStatsHandler(closedDB(t)).Routes() + for _, p := range []string{"/", "/top-remotes", "/top-files-by-hits", "/top-files-by-bandwidth"} { + if c := do(t, h, "GET", p, ""); c != 500 { + t.Errorf("stats %s = %d, want 500", p, c) + } + } +} + +func TestLocalErrorPaths(t *testing.T) { + h := NewLocalHandler(closedDB(t), nil).Routes() + // GetRemote fails on the closed db -> not found. + if c := do(t, h, "PUT", "/x/files/a.bin", "data"); c != 404 { + t.Errorf("upload unknown repo = %d, want 404", c) + } + // download / remove hit the db and 500. + if c := do(t, h, "GET", "/x/files/a.bin", ""); c != 500 { + t.Errorf("download = %d, want 500", c) + } + if c := do(t, h, "DELETE", "/x/files/a.bin", ""); c != 500 { + t.Errorf("remove = %d, want 500", c) + } +} + +func TestLocalHandlerDBAccessor(t *testing.T) { + db := closedDB(t) + if NewLocalHandler(db, nil).DB() != db { + t.Error("DB() should return the handler's database") + } +} diff --git a/internal/api/v2/local_fault_test.go b/internal/api/v2/local_fault_test.go new file mode 100644 index 0000000..3159411 --- /dev/null +++ b/internal/api/v2/local_fault_test.go @@ -0,0 +1,88 @@ +package v2 + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/storage" + "git.unkin.net/unkin/artifactapi/internal/testsupport" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// TestLocalUploadStoreFailure covers the upload handlers' store-error branches +// by killing the object store after a successful upload. +func TestLocalUploadStoreFailure(t *testing.T) { + if testDSN == "" { + t.Skip("Docker unavailable") + } + ctx := context.Background() + db, err := database.New(testDSN) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + conn, termMinio, err := testsupport.StartMinio(ctx) + if err != nil { + t.Skip("minio unavailable") + } + var store *storage.S3 + for i := 0; i < 20; i++ { + if store, err = storage.NewS3(conn.Endpoint, conn.AccessKey, conn.SecretKey, "fault", false, ""); err == nil { + break + } + time.Sleep(500 * time.Millisecond) + } + if err != nil { + termMinio() + t.Fatal(err) + } + + for _, pt := range []models.PackageType{models.PackageGeneric, models.PackagePyPI} { + if err := db.CreateRemote(ctx, &models.Remote{Name: "fault-" + string(pt), PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil { + t.Fatal(err) + } + } + + h := NewLocalHandler(db, store) + router := chi.NewRouter() + router.Route("/remotes/{name}/files", func(r chi.Router) { + r.Put("/*", h.Routes().ServeHTTP) + }) + srv := httptest.NewServer(router) + defer srv.Close() + + put := func(name, path, body string) int { + rq, _ := http.NewRequest("PUT", srv.URL+"/remotes/"+name+"/files/"+path, strings.NewReader(body)) + resp, err := http.DefaultClient.Do(rq) + if err != nil { + t.Fatalf("put: %v", err) + } + resp.Body.Close() + return resp.StatusCode + } + + // Sanity: uploads succeed while the store is up. + if c := put("fault-generic", "ok.bin", "data"); c != 201 { + t.Fatalf("generic upload while up = %d", c) + } + if c := put("fault-pypi", "foo-1.0-py3-none-any.whl", "wheel"); c != 201 { + t.Fatalf("pypi upload while up = %d", c) + } + + // Kill the store; subsequent CAS.Store calls fail -> 500. + termMinio() + if c := put("fault-generic", "after.bin", "data"); c != 500 { + t.Errorf("generic upload after store down = %d, want 500", c) + } + if c := put("fault-pypi", "bar-1.0-py3-none-any.whl", "wheel"); c != 500 { + t.Errorf("pypi upload after store down = %d, want 500", c) + } +} diff --git a/internal/auth/basic_test.go b/internal/auth/basic_test.go new file mode 100644 index 0000000..dd089d3 --- /dev/null +++ b/internal/auth/basic_test.go @@ -0,0 +1,23 @@ +package auth + +import ( + "encoding/base64" + "testing" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestBasicHeaders(t *testing.T) { + h := BasicHeaders(models.Remote{Username: "alice", Password: "secret"}) + got := h.Get("Authorization") + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("alice:secret")) + if got != want { + t.Errorf("Authorization = %q, want %q", got, want) + } +} + +func TestBasicHeadersNoUser(t *testing.T) { + if h := BasicHeaders(models.Remote{}); h.Get("Authorization") != "" { + t.Error("expected no Authorization header without a username") + } +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go new file mode 100644 index 0000000..5edbf41 --- /dev/null +++ b/internal/cache/cache_test.go @@ -0,0 +1,133 @@ +package cache + +import ( + "context" + "os" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/testsupport" +) + +var testRedis *Redis + +func TestMain(m *testing.M) { + ctx := context.Background() + url, terminate, err := testsupport.StartRedis(ctx) + if err != nil { + os.Exit(m.Run()) + } + r, err := NewRedis(url) + if err != nil { + terminate() + panic(err) + } + testRedis = r + code := m.Run() + r.Close() + terminate() + if code != 0 { + os.Exit(code) + } +} + +func requireRedis(t *testing.T) { + t.Helper() + if testRedis == nil { + t.Skip("Docker unavailable; skipping cache integration test") + } +} + +func TestNewRedisInvalid(t *testing.T) { + if _, err := NewRedis("://bad-url"); err == nil { + t.Error("expected error for invalid redis URL") + } +} + +func TestTTL(t *testing.T) { + requireRedis(t) + ctx := context.Background() + if fresh, _ := testRedis.CheckTTL(ctx, "r", "missing"); fresh { + t.Error("missing key should not be fresh") + } + if err := testRedis.SetTTL(ctx, "r", "p", time.Minute); err != nil { + t.Fatal(err) + } + if fresh, err := testRedis.CheckTTL(ctx, "r", "p"); err != nil || !fresh { + t.Errorf("expected fresh after SetTTL: %v %v", fresh, err) + } +} + +func TestLock(t *testing.T) { + requireRedis(t) + ctx := context.Background() + ok, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute) + if err != nil || !ok { + t.Fatalf("first acquire should succeed: %v %v", ok, err) + } + if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); ok { + t.Error("second acquire should fail while held") + } + if err := testRedis.ReleaseLock(ctx, "r", "lockpath"); err != nil { + t.Fatal(err) + } + if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); !ok { + t.Error("acquire should succeed after release") + } +} + +func TestETagAndToken(t *testing.T) { + requireRedis(t) + ctx := context.Background() + if v, _ := testRedis.GetETag(ctx, "r", "missing"); v != "" { + t.Error("missing etag should be empty") + } + testRedis.SetETag(ctx, "r", "p", `"abc"`, time.Minute) + if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` { + t.Errorf("etag = %q", v) + } + + if v, _ := testRedis.GetToken(ctx, "missing"); v != "" { + t.Error("missing token should be empty") + } + testRedis.SetToken(ctx, "key", "tok", time.Minute) + if v, _ := testRedis.GetToken(ctx, "key"); v != "tok" { + t.Errorf("token = %q", v) + } +} + +func TestCircuit(t *testing.T) { + requireRedis(t) + ctx := context.Background() + if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 0 { + t.Errorf("initial failures = %d", n) + } + n1, err := testRedis.IncrCircuitFailure(ctx, "cr", time.Minute) + if err != nil || n1 != 1 { + t.Fatalf("first incr = %d %v", n1, err) + } + n2, _ := testRedis.IncrCircuitFailure(ctx, "cr", time.Minute) + if n2 != 2 { + t.Errorf("second incr = %d", n2) + } + if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 2 { + t.Errorf("get failures = %d", n) + } + testRedis.ResetCircuit(ctx, "cr") + if n, _ := testRedis.GetCircuitFailures(ctx, "cr"); n != 0 { + t.Errorf("failures after reset = %d", n) + } +} + +func TestFlushRemote(t *testing.T) { + requireRedis(t) + ctx := context.Background() + testRedis.SetTTL(ctx, "flushme", "a", time.Hour) + testRedis.SetETag(ctx, "flushme", "a", "x", time.Hour) + if err := testRedis.FlushRemote(ctx, "flushme"); err != nil { + t.Fatal(err) + } + if fresh, _ := testRedis.CheckTTL(ctx, "flushme", "a"); fresh { + t.Error("expected keys flushed") + } +} diff --git a/internal/config/env_test.go b/internal/config/env_test.go new file mode 100644 index 0000000..179d314 --- /dev/null +++ b/internal/config/env_test.go @@ -0,0 +1,66 @@ +package config + +import ( + "os" + "testing" +) + +func TestLoadDefaults(t *testing.T) { + // Unset the vars Load reads so the fallback defaults are exercised. + for _, k := range []string{ + "LISTEN_ADDR", "DBHOST", "DBPORT", "DBUSER", "DBPASS", "DBNAME", "DBSSL", + "REDIS_URL", "MINIO_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", + "MINIO_BUCKET", "MINIO_SECURE", "MINIO_REGION", + } { + old, ok := os.LookupEnv(k) + os.Unsetenv(k) + if ok { + t.Cleanup(func() { os.Setenv(k, old) }) + } + } + + cfg, err := Load() + if err != nil { + t.Fatalf("load: %v", err) + } + if cfg.ListenAddr != ":8000" || cfg.DBPort != 5432 || cfg.DBUser != "artifacts" { + t.Errorf("unexpected defaults: %+v", cfg) + } + if cfg.RedisURL != "redis://localhost:6379" || cfg.S3Bucket != "artifacts" || cfg.S3Secure { + t.Errorf("unexpected defaults: %+v", cfg) + } +} + +func TestLoadOverrides(t *testing.T) { + t.Setenv("LISTEN_ADDR", ":9999") + t.Setenv("DBHOST", "db.example.com") + t.Setenv("DBPORT", "6000") + t.Setenv("DBUSER", "u") + t.Setenv("DBPASS", "pw") + t.Setenv("DBNAME", "n") + t.Setenv("DBSSL", "require") + t.Setenv("MINIO_SECURE", "true") + t.Setenv("MINIO_REGION", "us-east-1") + + cfg, err := Load() + if err != nil { + t.Fatalf("load: %v", err) + } + if cfg.ListenAddr != ":9999" || cfg.DBHost != "db.example.com" || cfg.DBPort != 6000 { + t.Errorf("overrides not applied: %+v", cfg) + } + if !cfg.S3Secure { + t.Error("MINIO_SECURE=true not parsed") + } + want := "postgres://u:pw@db.example.com:6000/n?sslmode=require" + if got := cfg.DatabaseDSN(); got != want { + t.Errorf("DSN = %q, want %q", got, want) + } +} + +func TestLoadInvalidPort(t *testing.T) { + t.Setenv("DBPORT", "not-a-number") + if _, err := Load(); err == nil { + t.Error("expected error for invalid DBPORT") + } +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go new file mode 100644 index 0000000..af3746e --- /dev/null +++ b/internal/database/database_test.go @@ -0,0 +1,334 @@ +package database + +import ( + "context" + "os" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/testsupport" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +var ( + testDB *DB + testDSN string +) + +func TestMain(m *testing.M) { + c := context.Background() + dsn, terminate, err := testsupport.StartPostgres(c) + if err != nil { + // Docker unavailable: run anyway so tests self-skip via requireDB. + os.Exit(m.Run()) + } + testDSN = dsn + db, err := New(dsn) + if err != nil { + terminate() + panic(err) + } + testDB = db + + code := m.Run() + + db.Close() + terminate() + // Return normally on success so the coverage profile is flushed; os.Exit + // would truncate it. + if code != 0 { + os.Exit(code) + } +} + +func requireDB(t *testing.T) { + t.Helper() + if testDB == nil { + t.Skip("Docker unavailable; skipping database integration test") + } +} + +func ctx() context.Context { return context.Background() } + +func seedRemote(t *testing.T, name string) { + t.Helper() + if err := testDB.CreateRemote(ctx(), &models.Remote{ + Name: name, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote, + BaseURL: "https://example.com", MutableTTL: 3600, + }); err != nil { + t.Fatalf("seed remote: %v", err) + } +} + +// seedBlob inserts a blob and returns its full content hash (sha256:), +// matching the reference convention used by artifacts and local files. +func seedBlob(t *testing.T, hash string) string { + t.Helper() + full := "sha256:" + hash + if err := testDB.UpsertBlob(ctx(), full, "blobs/sha256/"+hash, 10, "application/octet-stream"); err != nil { + t.Fatalf("seed blob: %v", err) + } + return full +} + +func TestRemotesCRUD(t *testing.T) { + requireDB(t) + seedRemote(t, "r-crud") + got, err := testDB.GetRemote(ctx(), "r-crud") + if err != nil || got.BaseURL != "https://example.com" { + t.Fatalf("get: %v %v", got, err) + } + got.BaseURL = "https://updated.example.com" + if err := testDB.UpdateRemote(ctx(), got); err != nil { + t.Fatalf("update: %v", err) + } + got, _ = testDB.GetRemote(ctx(), "r-crud") + if got.BaseURL != "https://updated.example.com" { + t.Errorf("update not applied: %v", got.BaseURL) + } + list, err := testDB.ListRemotes(ctx()) + if err != nil || len(list) == 0 { + t.Fatalf("list: %v %v", len(list), err) + } + if err := testDB.DeleteRemote(ctx(), "r-crud"); err != nil { + t.Fatalf("delete: %v", err) + } + if _, err := testDB.GetRemote(ctx(), "r-crud"); err == nil { + t.Error("expected error after delete") + } +} + +func TestArtifactsAndBlobs(t *testing.T) { + requireDB(t) + seedRemote(t, "r-art") + seedBlob(t, "aaaa") + hash := "sha256:aaaa" + if err := testDB.UpsertBlob(ctx(), hash, "blobs/sha256/aaaa", 10, "text/plain"); err != nil { + t.Fatal(err) + } + if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag1"); err != nil { + t.Fatal(err) + } + // Upsert again to exercise the ON CONFLICT update branch. + if err := testDB.UpsertArtifact(ctx(), "r-art", "path/a.txt", hash, "etag2"); err != nil { + t.Fatal(err) + } + art, err := testDB.GetArtifact(ctx(), "r-art", "path/a.txt") + if err != nil || art.ContentHash != hash { + t.Fatalf("get artifact: %v %v", art, err) + } + if err := testDB.TouchArtifactAccess(ctx(), "r-art", "path/a.txt"); err != nil { + t.Fatal(err) + } + arts, err := testDB.ListArtifacts(ctx(), "r-art", 10, 0) + if err != nil || len(arts) != 1 { + t.Fatalf("list artifacts: %v %v", len(arts), err) + } + if err := testDB.InsertAccessLog(ctx(), "r-art", "path/a.txt", true, 10, 5, "1.2.3.4"); err != nil { + t.Fatal(err) + } + if err := testDB.InsertAccessLogBatch(ctx(), []AccessLogEntry{ + {RemoteName: "r-art", Path: "b", CacheHit: false, SizeBytes: 20, UpstreamMS: 3}, + }); err != nil { + t.Fatal(err) + } + if err := testDB.InsertAccessLogBatch(ctx(), nil); err != nil { + t.Fatalf("empty batch should be a no-op: %v", err) + } + if err := testDB.DeleteArtifact(ctx(), "r-art", "path/a.txt"); err != nil { + t.Fatal(err) + } +} + +func TestOrphanAndColdCleanup(t *testing.T) { + requireDB(t) + seedBlob(t, "orphanhash") + // A blob with no artifact/local_file reference is orphaned, but only past + // the grace period. + if got, _ := testDB.FindOrphanedBlobs(ctx(), time.Hour); containsHash(got, "sha256:orphanhash") { + t.Error("fresh orphan should be excluded by grace period") + } + orphans, err := testDB.FindOrphanedBlobs(ctx(), -time.Hour) // cutoff in the future => include fresh + if err != nil { + t.Fatal(err) + } + if !containsHash(orphans, "sha256:orphanhash") { + t.Error("expected orphan to be found with zero grace") + } + if err := testDB.DeleteBlob(ctx(), "sha256:orphanhash"); err != nil { + t.Fatal(err) + } + + seedRemote(t, "r-cold") + seedBlob(t, "coldhash") + testDB.UpsertArtifact(ctx(), "r-cold", "cold.txt", "sha256:coldhash", "") + n, err := testDB.DeleteColdArtifacts(ctx(), "r-cold", -time.Hour) // negative => everything is "cold" + if err != nil || n < 1 { + t.Fatalf("delete cold: n=%d err=%v", n, err) + } +} + +func containsHash(blobs []models.Blob, hash string) bool { + for _, b := range blobs { + if b.ContentHash == hash { + return true + } + } + return false +} + +func TestLocalFiles(t *testing.T) { + requireDB(t) + seedRemote(t, "r-local") + seedBlob(t, "localhash") + hash := "sha256:localhash" + if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err != nil { + t.Fatal(err) + } + // Duplicate create must be rejected. + if err := testDB.CreateLocalFile(ctx(), "r-local", "foo/foo-1.0.whl", hash); err == nil { + t.Error("expected duplicate local file error") + } + f, err := testDB.GetLocalFile(ctx(), "r-local", "foo/foo-1.0.whl") + if err != nil || f == nil { + t.Fatalf("get local file: %v %v", f, err) + } + if files, err := testDB.ListLocalFiles(ctx(), "r-local", 10, 0); err != nil || len(files) != 1 { + t.Fatalf("list: %v %v", len(files), err) + } + if files, err := testDB.ListLocalFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(files) != 1 { + t.Fatalf("list by prefix: %v %v", len(files), err) + } + if entries, err := testDB.ListFilesByPrefix(ctx(), "r-local", "foo/"); err != nil || len(entries) != 1 { + t.Fatalf("provider list by prefix: %v %v", len(entries), err) + } + if pkgs, err := testDB.ListLocalFilePackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 { + t.Fatalf("list packages: %v %v", pkgs, err) + } + if pkgs, err := testDB.ListPackages(ctx(), "r-local"); err != nil || len(pkgs) == 0 { + t.Fatalf("provider list packages: %v %v", pkgs, err) + } + if err := testDB.DeleteLocalFile(ctx(), "r-local", "foo/foo-1.0.whl"); err != nil { + t.Fatal(err) + } +} + +func TestVirtualsCRUD(t *testing.T) { + requireDB(t) + if err := testDB.CreateVirtual(ctx(), &models.Virtual{ + Name: "v-crud", PackageType: models.PackageHelm, Members: []string{"a", "b"}, + }); err != nil { + t.Fatal(err) + } + v, err := testDB.GetVirtual(ctx(), "v-crud") + if err != nil || len(v.Members) != 2 { + t.Fatalf("get virtual: %v %v", v, err) + } + v.Members = []string{"a"} + if err := testDB.UpdateVirtual(ctx(), v); err != nil { + t.Fatal(err) + } + if vs, err := testDB.ListVirtuals(ctx()); err != nil || len(vs) == 0 { + t.Fatalf("list virtuals: %v %v", len(vs), err) + } + if err := testDB.DeleteVirtual(ctx(), "v-crud"); err != nil { + t.Fatal(err) + } +} + +func TestStats(t *testing.T) { + requireDB(t) + seedRemote(t, "r-stats") + seedBlob(t, "statshash") + testDB.UpsertArtifact(ctx(), "r-stats", "s.txt", "sha256:statshash", "") + testDB.InsertAccessLog(ctx(), "r-stats", "s.txt", true, 100, 2, "") + + if _, err := testDB.GetOverviewStats(ctx()); err != nil { + t.Fatalf("overview: %v", err) + } + if _, err := testDB.GetTopRemotes(ctx(), 5); err != nil { + t.Fatalf("top remotes: %v", err) + } + if _, err := testDB.GetTopFilesByHits(ctx(), 5); err != nil { + t.Fatalf("top files by hits: %v", err) + } + if _, err := testDB.GetTopFilesByBandwidth(ctx(), 5); err != nil { + t.Fatalf("top files by bandwidth: %v", err) + } +} + +func TestDatabaseErrorPaths(t *testing.T) { + requireDB(t) + bad, err := New(testDSN) + if err != nil { + t.Fatal(err) + } + bad.Close() // every query now fails + ctx := context.Background() + + if _, err := bad.ListRemotes(ctx); err == nil { + t.Error("ListRemotes should error on closed db") + } + if _, err := bad.ListVirtuals(ctx); err == nil { + t.Error("ListVirtuals should error") + } + if _, err := bad.ListArtifacts(ctx, "r", 10, 0); err == nil { + t.Error("ListArtifacts should error") + } + if _, err := bad.ListLocalFiles(ctx, "r", 10, 0); err == nil { + t.Error("ListLocalFiles should error") + } + if _, err := bad.ListLocalFilesByPrefix(ctx, "r", "p"); err == nil { + t.Error("ListLocalFilesByPrefix should error") + } + if _, err := bad.ListLocalFilePackages(ctx, "r"); err == nil { + t.Error("ListLocalFilePackages should error") + } + if _, err := bad.ListFilesByPrefix(ctx, "r", "p"); err == nil { + t.Error("ListFilesByPrefix should error") + } + if _, err := bad.ListPackages(ctx, "r"); err == nil { + t.Error("ListPackages should error") + } + if _, err := bad.FindOrphanedBlobs(ctx, 0); err == nil { + t.Error("FindOrphanedBlobs should error") + } + if _, err := bad.GetOverviewStats(ctx); err == nil { + t.Error("GetOverviewStats should error") + } + if _, err := bad.GetTopRemotes(ctx, 5); err == nil { + t.Error("GetTopRemotes should error") + } + if _, err := bad.GetTopFilesByHits(ctx, 5); err == nil { + t.Error("GetTopFilesByHits should error") + } + if _, err := bad.GetTopFilesByBandwidth(ctx, 5); err == nil { + t.Error("GetTopFilesByBandwidth should error") + } + if _, err := bad.ListRPMMetadataEntries(ctx, "r"); err == nil { + t.Error("ListRPMMetadataEntries should error") + } +} + +func TestRPMMetadata(t *testing.T) { + requireDB(t) + seedRemote(t, "r-rpm") + meta := &provider.RPMMetadata{ + RepoName: "r-rpm", FilePath: "Packages/x.rpm", ContentHash: "sha256:rpm", + Name: "x", Version: "1.0", Release: "1", Arch: "noarch", + Requires: []provider.RPMDep{{Name: "libc"}}, + Provides: []provider.RPMDep{{Name: "x"}}, + Files: []provider.RPMFile{}, + } + if err := testDB.InsertRPMMetadata(ctx(), meta); err != nil { + t.Fatal(err) + } + entries, err := testDB.ListRPMMetadataEntries(ctx(), "r-rpm") + if err != nil || len(entries) != 1 { + t.Fatalf("list rpm entries: %v %v", len(entries), err) + } + if rows, err := testDB.ListRPMMetadata(ctx(), "r-rpm"); err != nil || len(rows) != 1 { + t.Fatalf("list rpm rows: %v %v", len(rows), err) + } +} diff --git a/internal/gc/gc_integration_test.go b/internal/gc/gc_integration_test.go new file mode 100644 index 0000000..fa5b413 --- /dev/null +++ b/internal/gc/gc_integration_test.go @@ -0,0 +1,114 @@ +package gc + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/storage" + "git.unkin.net/unkin/artifactapi/internal/testsupport" +) + +var ( + testDB *database.DB + testStore *storage.S3 +) + +func TestMain(m *testing.M) { + ctx := context.Background() + dsn, termPG, err := testsupport.StartPostgres(ctx) + if err != nil { + os.Exit(m.Run()) + } + minio, termMinio, err := testsupport.StartMinio(ctx) + if err != nil { + termPG() + os.Exit(m.Run()) + } + db, err := database.New(dsn) + if err != nil { + panic(err) + } + var s3 *storage.S3 + for i := 0; i < 20; i++ { + if s3, err = storage.NewS3(minio.Endpoint, minio.AccessKey, minio.SecretKey, "gc-test", false, ""); err == nil { + break + } + time.Sleep(500 * time.Millisecond) + } + if err != nil { + panic(err) + } + testDB = db + testStore = s3 + + code := m.Run() + db.Close() + termMinio() + termPG() + if code != 0 { + os.Exit(code) + } +} + +func TestSweepDeletesOldOrphan(t *testing.T) { + if testDB == nil { + t.Skip("Docker unavailable") + } + ctx := context.Background() + hash := "sha256:gcorphan" + key := storage.BlobKey("gcorphan") + + if err := testStore.Upload(ctx, key, bytes.NewReader([]byte("orphan")), 6, "application/octet-stream"); err != nil { + t.Fatal(err) + } + if err := testDB.UpsertBlob(ctx, hash, key, 6, "application/octet-stream"); err != nil { + t.Fatal(err) + } + // Age the blob past the grace period. + if _, err := testDB.Pool.Exec(ctx, `UPDATE blobs SET created_at = now() - interval '2 hours' WHERE content_hash = $1`, hash); err != nil { + t.Fatal(err) + } + + c := New(testDB, testStore, time.Hour) + c.sweep(ctx) + + if exists, _ := testStore.Exists(ctx, key); exists { + t.Error("expected orphan object deleted from store") + } + orphans, _ := testDB.FindOrphanedBlobs(ctx, 0) + for _, b := range orphans { + if b.ContentHash == hash { + t.Error("expected orphan blob row deleted") + } + } +} + +func TestSweepNoOrphans(t *testing.T) { + if testDB == nil { + t.Skip("Docker unavailable") + } + // A sweep with nothing to collect should be a clean no-op. + New(testDB, testStore, time.Hour).sweep(context.Background()) +} + +func TestRunStopsOnContextCancel(t *testing.T) { + if testDB == nil { + t.Skip("Docker unavailable") + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + New(testDB, testStore, time.Hour).Run(ctx) + close(done) + }() + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run did not return after context cancel") + } +} diff --git a/internal/provider/alpine/alpine_test.go b/internal/provider/alpine/alpine_test.go new file mode 100644 index 0000000..a90c880 --- /dev/null +++ b/internal/provider/alpine/alpine_test.go @@ -0,0 +1,60 @@ +package alpine + +import ( + "context" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestType(t *testing.T) { + if (&Provider{}).Type() != models.PackageAlpine { + t.Fatal("wrong type") + } +} + +func TestClassify(t *testing.T) { + p := &Provider{} + if p.Classify("v3.19/main/x86_64/APKINDEX.tar.gz") != provider.Mutable { + t.Error("APKINDEX should be mutable") + } + if p.Classify("v3.19/main/x86_64/curl-8.0-r0.apk") != provider.Immutable { + t.Error("apk should be immutable") + } +} + +func TestContentType(t *testing.T) { + p := &Provider{} + cases := map[string]string{ + "pkg.apk": "application/vnd.android.package-archive", + "APKINDEX.tar.gz": "application/gzip", + "something.random": "application/octet-stream", + } + for path, want := range cases { + if got := p.ContentType(path); got != want { + t.Errorf("ContentType(%q) = %q, want %q", path, got, want) + } + } +} + +func TestUpstreamURL(t *testing.T) { + p := &Provider{} + got := p.UpstreamURL(models.Remote{BaseURL: "https://dl-cdn.alpinelinux.org/alpine/"}, "/v3.19/main/x86_64/curl.apk") + if got != "https://dl-cdn.alpinelinux.org/alpine/v3.19/main/x86_64/curl.apk" { + t.Errorf("got %q", got) + } +} + +func TestRewriteResponse(t *testing.T) { + if out, err := (&Provider{}).RewriteResponse([]byte("x"), models.Remote{}, "http://proxy"); out != nil || err != nil { + t.Error("alpine never rewrites") + } +} + +func TestAuthHeaders(t *testing.T) { + h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"}) + if h.Get("Authorization") == "" { + t.Error("expected auth header") + } +} diff --git a/internal/provider/docker/docker_extra_test.go b/internal/provider/docker/docker_extra_test.go new file mode 100644 index 0000000..e07f3e6 --- /dev/null +++ b/internal/provider/docker/docker_extra_test.go @@ -0,0 +1,53 @@ +package docker + +import ( + "context" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestDockerClassifyBranches(t *testing.T) { + p := &Provider{} + if p.Classify("library/nginx/tags/list") != provider.Mutable { + t.Error("tags/list should be mutable") + } + if p.Classify("library/nginx/manifests/latest") != provider.Mutable { + t.Error("tag manifest should be mutable") + } + if p.Classify("library/nginx/manifests/sha256:abcdef") != provider.Immutable { + t.Error("digest manifest should be immutable") + } + if p.Classify("library/nginx/blobs/sha256:abc") != provider.Immutable { + t.Error("blob should be immutable") + } +} + +func TestDockerContentType(t *testing.T) { + p := &Provider{} + if p.ContentType("x/blobs/sha256:abc") != "application/octet-stream" { + t.Error("blob content type") + } + if p.ContentType("x/manifests/latest") != "application/vnd.docker.distribution.manifest.v2+json" { + t.Error("manifest content type") + } + if p.ContentType("x/tags/list") != "application/json" { + t.Error("default content type") + } +} + +func TestDockerRewriteAndAuth(t *testing.T) { + p := &Provider{} + if out, err := p.RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil { + t.Error("docker never rewrites") + } + h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"}) + if h.Get("Authorization") == "" { + t.Error("expected basic auth header") + } + h, _ = p.AuthHeaders(context.Background(), models.Remote{}) + if h.Get("Authorization") != "" { + t.Error("no creds, no header") + } +} diff --git a/internal/provider/generic/generic_extra_test.go b/internal/provider/generic/generic_extra_test.go new file mode 100644 index 0000000..601d0f6 --- /dev/null +++ b/internal/provider/generic/generic_extra_test.go @@ -0,0 +1,13 @@ +package generic + +import ( + "testing" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestGenericRewriteResponse(t *testing.T) { + if out, err := (&Provider{}).RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil { + t.Error("generic never rewrites") + } +} diff --git a/internal/provider/goproxy/goproxy_extra_test.go b/internal/provider/goproxy/goproxy_extra_test.go new file mode 100644 index 0000000..eb2f3cf --- /dev/null +++ b/internal/provider/goproxy/goproxy_extra_test.go @@ -0,0 +1,27 @@ +package goproxy + +import ( + "context" + "testing" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestGoProxyURLAuthRewrite(t *testing.T) { + p := &Provider{} + if got := p.UpstreamURL(models.Remote{BaseURL: "https://proxy.golang.org/"}, "/mod/@v/list"); got != "https://proxy.golang.org/mod/@v/list" { + t.Errorf("upstream url %q", got) + } + if out, err := p.RewriteResponse([]byte("x"), models.Remote{}, "http://p"); out != nil || err != nil { + t.Error("goproxy never rewrites") + } + if h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"}); h.Get("Authorization") == "" { + t.Error("expected basic auth header") + } + if got := p.ContentType("mod/@v/v1.0.0.info"); got != "application/json" { + t.Errorf("info content type %q", got) + } + if got := p.ContentType("mod/@v/v1.0.0.mod"); got != "text/plain" { + t.Errorf("mod content type %q", got) + } +} diff --git a/internal/provider/helm/helm_extra_test.go b/internal/provider/helm/helm_extra_test.go new file mode 100644 index 0000000..2f93783 --- /dev/null +++ b/internal/provider/helm/helm_extra_test.go @@ -0,0 +1,18 @@ +package helm + +import "testing" + +func TestHelmContentTypeBranches(t *testing.T) { + p := &Provider{} + for path, want := range map[string]string{ + "charts/x-1.0.0.tgz": "application/gzip", + "x.tar.gz": "application/gzip", + "index.yaml": "text/yaml", + "x.yml": "text/yaml", + "other": "application/octet-stream", + } { + if got := p.ContentType(path); got != want { + t.Errorf("ContentType(%q)=%q want %q", path, got, want) + } + } +} diff --git a/internal/provider/npm/npm_test.go b/internal/provider/npm/npm_test.go new file mode 100644 index 0000000..5073e19 --- /dev/null +++ b/internal/provider/npm/npm_test.go @@ -0,0 +1,78 @@ +package npm + +import ( + "context" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestType(t *testing.T) { + if (&Provider{}).Type() != models.PackageNPM { + t.Fatal("wrong type") + } +} + +func TestClassify(t *testing.T) { + p := &Provider{} + if p.Classify("pkg/-/pkg-1.0.0.tgz") != provider.Immutable { + t.Error("tgz should be immutable") + } + if p.Classify("pkg") != provider.Mutable { + t.Error("metadata should be mutable") + } +} + +func TestContentType(t *testing.T) { + p := &Provider{} + if p.ContentType("pkg/-/pkg-1.0.0.tgz") != "application/gzip" { + t.Error("tgz content type") + } + if p.ContentType("pkg") != "application/json" { + t.Error("metadata content type") + } +} + +func TestUpstreamURL(t *testing.T) { + p := &Provider{} + got := p.UpstreamURL(models.Remote{BaseURL: "https://registry.npmjs.org/"}, "/pkg") + if got != "https://registry.npmjs.org/pkg" { + t.Errorf("got %q", got) + } +} + +func TestRewriteResponse(t *testing.T) { + p := &Provider{} + remote := models.Remote{Name: "npmjs", BaseURL: "https://registry.npmjs.org"} + + if out, _ := p.RewriteResponse([]byte(`{"a":1}`), remote, ""); out != nil { + t.Error("empty proxyBaseURL should be a no-op") + } + if out, _ := p.RewriteResponse([]byte("not json"), remote, "http://proxy"); out != nil { + t.Error("invalid json should be a no-op") + } + body := []byte(`{"tarball":"https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz"}`) + out, err := p.RewriteResponse(body, remote, "http://proxy") + if err != nil { + t.Fatal(err) + } + if string(out) != `{"tarball":"http://proxy/api/v1/remote/npmjs/pkg/-/pkg-1.0.0.tgz"}` { + t.Errorf("rewrite: %s", out) + } + if out, _ := p.RewriteResponse([]byte(`{"x":"unrelated"}`), remote, "http://proxy"); out != nil { + t.Error("no matching base URL should be a no-op") + } +} + +func TestAuthHeaders(t *testing.T) { + p := &Provider{} + h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "pw"}) + if h.Get("Authorization") == "" { + t.Error("expected auth header when credentials set") + } + h, _ = p.AuthHeaders(context.Background(), models.Remote{}) + if h.Get("Authorization") != "" { + t.Error("expected no auth header without credentials") + } +} diff --git a/internal/provider/puppet/puppet_test.go b/internal/provider/puppet/puppet_test.go new file mode 100644 index 0000000..33b76a3 --- /dev/null +++ b/internal/provider/puppet/puppet_test.go @@ -0,0 +1,78 @@ +package puppet + +import ( + "context" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestType(t *testing.T) { + if (&Provider{}).Type() != models.PackagePuppet { + t.Fatal("wrong type") + } +} + +func TestClassify(t *testing.T) { + p := &Provider{} + if p.Classify("v3/modules/puppetlabs-stdlib") != provider.Mutable { + t.Error("modules should be mutable") + } + if p.Classify("v3/releases?module=x") != provider.Mutable { + t.Error("releases should be mutable") + } + if p.Classify("v3/files/puppetlabs-stdlib-1.0.0.tar.gz") != provider.Immutable { + t.Error("files should be immutable") + } +} + +func TestContentType(t *testing.T) { + p := &Provider{} + if p.ContentType("x/mod-1.0.0.tar.gz") != "application/gzip" { + t.Error("tar.gz") + } + if p.ContentType("v3/modules/x") != "application/json" { + t.Error("v3 json") + } + if p.ContentType("other") != "application/octet-stream" { + t.Error("default") + } +} + +func TestUpstreamURL(t *testing.T) { + got := (&Provider{}).UpstreamURL(models.Remote{BaseURL: "https://forgeapi.puppet.com/"}, "/v3/modules/x") + if got != "https://forgeapi.puppet.com/v3/modules/x" { + t.Errorf("got %q", got) + } +} + +func TestRewriteResponse(t *testing.T) { + p := &Provider{} + remote := models.Remote{Name: "forge", BaseURL: "https://forgeapi.puppet.com"} + + if out, _ := p.RewriteResponse([]byte("x"), remote, ""); out != nil { + t.Error("empty proxyBaseURL is a no-op") + } + + body := []byte(`{"file_uri":"/v3/files/mod.tar.gz","home":"https://forgeapi.puppet.com/x"}`) + out, err := p.RewriteResponse(body, remote, "http://proxy") + if err != nil { + t.Fatal(err) + } + s := string(out) + if !strings.Contains(s, "http://proxy/api/v1/remote/forge/v3/files/mod.tar.gz") { + t.Errorf("v3/files not rewritten: %s", s) + } + if !strings.Contains(s, "http://proxy/api/v1/remote/forge/x") { + t.Errorf("base URL not rewritten: %s", s) + } +} + +func TestAuthHeaders(t *testing.T) { + h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{}) + if h.Get("Authorization") != "" { + t.Error("no credentials, no header") + } +} diff --git a/internal/provider/pypi/pypi_test.go b/internal/provider/pypi/pypi_test.go new file mode 100644 index 0000000..ecabe0e --- /dev/null +++ b/internal/provider/pypi/pypi_test.go @@ -0,0 +1,177 @@ +package pypi + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// fakeFileStore is an in-memory provider.FileStore for exercising local index +// generation without a database. +type fakeFileStore struct { + packages []string + files map[string][]provider.FileEntry +} + +func (f *fakeFileStore) ListPackages(_ context.Context, _ string) ([]string, error) { + return f.packages, nil +} + +func (f *fakeFileStore) ListFilesByPrefix(_ context.Context, _, prefix string) ([]provider.FileEntry, error) { + return f.files[prefix], nil +} + +func TestTypeClassifyContentType(t *testing.T) { + p := &Provider{} + if p.Type() != models.PackagePyPI { + t.Fatal("type") + } + if p.Classify("simple/foo/") != provider.Mutable { + t.Error("simple index should be mutable") + } + if p.Classify("packages/foo-1.0.whl") != provider.Immutable { + t.Error("wheel should be immutable") + } + cases := map[string]string{ + "foo-1.0-py3-none-any.whl": "application/zip", + "foo-1.0.zip": "application/zip", + "foo-1.0.tar.gz": "application/gzip", + "simple/foo/": "text/html", + "weird": "application/octet-stream", + } + for path, want := range cases { + if got := p.ContentType(path); got != want { + t.Errorf("ContentType(%q)=%q want %q", path, got, want) + } + } +} + +func TestUpstreamURL(t *testing.T) { + p := &Provider{} + if got := p.UpstreamURL(models.Remote{BaseURL: "https://files.example.com"}, "packages/foo.whl"); got != "https://files.example.com/packages/foo.whl" { + t.Errorf("got %q", got) + } + if got := p.UpstreamURL(models.Remote{BaseURL: "https://x"}, "simple/foo/"); got != "https://pypi.org/simple/foo/" { + t.Errorf("simple should hit pypi.org, got %q", got) + } +} + +func TestValidateUpload(t *testing.T) { + p := &Provider{} + sp, ct, err := p.ValidateUpload("numpy-1.26.0-cp311-cp311-linux_x86_64.whl") + if err != nil || sp != "numpy/numpy-1.26.0-cp311-cp311-linux_x86_64.whl" || ct != "application/zip" { + t.Errorf("wheel: sp=%q ct=%q err=%v", sp, ct, err) + } + sp, ct, err = p.ValidateUpload("requests-2.31.0.tar.gz") + if err != nil || sp != "requests/requests-2.31.0.tar.gz" || ct != "application/gzip" { + t.Errorf("sdist: sp=%q ct=%q err=%v", sp, ct, err) + } + if _, _, err := p.ValidateUpload("not-a-package.txt"); err == nil { + t.Error("expected error for bad extension") + } +} + +func TestPackageNameParsing(t *testing.T) { + if got := packageFromWheel("Foo_Bar-1.0-py3-none-any.whl"); got != "foo-bar" { + t.Errorf("wheel name = %q", got) + } + if got := packageFromWheel("noseparator.whl"); got != "" { + t.Errorf("expected empty for unparseable wheel, got %q", got) + } + if got := packageFromSdist("My.Pkg-2.0.tar.gz"); got != "my-pkg" { + t.Errorf("sdist name = %q", got) + } + if got := packageFromSdist("noseparator.zip"); got != "" { + t.Errorf("expected empty, got %q", got) + } +} + +func TestUploadResponse(t *testing.T) { + resp := (&Provider{}).UploadResponse("foo/foo-1.0.whl", "sha256:abc", 123) + if resp["filename"] != "foo-1.0.whl" || resp["package"] != "foo" || resp["content_hash"] != "sha256:abc" { + t.Errorf("unexpected upload response: %v", resp) + } +} + +func TestRewriteResponse(t *testing.T) { + p := &Provider{} + if out, _ := p.RewriteResponse([]byte("x"), models.Remote{Name: "pypi"}, ""); out != nil { + t.Error("empty proxyBaseURL is a no-op") + } + body := []byte(`foo.whl`) + out, err := p.RewriteResponse(body, models.Remote{Name: "pypi"}, "http://proxy") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), "http://proxy/api/v1/remote/pypi/") { + t.Errorf("not rewritten: %s", out) + } +} + +func TestGenerateLocalIndex(t *testing.T) { + p := &Provider{} + fs := &fakeFileStore{ + packages: []string{"foo", "bar"}, + files: map[string][]provider.FileEntry{ + "foo/": {{FilePath: "foo/foo-1.0-py3-none-any.whl", ContentHash: "sha256:aaa"}}, + }, + } + list, err := p.GenerateLocalIndex(context.Background(), fs, "local", "simple/") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(list), "foo") || !strings.Contains(string(list), "bar") { + t.Errorf("package list missing entries: %s", list) + } + + files, err := p.GenerateLocalIndex(context.Background(), fs, "local", "simple/foo/") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(files), "foo-1.0-py3-none-any.whl") { + t.Errorf("file list missing wheel: %s", files) + } + + if _, err := p.GenerateLocalIndex(context.Background(), fs, "local", "notsimple"); err == nil { + t.Error("expected error for non-simple path") + } +} + +func TestServeLocalIndexHTTP(t *testing.T) { + p := &Provider{} + fs := &fakeFileStore{ + packages: []string{"foo"}, + files: map[string][]provider.FileEntry{ + "foo/": {{FilePath: "foo/foo-1.0-py3-none-any.whl", ContentHash: "sha256:aaa"}}, + }, + } + serve := func(path string) (*httptest.ResponseRecorder, bool) { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/"+path, nil) + handled := p.ServeLocalIndex(w, r, fs, "local", path) + return w, handled + } + + if w, ok := serve("simple/"); !ok || w.Code != 200 || !strings.Contains(w.Body.String(), "foo") { + t.Errorf("simple index: handled=%v code=%d body=%s", ok, w.Code, w.Body.String()) + } + if w, ok := serve("simple/foo/"); !ok || w.Code != 200 || !strings.Contains(w.Body.String(), "foo-1.0-py3-none-any.whl") { + t.Errorf("package index: handled=%v code=%d body=%s", ok, w.Code, w.Body.String()) + } + // Non-simple paths are not handled. + if _, ok := serve("packages/foo.whl"); ok { + t.Error("non-index path should not be handled") + } +} + +func TestAuthHeaders(t *testing.T) { + h, _ := (&Provider{}).AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"}) + if h.Get("Authorization") == "" { + t.Error("expected auth header") + } +} diff --git a/internal/provider/rpm/rpm_meta_test.go b/internal/provider/rpm/rpm_meta_test.go new file mode 100644 index 0000000..1aa3c2f --- /dev/null +++ b/internal/provider/rpm/rpm_meta_test.go @@ -0,0 +1,276 @@ +package rpm + +import ( + "bytes" + "compress/gzip" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/testsupport" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +type fakeBlobReader struct{ data []byte } + +func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) { + return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil +} + +type fakeMetaStore struct{ inserted *provider.RPMMetadata } + +func (f *fakeMetaStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error { + f.inserted = m + return nil +} + +type fakeRPMReader struct{ metas []provider.RPMMetadata } + +func (f fakeRPMReader) ListRPMMetadataEntries(_ context.Context, _ string) ([]provider.RPMMetadata, error) { + return f.metas, nil +} +func (f fakeRPMReader) ListFilesByPrefix(_ context.Context, _, _ string) ([]provider.FileEntry, error) { + return nil, nil +} +func (f fakeRPMReader) ListPackages(_ context.Context, _ string) ([]string, error) { return nil, nil } + +func TestRPMPureFuncs(t *testing.T) { + p := &Provider{} + if p.Type() != models.PackageRPM { + t.Error("type") + } + if p.Classify("repodata/repomd.xml") != provider.Mutable { + t.Error("repomd should be mutable") + } + if p.Classify("Packages/foo.rpm") != provider.Immutable { + t.Error("rpm should be immutable") + } + if p.ContentType("x.rpm") != "application/x-rpm" { + t.Error("rpm content type") + } + if got := p.UpstreamURL(models.Remote{BaseURL: "https://mirror/"}, "/Packages/x.rpm"); got != "https://mirror/Packages/x.rpm" { + t.Errorf("upstream url %q", got) + } + if out, _ := p.RewriteResponse(nil, models.Remote{}, "http://p"); out != nil { + t.Error("rpm never rewrites") + } + h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"}) + if h.Get("Authorization") == "" { + t.Error("auth header") + } +} + +func TestRPMValidateUpload(t *testing.T) { + p := &Provider{} + sp, ct, err := p.ValidateUpload("dir/foo-1.0.noarch.rpm") + if err != nil || sp != "Packages/foo-1.0.noarch.rpm" || ct != "application/x-rpm" { + t.Errorf("sp=%q ct=%q err=%v", sp, ct, err) + } + if _, _, err := p.ValidateUpload("foo.txt"); err == nil { + t.Error("expected error for non-rpm") + } + resp := p.UploadResponse("Packages/foo.rpm", "sha256:abc", 10) + if resp["content_hash"] != "sha256:abc" { + t.Errorf("upload response %v", resp) + } +} + +func TestRPMAfterUpload(t *testing.T) { + data := testsupport.MinimalRPM("e2e-testpkg", "1.0", "1", "noarch") + store := &fakeMetaStore{} + (&Provider{}).AfterUpload(context.Background(), "myrepo", "Packages/e2e-testpkg-1.0-1.noarch.rpm", + "sha256:deadbeef", fakeBlobReader{data: data}, store) + + m := store.inserted + if m == nil { + t.Fatal("no metadata inserted") + } + if m.Name != "e2e-testpkg" || m.Version != "1.0" || m.Release != "1" || m.Arch != "noarch" { + t.Errorf("unexpected metadata: %+v", m) + } + if m.RPMSize != int64(len(data)) { + t.Errorf("RPMSize = %d, want %d", m.RPMSize, len(data)) + } + if len(m.Provides) == 0 { + t.Error("expected the package to provide itself") + } +} + +type errBlobReader struct{} + +func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) { + return nil, 0, io.ErrUnexpectedEOF +} + +func TestRPMAfterUploadErrors(t *testing.T) { + // Download failure: no metadata inserted, no panic. + store := &fakeMetaStore{} + (&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", errBlobReader{}, store) + if store.inserted != nil { + t.Error("no metadata should be inserted on download error") + } + // Parse failure: garbage bytes are not a valid RPM. + store2 := &fakeMetaStore{} + (&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", fakeBlobReader{data: []byte("not an rpm")}, store2) + if store2.inserted != nil { + t.Error("no metadata should be inserted on parse error") + } +} + +func TestRPMServeRepodata(t *testing.T) { + p := &Provider{} + reader := fakeRPMReader{metas: []provider.RPMMetadata{{ + Name: "e2e-testpkg", Version: "1.0", Release: "1", Arch: "noarch", + Summary: "test & ", + ContentHash: "sha256:abc", + Requires: []provider.RPMDep{{Name: "libc", Flags: "GE", Version: "2.0"}}, + Provides: []provider.RPMDep{{Name: "e2e-testpkg"}}, + Files: []provider.RPMFile{{Path: "/usr/share/e2e/README", Type: "file"}}, + Changelogs: []provider.RPMChangelog{{Author: "e2e", Date: 1, Text: "init"}}, + }}} + + serve := func(path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/"+path, nil) + if !p.ServeLocalIndex(w, r, reader, "myrepo", path) { + t.Fatalf("ServeLocalIndex returned false for %q", path) + } + return w + } + + if w := serve("repodata/repomd.xml"); w.Code != 200 || !strings.Contains(w.Body.String(), " 404. + if w := serve("repodata/bogus"); w.Code != http.StatusNotFound { + t.Errorf("bogus repodata: code %d", w.Code) + } + // Non-repodata path -> not handled. + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/Packages/x.rpm", nil) + if p.ServeLocalIndex(w, r, reader, "myrepo", "Packages/x.rpm") { + t.Error("expected ServeLocalIndex false for non-repodata path") + } +} + +type errRPMReader struct{} + +func (errRPMReader) ListRPMMetadataEntries(context.Context, string) ([]provider.RPMMetadata, error) { + return nil, io.ErrUnexpectedEOF +} +func (errRPMReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) { + return nil, nil +} +func (errRPMReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil } + +func TestRPMServeMetadataError(t *testing.T) { + p := &Provider{} + for _, path := range []string{"repodata/repomd.xml", "repodata/h-primary.xml.gz", "repodata/h-filelists.xml.gz", "repodata/h-other.xml.gz"} { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/"+path, nil) + p.ServeLocalIndex(w, r, errRPMReader{}, "repo", path) + if w.Code != 500 { + t.Errorf("%s with failing reader = %d, want 500", path, w.Code) + } + } +} + +func TestRPMFullMetadataXML(t *testing.T) { + // A fully-populated entry exercises every optional-field branch in the + // primary/filelists/other XML generators. + metas := []provider.RPMMetadata{{ + Name: "full", Epoch: 1, Version: "2.0", Release: "3", Arch: "x86_64", + Summary: "s", Description: "d", License: "MIT", Vendor: "acme", + Group: "System", BuildHost: "build.example.com", SourceRPM: "full-2.0.src.rpm", + URL: "https://example.com", Packager: "pkgr", ContentHash: "sha256:abc", + RPMSize: 100, InstalledSize: 200, + Requires: []provider.RPMDep{{Name: "libc", Flags: "GE", Epoch: "0", Version: "2.0", Release: "1"}}, + Provides: []provider.RPMDep{{Name: "full", Flags: "EQ", Version: "2.0"}}, + Files: []provider.RPMFile{{Path: "/usr/bin/full", Type: "file"}, {Path: "/etc/full", Type: "dir"}}, + Changelogs: []provider.RPMChangelog{{Author: "a", Date: 100, Text: "changed"}}, + }} + for _, gen := range []func([]provider.RPMMetadata) []byte{generatePrimaryXMLGZ, generateFilelistsXMLGZ, generateOtherXMLGZ} { + zr, err := gzip.NewReader(bytes.NewReader(gen(metas))) + if err != nil { + t.Fatal(err) + } + if _, err := io.ReadAll(zr); err != nil { + t.Error(err) + } + } +} + +func TestRPMPrimaryXMLContents(t *testing.T) { + // Exercise xmlEscape and dependency entry writing through the gzip'd XML. + metas := []provider.RPMMetadata{{ + Name: "pkg", Version: "1", Release: "1", Arch: "x86_64", Summary: "a & b", + Requires: []provider.RPMDep{{Name: "dep", Flags: "EQ", Version: "1.0", Epoch: "0"}}, + }} + gz := generatePrimaryXMLGZ(metas) + zr, err := gzip.NewReader(bytes.NewReader(gz)) + if err != nil { + t.Fatal(err) + } + out, _ := io.ReadAll(zr) + s := string(out) + if !strings.Contains(s, "a & b") { + t.Errorf("summary not xml-escaped: %s", s) + } + if !strings.Contains(s, "pkg") { + t.Errorf("package name missing: %s", s) + } +} + +func TestRPMContentTypeAndHelpers(t *testing.T) { + p := &Provider{} + for path, want := range map[string]string{ + "x.rpm": "application/x-rpm", + "repodata/repomd.xml": "application/xml", + "repodata/h-primary.xml.gz": "application/xml", + "repodata/h-primary.xml.xz": "application/xml", + "Packages/other": "application/octet-stream", + } { + if got := p.ContentType(path); got != want { + t.Errorf("ContentType(%q)=%q want %q", path, got, want) + } + } + + for flag, want := range map[int]string{ + 0x08 | 0x04: "GE", + 0x02 | 0x04: "LE", + 0x08: "GT", + 0x02: "LT", + 0x04: "EQ", + 0x00: "", + } { + if got := rpmFlagString(flag); got != want { + t.Errorf("rpmFlagString(%d)=%q want %q", flag, got, want) + } + } + + if firstGroup(nil) != "Unspecified" { + t.Error("empty groups should be Unspecified") + } + if firstGroup([]string{"System", "Base"}) != "System" { + t.Error("firstGroup should return the first") + } +} + +func TestGenerateLocalIndexUnsupported(t *testing.T) { + if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeRPMReader{}, "r", "simple/"); err == nil { + t.Error("expected unsupported error") + } +} diff --git a/internal/proxy/classifier_extra_test.go b/internal/proxy/classifier_extra_test.go new file mode 100644 index 0000000..dd5403a --- /dev/null +++ b/internal/proxy/classifier_extra_test.go @@ -0,0 +1,52 @@ +package proxy + +import ( + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + _ "git.unkin.net/unkin/artifactapi/internal/provider/generic" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestClassifierBranches(t *testing.T) { + gp, err := provider.Get(models.PackageGeneric) + if err != nil { + t.Fatal(err) + } + c := NewClassifier(gp) + + if c.Classify(models.Remote{Blocklist: []string{`\.exe$`}}, "x.exe") != ClassDenied { + t.Error("blocklist match should be denied") + } + // Allowlist present but path doesn't match -> denied. + allow := models.Remote{Patterns: []string{`^allowed/`}} + if c.Classify(allow, "other/x") != ClassDenied { + t.Error("non-allowlisted path should be denied") + } + if c.Classify(allow, "allowed/x") != ClassImmutable { + t.Error("allowlisted generic path should be immutable") + } + if c.Classify(models.Remote{MutablePatterns: []string{`index$`}}, "a/index") != ClassMutable { + t.Error("mutable pattern override failed") + } + if c.Classify(models.Remote{ImmutablePatterns: []string{`\.bin$`}}, "a.bin") != ClassImmutable { + t.Error("immutable pattern failed") + } + // An invalid regex is skipped (not treated as a match) rather than denying. + if c.Classify(models.Remote{Blocklist: []string{`[invalid`}}, "anything") == ClassDenied { + t.Error("invalid blocklist regex should be skipped, not deny everything") + } +} + +func TestClassificationString(t *testing.T) { + for c, want := range map[Classification]string{ + ClassImmutable: "immutable", + ClassMutable: "mutable", + ClassDenied: "denied", + Classification(99): "unknown", + } { + if c.String() != want { + t.Errorf("Classification(%d).String() = %q, want %q", c, c.String(), want) + } + } +} diff --git a/internal/proxy/engine_test.go b/internal/proxy/engine_test.go new file mode 100644 index 0000000..87ce690 --- /dev/null +++ b/internal/proxy/engine_test.go @@ -0,0 +1,557 @@ +package proxy + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/cache" + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/provider" + _ "git.unkin.net/unkin/artifactapi/internal/provider/generic" + _ "git.unkin.net/unkin/artifactapi/internal/provider/npm" + "git.unkin.net/unkin/artifactapi/internal/storage" + "git.unkin.net/unkin/artifactapi/internal/testsupport" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +var ( + testEngine *Engine + testCache *cache.Redis + testDB *database.DB + upstream *httptest.Server +) + +func TestMain(m *testing.M) { + ctx := context.Background() + dsn, termPG, err := testsupport.StartPostgres(ctx) + if err != nil { + os.Exit(m.Run()) + } + redisURL, termRedis, err := testsupport.StartRedis(ctx) + if err != nil { + termPG() + os.Exit(m.Run()) + } + minio, termMinio, err := testsupport.StartMinio(ctx) + if err != nil { + termPG() + termRedis() + os.Exit(m.Run()) + } + + db, err := database.New(dsn) + if err != nil { + panic(err) + } + redis, err := cache.NewRedis(redisURL) + if err != nil { + panic(err) + } + var s3 *storage.S3 + for i := 0; i < 20; i++ { + if s3, err = storage.NewS3(minio.Endpoint, minio.AccessKey, minio.SecretKey, "proxy-test", false, ""); err == nil { + break + } + time.Sleep(500 * time.Millisecond) + } + if err != nil { + panic(err) + } + + testCache = redis + testDB = db + testEngine = NewEngine(db, redis, s3) + upstream = httptest.NewServer(http.HandlerFunc(mockUpstream)) + + code := m.Run() + + upstream.Close() + db.Close() + termMinio() + termRedis() + termPG() + if code != 0 { + os.Exit(code) + } +} + +func mockUpstream(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/blob.bin": + w.Header().Set("Content-Type", "application/octet-stream") + w.Write([]byte("immutable blob")) + case "/pkg": // npm metadata: mutable, supports revalidation + if r.Method == http.MethodHead && r.Header.Get("If-None-Match") == `"v1"` { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", `"v1"`) + w.Write([]byte(`{"name":"pkg"}`)) + case "/protected.bin": // requires a bearer token obtained from /token + if r.Header.Get("Authorization") != "Bearer minted-token" { + w.Header().Set("Www-Authenticate", `Bearer realm="`+upstream.URL+`/token",service="reg",scope="repo:pull"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Write([]byte("protected payload")) + case "/protected2.bin": // same challenge as /protected.bin + if r.Header.Get("Authorization") != "Bearer minted-token" { + w.Header().Set("Www-Authenticate", `Bearer realm="`+upstream.URL+`/token",service="reg",scope="repo:pull"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Write([]byte("protected payload 2")) + case "/token": + w.Write([]byte(`{"token":"minted-token","expires_in":300}`)) + case "/token-at": + w.Write([]byte(`{"access_token":"at-token"}`)) + case "/token-500": + w.WriteHeader(http.StatusInternalServerError) + case "/err500": + w.WriteHeader(http.StatusInternalServerError) + case "/noauth": // 401 with an unusable challenge (no realm) + w.Header().Set("Www-Authenticate", `Bearer service="reg"`) + w.WriteHeader(http.StatusUnauthorized) + default: + http.NotFound(w, r) + } +} + +func requireStack(t *testing.T) { + t.Helper() + if testEngine == nil { + t.Skip("Docker unavailable; skipping proxy engine test") + } +} + +func genericRemote(name string) models.Remote { + return models.Remote{Name: name, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, StaleOnError: true} +} + +// seed inserts the remote so artifact rows (FK to remotes) can be stored. +func seed(t *testing.T, r models.Remote) models.Remote { + t.Helper() + rr := r + if err := testDB.CreateRemote(context.Background(), &rr); err != nil { + t.Fatalf("seed remote %s: %v", r.Name, err) + } + return r +} + +func prov(t *testing.T, pt models.PackageType) provider.Provider { + p, err := provider.Get(pt) + if err != nil { + t.Fatalf("provider %s: %v", pt, err) + } + return p +} + +func readAll(t *testing.T, res *FetchResult) string { + t.Helper() + defer res.Reader.Close() + b, _ := io.ReadAll(res.Reader) + return string(b) +} + +func TestFetchImmutableMissThenHit(t *testing.T) { + requireStack(t) + ctx := context.Background() + r := seed(t, genericRemote("eng-imm")) + p := prov(t, models.PackageGeneric) + + res, err := testEngine.Fetch(ctx, r, "blob.bin", p) + if err != nil { + t.Fatalf("fetch: %v", err) + } + if res.Source != "remote" || readAll(t, res) != "immutable blob" { + t.Errorf("miss: source=%s", res.Source) + } + res, err = testEngine.Fetch(ctx, r, "blob.bin", p) + if err != nil { + t.Fatal(err) + } + if res.Source != "cache" || readAll(t, res) != "immutable blob" { + t.Errorf("hit: source=%s", res.Source) + } +} + +func TestFetchDenied(t *testing.T) { + requireStack(t) + r := genericRemote("eng-deny") + r.Blocklist = []string{`\.secret$`} + _, err := testEngine.Fetch(context.Background(), r, "x.secret", prov(t, models.PackageGeneric)) + var pe *ProxyError + if err == nil || !asProxyError(err, &pe) || pe.Status != http.StatusForbidden { + t.Errorf("expected 403 ProxyError, got %v", err) + } +} + +func TestHead(t *testing.T) { + requireStack(t) + ctx := context.Background() + r := seed(t, genericRemote("eng-head")) + p := prov(t, models.PackageGeneric) + + // Uncached HEAD hits upstream. + h, err := testEngine.Head(ctx, r, "blob.bin", p) + if err != nil || h.Source != "remote" { + t.Fatalf("head uncached: %+v %v", h, err) + } + // Populate the cache, then HEAD should be served from metadata. + res, _ := testEngine.Fetch(ctx, r, "blob.bin", p) + res.Reader.Close() + h, err = testEngine.Head(ctx, r, "blob.bin", p) + if err != nil || h.Source != "cache" { + t.Errorf("head cached: %+v %v", h, err) + } + // Denied HEAD. + r.Blocklist = []string{".*"} + if _, err := testEngine.Head(ctx, r, "blob.bin", p); err == nil { + t.Error("expected denied head error") + } +} + +func TestStaleOnError(t *testing.T) { + requireStack(t) + ctx := context.Background() + r := seed(t, genericRemote("eng-stale")) + p := prov(t, models.PackageGeneric) + + if _, err := testEngine.Fetch(ctx, r, "blob.bin", p); err != nil { + t.Fatal(err) + } + // Drop cache freshness so the next fetch goes upstream, then point at a + // dead upstream: stale-on-error must serve the stored copy. + testCache.FlushRemote(ctx, "eng-stale") + r.BaseURL = "http://127.0.0.1:1" + res, err := testEngine.Fetch(ctx, r, "blob.bin", p) + if err != nil { + t.Fatalf("expected stale serve, got %v", err) + } + if res.Source != "cache" || readAll(t, res) != "immutable blob" { + t.Errorf("stale: source=%s", res.Source) + } +} + +func TestCircuitOpenServesStale(t *testing.T) { + requireStack(t) + ctx := context.Background() + r := seed(t, genericRemote("eng-circuit")) + p := prov(t, models.PackageGeneric) + if _, err := testEngine.Fetch(ctx, r, "blob.bin", p); err != nil { + t.Fatal(err) + } + testCache.FlushRemote(ctx, "eng-circuit") + for i := 0; i < 6; i++ { + testEngine.circuit.RecordFailure(ctx, "eng-circuit") + } + res, err := testEngine.Fetch(ctx, r, "blob.bin", p) + if err != nil { + t.Fatalf("circuit-open should serve stale: %v", err) + } + if res.Source != "cache" { + t.Errorf("expected stale from open circuit, got %s", res.Source) + } + res.Reader.Close() +} + +func TestMutableRevalidation(t *testing.T) { + requireStack(t) + ctx := context.Background() + r := seed(t, models.Remote{Name: "eng-npm", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600, StaleOnError: true}) + p := prov(t, models.PackageNPM) + + res, err := testEngine.Fetch(ctx, r, "pkg", p) + if err != nil { + t.Fatalf("initial mutable fetch: %v", err) + } + res.Reader.Close() + + // Expire only the freshness marker; the ETag persists, forcing a + // conditional revalidation that the upstream answers with 304. + testCache.SetTTL(ctx, "eng-npm", "pkg", time.Millisecond) + time.Sleep(10 * time.Millisecond) + res, err = testEngine.Fetch(ctx, r, "pkg", p) + if err != nil { + t.Fatalf("revalidation fetch: %v", err) + } + if res.Source != "cache" { + t.Errorf("revalidated response should come from cache, got %s", res.Source) + } + res.Reader.Close() +} + +func TestBearerTokenFlow(t *testing.T) { + requireStack(t) + ctx := context.Background() + r := seed(t, genericRemote("eng-bearer")) + p := prov(t, models.PackageGeneric) + + // GET: 401 challenge -> token endpoint -> retry with bearer -> 200. + res, err := testEngine.Fetch(ctx, r, "protected.bin", p) + if err != nil { + t.Fatalf("bearer fetch: %v", err) + } + if readAll(t, res) != "protected payload" { + t.Error("bearer-protected content mismatch") + } + + // A second protected path with the same challenge reuses the cached token. + res2, err := testEngine.Fetch(ctx, r, "protected2.bin", p) + if err != nil { + t.Fatalf("second bearer fetch: %v", err) + } + if readAll(t, res2) != "protected payload 2" { + t.Error("second bearer content mismatch") + } + + // HEAD path also negotiates a bearer token (uncached). + testCache.FlushRemote(ctx, "eng-bearer") + testDB.DeleteArtifact(ctx, "eng-bearer", "protected.bin") + if h, err := testEngine.Head(ctx, r, "protected.bin", p); err != nil || h.Source != "cache" && h.Source != "remote" { + t.Fatalf("bearer head: %+v %v", h, err) + } +} + +func TestFetchUpstreamError(t *testing.T) { + requireStack(t) + r := seed(t, genericRemote("eng-404")) + // Upstream 404 (no cached copy, stale-on-error can't help) -> ProxyError. + _, err := testEngine.Fetch(context.Background(), r, "missing", prov(t, models.PackageGeneric)) + var pe *ProxyError + if err == nil || !asProxyError(err, &pe) || pe.Status != http.StatusNotFound { + t.Errorf("expected 404 ProxyError, got %v", err) + } + // HEAD of a missing upstream path also errors. + if _, err := testEngine.Head(context.Background(), r, "missing", prov(t, models.PackageGeneric)); err == nil { + t.Error("expected head error for missing path") + } +} + +func TestFetchUpstreamStatusErrors(t *testing.T) { + requireStack(t) + ctx := context.Background() + p := prov(t, models.PackageGeneric) + + r := seed(t, genericRemote("eng-500")) + _, err := testEngine.Fetch(ctx, r, "err500", p) + var pe *ProxyError + if err == nil || !asProxyError(err, &pe) || pe.Status != http.StatusInternalServerError { + t.Errorf("expected 500 ProxyError, got %v", err) + } + + r = seed(t, genericRemote("eng-noauth")) + _, err = testEngine.Fetch(ctx, r, "noauth", p) + if err == nil || !asProxyError(err, &pe) || pe.Status != http.StatusUnauthorized { + t.Errorf("expected 401 ProxyError, got %v", err) + } +} + +func TestBearerTokenParsing(t *testing.T) { + // Non-Bearer challenges and missing realms are rejected. + if _, _, err := fetchBearerToken(context.Background(), "Basic realm=x", models.Remote{}); err == nil { + t.Error("expected error for non-Bearer challenge") + } + if _, _, err := fetchBearerToken(context.Background(), `Bearer service="reg"`, models.Remote{}); err == nil { + t.Error("expected error for missing realm") + } +} + +func TestWaitForStoreCoalesces(t *testing.T) { + requireStack(t) + ctx := context.Background() + r := seed(t, genericRemote("eng-herd")) + p := prov(t, models.PackageGeneric) + + // Fire concurrent cold-cache fetches: only one holds the lock, the others + // wait on the store (waitForStore) and pick up the result. + const n = 4 + done := make(chan string, n) + for i := 0; i < n; i++ { + go func() { + res, err := testEngine.Fetch(ctx, r, "blob.bin", p) + if err != nil { + done <- "err:" + err.Error() + return + } + done <- readAll(t, res) + }() + } + for i := 0; i < n; i++ { + if got := <-done; got != "immutable blob" { + t.Errorf("concurrent fetch got %q", got) + } + } +} + +func TestRevalidationUpstreamError(t *testing.T) { + requireStack(t) + ctx := context.Background() + r := seed(t, models.Remote{Name: "eng-reval-err", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600, StaleOnError: true}) + p := prov(t, models.PackageNPM) + + res, err := testEngine.Fetch(ctx, r, "pkg", p) + if err != nil { + t.Fatalf("initial fetch: %v", err) + } + res.Reader.Close() + + // Expire freshness but keep the ETag, then break the upstream: the + // conditional HEAD (checkUpstream) errors, and stale-on-error serves the + // stored index. + testCache.SetTTL(ctx, "eng-reval-err", "pkg", time.Millisecond) + time.Sleep(10 * time.Millisecond) + r.BaseURL = "http://127.0.0.1:1" + res, err = testEngine.Fetch(ctx, r, "pkg", p) + if err != nil { + t.Fatalf("expected stale serve on revalidation error, got %v", err) + } + if res.Source != "cache" { + t.Errorf("expected stale cache source, got %s", res.Source) + } + res.Reader.Close() +} + +func TestTTLFor(t *testing.T) { + e := &Engine{} + if got := e.ttlFor(models.Remote{ImmutableTTL: 100}, ClassImmutable); got != 100*time.Second { + t.Errorf("immutable ttl = %v", got) + } + if got := e.ttlFor(models.Remote{ImmutableTTL: 0}, ClassImmutable); got != 0 { + t.Errorf("immutable ttl=0 (forever) = %v", got) + } + if got := e.ttlFor(models.Remote{MutableTTL: 50}, ClassMutable); got != 50*time.Second { + t.Errorf("mutable ttl = %v", got) + } +} + +func TestHeadUpstreamStatusError(t *testing.T) { + requireStack(t) + r := seed(t, genericRemote("eng-head500")) + if _, err := testEngine.Head(context.Background(), r, "err500", prov(t, models.PackageGeneric)); err == nil { + t.Error("expected error for HEAD of 500 upstream") + } +} + +func TestHeadCachedIndex(t *testing.T) { + requireStack(t) + ctx := context.Background() + r := seed(t, models.Remote{Name: "eng-headidx", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600}) + p := prov(t, models.PackageNPM) + // Cache the mutable index, then HEAD is answered from the stored index. + res, err := testEngine.Fetch(ctx, r, "pkg", p) + if err != nil { + t.Fatal(err) + } + res.Reader.Close() + h, err := testEngine.Head(ctx, r, "pkg", p) + if err != nil || h.Source != "cache" { + t.Errorf("head of cached index: %+v %v", h, err) + } +} + +func TestFetchBearerTokenVariants(t *testing.T) { + requireStack(t) + ctx := context.Background() + + // access_token field + service/scope params + basic auth on the token req. + tok, _, err := fetchBearerToken(ctx, `Bearer realm="`+upstream.URL+`/token-at",service="reg",scope="repo:pull"`, models.Remote{Username: "u", Password: "p"}) + if err != nil || tok != "at-token" { + t.Errorf("access_token variant: tok=%q err=%v", tok, err) + } + // Token endpoint error status. + if _, _, err := fetchBearerToken(ctx, `Bearer realm="`+upstream.URL+`/token-500"`, models.Remote{}); err == nil { + t.Error("expected error for 500 token endpoint") + } +} + +func TestCheckUpstreamChanged(t *testing.T) { + requireStack(t) + ctx := context.Background() + r := genericRemote("eng-check") + // A non-matching ETag yields a normal 200 (not 304): not modified is false. + notModified, err := testEngine.checkUpstream(ctx, r, "pkg", `"stale-etag"`, prov(t, models.PackageNPM)) + if err != nil { + t.Fatalf("checkUpstream: %v", err) + } + if notModified { + t.Error("mismatched etag should report modified (notModified=false)") + } +} + +func TestUpstreamErrorUnwrap(t *testing.T) { + base := context.DeadlineExceeded + ue := &UpstreamError{Err: base} + if ue.Unwrap() != base { + t.Error("Unwrap should return the wrapped error") + } + if !isNetworkError(ue) { + t.Error("UpstreamError should be a network error") + } + if isNetworkError(context.Canceled) { + t.Error("plain error should not be a network error") + } +} + +func TestImmutableBlobDedup(t *testing.T) { + requireStack(t) + ctx := context.Background() + p := prov(t, models.PackageGeneric) + // Two remotes serving identical content: the second store hits the + // already-exists branch (blob content is deduplicated). + for _, name := range []string{"eng-dedup-a", "eng-dedup-b"} { + r := seed(t, genericRemote(name)) + res, err := testEngine.Fetch(ctx, r, "blob.bin", p) + if err != nil { + t.Fatalf("%s fetch: %v", name, err) + } + if readAll(t, res) != "immutable blob" { + t.Errorf("%s content mismatch", name) + } + } +} + +func TestCircuitBreakerStates(t *testing.T) { + requireStack(t) + ctx := context.Background() + cb := NewCircuitBreaker(testCache) + const key = "cb-states" + testCache.ResetCircuit(ctx, key) + + if cb.IsOpen(ctx, key) { + t.Error("fresh breaker should be closed") + } + if cb.Health(ctx, key).Status != "healthy" { + t.Error("fresh breaker should be healthy") + } + cb.RecordFailure(ctx, key) + if s := cb.Health(ctx, key).Status; s != "degraded" { + t.Errorf("one failure should be degraded, got %q", s) + } + for i := 0; i < 6; i++ { + cb.RecordFailure(ctx, key) + } + if !cb.IsOpen(ctx, key) { + t.Error("breaker should be open after threshold failures") + } + if s := cb.Health(ctx, key).Status; s != "down" { + t.Errorf("open breaker should be down, got %q", s) + } + cb.RecordSuccess(ctx, key) + if cb.IsOpen(ctx, key) { + t.Error("breaker should close after success") + } +} + +func asProxyError(err error, target **ProxyError) bool { + pe, ok := err.(*ProxyError) + if ok { + *target = pe + } + return ok +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 0000000..62b7bcf --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,620 @@ +package server + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strconv" + "strings" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/config" + "git.unkin.net/unkin/artifactapi/internal/testsupport" +) + +var ( + testTS *httptest.Server // the artifactapi router + upstream *httptest.Server // mock upstream the proxy fetches from + testSrv *Server +) + +func TestMain(m *testing.M) { + ctx := context.Background() + + dsn, termPG, err := testsupport.StartPostgres(ctx) + if err != nil { + os.Exit(m.Run()) + } + defer termPG() + redisURL, termRedis, err := testsupport.StartRedis(ctx) + if err != nil { + termPG() + os.Exit(m.Run()) + } + defer termRedis() + minio, termMinio, err := testsupport.StartMinio(ctx) + if err != nil { + termPG() + termRedis() + os.Exit(m.Run()) + } + defer termMinio() + + u, _ := url.Parse(dsn) + port, _ := strconv.Atoi(u.Port()) + cfg := &config.Config{ + ListenAddr: ":0", + DBHost: u.Hostname(), + DBPort: port, + DBUser: "artifacts", + DBPass: "artifacts123", + DBName: "artifacts", + DBSSL: "disable", + RedisURL: redisURL, + S3Endpoint: minio.Endpoint, + S3AccessKey: minio.AccessKey, + S3SecretKey: minio.SecretKey, + S3Bucket: "server-test", + } + + var srv *Server + for i := 0; i < 20; i++ { // tolerate MinIO reporting ready before bucket ops succeed + if srv, err = New(cfg, "test-version"); err == nil { + break + } + time.Sleep(500 * time.Millisecond) + } + if err != nil { + panic(err) + } + testSrv = srv + testTS = httptest.NewServer(srv.router) + upstream = httptest.NewServer(http.HandlerFunc(mockUpstream)) + + code := m.Run() + + testTS.Close() + upstream.Close() + termMinio() + termRedis() + termPG() + if code != 0 { + os.Exit(code) + } +} + +func mockUpstream(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/data/file.bin": + w.Write([]byte("upstream blob payload")) + case "/helm-a/index.yaml": + w.Write([]byte("apiVersion: v1\nentries:\n alpha:\n - name: alpha\n version: 1.0.0\n urls: [charts/alpha-1.0.0.tgz]\n")) + case "/helm-b/index.yaml": + w.Write([]byte("apiVersion: v1\nentries:\n beta:\n - name: beta\n version: 2.0.0\n urls: [charts/beta-2.0.0.tgz]\n")) + default: + http.NotFound(w, r) + } +} + +func requireStack(t *testing.T) { + t.Helper() + if testTS == nil { + t.Skip("Docker unavailable; skipping server integration test") + } +} + +func req(t *testing.T, method, path string, body string) (*http.Response, []byte) { + t.Helper() + var r io.Reader + if body != "" { + r = strings.NewReader(body) + } + rq, _ := http.NewRequest(method, testTS.URL+path, r) + if body != "" { + rq.Header.Set("Content-Type", "application/json") + } + resp, err := http.DefaultClient.Do(rq) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp, b +} + +func TestServerHealthAndRoot(t *testing.T) { + requireStack(t) + if resp, _ := req(t, "GET", "/health", ""); resp.StatusCode != 200 { + t.Errorf("health: %d", resp.StatusCode) + } + if resp, b := req(t, "GET", "/", ""); resp.StatusCode != 200 || !strings.Contains(string(b), "test-version") { + t.Errorf("root: %d %s", resp.StatusCode, b) + } + if resp, _ := req(t, "GET", "/api/v2/health", ""); resp.StatusCode != 200 { + t.Errorf("health v2: %d", resp.StatusCode) + } +} + +func TestServerRemoteAndProxy(t *testing.T) { + requireStack(t) + create := fmt.Sprintf(`{"name":"srv-remote","package_type":"generic","repo_type":"remote","base_url":%q,"stale_on_error":true}`, upstream.URL) + if resp, b := req(t, "POST", "/api/v2/remotes", create); resp.StatusCode != 201 { + t.Fatalf("create remote: %d %s", resp.StatusCode, b) + } + defer req(t, "DELETE", "/api/v2/remotes/srv-remote", "") + + if resp, _ := req(t, "GET", "/api/v2/remotes/srv-remote", ""); resp.StatusCode != 200 { + t.Errorf("get remote: %d", resp.StatusCode) + } + if resp, _ := req(t, "GET", "/api/v2/remotes", ""); resp.StatusCode != 200 { + t.Errorf("list remotes: %d", resp.StatusCode) + } + + // Proxy fetch: miss then hit. + resp, b := req(t, "GET", "/api/v1/remote/srv-remote/data/file.bin", "") + if resp.StatusCode != 200 || string(b) != "upstream blob payload" { + t.Fatalf("proxy miss: %d %s", resp.StatusCode, b) + } + if src := resp.Header.Get("X-Artifact-Source"); src != "remote" { + t.Errorf("expected remote source, got %q", src) + } + resp, _ = req(t, "GET", "/api/v1/remote/srv-remote/data/file.bin", "") + if resp.Header.Get("X-Artifact-Source") != "cache" { + t.Errorf("second fetch should be cache: %q", resp.Header.Get("X-Artifact-Source")) + } + + // Objects listing + stats now that we have an artifact. + if resp, _ := req(t, "GET", "/api/v2/remotes/srv-remote/objects", ""); resp.StatusCode != 200 { + t.Errorf("objects: %d", resp.StatusCode) + } + if resp, _ := req(t, "GET", "/api/v2/stats", ""); resp.StatusCode != 200 { + t.Errorf("stats: %d", resp.StatusCode) + } + for _, p := range []string{"/api/v2/stats/top-remotes", "/api/v2/stats/top-files-by-hits", "/api/v2/stats/top-files-by-bandwidth"} { + if resp, _ := req(t, "GET", p, ""); resp.StatusCode != 200 { + t.Errorf("%s: %d", p, resp.StatusCode) + } + } +} + +func TestServerLocalUpload(t *testing.T) { + requireStack(t) + if resp, b := req(t, "POST", "/api/v2/remotes", `{"name":"srv-local","package_type":"generic","repo_type":"local"}`); resp.StatusCode != 201 { + t.Fatalf("create local: %d %s", resp.StatusCode, b) + } + defer req(t, "DELETE", "/api/v2/remotes/srv-local", "") + + rq, _ := http.NewRequest("PUT", testTS.URL+"/api/v2/remotes/srv-local/files/dir/hello.bin", strings.NewReader("local payload")) + rq.Header.Set("Content-Type", "text/plain") // exercise the content-type branch + resp, err := http.DefaultClient.Do(rq) + if err != nil || resp.StatusCode != 201 { + t.Fatalf("upload: %v %d", err, resp.StatusCode) + } + resp.Body.Close() + + resp, b := req(t, "GET", "/api/v1/local/srv-local/dir/hello.bin", "") + if resp.StatusCode != 200 || string(b) != "local payload" { + t.Errorf("download local: %d %s", resp.StatusCode, b) + } + // Also download via the v2 files endpoint. + if resp, b := req(t, "GET", "/api/v2/remotes/srv-local/files/dir/hello.bin", ""); resp.StatusCode != 200 || string(b) != "local payload" { + t.Errorf("v2 download: %d %s", resp.StatusCode, b) + } +} + +func TestServerVirtualMerge(t *testing.T) { + requireStack(t) + for _, m := range []string{"a", "b"} { + body := fmt.Sprintf(`{"name":"srv-helm-%s","package_type":"helm","repo_type":"remote","base_url":"%s/helm-%s","stale_on_error":true}`, m, upstream.URL, m) + if resp, b := req(t, "POST", "/api/v2/remotes", body); resp.StatusCode != 201 { + t.Fatalf("create helm-%s: %d %s", m, resp.StatusCode, b) + } + defer req(t, "DELETE", "/api/v2/remotes/srv-helm-"+m, "") + } + if resp, b := req(t, "POST", "/api/v2/virtuals", `{"name":"srv-vh","package_type":"helm","members":["srv-helm-a","srv-helm-b"]}`); resp.StatusCode != 201 { + t.Fatalf("create virtual: %d %s", resp.StatusCode, b) + } + defer req(t, "DELETE", "/api/v2/virtuals/srv-vh", "") + + resp, b := req(t, "GET", "/api/v1/virtual/srv-vh/index.yaml", "") + if resp.StatusCode != 200 { + t.Fatalf("virtual fetch: %d %s", resp.StatusCode, b) + } + s := string(b) + if !strings.Contains(s, "alpha") || !strings.Contains(s, "beta") { + t.Errorf("merged index missing charts: %s", s) + } +} + +func TestServerProbe(t *testing.T) { + requireStack(t) + create := fmt.Sprintf(`{"name":"srv-probe","package_type":"generic","repo_type":"remote","base_url":%q,"stale_on_error":true}`, upstream.URL) + req(t, "POST", "/api/v2/remotes", create) + defer req(t, "DELETE", "/api/v2/remotes/srv-probe", "") + + // Reachable path -> status 200 in the probe body. + if resp, b := req(t, "POST", "/api/v2/probe", `{"remote":"srv-probe","path":"data/file.bin"}`); resp.StatusCode != 200 || !strings.Contains(string(b), `"status":200`) { + t.Errorf("probe reachable: %d %s", resp.StatusCode, b) + } + // Missing upstream path -> upstream error reported (502) in the body. + if resp, b := req(t, "POST", "/api/v2/probe", `{"remote":"srv-probe","path":"missing"}`); resp.StatusCode != 200 || !strings.Contains(string(b), `"status":502`) { + t.Errorf("probe missing: %d %s", resp.StatusCode, b) + } + // Unknown remote -> 404 in the body. + if resp, b := req(t, "POST", "/api/v2/probe", `{"remote":"nope","path":"x"}`); resp.StatusCode != 200 || !strings.Contains(string(b), `"status":404`) { + t.Errorf("probe unknown: %d %s", resp.StatusCode, b) + } + // Bad requests. + if resp, _ := req(t, "POST", "/api/v2/probe", `{}`); resp.StatusCode != 400 { + t.Errorf("probe missing fields: %d", resp.StatusCode) + } + if resp, _ := req(t, "POST", "/api/v2/probe", `not json`); resp.StatusCode != 400 { + t.Errorf("probe invalid json: %d", resp.StatusCode) + } +} + +func put(t *testing.T, path string, body []byte) (*http.Response, []byte) { + t.Helper() + rq, _ := http.NewRequest("PUT", testTS.URL+path, bytes.NewReader(body)) + resp, err := http.DefaultClient.Do(rq) + if err != nil { + t.Fatalf("PUT %s: %v", path, err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp, b +} + +func TestServerLocalPyPI(t *testing.T) { + requireStack(t) + if resp, b := req(t, "POST", "/api/v2/remotes", `{"name":"srv-pypi","package_type":"pypi","repo_type":"local"}`); resp.StatusCode != 201 { + t.Fatalf("create pypi local: %d %s", resp.StatusCode, b) + } + defer req(t, "DELETE", "/api/v2/remotes/srv-pypi", "") + + if resp, b := put(t, "/api/v2/remotes/srv-pypi/files/foo-1.0-py3-none-any.whl", []byte("wheel bytes")); resp.StatusCode != 201 { + t.Fatalf("upload wheel: %d %s", resp.StatusCode, b) + } + // Re-uploading the same file is rejected. + if resp, _ := put(t, "/api/v2/remotes/srv-pypi/files/foo-1.0-py3-none-any.whl", []byte("again")); resp.StatusCode != 409 { + t.Errorf("expected 409 on overwrite, got %d", resp.StatusCode) + } + // Invalid pypi filename rejected. + if resp, _ := put(t, "/api/v2/remotes/srv-pypi/files/not-a-package.txt", []byte("x")); resp.StatusCode != 400 { + t.Errorf("expected 400 for bad filename, got %d", resp.StatusCode) + } + + if resp, b := req(t, "GET", "/api/v1/local/srv-pypi/simple/", ""); resp.StatusCode != 200 || !strings.Contains(string(b), "foo") { + t.Errorf("simple index: %d %s", resp.StatusCode, b) + } + if resp, b := req(t, "GET", "/api/v1/local/srv-pypi/simple/foo/", ""); resp.StatusCode != 200 || !strings.Contains(string(b), "foo-1.0-py3-none-any.whl") { + t.Errorf("package index: %d %s", resp.StatusCode, b) + } +} + +func TestServerLocalRPMRepodata(t *testing.T) { + requireStack(t) + rpm := testsupport.MinimalRPM("e2e-testpkg", "1.0", "1", "noarch") + if resp, b := req(t, "POST", "/api/v2/remotes", `{"name":"srv-rpm","package_type":"rpm","repo_type":"local"}`); resp.StatusCode != 201 { + t.Fatalf("create rpm local: %d %s", resp.StatusCode, b) + } + defer req(t, "DELETE", "/api/v2/remotes/srv-rpm", "") + + if resp, b := put(t, "/api/v2/remotes/srv-rpm/files/e2e-testpkg-1.0-1.noarch.rpm", rpm); resp.StatusCode != 201 { + t.Fatalf("upload rpm: %d %s", resp.StatusCode, b) + } + + // repodata is generated asynchronously; poll for it. + var body []byte + for i := 0; i < 40; i++ { + var resp *http.Response + resp, body = req(t, "GET", "/api/v1/local/srv-rpm/repodata/repomd.xml", "") + if resp.StatusCode == 200 && strings.Contains(string(body), "= 400 { + t.Errorf("evict object: %d", resp.StatusCode) + } +} + +func TestServerValidationErrors(t *testing.T) { + requireStack(t) + if resp, _ := req(t, "POST", "/api/v2/remotes", `{"name":"bad","package_type":"bogus","base_url":"https://x"}`); resp.StatusCode != 400 { + t.Errorf("invalid package type: %d", resp.StatusCode) + } + if resp, _ := req(t, "POST", "/api/v2/remotes", `{"name":"bad","package_type":"generic","repo_type":"remote"}`); resp.StatusCode != 400 { + t.Errorf("missing base_url: %d", resp.StatusCode) + } + if resp, _ := req(t, "POST", "/api/v2/remotes", `not json`); resp.StatusCode != 400 { + t.Errorf("invalid json: %d", resp.StatusCode) + } + // Invalid regex pattern -> 400 from ValidatePatterns. + if resp, _ := req(t, "POST", "/api/v2/remotes", `{"name":"badre","package_type":"generic","repo_type":"remote","base_url":"https://x","blocklist":["[unterminated"]}`); resp.StatusCode != 400 { + t.Errorf("invalid regex: %d", resp.StatusCode) + } +} + +func TestServerDockerAndHead(t *testing.T) { + requireStack(t) + create := fmt.Sprintf(`{"name":"srv-docker","package_type":"generic","repo_type":"remote","base_url":%q,"stale_on_error":true}`, upstream.URL) + req(t, "POST", "/api/v2/remotes", create) + defer req(t, "DELETE", "/api/v2/remotes/srv-docker", "") + + // Docker registry ping. + if resp, _ := req(t, "GET", "/v2/", ""); resp.StatusCode != 200 { + t.Errorf("docker ping: %d", resp.StatusCode) + } + // HEAD through the docker route resolves metadata (uncached -> upstream). + rq, _ := http.NewRequest("HEAD", testTS.URL+"/v2/srv-docker/data/file.bin", nil) + resp, err := http.DefaultClient.Do(rq) + if err != nil { + t.Fatalf("head: %v", err) + } + resp.Body.Close() + if resp.StatusCode != 200 { + t.Errorf("head status: %d", resp.StatusCode) + } +} + +func TestServerRemoteUpdateAndVirtualCRUD(t *testing.T) { + requireStack(t) + req(t, "POST", "/api/v2/remotes", `{"name":"srv-upd","package_type":"helm","repo_type":"remote","base_url":"https://a.example.com","stale_on_error":true}`) + defer req(t, "DELETE", "/api/v2/remotes/srv-upd", "") + if resp, b := req(t, "PUT", "/api/v2/remotes/srv-upd", `{"package_type":"helm","base_url":"https://b.example.com","stale_on_error":true}`); resp.StatusCode != 200 { + t.Errorf("update remote: %d %s", resp.StatusCode, b) + } + + req(t, "POST", "/api/v2/virtuals", `{"name":"srv-v2","package_type":"helm","members":["srv-upd"]}`) + defer req(t, "DELETE", "/api/v2/virtuals/srv-v2", "") + if resp, _ := req(t, "GET", "/api/v2/virtuals/srv-v2", ""); resp.StatusCode != 200 { + t.Errorf("get virtual: %d", resp.StatusCode) + } + if resp, _ := req(t, "GET", "/api/v2/virtuals", ""); resp.StatusCode != 200 { + t.Errorf("list virtuals: %d", resp.StatusCode) + } + if resp, b := req(t, "PUT", "/api/v2/virtuals/srv-v2", `{"package_type":"helm","members":["srv-upd"]}`); resp.StatusCode != 200 { + t.Errorf("update virtual: %d %s", resp.StatusCode, b) + } +} + +func TestServerLocalRemoveAndMissing(t *testing.T) { + requireStack(t) + req(t, "POST", "/api/v2/remotes", `{"name":"srv-rm","package_type":"generic","repo_type":"local"}`) + defer req(t, "DELETE", "/api/v2/remotes/srv-rm", "") + + put(t, "/api/v2/remotes/srv-rm/files/a/b.bin", []byte("payload")) + if resp, _ := req(t, "DELETE", "/api/v2/remotes/srv-rm/files/a/b.bin", ""); resp.StatusCode >= 400 { + t.Errorf("delete local file: %d", resp.StatusCode) + } + if resp, _ := req(t, "GET", "/api/v1/local/srv-rm/a/b.bin", ""); resp.StatusCode != 404 { + t.Errorf("expected 404 for removed file, got %d", resp.StatusCode) + } +} + +func TestServerLocalUploadErrors(t *testing.T) { + requireStack(t) + // Uploading to a remote-type repo is rejected. + create := fmt.Sprintf(`{"name":"srv-uerr","package_type":"generic","repo_type":"remote","base_url":%q,"stale_on_error":true}`, upstream.URL) + req(t, "POST", "/api/v2/remotes", create) + defer req(t, "DELETE", "/api/v2/remotes/srv-uerr", "") + if resp, _ := put(t, "/api/v2/remotes/srv-uerr/files/x.bin", []byte("x")); resp.StatusCode != 400 { + t.Errorf("upload to remote repo should be 400, got %d", resp.StatusCode) + } + + // Duplicate generic upload is a conflict. + req(t, "POST", "/api/v2/remotes", `{"name":"srv-dup","package_type":"generic","repo_type":"local"}`) + defer req(t, "DELETE", "/api/v2/remotes/srv-dup", "") + put(t, "/api/v2/remotes/srv-dup/files/dup.bin", []byte("one")) + if resp, _ := put(t, "/api/v2/remotes/srv-dup/files/dup.bin", []byte("two")); resp.StatusCode != 409 { + t.Errorf("duplicate upload should be 409, got %d", resp.StatusCode) + } + + // Download of a missing local file is 404. + if resp, _ := req(t, "GET", "/api/v1/local/srv-dup/does/not/exist", ""); resp.StatusCode != 404 { + t.Errorf("missing local download should be 404, got %d", resp.StatusCode) + } + // Unknown virtual is 404. + if resp, _ := req(t, "GET", "/api/v1/virtual/nope/index.yaml", ""); resp.StatusCode != 404 { + t.Errorf("unknown virtual should be 404, got %d", resp.StatusCode) + } +} + +func TestServerEvents(t *testing.T) { + requireStack(t) + client := &http.Client{Timeout: 800 * time.Millisecond} + resp, err := client.Get(testTS.URL + "/api/v2/events") + if err == nil { + resp.Body.Close() + if resp.StatusCode != 200 { + t.Errorf("events status: %d", resp.StatusCode) + } + } + // A timeout is expected for a streaming endpoint; the handler still ran. +} + +func TestRunOnListener(t *testing.T) { + requireStack(t) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { errc <- testSrv.RunOnListener(ctx, ln) }() + + base := "http://" + ln.Addr().String() + ok := false + for i := 0; i < 50; i++ { + if resp, e := http.Get(base + "/health"); e == nil { + resp.Body.Close() + ok = resp.StatusCode == 200 + break + } + time.Sleep(20 * time.Millisecond) + } + if !ok { + t.Error("server did not serve /health") + } + cancel() + select { + case err := <-errc: + if err != nil { + t.Errorf("RunOnListener returned error: %v", err) + } + case <-time.After(12 * time.Second): + t.Fatal("RunOnListener did not shut down") + } +} + +func TestRun(t *testing.T) { + requireStack(t) + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { errc <- testSrv.Run(ctx) }() + time.Sleep(300 * time.Millisecond) // let it bind and start serving + cancel() + select { + case err := <-errc: + if err != nil { + t.Errorf("Run returned error: %v", err) + } + case <-time.After(12 * time.Second): + t.Fatal("Run did not shut down") + } +} + +func TestServerVirtualUnreachableMembers(t *testing.T) { + requireStack(t) + // A virtual whose only member does not exist -> no members reachable. + req(t, "POST", "/api/v2/virtuals", `{"name":"srv-vbad","package_type":"helm","members":["nonexistent-member"]}`) + defer req(t, "DELETE", "/api/v2/virtuals/srv-vbad", "") + if resp, _ := req(t, "GET", "/api/v1/virtual/srv-vbad/index.yaml", ""); resp.StatusCode != 502 { + t.Errorf("virtual with dead members = %d, want 502", resp.StatusCode) + } +} + +func TestServerVirtualLocalPyPIMerge(t *testing.T) { + requireStack(t) + for _, n := range []string{"a", "b"} { + req(t, "POST", "/api/v2/remotes", `{"name":"srv-pm-`+n+`","package_type":"pypi","repo_type":"local"}`) + defer req(t, "DELETE", "/api/v2/remotes/srv-pm-"+n, "") + } + put(t, "/api/v2/remotes/srv-pm-a/files/foo-1.0-py3-none-any.whl", []byte("foo")) + put(t, "/api/v2/remotes/srv-pm-b/files/bar-2.0-py3-none-any.whl", []byte("bar")) + req(t, "POST", "/api/v2/virtuals", `{"name":"srv-pmv","package_type":"pypi","members":["srv-pm-a","srv-pm-b"]}`) + defer req(t, "DELETE", "/api/v2/virtuals/srv-pmv", "") + + resp, b := req(t, "GET", "/api/v1/virtual/srv-pmv/simple/", "") + if resp.StatusCode != 200 { + t.Fatalf("virtual pypi index: %d %s", resp.StatusCode, b) + } + if s := string(b); !strings.Contains(s, "foo") || !strings.Contains(s, "bar") { + t.Errorf("merged local pypi index missing packages: %s", s) + } +} + +func TestServerProxyErrors(t *testing.T) { + requireStack(t) + // Blocklisted path -> 403 propagated through handleProxy. + block := fmt.Sprintf(`{"name":"srv-block","package_type":"generic","repo_type":"remote","base_url":%q,"blocklist":["\\.secret$"],"stale_on_error":true}`, upstream.URL) + req(t, "POST", "/api/v2/remotes", block) + defer req(t, "DELETE", "/api/v2/remotes/srv-block", "") + if resp, _ := req(t, "GET", "/api/v1/remote/srv-block/x.secret", ""); resp.StatusCode != 403 { + t.Errorf("blocklisted GET = %d, want 403", resp.StatusCode) + } + rq, _ := http.NewRequest("HEAD", testTS.URL+"/v2/srv-block/x.secret", nil) + if resp, err := http.DefaultClient.Do(rq); err == nil { + resp.Body.Close() + if resp.StatusCode != 403 { + t.Errorf("blocklisted HEAD = %d, want 403", resp.StatusCode) + } + } + + // Unreachable upstream, no stale copy -> 502 bad gateway. + dead := `{"name":"srv-dead","package_type":"generic","repo_type":"remote","base_url":"http://127.0.0.1:1","stale_on_error":false}` + req(t, "POST", "/api/v2/remotes", dead) + defer req(t, "DELETE", "/api/v2/remotes/srv-dead", "") + if resp, _ := req(t, "GET", "/api/v1/remote/srv-dead/x", ""); resp.StatusCode != 502 { + t.Errorf("dead upstream GET = %d, want 502", resp.StatusCode) + } +} + +func TestServerLocalMissingBlob(t *testing.T) { + requireStack(t) + req(t, "POST", "/api/v2/remotes", `{"name":"srv-ghost","package_type":"generic","repo_type":"local"}`) + defer req(t, "DELETE", "/api/v2/remotes/srv-ghost", "") + + ctx := context.Background() + // A local file whose blob object is absent from the store. + testSrv.db.UpsertBlob(ctx, "sha256:ghost", "blobs/sha256/ghost-missing", 5, "text/plain") + if err := testSrv.db.CreateLocalFile(ctx, "srv-ghost", "ghost.bin", "sha256:ghost"); err != nil { + t.Fatalf("create local file: %v", err) + } + + if resp, _ := req(t, "GET", "/api/v1/local/srv-ghost/ghost.bin", ""); resp.StatusCode != 500 { + t.Errorf("v1 download missing blob = %d, want 500", resp.StatusCode) + } + if resp, _ := req(t, "GET", "/api/v2/remotes/srv-ghost/files/ghost.bin", ""); resp.StatusCode != 500 { + t.Errorf("v2 download missing blob = %d, want 500", resp.StatusCode) + } +} + +func TestServerBogusProviderType(t *testing.T) { + requireStack(t) + // Insert a remote with an unregistered package type directly, bypassing + // validation, to exercise the provider-not-found branches. + _, err := testSrv.db.Pool.Exec(context.Background(), + `INSERT INTO remotes (name, package_type, repo_type, base_url) VALUES ($1,'bogus','remote','https://x')`, "srv-bogus") + if err != nil { + t.Fatalf("insert bogus remote: %v", err) + } + defer testSrv.db.Pool.Exec(context.Background(), `DELETE FROM remotes WHERE name='srv-bogus'`) + + if resp, _ := req(t, "GET", "/api/v1/remote/srv-bogus/x", ""); resp.StatusCode != 500 { + t.Errorf("bogus provider GET = %d, want 500", resp.StatusCode) + } + rq, _ := http.NewRequest("HEAD", testTS.URL+"/v2/srv-bogus/x", nil) + if resp, err := http.DefaultClient.Do(rq); err == nil { + resp.Body.Close() + if resp.StatusCode != 500 { + t.Errorf("bogus provider HEAD = %d, want 500", resp.StatusCode) + } + } + if resp, b := req(t, "POST", "/api/v2/probe", `{"remote":"srv-bogus","path":"x"}`); resp.StatusCode != 200 || !strings.Contains(string(b), `"status":500`) { + t.Errorf("bogus provider probe: %d %s", resp.StatusCode, b) + } +} + +func TestServerNotFound(t *testing.T) { + requireStack(t) + if resp, _ := req(t, "GET", "/api/v2/remotes/does-not-exist", ""); resp.StatusCode != 404 { + t.Errorf("expected 404, got %d", resp.StatusCode) + } + if resp, _ := req(t, "GET", "/api/v1/remote/nope/x", ""); resp.StatusCode != 404 { + t.Errorf("expected 404 for unknown remote, got %d", resp.StatusCode) + } + // Unknown local repo -> 404 in handleLocal. + if resp, _ := req(t, "GET", "/api/v1/local/nope/x", ""); resp.StatusCode != 404 { + t.Errorf("expected 404 for unknown local repo, got %d", resp.StatusCode) + } +} diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go new file mode 100644 index 0000000..0379900 --- /dev/null +++ b/internal/storage/storage_test.go @@ -0,0 +1,160 @@ +package storage + +import ( + "bytes" + "context" + "io" + "os" + "strings" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/testsupport" +) + +var ( + testS3 *S3 + testEndpoint string +) + +func TestMain(m *testing.M) { + ctx := context.Background() + conn, terminate, err := testsupport.StartMinio(ctx) + if err != nil { + os.Exit(m.Run()) + } + var s3 *S3 + for i := 0; i < 20; i++ { // MinIO can report ready before bucket ops succeed + if s3, err = NewS3(conn.Endpoint, conn.AccessKey, conn.SecretKey, "test-bucket", false, ""); err == nil { + break + } + time.Sleep(500 * time.Millisecond) + } + if err != nil { + terminate() + panic(err) + } + testS3 = s3 + testEndpoint = conn.Endpoint + code := m.Run() + terminate() + if code != 0 { + os.Exit(code) + } +} + +func requireS3(t *testing.T) { + t.Helper() + if testS3 == nil { + t.Skip("Docker unavailable; skipping storage integration test") + } +} + +func TestKeys(t *testing.T) { + if BlobKey("abc") != "blobs/sha256/abc" { + t.Error("BlobKey") + } + if IndexKey("remote", "path/to/x") != "indexes/remote/path/to/x" { + t.Error("IndexKey") + } +} + +func TestS3RoundTrip(t *testing.T) { + requireS3(t) + ctx := context.Background() + key := "blobs/sha256/test1" + content := []byte("hello storage") + + if err := testS3.Upload(ctx, key, bytes.NewReader(content), int64(len(content)), "text/plain"); err != nil { + t.Fatalf("upload: %v", err) + } + + exists, err := testS3.Exists(ctx, key) + if err != nil || !exists { + t.Fatalf("exists after upload: %v %v", exists, err) + } + + reader, info, err := testS3.Download(ctx, key) + if err != nil { + t.Fatalf("download: %v", err) + } + got, _ := io.ReadAll(reader) + reader.Close() + if !bytes.Equal(got, content) { + t.Errorf("content mismatch: %q", got) + } + if info.Size != int64(len(content)) || info.ContentType != "text/plain" { + t.Errorf("stat info wrong: size=%d ct=%s", info.Size, info.ContentType) + } + + if _, err := testS3.Stat(ctx, key); err != nil { + t.Errorf("stat: %v", err) + } + + if err := testS3.Delete(ctx, key); err != nil { + t.Fatalf("delete: %v", err) + } + if exists, _ := testS3.Exists(ctx, key); exists { + t.Error("expected object gone after delete") + } +} + +func TestNewS3ExistingBucket(t *testing.T) { + requireS3(t) + // The bucket already exists from TestMain, so ensureBucket takes the + // "already present" path. + if _, err := NewS3(testEndpoint, "minioadmin", "minioadmin", "test-bucket", false, ""); err != nil { + t.Fatalf("second NewS3: %v", err) + } +} + +func TestS3DownloadMissing(t *testing.T) { + requireS3(t) + if _, _, err := testS3.Download(context.Background(), "does/not/exist"); err == nil { + t.Error("expected error downloading missing key") + } + if _, err := testS3.Stat(context.Background(), "does/not/exist"); err == nil { + t.Error("expected error stat-ing missing key") + } + if exists, err := testS3.Exists(context.Background(), "does/not/exist"); err != nil || exists { + t.Errorf("Exists(missing) = %v, %v; want false, nil", exists, err) + } +} + +func TestCASStore(t *testing.T) { + requireS3(t) + ctx := context.Background() + cas := NewCAS(testS3) + content := "content-addressed payload" + + res, err := cas.Store(ctx, strings.NewReader(content), "text/plain") + if err != nil { + t.Fatalf("store: %v", err) + } + if res.AlreadyExists { + t.Error("first store should not report AlreadyExists") + } + if res.SizeBytes != int64(len(content)) || !strings.HasPrefix(res.ContentHash, "sha256:") { + t.Errorf("unexpected result: %+v", res) + } + + // Storing identical content again is deduplicated. + res2, err := cas.Store(ctx, strings.NewReader(content), "text/plain") + if err != nil { + t.Fatalf("store again: %v", err) + } + if !res2.AlreadyExists || res2.ContentHash != res.ContentHash { + t.Errorf("second store should dedup: %+v", res2) + } + + // The stored blob is retrievable. + reader, _, err := testS3.Download(ctx, res.S3Key) + if err != nil { + t.Fatalf("download stored blob: %v", err) + } + got, _ := io.ReadAll(reader) + reader.Close() + if string(got) != content { + t.Errorf("stored content mismatch: %q", got) + } +} diff --git a/internal/testsupport/containers.go b/internal/testsupport/containers.go new file mode 100644 index 0000000..0727571 --- /dev/null +++ b/internal/testsupport/containers.go @@ -0,0 +1,101 @@ +// Package testsupport starts throwaway backing containers (Postgres, Redis, +// MinIO) for integration-style unit tests. It is only ever imported from +// *_test.go files, so it never reaches the production binary. Each Start* +// function returns a connection detail plus a terminate func; callers wire +// them up in a TestMain and skip the package's tests when Docker is absent. +package testsupport + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + tcredis "github.com/testcontainers/testcontainers-go/modules/redis" + "github.com/testcontainers/testcontainers-go/wait" +) + +func init() { + // The Ryuk reaper container cannot start in this environment; each Start* + // returns an explicit terminate func for cleanup instead. + if _, ok := os.LookupEnv("TESTCONTAINERS_RYUK_DISABLED"); !ok { + os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true") + } +} + +// StartPostgres launches postgres:17-alpine and returns its DSN. +func StartPostgres(ctx context.Context) (dsn string, terminate func(), err error) { + c, err := tcpostgres.Run(ctx, + "postgres:17-alpine", + tcpostgres.WithDatabase("artifacts"), + tcpostgres.WithUsername("artifacts"), + tcpostgres.WithPassword("artifacts123"), + testcontainers.WithWaitStrategy( + // Postgres opens the port, runs init scripts, then restarts, so wait + // for the readiness log to appear twice to avoid connection resets. + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(60*time.Second), + ), + ) + if err != nil { + return "", nil, err + } + host, _ := c.Host(ctx) + port, _ := c.MappedPort(ctx, "5432/tcp") + dsn = fmt.Sprintf("postgres://artifacts:artifacts123@%s:%s/artifacts?sslmode=disable", host, port.Port()) + return dsn, func() { _ = c.Terminate(ctx) }, nil +} + +// StartRedis launches redis:7-alpine and returns its URL. +func StartRedis(ctx context.Context) (url string, terminate func(), err error) { + c, err := tcredis.Run(ctx, + "redis:7-alpine", + testcontainers.WithWaitStrategy( + wait.ForListeningPort("6379/tcp").WithStartupTimeout(60*time.Second), + ), + ) + if err != nil { + return "", nil, err + } + host, _ := c.Host(ctx) + port, _ := c.MappedPort(ctx, "6379/tcp") + url = fmt.Sprintf("redis://%s:%s", host, port.Port()) + return url, func() { _ = c.Terminate(ctx) }, nil +} + +// MinioConn holds MinIO connection details. +type MinioConn struct { + Endpoint string + AccessKey string + SecretKey string +} + +// StartMinio launches minio and returns its connection details. +func StartMinio(ctx context.Context) (conn MinioConn, terminate func(), err error) { + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "minio/minio:latest", + ExposedPorts: []string{"9000/tcp"}, + Cmd: []string{"server", "/data"}, + Env: map[string]string{ + "MINIO_ROOT_USER": "minioadmin", + "MINIO_ROOT_PASSWORD": "minioadmin", + }, + WaitingFor: wait.ForHTTP("/minio/health/ready").WithPort("9000/tcp").WithStartupTimeout(60 * time.Second), + }, + Started: true, + }) + if err != nil { + return MinioConn{}, nil, err + } + host, _ := c.Host(ctx) + port, _ := c.MappedPort(ctx, "9000/tcp") + return MinioConn{ + Endpoint: fmt.Sprintf("%s:%s", host, port.Port()), + AccessKey: "minioadmin", + SecretKey: "minioadmin", + }, func() { _ = c.Terminate(ctx) }, nil +} diff --git a/internal/testsupport/rpm.go b/internal/testsupport/rpm.go new file mode 100644 index 0000000..49f37bb --- /dev/null +++ b/internal/testsupport/rpm.go @@ -0,0 +1,59 @@ +package testsupport + +import ( + "bytes" + "encoding/binary" +) + +// MinimalRPM builds a valid-enough RPM package in pure Go (no committed binary +// fixture, no external rpmbuild). It carries just the header tags the provider +// reads: name/version/release/arch plus a single self Provides entry, which is +// enough for cavaliergopher/rpm to parse and for repodata generation. +func MinimalRPM(name, version, release, arch string) []byte { + type tag struct { + id, typ, count uint32 + data []byte + } + cstr := func(s string) []byte { return append([]byte(s), 0) } + tags := []tag{ + {1000, 6, 1, cstr(name)}, // RPMTAG_NAME (STRING) + {1001, 6, 1, cstr(version)}, // RPMTAG_VERSION + {1002, 6, 1, cstr(release)}, // RPMTAG_RELEASE + {1022, 6, 1, cstr(arch)}, // RPMTAG_ARCH + {1047, 8, 1, cstr(name)}, // RPMTAG_PROVIDENAME (STRING_ARRAY) + {1112, 4, 1, []byte{0, 0, 0, 0}}, // RPMTAG_PROVIDEFLAGS (INT32) + {1113, 8, 1, cstr(version)}, // RPMTAG_PROVIDEVERSION (STRING_ARRAY) + } + + buildHeader := func(entries []tag) []byte { + var index, store bytes.Buffer + for _, e := range entries { + off := uint32(store.Len()) + for _, v := range []uint32{e.id, e.typ, off, e.count} { + binary.Write(&index, binary.BigEndian, v) + } + store.Write(e.data) + } + var b bytes.Buffer + b.Write([]byte{0x8e, 0xad, 0xe8, 0x01, 0, 0, 0, 0}) // header magic + reserved + binary.Write(&b, binary.BigEndian, uint32(len(entries))) + binary.Write(&b, binary.BigEndian, uint32(store.Len())) + b.Write(index.Bytes()) + b.Write(store.Bytes()) + return b.Bytes() + } + + lead := make([]byte, 96) + copy(lead[0:4], []byte{0xed, 0xab, 0xee, 0xdb}) // lead magic + lead[4] = 3 // major version + binary.BigEndian.PutUint16(lead[8:10], 1) // archnum + copy(lead[10:76], name) // name (66 bytes, null-padded) + binary.BigEndian.PutUint16(lead[76:78], 1) // osnum + binary.BigEndian.PutUint16(lead[78:80], 5) // signature type + + var out bytes.Buffer + out.Write(lead) + out.Write(buildHeader(nil)) // empty signature header (16 bytes, 8-aligned) + out.Write(buildHeader(tags)) + return out.Bytes() +} diff --git a/internal/virtual/merger_test.go b/internal/virtual/merger_test.go new file mode 100644 index 0000000..2e117fe --- /dev/null +++ b/internal/virtual/merger_test.go @@ -0,0 +1,155 @@ +package virtual + +import ( + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestRegisterGetMerger(t *testing.T) { + if _, err := GetMerger(models.PackageHelm); err != nil { + t.Errorf("helm merger should be registered: %v", err) + } + if _, err := GetMerger(models.PackagePyPI); err != nil { + t.Errorf("pypi merger should be registered: %v", err) + } + if _, err := GetMerger(models.PackageType("nope")); err == nil { + t.Error("expected error for unknown merger") + } +} + +func TestPyPIMerge(t *testing.T) { + m := &PyPIMerger{} + members := []MemberIndex{ + {RemoteName: "a", RepoType: models.RepoTypeRemote, Body: []byte(`foo-1.0.whl`)}, + {RemoteName: "b", RepoType: models.RepoTypeLocal, Body: []byte(`bar-2.0.whl`)}, + } + out, err := m.MergeIndexes(members, "http://proxy") + if err != nil { + t.Fatal(err) + } + s := string(out) + if !strings.Contains(s, "foo-1.0.whl") || !strings.Contains(s, "bar-2.0.whl") { + t.Errorf("merged index missing entries: %s", s) + } + if !strings.Contains(s, "http://proxy/api/v1/remote/a/pkg/foo-1.0.whl") { + t.Errorf("remote href not rewritten: %s", s) + } + if !strings.Contains(s, "http://proxy/api/v1/local/b/bar-2.0.whl") { + t.Errorf("local href not rewritten: %s", s) + } + + // Sorted output: foo before... entries sorted by link text. + if strings.Index(s, "bar-2.0.whl") > strings.Index(s, "foo-1.0.whl") { + t.Error("entries should be sorted by text") + } + + // Duplicate link texts across members are de-duplicated. + dup := []MemberIndex{ + {RemoteName: "a", Body: []byte(`dup`)}, + {RemoteName: "b", Body: []byte(`dup`)}, + } + out, _ = m.MergeIndexes(dup, "") + if strings.Count(string(out), ">dup") != 1 { + t.Errorf("duplicate not de-duplicated: %s", out) + } +} + +func TestPyPIMergeNoProxyAndBadLinks(t *testing.T) { + m := &PyPIMerger{} + members := []MemberIndex{{ + RemoteName: "a", + Body: []byte("foo.whl\nno href\nnot a link"), + }} + // No proxy base URL: hrefs are left as-is. + out, err := m.MergeIndexes(members, "") + if err != nil { + t.Fatal(err) + } + s := string(out) + if !strings.Contains(s, ">foo.whl") { + t.Errorf("missing link: %s", s) + } +} + +func TestHelmMerge(t *testing.T) { + m := &HelmMerger{} + memberA := `apiVersion: v1 +entries: + alpha: + - name: alpha + version: 1.0.0 + urls: + - charts/alpha-1.0.0.tgz +` + memberB := `apiVersion: v1 +entries: + beta: + - name: beta + version: 2.0.0 + urls: + - https://charts.example.com/beta-2.0.0.tgz + gamma: + - name: gamma + version: 3.0.0 + urls: + - https://other-host.example.net/gamma-3.0.0.tgz +` + members := []MemberIndex{ + {RemoteName: "a", RepoType: models.RepoTypeLocal, BaseURL: "https://charts.example.com", Body: []byte(memberA)}, + {RemoteName: "b", RepoType: models.RepoTypeRemote, BaseURL: "https://charts.example.com", Body: []byte(memberB)}, + } + out, err := m.MergeIndexes(members, "http://proxy") + if err != nil { + t.Fatal(err) + } + s := string(out) + for _, chart := range []string{"alpha", "beta", "gamma"} { + if !strings.Contains(s, chart) { + t.Errorf("merged index missing chart %q: %s", chart, s) + } + } + // Relative URL from a local member is rewritten under /local/. + if !strings.Contains(s, "http://proxy/api/v1/local/a/charts/alpha-1.0.0.tgz") { + t.Errorf("relative local url not rewritten: %s", s) + } + // Same-host absolute URL from a remote member is rewritten under /remote/. + if !strings.Contains(s, "http://proxy/api/v1/remote/b/beta-2.0.0.tgz") { + t.Errorf("same-host absolute url not rewritten: %s", s) + } + // Cross-host absolute URL is left untouched. + if !strings.Contains(s, "https://other-host.example.net/gamma-3.0.0.tgz") { + t.Errorf("cross-host url should be preserved: %s", s) + } +} + +func TestHelmMergeDedup(t *testing.T) { + m := &HelmMerger{} + body := `apiVersion: v1 +entries: + alpha: + - name: alpha + version: 1.0.0 + urls: [charts/alpha-1.0.0.tgz] +` + members := []MemberIndex{ + {RemoteName: "a", BaseURL: "https://x", Body: []byte(body)}, + {RemoteName: "b", BaseURL: "https://x", Body: []byte(body)}, + } + out, _ := m.MergeIndexes(members, "") + if strings.Count(string(out), "version: 1.0.0") != 1 { + t.Errorf("duplicate chart version not de-duplicated: %s", out) + } +} + +func TestHelmMergeInvalidYAML(t *testing.T) { + m := &HelmMerger{} + out, err := m.MergeIndexes([]MemberIndex{{RemoteName: "a", Body: []byte("::: not yaml :::")}}, "") + if err != nil { + t.Fatalf("invalid member yaml should be skipped, not error: %v", err) + } + if !strings.Contains(string(out), "apiVersion") { + t.Errorf("expected a valid empty merged index: %s", out) + } +} diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go new file mode 100644 index 0000000..62f9fd5 --- /dev/null +++ b/pkg/client/client_test.go @@ -0,0 +1,145 @@ +package client + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func testServer(t *testing.T, h http.HandlerFunc) *Client { + t.Helper() + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + return New(srv.URL) +} + +func TestRemotesRoundTrip(t *testing.T) { + var gotMethod, gotPath string + c := testServer(t, func(w http.ResponseWriter, r *http.Request) { + gotMethod, gotPath = r.Method, r.URL.Path + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/remotes": + w.Write([]byte(`[{"name":"a"},{"name":"b"}]`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/remotes/a": + w.Write([]byte(`{"name":"a"}`)) + case r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{"name":"a"}`)) + } + }) + ctx := context.Background() + + remotes, err := c.ListRemotes(ctx) + if err != nil || len(remotes) != 2 { + t.Fatalf("ListRemotes: %v %v", remotes, err) + } + if r, err := c.GetRemote(ctx, "a"); err != nil || r.Name != "a" { + t.Fatalf("GetRemote: %v %v", r, err) + } + if err := c.CreateRemote(ctx, &models.Remote{Name: "a", PackageType: models.PackageGeneric}); err != nil { + t.Fatalf("CreateRemote: %v", err) + } + if err := c.UpdateRemote(ctx, &models.Remote{Name: "a"}); err != nil { + t.Fatalf("UpdateRemote: %v", err) + } + if err := c.DeleteRemote(ctx, "a"); err != nil { + t.Fatalf("DeleteRemote: %v", err) + } + if gotMethod != http.MethodDelete || gotPath != "/api/v2/remotes/a" { + t.Errorf("last call = %s %s", gotMethod, gotPath) + } +} + +func TestVirtualsRoundTrip(t *testing.T) { + c := testServer(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/virtuals"): + w.Write([]byte(`[{"name":"v"}]`)) + case r.Method == http.MethodGet: + w.Write([]byte(`{"name":"v"}`)) + case r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{"name":"v"}`)) + } + }) + ctx := context.Background() + if vs, err := c.ListVirtuals(ctx); err != nil || len(vs) != 1 { + t.Fatalf("ListVirtuals: %v %v", vs, err) + } + if v, err := c.GetVirtual(ctx, "v"); err != nil || v.Name != "v" { + t.Fatalf("GetVirtual: %v %v", v, err) + } + if err := c.CreateVirtual(ctx, &models.Virtual{Name: "v"}); err != nil { + t.Fatalf("CreateVirtual: %v", err) + } + if err := c.UpdateVirtual(ctx, &models.Virtual{Name: "v"}); err != nil { + t.Fatalf("UpdateVirtual: %v", err) + } + if err := c.DeleteVirtual(ctx, "v"); err != nil { + t.Fatalf("DeleteVirtual: %v", err) + } +} + +func TestStatsHealthObjects(t *testing.T) { + c := testServer(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/stats"): + w.Write([]byte(`{"total_remotes":3}`)) + case strings.HasSuffix(r.URL.Path, "/health"): + w.Write([]byte(`{"status":"ok"}`)) + case r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + w.Write([]byte(`[{"path":"p"}]`)) + } + }) + ctx := context.Background() + if _, err := c.Stats(ctx); err != nil { + t.Fatalf("Stats: %v", err) + } + if _, err := c.Health(ctx); err != nil { + t.Fatalf("Health: %v", err) + } + if objs, err := c.ListObjects(ctx, "r", 1, 50); err != nil || len(objs) != 1 { + t.Fatalf("ListObjects: %v %v", objs, err) + } + if err := c.EvictObject(ctx, "r", "some/path"); err != nil { + t.Fatalf("EvictObject: %v", err) + } +} + +func TestErrorResponses(t *testing.T) { + c := testServer(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusInternalServerError) + }) + ctx := context.Background() + _, err := c.GetRemote(ctx, "x") + if err == nil || !strings.Contains(err.Error(), "api error 500") { + t.Errorf("expected api error, got %v", err) + } +} + +func TestDecodeError(t *testing.T) { + c := testServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`not json`)) + }) + if _, err := c.ListRemotes(context.Background()); err == nil || !strings.Contains(err.Error(), "decode") { + t.Errorf("expected decode error, got %v", err) + } +} + +func TestRequestError(t *testing.T) { + // Invalid base URL triggers request construction failure. + c := New("http://[::1]:namedport") + if err := c.DeleteRemote(context.Background(), "x"); err == nil { + t.Error("expected request error for invalid URL") + } +} diff --git a/pkg/models/repotype_test.go b/pkg/models/repotype_test.go new file mode 100644 index 0000000..bedcc52 --- /dev/null +++ b/pkg/models/repotype_test.go @@ -0,0 +1,18 @@ +package models + +import "testing" + +func TestRepoType(t *testing.T) { + if RepoTypeRemote.String() != "remote" || RepoTypeLocal.String() != "local" { + t.Error("RepoType.String") + } + if !RepoTypeRemote.Valid() || RepoType("bogus").Valid() { + t.Error("RepoType.Valid") + } + if rt, err := ParseRepoType("local"); err != nil || rt != RepoTypeLocal { + t.Errorf("ParseRepoType(local) = %v %v", rt, err) + } + if _, err := ParseRepoType("nope"); err == nil { + t.Error("ParseRepoType should reject unknown") + } +}