artifactapi/internal/provider/deb/github_test.go

package deb

import (
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"git.unkin.net/unkin/artifactapi/internal/provider"
	"git.unkin.net/unkin/artifactapi/internal/testsupport"
	"git.unkin.net/unkin/artifactapi/pkg/models"
)

// fakeStore is an in-memory provider.RemoteMetadataStore + DebMetadataReader
// keyed by file_path, mirroring the (repo_name, file_path) uniqueness of the
// real deb_metadata table.
type fakeStore struct {
	mu   sync.Mutex
	rows map[string]provider.DebMetadata
}

func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.DebMetadata{}} }

func (f *fakeStore) InsertDebMetadata(_ context.Context, m *provider.DebMetadata) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	if _, ok := f.rows[m.FilePath]; ok {
		return nil // ON CONFLICT DO NOTHING
	}
	f.rows[m.FilePath] = *m
	return nil
}

func (f *fakeStore) DeleteDebMetadata(_ context.Context, _, filePath string) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	delete(f.rows, filePath)
	return nil
}

func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
func (f *fakeStore) DeleteRPMMetadata(context.Context, string, string) error        { return nil }
func (f *fakeStore) ListRPMMetadataEntries(context.Context, string) ([]provider.RPMMetadata, error) {
	return nil, nil
}

func (f *fakeStore) ListDebMetadataEntries(ctx context.Context, _ string) ([]provider.DebMetadata, error) {
	if err := ctx.Err(); err != nil {
		return nil, err
	}
	f.mu.Lock()
	defer f.mu.Unlock()
	out := make([]provider.DebMetadata, 0, len(f.rows))
	for _, m := range f.rows {
		out = append(out, m)
	}
	return out, nil
}

// githubFixture serves the releases API and the .deb asset downloads (with Range
// support) for a set of packages. digest controls whether the asset carries a
// sha256 digest (no-download path) or not (compute path).
type githubFixture struct {
	srv         *httptest.Server
	debBytes    map[string][]byte
	rangeHit    map[string]int
	fullHit     map[string]int
	etag        string
	releasesHit int
	notModHit   int
	releaseAuth string
	assetAuth   string
	mu          sync.Mutex
}

func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
	t.Helper()
	f := &githubFixture{
		debBytes: map[string][]byte{},
		rangeHit: map[string]int{},
		fullHit:  map[string]int{},
	}
	f.debBytes["demo_1.2-3_amd64.deb"] = testsupport.MinimalDeb("demo", "1.2-3", "amd64")

	mux := http.NewServeMux()
	mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) {
		page := r.URL.Query().Get("page")
		if page != "" && page != "1" {
			w.Write([]byte("[]"))
			return
		}
		f.mu.Lock()
		f.releasesHit++
		f.releaseAuth = r.Header.Get("Authorization")
		etag := f.etag
		if etag != "" && r.Header.Get("If-None-Match") == etag {
			f.notModHit++
			f.mu.Unlock()
			w.WriteHeader(http.StatusNotModified)
			return
		}
		f.mu.Unlock()
		if etag != "" {
			w.Header().Set("ETag", etag)
		}
		var assets []map[string]any
		for name := range f.debBytes {
			a := map[string]any{
				"name":                 name,
				"size":                 len(f.debBytes[name]),
				"browser_download_url": f.srv.URL + "/acme/tools/releases/download/v1.2-3/" + name,
			}
			if withDigest {
				sum := sha256.Sum256(f.debBytes[name])
				a["digest"] = "sha256:" + hex.EncodeToString(sum[:])
			}
			assets = append(assets, a)
		}
		rel := []map[string]any{{"tag_name": "v1.2-3", "draft": false, "assets": assets}}
		json.NewEncoder(w).Encode(rel)
	})
	mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) {
		name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
		body, ok := f.debBytes[name]
		if !ok {
			http.Error(w, "not found", 404)
			return
		}
		rng := r.Header.Get("Range")
		f.mu.Lock()
		f.assetAuth = r.Header.Get("Authorization")
		if rng != "" {
			f.rangeHit[name]++
		} else {
			f.fullHit[name]++
		}
		f.mu.Unlock()

		if rng == "" {
			w.WriteHeader(200)
			w.Write(body)
			return
		}
		var end int
		fmt.Sscanf(rng, "bytes=0-%d", &end)
		if end >= len(body)-1 {
			end = len(body) - 1
		}
		w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
		w.Header().Set("Content-Length", strconv.Itoa(end+1))
		w.WriteHeader(http.StatusPartialContent)
		w.Write(body[:end+1])
	})
	f.srv = httptest.NewServer(mux)
	t.Cleanup(f.srv.Close)
	return f
}

func (f *githubFixture) remote() models.Remote {
	return models.Remote{
		Name:           "acme-deb",
		PackageType:    models.PackageGitHubDeb,
		BaseURL:        f.srv.URL + "/repos/acme/tools",
		ReleasesRemote: "github",
		MutableTTL:     3600,
	}
}

func newTestProvider() *GitHubProvider {
	p := newGitHubProvider()
	p.headerInitial = 32 // force the ranged-fetch retry loop against the tiny fixture
	p.headerMax = 1 << 20
	return p
}

const demoPath = "acme/tools/releases/download/v1.2-3/demo_1.2-3_amd64.deb"

func TestGitHubScanDerivesControlFromPrefixAndDigest(t *testing.T) {
	fx := newGitHubFixture(t, true)
	p := newTestProvider()
	store := newFakeStore()

	if err := p.scan(context.Background(), fx.remote(), store); err != nil {
		t.Fatalf("scan: %v", err)
	}

	metas, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
	if len(metas) != 1 {
		t.Fatalf("want 1 metadata row, got %d", len(metas))
	}
	m := metas[0]
	if m.Name != "demo" || m.Version != "1.2-3" || m.Architecture != "amd64" {
		t.Fatalf("bad control fields: %+v", m)
	}
	if m.FilePath != demoPath {
		t.Fatalf("FilePath = %q, want %q", m.FilePath, demoPath)
	}
	if int(m.Size) != len(fx.debBytes["demo_1.2-3_amd64.deb"]) {
		t.Fatalf("Size = %d, want %d", m.Size, len(fx.debBytes["demo_1.2-3_amd64.deb"]))
	}
	sum := sha256.Sum256(fx.debBytes["demo_1.2-3_amd64.deb"])
	if m.SHA256 != hex.EncodeToString(sum[:]) {
		t.Fatalf("SHA256 = %q, want digest", m.SHA256)
	}
	if m.ContentHash != "sha256:"+hex.EncodeToString(sum[:]) {
		t.Fatalf("ContentHash = %q", m.ContentHash)
	}
	if m.MD5 != "" {
		t.Fatalf("MD5 should be unset for metadata-only derive, got %q", m.MD5)
	}
	if fx.fullHit["demo_1.2-3_amd64.deb"] != 0 {
		t.Fatalf("expected no full download when digest present, got %d", fx.fullHit["demo_1.2-3_amd64.deb"])
	}
	if fx.rangeHit["demo_1.2-3_amd64.deb"] == 0 {
		t.Fatalf("expected ranged control fetch")
	}
	if !strings.Contains(m.Control, "Package: demo") {
		t.Fatalf("raw control not captured: %q", m.Control)
	}
}

func TestGitHubChecksumComputedWhenDigestAbsent(t *testing.T) {
	fx := newGitHubFixture(t, false)
	p := newTestProvider()
	store := newFakeStore()

	if err := p.scan(context.Background(), fx.remote(), store); err != nil {
		t.Fatalf("scan: %v", err)
	}
	metas, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
	if len(metas) != 1 {
		t.Fatalf("want 1 row, got %d", len(metas))
	}
	sum := sha256.Sum256(fx.debBytes["demo_1.2-3_amd64.deb"])
	if metas[0].SHA256 != hex.EncodeToString(sum[:]) {
		t.Fatalf("computed checksum mismatch: %q", metas[0].SHA256)
	}
	if fx.fullHit["demo_1.2-3_amd64.deb"] == 0 {
		t.Fatalf("expected a full download to compute sha256 when digest absent")
	}
}

func TestGitHubServeRemoteIndexAndRedirect(t *testing.T) {
	fx := newGitHubFixture(t, true)
	p := newTestProvider()
	store := newFakeStore()
	remote := fx.remote()
	const proxyBase = "https://artifactapi.example"

	// Release is served and triggers the initial scan.
	rec := httptest.NewRecorder()
	req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Release", nil)
	if !p.ServeRemote(rec, req, remote, "Release", proxyBase, store) {
		t.Fatal("ServeRemote did not handle Release")
	}
	if rec.Code != 200 || !strings.Contains(rec.Body.String(), "Architectures:") {
		t.Fatalf("Release bad: code=%d body=%s", rec.Code, rec.Body.String())
	}
	if !strings.Contains(rec.Body.String(), "amd64") {
		t.Fatalf("Release missing arch: %s", rec.Body.String())
	}

	// Packages carries the package with a Filename that is the github-relative
	// download path (so it resolves back to this remote and redirects).
	rec = httptest.NewRecorder()
	req = httptest.NewRequest(http.MethodGet, "/x", nil)
	if !p.ServeRemote(rec, req, remote, "Packages", proxyBase, store) {
		t.Fatal("ServeRemote did not handle Packages")
	}
	pkgs := rec.Body.String()
	if !strings.Contains(pkgs, "Package: demo") {
		t.Fatalf("Packages missing package: %s", pkgs)
	}
	if !strings.Contains(pkgs, "Filename: "+demoPath) {
		t.Fatalf("Packages missing/incorrect Filename: %s", pkgs)
	}
	if !strings.Contains(pkgs, "SHA256: ") {
		t.Fatalf("Packages missing SHA256: %s", pkgs)
	}
	if strings.Contains(pkgs, "MD5sum:") {
		t.Fatalf("Packages should omit empty MD5sum: %s", pkgs)
	}

	// Packages.gz decompresses to the same content.
	rec = httptest.NewRecorder()
	req = httptest.NewRequest(http.MethodGet, "/x", nil)
	if !p.ServeRemote(rec, req, remote, "Packages.gz", proxyBase, store) {
		t.Fatal("ServeRemote did not handle Packages.gz")
	}
	gz, err := gzip.NewReader(rec.Body)
	if err != nil {
		t.Fatalf("gzip: %v", err)
	}
	unz, _ := io.ReadAll(gz)
	if !strings.Contains(string(unz), "Package: demo") {
		t.Fatalf("Packages.gz missing package: %s", unz)
	}

	// InRelease/Release.gpg 404 (unsigned, consumed via [trusted=yes]).
	for _, sp := range []string{"InRelease", "Release.gpg"} {
		rec = httptest.NewRecorder()
		req = httptest.NewRequest(http.MethodGet, "/x", nil)
		if !p.ServeRemote(rec, req, remote, sp, proxyBase, store) {
			t.Fatalf("ServeRemote did not handle %s", sp)
		}
		if rec.Code != http.StatusNotFound {
			t.Fatalf("%s want 404, got %d", sp, rec.Code)
		}
	}

	// A .deb request redirects to the backend releases_remote.
	rec = httptest.NewRecorder()
	req = httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/"+demoPath, nil)
	if !p.ServeRemote(rec, req, remote, demoPath, proxyBase, store) {
		t.Fatal("ServeRemote did not handle .deb")
	}
	if rec.Code != http.StatusFound {
		t.Fatalf("want 302, got %d", rec.Code)
	}
	wantLoc := proxyBase + "/api/v1/remote/github/" + demoPath
	if got := rec.Header().Get("Location"); got != wantLoc {
		t.Fatalf("Location = %q, want %q", got, wantLoc)
	}
}

// Real apt appends the flat-repo dist "./" verbatim, so the metadata-only remote
// receives "./Packages" / "./Release"; ServeRemote must collapse the dot-segment
// and synthesize the same index as the un-prefixed request.
func TestGitHubServeRemoteAptDotSegment(t *testing.T) {
	fx := newGitHubFixture(t, true)
	p := newTestProvider()
	store := newFakeStore()
	remote := fx.remote()
	const proxyBase = "https://artifactapi.example"

	serve := func(path string) *httptest.ResponseRecorder {
		rec := httptest.NewRecorder()
		req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/"+path, nil)
		if !p.ServeRemote(rec, req, remote, path, proxyBase, store) {
			t.Fatalf("ServeRemote did not handle %q", path)
		}
		return rec
	}

	// Packages is deterministic: byte-identical to the un-prefixed request.
	plain, dotted := serve("Packages"), serve("./Packages")
	if plain.Code != 200 || dotted.Code != 200 {
		t.Fatalf("Packages: plain=%d dotted=%d, want 200/200", plain.Code, dotted.Code)
	}
	if !strings.Contains(dotted.Body.String(), "Package: demo") {
		t.Fatalf("./Packages missing synthesized body: %s", dotted.Body.String())
	}
	if !bytes.Equal(plain.Body.Bytes(), dotted.Body.Bytes()) {
		t.Error("./Packages body differs from Packages body")
	}

	// Release carries a time.Now() Date: header; compare the rest.
	rPlain, rDotted := serve("Release"), serve("./Release")
	if rPlain.Code != 200 || rDotted.Code != 200 {
		t.Fatalf("Release: plain=%d dotted=%d, want 200/200", rPlain.Code, rDotted.Code)
	}
	if stripDate(rPlain.Body.String()) != stripDate(rDotted.Body.String()) {
		t.Error("./Release body differs from Release body (ignoring Date)")
	}
}

// A canceled inbound request must still serve the warm cache (detached context),
// not turn the metadata read into a 500.
func TestGitHubServeRemoteCanceledRequestServesCache(t *testing.T) {
	fx := newGitHubFixture(t, true)
	p := newTestProvider()
	store := newFakeStore()
	remote := fx.remote()

	if err := p.scan(context.Background(), remote, store); err != nil {
		t.Fatalf("warm scan: %v", err)
	}
	p.mu.Lock()
	p.lastScan[remote.Name] = time.Now()
	p.mu.Unlock()

	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	rec := httptest.NewRecorder()
	req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Packages", nil).WithContext(ctx)

	if !p.ServeRemote(rec, req, remote, "Packages", "https://x", store) {
		t.Fatal("ServeRemote did not handle Packages")
	}
	if rec.Code != http.StatusOK {
		t.Fatalf("canceled request must serve cache, not error; got code=%d body=%s", rec.Code, rec.Body.String())
	}
	if !strings.Contains(rec.Body.String(), "Package: demo") {
		t.Fatalf("expected Packages served from cache, got %s", rec.Body.String())
	}
}

func TestGitHubServeRemoteRedirectRequiresReleasesRemote(t *testing.T) {
	fx := newGitHubFixture(t, true)
	p := newTestProvider()
	store := newFakeStore()
	remote := fx.remote()
	remote.ReleasesRemote = ""

	rec := httptest.NewRecorder()
	req := httptest.NewRequest(http.MethodGet, "/x", nil)
	if !p.ServeRemote(rec, req, remote, demoPath, "https://x", store) {
		t.Fatal("expected handled")
	}
	if rec.Code != http.StatusInternalServerError {
		t.Fatalf("want 500 when releases_remote unset, got %d", rec.Code)
	}
}

func TestGitHubScanPrunesRemovedAssets(t *testing.T) {
	fx := newGitHubFixture(t, true)
	p := newTestProvider()
	store := newFakeStore()

	if err := p.scan(context.Background(), fx.remote(), store); err != nil {
		t.Fatalf("scan: %v", err)
	}
	if rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb"); len(rows) != 1 {
		t.Fatalf("want 1 row after first scan, got %d", len(rows))
	}

	delete(fx.debBytes, "demo_1.2-3_amd64.deb")
	if err := p.scan(context.Background(), fx.remote(), store); err != nil {
		t.Fatalf("rescan: %v", err)
	}
	if rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb"); len(rows) != 0 {
		t.Fatalf("want 0 rows after prune, got %d", len(rows))
	}
}

func TestGitHubAssetPatternFilter(t *testing.T) {
	fx := newGitHubFixture(t, true)
	fx.debBytes["other_9_arm64.deb"] = testsupport.MinimalDeb("other", "9", "arm64")
	p := newTestProvider()
	store := newFakeStore()
	remote := fx.remote()
	remote.Patterns = []string{`^demo_.*_amd64\.deb$`}

	if err := p.scan(context.Background(), remote, store); err != nil {
		t.Fatalf("scan: %v", err)
	}
	rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
	if len(rows) != 1 || rows[0].Name != "demo" {
		t.Fatalf("pattern filter failed, rows=%+v", rows)
	}
}