diff --git a/.gitignore b/.gitignore index 472957c..3ce9dad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ bin/ -terraform/ +/terraform/ # e2e-docker fixtures are real package files (.rpm, .tgz, .whl, .zip, ...) that # are intentionally tracked, overriding any global ignore of those extensions. diff --git a/README.md b/README.md index 3180214..e662976 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,36 @@ resource "artifactapi_virtual" "helm" { Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi) +### Serving providers as a registry + +A local `terraform` repo is a real provider registry: upload +`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under +`{namespace}/{type}/`, and Terraform installs them from a bare source address — +no `.terraformrc` mirror config: + +```hcl +terraform { + required_providers { + artifactapi = { + source = "artifactapi.k8s.syd1.au.unkin.net//" + version = "0.1.2" + } + } +} +``` + +The Terraform *namespace* segment is the artifactapi repo name; the provider is +matched by *type*. The registry serves service discovery +(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints, +and a GPG-signed `SHA256SUMS` per the provider registry protocol. + +Signing needs a GPG key. By default artifactapi generates one on first start and +stores it in the database (`signing_keys` table), so every replica shares it and +there's nothing to provision. To bring your own key instead, point +`TF_SIGNING_KEY_PATH` at an armored private key (optionally +`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. +`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. + ## Access Control | Field | Default | Behaviour | diff --git a/go.mod b/go.mod index 21d9ea0..b84cfa4 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/testcontainers/testcontainers-go v0.42.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 + golang.org/x/crypto v0.51.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -96,7 +97,6 @@ require ( go.opentelemetry.io/otel/trace v1.41.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.51.0 // indirect golang.org/x/net v0.53.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.44.0 // indirect diff --git a/internal/api/terraform/registry.go b/internal/api/terraform/registry.go new file mode 100644 index 0000000..7aee9de --- /dev/null +++ b/internal/api/terraform/registry.go @@ -0,0 +1,301 @@ +// Package terraform serves local terraform repos as a real Terraform provider +// registry: service discovery, version listing, and GPG-signed downloads, so +// `terraform init` installs from a bare source address with no client config. +package terraform + +import ( + "encoding/json" + "fmt" + "net/http" + "path" + "sort" + "strings" + + "github.com/go-chi/chi/v5" + + "git.unkin.net/unkin/artifactapi/internal/database" + tfprov "git.unkin.net/unkin/artifactapi/internal/provider/terraform" + "git.unkin.net/unkin/artifactapi/internal/tfsign" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// ProvidersV1Path is the base the service-discovery document advertises (Terraform +// appends "{namespace}/{type}/versions" etc). MountPath is the same prefix without +// the trailing slash, for chi.Mount. +const ( + ProvidersV1Path = "/terraform/v1/providers/" + MountPath = "/terraform/v1/providers" +) + +type Handler struct { + db *database.DB + signer *tfsign.Signer + protocols []string +} + +func NewHandler(db *database.DB, signer *tfsign.Signer, protocols string) *Handler { + var protos []string + for _, p := range strings.Split(protocols, ",") { + if p = strings.TrimSpace(p); p != "" { + protos = append(protos, p) + } + } + if len(protos) == 0 { + protos = []string{"5.0", "6.0"} + } + return &Handler{db: db, signer: signer, protocols: protos} +} + +// Enabled reports whether a signing key is configured. Without one the registry +// cannot produce the signed SHA256SUMS the protocol requires, so it stays off. +func (h *Handler) Enabled() bool { return h.signer != nil } + +func (h *Handler) Routes() chi.Router { + r := chi.NewRouter() + r.Get("/{namespace}/{type}/versions", h.versions) + r.Get("/{namespace}/{type}/{version}/download/{os}/{arch}", h.download) + r.Get("/{namespace}/{type}/{version}/sha256sums", h.sha256sums) + r.Get("/{namespace}/{type}/{version}/sha256sums.sig", h.sha256sumsSig) + return r +} + +// ServiceDiscovery answers /.well-known/terraform.json, pointing Terraform at the +// providers.v1 protocol base. +func (h *Handler) ServiceDiscovery(w http.ResponseWriter, r *http.Request) { + if !h.Enabled() { + http.NotFound(w, r) + return + } + writeJSON(w, map[string]string{"providers.v1": ProvidersV1Path}) +} + +// providerFile is one resolved platform artifact within a repo. +type providerFile struct { + version string + os string + arch string + filePath string // path within the repo, e.g. unkin/artifactapi/...zip + sha256 string // hex, no "sha256:" prefix +} + +// resolve finds every provider zip of the given type in the repo (namespace). +// The Terraform source namespace maps to the artifactapi repo name; the provider +// is matched by type across whatever in-repo folder it was uploaded under. +func (h *Handler) resolve(r *http.Request, namespace, typeName string) ([]providerFile, error) { + remote, err := h.db.GetRemote(r.Context(), namespace) + if err != nil || remote.PackageType != models.PackageTerraform { + return nil, nil + } + + rows, err := h.db.ListLocalFiles(r.Context(), namespace, 10000, 0) + if err != nil { + return nil, err + } + + var out []providerFile + for _, row := range rows { + parsed := tfprov.ParseProviderZip(path.Base(row.FilePath)) + if !parsed.Ok || parsed.Type != typeName { + continue + } + out = append(out, providerFile{ + version: parsed.Version, + os: parsed.OS, + arch: parsed.Arch, + filePath: row.FilePath, + sha256: strings.TrimPrefix(row.ContentHash, "sha256:"), + }) + } + return out, nil +} + +func (h *Handler) versions(w http.ResponseWriter, r *http.Request) { + if !h.Enabled() { + http.NotFound(w, r) + return + } + namespace := chi.URLParam(r, "namespace") + typeName := chi.URLParam(r, "type") + + files, err := h.resolve(r, namespace, typeName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if len(files) == 0 { + http.NotFound(w, r) + return + } + + // Group platforms by version, de-duplicated and stably ordered. + type platform struct { + OS string `json:"os"` + Arch string `json:"arch"` + } + platforms := map[string]map[string]platform{} + for _, f := range files { + if platforms[f.version] == nil { + platforms[f.version] = map[string]platform{} + } + platforms[f.version][f.os+"_"+f.arch] = platform{OS: f.os, Arch: f.arch} + } + + type versionEntry struct { + Version string `json:"version"` + Protocols []string `json:"protocols"` + Platforms []platform `json:"platforms"` + } + out := struct { + Versions []versionEntry `json:"versions"` + }{} + for version, plats := range platforms { + entry := versionEntry{Version: version, Protocols: h.protocols} + for _, p := range plats { + entry.Platforms = append(entry.Platforms, p) + } + sort.Slice(entry.Platforms, func(i, j int) bool { + return entry.Platforms[i].OS+entry.Platforms[i].Arch < entry.Platforms[j].OS+entry.Platforms[j].Arch + }) + out.Versions = append(out.Versions, entry) + } + sort.Slice(out.Versions, func(i, j int) bool { return out.Versions[i].Version < out.Versions[j].Version }) + + writeJSON(w, out) +} + +func (h *Handler) download(w http.ResponseWriter, r *http.Request) { + if !h.Enabled() { + http.NotFound(w, r) + return + } + namespace := chi.URLParam(r, "namespace") + typeName := chi.URLParam(r, "type") + version := chi.URLParam(r, "version") + osName := chi.URLParam(r, "os") + arch := chi.URLParam(r, "arch") + + files, err := h.resolve(r, namespace, typeName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var match *providerFile + for i := range files { + if files[i].version == version && files[i].os == osName && files[i].arch == arch { + match = &files[i] + break + } + } + if match == nil { + http.NotFound(w, r) + return + } + + base := baseURL(r) + verBase := fmt.Sprintf("%s%s/%s/%s", base+ProvidersV1Path, namespace, typeName, version) + + type gpgKey struct { + KeyID string `json:"key_id"` + ASCIIArmor string `json:"ascii_armor"` + } + resp := struct { + Protocols []string `json:"protocols"` + OS string `json:"os"` + Arch string `json:"arch"` + Filename string `json:"filename"` + DownloadURL string `json:"download_url"` + SHASumsURL string `json:"shasums_url"` + SHASumsSignatureURL string `json:"shasums_signature_url"` + SHASum string `json:"shasum"` + SigningKeys struct { + GPGPublicKeys []gpgKey `json:"gpg_public_keys"` + } `json:"signing_keys"` + }{ + Protocols: h.protocols, + OS: match.os, + Arch: match.arch, + Filename: path.Base(match.filePath), + DownloadURL: fmt.Sprintf("%s/api/v1/local/%s/%s", base, namespace, match.filePath), + SHASumsURL: verBase + "/sha256sums", + SHASumsSignatureURL: verBase + "/sha256sums.sig", + SHASum: match.sha256, + } + resp.SigningKeys.GPGPublicKeys = []gpgKey{{ + KeyID: h.signer.KeyID(), + ASCIIArmor: h.signer.PublicKeyArmor(), + }} + + writeJSON(w, resp) +} + +func (h *Handler) sha256sums(w http.ResponseWriter, r *http.Request) { + sums, ok := h.buildSums(w, r) + if !ok { + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Write(sums) +} + +func (h *Handler) sha256sumsSig(w http.ResponseWriter, r *http.Request) { + sums, ok := h.buildSums(w, r) + if !ok { + return + } + sig, err := h.signer.Sign(sums) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Write(sig) +} + +// buildSums renders the SHA256SUMS body for one version: one " " +// line per platform zip, sorted by filename so the signed bytes are stable. +func (h *Handler) buildSums(w http.ResponseWriter, r *http.Request) ([]byte, bool) { + if !h.Enabled() { + http.NotFound(w, r) + return nil, false + } + namespace := chi.URLParam(r, "namespace") + typeName := chi.URLParam(r, "type") + version := chi.URLParam(r, "version") + + files, err := h.resolve(r, namespace, typeName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return nil, false + } + + var lines []string + for _, f := range files { + if f.version != version { + continue + } + lines = append(lines, fmt.Sprintf("%s %s", f.sha256, path.Base(f.filePath))) + } + if len(lines) == 0 { + http.NotFound(w, r) + return nil, false + } + sort.Strings(lines) + return []byte(strings.Join(lines, "\n") + "\n"), true +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(v) +} + +func baseURL(r *http.Request) string { + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" { + scheme = fwd + } + return scheme + "://" + r.Host +} diff --git a/internal/api/terraform/registry_test.go b/internal/api/terraform/registry_test.go new file mode 100644 index 0000000..796e70c --- /dev/null +++ b/internal/api/terraform/registry_test.go @@ -0,0 +1,186 @@ +package terraform + +import ( + "bytes" + "context" + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/go-chi/chi/v5" + "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/armor" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/testsupport" + "git.unkin.net/unkin/artifactapi/internal/tfsign" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +var testDSN string + +func TestMain(m *testing.M) { + ctx := context.Background() + dsn, terminate, err := testsupport.StartPostgres(ctx) + if err != nil { + os.Exit(m.Run()) + } + testDSN = dsn + code := m.Run() + terminate() + os.Exit(code) +} + +// testSigner writes a throwaway armored key and loads it. +func testSigner(t *testing.T) *tfsign.Signer { + t.Helper() + e, err := openpgp.NewEntity("artifactapi test", "tf", "tf@example.com", nil) + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + w, _ := armor.Encode(&buf, openpgp.PrivateKeyType, nil) + if err := e.SerializePrivate(w, nil); err != nil { + t.Fatal(err) + } + w.Close() + p := filepath.Join(t.TempDir(), "private-key.asc") + if err := os.WriteFile(p, buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + s, err := tfsign.Load(p, "") + if err != nil { + t.Fatal(err) + } + return s +} + +func TestProviderRegistryFlow(t *testing.T) { + if testDSN == "" { + t.Skip("Docker unavailable") + } + ctx := context.Background() + db, err := database.New(testDSN) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + const repo = "tf-reg" // Terraform namespace == repo name + const filePath = "unkin/artifactapi/terraform-provider-artifactapi_1.2.3_linux_amd64.zip" + const hash = "sha256:983cdb25cb7b976538e4334d26e52dee5f44749b9be1500c760cf5cf66be659b" + const wantSha = "983cdb25cb7b976538e4334d26e52dee5f44749b9be1500c760cf5cf66be659b" + + if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageTerraform, RepoType: models.RepoTypeLocal}); err != nil { + t.Fatal(err) + } + if err := db.UpsertBlob(ctx, hash, "blobs/98/3c", 6381007, "application/zip"); err != nil { + t.Fatal(err) + } + if err := db.CreateLocalFile(ctx, repo, filePath, hash); err != nil { + t.Fatal(err) + } + + signer := testSigner(t) + h := NewHandler(db, signer, "5.0,6.0") + router := chi.NewRouter() + router.Get("/.well-known/terraform.json", h.ServiceDiscovery) + router.Mount(MountPath, h.Routes()) + + get := func(p string) *httptest.ResponseRecorder { + req := httptest.NewRequest("GET", p, nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + return w + } + + // Service discovery. + w := get("/.well-known/terraform.json") + if w.Code != 200 { + t.Fatalf("discovery = %d", w.Code) + } + var disc map[string]string + json.Unmarshal(w.Body.Bytes(), &disc) + if disc["providers.v1"] != ProvidersV1Path { + t.Errorf("providers.v1 = %q", disc["providers.v1"]) + } + + // Versions. + w = get("/terraform/v1/providers/tf-reg/artifactapi/versions") + if w.Code != 200 { + t.Fatalf("versions = %d %s", w.Code, w.Body) + } + var vresp struct { + Versions []struct { + Version string `json:"version"` + Protocols []string `json:"protocols"` + Platforms []map[string]string `json:"platforms"` + } `json:"versions"` + } + json.Unmarshal(w.Body.Bytes(), &vresp) + if len(vresp.Versions) != 1 || vresp.Versions[0].Version != "1.2.3" { + t.Fatalf("unexpected versions: %+v", vresp) + } + if len(vresp.Versions[0].Platforms) != 1 || vresp.Versions[0].Platforms[0]["os"] != "linux" { + t.Fatalf("unexpected platforms: %+v", vresp.Versions[0].Platforms) + } + + // Download. + w = get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/download/linux/amd64") + if w.Code != 200 { + t.Fatalf("download = %d %s", w.Code, w.Body) + } + var dl struct { + Filename string `json:"filename"` + DownloadURL string `json:"download_url"` + SHASumsURL string `json:"shasums_url"` + SHASumsSignatureURL string `json:"shasums_signature_url"` + SHASum string `json:"shasum"` + SigningKeys struct { + GPGPublicKeys []struct { + KeyID string `json:"key_id"` + ASCIIArmor string `json:"ascii_armor"` + } `json:"gpg_public_keys"` + } `json:"signing_keys"` + } + json.Unmarshal(w.Body.Bytes(), &dl) + if dl.SHASum != wantSha { + t.Errorf("shasum = %q", dl.SHASum) + } + wantURL := "http://example.com/api/v1/local/tf-reg/" + filePath + if dl.DownloadURL != wantURL { + t.Errorf("download_url = %q, want %q", dl.DownloadURL, wantURL) + } + if len(dl.SigningKeys.GPGPublicKeys) != 1 || dl.SigningKeys.GPGPublicKeys[0].KeyID != signer.KeyID() { + t.Errorf("signing key mismatch: %+v", dl.SigningKeys) + } + + // SHA256SUMS + signature verify against the advertised key. + sums := get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/sha256sums") + wantLine := wantSha + " terraform-provider-artifactapi_1.2.3_linux_amd64.zip\n" + if sums.Body.String() != wantLine { + t.Errorf("sha256sums = %q, want %q", sums.Body.String(), wantLine) + } + sig := get("/terraform/v1/providers/tf-reg/artifactapi/1.2.3/sha256sums.sig") + keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewReader([]byte(dl.SigningKeys.GPGPublicKeys[0].ASCIIArmor))) + if err != nil { + t.Fatal(err) + } + if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewReader(sums.Body.Bytes()), bytes.NewReader(sig.Body.Bytes())); err != nil { + t.Errorf("sha256sums.sig did not verify: %v", err) + } +} + +func TestRegistryDisabledWithoutSigner(t *testing.T) { + h := NewHandler(nil, nil, "") + router := chi.NewRouter() + router.Get("/.well-known/terraform.json", h.ServiceDiscovery) + req := httptest.NewRequest("GET", "/.well-known/terraform.json", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != 404 { + t.Errorf("disabled discovery = %d, want 404", w.Code) + } +} diff --git a/internal/config/env.go b/internal/config/env.go index 6d33d2a..c2ec5bd 100644 --- a/internal/config/env.go +++ b/internal/config/env.go @@ -24,6 +24,14 @@ type Config struct { S3Bucket string S3Secure bool S3Region string + + // Terraform provider registry signing. When TFSigningKeyPath points at a + // readable armored GPG private key, artifactapi serves local terraform + // repos as a real provider registry (service discovery + signed + // SHA256SUMS). Left empty, the registry endpoints stay disabled. + TFSigningKeyPath string + TFSigningKeyPassphrase string + TFProviderProtocols string } func (c *Config) DatabaseDSN() string { @@ -59,6 +67,10 @@ func Load() (*Config, error) { S3Bucket: getenv("MINIO_BUCKET", "artifacts"), S3Secure: s3Secure, S3Region: getenv("MINIO_REGION", ""), + + TFSigningKeyPath: getenv("TF_SIGNING_KEY_PATH", ""), + TFSigningKeyPassphrase: getenv("TF_SIGNING_KEY_PASSPHRASE", ""), + TFProviderProtocols: getenv("TF_PROVIDER_PROTOCOLS", "5.0,6.0"), } return cfg, nil diff --git a/internal/database/postgres.go b/internal/database/postgres.go index 32cc528..0e5b4e8 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -158,6 +158,13 @@ func (db *DB) migrate() error { ); CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name); + + CREATE TABLE IF NOT EXISTS signing_keys ( + purpose TEXT PRIMARY KEY, + private_key_armor TEXT NOT NULL, + key_id TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW() + ); `) return err } diff --git a/internal/database/signing_keys.go b/internal/database/signing_keys.go new file mode 100644 index 0000000..479f092 --- /dev/null +++ b/internal/database/signing_keys.go @@ -0,0 +1,35 @@ +package database + +import ( + "context" + "errors" + + "github.com/jackc/pgx/v5" +) + +// GetSigningKey returns the stored armored private key and key id for a purpose. +// found is false when no key has been generated yet. +func (db *DB) GetSigningKey(ctx context.Context, purpose string) (armor, keyID string, found bool, err error) { + row := db.Pool.QueryRow(ctx, ` + SELECT private_key_armor, key_id FROM signing_keys WHERE purpose = $1 + `, purpose) + if err := row.Scan(&armor, &keyID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return "", "", false, nil + } + return "", "", false, err + } + return armor, keyID, true, nil +} + +// InsertSigningKeyIfAbsent stores a freshly generated key, doing nothing if +// another replica already inserted one. Callers re-read with GetSigningKey to +// pick up whichever key won the race. +func (db *DB) InsertSigningKeyIfAbsent(ctx context.Context, purpose, armor, keyID string) error { + _, err := db.Pool.Exec(ctx, ` + INSERT INTO signing_keys (purpose, private_key_armor, key_id) + VALUES ($1, $2, $3) + ON CONFLICT (purpose) DO NOTHING + `, purpose, armor, keyID) + return err +} diff --git a/internal/database/signing_keys_test.go b/internal/database/signing_keys_test.go new file mode 100644 index 0000000..25f35d3 --- /dev/null +++ b/internal/database/signing_keys_test.go @@ -0,0 +1,31 @@ +package database + +import "testing" + +func TestSigningKeyRoundTripAndIdempotency(t *testing.T) { + requireDB(t) + + const purpose = "terraform-provider-test" + + // Absent to start. + if _, _, found, err := testDB.GetSigningKey(ctx(), purpose); err != nil || found { + t.Fatalf("expected no key, got found=%v err=%v", found, err) + } + + if err := testDB.InsertSigningKeyIfAbsent(ctx(), purpose, "ARMOR-1", "KEYID1"); err != nil { + t.Fatal(err) + } + + // A second insert must not overwrite (models the replica race). + if err := testDB.InsertSigningKeyIfAbsent(ctx(), purpose, "ARMOR-2", "KEYID2"); err != nil { + t.Fatal(err) + } + + armor, keyID, found, err := testDB.GetSigningKey(ctx(), purpose) + if err != nil || !found { + t.Fatalf("expected key, found=%v err=%v", found, err) + } + if armor != "ARMOR-1" || keyID != "KEYID1" { + t.Errorf("key was overwritten: armor=%q key_id=%q", armor, keyID) + } +} diff --git a/internal/provider/terraform/terraform.go b/internal/provider/terraform/terraform.go index f2633de..f29c33d 100644 --- a/internal/provider/terraform/terraform.go +++ b/internal/provider/terraform/terraform.go @@ -26,6 +26,27 @@ var providerZipRe = regexp.MustCompile( var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`) +// ParsedProviderZip describes a terraform-provider-{type}_{version}_{os}_{arch}.zip +// filename. Ok is false when the name doesn't match that convention. +type ParsedProviderZip struct { + Type string + Version string + OS string + Arch string + Ok bool +} + +// ParseProviderZip extracts the type, version and platform from a provider zip +// filename (the base name, not a full path). It's the canonical parser shared by +// the network-mirror index and the provider registry handler. +func ParseProviderZip(filename string) ParsedProviderZip { + m := providerZipRe.FindStringSubmatch(filename) + if m == nil { + return ParsedProviderZip{} + } + return ParsedProviderZip{Type: m[1], Version: m[2], OS: m[3], Arch: m[4], Ok: true} +} + type Provider struct{} func (p *Provider) Type() models.PackageType { return models.PackageTerraform } diff --git a/internal/provider/terraform/terraform_extra_test.go b/internal/provider/terraform/terraform_extra_test.go new file mode 100644 index 0000000..b089f9d --- /dev/null +++ b/internal/provider/terraform/terraform_extra_test.go @@ -0,0 +1,171 @@ +package terraform + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +type fakeFileStore struct{ entries []provider.FileEntry } + +func (f fakeFileStore) ListFilesByPrefix(_ context.Context, _, prefix string) ([]provider.FileEntry, error) { + var out []provider.FileEntry + for _, e := range f.entries { + if strings.HasPrefix(e.FilePath, prefix) { + out = append(out, e) + } + } + return out, nil +} +func (f fakeFileStore) ListPackages(_ context.Context, _ string) ([]string, error) { return nil, nil } + +func TestTFPureFuncs(t *testing.T) { + p := &Provider{} + if p.Classify("hashicorp/aws/versions") != provider.Mutable { + t.Error("versions should be mutable") + } + if p.Classify("hashicorp/aws/terraform-provider-aws_1.0.0_linux_amd64.zip") != provider.Immutable { + t.Error("zip should be immutable") + } + if got := p.UpstreamURL(models.Remote{BaseURL: "https://registry.terraform.io"}, "hashicorp/aws/versions"); got != "https://registry.terraform.io/v1/providers/hashicorp/aws/versions" { + t.Errorf("upstream url %q", got) + } + h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"}) + if h.Get("Authorization") == "" { + t.Error("auth header") + } + _ = p.ContentType("x.json") +} + +func TestTFValidateUpload(t *testing.T) { + p := &Provider{} + sp, ct, err := p.ValidateUpload("hashicorp/aws/terraform-provider-aws_1.2.3_linux_amd64.zip") + if err != nil || sp != "hashicorp/aws/terraform-provider-aws_1.2.3_linux_amd64.zip" || ct != "application/zip" { + t.Errorf("valid: sp=%q ct=%q err=%v", sp, ct, err) + } + if _, _, err := p.ValidateUpload("too/few"); err == nil { + t.Error("expected error for wrong path depth") + } + if _, _, err := p.ValidateUpload("ns/aws/not-a-provider.zip"); err == nil { + t.Error("expected error for bad filename") + } + if _, _, err := p.ValidateUpload("ns/gcp/terraform-provider-aws_1.0.0_linux_amd64.zip"); err == nil { + t.Error("expected error for type mismatch") + } +} + +func TestTFUploadResponse(t *testing.T) { + p := &Provider{} + resp := p.UploadResponse("hashicorp/aws/terraform-provider-aws_1.2.3_linux_amd64.zip", "sha256:abc", 100) + if resp["namespace"] != "hashicorp" || resp["type"] != "aws" || resp["version"] != "1.2.3" || resp["os"] != "linux" || resp["arch"] != "amd64" { + t.Errorf("structured response wrong: %v", resp) + } + fallback := p.UploadResponse("weird/path", "sha256:x", 1) + if fallback["path"] != "weird/path" { + t.Errorf("fallback response wrong: %v", fallback) + } +} + +func TestTFRewriteResponse(t *testing.T) { + p := &Provider{} + remote := models.Remote{Name: "tf", ReleasesRemote: "hashicorp-releases"} + + if out, _ := p.RewriteResponse([]byte(`{"download_url":"x"}`), models.Remote{}, "http://proxy"); out != nil { + t.Error("no ReleasesRemote should be a no-op") + } + if out, _ := p.RewriteResponse([]byte("not json"), remote, "http://proxy"); out != nil { + t.Error("invalid json should be a no-op") + } + body := []byte(`{"download_url":"https://releases.hashicorp.com/terraform-provider-aws/1.0/aws.zip"}`) + out, err := p.RewriteResponse(body, remote, "http://proxy") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), "http://proxy/api/v1/remote/hashicorp-releases/") { + t.Errorf("download_url not rewritten: %s", out) + } +} + +func TestTFServeLocalIndex(t *testing.T) { + p := &Provider{} + fs := fakeFileStore{entries: []provider.FileEntry{ + {FilePath: "hashicorp/aws/terraform-provider-aws_1.0.0_linux_amd64.zip", ContentHash: "sha256:deadbeef"}, + {FilePath: "hashicorp/aws/terraform-provider-aws_1.0.0_darwin_arm64.zip", ContentHash: "sha256:cafe"}, + }} + + serve := func(path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/"+path, nil) + p.ServeLocalIndex(w, r, fs, "repo", path) + return w + } + + if w := serve("hashicorp/aws/index.json"); w.Code != 200 || !strings.Contains(w.Body.String(), "1.0.0") { + t.Errorf("index.json: code=%d body=%s", w.Code, w.Body.String()) + } + if w := serve("hashicorp/aws/1.0.0.json"); w.Code != 200 || !strings.Contains(w.Body.String(), "linux_amd64") { + t.Errorf("version doc: code=%d body=%s", w.Code, w.Body.String()) + } + + // Not a terraform index path. + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/x", nil) + if p.ServeLocalIndex(w, r, fs, "repo", "hashicorp/aws/other.txt") { + t.Error("non-index path should return false") + } + if p.ServeLocalIndex(httptest.NewRecorder(), r, fs, "repo", "too/short") { + t.Error("short path should return false") + } +} + +func TestTFContentTypeAndEmptyIndex(t *testing.T) { + p := &Provider{} + for path, want := range map[string]string{ + "x.zip": "application/zip", + "x.sig": "application/octet-stream", + "index.json": "application/json", + } { + if got := p.ContentType(path); got != want { + t.Errorf("ContentType(%q)=%q want %q", path, got, want) + } + } + // index / version doc with no matching files -> 404. + empty := fakeFileStore{} + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/hashicorp/aws/index.json", nil) + p.ServeLocalIndex(w, r, empty, "repo", "hashicorp/aws/index.json") + if w.Code != http.StatusNotFound { + t.Errorf("empty index should be 404, got %d", w.Code) + } + w = httptest.NewRecorder() + p.ServeLocalIndex(w, r, empty, "repo", "hashicorp/aws/1.0.0.json") + if w.Code != http.StatusNotFound { + t.Errorf("empty version doc should be 404, got %d", w.Code) + } +} + +func TestRewriteDownloadURL(t *testing.T) { + // Empty proxy base -> unchanged. + if got := rewriteDownloadURL("https://x/a.zip", "rel", ""); got != "https://x/a.zip" { + t.Errorf("empty base: %q", got) + } + // Unparseable URL -> unchanged. + if got := rewriteDownloadURL("://bad", "rel", "http://p"); got != "://bad" { + t.Errorf("bad url: %q", got) + } + // Normal rewrite. + if got := rewriteDownloadURL("https://cdn/path/a.zip", "rel", "http://p"); got != "http://p/api/v1/remote/rel/path/a.zip" { + t.Errorf("rewrite: %q", got) + } +} + +func TestTFGenerateLocalIndexUnsupported(t *testing.T) { + if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeFileStore{}, "r", "x"); err == nil { + t.Error("expected unsupported error") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 04059c1..91114ec 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -12,6 +12,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + tfregistry "git.unkin.net/unkin/artifactapi/internal/api/terraform" v1 "git.unkin.net/unkin/artifactapi/internal/api/v1" v2 "git.unkin.net/unkin/artifactapi/internal/api/v2" "git.unkin.net/unkin/artifactapi/internal/cache" @@ -30,6 +31,7 @@ import ( _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform" "git.unkin.net/unkin/artifactapi/internal/proxy" "git.unkin.net/unkin/artifactapi/internal/storage" + "git.unkin.net/unkin/artifactapi/internal/tfsign" "git.unkin.net/unkin/artifactapi/internal/virtual" ) @@ -43,6 +45,7 @@ type Server struct { engine *proxy.Engine virtEngine *virtual.Engine localHandler *v2.LocalHandler + tfRegistry *tfregistry.Handler gc *gc.Collector } @@ -67,6 +70,25 @@ func New(cfg *config.Config, version string) (*Server, error) { virtEngine := virtual.NewEngine(db, engine) collector := gc.New(db, s3, 1*time.Hour) + // The terraform registry signs with a GPG key. A configured file wins (BYO + // key); otherwise artifactapi generates one on first start and persists it in + // the database so every replica shares it. A failure here must not take the + // server down — the registry just stays disabled. + var signer *tfsign.Signer + if cfg.TFSigningKeyPath != "" { + signer, err = tfsign.Load(cfg.TFSigningKeyPath, cfg.TFSigningKeyPassphrase) + } else { + signer, err = tfsign.LoadOrCreate(context.Background(), db, "terraform-provider") + } + if err != nil { + slog.Warn("terraform provider registry disabled", "error", err) + signer = nil + } + tfRegistry := tfregistry.NewHandler(db, signer, cfg.TFProviderProtocols) + if tfRegistry.Enabled() { + slog.Info("terraform provider registry enabled", "key_id", signer.KeyID()) + } + s := &Server{ cfg: cfg, version: version, @@ -76,6 +98,7 @@ func New(cfg *config.Config, version string) (*Server, error) { engine: engine, virtEngine: virtEngine, localHandler: localHandler, + tfRegistry: tfRegistry, gc: collector, } @@ -97,6 +120,11 @@ func (s *Server) routes() chi.Router { r.Get("/", s.handleRoot) r.Get("/version", s.handleVersion) + // Terraform provider registry: service discovery at the well-known path, + // providers.v1 protocol under /terraform/v1/providers. + r.Get("/.well-known/terraform.json", s.tfRegistry.ServiceDiscovery) + r.Mount(tfregistry.MountPath, s.tfRegistry.Routes()) + proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler) r.Mount("/api/v1", proxyHandler.Routes()) r.Mount("/v2", proxyHandler.DockerV2Routes()) diff --git a/internal/tfsign/signer.go b/internal/tfsign/signer.go new file mode 100644 index 0000000..5245448 --- /dev/null +++ b/internal/tfsign/signer.go @@ -0,0 +1,173 @@ +// Package tfsign loads a GPG signing key and produces the detached signatures +// the Terraform provider registry protocol requires over SHA256SUMS files. +package tfsign + +import ( + "bytes" + "context" + "fmt" + "os" + "strings" + + "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/armor" +) + +// KeyStore persists a generated signing key. *database.DB satisfies it. +type KeyStore interface { + GetSigningKey(ctx context.Context, purpose string) (armor, keyID string, found bool, err error) + InsertSigningKeyIfAbsent(ctx context.Context, purpose, armor, keyID string) error +} + +// LoadOrCreate returns a signer for purpose, generating and persisting a new key +// the first time it is needed. It is safe across replicas: a lost insert race +// just re-reads whichever key won. +func LoadOrCreate(ctx context.Context, store KeyStore, purpose string) (*Signer, error) { + armored, _, found, err := store.GetSigningKey(ctx, purpose) + if err != nil { + return nil, err + } + if !found { + newArmor, keyID, err := Generate() + if err != nil { + return nil, err + } + if err := store.InsertSigningKeyIfAbsent(ctx, purpose, newArmor, keyID); err != nil { + return nil, err + } + if armored, _, _, err = store.GetSigningKey(ctx, purpose); err != nil { + return nil, err + } + } + return LoadArmored(armored, "") +} + +// Signer holds a decrypted GPG entity and exposes what the registry download +// response needs: a detached signature, the armored public key, and the key ID. +type Signer struct { + entity *openpgp.Entity + publicASCII string + keyID string +} + +// Load reads an armored private key from path, decrypting it with passphrase if +// the key is protected. A blank path returns (nil, nil): a nil *Signer means the +// caller should fall back to another source (e.g. a DB-stored key). +func Load(path, passphrase string) (*Signer, error) { + if path == "" { + return nil, nil + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("open signing key: %w", err) + } + return fromArmor(string(data), passphrase, path) +} + +// LoadArmored builds a signer from an in-memory armored private key, e.g. one +// read from the database. A blank key returns (nil, nil). +func LoadArmored(armored, passphrase string) (*Signer, error) { + if armored == "" { + return nil, nil + } + return fromArmor(armored, passphrase, "stored key") +} + +// Generate creates a fresh signing keypair and returns the armored private key +// (to persist) and its uppercase key id. +func Generate() (armoredPrivateKey, keyID string, err error) { + entity, err := openpgp.NewEntity("artifactapi terraform registry", "provider signing", "artifactapi@localhost", nil) + if err != nil { + return "", "", err + } + var buf bytes.Buffer + w, err := armor.Encode(&buf, openpgp.PrivateKeyType, nil) + if err != nil { + return "", "", err + } + if err := entity.SerializePrivate(w, nil); err != nil { + return "", "", err + } + if err := w.Close(); err != nil { + return "", "", err + } + return buf.String(), strings.ToUpper(entity.PrimaryKey.KeyIdString()), nil +} + +func fromArmor(armored, passphrase, src string) (*Signer, error) { + keyring, err := openpgp.ReadArmoredKeyRing(strings.NewReader(armored)) + if err != nil { + return nil, fmt.Errorf("read signing key: %w", err) + } + if len(keyring) == 0 { + return nil, fmt.Errorf("signing key (%s) contains no entities", src) + } + entity := keyring[0] + + if entity.PrivateKey == nil { + return nil, fmt.Errorf("signing key (%s) has no private key material", src) + } + if entity.PrivateKey.Encrypted { + if err := decrypt(entity, passphrase); err != nil { + return nil, err + } + } + + pub, err := armorPublicKey(entity) + if err != nil { + return nil, err + } + + return &Signer{ + entity: entity, + publicASCII: pub, + keyID: entity.PrimaryKey.KeyIdString(), + }, nil +} + +// decrypt unlocks the entity's private key and all subkeys with the passphrase. +func decrypt(entity *openpgp.Entity, passphrase string) error { + pw := []byte(passphrase) + if err := entity.PrivateKey.Decrypt(pw); err != nil { + return fmt.Errorf("decrypt signing key: %w", err) + } + for _, sub := range entity.Subkeys { + if sub.PrivateKey != nil && sub.PrivateKey.Encrypted { + _ = sub.PrivateKey.Decrypt(pw) + } + } + return nil +} + +func armorPublicKey(entity *openpgp.Entity) (string, error) { + var buf bytes.Buffer + w, err := armor.Encode(&buf, openpgp.PublicKeyType, nil) + if err != nil { + return "", err + } + if err := entity.Serialize(w); err != nil { + return "", err + } + if err := w.Close(); err != nil { + return "", err + } + return buf.String(), nil +} + +// Sign returns a binary detached signature over message, matching the +// SHA256SUMS.sig format Terraform verifies. +func (s *Signer) Sign(message []byte) ([]byte, error) { + var buf bytes.Buffer + if err := openpgp.DetachSign(&buf, s.entity, bytes.NewReader(message), nil); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// PublicKeyArmor returns the ASCII-armored public key for the registry's +// signing_keys response. +func (s *Signer) PublicKeyArmor() string { return s.publicASCII } + +// KeyID returns the 16-hex-char uppercase key ID Terraform matches against the +// signature's issuer. +func (s *Signer) KeyID() string { return strings.ToUpper(s.keyID) } diff --git a/internal/tfsign/signer_test.go b/internal/tfsign/signer_test.go new file mode 100644 index 0000000..03a6a18 --- /dev/null +++ b/internal/tfsign/signer_test.go @@ -0,0 +1,155 @@ +package tfsign + +import ( + "bytes" + "context" + "os" + "path/filepath" + "regexp" + "testing" + + "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/armor" +) + +// armoredPrivateKey generates a throwaway armored private key for tests. +func armoredPrivateKey(t *testing.T) string { + t.Helper() + e, err := openpgp.NewEntity("artifactapi test", "tf registry", "tf@example.com", nil) + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + w, err := armor.Encode(&buf, openpgp.PrivateKeyType, nil) + if err != nil { + t.Fatal(err) + } + if err := e.SerializePrivate(w, nil); err != nil { + t.Fatal(err) + } + w.Close() + return buf.String() +} + +func writeKey(t *testing.T, contents string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "private-key.asc") + if err := os.WriteFile(p, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func TestLoadSignAndVerify(t *testing.T) { + path := writeKey(t, armoredPrivateKey(t)) + s, err := Load(path, "") + if err != nil { + t.Fatal(err) + } + if s == nil { + t.Fatal("expected a signer") + } + + if !regexp.MustCompile(`^[0-9A-F]{16}$`).MatchString(s.KeyID()) { + t.Errorf("key id %q is not 16 uppercase hex chars", s.KeyID()) + } + + msg := []byte("deadbeef terraform-provider-x_1.0.0_linux_amd64.zip\n") + sig, err := s.Sign(msg) + if err != nil { + t.Fatal(err) + } + + // The advertised public key must verify the signature over the same bytes. + keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewReader([]byte(s.PublicKeyArmor()))) + if err != nil { + t.Fatal(err) + } + if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewReader(msg), bytes.NewReader(sig)); err != nil { + t.Errorf("signature did not verify: %v", err) + } +} + +func TestGenerateAndLoadArmored(t *testing.T) { + priv, keyID, err := Generate() + if err != nil { + t.Fatal(err) + } + if !regexp.MustCompile(`^[0-9A-F]{16}$`).MatchString(keyID) { + t.Errorf("generated key id %q malformed", keyID) + } + + s, err := LoadArmored(priv, "") + if err != nil { + t.Fatal(err) + } + if s.KeyID() != keyID { + t.Errorf("loaded key id %q != generated %q", s.KeyID(), keyID) + } + + msg := []byte("abc terraform-provider-x_1.0.0_linux_amd64.zip\n") + sig, err := s.Sign(msg) + if err != nil { + t.Fatal(err) + } + keyring, _ := openpgp.ReadArmoredKeyRing(bytes.NewReader([]byte(s.PublicKeyArmor()))) + if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewReader(msg), bytes.NewReader(sig)); err != nil { + t.Errorf("signature did not verify: %v", err) + } +} + +// memStore is an in-memory KeyStore that records how many keys it accepted. +type memStore struct { + armor, keyID string + found bool + inserts int +} + +func (m *memStore) GetSigningKey(_ context.Context, _ string) (string, string, bool, error) { + return m.armor, m.keyID, m.found, nil +} + +func (m *memStore) InsertSigningKeyIfAbsent(_ context.Context, _, armor, keyID string) error { + if !m.found { // ON CONFLICT DO NOTHING + m.armor, m.keyID, m.found = armor, keyID, true + m.inserts++ + } + return nil +} + +func TestLoadOrCreateGeneratesOnceThenReuses(t *testing.T) { + store := &memStore{} + + first, err := LoadOrCreate(context.Background(), store, "terraform-provider") + if err != nil || first == nil { + t.Fatalf("first LoadOrCreate: signer=%v err=%v", first, err) + } + + second, err := LoadOrCreate(context.Background(), store, "terraform-provider") + if err != nil || second == nil { + t.Fatalf("second LoadOrCreate: signer=%v err=%v", second, err) + } + + if store.inserts != 1 { + t.Errorf("expected exactly one key generated, got %d", store.inserts) + } + if first.KeyID() != second.KeyID() { + t.Errorf("key id changed between loads: %q vs %q", first.KeyID(), second.KeyID()) + } +} + +func TestLoadEmptyPathDisabled(t *testing.T) { + s, err := Load("", "") + if err != nil { + t.Fatal(err) + } + if s != nil { + t.Error("empty path should yield a nil (disabled) signer") + } +} + +func TestLoadMissingFile(t *testing.T) { + if _, err := Load(filepath.Join(t.TempDir(), "nope.asc"), ""); err == nil { + t.Error("expected an error for a missing key file") + } +}