Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| be7be0bd04 | |||
| ac6e1bf798 | |||
| c6f7eb3eb7 | |||
| 8e96aabbef | |||
| 5a2bfdf73a | |||
| 8378701d9a | |||
| a51323c725 |
@@ -33,7 +33,7 @@ Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<
|
|||||||
- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)
|
- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)
|
||||||
- `generic`: [remote](docs/generic/remote.md)
|
- `generic`: [remote](docs/generic/remote.md)
|
||||||
- `gems`: [remote](docs/gems/remote.md), [local](docs/gems/local.md), [virtual](docs/gems/virtual.md)
|
- `gems`: [remote](docs/gems/remote.md), [local](docs/gems/local.md), [virtual](docs/gems/virtual.md)
|
||||||
- `goproxy`: [remote](docs/goproxy/remote.md), [local](docs/goproxy/local.md)
|
- `goproxy`: [remote](docs/goproxy/remote.md), [local](docs/goproxy/local.md), [virtual](docs/goproxy/virtual.md)
|
||||||
- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md)
|
- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md)
|
||||||
- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md)
|
- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md)
|
||||||
- `nuget`: [remote](docs/nuget/remote.md)
|
- `nuget`: [remote](docs/nuget/remote.md)
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
# `goproxy` virtual
|
||||||
|
|
||||||
|
A goproxy virtual serves one GOPROXY over local and remote goproxy members:
|
||||||
|
`GOPROXY=https://artifactapi.example.com/api/v1/virtual/<name>`.
|
||||||
|
|
||||||
|
- `<module>/@v/list` is the union of member versions, sorted by semver.
|
||||||
|
- `<module>/@latest` is the highest release in that union, else the highest
|
||||||
|
prerelease; with no listed versions it falls back to the first member that
|
||||||
|
answers `@latest`.
|
||||||
|
- `@latest` and `@v/<version>.{info,mod,zip}` 302 to the first member, in
|
||||||
|
member order, that lists the version (or, for unlisted pseudo-versions,
|
||||||
|
serves its `.info`). Redirects are root-relative and `no-store`.
|
||||||
|
- Module paths are case-encoded (uppercase as `!lower`).
|
||||||
|
- A member error fails the request with 502; a member answering 404 or 410 is
|
||||||
|
skipped. Merged lists are cached for 60s.
|
||||||
|
|
||||||
|
```hcl
|
||||||
|
resource "artifactapi_virtual" "go" {
|
||||||
|
name = "go"
|
||||||
|
package_type = "goproxy"
|
||||||
|
members = [artifactapi_local_goproxy.internal.name, artifactapi_remote_goproxy.goproxy.name]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export GOPROXY=https://artifactapi.example.com/api/v1/virtual/go GONOSUMDB=example.unkin.net
|
||||||
|
go get example.unkin.net/hello@latest golang.org/x/text@latest
|
||||||
|
```
|
||||||
|
|
||||||
|
Publish to the local member directly; the virtual is read-only.
|
||||||
@@ -38,7 +38,9 @@ already-running stack.
|
|||||||
client (merged versions, install through the member redirect); gems compact
|
client (merged versions, install through the member redirect); gems compact
|
||||||
index merge over two locals via stock `ruby:3.3` bundler (member order wins,
|
index merge over two locals via stock `ruby:3.3` bundler (member order wins,
|
||||||
downloads 302 to the owning member); ansible over two locals via stock
|
downloads 302 to the owning member); ansible over two locals via stock
|
||||||
`ansible-galaxy` (install through the virtual, download 302 to the owner).
|
`ansible-galaxy` (install through the virtual, download 302 to the owner);
|
||||||
|
goproxy over two locals via a stock `golang:1.25` client (merged versions,
|
||||||
|
member order wins, version files 302 to the owning member).
|
||||||
- **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors
|
- **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors
|
||||||
(pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors
|
(pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors
|
||||||
(constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary,
|
(constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary,
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
//go:build dockere2e
|
||||||
|
|
||||||
|
package e2edocker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/mod/module"
|
||||||
|
modzip "golang.org/x/mod/zip"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestVirtualGoProxy publishes to two local goproxy repos and drives a stock go
|
||||||
|
// client through a virtual over both: merged versions, member order winning a
|
||||||
|
// duplicate version, and version files redirected to the owning member.
|
||||||
|
func TestVirtualGoProxy(t *testing.T) {
|
||||||
|
network := os.Getenv("COMPOSE_NETWORK")
|
||||||
|
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||||
|
if network == "" || internal == "" {
|
||||||
|
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||||
|
}
|
||||||
|
if _, err := exec.LookPath("docker"); err != nil {
|
||||||
|
t.Skip("docker not available on the test host")
|
||||||
|
}
|
||||||
|
|
||||||
|
id := time.Now().UnixNano()
|
||||||
|
a, b, virt := fmt.Sprintf("e2e-go-a-%d", id), fmt.Sprintf("e2e-go-b-%d", id), fmt.Sprintf("e2e-go-v-%d", id)
|
||||||
|
for _, name := range []string{a, b} {
|
||||||
|
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"goproxy","repo_type":"local"}`, name))
|
||||||
|
defer deleteRepo(t, name)
|
||||||
|
}
|
||||||
|
createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"goproxy","members":[%q,%q]}`, virt, a, b))
|
||||||
|
defer deleteVirtual(t, virt)
|
||||||
|
|
||||||
|
const mod = "example.unkin.net/hello"
|
||||||
|
gomod := []byte("module " + mod + "\n\ngo 1.25\n")
|
||||||
|
for _, p := range [][2]string{{a, "v0.1.0"}, {b, "v0.1.0"}, {b, "v0.2.0"}} {
|
||||||
|
dir := t.TempDir()
|
||||||
|
os.WriteFile(filepath.Join(dir, "go.mod"), gomod, 0o600)
|
||||||
|
os.WriteFile(filepath.Join(dir, "hello.go"), []byte("package hello\n\nconst Greeting = \""+p[0]+":"+p[1]+"\"\n"), 0o600)
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if err := modzip.CreateFromDir(&buf, module.Version{Path: mod, Version: p[1]}, dir); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
uploadFile(t, p[0], mod+"/@v/"+p[1]+".zip", buf.Bytes(), "application/zip")
|
||||||
|
uploadFile(t, p[0], mod+"/@v/"+p[1]+".mod", gomod, "text/plain")
|
||||||
|
}
|
||||||
|
|
||||||
|
base := strings.TrimRight(internal, "/")
|
||||||
|
script := `set -euo pipefail
|
||||||
|
export GOPROXY="$BASE/api/v1/virtual/$V" GONOSUMDB=example.unkin.net GOFLAGS=-mod=mod GOTOOLCHAIN=local
|
||||||
|
mkdir -p /w && cd /w
|
||||||
|
printf 'module consumer\n\ngo 1.25\n' > go.mod
|
||||||
|
printf 'package main\n\nimport "example.unkin.net/hello"\n\nfunc main() { println("RUN " + hello.Greeting) }\n' > main.go
|
||||||
|
echo "VERSIONS $(go list -m -versions example.unkin.net/hello)"
|
||||||
|
go get example.unkin.net/hello@latest && go run . 2>&1
|
||||||
|
go get example.unkin.net/hello@v0.1.0 && go run . 2>&1
|
||||||
|
echo "REDIRECT $(curl -s -o /dev/null -w '%{redirect_url}' "$GOPROXY/example.unkin.net/hello/@v/v0.1.0.zip")"`
|
||||||
|
|
||||||
|
out, err := exec.Command("docker", "run", "--rm", "--network", network,
|
||||||
|
"-e", "BASE="+base, "-e", "V="+virt, "golang:1.25", "bash", "-c", script).CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("go run failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"VERSIONS example.unkin.net/hello v0.1.0 v0.2.0",
|
||||||
|
"RUN " + b + ":v0.2.0",
|
||||||
|
"RUN " + a + ":v0.1.0",
|
||||||
|
"REDIRECT " + base + "/api/v1/local/" + a + "/example.unkin.net/hello/@v/v0.1.0.zip",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(string(out), want) {
|
||||||
|
t.Fatalf("missing %q in go output:\n%s", want, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package v1
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||||
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// serveVirtualGoProxy serves the merged @v/list and 302s @latest and version
|
||||||
|
// files to the member owning that version.
|
||||||
|
func (h *ProxyHandler) serveVirtualGoProxy(w http.ResponseWriter, r *http.Request, virt models.Virtual, path string) {
|
||||||
|
body, loc, err := h.virtualEngine.GoProxy(r.Context(), virt, path, r.URL.RawQuery)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, virtual.ErrNotFound):
|
||||||
|
http.Error(w, "not found", http.StatusNotFound)
|
||||||
|
case errors.Is(err, virtual.ErrBadPath):
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
case err != nil:
|
||||||
|
slog.Error("virtual goproxy fetch failed", "virtual", virt.Name, "path", path, "error", err)
|
||||||
|
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||||
|
case loc != "":
|
||||||
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
|
http.Redirect(w, r, loc, http.StatusFound)
|
||||||
|
default:
|
||||||
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||||
|
w.Header().Set("X-Artifact-Source", "virtual")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -215,6 +215,10 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.serveVirtualAnsible(w, r, *virt, path, proxyBaseURL)
|
h.serveVirtualAnsible(w, r, *virt, path, proxyBaseURL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if virt.PackageType == models.PackageGoProxy {
|
||||||
|
h.serveVirtualGoProxy(w, r, *virt, path)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
|
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -141,7 +141,8 @@ func versions(ctx context.Context, files provider.FileStore, repoName, escMod st
|
|||||||
return versions, infos, nil
|
return versions, infos, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func latest(vs []string) string {
|
// Latest is the highest release in sorted vs, else the highest prerelease.
|
||||||
|
func Latest(vs []string) string {
|
||||||
for i := len(vs) - 1; i >= 0; i-- {
|
for i := len(vs) - 1; i >= 0; i-- {
|
||||||
if semver.Prerelease(vs[i]) == "" {
|
if semver.Prerelease(vs[i]) == "" {
|
||||||
return vs[i]
|
return vs[i]
|
||||||
@@ -205,7 +206,7 @@ func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileSt
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
v := latest(vs)
|
v := Latest(vs)
|
||||||
if v == "" {
|
if v == "" {
|
||||||
return nil, fmt.Errorf("no versions of %s", escMod)
|
return nil, fmt.Errorf("no versions of %s", escMod)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,6 +43,8 @@ type Engine struct {
|
|||||||
AnsibleLocal func(ctx context.Context, repo, path string) ([]byte, error)
|
AnsibleLocal func(ctx context.Context, repo, path string) ([]byte, error)
|
||||||
ansibleDoc func(ctx context.Context, remote models.Remote, path string) ([]byte, error)
|
ansibleDoc func(ctx context.Context, remote models.Remote, path string) ([]byte, error)
|
||||||
|
|
||||||
|
goMember func(ctx context.Context, name, path string) (*GoMember, error)
|
||||||
|
|
||||||
mergeTTL time.Duration
|
mergeTTL time.Duration
|
||||||
|
|
||||||
sf singleflight.Group
|
sf singleflight.Group
|
||||||
@@ -74,6 +76,7 @@ func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
|
|||||||
e.npmMember = e.fetchNPMMember
|
e.npmMember = e.fetchNPMMember
|
||||||
e.gemsMember = e.fetchGemsMember
|
e.gemsMember = e.fetchGemsMember
|
||||||
e.ansibleDoc = e.ansibleMemberDoc
|
e.ansibleDoc = e.ansibleMemberDoc
|
||||||
|
e.goMember = e.fetchGoMember
|
||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -428,6 +431,9 @@ func (e *Engine) fetchMemberPath(ctx context.Context, remote models.Remote, path
|
|||||||
if serve != nil {
|
if serve != nil {
|
||||||
rec := httptest.NewRecorder()
|
rec := httptest.NewRecorder()
|
||||||
if serve(rec, httptest.NewRequestWithContext(ctx, http.MethodGet, "/"+path, nil)) {
|
if serve(rec, httptest.NewRequestWithContext(ctx, http.MethodGet, "/"+path, nil)) {
|
||||||
|
if rec.Code == http.StatusNotFound || rec.Code == http.StatusGone {
|
||||||
|
return nil, fmt.Errorf("%s/%s: %w", remote.Name, path, ErrNotFound)
|
||||||
|
}
|
||||||
if rec.Code != http.StatusOK {
|
if rec.Code != http.StatusOK {
|
||||||
return nil, fmt.Errorf("%s/%s: status %d", remote.Name, path, rec.Code)
|
return nil, fmt.Errorf("%s/%s: status %d", remote.Name, path, rec.Code)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,228 @@
|
|||||||
|
package virtual
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"net/http"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"golang.org/x/mod/module"
|
||||||
|
"golang.org/x/mod/semver"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||||
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxGoModules = 1024
|
||||||
|
|
||||||
|
// GoMember is one member's response for a module path; Body is nil when the
|
||||||
|
// member does not have it (404/410).
|
||||||
|
type GoMember struct {
|
||||||
|
RemoteName string
|
||||||
|
Body []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoDoc is a goproxy virtual's merged @v/list for one module. Owners maps each
|
||||||
|
// version to the first member listing it; missing caches that no member has it.
|
||||||
|
type GoDoc struct {
|
||||||
|
Body []byte
|
||||||
|
Owners map[string]string
|
||||||
|
Latest string
|
||||||
|
missing bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *GoDoc) index() []byte {
|
||||||
|
if d.missing {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
b := append(slices.Clone(d.Body), 0)
|
||||||
|
for _, v := range slices.Sorted(maps.Keys(d.Owners)) {
|
||||||
|
b = append(b, v+"="+d.Owners[v]+"\n"...)
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *GoDoc) files() map[string][]byte { return nil }
|
||||||
|
|
||||||
|
// MergeGoList unions member @v/list documents, sorted by semver; the first
|
||||||
|
// member listing a version owns it. ErrNotFound means no member has the module.
|
||||||
|
func MergeGoList(members []GoMember) (*GoDoc, error) {
|
||||||
|
d := &GoDoc{Owners: map[string]string{}}
|
||||||
|
found := false
|
||||||
|
var vs []string
|
||||||
|
for _, m := range members {
|
||||||
|
if m.Body == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
found = true
|
||||||
|
for line := range strings.Lines(string(m.Body)) {
|
||||||
|
f := strings.Fields(line)
|
||||||
|
if len(f) == 0 || !semver.IsValid(f[0]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, dup := d.Owners[f[0]]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
d.Owners[f[0]] = m.RemoteName
|
||||||
|
vs = append(vs, f[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
semver.Sort(vs)
|
||||||
|
for _, v := range vs {
|
||||||
|
d.Body = append(d.Body, v+"\n"...)
|
||||||
|
}
|
||||||
|
if d.Body == nil {
|
||||||
|
d.Body = []byte{}
|
||||||
|
}
|
||||||
|
d.Latest = goproxy.Latest(vs)
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoProxy serves a goproxy virtual path: the merged <module>/@v/list body, or a
|
||||||
|
// root-relative redirect for @latest and @v/<version>.{info,mod,zip} to the
|
||||||
|
// first member, in member order, that has that version.
|
||||||
|
func (e *Engine) GoProxy(ctx context.Context, virt models.Virtual, path, rawQuery string) (body []byte, loc string, err error) {
|
||||||
|
escMod, rest, ok := strings.Cut(path, "/@")
|
||||||
|
if !ok {
|
||||||
|
return nil, "", ErrNotFound
|
||||||
|
}
|
||||||
|
if _, err := module.UnescapePath(escMod); err != nil {
|
||||||
|
return nil, "", ErrNotFound
|
||||||
|
}
|
||||||
|
if rest == "v/list" {
|
||||||
|
d, err := e.goList(ctx, virt, escMod)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
return d.Body, "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var target, owner string
|
||||||
|
var d *GoDoc
|
||||||
|
switch {
|
||||||
|
case rest == "latest":
|
||||||
|
if d, err = e.goList(ctx, virt, escMod); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if d.Latest == "" {
|
||||||
|
target = path
|
||||||
|
owner, err = e.goProbe(ctx, virt, target)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
escVer, _ := module.EscapeVersion(d.Latest)
|
||||||
|
target, owner = escMod+"/@v/"+escVer+".info", d.Owners[d.Latest]
|
||||||
|
case strings.HasPrefix(rest, "v/"):
|
||||||
|
escVer, ext, ok := cutGoExt(rest[len("v/"):])
|
||||||
|
if !ok {
|
||||||
|
return nil, "", ErrNotFound
|
||||||
|
}
|
||||||
|
ver, verr := module.UnescapeVersion(escVer)
|
||||||
|
if verr != nil {
|
||||||
|
return nil, "", ErrNotFound
|
||||||
|
}
|
||||||
|
if d, err = e.goList(ctx, virt, escMod); err != nil && !errors.Is(err, ErrNotFound) {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if d != nil {
|
||||||
|
owner = d.Owners[ver]
|
||||||
|
}
|
||||||
|
if owner == "" {
|
||||||
|
owner, err = e.goProbe(ctx, virt, escMod+"/@v/"+escVer+".info")
|
||||||
|
}
|
||||||
|
target = escMod + "/@v/" + escVer + "." + ext
|
||||||
|
default:
|
||||||
|
return nil, "", ErrNotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
loc, ok, err = e.memberLocation(ctx, owner, target, "", "")
|
||||||
|
if err != nil || !ok {
|
||||||
|
return nil, "", errors.Join(err, ErrNotFound)
|
||||||
|
}
|
||||||
|
// module paths and versions are validated, so the only escape is the case-encoding '!'
|
||||||
|
loc = strings.ReplaceAll(loc, "%21", "!")
|
||||||
|
if rawQuery != "" {
|
||||||
|
loc += "?" + rawQuery
|
||||||
|
}
|
||||||
|
return nil, loc, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cutGoExt(file string) (escVer, ext string, ok bool) {
|
||||||
|
dot := strings.LastIndex(file, ".")
|
||||||
|
if dot <= 0 || strings.Contains(file, "/") {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
switch ext = file[dot+1:]; ext {
|
||||||
|
case "info", "mod", "zip":
|
||||||
|
return file[:dot], ext, true
|
||||||
|
}
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) goList(ctx context.Context, virt models.Virtual, escMod string) (*GoDoc, error) {
|
||||||
|
prefix := "goproxy\x00" + virt.Name + "\x00"
|
||||||
|
inserted := false
|
||||||
|
repo, err := e.merged(ctx, prefix+escMod, func(ctx context.Context) (mergedRepo, error) {
|
||||||
|
inserted = true
|
||||||
|
members, err := fetchMembers(virt, func(name string) (*GoMember, error) { return e.goMember(ctx, name, escMod+"/@v/list") })
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
d, err := MergeGoList(members)
|
||||||
|
if errors.Is(err, ErrNotFound) {
|
||||||
|
return &GoDoc{missing: true}, nil
|
||||||
|
}
|
||||||
|
return d, err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if inserted {
|
||||||
|
e.evictOldest(prefix, maxGoModules)
|
||||||
|
}
|
||||||
|
if d := repo.(*GoDoc); !d.missing {
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
// goProbe returns the first member, in member order, that serves path.
|
||||||
|
func (e *Engine) goProbe(ctx context.Context, virt models.Virtual, path string) (string, error) {
|
||||||
|
for _, name := range virt.Members {
|
||||||
|
m, err := e.goMember(ctx, name, path)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("virtual %q: member %q: %v", virt.Name, name, err)
|
||||||
|
}
|
||||||
|
if m.Body != nil {
|
||||||
|
return name, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) fetchGoMember(ctx context.Context, name, path string) (*GoMember, error) {
|
||||||
|
remote, err := e.getRemote(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("remote %q: %w", name, err)
|
||||||
|
}
|
||||||
|
body, err := e.fetchMemberPath(ctx, *remote, path)
|
||||||
|
var pe *proxy.ProxyError
|
||||||
|
if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && (pe.Status == http.StatusNotFound || pe.Status == http.StatusGone) {
|
||||||
|
return &GoMember{RemoteName: name}, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if body == nil {
|
||||||
|
body = []byte{}
|
||||||
|
}
|
||||||
|
return &GoMember{RemoteName: name, Body: body}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
package virtual
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||||
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMergeGoList(t *testing.T) {
|
||||||
|
d, err := MergeGoList([]GoMember{
|
||||||
|
{RemoteName: "internal", Body: []byte("v0.2.0\nv0.1.0\n")},
|
||||||
|
{RemoteName: "absent"},
|
||||||
|
{RemoteName: "goproxy", Body: []byte("v0.1.0 extra\nv1.0.0-rc.1\n\nnot-a-version\nv0.3.0\n")},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if want := "v0.1.0\nv0.2.0\nv0.3.0\nv1.0.0-rc.1\n"; string(d.Body) != want {
|
||||||
|
t.Fatalf("list = %q, want %q", d.Body, want)
|
||||||
|
}
|
||||||
|
if d.Latest != "v0.3.0" {
|
||||||
|
t.Fatalf("latest = %q, want the highest release over the prerelease", d.Latest)
|
||||||
|
}
|
||||||
|
for v, owner := range map[string]string{"v0.1.0": "internal", "v0.2.0": "internal", "v0.3.0": "goproxy", "v1.0.0-rc.1": "goproxy"} {
|
||||||
|
if d.Owners[v] != owner {
|
||||||
|
t.Errorf("owner of %s = %q, want %q", v, d.Owners[v], owner)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
d, _ = MergeGoList([]GoMember{{RemoteName: "a", Body: []byte("v1.0.0-beta\nv1.0.0-alpha\n")}})
|
||||||
|
if d.Latest != "v1.0.0-beta" {
|
||||||
|
t.Fatalf("latest with only prereleases = %q", d.Latest)
|
||||||
|
}
|
||||||
|
if d, err := MergeGoList([]GoMember{{RemoteName: "a", Body: []byte{}}}); err != nil || string(d.Body) != "" || d.Latest != "" {
|
||||||
|
t.Fatalf("empty list must merge to an empty 200 body: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := MergeGoList([]GoMember{{RemoteName: "a"}, {RemoteName: "b"}}); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("want ErrNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func goEngine(docs map[string]string, fail string) (*Engine, map[string]int) {
|
||||||
|
var mu sync.Mutex
|
||||||
|
asked := map[string]int{}
|
||||||
|
return &Engine{
|
||||||
|
mergeTTL: time.Minute,
|
||||||
|
getRemote: func(_ context.Context, name string) (*models.Remote, error) {
|
||||||
|
rt := models.RepoTypeRemote
|
||||||
|
if name == "internal" {
|
||||||
|
rt = models.RepoTypeLocal
|
||||||
|
}
|
||||||
|
return &models.Remote{Name: name, RepoType: rt}, nil
|
||||||
|
},
|
||||||
|
goMember: func(_ context.Context, name, path string) (*GoMember, error) {
|
||||||
|
mu.Lock()
|
||||||
|
asked[name+" "+path]++
|
||||||
|
mu.Unlock()
|
||||||
|
if name == fail {
|
||||||
|
return nil, errors.New("upstream down")
|
||||||
|
}
|
||||||
|
if d, ok := docs[name+" "+path]; ok {
|
||||||
|
return &GoMember{RemoteName: name, Body: []byte(d)}, nil
|
||||||
|
}
|
||||||
|
return &GoMember{RemoteName: name}, nil
|
||||||
|
},
|
||||||
|
}, asked
|
||||||
|
}
|
||||||
|
|
||||||
|
func goVirt() models.Virtual {
|
||||||
|
return models.Virtual{Name: "go", PackageType: models.PackageGoProxy, Members: []string{"internal", "goproxy"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGoProxyRedirectsToFirstOwner(t *testing.T) {
|
||||||
|
e, asked := goEngine(map[string]string{
|
||||||
|
"internal example.unkin.net/!hello/@v/list": "v0.1.0\nv0.2.0\n",
|
||||||
|
"goproxy example.unkin.net/!hello/@v/list": "v0.2.0\nv0.3.0-rc.1\n",
|
||||||
|
"goproxy golang.org/x/text/@v/list": "v0.1.0\n",
|
||||||
|
"goproxy example.unkin.net/!hello/@v/v0.0.0-20260101-abcdef.info": "{}",
|
||||||
|
"goproxy example.com/notags/@v/list": "",
|
||||||
|
"goproxy example.com/notags/@latest": "{}",
|
||||||
|
}, "")
|
||||||
|
virt := goVirt()
|
||||||
|
body, loc, err := e.GoProxy(context.Background(), virt, "example.unkin.net/!hello/@v/list", "")
|
||||||
|
if err != nil || loc != "" || string(body) != "v0.1.0\nv0.2.0\nv0.3.0-rc.1\n" {
|
||||||
|
t.Fatalf("list = %q %q %v", body, loc, err)
|
||||||
|
}
|
||||||
|
for path, want := range map[string]string{
|
||||||
|
"example.unkin.net/!hello/@v/v0.2.0.zip": "/api/v1/local/internal/example.unkin.net/!hello/@v/v0.2.0.zip",
|
||||||
|
"example.unkin.net/!hello/@v/v0.3.0-rc.1.mod": "/api/v1/remote/goproxy/example.unkin.net/!hello/@v/v0.3.0-rc.1.mod",
|
||||||
|
"example.unkin.net/!hello/@latest": "/api/v1/local/internal/example.unkin.net/!hello/@v/v0.2.0.info",
|
||||||
|
"example.unkin.net/!hello/@v/v0.0.0-20260101-abcdef.zip": "/api/v1/remote/goproxy/example.unkin.net/!hello/@v/v0.0.0-20260101-abcdef.zip",
|
||||||
|
"golang.org/x/text/@v/v0.1.0.info": "/api/v1/remote/goproxy/golang.org/x/text/@v/v0.1.0.info",
|
||||||
|
"example.com/notags/@latest": "/api/v1/remote/goproxy/example.com/notags/@latest",
|
||||||
|
} {
|
||||||
|
_, loc, err := e.GoProxy(context.Background(), virt, path, "")
|
||||||
|
if err != nil || loc != want {
|
||||||
|
t.Errorf("%s -> %q %v, want %q", path, loc, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, path := range []string{
|
||||||
|
"example.unkin.net/!hello/@v/v9.9.9.zip",
|
||||||
|
"example.unkin.net/Hello/@v/list",
|
||||||
|
"example.unkin.net/!hello/@v/v0.1.0.tar",
|
||||||
|
"example.unkin.net/!hello/@v/../v0.1.0.zip",
|
||||||
|
"example.unkin.net/!hello",
|
||||||
|
"missing.example/mod/@v/list",
|
||||||
|
"missing.example/mod/@latest",
|
||||||
|
} {
|
||||||
|
if _, _, err := e.GoProxy(context.Background(), virt, path, ""); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Errorf("%s: want ErrNotFound, got %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := asked["internal example.unkin.net/!hello/@v/list"]; n != 1 {
|
||||||
|
t.Fatalf("merged list must be cached, internal asked %d times", n)
|
||||||
|
}
|
||||||
|
if n := asked["goproxy missing.example/mod/@v/list"]; n != 1 {
|
||||||
|
t.Fatalf("a module no member has must be cached, goproxy asked %d times", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGoProxyFollowsOwnerChange(t *testing.T) {
|
||||||
|
docs := map[string]string{"internal example.com/mod/@v/list": "v0.1.0\n"}
|
||||||
|
e, _ := goEngine(docs, "")
|
||||||
|
e.mergeTTL = 0
|
||||||
|
zip := func() string {
|
||||||
|
t.Helper()
|
||||||
|
_, loc, err := e.GoProxy(context.Background(), goVirt(), "example.com/mod/@v/v0.1.0.zip", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return loc
|
||||||
|
}
|
||||||
|
if loc := zip(); loc != "/api/v1/local/internal/example.com/mod/@v/v0.1.0.zip" {
|
||||||
|
t.Fatalf("before move: %q", loc)
|
||||||
|
}
|
||||||
|
delete(docs, "internal example.com/mod/@v/list")
|
||||||
|
docs["goproxy example.com/mod/@v/list"] = "v0.1.0\n"
|
||||||
|
if loc := zip(); loc != "/api/v1/remote/goproxy/example.com/mod/@v/v0.1.0.zip" {
|
||||||
|
t.Fatalf("same list, new owner: redirect still %q", loc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGoProxyFailsClosed(t *testing.T) {
|
||||||
|
e, _ := goEngine(map[string]string{"internal example.unkin.net/hello/@v/list": "v0.1.0\n"}, "goproxy")
|
||||||
|
for _, path := range []string{"example.unkin.net/hello/@v/list", "example.unkin.net/hello/@latest", "example.unkin.net/hello/@v/v0.1.0.zip"} {
|
||||||
|
if _, _, err := e.GoProxy(context.Background(), goVirt(), path, ""); err == nil || errors.Is(err, ErrNotFound) {
|
||||||
|
t.Errorf("%s: want member error, got %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGoProxyLocalMemberNotFound(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
dsn, terminate, err := testsupport.StartPostgres(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("Docker unavailable")
|
||||||
|
}
|
||||||
|
defer terminate()
|
||||||
|
db, err := database.New(dsn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
local := models.Remote{Name: "internal", PackageType: models.PackageGoProxy, RepoType: models.RepoTypeLocal}
|
||||||
|
if err := db.CreateRemote(ctx, &local); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
e := NewEngine(db, nil)
|
||||||
|
if _, err := e.fetchMemberPath(ctx, local, "example.com/mod/@latest"); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("local 404 must be ErrNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
virt := models.Virtual{Name: "go", PackageType: models.PackageGoProxy, Members: []string{"internal"}}
|
||||||
|
if _, _, err := e.GoProxy(ctx, virt, "example.com/mod/@latest", ""); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("module only absent from a local member must be not-found, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user