From f59d6cf8d865921369e77d3ba603e24b2f38fab9 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Fri, 9 Oct 2026 19:17:13 +1100 Subject: [PATCH] add rpm support to virtual repositories --- internal/virtual/rpm_merger.go | 279 ++++++++++++++++++++++++++++ internal/virtual/rpm_merger_test.go | 268 ++++++++++++++++++++++++++ 2 files changed, 547 insertions(+) create mode 100644 internal/virtual/rpm_merger.go create mode 100644 internal/virtual/rpm_merger_test.go diff --git a/internal/virtual/rpm_merger.go b/internal/virtual/rpm_merger.go new file mode 100644 index 0000000..2fda0b4 --- /dev/null +++ b/internal/virtual/rpm_merger.go @@ -0,0 +1,279 @@ +package virtual + +import ( + "bytes" + "compress/bzip2" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "regexp" + "strings" + + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +var rpmDataTypes = []string{"primary", "filelists", "other"} + +var rpmRoots = map[string]string{ + "primary": ``, + "filelists": ``, + "other": ``, +} + +var rpmRootClose = map[string]string{"primary": "", "filelists": "", "other": ""} + +var locationRe = regexp.MustCompile(`]*/>`) + +// RPMMember is one member repo's decompressed primary/filelists/other XML, +// keyed by repomd data type. Missing types contribute no packages. +type RPMMember struct { + RemoteName string + Timestamp int64 + Data map[string][]byte +} + +// RPMRepo is a merged yum repo: repomd.xml plus the files it references, keyed +// by their path under the repo root. +type RPMRepo struct { + Repomd []byte + Files map[string][]byte +} + +type repomdDoc struct { + Revision string `xml:"revision"` + Data []repomdData `xml:"data"` +} + +type repomdData struct { + Type string `xml:"type,attr"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + Timestamp int64 `xml:"timestamp"` +} + +type rpmPkgVersion struct { + Epoch string `xml:"epoch,attr"` + Ver string `xml:"ver,attr"` + Rel string `xml:"rel,attr"` +} + +type primaryPkg struct { + Name string `xml:"name"` + Arch string `xml:"arch"` + Version rpmPkgVersion `xml:"version"` + Checksum string `xml:"checksum"` + Location struct { + Href string `xml:"href,attr"` + Base string `xml:"http://www.w3.org/XML/1998/namespace base,attr"` + } `xml:"location"` +} + +type pkgidPkg struct { + PkgID string `xml:"pkgid,attr"` +} + +// parseRepomd returns the member's primary/filelists/other locations and its +// newest data timestamp. +func parseRepomd(body []byte) (map[string]string, int64, error) { + var doc repomdDoc + if err := xml.Unmarshal(body, &doc); err != nil { + return nil, 0, fmt.Errorf("parse repomd.xml: %w", err) + } + locs := map[string]string{} + var ts int64 + for _, d := range doc.Data { + for _, t := range rpmDataTypes { + if d.Type == t && d.Location.Href != "" { + locs[t] = d.Location.Href + ts = max(ts, d.Timestamp) + } + } + } + if locs["primary"] == "" { + return nil, 0, errors.New("repomd.xml has no primary data") + } + return locs, ts, nil +} + +func decompress(href string, body []byte) ([]byte, error) { + var r io.Reader + switch { + case strings.HasSuffix(href, ".gz"): + gz, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + r = gz + case strings.HasSuffix(href, ".xz"): + x, err := xz.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + r = x + case strings.HasSuffix(href, ".zst"): + z, err := zstd.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer z.Close() + r = z + case strings.HasSuffix(href, ".bz2"): + r = bzip2.NewReader(bytes.NewReader(body)) + default: + return body, nil + } + return io.ReadAll(r) +} + +// splitPackages returns the raw bytes of each top-level element. +func splitPackages(doc []byte) ([][]byte, error) { + d := xml.NewDecoder(bytes.NewReader(doc)) + var pkgs [][]byte + depth := 0 + for { + start := d.InputOffset() + tok, err := d.Token() + if err == io.EOF { + return pkgs, nil + } + if err != nil { + return nil, err + } + switch t := tok.(type) { + case xml.StartElement: + if depth == 1 && t.Name.Local == "package" { + if err := d.Skip(); err != nil { + return nil, err + } + pkgs = append(pkgs, doc[start:d.InputOffset()]) + continue + } + depth++ + case xml.EndElement: + depth-- + } + } +} + +func xmlEscape(s string) string { + var b bytes.Buffer + _ = xml.EscapeText(&b, []byte(s)) + return b.String() +} + +// MergeRPM merges members into one repo. Members are in priority order: the +// first member to carry a NEVRA wins. Package locations are prefixed with the +// owning member's name so the virtual can route downloads back to it. +func MergeRPM(members []RPMMember) (*RPMRepo, error) { + kept := map[string]bool{} + seen := map[string]bool{} + out := map[string][][]byte{} + + for i, m := range members { + pkgs, err := splitPackages(m.Data["primary"]) + if err != nil { + return nil, fmt.Errorf("member %q primary: %w", m.RemoteName, err) + } + for _, raw := range pkgs { + var p primaryPkg + if err := xml.Unmarshal(raw, &p); err != nil { + return nil, fmt.Errorf("member %q primary package: %w", m.RemoteName, err) + } + epoch := p.Version.Epoch + if epoch == "" { + epoch = "0" + } + nevra := fmt.Sprintf("%s-%s:%s-%s.%s", p.Name, epoch, p.Version.Ver, p.Version.Rel, p.Arch) + if seen[nevra] { + continue + } + seen[nevra] = true + kept[fmt.Sprintf("%d/%s", i, p.Checksum)] = true + + if p.Location.Base == "" { + loc := `` + raw = locationRe.ReplaceAll(raw, []byte(loc)) + } + out["primary"] = append(out["primary"], raw) + } + } + + for _, t := range rpmDataTypes[1:] { + emitted := map[string]bool{} + for i, m := range members { + pkgs, err := splitPackages(m.Data[t]) + if err != nil { + return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, t, err) + } + for _, raw := range pkgs { + var p pkgidPkg + if err := xml.Unmarshal(raw, &p); err != nil { + return nil, fmt.Errorf("member %q %s package: %w", m.RemoteName, t, err) + } + key := fmt.Sprintf("%d/%s", i, p.PkgID) + if kept[key] && !emitted[key] { + emitted[key] = true + out[t] = append(out[t], raw) + } + } + } + } + + var ts int64 + for _, m := range members { + ts = max(ts, m.Timestamp) + } + + repo := &RPMRepo{Files: map[string][]byte{}} + var md bytes.Buffer + md.WriteString(xml.Header) + md.WriteString(`` + "\n") + fmt.Fprintf(&md, " %d\n", ts) + for _, t := range rpmDataTypes { + var doc bytes.Buffer + doc.WriteString(xml.Header) + fmt.Fprintf(&doc, rpmRoots[t]+"\n", len(out[t])) + for _, raw := range out[t] { + doc.Write(raw) + doc.WriteString("\n") + } + doc.WriteString(rpmRootClose[t] + "\n") + + gz := gzipDeterministic(doc.Bytes()) + sum := sha256Hex(gz) + href := fmt.Sprintf("repodata/%s-%s.xml.gz", sum, t) + repo.Files[href] = gz + + fmt.Fprintf(&md, " \n", t) + fmt.Fprintf(&md, " %s\n", sum) + fmt.Fprintf(&md, " %s\n", sha256Hex(doc.Bytes())) + fmt.Fprintf(&md, " \n", href) + fmt.Fprintf(&md, " %d\n", ts) + fmt.Fprintf(&md, " %d\n", len(gz)) + fmt.Fprintf(&md, " %d\n", doc.Len()) + md.WriteString(" \n") + } + md.WriteString("\n") + repo.Repomd = md.Bytes() + return repo, nil +} + +func gzipDeterministic(data []byte) []byte { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + gz.Header = gzip.Header{OS: 255} + _, _ = gz.Write(data) + _ = gz.Close() + return buf.Bytes() +} + +func sha256Hex(data []byte) string { + h := sha256.Sum256(data) + return hex.EncodeToString(h[:]) +} diff --git a/internal/virtual/rpm_merger_test.go b/internal/virtual/rpm_merger_test.go new file mode 100644 index 0000000..144cfeb --- /dev/null +++ b/internal/virtual/rpm_merger_test.go @@ -0,0 +1,268 @@ +package virtual + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "fmt" + "io" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +func primaryXML(pkgs ...string) []byte { + return []byte(` + +` + strings.Join(pkgs, "\n") + ` +`) +} + +func primaryPkgXML(name, ver, pkgid, href string) string { + return ` + ` + name + ` + x86_64 + + ` + pkgid + ` + + + + +` +} + +func filelistsXML(pkgids ...string) []byte { + var b strings.Builder + b.WriteString(``) + for _, id := range pkgids { + b.WriteString(`/usr/bin/` + id + ``) + } + b.WriteString(``) + return []byte(b.String()) +} + +func gunzip(t *testing.T, b []byte) []byte { + t.Helper() + r, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + out, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + return out +} + +func mergedFile(t *testing.T, repo *RPMRepo, dtype string) []byte { + t.Helper() + for href, body := range repo.Files { + if strings.HasSuffix(href, "-"+dtype+".xml.gz") { + return gunzip(t, body) + } + } + t.Fatalf("no %s file in merged repo", dtype) + return nil +} + +func TestMergeRPMDedupePriority(t *testing.T) { + members := []RPMMember{ + {RemoteName: "first", Timestamp: 100, Data: map[string][]byte{ + "primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo-1.0.rpm")), + "filelists": filelistsXML("aaa"), + }}, + {RemoteName: "second", Timestamp: 200, Data: map[string][]byte{ + "primary": primaryXML( + primaryPkgXML("foo", "1.0", "bbb", "Packages/foo-1.0-rebuilt.rpm"), + primaryPkgXML("bar", "2.0", "ccc", "Packages/bar-2.0.rpm"), + ), + "filelists": filelistsXML("bbb", "ccc"), + }}, + } + repo, err := MergeRPM(members) + if err != nil { + t.Fatal(err) + } + + primary := string(mergedFile(t, repo, "primary")) + if strings.Count(primary, "foo") != 1 { + t.Fatalf("duplicate NEVRA not deduped:\n%s", primary) + } + if !strings.Contains(primary, `href="first/Packages/foo-1.0.rpm"`) || strings.Contains(primary, "foo-1.0-rebuilt") { + t.Fatalf("first member should win duplicate NEVRA:\n%s", primary) + } + if !strings.Contains(primary, `href="second/Packages/bar-2.0.rpm"`) { + t.Fatalf("unique package from second member missing:\n%s", primary) + } + if !strings.Contains(primary, `packages="2"`) { + t.Fatalf("package count wrong:\n%s", primary) + } + + filelists := string(mergedFile(t, repo, "filelists")) + if !strings.Contains(filelists, "/usr/bin/aaa") || !strings.Contains(filelists, "/usr/bin/ccc") || strings.Contains(filelists, "/usr/bin/bbb") { + t.Fatalf("filelists must follow the winning primary entries:\n%s", filelists) + } + + other := string(mergedFile(t, repo, "other")) + if !strings.Contains(other, `packages="0"`) { + t.Fatalf("members without other data should yield an empty other.xml:\n%s", other) + } + + if !strings.Contains(string(repo.Repomd), "200") { + t.Fatalf("revision should be the newest member timestamp:\n%s", repo.Repomd) + } +} + +func TestMergeRPMOutputIsWellFormed(t *testing.T) { + repo, err := MergeRPM([]RPMMember{{RemoteName: "a", Data: map[string][]byte{ + "primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo.rpm")), + }}}) + if err != nil { + t.Fatal(err) + } + var doc struct { + XMLName xml.Name + Packages []struct { + Name string `xml:"name"` + Provides []struct { + Name string `xml:"name,attr"` + } `xml:"format>provides>entry"` + } `xml:"package"` + } + if err := xml.Unmarshal(mergedFile(t, repo, "primary"), &doc); err != nil { + t.Fatal(err) + } + if doc.XMLName.Space != "http://linux.duke.edu/metadata/common" || len(doc.Packages) != 1 { + t.Fatalf("unexpected merged primary: %+v", doc) + } + if len(doc.Packages[0].Provides) != 1 || doc.Packages[0].Provides[0].Name != "foo" { + t.Fatalf("rpm: namespaced format lost: %+v", doc.Packages[0]) + } +} + +func TestMergeRPMHrefRewriting(t *testing.T) { + based := `extnoarch` + + `eee` + repo, err := MergeRPM([]RPMMember{{RemoteName: "gh", Data: map[string][]byte{ + "primary": primaryXML( + primaryPkgXML("foo", "1.0", "aaa", "storytold/photocraft/releases/download/v1.0/foo&bar.rpm"), + primaryPkgXML("lead", "1.0", "bbb", "/Packages/lead.rpm"), + based, + ), + }}}) + if err != nil { + t.Fatal(err) + } + primary := string(mergedFile(t, repo, "primary")) + for _, want := range []string{ + ``, + ``, + ``, + } { + if !strings.Contains(primary, want) { + t.Errorf("missing %s in:\n%s", want, primary) + } + } +} + +func TestMergeRPMChecksums(t *testing.T) { + repo, err := MergeRPM([]RPMMember{{RemoteName: "a", Timestamp: 42, Data: map[string][]byte{ + "primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo.rpm")), + }}}) + if err != nil { + t.Fatal(err) + } + + var md struct { + Data []struct { + Type string `xml:"type,attr"` + Checksum string `xml:"checksum"` + OpenChecksum string `xml:"open-checksum"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + Timestamp int64 `xml:"timestamp"` + Size int `xml:"size"` + OpenSize int `xml:"open-size"` + } `xml:"data"` + } + if err := xml.Unmarshal(repo.Repomd, &md); err != nil { + t.Fatal(err) + } + if len(md.Data) != 3 { + t.Fatalf("want primary/filelists/other, got %d entries", len(md.Data)) + } + for _, d := range md.Data { + body, ok := repo.Files[d.Location.Href] + if !ok { + t.Fatalf("%s: location %q not served", d.Type, d.Location.Href) + } + sum := sha256.Sum256(body) + if hex.EncodeToString(sum[:]) != d.Checksum || len(body) != d.Size { + t.Errorf("%s: checksum/size do not match served bytes", d.Type) + } + open := gunzip(t, body) + osum := sha256.Sum256(open) + if hex.EncodeToString(osum[:]) != d.OpenChecksum || len(open) != d.OpenSize { + t.Errorf("%s: open-checksum/open-size do not match decompressed bytes", d.Type) + } + if d.Timestamp != 42 { + t.Errorf("%s: timestamp %d, want 42", d.Type, d.Timestamp) + } + } + + again, _ := MergeRPM([]RPMMember{{RemoteName: "a", Timestamp: 42, Data: map[string][]byte{ + "primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo.rpm")), + }}}) + if !bytes.Equal(repo.Repomd, again.Repomd) { + t.Error("merge must be deterministic so repomd checksums stay valid across requests") + } +} + +func TestParseRepomd(t *testing.T) { + locs, ts, err := parseRepomd([]byte(` + 10 + 99 + 20 +`)) + if err != nil { + t.Fatal(err) + } + if locs["primary"] != "repodata/p-primary.xml.zst" || locs["other"] != "repodata/o-other.xml.gz" || len(locs) != 2 { + t.Fatalf("unexpected locations: %v", locs) + } + if ts != 20 { + t.Fatalf("timestamp %d, want 20 (sqlite entries ignored)", ts) + } + if _, _, err := parseRepomd([]byte(``)); err == nil { + t.Fatal("repomd without primary should error") + } +} + +func TestDecompress(t *testing.T) { + plain := []byte("") + + var xzBuf bytes.Buffer + xw, _ := xz.NewWriter(&xzBuf) + _, _ = xw.Write(plain) + _ = xw.Close() + + zw, _ := zstd.NewWriter(nil) + zst := zw.EncodeAll(plain, nil) + + for href, body := range map[string][]byte{ + "p.xml": plain, + "p.xml.gz": gzipDeterministic(plain), + "p.xml.xz": xzBuf.Bytes(), + "p.xml.zst": zst, + } { + got, err := decompress(href, body) + if err != nil || !bytes.Equal(got, plain) { + t.Errorf("%s: got %q, %v", href, got, err) + } + } +}