diff --git a/internal/virtual/rpm_merger.go b/internal/virtual/rpm_merger.go
new file mode 100644
index 0000000..2fda0b4
--- /dev/null
+++ b/internal/virtual/rpm_merger.go
@@ -0,0 +1,279 @@
+package virtual
+
+import (
+ "bytes"
+ "compress/bzip2"
+ "compress/gzip"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "io"
+ "regexp"
+ "strings"
+
+ "github.com/klauspost/compress/zstd"
+ "github.com/ulikunitz/xz"
+)
+
+var rpmDataTypes = []string{"primary", "filelists", "other"}
+
+var rpmRoots = map[string]string{
+ "primary": ``,
+ "filelists": ``,
+ "other": ``,
+}
+
+var rpmRootClose = map[string]string{"primary": "", "filelists": "", "other": ""}
+
+var locationRe = regexp.MustCompile(`]*/>`)
+
+// RPMMember is one member repo's decompressed primary/filelists/other XML,
+// keyed by repomd data type. Missing types contribute no packages.
+type RPMMember struct {
+ RemoteName string
+ Timestamp int64
+ Data map[string][]byte
+}
+
+// RPMRepo is a merged yum repo: repomd.xml plus the files it references, keyed
+// by their path under the repo root.
+type RPMRepo struct {
+ Repomd []byte
+ Files map[string][]byte
+}
+
+type repomdDoc struct {
+ Revision string `xml:"revision"`
+ Data []repomdData `xml:"data"`
+}
+
+type repomdData struct {
+ Type string `xml:"type,attr"`
+ Location struct {
+ Href string `xml:"href,attr"`
+ } `xml:"location"`
+ Timestamp int64 `xml:"timestamp"`
+}
+
+type rpmPkgVersion struct {
+ Epoch string `xml:"epoch,attr"`
+ Ver string `xml:"ver,attr"`
+ Rel string `xml:"rel,attr"`
+}
+
+type primaryPkg struct {
+ Name string `xml:"name"`
+ Arch string `xml:"arch"`
+ Version rpmPkgVersion `xml:"version"`
+ Checksum string `xml:"checksum"`
+ Location struct {
+ Href string `xml:"href,attr"`
+ Base string `xml:"http://www.w3.org/XML/1998/namespace base,attr"`
+ } `xml:"location"`
+}
+
+type pkgidPkg struct {
+ PkgID string `xml:"pkgid,attr"`
+}
+
+// parseRepomd returns the member's primary/filelists/other locations and its
+// newest data timestamp.
+func parseRepomd(body []byte) (map[string]string, int64, error) {
+ var doc repomdDoc
+ if err := xml.Unmarshal(body, &doc); err != nil {
+ return nil, 0, fmt.Errorf("parse repomd.xml: %w", err)
+ }
+ locs := map[string]string{}
+ var ts int64
+ for _, d := range doc.Data {
+ for _, t := range rpmDataTypes {
+ if d.Type == t && d.Location.Href != "" {
+ locs[t] = d.Location.Href
+ ts = max(ts, d.Timestamp)
+ }
+ }
+ }
+ if locs["primary"] == "" {
+ return nil, 0, errors.New("repomd.xml has no primary data")
+ }
+ return locs, ts, nil
+}
+
+func decompress(href string, body []byte) ([]byte, error) {
+ var r io.Reader
+ switch {
+ case strings.HasSuffix(href, ".gz"):
+ gz, err := gzip.NewReader(bytes.NewReader(body))
+ if err != nil {
+ return nil, err
+ }
+ r = gz
+ case strings.HasSuffix(href, ".xz"):
+ x, err := xz.NewReader(bytes.NewReader(body))
+ if err != nil {
+ return nil, err
+ }
+ r = x
+ case strings.HasSuffix(href, ".zst"):
+ z, err := zstd.NewReader(bytes.NewReader(body))
+ if err != nil {
+ return nil, err
+ }
+ defer z.Close()
+ r = z
+ case strings.HasSuffix(href, ".bz2"):
+ r = bzip2.NewReader(bytes.NewReader(body))
+ default:
+ return body, nil
+ }
+ return io.ReadAll(r)
+}
+
+// splitPackages returns the raw bytes of each top-level element.
+func splitPackages(doc []byte) ([][]byte, error) {
+ d := xml.NewDecoder(bytes.NewReader(doc))
+ var pkgs [][]byte
+ depth := 0
+ for {
+ start := d.InputOffset()
+ tok, err := d.Token()
+ if err == io.EOF {
+ return pkgs, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ switch t := tok.(type) {
+ case xml.StartElement:
+ if depth == 1 && t.Name.Local == "package" {
+ if err := d.Skip(); err != nil {
+ return nil, err
+ }
+ pkgs = append(pkgs, doc[start:d.InputOffset()])
+ continue
+ }
+ depth++
+ case xml.EndElement:
+ depth--
+ }
+ }
+}
+
+func xmlEscape(s string) string {
+ var b bytes.Buffer
+ _ = xml.EscapeText(&b, []byte(s))
+ return b.String()
+}
+
+// MergeRPM merges members into one repo. Members are in priority order: the
+// first member to carry a NEVRA wins. Package locations are prefixed with the
+// owning member's name so the virtual can route downloads back to it.
+func MergeRPM(members []RPMMember) (*RPMRepo, error) {
+ kept := map[string]bool{}
+ seen := map[string]bool{}
+ out := map[string][][]byte{}
+
+ for i, m := range members {
+ pkgs, err := splitPackages(m.Data["primary"])
+ if err != nil {
+ return nil, fmt.Errorf("member %q primary: %w", m.RemoteName, err)
+ }
+ for _, raw := range pkgs {
+ var p primaryPkg
+ if err := xml.Unmarshal(raw, &p); err != nil {
+ return nil, fmt.Errorf("member %q primary package: %w", m.RemoteName, err)
+ }
+ epoch := p.Version.Epoch
+ if epoch == "" {
+ epoch = "0"
+ }
+ nevra := fmt.Sprintf("%s-%s:%s-%s.%s", p.Name, epoch, p.Version.Ver, p.Version.Rel, p.Arch)
+ if seen[nevra] {
+ continue
+ }
+ seen[nevra] = true
+ kept[fmt.Sprintf("%d/%s", i, p.Checksum)] = true
+
+ if p.Location.Base == "" {
+ loc := ``
+ raw = locationRe.ReplaceAll(raw, []byte(loc))
+ }
+ out["primary"] = append(out["primary"], raw)
+ }
+ }
+
+ for _, t := range rpmDataTypes[1:] {
+ emitted := map[string]bool{}
+ for i, m := range members {
+ pkgs, err := splitPackages(m.Data[t])
+ if err != nil {
+ return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, t, err)
+ }
+ for _, raw := range pkgs {
+ var p pkgidPkg
+ if err := xml.Unmarshal(raw, &p); err != nil {
+ return nil, fmt.Errorf("member %q %s package: %w", m.RemoteName, t, err)
+ }
+ key := fmt.Sprintf("%d/%s", i, p.PkgID)
+ if kept[key] && !emitted[key] {
+ emitted[key] = true
+ out[t] = append(out[t], raw)
+ }
+ }
+ }
+ }
+
+ var ts int64
+ for _, m := range members {
+ ts = max(ts, m.Timestamp)
+ }
+
+ repo := &RPMRepo{Files: map[string][]byte{}}
+ var md bytes.Buffer
+ md.WriteString(xml.Header)
+ md.WriteString(`` + "\n")
+ fmt.Fprintf(&md, " %d\n", ts)
+ for _, t := range rpmDataTypes {
+ var doc bytes.Buffer
+ doc.WriteString(xml.Header)
+ fmt.Fprintf(&doc, rpmRoots[t]+"\n", len(out[t]))
+ for _, raw := range out[t] {
+ doc.Write(raw)
+ doc.WriteString("\n")
+ }
+ doc.WriteString(rpmRootClose[t] + "\n")
+
+ gz := gzipDeterministic(doc.Bytes())
+ sum := sha256Hex(gz)
+ href := fmt.Sprintf("repodata/%s-%s.xml.gz", sum, t)
+ repo.Files[href] = gz
+
+ fmt.Fprintf(&md, " \n", t)
+ fmt.Fprintf(&md, " %s\n", sum)
+ fmt.Fprintf(&md, " %s\n", sha256Hex(doc.Bytes()))
+ fmt.Fprintf(&md, " \n", href)
+ fmt.Fprintf(&md, " %d\n", ts)
+ fmt.Fprintf(&md, " %d\n", len(gz))
+ fmt.Fprintf(&md, " %d\n", doc.Len())
+ md.WriteString(" \n")
+ }
+ md.WriteString("\n")
+ repo.Repomd = md.Bytes()
+ return repo, nil
+}
+
+func gzipDeterministic(data []byte) []byte {
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ gz.Header = gzip.Header{OS: 255}
+ _, _ = gz.Write(data)
+ _ = gz.Close()
+ return buf.Bytes()
+}
+
+func sha256Hex(data []byte) string {
+ h := sha256.Sum256(data)
+ return hex.EncodeToString(h[:])
+}
diff --git a/internal/virtual/rpm_merger_test.go b/internal/virtual/rpm_merger_test.go
new file mode 100644
index 0000000..144cfeb
--- /dev/null
+++ b/internal/virtual/rpm_merger_test.go
@@ -0,0 +1,268 @@
+package virtual
+
+import (
+ "bytes"
+ "compress/gzip"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/xml"
+ "fmt"
+ "io"
+ "strings"
+ "testing"
+
+ "github.com/klauspost/compress/zstd"
+ "github.com/ulikunitz/xz"
+)
+
+func primaryXML(pkgs ...string) []byte {
+ return []byte(`
+
+` + strings.Join(pkgs, "\n") + `
+`)
+}
+
+func primaryPkgXML(name, ver, pkgid, href string) string {
+ return `
+ ` + name + `
+ x86_64
+
+ ` + pkgid + `
+
+
+
+
+`
+}
+
+func filelistsXML(pkgids ...string) []byte {
+ var b strings.Builder
+ b.WriteString(``)
+ for _, id := range pkgids {
+ b.WriteString(`/usr/bin/` + id + ``)
+ }
+ b.WriteString(``)
+ return []byte(b.String())
+}
+
+func gunzip(t *testing.T, b []byte) []byte {
+ t.Helper()
+ r, err := gzip.NewReader(bytes.NewReader(b))
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := io.ReadAll(r)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return out
+}
+
+func mergedFile(t *testing.T, repo *RPMRepo, dtype string) []byte {
+ t.Helper()
+ for href, body := range repo.Files {
+ if strings.HasSuffix(href, "-"+dtype+".xml.gz") {
+ return gunzip(t, body)
+ }
+ }
+ t.Fatalf("no %s file in merged repo", dtype)
+ return nil
+}
+
+func TestMergeRPMDedupePriority(t *testing.T) {
+ members := []RPMMember{
+ {RemoteName: "first", Timestamp: 100, Data: map[string][]byte{
+ "primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo-1.0.rpm")),
+ "filelists": filelistsXML("aaa"),
+ }},
+ {RemoteName: "second", Timestamp: 200, Data: map[string][]byte{
+ "primary": primaryXML(
+ primaryPkgXML("foo", "1.0", "bbb", "Packages/foo-1.0-rebuilt.rpm"),
+ primaryPkgXML("bar", "2.0", "ccc", "Packages/bar-2.0.rpm"),
+ ),
+ "filelists": filelistsXML("bbb", "ccc"),
+ }},
+ }
+ repo, err := MergeRPM(members)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ primary := string(mergedFile(t, repo, "primary"))
+ if strings.Count(primary, "foo") != 1 {
+ t.Fatalf("duplicate NEVRA not deduped:\n%s", primary)
+ }
+ if !strings.Contains(primary, `href="first/Packages/foo-1.0.rpm"`) || strings.Contains(primary, "foo-1.0-rebuilt") {
+ t.Fatalf("first member should win duplicate NEVRA:\n%s", primary)
+ }
+ if !strings.Contains(primary, `href="second/Packages/bar-2.0.rpm"`) {
+ t.Fatalf("unique package from second member missing:\n%s", primary)
+ }
+ if !strings.Contains(primary, `packages="2"`) {
+ t.Fatalf("package count wrong:\n%s", primary)
+ }
+
+ filelists := string(mergedFile(t, repo, "filelists"))
+ if !strings.Contains(filelists, "/usr/bin/aaa") || !strings.Contains(filelists, "/usr/bin/ccc") || strings.Contains(filelists, "/usr/bin/bbb") {
+ t.Fatalf("filelists must follow the winning primary entries:\n%s", filelists)
+ }
+
+ other := string(mergedFile(t, repo, "other"))
+ if !strings.Contains(other, `packages="0"`) {
+ t.Fatalf("members without other data should yield an empty other.xml:\n%s", other)
+ }
+
+ if !strings.Contains(string(repo.Repomd), "200") {
+ t.Fatalf("revision should be the newest member timestamp:\n%s", repo.Repomd)
+ }
+}
+
+func TestMergeRPMOutputIsWellFormed(t *testing.T) {
+ repo, err := MergeRPM([]RPMMember{{RemoteName: "a", Data: map[string][]byte{
+ "primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo.rpm")),
+ }}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var doc struct {
+ XMLName xml.Name
+ Packages []struct {
+ Name string `xml:"name"`
+ Provides []struct {
+ Name string `xml:"name,attr"`
+ } `xml:"format>provides>entry"`
+ } `xml:"package"`
+ }
+ if err := xml.Unmarshal(mergedFile(t, repo, "primary"), &doc); err != nil {
+ t.Fatal(err)
+ }
+ if doc.XMLName.Space != "http://linux.duke.edu/metadata/common" || len(doc.Packages) != 1 {
+ t.Fatalf("unexpected merged primary: %+v", doc)
+ }
+ if len(doc.Packages[0].Provides) != 1 || doc.Packages[0].Provides[0].Name != "foo" {
+ t.Fatalf("rpm: namespaced format lost: %+v", doc.Packages[0])
+ }
+}
+
+func TestMergeRPMHrefRewriting(t *testing.T) {
+ based := `extnoarch` +
+ `eee`
+ repo, err := MergeRPM([]RPMMember{{RemoteName: "gh", Data: map[string][]byte{
+ "primary": primaryXML(
+ primaryPkgXML("foo", "1.0", "aaa", "storytold/photocraft/releases/download/v1.0/foo&bar.rpm"),
+ primaryPkgXML("lead", "1.0", "bbb", "/Packages/lead.rpm"),
+ based,
+ ),
+ }}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ primary := string(mergedFile(t, repo, "primary"))
+ for _, want := range []string{
+ ``,
+ ``,
+ ``,
+ } {
+ if !strings.Contains(primary, want) {
+ t.Errorf("missing %s in:\n%s", want, primary)
+ }
+ }
+}
+
+func TestMergeRPMChecksums(t *testing.T) {
+ repo, err := MergeRPM([]RPMMember{{RemoteName: "a", Timestamp: 42, Data: map[string][]byte{
+ "primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo.rpm")),
+ }}})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var md struct {
+ Data []struct {
+ Type string `xml:"type,attr"`
+ Checksum string `xml:"checksum"`
+ OpenChecksum string `xml:"open-checksum"`
+ Location struct {
+ Href string `xml:"href,attr"`
+ } `xml:"location"`
+ Timestamp int64 `xml:"timestamp"`
+ Size int `xml:"size"`
+ OpenSize int `xml:"open-size"`
+ } `xml:"data"`
+ }
+ if err := xml.Unmarshal(repo.Repomd, &md); err != nil {
+ t.Fatal(err)
+ }
+ if len(md.Data) != 3 {
+ t.Fatalf("want primary/filelists/other, got %d entries", len(md.Data))
+ }
+ for _, d := range md.Data {
+ body, ok := repo.Files[d.Location.Href]
+ if !ok {
+ t.Fatalf("%s: location %q not served", d.Type, d.Location.Href)
+ }
+ sum := sha256.Sum256(body)
+ if hex.EncodeToString(sum[:]) != d.Checksum || len(body) != d.Size {
+ t.Errorf("%s: checksum/size do not match served bytes", d.Type)
+ }
+ open := gunzip(t, body)
+ osum := sha256.Sum256(open)
+ if hex.EncodeToString(osum[:]) != d.OpenChecksum || len(open) != d.OpenSize {
+ t.Errorf("%s: open-checksum/open-size do not match decompressed bytes", d.Type)
+ }
+ if d.Timestamp != 42 {
+ t.Errorf("%s: timestamp %d, want 42", d.Type, d.Timestamp)
+ }
+ }
+
+ again, _ := MergeRPM([]RPMMember{{RemoteName: "a", Timestamp: 42, Data: map[string][]byte{
+ "primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo.rpm")),
+ }}})
+ if !bytes.Equal(repo.Repomd, again.Repomd) {
+ t.Error("merge must be deterministic so repomd checksums stay valid across requests")
+ }
+}
+
+func TestParseRepomd(t *testing.T) {
+ locs, ts, err := parseRepomd([]byte(`
+ 10
+ 99
+ 20
+`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if locs["primary"] != "repodata/p-primary.xml.zst" || locs["other"] != "repodata/o-other.xml.gz" || len(locs) != 2 {
+ t.Fatalf("unexpected locations: %v", locs)
+ }
+ if ts != 20 {
+ t.Fatalf("timestamp %d, want 20 (sqlite entries ignored)", ts)
+ }
+ if _, _, err := parseRepomd([]byte(``)); err == nil {
+ t.Fatal("repomd without primary should error")
+ }
+}
+
+func TestDecompress(t *testing.T) {
+ plain := []byte("")
+
+ var xzBuf bytes.Buffer
+ xw, _ := xz.NewWriter(&xzBuf)
+ _, _ = xw.Write(plain)
+ _ = xw.Close()
+
+ zw, _ := zstd.NewWriter(nil)
+ zst := zw.EncodeAll(plain, nil)
+
+ for href, body := range map[string][]byte{
+ "p.xml": plain,
+ "p.xml.gz": gzipDeterministic(plain),
+ "p.xml.xz": xzBuf.Bytes(),
+ "p.xml.zst": zst,
+ } {
+ got, err := decompress(href, body)
+ if err != nil || !bytes.Equal(got, plain) {
+ t.Errorf("%s: got %q, %v", href, got, err)
+ }
+ }
+}