From 591a6a2e7ad09417cf31f3b7219b4c0e99c27b30 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 18 Jul 2026 00:09:38 +1000 Subject: [PATCH] Publish vendor RPMs to artifactapi rpm-vendor, one package per job Builds currently push to Gitea's RPM registry and run build-all in a single job. Publish instead to the artifactapi rpm-vendor local repos (real yum repos; repodata regenerates automatically) and give each vendor package its own job so failures are isolated and builds parallelise. Because builds no longer land in Gitea, the "skip already-built version" check now probes artifactapi. - tools/build: repoint check_package_exists from the Gitea packages API to the artifactapi rpm-vendor repo (GET the served Packages/.rpm path). Add get_vendor_repo() (distro -> rpm-vendor-) and get_rpm_arch() (amd64 -> x86_64), and carry arch on PackageInfo so the probed filename matches the built RPM. - .woodpecker: replace the per-distro build-all pipelines with per-distro matrix pipelines over PACKAGE, so each package builds in its own job. deploy-*.yaml (master push) build each package and PUT its RPM to the distro's rpm-vendor repo; build-*.yaml (pull_request) build as a check. - tools/gen-pipelines + `make pipelines`: generate the .woodpecker matrix pipelines from rpms/*/metadata.yaml (source of truth for target distros); regenerate after adding or removing packages. --- .woodpecker/build-almalinux8.yaml | 84 +++++++++++++++-- .woodpecker/build-almalinux9.yaml | 85 +++++++++++++++-- .woodpecker/build-fedora42.yaml | 21 +++-- .woodpecker/build-fedora43.yaml | 21 +++-- .woodpecker/build-fedora44.yaml | 21 +++-- .woodpecker/deploy-almalinux8.yaml | 109 ++++++++++++++++++--- .woodpecker/deploy-almalinux9.yaml | 110 ++++++++++++++++++--- .woodpecker/deploy-fedora42.yaml | 48 ++++++---- .woodpecker/deploy-fedora43.yaml | 48 ++++++---- .woodpecker/deploy-fedora44.yaml | 48 ++++++---- Makefile | 7 +- tools/build | 107 ++++++++++++--------- tools/gen-pipelines | 147 +++++++++++++++++++++++++++++ 13 files changed, 695 insertions(+), 161 deletions(-) create mode 100755 tools/gen-pipelines diff --git a/.woodpecker/build-almalinux8.yaml b/.woodpecker/build-almalinux8.yaml index 0ae34c7..3dda3cb 100644 --- a/.woodpecker/build-almalinux8.yaml +++ b/.woodpecker/build-almalinux8.yaml @@ -1,13 +1,88 @@ +# Generated by tools/gen-pipelines - do not edit by hand. when: - event: pull_request +matrix: + PACKAGE: + - act_runner + - bind_exporter + - boilerplate + - claude-code + - cni-plugins + - code-server + - consul + - consul-cni + - etcd + - exportarr + - frr_exporter + - g10k + - git-delta + - hadolint + - helm + - helmfile + - jellyfin-ffmpeg-bin + - jellyfin-server + - jellyfin-web + - jsonnet + - jsonnet-language-server + - jsonnet-lint + - kubecolor + - kubeconform + - kubectx + - kustomize + - lazydocker + - libfoundationdb + - neovim + - neovim-glibc-2.17 + - nfpm + - node_exporter + - nomad + - nomad-autoscaler + - nzbget + - nzbget_exporter + - openbao-plugin-secret-consul + - openbao-plugin-secret-nomad + - openbao-plugins + - packer + - pgbouncer_exporter + - postgres_exporter + - puppet-initial + - puppetdb_exporter + - ruff + - sops + - stalwart + - stalwart-cli + - stalwart-foundationdb + - starship + - stern + - tea + - terraform + - terragrunt + - tflint + - unkin-ca-certificates + - unkin-undionly-kpxe + - unrar + - uv + - vals + - vault + - victoria-logs + - vlutils + - vmagent + - vmalert + - vminsert + - vmselect + - vmstorage + - vmutils + - xh + - yq + steps: - - name: build rpms + - name: build image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - mkdir -p /woodpecker/rpms - ln -s /woodpecker/rpms /workspace - - ./tools/build build-all --distro almalinux/el8 --buildah + - ./tools/build build --distro almalinux/el8 --buildah "$PACKAGE" privileged: true backend_options: kubernetes: @@ -19,8 +94,3 @@ steps: limits: memory: 2Gi cpu: 2 - - - name: show rpms - image: git.unkin.net/unkin/almalinux8-base:latest - commands: - - find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm" diff --git a/.woodpecker/build-almalinux9.yaml b/.woodpecker/build-almalinux9.yaml index 957dd9f..fb066cc 100644 --- a/.woodpecker/build-almalinux9.yaml +++ b/.woodpecker/build-almalinux9.yaml @@ -1,13 +1,89 @@ +# Generated by tools/gen-pipelines - do not edit by hand. when: - event: pull_request +matrix: + PACKAGE: + - act_runner + - bind_exporter + - boilerplate + - claude-code + - cni-plugins + - code-server + - consul + - consul-cni + - etcd + - exportarr + - frr_exporter + - g10k + - git-delta + - hadolint + - helm + - helmfile + - incus + - jellyfin-ffmpeg-bin + - jellyfin-server + - jellyfin-web + - jsonnet + - jsonnet-language-server + - jsonnet-lint + - kubecolor + - kubeconform + - kubectx + - kustomize + - lazydocker + - libfoundationdb + - neovim + - neovim-glibc-2.17 + - nfpm + - node_exporter + - nomad + - nomad-autoscaler + - nzbget + - nzbget_exporter + - openbao-plugin-secret-consul + - openbao-plugin-secret-nomad + - openbao-plugins + - packer + - pgbouncer_exporter + - postgres_exporter + - puppet-initial + - puppetdb_exporter + - ruff + - sops + - stalwart + - stalwart-cli + - stalwart-foundationdb + - starship + - stern + - tea + - terraform + - terragrunt + - tflint + - unkin-ca-certificates + - unkin-undionly-kpxe + - unrar + - uv + - vals + - vault + - victoria-logs + - vlutils + - vmagent + - vmalert + - vminsert + - vmselect + - vmstorage + - vmutils + - xh + - yq + steps: - - name: build rpms + - name: build image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - mkdir -p /woodpecker/rpms - ln -s /woodpecker/rpms /workspace - - ./tools/build build-all --distro almalinux/el9 --buildah + - ./tools/build build --distro almalinux/el9 --buildah "$PACKAGE" privileged: true backend_options: kubernetes: @@ -19,8 +95,3 @@ steps: limits: memory: 2Gi cpu: 2 - - - name: show rpms - image: git.unkin.net/unkin/almalinux9-base:latest - commands: - - find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm" diff --git a/.woodpecker/build-fedora42.yaml b/.woodpecker/build-fedora42.yaml index da40626..e44799a 100644 --- a/.woodpecker/build-fedora42.yaml +++ b/.woodpecker/build-fedora42.yaml @@ -1,13 +1,25 @@ +# Generated by tools/gen-pipelines - do not edit by hand. when: - event: pull_request +matrix: + PACKAGE: + - consul + - nfpm + - tea + - unkin-ca-certificates + - uv + - vault + - xh + - yq + steps: - - name: build rpms + - name: build image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - mkdir -p /woodpecker/rpms - ln -s /woodpecker/rpms /workspace - - ./tools/build build-all --distro fedora/42 --buildah + - ./tools/build build --distro fedora/42 --buildah "$PACKAGE" privileged: true backend_options: kubernetes: @@ -19,8 +31,3 @@ steps: limits: memory: 2Gi cpu: 2 - - - name: show rpms - image: git.unkin.net/unkin/fedora42-base:latest - commands: - - find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm" diff --git a/.woodpecker/build-fedora43.yaml b/.woodpecker/build-fedora43.yaml index aee2e70..b450571 100644 --- a/.woodpecker/build-fedora43.yaml +++ b/.woodpecker/build-fedora43.yaml @@ -1,13 +1,25 @@ +# Generated by tools/gen-pipelines - do not edit by hand. when: - event: pull_request +matrix: + PACKAGE: + - consul + - nfpm + - tea + - unkin-ca-certificates + - uv + - vault + - xh + - yq + steps: - - name: build rpms + - name: build image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - mkdir -p /woodpecker/rpms - ln -s /woodpecker/rpms /workspace - - ./tools/build build-all --distro fedora/43 --buildah + - ./tools/build build --distro fedora/43 --buildah "$PACKAGE" privileged: true backend_options: kubernetes: @@ -19,8 +31,3 @@ steps: limits: memory: 2Gi cpu: 2 - - - name: show rpms - image: git.unkin.net/unkin/fedora43-base:latest - commands: - - find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm" diff --git a/.woodpecker/build-fedora44.yaml b/.woodpecker/build-fedora44.yaml index 063ad13..3778339 100644 --- a/.woodpecker/build-fedora44.yaml +++ b/.woodpecker/build-fedora44.yaml @@ -1,13 +1,25 @@ +# Generated by tools/gen-pipelines - do not edit by hand. when: - event: pull_request +matrix: + PACKAGE: + - consul + - nfpm + - tea + - unkin-ca-certificates + - uv + - vault + - xh + - yq + steps: - - name: build rpms + - name: build image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - mkdir -p /woodpecker/rpms - ln -s /woodpecker/rpms /workspace - - ./tools/build build-all --distro fedora/44 --buildah + - ./tools/build build --distro fedora/44 --buildah "$PACKAGE" privileged: true backend_options: kubernetes: @@ -19,8 +31,3 @@ steps: limits: memory: 2Gi cpu: 2 - - - name: show rpms - image: git.unkin.net/unkin/fedora44-base:latest - commands: - - find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm" diff --git a/.woodpecker/deploy-almalinux8.yaml b/.woodpecker/deploy-almalinux8.yaml index 26de4fe..7a58fe1 100644 --- a/.woodpecker/deploy-almalinux8.yaml +++ b/.woodpecker/deploy-almalinux8.yaml @@ -1,14 +1,89 @@ +# Generated by tools/gen-pipelines - do not edit by hand. when: - event: push branch: master +matrix: + PACKAGE: + - act_runner + - bind_exporter + - boilerplate + - claude-code + - cni-plugins + - code-server + - consul + - consul-cni + - etcd + - exportarr + - frr_exporter + - g10k + - git-delta + - hadolint + - helm + - helmfile + - jellyfin-ffmpeg-bin + - jellyfin-server + - jellyfin-web + - jsonnet + - jsonnet-language-server + - jsonnet-lint + - kubecolor + - kubeconform + - kubectx + - kustomize + - lazydocker + - libfoundationdb + - neovim + - neovim-glibc-2.17 + - nfpm + - node_exporter + - nomad + - nomad-autoscaler + - nzbget + - nzbget_exporter + - openbao-plugin-secret-consul + - openbao-plugin-secret-nomad + - openbao-plugins + - packer + - pgbouncer_exporter + - postgres_exporter + - puppet-initial + - puppetdb_exporter + - ruff + - sops + - stalwart + - stalwart-cli + - stalwart-foundationdb + - starship + - stern + - tea + - terraform + - terragrunt + - tflint + - unkin-ca-certificates + - unkin-undionly-kpxe + - unrar + - uv + - vals + - vault + - victoria-logs + - vlutils + - vmagent + - vmalert + - vminsert + - vmselect + - vmstorage + - vmutils + - xh + - yq + steps: - - name: build-rpms + - name: build image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - mkdir -p /woodpecker/rpms - ln -s /woodpecker/rpms /workspace - - ./tools/build build-all --distro almalinux/el8 --buildah + - ./tools/build build --distro almalinux/el8 --buildah "$PACKAGE" privileged: true backend_options: kubernetes: @@ -21,22 +96,27 @@ steps: memory: 2Gi cpu: 2 - - name: show-rpms + - name: upload image: git.unkin.net/unkin/almalinux9-base:latest - commands: - - find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm" - depends_on: [build-rpms] - - - name: deploy-rpms - image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - | - for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do - curl --user droneci:$${DRONECI_PASSWORD} --upload-file $rpm https://git.unkin.net/api/packages/unkin/rpm/almalinux/el8/upload + HOST="https://artifactapi.k8s.syd1.au.unkin.net" + REPO="rpm-vendor-el8" + for rpm in $(find dist -type f -name '*.rpm'); do + FILE=$$(basename "$$rpm") + # artifactapi has no HEAD route (405); probe the served Packages path. + code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true) + if [ "$$code" = "200" ]; then + echo "$$FILE already exists in $$REPO (HTTP $$code); skipping upload" + continue + fi + echo "Uploading $$FILE to $$REPO (existence probe returned $$code)" + curl -f -X PUT \ + "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" \ + -H "Content-Type: application/x-rpm" \ + --data-binary @"$$rpm" done - environment: - DRONECI_PASSWORD: - from_secret: DRONECI_PASSWORD + depends_on: [build] backend_options: kubernetes: serviceAccountName: default @@ -47,4 +127,3 @@ steps: limits: memory: 512Mi cpu: 500m - depends_on: [build-rpms, show-rpms] diff --git a/.woodpecker/deploy-almalinux9.yaml b/.woodpecker/deploy-almalinux9.yaml index 9b4d89a..5d21762 100644 --- a/.woodpecker/deploy-almalinux9.yaml +++ b/.woodpecker/deploy-almalinux9.yaml @@ -1,14 +1,90 @@ +# Generated by tools/gen-pipelines - do not edit by hand. when: - event: push branch: master +matrix: + PACKAGE: + - act_runner + - bind_exporter + - boilerplate + - claude-code + - cni-plugins + - code-server + - consul + - consul-cni + - etcd + - exportarr + - frr_exporter + - g10k + - git-delta + - hadolint + - helm + - helmfile + - incus + - jellyfin-ffmpeg-bin + - jellyfin-server + - jellyfin-web + - jsonnet + - jsonnet-language-server + - jsonnet-lint + - kubecolor + - kubeconform + - kubectx + - kustomize + - lazydocker + - libfoundationdb + - neovim + - neovim-glibc-2.17 + - nfpm + - node_exporter + - nomad + - nomad-autoscaler + - nzbget + - nzbget_exporter + - openbao-plugin-secret-consul + - openbao-plugin-secret-nomad + - openbao-plugins + - packer + - pgbouncer_exporter + - postgres_exporter + - puppet-initial + - puppetdb_exporter + - ruff + - sops + - stalwart + - stalwart-cli + - stalwart-foundationdb + - starship + - stern + - tea + - terraform + - terragrunt + - tflint + - unkin-ca-certificates + - unkin-undionly-kpxe + - unrar + - uv + - vals + - vault + - victoria-logs + - vlutils + - vmagent + - vmalert + - vminsert + - vmselect + - vmstorage + - vmutils + - xh + - yq + steps: - - name: build-rpms + - name: build image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - mkdir -p /woodpecker/rpms - ln -s /woodpecker/rpms /workspace - - ./tools/build build-all --distro almalinux/el9 --buildah + - ./tools/build build --distro almalinux/el9 --buildah "$PACKAGE" privileged: true backend_options: kubernetes: @@ -21,22 +97,27 @@ steps: memory: 2Gi cpu: 2 - - name: show-rpms + - name: upload image: git.unkin.net/unkin/almalinux9-base:latest - commands: - - find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm" - depends_on: [build-rpms] - - - name: deploy-rpms - image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - | - for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do - curl --user droneci:$${DRONECI_PASSWORD} --upload-file $rpm https://git.unkin.net/api/packages/unkin/rpm/almalinux/el9/upload + HOST="https://artifactapi.k8s.syd1.au.unkin.net" + REPO="rpm-vendor-el9" + for rpm in $(find dist -type f -name '*.rpm'); do + FILE=$$(basename "$$rpm") + # artifactapi has no HEAD route (405); probe the served Packages path. + code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true) + if [ "$$code" = "200" ]; then + echo "$$FILE already exists in $$REPO (HTTP $$code); skipping upload" + continue + fi + echo "Uploading $$FILE to $$REPO (existence probe returned $$code)" + curl -f -X PUT \ + "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" \ + -H "Content-Type: application/x-rpm" \ + --data-binary @"$$rpm" done - environment: - DRONECI_PASSWORD: - from_secret: DRONECI_PASSWORD + depends_on: [build] backend_options: kubernetes: serviceAccountName: default @@ -47,4 +128,3 @@ steps: limits: memory: 512Mi cpu: 500m - depends_on: [build-rpms, show-rpms] diff --git a/.woodpecker/deploy-fedora42.yaml b/.woodpecker/deploy-fedora42.yaml index 6726501..213b2f5 100644 --- a/.woodpecker/deploy-fedora42.yaml +++ b/.woodpecker/deploy-fedora42.yaml @@ -1,14 +1,26 @@ +# Generated by tools/gen-pipelines - do not edit by hand. when: - event: push branch: master +matrix: + PACKAGE: + - consul + - nfpm + - tea + - unkin-ca-certificates + - uv + - vault + - xh + - yq + steps: - - name: build-rpms + - name: build image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - mkdir -p /woodpecker/rpms - ln -s /woodpecker/rpms /workspace - - ./tools/build build-all --distro fedora/42 --buildah + - ./tools/build build --distro fedora/42 --buildah "$PACKAGE" privileged: true backend_options: kubernetes: @@ -21,22 +33,27 @@ steps: memory: 2Gi cpu: 2 - - name: show-rpms - image: git.unkin.net/unkin/fedora42-rpmbuilder:latest - commands: - - find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm" - depends_on: [build-rpms] - - - name: deploy-rpms - image: git.unkin.net/unkin/fedora42-rpmbuilder:latest + - name: upload + image: git.unkin.net/unkin/almalinux9-base:latest commands: - | - for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do - curl --user droneci:$${DRONECI_PASSWORD} --upload-file $rpm https://git.unkin.net/api/packages/unkin/rpm/fedora/42/upload + HOST="https://artifactapi.k8s.syd1.au.unkin.net" + REPO="rpm-vendor-f42" + for rpm in $(find dist -type f -name '*.rpm'); do + FILE=$$(basename "$$rpm") + # artifactapi has no HEAD route (405); probe the served Packages path. + code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true) + if [ "$$code" = "200" ]; then + echo "$$FILE already exists in $$REPO (HTTP $$code); skipping upload" + continue + fi + echo "Uploading $$FILE to $$REPO (existence probe returned $$code)" + curl -f -X PUT \ + "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" \ + -H "Content-Type: application/x-rpm" \ + --data-binary @"$$rpm" done - environment: - DRONECI_PASSWORD: - from_secret: DRONECI_PASSWORD + depends_on: [build] backend_options: kubernetes: serviceAccountName: default @@ -47,4 +64,3 @@ steps: limits: memory: 512Mi cpu: 500m - depends_on: [build-rpms, show-rpms] diff --git a/.woodpecker/deploy-fedora43.yaml b/.woodpecker/deploy-fedora43.yaml index be50719..941682b 100644 --- a/.woodpecker/deploy-fedora43.yaml +++ b/.woodpecker/deploy-fedora43.yaml @@ -1,14 +1,26 @@ +# Generated by tools/gen-pipelines - do not edit by hand. when: - event: push branch: master +matrix: + PACKAGE: + - consul + - nfpm + - tea + - unkin-ca-certificates + - uv + - vault + - xh + - yq + steps: - - name: build-rpms + - name: build image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - mkdir -p /woodpecker/rpms - ln -s /woodpecker/rpms /workspace - - ./tools/build build-all --distro fedora/43 --buildah + - ./tools/build build --distro fedora/43 --buildah "$PACKAGE" privileged: true backend_options: kubernetes: @@ -21,22 +33,27 @@ steps: memory: 2Gi cpu: 2 - - name: show-rpms - image: git.unkin.net/unkin/fedora43-rpmbuilder:latest - commands: - - find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm" - depends_on: [build-rpms] - - - name: deploy-rpms - image: git.unkin.net/unkin/fedora43-rpmbuilder:latest + - name: upload + image: git.unkin.net/unkin/almalinux9-base:latest commands: - | - for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do - curl --user droneci:$${DRONECI_PASSWORD} --upload-file $rpm https://git.unkin.net/api/packages/unkin/rpm/fedora/43/upload + HOST="https://artifactapi.k8s.syd1.au.unkin.net" + REPO="rpm-vendor-f43" + for rpm in $(find dist -type f -name '*.rpm'); do + FILE=$$(basename "$$rpm") + # artifactapi has no HEAD route (405); probe the served Packages path. + code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true) + if [ "$$code" = "200" ]; then + echo "$$FILE already exists in $$REPO (HTTP $$code); skipping upload" + continue + fi + echo "Uploading $$FILE to $$REPO (existence probe returned $$code)" + curl -f -X PUT \ + "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" \ + -H "Content-Type: application/x-rpm" \ + --data-binary @"$$rpm" done - environment: - DRONECI_PASSWORD: - from_secret: DRONECI_PASSWORD + depends_on: [build] backend_options: kubernetes: serviceAccountName: default @@ -47,4 +64,3 @@ steps: limits: memory: 512Mi cpu: 500m - depends_on: [build-rpms, show-rpms] diff --git a/.woodpecker/deploy-fedora44.yaml b/.woodpecker/deploy-fedora44.yaml index 42d4c45..93c1ff2 100644 --- a/.woodpecker/deploy-fedora44.yaml +++ b/.woodpecker/deploy-fedora44.yaml @@ -1,14 +1,26 @@ +# Generated by tools/gen-pipelines - do not edit by hand. when: - event: push branch: master +matrix: + PACKAGE: + - consul + - nfpm + - tea + - unkin-ca-certificates + - uv + - vault + - xh + - yq + steps: - - name: build-rpms + - name: build image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest commands: - mkdir -p /woodpecker/rpms - ln -s /woodpecker/rpms /workspace - - ./tools/build build-all --distro fedora/44 --buildah + - ./tools/build build --distro fedora/44 --buildah "$PACKAGE" privileged: true backend_options: kubernetes: @@ -21,22 +33,27 @@ steps: memory: 2Gi cpu: 2 - - name: show-rpms - image: git.unkin.net/unkin/fedora44-rpmbuilder:latest - commands: - - find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm" - depends_on: [build-rpms] - - - name: deploy-rpms - image: git.unkin.net/unkin/fedora44-rpmbuilder:latest + - name: upload + image: git.unkin.net/unkin/almalinux9-base:latest commands: - | - for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do - curl --user droneci:$${DRONECI_PASSWORD} --upload-file $rpm https://git.unkin.net/api/packages/unkin/rpm/fedora/44/upload + HOST="https://artifactapi.k8s.syd1.au.unkin.net" + REPO="rpm-vendor-f44" + for rpm in $(find dist -type f -name '*.rpm'); do + FILE=$$(basename "$$rpm") + # artifactapi has no HEAD route (405); probe the served Packages path. + code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true) + if [ "$$code" = "200" ]; then + echo "$$FILE already exists in $$REPO (HTTP $$code); skipping upload" + continue + fi + echo "Uploading $$FILE to $$REPO (existence probe returned $$code)" + curl -f -X PUT \ + "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" \ + -H "Content-Type: application/x-rpm" \ + --data-binary @"$$rpm" done - environment: - DRONECI_PASSWORD: - from_secret: DRONECI_PASSWORD + depends_on: [build] backend_options: kubernetes: serviceAccountName: default @@ -47,4 +64,3 @@ steps: limits: memory: 512Mi cpu: 500m - depends_on: [build-rpms, show-rpms] diff --git a/Makefile b/Makefile index bc97295..ff70173 100644 --- a/Makefile +++ b/Makefile @@ -11,9 +11,14 @@ DISTRO ?= almalinux/el9 PACKAGES := $(shell find $(ROOT_DIR)/rpms -mindepth 1 -maxdepth 1 -type d -exec test -f {}/metadata.yaml \; -print | xargs -n1 basename | sort) # Default target to build all packages -.PHONY: all list build clean test +.PHONY: all list build clean test pipelines all: build-all +# Regenerate the per-distro Woodpecker matrix pipelines from rpms/*/metadata.yaml +pipelines: + @echo "Regenerating .woodpecker pipelines..." + $(ROOT_DIR)/tools/gen-pipelines + # List all available packages list: @echo "Available packages:" diff --git a/tools/build b/tools/build index f4f4c70..e40cb0d 100755 --- a/tools/build +++ b/tools/build @@ -63,6 +63,7 @@ def _make_session(retries: int = 3, backoff_factor: float = 0.5) -> requests.Ses # requests.Session is thread-safe for concurrent requests. _gitea_session = _make_session() _github_session = _make_session() +_artifactapi_session = _make_session() # ==================== VALIDATION SCHEMA ==================== @@ -567,68 +568,77 @@ def effective_release(base_release: str, distro: str, use_dist_tag: bool) -> str return f"{base_release}.{tag}" if tag else base_release -def check_package_exists(package_name: str, version: str, release: str) -> bool: +def get_vendor_repo(distro: str) -> str: """ - Check if a package version exists in the Gitea package registry. + Map a distro path to its artifactapi rpm-vendor local repo name. - Distro disambiguation is handled by the caller via the release string: - when dist_tag is enabled the caller appends the dist tag to release - (e.g. '1.el9', '1.fc43') so each distro maps to a unique version entry. + Examples: + 'almalinux/el8' -> 'rpm-vendor-el8' + 'almalinux/el9' -> 'rpm-vendor-el9' + 'fedora/42' -> 'rpm-vendor-f42' + """ + if distro.startswith('almalinux/'): + return 'rpm-vendor-' + distro.split('/', 1)[1] + if distro.startswith('fedora/'): + return 'rpm-vendor-f' + distro.split('/', 1)[1] + return 'rpm-vendor' + + +def get_rpm_arch(arch: str) -> str: + """Map a metadata arch to the RPM architecture used in built filenames.""" + return {'amd64': 'x86_64', 'arm64': 'aarch64'}.get(arch, arch) + + +def check_package_exists( + package_name: str, + version: str, + release: str, + distro: str, + arch: str = 'amd64', +) -> bool: + """ + Check if a built RPM already exists in the artifactapi rpm-vendor repo. + + Each distro publishes to its own local repo (rpm-vendor-el9, rpm-vendor-f42, + ...), and dist_tag disambiguation is carried in the release string (e.g. + '1.el9', '1.fc43'), so the served RPM filename is unique per distro. We probe + the served path under Packages/ with a GET (artifactapi has no HEAD route). Args: package_name: Name of the package version: Version string - release: Effective release (may include dist tag suffix) + release: Effective release (may include dist tag suffix, e.g. '1.el9') + distro: Target distro (e.g. 'almalinux/el9', 'fedora/42') + arch: Package arch from metadata (default 'amd64') Returns: - True if package exists, False otherwise + True if the RPM is already published, False otherwise """ logger = logging.getLogger(__name__) - # Get configuration from environment - base_url = os.getenv('GITEA_URL', 'https://git.unkin.net') - owner = os.getenv('GITEA_OWNER', 'unkin') - package_type = os.getenv('GITEA_PACKAGE_TYPE', 'rpm') - - # Get Gitea token from Vault - fail hard if unavailable - gitea_token = get_gitea_token() + base_url = os.getenv('ARTIFACTAPI_URL', 'https://artifactapi.k8s.syd1.au.unkin.net') + repo = get_vendor_repo(distro) + rpm_file = f"{package_name}-{version}-{release}.{get_rpm_arch(arch)}.rpm" + url = f"{base_url}/api/v2/remotes/{repo}/files/Packages/{rpm_file}" try: - # Normalize version by removing leading zeros (Gitea does this automatically) - # e.g., "2025.08.03" becomes "2025.8.3" - normalized_version = normalize_version(version) - full_version = f"{normalized_version}-{release}" - url = ( - f"{base_url}/api/v1/packages/{owner}/" - f"{package_type}/{package_name}/{full_version}" - ) - - _gitea_session.headers.update({'Authorization': f'token {gitea_token}'}) - logger.debug(f"Checking package existence: {url}") - response = _gitea_session.get(url, timeout=10) + response = _artifactapi_session.get(url, timeout=10) if response.status_code == 200: - package_info = response.json() - exists = bool(package_info.get('id')) - logger.debug(f"Package {package_name}:{full_version} {'exists' if exists else 'not found'}") - return exists - - elif response.status_code == 404: - logger.debug(f"Package {package_name}:{full_version} not found (404)") - return False - elif response.status_code == 401: - logger.error("Authentication failed. Check GITEA_API_TOKEN.") - return False - else: - logger.warning( - f"Unexpected response checking package {package_name}:{full_version}: " - f"{response.status_code} - {response.text}" - ) + logger.debug(f"Package {rpm_file} exists in {repo}") + return True + if response.status_code == 404: + logger.debug(f"Package {rpm_file} not found in {repo} (404)") return False + logger.warning( + f"Unexpected response checking {rpm_file} in {repo}: " + f"{response.status_code}" + ) + return False except requests.RequestException as e: - logger.error(f"Failed to check package {package_name}:{version}-{release}: {e}") + logger.error(f"Failed to check {rpm_file} in {repo}: {e}") return False @@ -1251,6 +1261,7 @@ class PackageInfo: directory: Path distro: str = 'almalinux/el9' base_image: str = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest" + arch: str = 'amd64' def __str__(self): return f"{self.name}-{self.version}-{self.release} ({self.distro})" @@ -1328,7 +1339,7 @@ class Builder: # Use the first repository as the distro identifier build_distro = repositories[0] if repositories else 'unknown' rel = effective_release(str(release), build_distro, use_dist_tag) - packages.append(PackageInfo(package_name, version, rel, package_dir, build_distro, base_image)) + packages.append(PackageInfo(package_name, version, rel, package_dir, build_distro, base_image, arch=metadata.get('arch', 'amd64'))) else: # Build for specific distro for build_config in build_configs: @@ -1341,7 +1352,7 @@ class Builder: # Check if the target distro matches any repository if distro in repositories and base_image and version and release: rel = effective_release(str(release), distro, use_dist_tag) - packages.append(PackageInfo(package_name, version, rel, package_dir, distro, base_image)) + packages.append(PackageInfo(package_name, version, rel, package_dir, distro, base_image, arch=metadata.get('arch', 'amd64'))) break else: # If no matching distro found, log a warning @@ -1440,7 +1451,7 @@ class Builder: return False rel = effective_release(release, distro, use_dist_tag) - package_info = PackageInfo(package, version, rel, package_dir, distro, base_image) + package_info = PackageInfo(package, version, rel, package_dir, distro, base_image, arch=metadata.get('arch', 'amd64')) return self._build_package(package_info, dry_run, force, native, buildah) def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9', native: bool = False, buildah: bool = False) -> bool: @@ -1522,7 +1533,9 @@ class Builder: if check_package_exists( package_info.name, package_info.version, - package_info.release + package_info.release, + package_info.distro, + package_info.arch, ): self.logger.info( f"Skipping {package_info} (already exists in repository)" diff --git a/tools/gen-pipelines b/tools/gen-pipelines new file mode 100755 index 0000000..de36cb7 --- /dev/null +++ b/tools/gen-pipelines @@ -0,0 +1,147 @@ +#!/usr/bin/env -S uv run --script +# /// script +# dependencies = [ +# "pyyaml", +# ] +# /// + +# vim: filetype=python + +""" +Generate Woodpecker pipelines for rpmbuilder. + +One pipeline file per distro (build-.yaml for pull_request checks and +deploy-.yaml for master pushes). Each pipeline uses a Woodpecker matrix +over the packages that target that distro, so every vendor package builds in its +own job. Deploy jobs publish to the distro's artifactapi rpm-vendor local repo. + +The matrix package lists are derived from each rpms/*/metadata.yaml, so this +script is the source of truth: run `make pipelines` (or ./tools/gen-pipelines) +after adding/removing a package or changing its target distros, and commit the +regenerated .woodpecker/*.yaml. +""" + +import glob +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parent.parent +WOODPECKER = ROOT / ".woodpecker" + +# Distro -> pipeline settings. `builder` runs buildah/the build tool (the actual +# RPM build happens in a nested container using each package's metadata image); +# `upload` only needs curl. rpm-vendor- mirrors the artifactapi repos. +DISTROS = [ + {"key": "almalinux8", "distro": "almalinux/el8", "repo": "rpm-vendor-el8"}, + {"key": "almalinux9", "distro": "almalinux/el9", "repo": "rpm-vendor-el9"}, + {"key": "fedora42", "distro": "fedora/42", "repo": "rpm-vendor-f42"}, + {"key": "fedora43", "distro": "fedora/43", "repo": "rpm-vendor-f43"}, + {"key": "fedora44", "distro": "fedora/44", "repo": "rpm-vendor-f44"}, +] + +BUILDER_IMAGE = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest" +UPLOAD_IMAGE = "git.unkin.net/unkin/almalinux9-base:latest" +ARTIFACTAPI = "https://artifactapi.k8s.syd1.au.unkin.net" + +HEADER = "# Generated by tools/gen-pipelines - do not edit by hand.\n" + + +def packages_for(distro: str) -> list[str]: + names = [] + for mf in sorted(glob.glob(str(ROOT / "rpms" / "*" / "metadata.yaml"))): + meta = yaml.safe_load(open(mf)) or {} + distros = { + repo + for build in meta.get("builds", []) + for repo in build.get("repository", []) + } + if distro in distros: + names.append(meta.get("name", Path(mf).parent.name)) + return sorted(names) + + +def matrix_block(packages: list[str]) -> str: + lines = "\n".join(f" - {p}" for p in packages) + return f"matrix:\n PACKAGE:\n{lines}\n" + + +def build_step() -> str: + return f""" - name: build + image: {BUILDER_IMAGE} + commands: + - mkdir -p /woodpecker/rpms + - ln -s /woodpecker/rpms /workspace + - ./tools/build build --distro DISTRO --buildah "$PACKAGE" + privileged: true + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 512Mi + cpu: 1 + limits: + memory: 2Gi + cpu: 2 +""" + + +def upload_step(repo: str) -> str: + return f""" - name: upload + image: {UPLOAD_IMAGE} + commands: + - | + HOST="{ARTIFACTAPI}" + REPO="{repo}" + for rpm in $(find dist -type f -name '*.rpm'); do + FILE=$$(basename "$$rpm") + # artifactapi has no HEAD route (405); probe the served Packages path. + code=$$(curl -s -o /dev/null -w '%{{http_code}}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true) + if [ "$$code" = "200" ]; then + echo "$$FILE already exists in $$REPO (HTTP $$code); skipping upload" + continue + fi + echo "Uploading $$FILE to $$REPO (existence probe returned $$code)" + curl -f -X PUT \\ + "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" \\ + -H "Content-Type: application/x-rpm" \\ + --data-binary @"$$rpm" + done + depends_on: [build] + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 512Mi + cpu: 500m +""" + + +def render(distro_cfg: dict, deploy: bool) -> str: + packages = packages_for(distro_cfg["distro"]) + if deploy: + when = "when:\n - event: push\n branch: master\n" + else: + when = "when:\n - event: pull_request\n" + body = HEADER + when + "\n" + matrix_block(packages) + "\nsteps:\n" + body += build_step().replace("DISTRO", distro_cfg["distro"]) + if deploy: + body += "\n" + upload_step(distro_cfg["repo"]) + return body + + +def main() -> None: + for cfg in DISTROS: + (WOODPECKER / f"build-{cfg['key']}.yaml").write_text(render(cfg, deploy=False)) + (WOODPECKER / f"deploy-{cfg['key']}.yaml").write_text(render(cfg, deploy=True)) + print(f"generated build-{cfg['key']}.yaml deploy-{cfg['key']}.yaml " + f"({len(packages_for(cfg['distro']))} packages)") + + +if __name__ == "__main__": + main()