Publish vendor RPMs to artifactapi rpm-vendor, one package per job
ci/woodpecker/pr/build-almalinux9/8 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/26 Pipeline was successful
ci/woodpecker/pr/build-fedora42/1 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/23 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/18 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/27 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/16 Pipeline was successful
ci/woodpecker/pr/build-fedora42/2 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/12 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/19 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/10 Pipeline was successful
ci/woodpecker/pr/build-fedora42/6 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/14 Pipeline was successful
ci/woodpecker/pr/build-fedora43/4 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/11 Pipeline was successful
ci/woodpecker/pr/build-fedora43/5 Pipeline was successful
ci/woodpecker/pr/build-fedora42/4 Pipeline was successful
ci/woodpecker/pr/build-fedora42/8 Pipeline was successful
ci/woodpecker/pr/build-fedora42/3 Pipeline was successful
ci/woodpecker/pr/build-fedora43/2 Pipeline was successful
ci/woodpecker/pr/build-fedora44/5 Pipeline was successful
ci/woodpecker/pr/build-fedora43/7 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/25 Pipeline was successful
ci/woodpecker/pr/build-fedora44/8 Pipeline was successful
ci/woodpecker/pr/build-fedora43/3 Pipeline failed
ci/woodpecker/pr/build-fedora43/1 Pipeline was successful
ci/woodpecker/pr/build-fedora42/7 Pipeline was successful
ci/woodpecker/pr/build-fedora43/8 Pipeline was successful
ci/woodpecker/pr/build-fedora44/1 Pipeline was successful
ci/woodpecker/pr/build-fedora43/6 Pipeline was successful
ci/woodpecker/pr/build-fedora44/6 Pipeline was successful
ci/woodpecker/pr/build-fedora44/4 Pipeline was successful
ci/woodpecker/pr/build-fedora44/3 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/20 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/22 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/11 Pipeline was successful
ci/woodpecker/pr/build-fedora42/5 Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/build-fedora44/7 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/17 Pipeline was successful
ci/woodpecker/pr/build-fedora44/2 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/13 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/6 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/1 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/12 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/5 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/21 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/22 Pipeline failed
ci/woodpecker/pr/build-almalinux8/10 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/20 Pipeline failed
ci/woodpecker/pr/build-almalinux8/9 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/19 Pipeline failed
ci/woodpecker/pr/build-almalinux8/24 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/15 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/16 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/3 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/7 Pipeline failed
ci/woodpecker/pr/build-almalinux8/4 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/2 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/18 Pipeline failed
ci/woodpecker/pr/build-almalinux8/8 Pipeline failed
ci/woodpecker/pr/build-almalinux9/3 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/2 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/1 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/26 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/23 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/6 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/13 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/15 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/27 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/17 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/7 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/5 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/4 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/24 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/14 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/9 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/25 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/21 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/8 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/26 Pipeline was successful
ci/woodpecker/pr/build-fedora42/1 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/23 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/18 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/27 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/16 Pipeline was successful
ci/woodpecker/pr/build-fedora42/2 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/12 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/19 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/10 Pipeline was successful
ci/woodpecker/pr/build-fedora42/6 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/14 Pipeline was successful
ci/woodpecker/pr/build-fedora43/4 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/11 Pipeline was successful
ci/woodpecker/pr/build-fedora43/5 Pipeline was successful
ci/woodpecker/pr/build-fedora42/4 Pipeline was successful
ci/woodpecker/pr/build-fedora42/8 Pipeline was successful
ci/woodpecker/pr/build-fedora42/3 Pipeline was successful
ci/woodpecker/pr/build-fedora43/2 Pipeline was successful
ci/woodpecker/pr/build-fedora44/5 Pipeline was successful
ci/woodpecker/pr/build-fedora43/7 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/25 Pipeline was successful
ci/woodpecker/pr/build-fedora44/8 Pipeline was successful
ci/woodpecker/pr/build-fedora43/3 Pipeline failed
ci/woodpecker/pr/build-fedora43/1 Pipeline was successful
ci/woodpecker/pr/build-fedora42/7 Pipeline was successful
ci/woodpecker/pr/build-fedora43/8 Pipeline was successful
ci/woodpecker/pr/build-fedora44/1 Pipeline was successful
ci/woodpecker/pr/build-fedora43/6 Pipeline was successful
ci/woodpecker/pr/build-fedora44/6 Pipeline was successful
ci/woodpecker/pr/build-fedora44/4 Pipeline was successful
ci/woodpecker/pr/build-fedora44/3 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/20 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/22 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/11 Pipeline was successful
ci/woodpecker/pr/build-fedora42/5 Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/build-fedora44/7 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/17 Pipeline was successful
ci/woodpecker/pr/build-fedora44/2 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/13 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/6 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/1 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/12 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/5 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/21 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/22 Pipeline failed
ci/woodpecker/pr/build-almalinux8/10 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/20 Pipeline failed
ci/woodpecker/pr/build-almalinux8/9 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/19 Pipeline failed
ci/woodpecker/pr/build-almalinux8/24 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/15 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/16 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/3 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/7 Pipeline failed
ci/woodpecker/pr/build-almalinux8/4 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/2 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/18 Pipeline failed
ci/woodpecker/pr/build-almalinux8/8 Pipeline failed
ci/woodpecker/pr/build-almalinux9/3 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/2 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/1 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/26 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/23 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/6 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/13 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/15 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/27 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/17 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/7 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/5 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/4 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/24 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/14 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/9 Pipeline was successful
ci/woodpecker/pr/build-almalinux8/25 Pipeline was successful
ci/woodpecker/pr/build-almalinux9/21 Pipeline was successful
Builds currently push to Gitea's RPM registry and run build-all in a single job. Publish instead to the artifactapi rpm-vendor local repos (real yum repos; repodata regenerates automatically) and give each vendor package its own job so failures are isolated and builds parallelise. Because builds no longer land in Gitea, the "skip already-built version" check now probes artifactapi. - tools/build: repoint check_package_exists from the Gitea packages API to the artifactapi rpm-vendor repo (GET the served Packages/<file>.rpm path). Add get_vendor_repo() (distro -> rpm-vendor-<suffix>) and get_rpm_arch() (amd64 -> x86_64), and carry arch on PackageInfo so the probed filename matches the built RPM. - .woodpecker: replace the per-distro build-all pipelines with per-distro matrix pipelines over PACKAGE, so each package builds in its own job. deploy-*.yaml (master push) build each package and PUT its RPM to the distro's rpm-vendor repo; build-*.yaml (pull_request) build as a check. - tools/gen-pipelines + `make pipelines`: generate the .woodpecker matrix pipelines from rpms/*/metadata.yaml (source of truth for target distros); regenerate after adding or removing packages.
This commit is contained in:
Executable
+147
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "pyyaml",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# vim: filetype=python
|
||||
|
||||
"""
|
||||
Generate Woodpecker pipelines for rpmbuilder.
|
||||
|
||||
One pipeline file per distro (build-<distro>.yaml for pull_request checks and
|
||||
deploy-<distro>.yaml for master pushes). Each pipeline uses a Woodpecker matrix
|
||||
over the packages that target that distro, so every vendor package builds in its
|
||||
own job. Deploy jobs publish to the distro's artifactapi rpm-vendor local repo.
|
||||
|
||||
The matrix package lists are derived from each rpms/*/metadata.yaml, so this
|
||||
script is the source of truth: run `make pipelines` (or ./tools/gen-pipelines)
|
||||
after adding/removing a package or changing its target distros, and commit the
|
||||
regenerated .woodpecker/*.yaml.
|
||||
"""
|
||||
|
||||
import glob
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
WOODPECKER = ROOT / ".woodpecker"
|
||||
|
||||
# Distro -> pipeline settings. `builder` runs buildah/the build tool (the actual
|
||||
# RPM build happens in a nested container using each package's metadata image);
|
||||
# `upload` only needs curl. rpm-vendor-<suffix> mirrors the artifactapi repos.
|
||||
DISTROS = [
|
||||
{"key": "almalinux8", "distro": "almalinux/el8", "repo": "rpm-vendor-el8"},
|
||||
{"key": "almalinux9", "distro": "almalinux/el9", "repo": "rpm-vendor-el9"},
|
||||
{"key": "fedora42", "distro": "fedora/42", "repo": "rpm-vendor-f42"},
|
||||
{"key": "fedora43", "distro": "fedora/43", "repo": "rpm-vendor-f43"},
|
||||
{"key": "fedora44", "distro": "fedora/44", "repo": "rpm-vendor-f44"},
|
||||
]
|
||||
|
||||
BUILDER_IMAGE = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest"
|
||||
UPLOAD_IMAGE = "git.unkin.net/unkin/almalinux9-base:latest"
|
||||
ARTIFACTAPI = "https://artifactapi.k8s.syd1.au.unkin.net"
|
||||
|
||||
HEADER = "# Generated by tools/gen-pipelines - do not edit by hand.\n"
|
||||
|
||||
|
||||
def packages_for(distro: str) -> list[str]:
|
||||
names = []
|
||||
for mf in sorted(glob.glob(str(ROOT / "rpms" / "*" / "metadata.yaml"))):
|
||||
meta = yaml.safe_load(open(mf)) or {}
|
||||
distros = {
|
||||
repo
|
||||
for build in meta.get("builds", [])
|
||||
for repo in build.get("repository", [])
|
||||
}
|
||||
if distro in distros:
|
||||
names.append(meta.get("name", Path(mf).parent.name))
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def matrix_block(packages: list[str]) -> str:
|
||||
lines = "\n".join(f" - {p}" for p in packages)
|
||||
return f"matrix:\n PACKAGE:\n{lines}\n"
|
||||
|
||||
|
||||
def build_step() -> str:
|
||||
return f""" - name: build
|
||||
image: {BUILDER_IMAGE}
|
||||
commands:
|
||||
- mkdir -p /woodpecker/rpms
|
||||
- ln -s /woodpecker/rpms /workspace
|
||||
- ./tools/build build --distro DISTRO --buildah "$PACKAGE"
|
||||
privileged: true
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
"""
|
||||
|
||||
|
||||
def upload_step(repo: str) -> str:
|
||||
return f""" - name: upload
|
||||
image: {UPLOAD_IMAGE}
|
||||
commands:
|
||||
- |
|
||||
HOST="{ARTIFACTAPI}"
|
||||
REPO="{repo}"
|
||||
for rpm in $(find dist -type f -name '*.rpm'); do
|
||||
FILE=$$(basename "$$rpm")
|
||||
# artifactapi has no HEAD route (405); probe the served Packages path.
|
||||
code=$$(curl -s -o /dev/null -w '%{{http_code}}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true)
|
||||
if [ "$$code" = "200" ]; then
|
||||
echo "$$FILE already exists in $$REPO (HTTP $$code); skipping upload"
|
||||
continue
|
||||
fi
|
||||
echo "Uploading $$FILE to $$REPO (existence probe returned $$code)"
|
||||
curl -f -X PUT \\
|
||||
"$$HOST/api/v2/remotes/$$REPO/files/$$FILE" \\
|
||||
-H "Content-Type: application/x-rpm" \\
|
||||
--data-binary @"$$rpm"
|
||||
done
|
||||
depends_on: [build]
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 128Mi
|
||||
cpu: 100m
|
||||
limits:
|
||||
memory: 512Mi
|
||||
cpu: 500m
|
||||
"""
|
||||
|
||||
|
||||
def render(distro_cfg: dict, deploy: bool) -> str:
|
||||
packages = packages_for(distro_cfg["distro"])
|
||||
if deploy:
|
||||
when = "when:\n - event: push\n branch: master\n"
|
||||
else:
|
||||
when = "when:\n - event: pull_request\n"
|
||||
body = HEADER + when + "\n" + matrix_block(packages) + "\nsteps:\n"
|
||||
body += build_step().replace("DISTRO", distro_cfg["distro"])
|
||||
if deploy:
|
||||
body += "\n" + upload_step(distro_cfg["repo"])
|
||||
return body
|
||||
|
||||
|
||||
def main() -> None:
|
||||
for cfg in DISTROS:
|
||||
(WOODPECKER / f"build-{cfg['key']}.yaml").write_text(render(cfg, deploy=False))
|
||||
(WOODPECKER / f"deploy-{cfg['key']}.yaml").write_text(render(cfg, deploy=True))
|
||||
print(f"generated build-{cfg['key']}.yaml deploy-{cfg['key']}.yaml "
|
||||
f"({len(packages_for(cfg['distro']))} packages)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user