Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 173ffd0f22 |
@@ -31,12 +31,18 @@ steps:
|
|||||||
image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest
|
image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
|
HOST="https://artifactapi.k8s.syd1.au.unkin.net"
|
||||||
|
REPO="rpm-vendor-el8"
|
||||||
for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do
|
for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do
|
||||||
curl --user droneci:$${DRONECI_PASSWORD} --upload-file $rpm https://git.unkin.net/api/packages/unkin/rpm/almalinux/el8/upload
|
FILE=$$(basename "$$rpm")
|
||||||
|
# artifactapi has no HEAD route (405); probe the served Packages path.
|
||||||
|
code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true)
|
||||||
|
if [ "$$code" = "200" ]; then
|
||||||
|
echo "$$FILE already exists in $$REPO; skipping"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
curl -f -X PUT "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" -H "Content-Type: application/x-rpm" --data-binary @"$$rpm"
|
||||||
done
|
done
|
||||||
environment:
|
|
||||||
DRONECI_PASSWORD:
|
|
||||||
from_secret: DRONECI_PASSWORD
|
|
||||||
backend_options:
|
backend_options:
|
||||||
kubernetes:
|
kubernetes:
|
||||||
serviceAccountName: default
|
serviceAccountName: default
|
||||||
|
|||||||
@@ -31,12 +31,18 @@ steps:
|
|||||||
image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest
|
image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
|
HOST="https://artifactapi.k8s.syd1.au.unkin.net"
|
||||||
|
REPO="rpm-vendor-el9"
|
||||||
for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do
|
for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do
|
||||||
curl --user droneci:$${DRONECI_PASSWORD} --upload-file $rpm https://git.unkin.net/api/packages/unkin/rpm/almalinux/el9/upload
|
FILE=$$(basename "$$rpm")
|
||||||
|
# artifactapi has no HEAD route (405); probe the served Packages path.
|
||||||
|
code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true)
|
||||||
|
if [ "$$code" = "200" ]; then
|
||||||
|
echo "$$FILE already exists in $$REPO; skipping"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
curl -f -X PUT "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" -H "Content-Type: application/x-rpm" --data-binary @"$$rpm"
|
||||||
done
|
done
|
||||||
environment:
|
|
||||||
DRONECI_PASSWORD:
|
|
||||||
from_secret: DRONECI_PASSWORD
|
|
||||||
backend_options:
|
backend_options:
|
||||||
kubernetes:
|
kubernetes:
|
||||||
serviceAccountName: default
|
serviceAccountName: default
|
||||||
|
|||||||
@@ -31,12 +31,18 @@ steps:
|
|||||||
image: git.unkin.net/unkin/fedora42-rpmbuilder:latest
|
image: git.unkin.net/unkin/fedora42-rpmbuilder:latest
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
|
HOST="https://artifactapi.k8s.syd1.au.unkin.net"
|
||||||
|
REPO="rpm-vendor-f42"
|
||||||
for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do
|
for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do
|
||||||
curl --user droneci:$${DRONECI_PASSWORD} --upload-file $rpm https://git.unkin.net/api/packages/unkin/rpm/fedora/42/upload
|
FILE=$$(basename "$$rpm")
|
||||||
|
# artifactapi has no HEAD route (405); probe the served Packages path.
|
||||||
|
code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true)
|
||||||
|
if [ "$$code" = "200" ]; then
|
||||||
|
echo "$$FILE already exists in $$REPO; skipping"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
curl -f -X PUT "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" -H "Content-Type: application/x-rpm" --data-binary @"$$rpm"
|
||||||
done
|
done
|
||||||
environment:
|
|
||||||
DRONECI_PASSWORD:
|
|
||||||
from_secret: DRONECI_PASSWORD
|
|
||||||
backend_options:
|
backend_options:
|
||||||
kubernetes:
|
kubernetes:
|
||||||
serviceAccountName: default
|
serviceAccountName: default
|
||||||
|
|||||||
@@ -31,12 +31,18 @@ steps:
|
|||||||
image: git.unkin.net/unkin/fedora43-rpmbuilder:latest
|
image: git.unkin.net/unkin/fedora43-rpmbuilder:latest
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
|
HOST="https://artifactapi.k8s.syd1.au.unkin.net"
|
||||||
|
REPO="rpm-vendor-f43"
|
||||||
for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do
|
for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do
|
||||||
curl --user droneci:$${DRONECI_PASSWORD} --upload-file $rpm https://git.unkin.net/api/packages/unkin/rpm/fedora/43/upload
|
FILE=$$(basename "$$rpm")
|
||||||
|
# artifactapi has no HEAD route (405); probe the served Packages path.
|
||||||
|
code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true)
|
||||||
|
if [ "$$code" = "200" ]; then
|
||||||
|
echo "$$FILE already exists in $$REPO; skipping"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
curl -f -X PUT "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" -H "Content-Type: application/x-rpm" --data-binary @"$$rpm"
|
||||||
done
|
done
|
||||||
environment:
|
|
||||||
DRONECI_PASSWORD:
|
|
||||||
from_secret: DRONECI_PASSWORD
|
|
||||||
backend_options:
|
backend_options:
|
||||||
kubernetes:
|
kubernetes:
|
||||||
serviceAccountName: default
|
serviceAccountName: default
|
||||||
|
|||||||
@@ -31,12 +31,18 @@ steps:
|
|||||||
image: git.unkin.net/unkin/fedora44-rpmbuilder:latest
|
image: git.unkin.net/unkin/fedora44-rpmbuilder:latest
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
|
HOST="https://artifactapi.k8s.syd1.au.unkin.net"
|
||||||
|
REPO="rpm-vendor-f44"
|
||||||
for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do
|
for rpm in $(find /woodpecker/src/git.unkin.net/unkin/rpmbuilder/ -type f -name "*.rpm"); do
|
||||||
curl --user droneci:$${DRONECI_PASSWORD} --upload-file $rpm https://git.unkin.net/api/packages/unkin/rpm/fedora/44/upload
|
FILE=$$(basename "$$rpm")
|
||||||
|
# artifactapi has no HEAD route (405); probe the served Packages path.
|
||||||
|
code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true)
|
||||||
|
if [ "$$code" = "200" ]; then
|
||||||
|
echo "$$FILE already exists in $$REPO; skipping"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
curl -f -X PUT "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" -H "Content-Type: application/x-rpm" --data-binary @"$$rpm"
|
||||||
done
|
done
|
||||||
environment:
|
|
||||||
DRONECI_PASSWORD:
|
|
||||||
from_secret: DRONECI_PASSWORD
|
|
||||||
backend_options:
|
backend_options:
|
||||||
kubernetes:
|
kubernetes:
|
||||||
serviceAccountName: default
|
serviceAccountName: default
|
||||||
|
|||||||
+66
-44
@@ -63,6 +63,16 @@ def _make_session(retries: int = 3, backoff_factor: float = 0.5) -> requests.Ses
|
|||||||
# requests.Session is thread-safe for concurrent requests.
|
# requests.Session is thread-safe for concurrent requests.
|
||||||
_gitea_session = _make_session()
|
_gitea_session = _make_session()
|
||||||
_github_session = _make_session()
|
_github_session = _make_session()
|
||||||
|
_artifactapi_session = _make_session()
|
||||||
|
# artifactapi serves an internally-signed TLS cert. Verify against the OS CA
|
||||||
|
# bundle (which includes the internal CA, as dnf/curl in the builder images do)
|
||||||
|
# rather than certifi's Mozilla bundle, which does not trust it.
|
||||||
|
_artifactapi_session.verify = os.getenv('ARTIFACTAPI_CA_BUNDLE') or next(
|
||||||
|
(p for p in ('/etc/pki/tls/certs/ca-bundle.crt',
|
||||||
|
'/etc/ssl/certs/ca-certificates.crt')
|
||||||
|
if os.path.exists(p)),
|
||||||
|
True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ==================== VALIDATION SCHEMA ====================
|
# ==================== VALIDATION SCHEMA ====================
|
||||||
@@ -567,68 +577,77 @@ def effective_release(base_release: str, distro: str, use_dist_tag: bool) -> str
|
|||||||
return f"{base_release}.{tag}" if tag else base_release
|
return f"{base_release}.{tag}" if tag else base_release
|
||||||
|
|
||||||
|
|
||||||
def check_package_exists(package_name: str, version: str, release: str) -> bool:
|
def get_vendor_repo(distro: str) -> str:
|
||||||
"""
|
"""
|
||||||
Check if a package version exists in the Gitea package registry.
|
Map a distro path to its artifactapi rpm-vendor local repo name.
|
||||||
|
|
||||||
Distro disambiguation is handled by the caller via the release string:
|
Examples:
|
||||||
when dist_tag is enabled the caller appends the dist tag to release
|
'almalinux/el8' -> 'rpm-vendor-el8'
|
||||||
(e.g. '1.el9', '1.fc43') so each distro maps to a unique version entry.
|
'almalinux/el9' -> 'rpm-vendor-el9'
|
||||||
|
'fedora/42' -> 'rpm-vendor-f42'
|
||||||
|
"""
|
||||||
|
if distro.startswith('almalinux/'):
|
||||||
|
return 'rpm-vendor-' + distro.split('/', 1)[1]
|
||||||
|
if distro.startswith('fedora/'):
|
||||||
|
return 'rpm-vendor-f' + distro.split('/', 1)[1]
|
||||||
|
return 'rpm-vendor'
|
||||||
|
|
||||||
|
|
||||||
|
def get_rpm_arch(arch: str) -> str:
|
||||||
|
"""Map a metadata arch to the RPM architecture used in built filenames."""
|
||||||
|
return {'amd64': 'x86_64', 'arm64': 'aarch64'}.get(arch, arch)
|
||||||
|
|
||||||
|
|
||||||
|
def check_package_exists(
|
||||||
|
package_name: str,
|
||||||
|
version: str,
|
||||||
|
release: str,
|
||||||
|
distro: str,
|
||||||
|
arch: str = 'amd64',
|
||||||
|
) -> bool:
|
||||||
|
"""
|
||||||
|
Check if a built RPM already exists in the artifactapi rpm-vendor repo.
|
||||||
|
|
||||||
|
Each distro publishes to its own local repo (rpm-vendor-el9, rpm-vendor-f42,
|
||||||
|
...), and dist_tag disambiguation is carried in the release string (e.g.
|
||||||
|
'1.el9', '1.fc43'), so the served RPM filename is unique per distro. We probe
|
||||||
|
the served path under Packages/ with a GET (artifactapi has no HEAD route).
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
package_name: Name of the package
|
package_name: Name of the package
|
||||||
version: Version string
|
version: Version string
|
||||||
release: Effective release (may include dist tag suffix)
|
release: Effective release (may include dist tag suffix, e.g. '1.el9')
|
||||||
|
distro: Target distro (e.g. 'almalinux/el9', 'fedora/42')
|
||||||
|
arch: Package arch from metadata (default 'amd64')
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if package exists, False otherwise
|
True if the RPM is already published, False otherwise
|
||||||
"""
|
"""
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Get configuration from environment
|
base_url = os.getenv('ARTIFACTAPI_URL', 'https://artifactapi.k8s.syd1.au.unkin.net')
|
||||||
base_url = os.getenv('GITEA_URL', 'https://git.unkin.net')
|
repo = get_vendor_repo(distro)
|
||||||
owner = os.getenv('GITEA_OWNER', 'unkin')
|
rpm_file = f"{package_name}-{version}-{release}.{get_rpm_arch(arch)}.rpm"
|
||||||
package_type = os.getenv('GITEA_PACKAGE_TYPE', 'rpm')
|
url = f"{base_url}/api/v2/remotes/{repo}/files/Packages/{rpm_file}"
|
||||||
|
|
||||||
# Get Gitea token from Vault - fail hard if unavailable
|
|
||||||
gitea_token = get_gitea_token()
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Normalize version by removing leading zeros (Gitea does this automatically)
|
|
||||||
# e.g., "2025.08.03" becomes "2025.8.3"
|
|
||||||
normalized_version = normalize_version(version)
|
|
||||||
full_version = f"{normalized_version}-{release}"
|
|
||||||
url = (
|
|
||||||
f"{base_url}/api/v1/packages/{owner}/"
|
|
||||||
f"{package_type}/{package_name}/{full_version}"
|
|
||||||
)
|
|
||||||
|
|
||||||
_gitea_session.headers.update({'Authorization': f'token {gitea_token}'})
|
|
||||||
|
|
||||||
logger.debug(f"Checking package existence: {url}")
|
logger.debug(f"Checking package existence: {url}")
|
||||||
response = _gitea_session.get(url, timeout=10)
|
response = _artifactapi_session.get(url, timeout=10)
|
||||||
|
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
package_info = response.json()
|
logger.debug(f"Package {rpm_file} exists in {repo}")
|
||||||
exists = bool(package_info.get('id'))
|
return True
|
||||||
logger.debug(f"Package {package_name}:{full_version} {'exists' if exists else 'not found'}")
|
if response.status_code == 404:
|
||||||
return exists
|
logger.debug(f"Package {rpm_file} not found in {repo} (404)")
|
||||||
|
|
||||||
elif response.status_code == 404:
|
|
||||||
logger.debug(f"Package {package_name}:{full_version} not found (404)")
|
|
||||||
return False
|
return False
|
||||||
elif response.status_code == 401:
|
|
||||||
logger.error("Authentication failed. Check GITEA_API_TOKEN.")
|
|
||||||
return False
|
|
||||||
else:
|
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"Unexpected response checking package {package_name}:{full_version}: "
|
f"Unexpected response checking {rpm_file} in {repo}: "
|
||||||
f"{response.status_code} - {response.text}"
|
f"{response.status_code}"
|
||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
except requests.RequestException as e:
|
except requests.RequestException as e:
|
||||||
logger.error(f"Failed to check package {package_name}:{version}-{release}: {e}")
|
logger.error(f"Failed to check {rpm_file} in {repo}: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
@@ -1251,6 +1270,7 @@ class PackageInfo:
|
|||||||
directory: Path
|
directory: Path
|
||||||
distro: str = 'almalinux/el9'
|
distro: str = 'almalinux/el9'
|
||||||
base_image: str = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest"
|
base_image: str = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest"
|
||||||
|
arch: str = 'amd64'
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return f"{self.name}-{self.version}-{self.release} ({self.distro})"
|
return f"{self.name}-{self.version}-{self.release} ({self.distro})"
|
||||||
@@ -1328,7 +1348,7 @@ class Builder:
|
|||||||
# Use the first repository as the distro identifier
|
# Use the first repository as the distro identifier
|
||||||
build_distro = repositories[0] if repositories else 'unknown'
|
build_distro = repositories[0] if repositories else 'unknown'
|
||||||
rel = effective_release(str(release), build_distro, use_dist_tag)
|
rel = effective_release(str(release), build_distro, use_dist_tag)
|
||||||
packages.append(PackageInfo(package_name, version, rel, package_dir, build_distro, base_image))
|
packages.append(PackageInfo(package_name, version, rel, package_dir, build_distro, base_image, arch=metadata.get('arch', 'amd64')))
|
||||||
else:
|
else:
|
||||||
# Build for specific distro
|
# Build for specific distro
|
||||||
for build_config in build_configs:
|
for build_config in build_configs:
|
||||||
@@ -1341,7 +1361,7 @@ class Builder:
|
|||||||
# Check if the target distro matches any repository
|
# Check if the target distro matches any repository
|
||||||
if distro in repositories and base_image and version and release:
|
if distro in repositories and base_image and version and release:
|
||||||
rel = effective_release(str(release), distro, use_dist_tag)
|
rel = effective_release(str(release), distro, use_dist_tag)
|
||||||
packages.append(PackageInfo(package_name, version, rel, package_dir, distro, base_image))
|
packages.append(PackageInfo(package_name, version, rel, package_dir, distro, base_image, arch=metadata.get('arch', 'amd64')))
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
# If no matching distro found, log a warning
|
# If no matching distro found, log a warning
|
||||||
@@ -1440,7 +1460,7 @@ class Builder:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
rel = effective_release(release, distro, use_dist_tag)
|
rel = effective_release(release, distro, use_dist_tag)
|
||||||
package_info = PackageInfo(package, version, rel, package_dir, distro, base_image)
|
package_info = PackageInfo(package, version, rel, package_dir, distro, base_image, arch=metadata.get('arch', 'amd64'))
|
||||||
return self._build_package(package_info, dry_run, force, native, buildah)
|
return self._build_package(package_info, dry_run, force, native, buildah)
|
||||||
|
|
||||||
def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9', native: bool = False, buildah: bool = False) -> bool:
|
def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9', native: bool = False, buildah: bool = False) -> bool:
|
||||||
@@ -1522,7 +1542,9 @@ class Builder:
|
|||||||
if check_package_exists(
|
if check_package_exists(
|
||||||
package_info.name,
|
package_info.name,
|
||||||
package_info.version,
|
package_info.version,
|
||||||
package_info.release
|
package_info.release,
|
||||||
|
package_info.distro,
|
||||||
|
package_info.arch,
|
||||||
):
|
):
|
||||||
self.logger.info(
|
self.logger.info(
|
||||||
f"Skipping {package_info} (already exists in repository)"
|
f"Skipping {package_info} (already exists in repository)"
|
||||||
|
|||||||
Reference in New Issue
Block a user