feat: migrate to woodpeckerci
Build / build-8 (pull_request) Successful in 10s
Build / build-9 (pull_request) Successful in 11s
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/build-almalinux9 Pipeline was successful
ci/woodpecker/pr/build-almalinux8 Pipeline was successful

- update build tool for kubernetes auth
- update build tool to build packages without docker (native + buildah)
- add woodpecker pre-commit and build jobs
This commit is contained in:
2026-03-07 11:19:13 +11:00
parent cede57a565
commit 1657c74b52
6 changed files with 516 additions and 43 deletions
+20
View File
@@ -0,0 +1,20 @@
when:
- event: pull_request
steps:
- name: build rpms
image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest
commands:
- mkdir -p /woodpecker/rpms /apps/
- ln -s /woodpecker/rpms /apps/dist
- dnf install buildah -y
- ./tools/build build-all --distro almalinux/el8 --buildah
privileged: true
backend_options:
kubernetes:
serviceAccountName: default
- name: show rpms
image: git.unkin.net/unkin/almalinux8-base:latest
commands:
- find /woodpecker/rpms -type f -name "*.rpm"
+20
View File
@@ -0,0 +1,20 @@
when:
- event: pull_request
steps:
- name: build rpms
image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest
commands:
- mkdir -p /woodpecker/rpms /apps/
- ln -s /woodpecker/rpms /apps/dist
- dnf install buildah -y
- ./tools/build build-all --distro almalinux/el9 --buildah
privileged: true
backend_options:
kubernetes:
serviceAccountName: default
- name: show rpms
image: git.unkin.net/unkin/almalinux9-base:latest
commands:
- find /woodpecker/rpms -type f -name "*.rpm"
+9
View File
@@ -0,0 +1,9 @@
when:
- event: pull_request
steps:
- name: pre-commit
image: git.unkin.net/unkin/almalinux9-base:latest
commands:
- dnf install uv make -y
- uvx pre-commit run --all-files
+14
View File
@@ -3,6 +3,10 @@ ROOT_DIR := $(PWD)
BUILD_TOOL := $(ROOT_DIR)/tools/build
DISTRO ?= almalinux/el9
# Authentication variables (optional)
# VAULT_ROLE_ID - Use AppRole authentication if set
# VAULT_ROLE - Kubernetes role for service account authentication (default: rpmbuilder)
# Automatically find all packages with metadata.yaml
PACKAGES := $(shell find $(ROOT_DIR)/rpms -mindepth 1 -maxdepth 1 -type d -exec test -f {}/metadata.yaml \; -print | xargs -n1 basename | sort)
@@ -22,6 +26,16 @@ build-all:
@echo "Building all packages using Python tooling for distro $(DISTRO)..."
$(BUILD_TOOL) build-all --distro $(DISTRO)
# Build all packages using native build (no Docker)
build-all-native:
@echo "Building all packages natively (no Docker) for distro $(DISTRO)..."
$(BUILD_TOOL) build-all --distro $(DISTRO) --native
# Build all packages using Buildah
build-all-buildah:
@echo "Building all packages using Buildah for distro $(DISTRO)..."
$(BUILD_TOOL) build-all --distro $(DISTRO) --buildah
# Build specific package using Python tool
.PHONY: $(PACKAGES)
$(PACKAGES):
+1 -1
View File
@@ -5,7 +5,7 @@ set -e
# Download and extract unrar (with version formatting)
export DOWNLOAD_VERSION=$(echo $PACKAGE_VERSION | sed s/\\.//)
curl -L -o /app/rarlinux.tar.gz https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/rarlab/rar/rarlinux-x64-${DOWNLOAD_VERSION}.tar.gz
tar xf /app/rarlinux.tar.gz
tar xf /app/rarlinux.tar.gz -C /app
mv /app/rar/unrar /app/
# Process the nfpm.yaml template with environment variables
+452 -42
View File
@@ -156,7 +156,7 @@ class PackageMetadata:
def get_vault_client() -> hvac.Client:
"""
Initialize and authenticate Vault client using AppRole authentication.
Initialize and authenticate Vault client using AppRole or Kubernetes authentication.
Returns:
Authenticated HVAC client
@@ -166,10 +166,7 @@ def get_vault_client() -> hvac.Client:
# Get required environment variables
vault_addr = os.getenv('VAULT_ADDR', 'https://vault.service.consul:8200')
vault_role_id = os.getenv('VAULT_ROLE_ID')
if not vault_role_id:
logger.error("VAULT_ROLE_ID environment variable is required")
sys.exit(1)
vault_role = os.getenv('VAULT_ROLE', 'rpmbuilder')
# Initialize Vault client with CA certificate
client = hvac.Client(
@@ -177,21 +174,55 @@ def get_vault_client() -> hvac.Client:
verify='/etc/pki/tls/cert.pem'
)
# Authenticate using AppRole
try:
logger.debug(f"Authenticating to Vault at {vault_addr}")
client.auth.approle.login(role_id=vault_role_id)
# Use AppRole authentication if VAULT_ROLE_ID is available
if vault_role_id:
try:
logger.debug(f"Authenticating to Vault at {vault_addr} using AppRole")
client.auth.approle.login(role_id=vault_role_id)
if not client.is_authenticated():
logger.error("Failed to authenticate with Vault")
if not client.is_authenticated():
logger.error("Failed to authenticate with Vault using AppRole")
sys.exit(1)
logger.debug("Successfully authenticated with Vault using AppRole")
return client
except Exception as e:
logger.error(f"AppRole authentication failed: {e}")
sys.exit(1)
logger.debug("Successfully authenticated with Vault")
return client
# Fallback to Kubernetes authentication if service account token is available
service_account_token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token'
except Exception as e:
logger.error(f"Vault authentication failed: {e}")
sys.exit(1)
if os.path.exists(service_account_token_path):
try:
logger.debug(f"Attempting Kubernetes authentication to Vault at {vault_addr}")
# Read the service account token
with open(service_account_token_path, 'r') as f:
jwt_token = f.read().strip()
# Authenticate using Kubernetes auth method
client.auth.kubernetes.login(
role=vault_role,
jwt=jwt_token,
mount_point='k8s/au/syd1'
)
if not client.is_authenticated():
logger.error("Failed to authenticate with Vault using Kubernetes auth")
sys.exit(1)
logger.debug("Successfully authenticated with Vault using Kubernetes auth")
return client
except Exception as e:
logger.error(f"Kubernetes authentication failed: {e}")
sys.exit(1)
# No authentication method available
logger.error("Neither VAULT_ROLE_ID environment variable nor Kubernetes service account token is available")
sys.exit(1)
def get_gitea_token() -> str:
@@ -570,6 +601,50 @@ def check_docker_available() -> bool:
return False
def check_native_build_deps() -> bool:
"""
Check if native build dependencies are available (nfpm, envsubst, etc.).
Returns:
True if native build dependencies are available, False otherwise
"""
required_commands = ['nfpm', 'envsubst', 'wget', 'tar']
for cmd in required_commands:
try:
result = subprocess.run(
[cmd, '--version'],
capture_output=True,
text=True,
timeout=5
)
if result.returncode != 0:
return False
except (subprocess.TimeoutExpired, FileNotFoundError):
return False
return True
def check_buildah_available() -> bool:
"""
Check if Buildah is available.
Returns:
True if Buildah is available, False otherwise
"""
try:
result = subprocess.run(
['buildah', 'version'],
capture_output=True,
text=True,
timeout=10
)
return result.returncode == 0
except (subprocess.TimeoutExpired, FileNotFoundError):
return False
def cleanup_container(container_name: str) -> None:
"""
Remove a Docker container.
@@ -777,6 +852,302 @@ def build_package_docker(
return False
def build_package_native(
package_dir: Path,
package_name: str,
package_version: str,
package_release: str,
dist_dir: Path,
repository: str,
dry_run: bool = False
) -> bool:
"""
Build a package natively without Docker, running build scripts directly.
Args:
package_dir: Directory containing the package resources
package_name: Name of the package
package_version: Package version
package_release: Package release number
dist_dir: Directory to store built packages
repository: Repository path (e.g., 'almalinux/el9')
dry_run: If True, only show what would be done
Returns:
True if build succeeded, False otherwise
"""
logger = logging.getLogger(__name__)
try:
# Ensure dist directory exists with repository structure
package_dist_dir = dist_dir / repository
if not dry_run:
package_dist_dir.mkdir(parents=True, exist_ok=True)
# Create a temporary workspace for native build
import tempfile
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
app_dir = temp_path / "app"
app_dist_dir = app_dir / "dist"
# Create directories
if not dry_run:
app_dir.mkdir(parents=True, exist_ok=True)
app_dist_dir.mkdir(parents=True, exist_ok=True)
# Copy package resources to temp directory
import shutil
temp_resources_dir = app_dir / "resources"
if not dry_run:
shutil.copytree(package_dir / "resources", temp_resources_dir)
# Read metadata.yaml to get all package fields
metadata_file = package_dir / "metadata.yaml"
metadata = {}
if metadata_file.exists():
try:
with open(metadata_file, 'r') as f:
metadata = yaml.safe_load(f) or {}
except Exception as e:
logger.warning(f"Could not read metadata.yaml: {e}")
logger.info(f"Building RPM for {package_name} version {package_version} natively")
if dry_run:
logger.info(f"[DRY RUN] Would build natively from: {temp_path}")
logger.info("[DRY RUN] Would set environment variables:")
logger.info(f"[DRY RUN] PACKAGE_NAME={package_name}")
logger.info(f"[DRY RUN] PACKAGE_VERSION={package_version}")
logger.info(f"[DRY RUN] PACKAGE_RELEASE={package_release}")
logger.info(f"[DRY RUN] PACKAGE_DESCRIPTION={metadata.get('description', '')}")
logger.info(f"[DRY RUN] Would run: {temp_resources_dir / 'build.sh'}")
logger.info(f"[DRY RUN] Would copy artifacts to: {package_dist_dir}")
return True
# Set up environment variables like the Dockerfile does
build_env = os.environ.copy()
build_env.update({
'PACKAGE_NAME': package_name,
'PACKAGE_VERSION': package_version,
'PACKAGE_RELEASE': package_release,
'PACKAGE_DESCRIPTION': metadata.get('description', ''),
'PACKAGE_MAINTAINER': metadata.get('maintainer', ''),
'PACKAGE_HOMEPAGE': metadata.get('homepage', ''),
'PACKAGE_LICENSE': metadata.get('license', ''),
'PACKAGE_ARCH': metadata.get('arch', 'amd64'),
'PACKAGE_PLATFORM': metadata.get('platform', 'linux')
})
# Run the build script
build_script = temp_resources_dir / "build.sh"
if not build_script.exists():
logger.error(f"Build script not found: {build_script}")
return False
# Make build script executable
build_script.chmod(0o755)
logger.debug(f"Running build script: {build_script}")
result = subprocess.run(
[str(build_script)],
cwd=app_dir,
env=build_env,
capture_output=True,
text=True
)
if result.returncode != 0:
logger.error(f"Native build failed for {package_name}")
logger.error(f"stdout: {result.stdout}")
logger.error(f"stderr: {result.stderr}")
return False
# Copy artifacts to final destination
if app_dist_dir.exists():
for item in app_dist_dir.iterdir():
if item.is_file():
shutil.copy2(item, package_dist_dir)
logger.info(f"Successfully built {package_name}-{package_version}-{package_release} natively")
return True
except Exception as e:
logger.error(f"Unexpected error building {package_name} natively: {e}")
return False
def build_package_buildah(
package_dir: Path,
package_name: str,
package_version: str,
package_release: str,
dist_dir: Path,
repository: str,
base_image: str = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest",
dry_run: bool = False
) -> bool:
"""
Build a package using Buildah without Docker daemon.
Args:
package_dir: Directory containing the package resources
package_name: Name of the package
package_version: Package version
package_release: Package release number
dist_dir: Directory to store built packages
repository: Repository path (e.g., 'almalinux/el9')
base_image: Base Docker image to use for building
dry_run: If True, only show what would be done
Returns:
True if build succeeded, False otherwise
"""
logger = logging.getLogger(__name__)
try:
# Ensure dist directory exists with repository structure
package_dist_dir = dist_dir / repository
if not dry_run:
package_dist_dir.mkdir(parents=True, exist_ok=True)
# Generate container name
container_name = f"{package_name}-{package_version}-buildah"
# Read metadata.yaml to get all package fields
metadata_file = package_dir / "metadata.yaml"
metadata = {}
if metadata_file.exists():
try:
with open(metadata_file, 'r') as f:
metadata = yaml.safe_load(f) or {}
except Exception as e:
logger.warning(f"Could not read metadata.yaml: {e}")
logger.info(f"Building RPM for {package_name} version {package_version} using Buildah")
if dry_run:
logger.info(f"[DRY RUN] Would use Buildah to build from: {base_image}")
logger.info("[DRY RUN] Would set environment variables:")
logger.info(f"[DRY RUN] PACKAGE_NAME={package_name}")
logger.info(f"[DRY RUN] PACKAGE_VERSION={package_version}")
logger.info(f"[DRY RUN] PACKAGE_RELEASE={package_release}")
logger.info(f"[DRY RUN] Would copy artifacts to: {package_dist_dir}")
return True
try:
# Step 1: Create a working container from base image
from_args = ['buildah', 'from', '--name', container_name, base_image]
logger.debug(f"Running: {' '.join(from_args)}")
result = subprocess.run(from_args, capture_output=True, text=True)
if result.returncode != 0:
logger.error(f"Failed to create Buildah container from {base_image}")
logger.error(f"stderr: {result.stderr}")
return False
# Step 2: Set environment variables
env_vars = {
'PACKAGE_NAME': package_name,
'PACKAGE_VERSION': package_version,
'PACKAGE_RELEASE': package_release,
'PACKAGE_DESCRIPTION': metadata.get('description', ''),
'PACKAGE_MAINTAINER': metadata.get('maintainer', ''),
'PACKAGE_HOMEPAGE': metadata.get('homepage', ''),
'PACKAGE_LICENSE': metadata.get('license', ''),
'PACKAGE_ARCH': metadata.get('arch', 'amd64'),
'PACKAGE_PLATFORM': metadata.get('platform', 'linux')
}
for key, value in env_vars.items():
config_args = ['buildah', 'config', '--env', f'{key}={value}', container_name]
logger.debug(f"Running: {' '.join(config_args)}")
result = subprocess.run(config_args, capture_output=True, text=True)
if result.returncode != 0:
logger.error(f"Failed to set environment variable {key}")
return False
# Step 3: Copy resources to container
copy_args = [
'buildah', 'copy', container_name,
str(package_dir / "resources"), '/app/resources'
]
logger.debug(f"Running: {' '.join(copy_args)}")
result = subprocess.run(copy_args, capture_output=True, text=True)
if result.returncode != 0:
logger.error(f"Failed to copy resources to container")
logger.error(f"stderr: {result.stderr}")
return False
# Step 4: Create dist directory in container
run_args = ['buildah', 'run', container_name, 'mkdir', '-p', '/app/dist']
logger.debug(f"Running: {' '.join(run_args)}")
result = subprocess.run(run_args, capture_output=True, text=True)
# Step 5: Run the build script
run_args = ['buildah', 'run', '--workingdir', '/app', container_name, '/app/resources/build.sh']
logger.debug(f"Running: {' '.join(run_args)}")
result = subprocess.run(run_args, capture_output=True, text=True)
if result.returncode != 0:
logger.error(f"Buildah build script failed for {package_name}")
logger.error(f"stdout: {result.stdout}")
logger.error(f"stderr: {result.stderr}")
return False
# Step 6: Copy artifacts from container to host
copy_out_args = [
'buildah', 'run', container_name,
'find', '/app/dist', '-type', 'f', '-name', '*.rpm'
]
result = subprocess.run(copy_out_args, capture_output=True, text=True)
logger.debug(f"Find RPMs result: {result.stdout}")
if result.returncode == 0 and result.stdout.strip():
rpm_files = result.stdout.strip().split('\n')
logger.info(f"Found {len(rpm_files)} RPM files to copy")
for rpm_file in rpm_files:
rpm_file = rpm_file.strip()
if rpm_file:
# Use buildah mount to copy files out
mount_args = ['buildah', 'mount', container_name]
mount_result = subprocess.run(mount_args, capture_output=True, text=True)
if mount_result.returncode == 0:
container_path = mount_result.stdout.strip()
source_file = Path(container_path) / rpm_file.lstrip('/')
if source_file.exists():
import shutil
dest_file = package_dist_dir / source_file.name
shutil.copy2(source_file, dest_file)
logger.debug(f"Copied {source_file} to {dest_file}")
else:
logger.error(f"Source file not found: {source_file}")
# Unmount
subprocess.run(['buildah', 'unmount', container_name], capture_output=True)
else:
logger.error(f"Failed to mount container: {mount_result.stderr}")
else:
logger.warning(f"No RPM files found or find command failed: {result.stderr}")
logger.info(f"Successfully built {package_name}-{package_version}-{package_release} using Buildah")
return True
finally:
# Step 7: Clean up container
rm_args = ['buildah', 'rm', container_name]
logger.debug(f"Running: {' '.join(rm_args)}")
subprocess.run(rm_args, capture_output=True, text=True)
except Exception as e:
logger.error(f"Unexpected error building {package_name} with Buildah: {e}")
return False
def cleanup_images(image_pattern: str = "*-builder") -> None:
"""
Clean up Docker images matching a pattern.
@@ -917,7 +1288,9 @@ class Builder:
release: str,
dry_run: bool = False,
force: bool = False,
distro: str = 'almalinux/el9'
distro: str = 'almalinux/el9',
native: bool = False,
buildah: bool = False
) -> bool:
"""
Build a single package.
@@ -993,9 +1366,9 @@ class Builder:
return False
package_info = PackageInfo(package, version, release, package_dir, distro, base_image)
return self._build_package(package_info, dry_run, force)
return self._build_package(package_info, dry_run, force, native, buildah)
def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9') -> bool:
def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9', native: bool = False, buildah: bool = False) -> bool:
"""
Build all packages.
@@ -1017,29 +1390,29 @@ class Builder:
self.logger.info(f"Found {len(packages)} packages to process")
if parallel == 1:
return self._build_sequential(packages, dry_run, force)
return self._build_sequential(packages, dry_run, force, native, buildah)
else:
return self._build_parallel(packages, dry_run, force, parallel)
return self._build_parallel(packages, dry_run, force, parallel, native, buildah)
def _build_sequential(self, packages: List[PackageInfo], dry_run: bool, force: bool) -> bool:
def _build_sequential(self, packages: List[PackageInfo], dry_run: bool, force: bool, native: bool, buildah: bool) -> bool:
"""Build packages sequentially."""
success_count = 0
for package_info in packages:
if self._build_package(package_info, dry_run, force):
if self._build_package(package_info, dry_run, force, native, buildah):
success_count += 1
self.logger.info(f"Built {success_count}/{len(packages)} packages successfully")
return success_count == len(packages)
def _build_parallel(self, packages: List[PackageInfo], dry_run: bool, force: bool, parallel: int) -> bool:
def _build_parallel(self, packages: List[PackageInfo], dry_run: bool, force: bool, parallel: int, native: bool, buildah: bool) -> bool:
"""Build packages in parallel."""
success_count = 0
with ThreadPoolExecutor(max_workers=parallel) as executor:
# Submit all build tasks
future_to_package = {
executor.submit(self._build_package, pkg, dry_run, force): pkg
executor.submit(self._build_package, pkg, dry_run, force, native, buildah): pkg
for pkg in packages
}
@@ -1056,7 +1429,7 @@ class Builder:
self.logger.info(f"Built {success_count}/{len(packages)} packages successfully")
return success_count == len(packages)
def _build_package(self, package_info: PackageInfo, dry_run: bool, force: bool) -> bool:
def _build_package(self, package_info: PackageInfo, dry_run: bool, force: bool, native: bool = False, buildah: bool = False) -> bool:
"""
Build a single package.
@@ -1081,22 +1454,51 @@ class Builder:
)
return True
# Check Docker is available (unless dry run)
if not dry_run and not check_docker_available():
self.logger.error("Docker is not available or running")
# Check build tool availability (unless dry run)
use_native = native or (not buildah and check_native_build_deps())
use_buildah = buildah or (not use_native and check_buildah_available())
use_docker = not use_native and not use_buildah and check_docker_available()
if not dry_run and not use_native and not use_buildah and not use_docker:
self.logger.error("No build tools available (tried native, Buildah, Docker)")
return False
# Build the package
return build_package_docker(
package_dir=package_info.directory,
package_name=package_info.name,
package_version=package_info.version,
package_release=package_info.release,
dist_dir=self.dist_dir,
repository=package_info.distro,
base_image=package_info.base_image,
dry_run=dry_run
)
# Build the package using available tool
if use_native:
self.logger.debug(f"Using native build for {package_info.name}")
return build_package_native(
package_dir=package_info.directory,
package_name=package_info.name,
package_version=package_info.version,
package_release=package_info.release,
dist_dir=self.dist_dir,
repository=package_info.distro,
dry_run=dry_run
)
elif use_buildah:
self.logger.debug(f"Using Buildah to build {package_info.name}")
return build_package_buildah(
package_dir=package_info.directory,
package_name=package_info.name,
package_version=package_info.version,
package_release=package_info.release,
dist_dir=self.dist_dir,
repository=package_info.distro,
base_image=package_info.base_image,
dry_run=dry_run
)
else:
self.logger.debug(f"Using Docker to build {package_info.name}")
return build_package_docker(
package_dir=package_info.directory,
package_name=package_info.name,
package_version=package_info.version,
package_release=package_info.release,
dist_dir=self.dist_dir,
repository=package_info.distro,
base_image=package_info.base_image,
dry_run=dry_run
)
except Exception as e:
self.logger.error(f"Failed to build {package_info}: {e}")
@@ -1358,6 +1760,8 @@ def build(
distro: str = typer.Option("almalinux/el9", help="Target distro (default: almalinux/el9)"),
dry_run: bool = typer.Option(False, "--dry-run", help="Show what would be built without building"),
force: bool = typer.Option(False, "--force", help="Build even if package exists in registry"),
native: bool = typer.Option(False, "--native", help="Force native build (skip Docker even if available)"),
buildah: bool = typer.Option(False, "--buildah", help="Force Buildah build (requires Buildah)"),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose logging")
):
"""Build a specific package."""
@@ -1419,7 +1823,9 @@ def build(
release=str(release),
dry_run=dry_run,
force=force,
distro=distro
distro=distro,
native=native,
buildah=buildah
)
if not success:
@@ -1435,6 +1841,8 @@ def build_all(
force: bool = typer.Option(False, "--force", help="Build even if packages exist in registry"),
parallel: int = typer.Option(4, help="Number of parallel builds"),
distro: str = typer.Option("almalinux/el9", help="Target distro (almalinux/el8, almalinux/el9, or 'all' for all distros)"),
native: bool = typer.Option(False, "--native", help="Force native build (skip Docker even if available)"),
buildah: bool = typer.Option(False, "--buildah", help="Force Buildah build (requires Buildah)"),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose logging")
):
"""Build all packages."""
@@ -1448,7 +1856,9 @@ def build_all(
dry_run=dry_run,
force=force,
parallel=parallel,
distro=distro
distro=distro,
native=native,
buildah=buildah
)
if not success: