From facdfb4ba5d753ea2a0ddd010de4d2b2eee84776 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 7 Mar 2026 11:19:13 +1100 Subject: [PATCH] feat: migrate to woodpeckerci - update build tool for kubernetes auth - update build tool to build packages without docker (native + buildah) - add woodpecker pre-commit and build jobs --- .woodpecker/build-almalinux8.yaml | 18 ++ .woodpecker/build-almalinux9.yaml | 18 ++ .woodpecker/pre-commit.yaml | 9 + Makefile | 14 + rpms/unrar/resources/build.sh | 2 +- tools/build | 475 +++++++++++++++++++++++++++--- 6 files changed, 493 insertions(+), 43 deletions(-) create mode 100644 .woodpecker/build-almalinux8.yaml create mode 100644 .woodpecker/build-almalinux9.yaml create mode 100644 .woodpecker/pre-commit.yaml diff --git a/.woodpecker/build-almalinux8.yaml b/.woodpecker/build-almalinux8.yaml new file mode 100644 index 0000000..9695d4e --- /dev/null +++ b/.woodpecker/build-almalinux8.yaml @@ -0,0 +1,18 @@ +when: + - event: pull_request + +steps: + - name: build rpms + image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest + commands: + - mkdir -p /app/dist + - dnf install buildah -y + - ./tools/build build-all --distro almalinux/el8 --buildah + privileged: true + backend_options: + kubernetes: + serviceAccountName: default + - name: show rpms + image: git.unkin.net/unkin/almalinux8-base:latest + commands: + - find /app/dist -type f -name "*.rpm" diff --git a/.woodpecker/build-almalinux9.yaml b/.woodpecker/build-almalinux9.yaml new file mode 100644 index 0000000..08e7712 --- /dev/null +++ b/.woodpecker/build-almalinux9.yaml @@ -0,0 +1,18 @@ +when: + - event: pull_request + +steps: + - name: build rpms + image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest + commands: + - mkdir -p /app/dist/ + - dnf install buildah -y + - ./tools/build build-all --distro almalinux/el9 --buildah + privileged: true + backend_options: + kubernetes: + serviceAccountName: default + - name: show rpms + image: git.unkin.net/unkin/almalinux9-base:latest + commands: + - find /app/dist -type f -name "*.rpm" diff --git a/.woodpecker/pre-commit.yaml b/.woodpecker/pre-commit.yaml new file mode 100644 index 0000000..bf6529f --- /dev/null +++ b/.woodpecker/pre-commit.yaml @@ -0,0 +1,9 @@ +when: + - event: pull_request + +steps: + - name: pre-commit + image: git.unkin.net/unkin/almalinux9-base:latest + commands: + - dnf install uv make -y + - uvx pre-commit run --all-files diff --git a/Makefile b/Makefile index 8c56409..70ee592 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,10 @@ ROOT_DIR := $(PWD) BUILD_TOOL := $(ROOT_DIR)/tools/build DISTRO ?= almalinux/el9 +# Authentication variables (optional) +# VAULT_ROLE_ID - Use AppRole authentication if set +# VAULT_ROLE - Kubernetes role for service account authentication (default: rpmbuilder) + # Automatically find all packages with metadata.yaml PACKAGES := $(shell find $(ROOT_DIR)/rpms -mindepth 1 -maxdepth 1 -type d -exec test -f {}/metadata.yaml \; -print | xargs -n1 basename | sort) @@ -22,6 +26,16 @@ build-all: @echo "Building all packages using Python tooling for distro $(DISTRO)..." $(BUILD_TOOL) build-all --distro $(DISTRO) +# Build all packages using native build (no Docker) +build-all-native: + @echo "Building all packages natively (no Docker) for distro $(DISTRO)..." + $(BUILD_TOOL) build-all --distro $(DISTRO) --native + +# Build all packages using Buildah +build-all-buildah: + @echo "Building all packages using Buildah for distro $(DISTRO)..." + $(BUILD_TOOL) build-all --distro $(DISTRO) --buildah + # Build specific package using Python tool .PHONY: $(PACKAGES) $(PACKAGES): diff --git a/rpms/unrar/resources/build.sh b/rpms/unrar/resources/build.sh index 47c0e2b..38c585c 100755 --- a/rpms/unrar/resources/build.sh +++ b/rpms/unrar/resources/build.sh @@ -5,7 +5,7 @@ set -e # Download and extract unrar (with version formatting) export DOWNLOAD_VERSION=$(echo $PACKAGE_VERSION | sed s/\\.//) curl -L -o /app/rarlinux.tar.gz https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/rarlab/rar/rarlinux-x64-${DOWNLOAD_VERSION}.tar.gz -tar xf /app/rarlinux.tar.gz +tar xf /app/rarlinux.tar.gz -C /app mv /app/rar/unrar /app/ # Process the nfpm.yaml template with environment variables diff --git a/tools/build b/tools/build index ebc2c0a..4ea70c5 100755 --- a/tools/build +++ b/tools/build @@ -156,7 +156,7 @@ class PackageMetadata: def get_vault_client() -> hvac.Client: """ - Initialize and authenticate Vault client using AppRole authentication. + Initialize and authenticate Vault client using AppRole or Kubernetes authentication. Returns: Authenticated HVAC client @@ -166,10 +166,7 @@ def get_vault_client() -> hvac.Client: # Get required environment variables vault_addr = os.getenv('VAULT_ADDR', 'https://vault.service.consul:8200') vault_role_id = os.getenv('VAULT_ROLE_ID') - - if not vault_role_id: - logger.error("VAULT_ROLE_ID environment variable is required") - sys.exit(1) + vault_role = os.getenv('VAULT_ROLE', 'rpmbuilder') # Initialize Vault client with CA certificate client = hvac.Client( @@ -177,21 +174,55 @@ def get_vault_client() -> hvac.Client: verify='/etc/pki/tls/cert.pem' ) - # Authenticate using AppRole - try: - logger.debug(f"Authenticating to Vault at {vault_addr}") - client.auth.approle.login(role_id=vault_role_id) + # Use AppRole authentication if VAULT_ROLE_ID is available + if vault_role_id: + try: + logger.debug(f"Authenticating to Vault at {vault_addr} using AppRole") + client.auth.approle.login(role_id=vault_role_id) - if not client.is_authenticated(): - logger.error("Failed to authenticate with Vault") + if not client.is_authenticated(): + logger.error("Failed to authenticate with Vault using AppRole") + sys.exit(1) + + logger.debug("Successfully authenticated with Vault using AppRole") + return client + + except Exception as e: + logger.error(f"AppRole authentication failed: {e}") sys.exit(1) - logger.debug("Successfully authenticated with Vault") - return client + # Fallback to Kubernetes authentication if service account token is available + service_account_token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' - except Exception as e: - logger.error(f"Vault authentication failed: {e}") - sys.exit(1) + if os.path.exists(service_account_token_path): + try: + logger.debug(f"Attempting Kubernetes authentication to Vault at {vault_addr}") + + # Read the service account token + with open(service_account_token_path, 'r') as f: + jwt_token = f.read().strip() + + # Authenticate using Kubernetes auth method + client.auth.kubernetes.login( + role=vault_role, + jwt=jwt_token, + mount_point='k8s/au/syd1' + ) + + if not client.is_authenticated(): + logger.error("Failed to authenticate with Vault using Kubernetes auth") + sys.exit(1) + + logger.debug("Successfully authenticated with Vault using Kubernetes auth") + return client + + except Exception as e: + logger.error(f"Kubernetes authentication failed: {e}") + sys.exit(1) + + # No authentication method available + logger.error("Neither VAULT_ROLE_ID environment variable nor Kubernetes service account token is available") + sys.exit(1) def get_gitea_token() -> str: @@ -570,6 +601,50 @@ def check_docker_available() -> bool: return False +def check_native_build_deps() -> bool: + """ + Check if native build dependencies are available (nfpm, envsubst, etc.). + + Returns: + True if native build dependencies are available, False otherwise + """ + required_commands = ['nfpm', 'envsubst', 'wget', 'tar'] + + for cmd in required_commands: + try: + result = subprocess.run( + [cmd, '--version'], + capture_output=True, + text=True, + timeout=5 + ) + if result.returncode != 0: + return False + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + return True + + +def check_buildah_available() -> bool: + """ + Check if Buildah is available. + + Returns: + True if Buildah is available, False otherwise + """ + try: + result = subprocess.run( + ['buildah', 'version'], + capture_output=True, + text=True, + timeout=10 + ) + return result.returncode == 0 + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + def cleanup_container(container_name: str) -> None: """ Remove a Docker container. @@ -777,6 +852,283 @@ def build_package_docker( return False +def build_package_native( + package_dir: Path, + package_name: str, + package_version: str, + package_release: str, + dist_dir: Path, + repository: str, + dry_run: bool = False +) -> bool: + """ + Build a package natively without Docker, running build scripts directly. + + Args: + package_dir: Directory containing the package resources + package_name: Name of the package + package_version: Package version + package_release: Package release number + dist_dir: Directory to store built packages + repository: Repository path (e.g., 'almalinux/el9') + dry_run: If True, only show what would be done + + Returns: + True if build succeeded, False otherwise + """ + logger = logging.getLogger(__name__) + + try: + # Ensure dist directory exists with repository structure + package_dist_dir = dist_dir / repository + if not dry_run: + package_dist_dir.mkdir(parents=True, exist_ok=True) + + # Create a temporary workspace for native build + import tempfile + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + app_dir = temp_path / "app" + app_dist_dir = app_dir / "dist" + + # Create directories + if not dry_run: + app_dir.mkdir(parents=True, exist_ok=True) + app_dist_dir.mkdir(parents=True, exist_ok=True) + + # Copy package resources to temp directory + import shutil + temp_resources_dir = app_dir / "resources" + if not dry_run: + shutil.copytree(package_dir / "resources", temp_resources_dir) + + # Read metadata.yaml to get all package fields + metadata_file = package_dir / "metadata.yaml" + metadata = {} + if metadata_file.exists(): + try: + with open(metadata_file, 'r') as f: + metadata = yaml.safe_load(f) or {} + except Exception as e: + logger.warning(f"Could not read metadata.yaml: {e}") + + logger.info(f"Building RPM for {package_name} version {package_version} natively") + + if dry_run: + logger.info(f"[DRY RUN] Would build natively from: {temp_path}") + logger.info("[DRY RUN] Would set environment variables:") + logger.info(f"[DRY RUN] PACKAGE_NAME={package_name}") + logger.info(f"[DRY RUN] PACKAGE_VERSION={package_version}") + logger.info(f"[DRY RUN] PACKAGE_RELEASE={package_release}") + logger.info(f"[DRY RUN] PACKAGE_DESCRIPTION={metadata.get('description', '')}") + logger.info(f"[DRY RUN] Would run: {temp_resources_dir / 'build.sh'}") + logger.info(f"[DRY RUN] Would copy artifacts to: {package_dist_dir}") + return True + + # Set up environment variables like the Dockerfile does + build_env = os.environ.copy() + build_env.update({ + 'PACKAGE_NAME': package_name, + 'PACKAGE_VERSION': package_version, + 'PACKAGE_RELEASE': package_release, + 'PACKAGE_DESCRIPTION': metadata.get('description', ''), + 'PACKAGE_MAINTAINER': metadata.get('maintainer', ''), + 'PACKAGE_HOMEPAGE': metadata.get('homepage', ''), + 'PACKAGE_LICENSE': metadata.get('license', ''), + 'PACKAGE_ARCH': metadata.get('arch', 'amd64'), + 'PACKAGE_PLATFORM': metadata.get('platform', 'linux') + }) + + # Run the build script + build_script = temp_resources_dir / "build.sh" + if not build_script.exists(): + logger.error(f"Build script not found: {build_script}") + return False + + # Make build script executable + build_script.chmod(0o755) + + logger.debug(f"Running build script: {build_script}") + result = subprocess.run( + [str(build_script)], + cwd=app_dir, + env=build_env, + capture_output=True, + text=True + ) + + if result.returncode != 0: + logger.error(f"Native build failed for {package_name}") + logger.error(f"stdout: {result.stdout}") + logger.error(f"stderr: {result.stderr}") + return False + + # Copy artifacts to final destination + if app_dist_dir.exists(): + for item in app_dist_dir.iterdir(): + if item.is_file(): + shutil.copy2(item, package_dist_dir) + + logger.info(f"Successfully built {package_name}-{package_version}-{package_release} natively") + return True + + except Exception as e: + logger.error(f"Unexpected error building {package_name} natively: {e}") + return False + + +def build_package_buildah( + package_dir: Path, + package_name: str, + package_version: str, + package_release: str, + dist_dir: Path, + repository: str, + base_image: str = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest", + dry_run: bool = False +) -> bool: + """ + Build a package using Buildah without Docker daemon. + + Args: + package_dir: Directory containing the package resources + package_name: Name of the package + package_version: Package version + package_release: Package release number + dist_dir: Directory to store built packages + repository: Repository path (e.g., 'almalinux/el9') + base_image: Base Docker image to use for building + dry_run: If True, only show what would be done + + Returns: + True if build succeeded, False otherwise + """ + logger = logging.getLogger(__name__) + + try: + # Ensure dist directory exists with repository structure + package_dist_dir = dist_dir / repository + if not dry_run: + package_dist_dir.mkdir(parents=True, exist_ok=True) + + # Generate container name + container_name = f"{package_name}-{package_version}-buildah" + + # Read metadata.yaml to get all package fields + metadata_file = package_dir / "metadata.yaml" + metadata = {} + if metadata_file.exists(): + try: + with open(metadata_file, 'r') as f: + metadata = yaml.safe_load(f) or {} + except Exception as e: + logger.warning(f"Could not read metadata.yaml: {e}") + + logger.info(f"Building RPM for {package_name} version {package_version} using Buildah") + + if dry_run: + logger.info(f"[DRY RUN] Would use Buildah to build from: {base_image}") + logger.info("[DRY RUN] Would set environment variables:") + logger.info(f"[DRY RUN] PACKAGE_NAME={package_name}") + logger.info(f"[DRY RUN] PACKAGE_VERSION={package_version}") + logger.info(f"[DRY RUN] PACKAGE_RELEASE={package_release}") + logger.info(f"[DRY RUN] Would copy artifacts to: {package_dist_dir}") + return True + + try: + # Step 1: Create a working container from base image + from_args = ['buildah', 'from', '--name', container_name, base_image] + logger.debug(f"Running: {' '.join(from_args)}") + result = subprocess.run(from_args, capture_output=True, text=True) + + if result.returncode != 0: + logger.error(f"Failed to create Buildah container from {base_image}") + logger.error(f"stderr: {result.stderr}") + return False + + # Step 2: Set environment variables + env_vars = { + 'PACKAGE_NAME': package_name, + 'PACKAGE_VERSION': package_version, + 'PACKAGE_RELEASE': package_release, + 'PACKAGE_DESCRIPTION': metadata.get('description', ''), + 'PACKAGE_MAINTAINER': metadata.get('maintainer', ''), + 'PACKAGE_HOMEPAGE': metadata.get('homepage', ''), + 'PACKAGE_LICENSE': metadata.get('license', ''), + 'PACKAGE_ARCH': metadata.get('arch', 'amd64'), + 'PACKAGE_PLATFORM': metadata.get('platform', 'linux') + } + + for key, value in env_vars.items(): + config_args = ['buildah', 'config', '--env', f'{key}={value}', container_name] + logger.debug(f"Running: {' '.join(config_args)}") + result = subprocess.run(config_args, capture_output=True, text=True) + if result.returncode != 0: + logger.error(f"Failed to set environment variable {key}") + return False + + # Step 3: Copy resources to container + copy_args = [ + 'buildah', 'copy', container_name, + str(package_dir / "resources"), '/app/resources' + ] + logger.debug(f"Running: {' '.join(copy_args)}") + result = subprocess.run(copy_args, capture_output=True, text=True) + + if result.returncode != 0: + logger.error(f"Failed to copy resources to container") + logger.error(f"stderr: {result.stderr}") + return False + + # Step 4: Create dist directory in container + run_args = ['buildah', 'run', container_name, 'mkdir', '-p', '/app/dist'] + logger.debug(f"Running: {' '.join(run_args)}") + result = subprocess.run(run_args, capture_output=True, text=True) + + # Step 5: Run the build script + run_args = ['buildah', 'run', '--workingdir', '/app', container_name, '/app/resources/build.sh'] + logger.debug(f"Running: {' '.join(run_args)}") + result = subprocess.run(run_args, capture_output=True, text=True) + + if result.returncode != 0: + logger.error(f"Buildah build script failed for {package_name}") + logger.error(f"stdout: {result.stdout}") + logger.error(f"stderr: {result.stderr}") + return False + + # Step 6: Copy artifacts from container to host + copy_out_args = [ + 'buildah', 'run', container_name, + 'find', '/app/dist', '-type', 'f', '-name', '*.rpm' + ] + result = subprocess.run(copy_out_args, capture_output=True, text=True) + + if result.returncode == 0 and result.stdout.strip(): + rpm_files = result.stdout.strip().split('\n') + for rpm_file in rpm_files: + rpm_file = rpm_file.strip() + if rpm_file: + copy_args = [ + 'buildah', 'copy', '--from', container_name, + rpm_file, str(package_dist_dir) + ] + logger.debug(f"Running: {' '.join(copy_args)}") + subprocess.run(copy_args, capture_output=True, text=True) + + logger.info(f"Successfully built {package_name}-{package_version}-{package_release} using Buildah") + return True + + finally: + # Step 7: Clean up container + rm_args = ['buildah', 'rm', container_name] + logger.debug(f"Running: {' '.join(rm_args)}") + subprocess.run(rm_args, capture_output=True, text=True) + + except Exception as e: + logger.error(f"Unexpected error building {package_name} with Buildah: {e}") + return False + + def cleanup_images(image_pattern: str = "*-builder") -> None: """ Clean up Docker images matching a pattern. @@ -917,7 +1269,9 @@ class Builder: release: str, dry_run: bool = False, force: bool = False, - distro: str = 'almalinux/el9' + distro: str = 'almalinux/el9', + native: bool = False, + buildah: bool = False ) -> bool: """ Build a single package. @@ -993,9 +1347,9 @@ class Builder: return False package_info = PackageInfo(package, version, release, package_dir, distro, base_image) - return self._build_package(package_info, dry_run, force) + return self._build_package(package_info, dry_run, force, native, buildah) - def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9') -> bool: + def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9', native: bool = False, buildah: bool = False) -> bool: """ Build all packages. @@ -1017,29 +1371,29 @@ class Builder: self.logger.info(f"Found {len(packages)} packages to process") if parallel == 1: - return self._build_sequential(packages, dry_run, force) + return self._build_sequential(packages, dry_run, force, native, buildah) else: - return self._build_parallel(packages, dry_run, force, parallel) + return self._build_parallel(packages, dry_run, force, parallel, native, buildah) - def _build_sequential(self, packages: List[PackageInfo], dry_run: bool, force: bool) -> bool: + def _build_sequential(self, packages: List[PackageInfo], dry_run: bool, force: bool, native: bool, buildah: bool) -> bool: """Build packages sequentially.""" success_count = 0 for package_info in packages: - if self._build_package(package_info, dry_run, force): + if self._build_package(package_info, dry_run, force, native, buildah): success_count += 1 self.logger.info(f"Built {success_count}/{len(packages)} packages successfully") return success_count == len(packages) - def _build_parallel(self, packages: List[PackageInfo], dry_run: bool, force: bool, parallel: int) -> bool: + def _build_parallel(self, packages: List[PackageInfo], dry_run: bool, force: bool, parallel: int, native: bool, buildah: bool) -> bool: """Build packages in parallel.""" success_count = 0 with ThreadPoolExecutor(max_workers=parallel) as executor: # Submit all build tasks future_to_package = { - executor.submit(self._build_package, pkg, dry_run, force): pkg + executor.submit(self._build_package, pkg, dry_run, force, native, buildah): pkg for pkg in packages } @@ -1056,7 +1410,7 @@ class Builder: self.logger.info(f"Built {success_count}/{len(packages)} packages successfully") return success_count == len(packages) - def _build_package(self, package_info: PackageInfo, dry_run: bool, force: bool) -> bool: + def _build_package(self, package_info: PackageInfo, dry_run: bool, force: bool, native: bool = False, buildah: bool = False) -> bool: """ Build a single package. @@ -1081,22 +1435,51 @@ class Builder: ) return True - # Check Docker is available (unless dry run) - if not dry_run and not check_docker_available(): - self.logger.error("Docker is not available or running") + # Check build tool availability (unless dry run) + use_native = native or (not buildah and check_native_build_deps()) + use_buildah = buildah or (not use_native and check_buildah_available()) + use_docker = not use_native and not use_buildah and check_docker_available() + + if not dry_run and not use_native and not use_buildah and not use_docker: + self.logger.error("No build tools available (tried native, Buildah, Docker)") return False - # Build the package - return build_package_docker( - package_dir=package_info.directory, - package_name=package_info.name, - package_version=package_info.version, - package_release=package_info.release, - dist_dir=self.dist_dir, - repository=package_info.distro, - base_image=package_info.base_image, - dry_run=dry_run - ) + # Build the package using available tool + if use_native: + self.logger.debug(f"Using native build for {package_info.name}") + return build_package_native( + package_dir=package_info.directory, + package_name=package_info.name, + package_version=package_info.version, + package_release=package_info.release, + dist_dir=self.dist_dir, + repository=package_info.distro, + dry_run=dry_run + ) + elif use_buildah: + self.logger.debug(f"Using Buildah to build {package_info.name}") + return build_package_buildah( + package_dir=package_info.directory, + package_name=package_info.name, + package_version=package_info.version, + package_release=package_info.release, + dist_dir=self.dist_dir, + repository=package_info.distro, + base_image=package_info.base_image, + dry_run=dry_run + ) + else: + self.logger.debug(f"Using Docker to build {package_info.name}") + return build_package_docker( + package_dir=package_info.directory, + package_name=package_info.name, + package_version=package_info.version, + package_release=package_info.release, + dist_dir=self.dist_dir, + repository=package_info.distro, + base_image=package_info.base_image, + dry_run=dry_run + ) except Exception as e: self.logger.error(f"Failed to build {package_info}: {e}") @@ -1358,6 +1741,8 @@ def build( distro: str = typer.Option("almalinux/el9", help="Target distro (default: almalinux/el9)"), dry_run: bool = typer.Option(False, "--dry-run", help="Show what would be built without building"), force: bool = typer.Option(False, "--force", help="Build even if package exists in registry"), + native: bool = typer.Option(False, "--native", help="Force native build (skip Docker even if available)"), + buildah: bool = typer.Option(False, "--buildah", help="Force Buildah build (requires Buildah)"), verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose logging") ): """Build a specific package.""" @@ -1419,7 +1804,9 @@ def build( release=str(release), dry_run=dry_run, force=force, - distro=distro + distro=distro, + native=native, + buildah=buildah ) if not success: @@ -1435,6 +1822,8 @@ def build_all( force: bool = typer.Option(False, "--force", help="Build even if packages exist in registry"), parallel: int = typer.Option(4, help="Number of parallel builds"), distro: str = typer.Option("almalinux/el9", help="Target distro (almalinux/el8, almalinux/el9, or 'all' for all distros)"), + native: bool = typer.Option(False, "--native", help="Force native build (skip Docker even if available)"), + buildah: bool = typer.Option(False, "--buildah", help="Force Buildah build (requires Buildah)"), verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose logging") ): """Build all packages.""" @@ -1448,7 +1837,9 @@ def build_all( dry_run=dry_run, force=force, parallel=parallel, - distro=distro + distro=distro, + native=native, + buildah=buildah ) if not success: