From 737e70533b461a6c942953e95cd1e8ad0a7cffb4 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 7 Mar 2026 11:19:13 +1100 Subject: [PATCH] feat: migrate to woodpeckerci - update build tool for kubernetes auth - update build tool to build packages without docker - add woodpecker pre-commit and build jobs --- .woodpecker/build-almalinux8.yaml | 16 ++ .woodpecker/build-almalinux9.yaml | 16 ++ .woodpecker/pre-commit.yaml | 9 + Makefile | 9 + tools/build | 286 +++++++++++++++++++++++++----- 5 files changed, 294 insertions(+), 42 deletions(-) create mode 100644 .woodpecker/build-almalinux8.yaml create mode 100644 .woodpecker/build-almalinux9.yaml create mode 100644 .woodpecker/pre-commit.yaml diff --git a/.woodpecker/build-almalinux8.yaml b/.woodpecker/build-almalinux8.yaml new file mode 100644 index 0000000..4eb25fd --- /dev/null +++ b/.woodpecker/build-almalinux8.yaml @@ -0,0 +1,16 @@ +when: + - event: pull_request + +steps: + - name: build rpms + image: git.unkin.net/unkin/almalinux8-rpmbuilder:latest + commands: + - mkdir -p /app/dist + - ./tools/build build-all --distro almalinux/el8 --native + backend_options: + kubernetes: + serviceAccountName: default + - name: show rpms + image: git.unkin.net/unkin/almalinux8-base:latest + commands: + - find /app/dist -type f -name "*.rpm" diff --git a/.woodpecker/build-almalinux9.yaml b/.woodpecker/build-almalinux9.yaml new file mode 100644 index 0000000..14071b3 --- /dev/null +++ b/.woodpecker/build-almalinux9.yaml @@ -0,0 +1,16 @@ +when: + - event: pull_request + +steps: + - name: build rpms + image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest + commands: + - mkdir -p /app/dist/ + - ./tools/build build-all --distro almalinux/el9 --native + backend_options: + kubernetes: + serviceAccountName: default + - name: show rpms + image: git.unkin.net/unkin/almalinux9-base:latest + commands: + - find /app/dist -type f -name "*.rpm" diff --git a/.woodpecker/pre-commit.yaml b/.woodpecker/pre-commit.yaml new file mode 100644 index 0000000..bf6529f --- /dev/null +++ b/.woodpecker/pre-commit.yaml @@ -0,0 +1,9 @@ +when: + - event: pull_request + +steps: + - name: pre-commit + image: git.unkin.net/unkin/almalinux9-base:latest + commands: + - dnf install uv make -y + - uvx pre-commit run --all-files diff --git a/Makefile b/Makefile index 8c56409..2aa4dc0 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,10 @@ ROOT_DIR := $(PWD) BUILD_TOOL := $(ROOT_DIR)/tools/build DISTRO ?= almalinux/el9 +# Authentication variables (optional) +# VAULT_ROLE_ID - Use AppRole authentication if set +# VAULT_ROLE - Kubernetes role for service account authentication (default: rpmbuilder) + # Automatically find all packages with metadata.yaml PACKAGES := $(shell find $(ROOT_DIR)/rpms -mindepth 1 -maxdepth 1 -type d -exec test -f {}/metadata.yaml \; -print | xargs -n1 basename | sort) @@ -22,6 +26,11 @@ build-all: @echo "Building all packages using Python tooling for distro $(DISTRO)..." $(BUILD_TOOL) build-all --distro $(DISTRO) +# Build all packages using native build (no Docker) +build-all-native: + @echo "Building all packages natively (no Docker) for distro $(DISTRO)..." + $(BUILD_TOOL) build-all --distro $(DISTRO) --native + # Build specific package using Python tool .PHONY: $(PACKAGES) $(PACKAGES): diff --git a/tools/build b/tools/build index ebc2c0a..5d89dbc 100755 --- a/tools/build +++ b/tools/build @@ -156,7 +156,7 @@ class PackageMetadata: def get_vault_client() -> hvac.Client: """ - Initialize and authenticate Vault client using AppRole authentication. + Initialize and authenticate Vault client using AppRole or Kubernetes authentication. Returns: Authenticated HVAC client @@ -166,10 +166,7 @@ def get_vault_client() -> hvac.Client: # Get required environment variables vault_addr = os.getenv('VAULT_ADDR', 'https://vault.service.consul:8200') vault_role_id = os.getenv('VAULT_ROLE_ID') - - if not vault_role_id: - logger.error("VAULT_ROLE_ID environment variable is required") - sys.exit(1) + vault_role = os.getenv('VAULT_ROLE', 'rpmbuilder') # Initialize Vault client with CA certificate client = hvac.Client( @@ -177,21 +174,55 @@ def get_vault_client() -> hvac.Client: verify='/etc/pki/tls/cert.pem' ) - # Authenticate using AppRole - try: - logger.debug(f"Authenticating to Vault at {vault_addr}") - client.auth.approle.login(role_id=vault_role_id) + # Use AppRole authentication if VAULT_ROLE_ID is available + if vault_role_id: + try: + logger.debug(f"Authenticating to Vault at {vault_addr} using AppRole") + client.auth.approle.login(role_id=vault_role_id) - if not client.is_authenticated(): - logger.error("Failed to authenticate with Vault") + if not client.is_authenticated(): + logger.error("Failed to authenticate with Vault using AppRole") + sys.exit(1) + + logger.debug("Successfully authenticated with Vault using AppRole") + return client + + except Exception as e: + logger.error(f"AppRole authentication failed: {e}") sys.exit(1) - logger.debug("Successfully authenticated with Vault") - return client + # Fallback to Kubernetes authentication if service account token is available + service_account_token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' - except Exception as e: - logger.error(f"Vault authentication failed: {e}") - sys.exit(1) + if os.path.exists(service_account_token_path): + try: + logger.debug(f"Attempting Kubernetes authentication to Vault at {vault_addr}") + + # Read the service account token + with open(service_account_token_path, 'r') as f: + jwt_token = f.read().strip() + + # Authenticate using Kubernetes auth method + client.auth.kubernetes.login( + role=vault_role, + jwt=jwt_token, + mount_point='k8s/au/syd1' + ) + + if not client.is_authenticated(): + logger.error("Failed to authenticate with Vault using Kubernetes auth") + sys.exit(1) + + logger.debug("Successfully authenticated with Vault using Kubernetes auth") + return client + + except Exception as e: + logger.error(f"Kubernetes authentication failed: {e}") + sys.exit(1) + + # No authentication method available + logger.error("Neither VAULT_ROLE_ID environment variable nor Kubernetes service account token is available") + sys.exit(1) def get_gitea_token() -> str: @@ -570,6 +601,31 @@ def check_docker_available() -> bool: return False +def check_native_build_deps() -> bool: + """ + Check if native build dependencies are available (nfpm, envsubst, etc.). + + Returns: + True if native build dependencies are available, False otherwise + """ + required_commands = ['nfpm', 'envsubst', 'wget', 'tar'] + + for cmd in required_commands: + try: + result = subprocess.run( + [cmd, '--version'], + capture_output=True, + text=True, + timeout=5 + ) + if result.returncode != 0: + return False + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + return True + + def cleanup_container(container_name: str) -> None: """ Remove a Docker container. @@ -777,6 +833,131 @@ def build_package_docker( return False +def build_package_native( + package_dir: Path, + package_name: str, + package_version: str, + package_release: str, + dist_dir: Path, + repository: str, + dry_run: bool = False +) -> bool: + """ + Build a package natively without Docker, running build scripts directly. + + Args: + package_dir: Directory containing the package resources + package_name: Name of the package + package_version: Package version + package_release: Package release number + dist_dir: Directory to store built packages + repository: Repository path (e.g., 'almalinux/el9') + dry_run: If True, only show what would be done + + Returns: + True if build succeeded, False otherwise + """ + logger = logging.getLogger(__name__) + + try: + # Ensure dist directory exists with repository structure + package_dist_dir = dist_dir / repository + if not dry_run: + package_dist_dir.mkdir(parents=True, exist_ok=True) + + # Create a temporary workspace for native build + import tempfile + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + app_dir = temp_path / "app" + app_dist_dir = app_dir / "dist" + + # Create directories + if not dry_run: + app_dir.mkdir(parents=True, exist_ok=True) + app_dist_dir.mkdir(parents=True, exist_ok=True) + + # Copy package resources to temp directory + import shutil + temp_resources_dir = app_dir / "resources" + if not dry_run: + shutil.copytree(package_dir / "resources", temp_resources_dir) + + # Read metadata.yaml to get all package fields + metadata_file = package_dir / "metadata.yaml" + metadata = {} + if metadata_file.exists(): + try: + with open(metadata_file, 'r') as f: + metadata = yaml.safe_load(f) or {} + except Exception as e: + logger.warning(f"Could not read metadata.yaml: {e}") + + logger.info(f"Building RPM for {package_name} version {package_version} natively") + + if dry_run: + logger.info(f"[DRY RUN] Would build natively from: {temp_path}") + logger.info("[DRY RUN] Would set environment variables:") + logger.info(f"[DRY RUN] PACKAGE_NAME={package_name}") + logger.info(f"[DRY RUN] PACKAGE_VERSION={package_version}") + logger.info(f"[DRY RUN] PACKAGE_RELEASE={package_release}") + logger.info(f"[DRY RUN] PACKAGE_DESCRIPTION={metadata.get('description', '')}") + logger.info(f"[DRY RUN] Would run: {temp_resources_dir / 'build.sh'}") + logger.info(f"[DRY RUN] Would copy artifacts to: {package_dist_dir}") + return True + + # Set up environment variables like the Dockerfile does + build_env = os.environ.copy() + build_env.update({ + 'PACKAGE_NAME': package_name, + 'PACKAGE_VERSION': package_version, + 'PACKAGE_RELEASE': package_release, + 'PACKAGE_DESCRIPTION': metadata.get('description', ''), + 'PACKAGE_MAINTAINER': metadata.get('maintainer', ''), + 'PACKAGE_HOMEPAGE': metadata.get('homepage', ''), + 'PACKAGE_LICENSE': metadata.get('license', ''), + 'PACKAGE_ARCH': metadata.get('arch', 'amd64'), + 'PACKAGE_PLATFORM': metadata.get('platform', 'linux') + }) + + # Run the build script + build_script = temp_resources_dir / "build.sh" + if not build_script.exists(): + logger.error(f"Build script not found: {build_script}") + return False + + # Make build script executable + build_script.chmod(0o755) + + logger.debug(f"Running build script: {build_script}") + result = subprocess.run( + [str(build_script)], + cwd=app_dir, + env=build_env, + capture_output=True, + text=True + ) + + if result.returncode != 0: + logger.error(f"Native build failed for {package_name}") + logger.error(f"stdout: {result.stdout}") + logger.error(f"stderr: {result.stderr}") + return False + + # Copy artifacts to final destination + if app_dist_dir.exists(): + for item in app_dist_dir.iterdir(): + if item.is_file(): + shutil.copy2(item, package_dist_dir) + + logger.info(f"Successfully built {package_name}-{package_version}-{package_release} natively") + return True + + except Exception as e: + logger.error(f"Unexpected error building {package_name} natively: {e}") + return False + + def cleanup_images(image_pattern: str = "*-builder") -> None: """ Clean up Docker images matching a pattern. @@ -917,7 +1098,8 @@ class Builder: release: str, dry_run: bool = False, force: bool = False, - distro: str = 'almalinux/el9' + distro: str = 'almalinux/el9', + native: bool = False ) -> bool: """ Build a single package. @@ -993,9 +1175,9 @@ class Builder: return False package_info = PackageInfo(package, version, release, package_dir, distro, base_image) - return self._build_package(package_info, dry_run, force) + return self._build_package(package_info, dry_run, force, native) - def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9') -> bool: + def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9', native: bool = False) -> bool: """ Build all packages. @@ -1017,29 +1199,29 @@ class Builder: self.logger.info(f"Found {len(packages)} packages to process") if parallel == 1: - return self._build_sequential(packages, dry_run, force) + return self._build_sequential(packages, dry_run, force, native) else: - return self._build_parallel(packages, dry_run, force, parallel) + return self._build_parallel(packages, dry_run, force, parallel, native) - def _build_sequential(self, packages: List[PackageInfo], dry_run: bool, force: bool) -> bool: + def _build_sequential(self, packages: List[PackageInfo], dry_run: bool, force: bool, native: bool) -> bool: """Build packages sequentially.""" success_count = 0 for package_info in packages: - if self._build_package(package_info, dry_run, force): + if self._build_package(package_info, dry_run, force, native): success_count += 1 self.logger.info(f"Built {success_count}/{len(packages)} packages successfully") return success_count == len(packages) - def _build_parallel(self, packages: List[PackageInfo], dry_run: bool, force: bool, parallel: int) -> bool: + def _build_parallel(self, packages: List[PackageInfo], dry_run: bool, force: bool, parallel: int, native: bool) -> bool: """Build packages in parallel.""" success_count = 0 with ThreadPoolExecutor(max_workers=parallel) as executor: # Submit all build tasks future_to_package = { - executor.submit(self._build_package, pkg, dry_run, force): pkg + executor.submit(self._build_package, pkg, dry_run, force, native): pkg for pkg in packages } @@ -1056,7 +1238,7 @@ class Builder: self.logger.info(f"Built {success_count}/{len(packages)} packages successfully") return success_count == len(packages) - def _build_package(self, package_info: PackageInfo, dry_run: bool, force: bool) -> bool: + def _build_package(self, package_info: PackageInfo, dry_run: bool, force: bool, native: bool = False) -> bool: """ Build a single package. @@ -1081,22 +1263,38 @@ class Builder: ) return True - # Check Docker is available (unless dry run) - if not dry_run and not check_docker_available(): - self.logger.error("Docker is not available or running") + # Check build tool availability (unless dry run) + use_native = native or check_native_build_deps() + use_docker = not use_native and check_docker_available() + + if not dry_run and not use_native and not use_docker: + self.logger.error("Neither native build dependencies nor Docker is available") return False - # Build the package - return build_package_docker( - package_dir=package_info.directory, - package_name=package_info.name, - package_version=package_info.version, - package_release=package_info.release, - dist_dir=self.dist_dir, - repository=package_info.distro, - base_image=package_info.base_image, - dry_run=dry_run - ) + # Build the package using available tool + if use_native: + self.logger.debug(f"Using native build for {package_info.name}") + return build_package_native( + package_dir=package_info.directory, + package_name=package_info.name, + package_version=package_info.version, + package_release=package_info.release, + dist_dir=self.dist_dir, + repository=package_info.distro, + dry_run=dry_run + ) + else: + self.logger.debug(f"Using Docker to build {package_info.name}") + return build_package_docker( + package_dir=package_info.directory, + package_name=package_info.name, + package_version=package_info.version, + package_release=package_info.release, + dist_dir=self.dist_dir, + repository=package_info.distro, + base_image=package_info.base_image, + dry_run=dry_run + ) except Exception as e: self.logger.error(f"Failed to build {package_info}: {e}") @@ -1358,6 +1556,7 @@ def build( distro: str = typer.Option("almalinux/el9", help="Target distro (default: almalinux/el9)"), dry_run: bool = typer.Option(False, "--dry-run", help="Show what would be built without building"), force: bool = typer.Option(False, "--force", help="Build even if package exists in registry"), + native: bool = typer.Option(False, "--native", help="Force native build (skip Docker even if available)"), verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose logging") ): """Build a specific package.""" @@ -1419,7 +1618,8 @@ def build( release=str(release), dry_run=dry_run, force=force, - distro=distro + distro=distro, + native=native ) if not success: @@ -1435,6 +1635,7 @@ def build_all( force: bool = typer.Option(False, "--force", help="Build even if packages exist in registry"), parallel: int = typer.Option(4, help="Number of parallel builds"), distro: str = typer.Option("almalinux/el9", help="Target distro (almalinux/el8, almalinux/el9, or 'all' for all distros)"), + native: bool = typer.Option(False, "--native", help="Force native build (skip Docker even if available)"), verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose logging") ): """Build all packages.""" @@ -1448,7 +1649,8 @@ def build_all( dry_run=dry_run, force=force, parallel=parallel, - distro=distro + distro=distro, + native=native ) if not success: