diff --git a/.woodpecker/build-almalinux8.yaml b/.woodpecker/build-almalinux8.yaml new file mode 100644 index 0000000..ac6f57f --- /dev/null +++ b/.woodpecker/build-almalinux8.yaml @@ -0,0 +1,15 @@ +when: + - event: pull_request + +steps: + - name: build rpms + image: gcr.io/kaniko-project/executor:latest + commands: + - ./tools/build build-all --distro almalinux/el8 --use-kaniko + backend_options: + kubernetes: + serviceAccountName: default + - name: show rpms + image: git.unkin.net/unkin/almalinux8-base:latest + commands: + - find /workspace -type f -name "*.rpm" diff --git a/.woodpecker/build-almalinux9.yaml b/.woodpecker/build-almalinux9.yaml new file mode 100644 index 0000000..ac6f57f --- /dev/null +++ b/.woodpecker/build-almalinux9.yaml @@ -0,0 +1,15 @@ +when: + - event: pull_request + +steps: + - name: build rpms + image: gcr.io/kaniko-project/executor:latest + commands: + - ./tools/build build-all --distro almalinux/el8 --use-kaniko + backend_options: + kubernetes: + serviceAccountName: default + - name: show rpms + image: git.unkin.net/unkin/almalinux8-base:latest + commands: + - find /workspace -type f -name "*.rpm" diff --git a/.woodpecker/pre-commit.yaml b/.woodpecker/pre-commit.yaml new file mode 100644 index 0000000..bf6529f --- /dev/null +++ b/.woodpecker/pre-commit.yaml @@ -0,0 +1,9 @@ +when: + - event: pull_request + +steps: + - name: pre-commit + image: git.unkin.net/unkin/almalinux9-base:latest + commands: + - dnf install uv make -y + - uvx pre-commit run --all-files diff --git a/Makefile b/Makefile index 8c56409..ad08c27 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,10 @@ ROOT_DIR := $(PWD) BUILD_TOOL := $(ROOT_DIR)/tools/build DISTRO ?= almalinux/el9 +# Authentication variables (optional) +# VAULT_ROLE_ID - Use AppRole authentication if set +# VAULT_ROLE - Kubernetes role for service account authentication (default: rpmbuilder) + # Automatically find all packages with metadata.yaml PACKAGES := $(shell find $(ROOT_DIR)/rpms -mindepth 1 -maxdepth 1 -type d -exec test -f {}/metadata.yaml \; -print | xargs -n1 basename | sort) diff --git a/tools/build b/tools/build index ebc2c0a..87913bc 100755 --- a/tools/build +++ b/tools/build @@ -156,7 +156,7 @@ class PackageMetadata: def get_vault_client() -> hvac.Client: """ - Initialize and authenticate Vault client using AppRole authentication. + Initialize and authenticate Vault client using AppRole or Kubernetes authentication. Returns: Authenticated HVAC client @@ -166,10 +166,7 @@ def get_vault_client() -> hvac.Client: # Get required environment variables vault_addr = os.getenv('VAULT_ADDR', 'https://vault.service.consul:8200') vault_role_id = os.getenv('VAULT_ROLE_ID') - - if not vault_role_id: - logger.error("VAULT_ROLE_ID environment variable is required") - sys.exit(1) + vault_role = os.getenv('VAULT_ROLE', 'rpmbuilder') # Initialize Vault client with CA certificate client = hvac.Client( @@ -177,21 +174,55 @@ def get_vault_client() -> hvac.Client: verify='/etc/pki/tls/cert.pem' ) - # Authenticate using AppRole - try: - logger.debug(f"Authenticating to Vault at {vault_addr}") - client.auth.approle.login(role_id=vault_role_id) + # Use AppRole authentication if VAULT_ROLE_ID is available + if vault_role_id: + try: + logger.debug(f"Authenticating to Vault at {vault_addr} using AppRole") + client.auth.approle.login(role_id=vault_role_id) - if not client.is_authenticated(): - logger.error("Failed to authenticate with Vault") + if not client.is_authenticated(): + logger.error("Failed to authenticate with Vault using AppRole") + sys.exit(1) + + logger.debug("Successfully authenticated with Vault using AppRole") + return client + + except Exception as e: + logger.error(f"AppRole authentication failed: {e}") sys.exit(1) - logger.debug("Successfully authenticated with Vault") - return client + # Fallback to Kubernetes authentication if service account token is available + service_account_token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' - except Exception as e: - logger.error(f"Vault authentication failed: {e}") - sys.exit(1) + if os.path.exists(service_account_token_path): + try: + logger.debug(f"Attempting Kubernetes authentication to Vault at {vault_addr}") + + # Read the service account token + with open(service_account_token_path, 'r') as f: + jwt_token = f.read().strip() + + # Authenticate using Kubernetes auth method + client.auth.kubernetes.login( + role=vault_role, + jwt=jwt_token, + mount_point='k8s/au/syd1' + ) + + if not client.is_authenticated(): + logger.error("Failed to authenticate with Vault using Kubernetes auth") + sys.exit(1) + + logger.debug("Successfully authenticated with Vault using Kubernetes auth") + return client + + except Exception as e: + logger.error(f"Kubernetes authentication failed: {e}") + sys.exit(1) + + # No authentication method available + logger.error("Neither VAULT_ROLE_ID environment variable nor Kubernetes service account token is available") + sys.exit(1) def get_gitea_token() -> str: @@ -570,6 +601,25 @@ def check_docker_available() -> bool: return False +def check_kaniko_available() -> bool: + """ + Check if Kaniko executor is available. + + Returns: + True if Kaniko is available, False otherwise + """ + try: + result = subprocess.run( + ['/kaniko/executor', '--version'], + capture_output=True, + text=True, + timeout=10 + ) + return result.returncode == 0 + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + def cleanup_container(container_name: str) -> None: """ Remove a Docker container. @@ -777,6 +827,140 @@ def build_package_docker( return False +def build_package_kaniko( + package_dir: Path, + package_name: str, + package_version: str, + package_release: str, + dist_dir: Path, + repository: str, + base_image: str = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest", + dry_run: bool = False +) -> bool: + """ + Build a package using Kaniko without Docker daemon. + + Args: + package_dir: Directory containing the package resources + package_name: Name of the package + package_version: Package version + package_release: Package release number + dist_dir: Directory to store built packages + repository: Repository path (e.g., 'almalinux/el9') + base_image: Base Docker image to use for building + dry_run: If True, only show what would be done + + Returns: + True if build succeeded, False otherwise + """ + logger = logging.getLogger(__name__) + + try: + # Ensure dist directory exists with repository structure + package_dist_dir = dist_dir / repository + if not dry_run: + package_dist_dir.mkdir(parents=True, exist_ok=True) + + # Create a temporary workspace for Kaniko + import tempfile + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Copy package resources to temp directory + import shutil + temp_resources_dir = temp_path / "resources" + shutil.copytree(package_dir / "resources", temp_resources_dir) + + # Copy Dockerfile to temp directory + central_dockerfile = package_dir.parent.parent / "Dockerfile" + shutil.copy2(central_dockerfile, temp_path / "Dockerfile") + + # Read metadata.yaml to get all package fields + metadata_file = package_dir / "metadata.yaml" + metadata = {} + if metadata_file.exists(): + try: + with open(metadata_file, 'r') as f: + metadata = yaml.safe_load(f) or {} + except Exception as e: + logger.warning(f"Could not read metadata.yaml: {e}") + + logger.info(f"Building RPM for {package_name} version {package_version} using Kaniko") + + if dry_run: + logger.info(f"[DRY RUN] Would use Kaniko to build from: {temp_path}") + logger.info(f"[DRY RUN] Would use base image: {base_image}") + logger.info("[DRY RUN] Would pass build arguments:") + logger.info(f"[DRY RUN] PACKAGE_NAME={package_name}") + logger.info(f"[DRY RUN] PACKAGE_VERSION={package_version}") + logger.info(f"[DRY RUN] PACKAGE_RELEASE={package_release}") + logger.info(f"[DRY RUN] Would copy artifacts to: {package_dist_dir}") + return True + + # Build using Kaniko + kaniko_args = [ + '/kaniko/executor', + '--context', str(temp_path), + '--dockerfile', str(temp_path / "Dockerfile"), + '--build-arg', f'BASE_IMAGE={base_image}', + '--build-arg', f'PACKAGE_NAME={package_name}', + '--build-arg', f'PACKAGE_VERSION={package_version}', + '--build-arg', f'PACKAGE_RELEASE={package_release}', + '--build-arg', f'PACKAGE_DESCRIPTION={metadata.get("description", "")}', + '--build-arg', f'PACKAGE_MAINTAINER={metadata.get("maintainer", "")}', + '--build-arg', f'PACKAGE_HOMEPAGE={metadata.get("homepage", "")}', + '--build-arg', f'PACKAGE_LICENSE={metadata.get("license", "")}', + '--build-arg', f'PACKAGE_ARCH={metadata.get("arch", "amd64")}', + '--build-arg', f'PACKAGE_PLATFORM={metadata.get("platform", "linux")}', + '--no-push', # Don't push to registry, just build + '--tar-path', str(temp_path / "image.tar") + ] + + logger.debug(f"Running: {' '.join(kaniko_args)}") + result = subprocess.run( + kaniko_args, + capture_output=True, + text=True, + cwd=temp_path + ) + + if result.returncode != 0: + logger.error(f"Kaniko build failed for {package_name}") + logger.error(f"stdout: {result.stdout}") + logger.error(f"stderr: {result.stderr}") + return False + + # Extract the artifacts from the built image + extract_args = [ + 'tar', '-xf', str(temp_path / "image.tar"), + '-C', str(temp_path), + '--strip-components=1', + 'app/dist' + ] + + logger.debug(f"Running: {' '.join(extract_args)}") + result = subprocess.run(extract_args, capture_output=True, text=True) + + if result.returncode != 0: + logger.error(f"Failed to extract artifacts for {package_name}") + logger.error(f"stderr: {result.stderr}") + return False + + # Copy artifacts to final destination + extracted_dist = temp_path / "app" / "dist" + if extracted_dist.exists(): + for item in extracted_dist.iterdir(): + if item.is_file(): + shutil.copy2(item, package_dist_dir) + + logger.info(f"Successfully built {package_name}-{package_version}-{package_release} using Kaniko") + return True + + except Exception as e: + logger.error(f"Unexpected error building {package_name} with Kaniko: {e}") + return False + + def cleanup_images(image_pattern: str = "*-builder") -> None: """ Clean up Docker images matching a pattern. @@ -1081,22 +1265,39 @@ class Builder: ) return True - # Check Docker is available (unless dry run) - if not dry_run and not check_docker_available(): - self.logger.error("Docker is not available or running") + # Check build tool availability (unless dry run) + use_kaniko = check_kaniko_available() + use_docker = not use_kaniko and check_docker_available() + + if not dry_run and not use_kaniko and not use_docker: + self.logger.error("Neither Kaniko nor Docker is available") return False - # Build the package - return build_package_docker( - package_dir=package_info.directory, - package_name=package_info.name, - package_version=package_info.version, - package_release=package_info.release, - dist_dir=self.dist_dir, - repository=package_info.distro, - base_image=package_info.base_image, - dry_run=dry_run - ) + # Build the package using available tool + if use_kaniko: + self.logger.debug(f"Using Kaniko to build {package_info.name}") + return build_package_kaniko( + package_dir=package_info.directory, + package_name=package_info.name, + package_version=package_info.version, + package_release=package_info.release, + dist_dir=self.dist_dir, + repository=package_info.distro, + base_image=package_info.base_image, + dry_run=dry_run + ) + else: + self.logger.debug(f"Using Docker to build {package_info.name}") + return build_package_docker( + package_dir=package_info.directory, + package_name=package_info.name, + package_version=package_info.version, + package_release=package_info.release, + dist_dir=self.dist_dir, + repository=package_info.distro, + base_image=package_info.base_image, + dry_run=dry_run + ) except Exception as e: self.logger.error(f"Failed to build {package_info}: {e}")