feat: migrate to woodpeckerci
Build / build-8 (pull_request) Successful in 13s
Build / build-9 (pull_request) Successful in 14s
ci/woodpecker/pr/build-almalinux8 Pipeline failed
ci/woodpecker/pr/build-almalinux9 Pipeline failed
ci/woodpecker/pr/pre-commit Pipeline failed

- update build tool for kubernetes auth
- update build tool for kaniko
- add woodpecker pre-commit and build jobs
This commit is contained in:
2026-03-07 11:19:13 +11:00
parent b585cca9f0
commit 05ffdfed83
5 changed files with 274 additions and 30 deletions
+15
View File
@@ -0,0 +1,15 @@
when:
- event: pull_request
steps:
- name: build rpms
image: gcr.io/kaniko-project/executor:latest
commands:
- ./tools/build build-all --distro almalinux/el8 --use-kaniko
backend_options:
kubernetes:
serviceAccountName: default
- name: show rpms
image: git.unkin.net/unkin/almalinux8-base:latest
commands:
- find /workspace -type f -name "*.rpm"
+15
View File
@@ -0,0 +1,15 @@
when:
- event: pull_request
steps:
- name: build rpms
image: gcr.io/kaniko-project/executor:latest
commands:
- ./tools/build build-all --distro almalinux/el8 --use-kaniko
backend_options:
kubernetes:
serviceAccountName: default
- name: show rpms
image: git.unkin.net/unkin/almalinux8-base:latest
commands:
- find /workspace -type f -name "*.rpm"
+9
View File
@@ -0,0 +1,9 @@
when:
- event: pull_request
steps:
- name: pre-commit
image: git.unkin.net/unkin/almalinux9-base:latest
commands:
- dnf install uv make -y
- uvx pre-commit run --all-files
+4
View File
@@ -3,6 +3,10 @@ ROOT_DIR := $(PWD)
BUILD_TOOL := $(ROOT_DIR)/tools/build BUILD_TOOL := $(ROOT_DIR)/tools/build
DISTRO ?= almalinux/el9 DISTRO ?= almalinux/el9
# Authentication variables (optional)
# VAULT_ROLE_ID - Use AppRole authentication if set
# VAULT_ROLE - Kubernetes role for service account authentication (default: rpmbuilder)
# Automatically find all packages with metadata.yaml # Automatically find all packages with metadata.yaml
PACKAGES := $(shell find $(ROOT_DIR)/rpms -mindepth 1 -maxdepth 1 -type d -exec test -f {}/metadata.yaml \; -print | xargs -n1 basename | sort) PACKAGES := $(shell find $(ROOT_DIR)/rpms -mindepth 1 -maxdepth 1 -type d -exec test -f {}/metadata.yaml \; -print | xargs -n1 basename | sort)
+215 -14
View File
@@ -156,7 +156,7 @@ class PackageMetadata:
def get_vault_client() -> hvac.Client: def get_vault_client() -> hvac.Client:
""" """
Initialize and authenticate Vault client using AppRole authentication. Initialize and authenticate Vault client using AppRole or Kubernetes authentication.
Returns: Returns:
Authenticated HVAC client Authenticated HVAC client
@@ -166,10 +166,7 @@ def get_vault_client() -> hvac.Client:
# Get required environment variables # Get required environment variables
vault_addr = os.getenv('VAULT_ADDR', 'https://vault.service.consul:8200') vault_addr = os.getenv('VAULT_ADDR', 'https://vault.service.consul:8200')
vault_role_id = os.getenv('VAULT_ROLE_ID') vault_role_id = os.getenv('VAULT_ROLE_ID')
vault_role = os.getenv('VAULT_ROLE', 'rpmbuilder')
if not vault_role_id:
logger.error("VAULT_ROLE_ID environment variable is required")
sys.exit(1)
# Initialize Vault client with CA certificate # Initialize Vault client with CA certificate
client = hvac.Client( client = hvac.Client(
@@ -177,20 +174,54 @@ def get_vault_client() -> hvac.Client:
verify='/etc/pki/tls/cert.pem' verify='/etc/pki/tls/cert.pem'
) )
# Authenticate using AppRole # Use AppRole authentication if VAULT_ROLE_ID is available
if vault_role_id:
try: try:
logger.debug(f"Authenticating to Vault at {vault_addr}") logger.debug(f"Authenticating to Vault at {vault_addr} using AppRole")
client.auth.approle.login(role_id=vault_role_id) client.auth.approle.login(role_id=vault_role_id)
if not client.is_authenticated(): if not client.is_authenticated():
logger.error("Failed to authenticate with Vault") logger.error("Failed to authenticate with Vault using AppRole")
sys.exit(1) sys.exit(1)
logger.debug("Successfully authenticated with Vault") logger.debug("Successfully authenticated with Vault using AppRole")
return client return client
except Exception as e: except Exception as e:
logger.error(f"Vault authentication failed: {e}") logger.error(f"AppRole authentication failed: {e}")
sys.exit(1)
# Fallback to Kubernetes authentication if service account token is available
service_account_token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token'
if os.path.exists(service_account_token_path):
try:
logger.debug(f"Attempting Kubernetes authentication to Vault at {vault_addr}")
# Read the service account token
with open(service_account_token_path, 'r') as f:
jwt_token = f.read().strip()
# Authenticate using Kubernetes auth method
client.auth.kubernetes.login(
role=vault_role,
jwt=jwt_token,
mount_point='k8s/au/syd1'
)
if not client.is_authenticated():
logger.error("Failed to authenticate with Vault using Kubernetes auth")
sys.exit(1)
logger.debug("Successfully authenticated with Vault using Kubernetes auth")
return client
except Exception as e:
logger.error(f"Kubernetes authentication failed: {e}")
sys.exit(1)
# No authentication method available
logger.error("Neither VAULT_ROLE_ID environment variable nor Kubernetes service account token is available")
sys.exit(1) sys.exit(1)
@@ -570,6 +601,25 @@ def check_docker_available() -> bool:
return False return False
def check_kaniko_available() -> bool:
"""
Check if Kaniko executor is available.
Returns:
True if Kaniko is available, False otherwise
"""
try:
result = subprocess.run(
['/kaniko/executor', '--version'],
capture_output=True,
text=True,
timeout=10
)
return result.returncode == 0
except (subprocess.TimeoutExpired, FileNotFoundError):
return False
def cleanup_container(container_name: str) -> None: def cleanup_container(container_name: str) -> None:
""" """
Remove a Docker container. Remove a Docker container.
@@ -777,6 +827,140 @@ def build_package_docker(
return False return False
def build_package_kaniko(
package_dir: Path,
package_name: str,
package_version: str,
package_release: str,
dist_dir: Path,
repository: str,
base_image: str = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest",
dry_run: bool = False
) -> bool:
"""
Build a package using Kaniko without Docker daemon.
Args:
package_dir: Directory containing the package resources
package_name: Name of the package
package_version: Package version
package_release: Package release number
dist_dir: Directory to store built packages
repository: Repository path (e.g., 'almalinux/el9')
base_image: Base Docker image to use for building
dry_run: If True, only show what would be done
Returns:
True if build succeeded, False otherwise
"""
logger = logging.getLogger(__name__)
try:
# Ensure dist directory exists with repository structure
package_dist_dir = dist_dir / repository
if not dry_run:
package_dist_dir.mkdir(parents=True, exist_ok=True)
# Create a temporary workspace for Kaniko
import tempfile
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
# Copy package resources to temp directory
import shutil
temp_resources_dir = temp_path / "resources"
shutil.copytree(package_dir / "resources", temp_resources_dir)
# Copy Dockerfile to temp directory
central_dockerfile = package_dir.parent.parent / "Dockerfile"
shutil.copy2(central_dockerfile, temp_path / "Dockerfile")
# Read metadata.yaml to get all package fields
metadata_file = package_dir / "metadata.yaml"
metadata = {}
if metadata_file.exists():
try:
with open(metadata_file, 'r') as f:
metadata = yaml.safe_load(f) or {}
except Exception as e:
logger.warning(f"Could not read metadata.yaml: {e}")
logger.info(f"Building RPM for {package_name} version {package_version} using Kaniko")
if dry_run:
logger.info(f"[DRY RUN] Would use Kaniko to build from: {temp_path}")
logger.info(f"[DRY RUN] Would use base image: {base_image}")
logger.info("[DRY RUN] Would pass build arguments:")
logger.info(f"[DRY RUN] PACKAGE_NAME={package_name}")
logger.info(f"[DRY RUN] PACKAGE_VERSION={package_version}")
logger.info(f"[DRY RUN] PACKAGE_RELEASE={package_release}")
logger.info(f"[DRY RUN] Would copy artifacts to: {package_dist_dir}")
return True
# Build using Kaniko
kaniko_args = [
'/kaniko/executor',
'--context', str(temp_path),
'--dockerfile', str(temp_path / "Dockerfile"),
'--build-arg', f'BASE_IMAGE={base_image}',
'--build-arg', f'PACKAGE_NAME={package_name}',
'--build-arg', f'PACKAGE_VERSION={package_version}',
'--build-arg', f'PACKAGE_RELEASE={package_release}',
'--build-arg', f'PACKAGE_DESCRIPTION={metadata.get("description", "")}',
'--build-arg', f'PACKAGE_MAINTAINER={metadata.get("maintainer", "")}',
'--build-arg', f'PACKAGE_HOMEPAGE={metadata.get("homepage", "")}',
'--build-arg', f'PACKAGE_LICENSE={metadata.get("license", "")}',
'--build-arg', f'PACKAGE_ARCH={metadata.get("arch", "amd64")}',
'--build-arg', f'PACKAGE_PLATFORM={metadata.get("platform", "linux")}',
'--no-push', # Don't push to registry, just build
'--tar-path', str(temp_path / "image.tar")
]
logger.debug(f"Running: {' '.join(kaniko_args)}")
result = subprocess.run(
kaniko_args,
capture_output=True,
text=True,
cwd=temp_path
)
if result.returncode != 0:
logger.error(f"Kaniko build failed for {package_name}")
logger.error(f"stdout: {result.stdout}")
logger.error(f"stderr: {result.stderr}")
return False
# Extract the artifacts from the built image
extract_args = [
'tar', '-xf', str(temp_path / "image.tar"),
'-C', str(temp_path),
'--strip-components=1',
'app/dist'
]
logger.debug(f"Running: {' '.join(extract_args)}")
result = subprocess.run(extract_args, capture_output=True, text=True)
if result.returncode != 0:
logger.error(f"Failed to extract artifacts for {package_name}")
logger.error(f"stderr: {result.stderr}")
return False
# Copy artifacts to final destination
extracted_dist = temp_path / "app" / "dist"
if extracted_dist.exists():
for item in extracted_dist.iterdir():
if item.is_file():
shutil.copy2(item, package_dist_dir)
logger.info(f"Successfully built {package_name}-{package_version}-{package_release} using Kaniko")
return True
except Exception as e:
logger.error(f"Unexpected error building {package_name} with Kaniko: {e}")
return False
def cleanup_images(image_pattern: str = "*-builder") -> None: def cleanup_images(image_pattern: str = "*-builder") -> None:
""" """
Clean up Docker images matching a pattern. Clean up Docker images matching a pattern.
@@ -1081,12 +1265,29 @@ class Builder:
) )
return True return True
# Check Docker is available (unless dry run) # Check build tool availability (unless dry run)
if not dry_run and not check_docker_available(): use_kaniko = check_kaniko_available()
self.logger.error("Docker is not available or running") use_docker = not use_kaniko and check_docker_available()
if not dry_run and not use_kaniko and not use_docker:
self.logger.error("Neither Kaniko nor Docker is available")
return False return False
# Build the package # Build the package using available tool
if use_kaniko:
self.logger.debug(f"Using Kaniko to build {package_info.name}")
return build_package_kaniko(
package_dir=package_info.directory,
package_name=package_info.name,
package_version=package_info.version,
package_release=package_info.release,
dist_dir=self.dist_dir,
repository=package_info.distro,
base_image=package_info.base_image,
dry_run=dry_run
)
else:
self.logger.debug(f"Using Docker to build {package_info.name}")
return build_package_docker( return build_package_docker(
package_dir=package_info.directory, package_dir=package_info.directory,
package_name=package_info.name, package_name=package_info.name,