feat: migrate to woodpeckerci
Build / build-9 (pull_request) Successful in 10s
Build / build-8 (pull_request) Successful in 10s
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/build-almalinux9 Pipeline was successful
ci/woodpecker/pr/build-almalinux8 Pipeline was successful
Build / build-9 (pull_request) Successful in 10s
Build / build-8 (pull_request) Successful in 10s
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/build-almalinux9 Pipeline was successful
ci/woodpecker/pr/build-almalinux8 Pipeline was successful
- update build tool for kubernetes auth - update build tool to build packages without docker (native + buildah) - add woodpecker pre-commit and build jobs
This commit is contained in:
@@ -0,0 +1,20 @@
|
|||||||
|
when:
|
||||||
|
- event: pull_request
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: build rpms
|
||||||
|
image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest
|
||||||
|
commands:
|
||||||
|
- mkdir -p /woodpecker/rpms /apps/
|
||||||
|
- ln -s /woodpecker/rpms /apps/dist
|
||||||
|
- dnf install buildah -y
|
||||||
|
- ./tools/build build-all --distro almalinux/el8 --buildah
|
||||||
|
privileged: true
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
|
||||||
|
- name: show rpms
|
||||||
|
image: git.unkin.net/unkin/almalinux8-base:latest
|
||||||
|
commands:
|
||||||
|
- find /woodpecker/rpms -type f -name "*.rpm"
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
when:
|
||||||
|
- event: pull_request
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: build rpms
|
||||||
|
image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest
|
||||||
|
commands:
|
||||||
|
- mkdir -p /woodpecker/rpms /apps/
|
||||||
|
- ln -s /woodpecker/rpms /apps/dist
|
||||||
|
- dnf install buildah -y
|
||||||
|
- ./tools/build build-all --distro almalinux/el9 --buildah
|
||||||
|
privileged: true
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
|
||||||
|
- name: show rpms
|
||||||
|
image: git.unkin.net/unkin/almalinux9-base:latest
|
||||||
|
commands:
|
||||||
|
- find /woodpecker/rpms -type f -name "*.rpm"
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
when:
|
||||||
|
- event: pull_request
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: pre-commit
|
||||||
|
image: git.unkin.net/unkin/almalinux9-base:latest
|
||||||
|
commands:
|
||||||
|
- dnf install uv make -y
|
||||||
|
- uvx pre-commit run --all-files
|
||||||
@@ -3,6 +3,10 @@ ROOT_DIR := $(PWD)
|
|||||||
BUILD_TOOL := $(ROOT_DIR)/tools/build
|
BUILD_TOOL := $(ROOT_DIR)/tools/build
|
||||||
DISTRO ?= almalinux/el9
|
DISTRO ?= almalinux/el9
|
||||||
|
|
||||||
|
# Authentication variables (optional)
|
||||||
|
# VAULT_ROLE_ID - Use AppRole authentication if set
|
||||||
|
# VAULT_ROLE - Kubernetes role for service account authentication (default: rpmbuilder)
|
||||||
|
|
||||||
# Automatically find all packages with metadata.yaml
|
# Automatically find all packages with metadata.yaml
|
||||||
PACKAGES := $(shell find $(ROOT_DIR)/rpms -mindepth 1 -maxdepth 1 -type d -exec test -f {}/metadata.yaml \; -print | xargs -n1 basename | sort)
|
PACKAGES := $(shell find $(ROOT_DIR)/rpms -mindepth 1 -maxdepth 1 -type d -exec test -f {}/metadata.yaml \; -print | xargs -n1 basename | sort)
|
||||||
|
|
||||||
@@ -22,6 +26,16 @@ build-all:
|
|||||||
@echo "Building all packages using Python tooling for distro $(DISTRO)..."
|
@echo "Building all packages using Python tooling for distro $(DISTRO)..."
|
||||||
$(BUILD_TOOL) build-all --distro $(DISTRO)
|
$(BUILD_TOOL) build-all --distro $(DISTRO)
|
||||||
|
|
||||||
|
# Build all packages using native build (no Docker)
|
||||||
|
build-all-native:
|
||||||
|
@echo "Building all packages natively (no Docker) for distro $(DISTRO)..."
|
||||||
|
$(BUILD_TOOL) build-all --distro $(DISTRO) --native
|
||||||
|
|
||||||
|
# Build all packages using Buildah
|
||||||
|
build-all-buildah:
|
||||||
|
@echo "Building all packages using Buildah for distro $(DISTRO)..."
|
||||||
|
$(BUILD_TOOL) build-all --distro $(DISTRO) --buildah
|
||||||
|
|
||||||
# Build specific package using Python tool
|
# Build specific package using Python tool
|
||||||
.PHONY: $(PACKAGES)
|
.PHONY: $(PACKAGES)
|
||||||
$(PACKAGES):
|
$(PACKAGES):
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ set -e
|
|||||||
# Download and extract unrar (with version formatting)
|
# Download and extract unrar (with version formatting)
|
||||||
export DOWNLOAD_VERSION=$(echo $PACKAGE_VERSION | sed s/\\.//)
|
export DOWNLOAD_VERSION=$(echo $PACKAGE_VERSION | sed s/\\.//)
|
||||||
curl -L -o /app/rarlinux.tar.gz https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/rarlab/rar/rarlinux-x64-${DOWNLOAD_VERSION}.tar.gz
|
curl -L -o /app/rarlinux.tar.gz https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/rarlab/rar/rarlinux-x64-${DOWNLOAD_VERSION}.tar.gz
|
||||||
tar xf /app/rarlinux.tar.gz
|
tar xf /app/rarlinux.tar.gz -C /app
|
||||||
mv /app/rar/unrar /app/
|
mv /app/rar/unrar /app/
|
||||||
|
|
||||||
# Process the nfpm.yaml template with environment variables
|
# Process the nfpm.yaml template with environment variables
|
||||||
|
|||||||
+433
-42
@@ -156,7 +156,7 @@ class PackageMetadata:
|
|||||||
|
|
||||||
def get_vault_client() -> hvac.Client:
|
def get_vault_client() -> hvac.Client:
|
||||||
"""
|
"""
|
||||||
Initialize and authenticate Vault client using AppRole authentication.
|
Initialize and authenticate Vault client using AppRole or Kubernetes authentication.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Authenticated HVAC client
|
Authenticated HVAC client
|
||||||
@@ -166,10 +166,7 @@ def get_vault_client() -> hvac.Client:
|
|||||||
# Get required environment variables
|
# Get required environment variables
|
||||||
vault_addr = os.getenv('VAULT_ADDR', 'https://vault.service.consul:8200')
|
vault_addr = os.getenv('VAULT_ADDR', 'https://vault.service.consul:8200')
|
||||||
vault_role_id = os.getenv('VAULT_ROLE_ID')
|
vault_role_id = os.getenv('VAULT_ROLE_ID')
|
||||||
|
vault_role = os.getenv('VAULT_ROLE', 'rpmbuilder')
|
||||||
if not vault_role_id:
|
|
||||||
logger.error("VAULT_ROLE_ID environment variable is required")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# Initialize Vault client with CA certificate
|
# Initialize Vault client with CA certificate
|
||||||
client = hvac.Client(
|
client = hvac.Client(
|
||||||
@@ -177,21 +174,55 @@ def get_vault_client() -> hvac.Client:
|
|||||||
verify='/etc/pki/tls/cert.pem'
|
verify='/etc/pki/tls/cert.pem'
|
||||||
)
|
)
|
||||||
|
|
||||||
# Authenticate using AppRole
|
# Use AppRole authentication if VAULT_ROLE_ID is available
|
||||||
try:
|
if vault_role_id:
|
||||||
logger.debug(f"Authenticating to Vault at {vault_addr}")
|
try:
|
||||||
client.auth.approle.login(role_id=vault_role_id)
|
logger.debug(f"Authenticating to Vault at {vault_addr} using AppRole")
|
||||||
|
client.auth.approle.login(role_id=vault_role_id)
|
||||||
|
|
||||||
if not client.is_authenticated():
|
if not client.is_authenticated():
|
||||||
logger.error("Failed to authenticate with Vault")
|
logger.error("Failed to authenticate with Vault using AppRole")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
logger.debug("Successfully authenticated with Vault using AppRole")
|
||||||
|
return client
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"AppRole authentication failed: {e}")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
logger.debug("Successfully authenticated with Vault")
|
# Fallback to Kubernetes authentication if service account token is available
|
||||||
return client
|
service_account_token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token'
|
||||||
|
|
||||||
except Exception as e:
|
if os.path.exists(service_account_token_path):
|
||||||
logger.error(f"Vault authentication failed: {e}")
|
try:
|
||||||
sys.exit(1)
|
logger.debug(f"Attempting Kubernetes authentication to Vault at {vault_addr}")
|
||||||
|
|
||||||
|
# Read the service account token
|
||||||
|
with open(service_account_token_path, 'r') as f:
|
||||||
|
jwt_token = f.read().strip()
|
||||||
|
|
||||||
|
# Authenticate using Kubernetes auth method
|
||||||
|
client.auth.kubernetes.login(
|
||||||
|
role=vault_role,
|
||||||
|
jwt=jwt_token,
|
||||||
|
mount_point='k8s/au/syd1'
|
||||||
|
)
|
||||||
|
|
||||||
|
if not client.is_authenticated():
|
||||||
|
logger.error("Failed to authenticate with Vault using Kubernetes auth")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
logger.debug("Successfully authenticated with Vault using Kubernetes auth")
|
||||||
|
return client
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Kubernetes authentication failed: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# No authentication method available
|
||||||
|
logger.error("Neither VAULT_ROLE_ID environment variable nor Kubernetes service account token is available")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
def get_gitea_token() -> str:
|
def get_gitea_token() -> str:
|
||||||
@@ -570,6 +601,50 @@ def check_docker_available() -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def check_native_build_deps() -> bool:
|
||||||
|
"""
|
||||||
|
Check if native build dependencies are available (nfpm, envsubst, etc.).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if native build dependencies are available, False otherwise
|
||||||
|
"""
|
||||||
|
required_commands = ['nfpm', 'envsubst', 'wget', 'tar']
|
||||||
|
|
||||||
|
for cmd in required_commands:
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
[cmd, '--version'],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=5
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return False
|
||||||
|
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def check_buildah_available() -> bool:
|
||||||
|
"""
|
||||||
|
Check if Buildah is available.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if Buildah is available, False otherwise
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
['buildah', 'version'],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=10
|
||||||
|
)
|
||||||
|
return result.returncode == 0
|
||||||
|
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def cleanup_container(container_name: str) -> None:
|
def cleanup_container(container_name: str) -> None:
|
||||||
"""
|
"""
|
||||||
Remove a Docker container.
|
Remove a Docker container.
|
||||||
@@ -777,6 +852,283 @@ def build_package_docker(
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def build_package_native(
|
||||||
|
package_dir: Path,
|
||||||
|
package_name: str,
|
||||||
|
package_version: str,
|
||||||
|
package_release: str,
|
||||||
|
dist_dir: Path,
|
||||||
|
repository: str,
|
||||||
|
dry_run: bool = False
|
||||||
|
) -> bool:
|
||||||
|
"""
|
||||||
|
Build a package natively without Docker, running build scripts directly.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
package_dir: Directory containing the package resources
|
||||||
|
package_name: Name of the package
|
||||||
|
package_version: Package version
|
||||||
|
package_release: Package release number
|
||||||
|
dist_dir: Directory to store built packages
|
||||||
|
repository: Repository path (e.g., 'almalinux/el9')
|
||||||
|
dry_run: If True, only show what would be done
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if build succeeded, False otherwise
|
||||||
|
"""
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Ensure dist directory exists with repository structure
|
||||||
|
package_dist_dir = dist_dir / repository
|
||||||
|
if not dry_run:
|
||||||
|
package_dist_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Create a temporary workspace for native build
|
||||||
|
import tempfile
|
||||||
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
|
temp_path = Path(temp_dir)
|
||||||
|
app_dir = temp_path / "app"
|
||||||
|
app_dist_dir = app_dir / "dist"
|
||||||
|
|
||||||
|
# Create directories
|
||||||
|
if not dry_run:
|
||||||
|
app_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
app_dist_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Copy package resources to temp directory
|
||||||
|
import shutil
|
||||||
|
temp_resources_dir = app_dir / "resources"
|
||||||
|
if not dry_run:
|
||||||
|
shutil.copytree(package_dir / "resources", temp_resources_dir)
|
||||||
|
|
||||||
|
# Read metadata.yaml to get all package fields
|
||||||
|
metadata_file = package_dir / "metadata.yaml"
|
||||||
|
metadata = {}
|
||||||
|
if metadata_file.exists():
|
||||||
|
try:
|
||||||
|
with open(metadata_file, 'r') as f:
|
||||||
|
metadata = yaml.safe_load(f) or {}
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Could not read metadata.yaml: {e}")
|
||||||
|
|
||||||
|
logger.info(f"Building RPM for {package_name} version {package_version} natively")
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
logger.info(f"[DRY RUN] Would build natively from: {temp_path}")
|
||||||
|
logger.info("[DRY RUN] Would set environment variables:")
|
||||||
|
logger.info(f"[DRY RUN] PACKAGE_NAME={package_name}")
|
||||||
|
logger.info(f"[DRY RUN] PACKAGE_VERSION={package_version}")
|
||||||
|
logger.info(f"[DRY RUN] PACKAGE_RELEASE={package_release}")
|
||||||
|
logger.info(f"[DRY RUN] PACKAGE_DESCRIPTION={metadata.get('description', '')}")
|
||||||
|
logger.info(f"[DRY RUN] Would run: {temp_resources_dir / 'build.sh'}")
|
||||||
|
logger.info(f"[DRY RUN] Would copy artifacts to: {package_dist_dir}")
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Set up environment variables like the Dockerfile does
|
||||||
|
build_env = os.environ.copy()
|
||||||
|
build_env.update({
|
||||||
|
'PACKAGE_NAME': package_name,
|
||||||
|
'PACKAGE_VERSION': package_version,
|
||||||
|
'PACKAGE_RELEASE': package_release,
|
||||||
|
'PACKAGE_DESCRIPTION': metadata.get('description', ''),
|
||||||
|
'PACKAGE_MAINTAINER': metadata.get('maintainer', ''),
|
||||||
|
'PACKAGE_HOMEPAGE': metadata.get('homepage', ''),
|
||||||
|
'PACKAGE_LICENSE': metadata.get('license', ''),
|
||||||
|
'PACKAGE_ARCH': metadata.get('arch', 'amd64'),
|
||||||
|
'PACKAGE_PLATFORM': metadata.get('platform', 'linux')
|
||||||
|
})
|
||||||
|
|
||||||
|
# Run the build script
|
||||||
|
build_script = temp_resources_dir / "build.sh"
|
||||||
|
if not build_script.exists():
|
||||||
|
logger.error(f"Build script not found: {build_script}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Make build script executable
|
||||||
|
build_script.chmod(0o755)
|
||||||
|
|
||||||
|
logger.debug(f"Running build script: {build_script}")
|
||||||
|
result = subprocess.run(
|
||||||
|
[str(build_script)],
|
||||||
|
cwd=app_dir,
|
||||||
|
env=build_env,
|
||||||
|
capture_output=True,
|
||||||
|
text=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.error(f"Native build failed for {package_name}")
|
||||||
|
logger.error(f"stdout: {result.stdout}")
|
||||||
|
logger.error(f"stderr: {result.stderr}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Copy artifacts to final destination
|
||||||
|
if app_dist_dir.exists():
|
||||||
|
for item in app_dist_dir.iterdir():
|
||||||
|
if item.is_file():
|
||||||
|
shutil.copy2(item, package_dist_dir)
|
||||||
|
|
||||||
|
logger.info(f"Successfully built {package_name}-{package_version}-{package_release} natively")
|
||||||
|
return True
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Unexpected error building {package_name} natively: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def build_package_buildah(
|
||||||
|
package_dir: Path,
|
||||||
|
package_name: str,
|
||||||
|
package_version: str,
|
||||||
|
package_release: str,
|
||||||
|
dist_dir: Path,
|
||||||
|
repository: str,
|
||||||
|
base_image: str = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest",
|
||||||
|
dry_run: bool = False
|
||||||
|
) -> bool:
|
||||||
|
"""
|
||||||
|
Build a package using Buildah without Docker daemon.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
package_dir: Directory containing the package resources
|
||||||
|
package_name: Name of the package
|
||||||
|
package_version: Package version
|
||||||
|
package_release: Package release number
|
||||||
|
dist_dir: Directory to store built packages
|
||||||
|
repository: Repository path (e.g., 'almalinux/el9')
|
||||||
|
base_image: Base Docker image to use for building
|
||||||
|
dry_run: If True, only show what would be done
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if build succeeded, False otherwise
|
||||||
|
"""
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Ensure dist directory exists with repository structure
|
||||||
|
package_dist_dir = dist_dir / repository
|
||||||
|
if not dry_run:
|
||||||
|
package_dist_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Generate container name
|
||||||
|
container_name = f"{package_name}-{package_version}-buildah"
|
||||||
|
|
||||||
|
# Read metadata.yaml to get all package fields
|
||||||
|
metadata_file = package_dir / "metadata.yaml"
|
||||||
|
metadata = {}
|
||||||
|
if metadata_file.exists():
|
||||||
|
try:
|
||||||
|
with open(metadata_file, 'r') as f:
|
||||||
|
metadata = yaml.safe_load(f) or {}
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Could not read metadata.yaml: {e}")
|
||||||
|
|
||||||
|
logger.info(f"Building RPM for {package_name} version {package_version} using Buildah")
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
logger.info(f"[DRY RUN] Would use Buildah to build from: {base_image}")
|
||||||
|
logger.info("[DRY RUN] Would set environment variables:")
|
||||||
|
logger.info(f"[DRY RUN] PACKAGE_NAME={package_name}")
|
||||||
|
logger.info(f"[DRY RUN] PACKAGE_VERSION={package_version}")
|
||||||
|
logger.info(f"[DRY RUN] PACKAGE_RELEASE={package_release}")
|
||||||
|
logger.info(f"[DRY RUN] Would copy artifacts to: {package_dist_dir}")
|
||||||
|
return True
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Step 1: Create a working container from base image
|
||||||
|
from_args = ['buildah', 'from', '--name', container_name, base_image]
|
||||||
|
logger.debug(f"Running: {' '.join(from_args)}")
|
||||||
|
result = subprocess.run(from_args, capture_output=True, text=True)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.error(f"Failed to create Buildah container from {base_image}")
|
||||||
|
logger.error(f"stderr: {result.stderr}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Step 2: Set environment variables
|
||||||
|
env_vars = {
|
||||||
|
'PACKAGE_NAME': package_name,
|
||||||
|
'PACKAGE_VERSION': package_version,
|
||||||
|
'PACKAGE_RELEASE': package_release,
|
||||||
|
'PACKAGE_DESCRIPTION': metadata.get('description', ''),
|
||||||
|
'PACKAGE_MAINTAINER': metadata.get('maintainer', ''),
|
||||||
|
'PACKAGE_HOMEPAGE': metadata.get('homepage', ''),
|
||||||
|
'PACKAGE_LICENSE': metadata.get('license', ''),
|
||||||
|
'PACKAGE_ARCH': metadata.get('arch', 'amd64'),
|
||||||
|
'PACKAGE_PLATFORM': metadata.get('platform', 'linux')
|
||||||
|
}
|
||||||
|
|
||||||
|
for key, value in env_vars.items():
|
||||||
|
config_args = ['buildah', 'config', '--env', f'{key}={value}', container_name]
|
||||||
|
logger.debug(f"Running: {' '.join(config_args)}")
|
||||||
|
result = subprocess.run(config_args, capture_output=True, text=True)
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.error(f"Failed to set environment variable {key}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Step 3: Copy resources to container
|
||||||
|
copy_args = [
|
||||||
|
'buildah', 'copy', container_name,
|
||||||
|
str(package_dir / "resources"), '/app/resources'
|
||||||
|
]
|
||||||
|
logger.debug(f"Running: {' '.join(copy_args)}")
|
||||||
|
result = subprocess.run(copy_args, capture_output=True, text=True)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.error(f"Failed to copy resources to container")
|
||||||
|
logger.error(f"stderr: {result.stderr}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Step 4: Create dist directory in container
|
||||||
|
run_args = ['buildah', 'run', container_name, 'mkdir', '-p', '/app/dist']
|
||||||
|
logger.debug(f"Running: {' '.join(run_args)}")
|
||||||
|
result = subprocess.run(run_args, capture_output=True, text=True)
|
||||||
|
|
||||||
|
# Step 5: Run the build script
|
||||||
|
run_args = ['buildah', 'run', '--workingdir', '/app', container_name, '/app/resources/build.sh']
|
||||||
|
logger.debug(f"Running: {' '.join(run_args)}")
|
||||||
|
result = subprocess.run(run_args, capture_output=True, text=True)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.error(f"Buildah build script failed for {package_name}")
|
||||||
|
logger.error(f"stdout: {result.stdout}")
|
||||||
|
logger.error(f"stderr: {result.stderr}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Step 6: Copy artifacts from container to host
|
||||||
|
copy_out_args = [
|
||||||
|
'buildah', 'run', container_name,
|
||||||
|
'find', '/app/dist', '-type', 'f', '-name', '*.rpm'
|
||||||
|
]
|
||||||
|
result = subprocess.run(copy_out_args, capture_output=True, text=True)
|
||||||
|
|
||||||
|
if result.returncode == 0 and result.stdout.strip():
|
||||||
|
rpm_files = result.stdout.strip().split('\n')
|
||||||
|
for rpm_file in rpm_files:
|
||||||
|
rpm_file = rpm_file.strip()
|
||||||
|
if rpm_file:
|
||||||
|
copy_args = [
|
||||||
|
'buildah', 'copy', '--from', container_name,
|
||||||
|
rpm_file, str(package_dist_dir)
|
||||||
|
]
|
||||||
|
logger.debug(f"Running: {' '.join(copy_args)}")
|
||||||
|
subprocess.run(copy_args, capture_output=True, text=True)
|
||||||
|
|
||||||
|
logger.info(f"Successfully built {package_name}-{package_version}-{package_release} using Buildah")
|
||||||
|
return True
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# Step 7: Clean up container
|
||||||
|
rm_args = ['buildah', 'rm', container_name]
|
||||||
|
logger.debug(f"Running: {' '.join(rm_args)}")
|
||||||
|
subprocess.run(rm_args, capture_output=True, text=True)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Unexpected error building {package_name} with Buildah: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def cleanup_images(image_pattern: str = "*-builder") -> None:
|
def cleanup_images(image_pattern: str = "*-builder") -> None:
|
||||||
"""
|
"""
|
||||||
Clean up Docker images matching a pattern.
|
Clean up Docker images matching a pattern.
|
||||||
@@ -917,7 +1269,9 @@ class Builder:
|
|||||||
release: str,
|
release: str,
|
||||||
dry_run: bool = False,
|
dry_run: bool = False,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
distro: str = 'almalinux/el9'
|
distro: str = 'almalinux/el9',
|
||||||
|
native: bool = False,
|
||||||
|
buildah: bool = False
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""
|
"""
|
||||||
Build a single package.
|
Build a single package.
|
||||||
@@ -993,9 +1347,9 @@ class Builder:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
package_info = PackageInfo(package, version, release, package_dir, distro, base_image)
|
package_info = PackageInfo(package, version, release, package_dir, distro, base_image)
|
||||||
return self._build_package(package_info, dry_run, force)
|
return self._build_package(package_info, dry_run, force, native, buildah)
|
||||||
|
|
||||||
def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9') -> bool:
|
def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9', native: bool = False, buildah: bool = False) -> bool:
|
||||||
"""
|
"""
|
||||||
Build all packages.
|
Build all packages.
|
||||||
|
|
||||||
@@ -1017,29 +1371,29 @@ class Builder:
|
|||||||
self.logger.info(f"Found {len(packages)} packages to process")
|
self.logger.info(f"Found {len(packages)} packages to process")
|
||||||
|
|
||||||
if parallel == 1:
|
if parallel == 1:
|
||||||
return self._build_sequential(packages, dry_run, force)
|
return self._build_sequential(packages, dry_run, force, native, buildah)
|
||||||
else:
|
else:
|
||||||
return self._build_parallel(packages, dry_run, force, parallel)
|
return self._build_parallel(packages, dry_run, force, parallel, native, buildah)
|
||||||
|
|
||||||
def _build_sequential(self, packages: List[PackageInfo], dry_run: bool, force: bool) -> bool:
|
def _build_sequential(self, packages: List[PackageInfo], dry_run: bool, force: bool, native: bool, buildah: bool) -> bool:
|
||||||
"""Build packages sequentially."""
|
"""Build packages sequentially."""
|
||||||
success_count = 0
|
success_count = 0
|
||||||
|
|
||||||
for package_info in packages:
|
for package_info in packages:
|
||||||
if self._build_package(package_info, dry_run, force):
|
if self._build_package(package_info, dry_run, force, native, buildah):
|
||||||
success_count += 1
|
success_count += 1
|
||||||
|
|
||||||
self.logger.info(f"Built {success_count}/{len(packages)} packages successfully")
|
self.logger.info(f"Built {success_count}/{len(packages)} packages successfully")
|
||||||
return success_count == len(packages)
|
return success_count == len(packages)
|
||||||
|
|
||||||
def _build_parallel(self, packages: List[PackageInfo], dry_run: bool, force: bool, parallel: int) -> bool:
|
def _build_parallel(self, packages: List[PackageInfo], dry_run: bool, force: bool, parallel: int, native: bool, buildah: bool) -> bool:
|
||||||
"""Build packages in parallel."""
|
"""Build packages in parallel."""
|
||||||
success_count = 0
|
success_count = 0
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=parallel) as executor:
|
with ThreadPoolExecutor(max_workers=parallel) as executor:
|
||||||
# Submit all build tasks
|
# Submit all build tasks
|
||||||
future_to_package = {
|
future_to_package = {
|
||||||
executor.submit(self._build_package, pkg, dry_run, force): pkg
|
executor.submit(self._build_package, pkg, dry_run, force, native, buildah): pkg
|
||||||
for pkg in packages
|
for pkg in packages
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1056,7 +1410,7 @@ class Builder:
|
|||||||
self.logger.info(f"Built {success_count}/{len(packages)} packages successfully")
|
self.logger.info(f"Built {success_count}/{len(packages)} packages successfully")
|
||||||
return success_count == len(packages)
|
return success_count == len(packages)
|
||||||
|
|
||||||
def _build_package(self, package_info: PackageInfo, dry_run: bool, force: bool) -> bool:
|
def _build_package(self, package_info: PackageInfo, dry_run: bool, force: bool, native: bool = False, buildah: bool = False) -> bool:
|
||||||
"""
|
"""
|
||||||
Build a single package.
|
Build a single package.
|
||||||
|
|
||||||
@@ -1081,22 +1435,51 @@ class Builder:
|
|||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# Check Docker is available (unless dry run)
|
# Check build tool availability (unless dry run)
|
||||||
if not dry_run and not check_docker_available():
|
use_native = native or (not buildah and check_native_build_deps())
|
||||||
self.logger.error("Docker is not available or running")
|
use_buildah = buildah or (not use_native and check_buildah_available())
|
||||||
|
use_docker = not use_native and not use_buildah and check_docker_available()
|
||||||
|
|
||||||
|
if not dry_run and not use_native and not use_buildah and not use_docker:
|
||||||
|
self.logger.error("No build tools available (tried native, Buildah, Docker)")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Build the package
|
# Build the package using available tool
|
||||||
return build_package_docker(
|
if use_native:
|
||||||
package_dir=package_info.directory,
|
self.logger.debug(f"Using native build for {package_info.name}")
|
||||||
package_name=package_info.name,
|
return build_package_native(
|
||||||
package_version=package_info.version,
|
package_dir=package_info.directory,
|
||||||
package_release=package_info.release,
|
package_name=package_info.name,
|
||||||
dist_dir=self.dist_dir,
|
package_version=package_info.version,
|
||||||
repository=package_info.distro,
|
package_release=package_info.release,
|
||||||
base_image=package_info.base_image,
|
dist_dir=self.dist_dir,
|
||||||
dry_run=dry_run
|
repository=package_info.distro,
|
||||||
)
|
dry_run=dry_run
|
||||||
|
)
|
||||||
|
elif use_buildah:
|
||||||
|
self.logger.debug(f"Using Buildah to build {package_info.name}")
|
||||||
|
return build_package_buildah(
|
||||||
|
package_dir=package_info.directory,
|
||||||
|
package_name=package_info.name,
|
||||||
|
package_version=package_info.version,
|
||||||
|
package_release=package_info.release,
|
||||||
|
dist_dir=self.dist_dir,
|
||||||
|
repository=package_info.distro,
|
||||||
|
base_image=package_info.base_image,
|
||||||
|
dry_run=dry_run
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.logger.debug(f"Using Docker to build {package_info.name}")
|
||||||
|
return build_package_docker(
|
||||||
|
package_dir=package_info.directory,
|
||||||
|
package_name=package_info.name,
|
||||||
|
package_version=package_info.version,
|
||||||
|
package_release=package_info.release,
|
||||||
|
dist_dir=self.dist_dir,
|
||||||
|
repository=package_info.distro,
|
||||||
|
base_image=package_info.base_image,
|
||||||
|
dry_run=dry_run
|
||||||
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Failed to build {package_info}: {e}")
|
self.logger.error(f"Failed to build {package_info}: {e}")
|
||||||
@@ -1358,6 +1741,8 @@ def build(
|
|||||||
distro: str = typer.Option("almalinux/el9", help="Target distro (default: almalinux/el9)"),
|
distro: str = typer.Option("almalinux/el9", help="Target distro (default: almalinux/el9)"),
|
||||||
dry_run: bool = typer.Option(False, "--dry-run", help="Show what would be built without building"),
|
dry_run: bool = typer.Option(False, "--dry-run", help="Show what would be built without building"),
|
||||||
force: bool = typer.Option(False, "--force", help="Build even if package exists in registry"),
|
force: bool = typer.Option(False, "--force", help="Build even if package exists in registry"),
|
||||||
|
native: bool = typer.Option(False, "--native", help="Force native build (skip Docker even if available)"),
|
||||||
|
buildah: bool = typer.Option(False, "--buildah", help="Force Buildah build (requires Buildah)"),
|
||||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose logging")
|
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose logging")
|
||||||
):
|
):
|
||||||
"""Build a specific package."""
|
"""Build a specific package."""
|
||||||
@@ -1419,7 +1804,9 @@ def build(
|
|||||||
release=str(release),
|
release=str(release),
|
||||||
dry_run=dry_run,
|
dry_run=dry_run,
|
||||||
force=force,
|
force=force,
|
||||||
distro=distro
|
distro=distro,
|
||||||
|
native=native,
|
||||||
|
buildah=buildah
|
||||||
)
|
)
|
||||||
|
|
||||||
if not success:
|
if not success:
|
||||||
@@ -1435,6 +1822,8 @@ def build_all(
|
|||||||
force: bool = typer.Option(False, "--force", help="Build even if packages exist in registry"),
|
force: bool = typer.Option(False, "--force", help="Build even if packages exist in registry"),
|
||||||
parallel: int = typer.Option(4, help="Number of parallel builds"),
|
parallel: int = typer.Option(4, help="Number of parallel builds"),
|
||||||
distro: str = typer.Option("almalinux/el9", help="Target distro (almalinux/el8, almalinux/el9, or 'all' for all distros)"),
|
distro: str = typer.Option("almalinux/el9", help="Target distro (almalinux/el8, almalinux/el9, or 'all' for all distros)"),
|
||||||
|
native: bool = typer.Option(False, "--native", help="Force native build (skip Docker even if available)"),
|
||||||
|
buildah: bool = typer.Option(False, "--buildah", help="Force Buildah build (requires Buildah)"),
|
||||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose logging")
|
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose logging")
|
||||||
):
|
):
|
||||||
"""Build all packages."""
|
"""Build all packages."""
|
||||||
@@ -1448,7 +1837,9 @@ def build_all(
|
|||||||
dry_run=dry_run,
|
dry_run=dry_run,
|
||||||
force=force,
|
force=force,
|
||||||
parallel=parallel,
|
parallel=parallel,
|
||||||
distro=distro
|
distro=distro,
|
||||||
|
native=native,
|
||||||
|
buildah=buildah
|
||||||
)
|
)
|
||||||
|
|
||||||
if not success:
|
if not success:
|
||||||
|
|||||||
Reference in New Issue
Block a user