refactor: modernise RPM builder with Python tooling
- Replace Makefile version/release file system with metadata.yaml only - Add Python build automation (./tools/build) with Gitea API integration - Add GitHub release updater (./tools/update-gh) for version management - Centralize Dockerfiles into single parameterized Dockerfile - Remove 54+ individual package Dockerfiles and version directories - Update Makefile to use new Python tooling - Add GITEA_API_TOKEN validation to prevent duplicate builds - Support both explicit version/release args and metadata.yaml reading
This commit is contained in:
Executable
+812
@@ -0,0 +1,812 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "requests",
|
||||
# "pyyaml"
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# vim: filetype=python
|
||||
|
||||
"""
|
||||
RPM Builder Tool
|
||||
|
||||
A Python replacement for the Makefile-based build system.
|
||||
Builds RPM packages using Docker and checks for existing packages via Gitea API.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import logging
|
||||
import subprocess
|
||||
import tempfile
|
||||
import shutil
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Optional, Dict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
|
||||
# ==================== GITEA API FUNCTIONS ====================
|
||||
|
||||
def normalize_version(version: str) -> str:
|
||||
"""
|
||||
Normalize version string by removing leading zeros from numeric components.
|
||||
Gitea automatically does this normalization.
|
||||
|
||||
Examples:
|
||||
"2025.08.03" -> "2025.8.3"
|
||||
"1.05.0" -> "1.5.0"
|
||||
"0.6.1" -> "0.6.1" (no change needed)
|
||||
|
||||
Args:
|
||||
version: Original version string
|
||||
|
||||
Returns:
|
||||
Normalized version string
|
||||
"""
|
||||
import re
|
||||
|
||||
# Split by common separators and normalize each numeric part
|
||||
parts = re.split(r'([.\-_])', version)
|
||||
normalized_parts = []
|
||||
|
||||
for part in parts:
|
||||
# If this part is purely numeric and has leading zeros, remove them
|
||||
if part.isdigit() and len(part) > 1 and part.startswith('0'):
|
||||
# Remove leading zeros but keep at least one digit
|
||||
normalized_parts.append(str(int(part)))
|
||||
else:
|
||||
normalized_parts.append(part)
|
||||
|
||||
return ''.join(normalized_parts)
|
||||
|
||||
|
||||
def check_package_exists(package_name: str, version: str, release: str) -> bool:
|
||||
"""
|
||||
Check if a package version exists in the Gitea package registry.
|
||||
|
||||
Args:
|
||||
package_name: Name of the package
|
||||
version: Version string
|
||||
release: Release number
|
||||
|
||||
Returns:
|
||||
True if package exists, False otherwise
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Get configuration from environment
|
||||
base_url = os.getenv('GITEA_URL', 'https://git.unkin.net')
|
||||
api_token = os.getenv('GITEA_API_TOKEN')
|
||||
owner = os.getenv('GITEA_OWNER', 'unkin')
|
||||
package_type = os.getenv('GITEA_PACKAGE_TYPE', 'rpm')
|
||||
|
||||
if not api_token:
|
||||
logger.warning("No GITEA_API_TOKEN found. API requests may fail for private repos.")
|
||||
|
||||
try:
|
||||
# Normalize version by removing leading zeros (Gitea does this automatically)
|
||||
# e.g., "2025.08.03" becomes "2025.8.3"
|
||||
normalized_version = normalize_version(version)
|
||||
full_version = f"{normalized_version}-{release}"
|
||||
url = (
|
||||
f"{base_url}/api/v1/packages/{owner}/"
|
||||
f"{package_type}/{package_name}/{full_version}"
|
||||
)
|
||||
|
||||
headers = {}
|
||||
if api_token:
|
||||
headers['Authorization'] = f'token {api_token}'
|
||||
|
||||
logger.debug(f"Checking package existence: {url}")
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
|
||||
if response.status_code == 200:
|
||||
package_info = response.json()
|
||||
# Package exists if we get package info back
|
||||
exists = bool(package_info.get('id'))
|
||||
logger.debug(f"Package {package_name}:{full_version} {'exists' if exists else 'not found'}")
|
||||
return exists
|
||||
elif response.status_code == 404:
|
||||
logger.debug(f"Package {package_name}:{full_version} not found (404)")
|
||||
return False
|
||||
elif response.status_code == 401:
|
||||
logger.error("Authentication failed. Check GITEA_API_TOKEN.")
|
||||
return False
|
||||
else:
|
||||
logger.warning(
|
||||
f"Unexpected response checking package {package_name}:{full_version}: "
|
||||
f"{response.status_code} - {response.text}"
|
||||
)
|
||||
return False
|
||||
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Failed to check package {package_name}:{version}-{release}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def get_package_full_name(package_name: str, version: str, release: str) -> str:
|
||||
"""
|
||||
Generate the full package name as used in the registry.
|
||||
|
||||
Args:
|
||||
package_name: Package name
|
||||
version: Version string
|
||||
release: Release number
|
||||
|
||||
Returns:
|
||||
Full package name string
|
||||
"""
|
||||
return f"{package_name}-{version}-{release}"
|
||||
|
||||
|
||||
# ==================== DOCKER FUNCTIONS ====================
|
||||
|
||||
def check_docker_available() -> bool:
|
||||
"""
|
||||
Check if Docker is available and running.
|
||||
|
||||
Returns:
|
||||
True if Docker is available, False otherwise
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['docker', 'version'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10
|
||||
)
|
||||
return result.returncode == 0
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return False
|
||||
|
||||
|
||||
def cleanup_container(container_name: str) -> None:
|
||||
"""
|
||||
Remove a Docker container.
|
||||
|
||||
Args:
|
||||
container_name: Name of the container to remove
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
try:
|
||||
remove_args = ['docker', 'rm', container_name]
|
||||
logger.debug(f"Running: {' '.join(remove_args)}")
|
||||
subprocess.run(remove_args, capture_output=True, text=True)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to remove container {container_name}: {e}")
|
||||
|
||||
|
||||
def get_base_image_from_metadata(package_dir: Path, distro: str = "el/9") -> str:
|
||||
"""
|
||||
Get the base image from package metadata.yaml.
|
||||
|
||||
Args:
|
||||
package_dir: Directory containing the package
|
||||
distro: Target distro (default: el/9)
|
||||
|
||||
Returns:
|
||||
Base image URL or default if not found
|
||||
"""
|
||||
metadata_file = package_dir / "metadata.yaml"
|
||||
default_image = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest"
|
||||
|
||||
if not metadata_file.exists():
|
||||
return default_image
|
||||
|
||||
try:
|
||||
import yaml
|
||||
with open(metadata_file, 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
|
||||
build_configs = metadata.get('build', [])
|
||||
for config in build_configs:
|
||||
if config.get('distro') == distro:
|
||||
return config.get('image', default_image)
|
||||
|
||||
# If no matching distro found, return first image or default
|
||||
if build_configs:
|
||||
return build_configs[0].get('image', default_image)
|
||||
|
||||
return default_image
|
||||
|
||||
except Exception:
|
||||
return default_image
|
||||
|
||||
|
||||
def build_package_docker(
|
||||
package_dir: Path,
|
||||
package_name: str,
|
||||
package_version: str,
|
||||
package_release: str,
|
||||
dist_dir: Path,
|
||||
base_image: str = "git.unkin.net/unkin/almalinux9-rpmbuilder:latest",
|
||||
dry_run: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Build a package using Docker with central Dockerfile.
|
||||
|
||||
Args:
|
||||
package_dir: Directory containing the package resources
|
||||
package_name: Name of the package
|
||||
package_version: Package version
|
||||
package_release: Package release number
|
||||
dist_dir: Directory to store built packages
|
||||
base_image: Base Docker image to use for building
|
||||
dry_run: If True, only show what would be done
|
||||
|
||||
Returns:
|
||||
True if build succeeded, False otherwise
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
# Ensure dist directory exists
|
||||
package_dist_dir = dist_dir / package_name
|
||||
if not dry_run:
|
||||
package_dist_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Generate Docker image name
|
||||
image_name = f"{package_name.lower()}-builder"
|
||||
container_name = f"{package_name}-{package_version}-builder"
|
||||
|
||||
logger.info(f"Building RPM for {package_name} version {package_version}")
|
||||
|
||||
if dry_run:
|
||||
logger.info(f"[DRY RUN] Would build Docker image: {image_name}")
|
||||
logger.info(f"[DRY RUN] Would use base image: {base_image}")
|
||||
logger.info(f"[DRY RUN] Would create container: {container_name}")
|
||||
logger.info(f"[DRY RUN] Would copy artifacts to: {package_dist_dir}")
|
||||
return True
|
||||
|
||||
# Step 1: Build Docker image using central Dockerfile
|
||||
central_dockerfile = package_dir.parent.parent / "Dockerfile"
|
||||
build_args = [
|
||||
'docker', 'build',
|
||||
'-f', str(central_dockerfile),
|
||||
'--build-arg', f'BASE_IMAGE={base_image}',
|
||||
'--build-arg', f'PACKAGE_VERSION={package_version}',
|
||||
'--build-arg', f'PACKAGE_RELEASE={package_release}',
|
||||
'-t', image_name,
|
||||
str(package_dir)
|
||||
]
|
||||
|
||||
logger.debug(f"Running: {' '.join(build_args)}")
|
||||
result = subprocess.run(
|
||||
build_args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=package_dir
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Docker build failed for {package_name}")
|
||||
logger.error(f"stdout: {result.stdout}")
|
||||
logger.error(f"stderr: {result.stderr}")
|
||||
return False
|
||||
|
||||
# Step 2: Create and start container
|
||||
create_args = [
|
||||
'docker', 'create',
|
||||
'--name', container_name,
|
||||
image_name
|
||||
]
|
||||
|
||||
logger.debug(f"Running: {' '.join(create_args)}")
|
||||
result = subprocess.run(create_args, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Container creation failed for {package_name}")
|
||||
logger.error(f"stderr: {result.stderr}")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Step 3: Start container
|
||||
start_args = ['docker', 'start', '-a', container_name]
|
||||
logger.debug(f"Running: {' '.join(start_args)}")
|
||||
result = subprocess.run(start_args, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Container execution failed for {package_name}")
|
||||
logger.error(f"stdout: {result.stdout}")
|
||||
logger.error(f"stderr: {result.stderr}")
|
||||
return False
|
||||
|
||||
# Step 4: Copy artifacts
|
||||
copy_args = [
|
||||
'docker', 'cp',
|
||||
f"{container_name}:/app/dist/.",
|
||||
str(package_dist_dir) + "/"
|
||||
]
|
||||
|
||||
logger.debug(f"Running: {' '.join(copy_args)}")
|
||||
result = subprocess.run(copy_args, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Failed to copy artifacts for {package_name}")
|
||||
logger.error(f"stderr: {result.stderr}")
|
||||
return False
|
||||
|
||||
logger.info(f"Successfully built {package_name}-{package_version}-{package_release}")
|
||||
return True
|
||||
|
||||
finally:
|
||||
# Step 5: Clean up container
|
||||
cleanup_container(container_name)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error building {package_name}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def cleanup_images(image_pattern: str = "*-builder") -> None:
|
||||
"""
|
||||
Clean up Docker images matching a pattern.
|
||||
|
||||
Args:
|
||||
image_pattern: Pattern to match image names
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
try:
|
||||
# List images matching pattern
|
||||
list_args = ['docker', 'images', '--format', '{{.Repository}}', '--filter', f'reference={image_pattern}']
|
||||
result = subprocess.run(list_args, capture_output=True, text=True)
|
||||
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
images = result.stdout.strip().split('\n')
|
||||
if images:
|
||||
remove_args = ['docker', 'rmi'] + images
|
||||
subprocess.run(remove_args, capture_output=True, text=True)
|
||||
logger.info(f"Cleaned up {len(images)} Docker images")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up Docker images: {e}")
|
||||
|
||||
|
||||
# ==================== PACKAGE INFO CLASS ====================
|
||||
|
||||
class PackageInfo:
|
||||
"""Information about a package to build."""
|
||||
|
||||
def __init__(self, name: str, version: str, release: str, directory: Path, distro: str = 'el/9', base_image: str = None):
|
||||
self.name = name
|
||||
self.version = version
|
||||
self.release = release
|
||||
self.directory = directory
|
||||
self.distro = distro
|
||||
self.base_image = base_image or "git.unkin.net/unkin/almalinux9-rpmbuilder:latest"
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.name}-{self.version}-{self.release} ({self.distro})"
|
||||
|
||||
|
||||
# ==================== BUILDER CLASS ====================
|
||||
|
||||
class Builder:
|
||||
"""Main builder class that orchestrates package building."""
|
||||
|
||||
def __init__(self, root_dir: Path):
|
||||
"""
|
||||
Initialize the builder.
|
||||
|
||||
Args:
|
||||
root_dir: Root directory of the project
|
||||
"""
|
||||
self.root_dir = root_dir
|
||||
self.rpms_dir = root_dir / "rpms"
|
||||
self.dist_dir = root_dir / "dist"
|
||||
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
# Ensure dist directory exists
|
||||
self.dist_dir.mkdir(exist_ok=True)
|
||||
|
||||
def discover_packages(self, distro: str = 'el/9') -> List[PackageInfo]:
|
||||
"""
|
||||
Discover all packages and their versions from metadata.yaml files.
|
||||
|
||||
Args:
|
||||
distro: Target distro (e.g., 'el/8', 'el/9', 'all')
|
||||
|
||||
Returns:
|
||||
List of PackageInfo objects
|
||||
"""
|
||||
packages = []
|
||||
|
||||
if not self.rpms_dir.exists():
|
||||
self.logger.error(f"RPMs directory not found: {self.rpms_dir}")
|
||||
return packages
|
||||
|
||||
for package_dir in self.rpms_dir.iterdir():
|
||||
if not package_dir.is_dir() or package_dir.name.startswith('.'):
|
||||
continue
|
||||
|
||||
metadata_file = package_dir / "metadata.yaml"
|
||||
if not metadata_file.exists():
|
||||
self.logger.warning(f"No metadata.yaml found for {package_dir.name}")
|
||||
continue
|
||||
|
||||
try:
|
||||
import yaml
|
||||
with open(metadata_file, 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
|
||||
package_name = metadata.get('name', package_dir.name)
|
||||
version = metadata.get('version')
|
||||
release = metadata.get('release')
|
||||
build_configs = metadata.get('build', [])
|
||||
|
||||
if not version:
|
||||
self.logger.warning(f"No version in metadata.yaml for {package_name}")
|
||||
continue
|
||||
|
||||
if not release:
|
||||
self.logger.warning(f"No release in metadata.yaml for {package_name}")
|
||||
continue
|
||||
|
||||
# Handle distro filtering
|
||||
if distro == 'all':
|
||||
# Build for all configured distros
|
||||
for build_config in build_configs:
|
||||
if isinstance(build_config, dict):
|
||||
build_distro = build_config.get('distro')
|
||||
base_image = build_config.get('image')
|
||||
if build_distro and base_image:
|
||||
packages.append(PackageInfo(package_name, version, str(release), package_dir, build_distro, base_image))
|
||||
else:
|
||||
# Build for specific distro
|
||||
for build_config in build_configs:
|
||||
if isinstance(build_config, dict) and build_config.get('distro') == distro:
|
||||
base_image = build_config.get('image')
|
||||
if base_image:
|
||||
packages.append(PackageInfo(package_name, version, str(release), package_dir, distro, base_image))
|
||||
break
|
||||
else:
|
||||
# If no matching distro found, log a warning
|
||||
self.logger.debug(f"No build config for {distro} found for {package_name}")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error reading metadata.yaml for {package_dir.name}: {e}")
|
||||
continue
|
||||
|
||||
return packages
|
||||
|
||||
def build_single(
|
||||
self,
|
||||
package: str,
|
||||
version: str,
|
||||
release: str,
|
||||
dry_run: bool = False,
|
||||
force: bool = False,
|
||||
distro: str = 'el/9'
|
||||
) -> bool:
|
||||
"""
|
||||
Build a single package.
|
||||
|
||||
Args:
|
||||
package: Package name
|
||||
version: Package version
|
||||
release: Package release
|
||||
dry_run: If True, only show what would be done
|
||||
force: If True, build even if package exists
|
||||
distro: Target distro (e.g., 'el/8', 'el/9', 'all')
|
||||
|
||||
Returns:
|
||||
True if build succeeded, False otherwise
|
||||
"""
|
||||
package_dir = self.rpms_dir / package
|
||||
|
||||
if not package_dir.exists():
|
||||
self.logger.error(f"Package directory not found: {package_dir}")
|
||||
return False
|
||||
|
||||
# Read metadata.yaml to validate version/release
|
||||
metadata_file = package_dir / "metadata.yaml"
|
||||
if not metadata_file.exists():
|
||||
self.logger.error(f"metadata.yaml not found: {metadata_file}")
|
||||
return False
|
||||
|
||||
try:
|
||||
import yaml
|
||||
with open(metadata_file, 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
|
||||
metadata_version = metadata.get('version')
|
||||
metadata_release = metadata.get('release')
|
||||
|
||||
if metadata_version != version:
|
||||
self.logger.error(
|
||||
f"Version mismatch for {package}: "
|
||||
f"provided {version} but metadata.yaml has {metadata_version}"
|
||||
)
|
||||
return False
|
||||
|
||||
if str(metadata_release) != str(release):
|
||||
self.logger.error(
|
||||
f"Release mismatch for {package}: "
|
||||
f"provided {release} but metadata.yaml has {metadata_release}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Find base image for the specified distro
|
||||
build_configs = metadata.get('build', [])
|
||||
base_image = None
|
||||
|
||||
if distro == 'all':
|
||||
# For single package build, 'all' doesn't make sense, default to el/9
|
||||
distro = 'el/9'
|
||||
|
||||
for build_config in build_configs:
|
||||
if isinstance(build_config, dict) and build_config.get('distro') == distro:
|
||||
base_image = build_config.get('image')
|
||||
break
|
||||
|
||||
if not base_image:
|
||||
self.logger.error(f"No build configuration found for distro {distro} in {package}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error reading metadata.yaml for {package}: {e}")
|
||||
return False
|
||||
|
||||
package_info = PackageInfo(package, version, release, package_dir, distro, base_image)
|
||||
return self._build_package(package_info, dry_run, force)
|
||||
|
||||
def build_all(self, dry_run: bool = False, force: bool = False, parallel: int = 4, distro: str = 'el/9') -> bool:
|
||||
"""
|
||||
Build all packages.
|
||||
|
||||
Args:
|
||||
dry_run: If True, only show what would be done
|
||||
force: If True, build even if packages exist
|
||||
parallel: Number of parallel builds
|
||||
distro: Target distro (e.g., 'el/8', 'el/9', 'all')
|
||||
|
||||
Returns:
|
||||
True if all builds succeeded, False otherwise
|
||||
"""
|
||||
packages = self.discover_packages(distro)
|
||||
|
||||
if not packages:
|
||||
self.logger.warning("No packages found to build")
|
||||
return True
|
||||
|
||||
self.logger.info(f"Found {len(packages)} packages to process")
|
||||
|
||||
if parallel == 1:
|
||||
return self._build_sequential(packages, dry_run, force)
|
||||
else:
|
||||
return self._build_parallel(packages, dry_run, force, parallel)
|
||||
|
||||
def _build_sequential(self, packages: List[PackageInfo], dry_run: bool, force: bool) -> bool:
|
||||
"""Build packages sequentially."""
|
||||
success_count = 0
|
||||
|
||||
for package_info in packages:
|
||||
if self._build_package(package_info, dry_run, force):
|
||||
success_count += 1
|
||||
|
||||
self.logger.info(f"Built {success_count}/{len(packages)} packages successfully")
|
||||
return success_count == len(packages)
|
||||
|
||||
def _build_parallel(self, packages: List[PackageInfo], dry_run: bool, force: bool, parallel: int) -> bool:
|
||||
"""Build packages in parallel."""
|
||||
success_count = 0
|
||||
|
||||
with ThreadPoolExecutor(max_workers=parallel) as executor:
|
||||
# Submit all build tasks
|
||||
future_to_package = {
|
||||
executor.submit(self._build_package, pkg, dry_run, force): pkg
|
||||
for pkg in packages
|
||||
}
|
||||
|
||||
# Process completed builds
|
||||
for future in as_completed(future_to_package):
|
||||
package_info = future_to_package[future]
|
||||
try:
|
||||
success = future.result()
|
||||
if success:
|
||||
success_count += 1
|
||||
except Exception as e:
|
||||
self.logger.error(f"Build failed for {package_info}: {e}")
|
||||
|
||||
self.logger.info(f"Built {success_count}/{len(packages)} packages successfully")
|
||||
return success_count == len(packages)
|
||||
|
||||
def _build_package(self, package_info: PackageInfo, dry_run: bool, force: bool) -> bool:
|
||||
"""
|
||||
Build a single package.
|
||||
|
||||
Args:
|
||||
package_info: Package information
|
||||
dry_run: If True, only show what would be done
|
||||
force: If True, build even if package exists
|
||||
|
||||
Returns:
|
||||
True if build succeeded, False otherwise
|
||||
"""
|
||||
try:
|
||||
# Check if package already exists (unless forced)
|
||||
if not force:
|
||||
# Check if we have GITEA_API_TOKEN for reliable package checking
|
||||
api_token = os.getenv('GITEA_API_TOKEN')
|
||||
if not api_token:
|
||||
self.logger.error(
|
||||
f"Skipping {package_info} - GITEA_API_TOKEN missing. "
|
||||
"Cannot verify if package already exists. Use --force to build anyway."
|
||||
)
|
||||
return False
|
||||
|
||||
if check_package_exists(
|
||||
package_info.name,
|
||||
package_info.version,
|
||||
package_info.release
|
||||
):
|
||||
self.logger.info(
|
||||
f"Skipping {package_info} (already exists in repository)"
|
||||
)
|
||||
return True
|
||||
|
||||
# Check Docker is available (unless dry run)
|
||||
if not dry_run and not check_docker_available():
|
||||
self.logger.error("Docker is not available or running")
|
||||
return False
|
||||
|
||||
# Build the package
|
||||
return build_package_docker(
|
||||
package_dir=package_info.directory,
|
||||
package_name=package_info.name,
|
||||
package_version=package_info.version,
|
||||
package_release=package_info.release,
|
||||
dist_dir=self.dist_dir,
|
||||
base_image=package_info.base_image,
|
||||
dry_run=dry_run
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to build {package_info}: {e}")
|
||||
return False
|
||||
|
||||
def list_packages(self) -> None:
|
||||
"""List all available packages."""
|
||||
packages = self.discover_packages()
|
||||
|
||||
if not packages:
|
||||
print("No packages found")
|
||||
return
|
||||
|
||||
print("Available packages:")
|
||||
for package_info in sorted(packages, key=lambda p: (p.name, p.version)):
|
||||
print(f" {package_info}")
|
||||
|
||||
def clean_dist(self) -> None:
|
||||
"""Clean the dist directory."""
|
||||
if self.dist_dir.exists():
|
||||
import shutil
|
||||
shutil.rmtree(self.dist_dir)
|
||||
self.dist_dir.mkdir()
|
||||
self.logger.info("Cleaned dist directory")
|
||||
|
||||
|
||||
# ==================== MAIN FUNCTIONS ====================
|
||||
|
||||
def setup_logging(verbose=False):
|
||||
"""Set up logging configuration."""
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
datefmt='%H:%M:%S'
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Build RPM packages using Docker',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s --package consul --version 1.21.1 --release 1
|
||||
%(prog)s --package consul (uses version/release from metadata.yaml)
|
||||
%(prog)s --package consul --distro el/8 (build for el/8)
|
||||
%(prog)s --all (builds all packages for el/9 by default)
|
||||
%(prog)s --all --distro el/8 (builds all packages for el/8)
|
||||
%(prog)s --all --distro all (builds all packages for all distros)
|
||||
%(prog)s --all --dry-run
|
||||
"""
|
||||
)
|
||||
|
||||
# Package selection arguments
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--package', help='Package name to build')
|
||||
group.add_argument('--all', action='store_true', help='Build all packages')
|
||||
|
||||
# Version and release (optional for single package builds, read from metadata.yaml if not provided)
|
||||
parser.add_argument('--version', help='Package version (optional, read from metadata.yaml if not provided)')
|
||||
parser.add_argument('--release', help='Package release number (optional, read from metadata.yaml if not provided)')
|
||||
|
||||
# Optional arguments
|
||||
parser.add_argument('--distro', default='el/9', help='Build for specific distro (default: el/9). Use "all" to build for all distros.')
|
||||
parser.add_argument('--dry-run', action='store_true', help='Show what would be built without building')
|
||||
parser.add_argument('--force', action='store_true', help='Build even if package exists in registry')
|
||||
parser.add_argument('--verbose', '-v', action='store_true', help='Enable verbose logging')
|
||||
parser.add_argument('--parallel', type=int, default=4, help='Number of parallel builds (default: 4)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# No validation needed - version/release will be read from metadata.yaml if not provided
|
||||
|
||||
setup_logging(args.verbose)
|
||||
|
||||
try:
|
||||
# Initialize components
|
||||
root_dir = Path(__file__).parent.parent
|
||||
builder = Builder(root_dir)
|
||||
|
||||
# Execute build
|
||||
if args.all:
|
||||
success = builder.build_all(
|
||||
dry_run=args.dry_run,
|
||||
force=args.force,
|
||||
parallel=args.parallel,
|
||||
distro=args.distro
|
||||
)
|
||||
else:
|
||||
# Read version/release from metadata.yaml if not provided
|
||||
version = args.version
|
||||
release = args.release
|
||||
|
||||
if not version or not release:
|
||||
package_dir = builder.rpms_dir / args.package
|
||||
metadata_file = package_dir / "metadata.yaml"
|
||||
|
||||
if not metadata_file.exists():
|
||||
logging.error(f"metadata.yaml not found for package {args.package}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import yaml
|
||||
with open(metadata_file, 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
|
||||
if not version:
|
||||
version = metadata.get('version')
|
||||
if not version:
|
||||
logging.error(f"No version in metadata.yaml for {args.package}")
|
||||
sys.exit(1)
|
||||
|
||||
if not release:
|
||||
release = metadata.get('release')
|
||||
if not release:
|
||||
logging.error(f"No release in metadata.yaml for {args.package}")
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error reading metadata.yaml for {args.package}: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
success = builder.build_single(
|
||||
package=args.package,
|
||||
version=version,
|
||||
release=str(release),
|
||||
dry_run=args.dry_run,
|
||||
force=args.force,
|
||||
distro=args.distro
|
||||
)
|
||||
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logging.info("Build interrupted by user")
|
||||
sys.exit(130)
|
||||
except Exception as e:
|
||||
logging.error(f"Build failed: {e}")
|
||||
if args.verbose:
|
||||
logging.exception("Full traceback:")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+380
@@ -0,0 +1,380 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "requests",
|
||||
# "pyyaml"
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# vim: filetype=python
|
||||
|
||||
"""
|
||||
GitHub Release Update Tool
|
||||
|
||||
Checks GitHub releases for packages and updates metadata.yaml and release files
|
||||
when newer versions are available.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import logging
|
||||
import requests
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, List
|
||||
import re
|
||||
|
||||
|
||||
def setup_logging(verbose=False):
|
||||
"""Set up logging configuration."""
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
datefmt='%H:%M:%S'
|
||||
)
|
||||
|
||||
|
||||
def load_env_vars(env_file: Path) -> Dict[str, str]:
|
||||
"""
|
||||
Load environment variables from env file.
|
||||
|
||||
Args:
|
||||
env_file: Path to the env file
|
||||
|
||||
Returns:
|
||||
Dictionary of environment variables
|
||||
"""
|
||||
env_vars = {}
|
||||
|
||||
if not env_file.exists():
|
||||
return env_vars
|
||||
|
||||
with open(env_file, 'r') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#') and '=' in line:
|
||||
# Handle export statements
|
||||
if line.startswith('export '):
|
||||
line = line[7:] # Remove 'export '
|
||||
|
||||
key, value = line.split('=', 1)
|
||||
env_vars[key.strip()] = value.strip()
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
def get_github_latest_release(repo: str, github_token: str) -> Optional[Dict]:
|
||||
"""
|
||||
Get the latest release from GitHub API.
|
||||
|
||||
Args:
|
||||
repo: GitHub repository in format "owner/repo"
|
||||
github_token: GitHub API token
|
||||
|
||||
Returns:
|
||||
Latest release info or None if not found
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
url = f"https://api.github.com/repos/{repo}/releases/latest"
|
||||
headers = {
|
||||
'Authorization': f'token {github_token}',
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
}
|
||||
|
||||
logger.debug(f"Checking GitHub releases: {url}")
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
|
||||
if response.status_code == 200:
|
||||
release = response.json()
|
||||
logger.debug(f"Latest release for {repo}: {release.get('tag_name', 'unknown')}")
|
||||
return release
|
||||
elif response.status_code == 404:
|
||||
logger.warning(f"No releases found for {repo}")
|
||||
return None
|
||||
elif response.status_code == 401:
|
||||
logger.error("GitHub authentication failed. Check GITHUB_API_TOKEN.")
|
||||
return None
|
||||
else:
|
||||
logger.warning(
|
||||
f"Unexpected response from GitHub API for {repo}: "
|
||||
f"{response.status_code} - {response.text}"
|
||||
)
|
||||
return None
|
||||
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Failed to check GitHub releases for {repo}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def normalize_version(version: str) -> str:
|
||||
"""
|
||||
Normalize version string by removing 'v' prefix if present.
|
||||
|
||||
Args:
|
||||
version: Version string (e.g., "v1.2.3" or "1.2.3")
|
||||
|
||||
Returns:
|
||||
Normalized version string (e.g., "1.2.3")
|
||||
"""
|
||||
if version.startswith('v'):
|
||||
return version[1:]
|
||||
return version
|
||||
|
||||
|
||||
def compare_versions(current: str, latest: str) -> bool:
|
||||
"""
|
||||
Compare version strings to determine if latest is newer.
|
||||
This is a simple string comparison that works for semantic versions.
|
||||
|
||||
Args:
|
||||
current: Current version string
|
||||
latest: Latest version string
|
||||
|
||||
Returns:
|
||||
True if latest is newer than current
|
||||
"""
|
||||
def version_tuple(v):
|
||||
# Split by dots and convert to integers where possible
|
||||
parts = []
|
||||
for part in v.split('.'):
|
||||
try:
|
||||
parts.append(int(part))
|
||||
except ValueError:
|
||||
# Handle non-numeric parts
|
||||
parts.append(part)
|
||||
return tuple(parts)
|
||||
|
||||
try:
|
||||
return version_tuple(latest) > version_tuple(current)
|
||||
except:
|
||||
# Fallback to string comparison
|
||||
return latest != current
|
||||
|
||||
|
||||
def update_package_metadata(package_dir: Path, new_version: str, dry_run: bool = False) -> bool:
|
||||
"""
|
||||
Update package metadata.yaml with new version.
|
||||
|
||||
Args:
|
||||
package_dir: Path to package directory
|
||||
new_version: New version to update to
|
||||
dry_run: If True, only show what would be done
|
||||
|
||||
Returns:
|
||||
True if update was successful
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
metadata_file = package_dir / "metadata.yaml"
|
||||
|
||||
try:
|
||||
# Load current metadata
|
||||
with open(metadata_file, 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
|
||||
old_version = metadata.get('version', 'unknown')
|
||||
logger.info(f"Updating {metadata.get('name', 'unknown')} from {old_version} to {new_version}")
|
||||
|
||||
if dry_run:
|
||||
logger.info(f"[DRY RUN] Would update metadata.yaml version to {new_version}")
|
||||
return True
|
||||
|
||||
# Update version in metadata and reset release to 1
|
||||
metadata['version'] = new_version
|
||||
metadata['release'] = 1
|
||||
|
||||
# Write updated metadata
|
||||
with open(metadata_file, 'w') as f:
|
||||
yaml.dump(metadata, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
logger.info(f"Successfully updated {metadata.get('name')} to version {new_version}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update package metadata: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def check_package_updates(package_dir: Path, github_token: str, dry_run: bool = False) -> bool:
|
||||
"""
|
||||
Check for updates for a single package.
|
||||
|
||||
Args:
|
||||
package_dir: Path to package directory
|
||||
github_token: GitHub API token
|
||||
dry_run: If True, only show what would be done
|
||||
|
||||
Returns:
|
||||
True if package was updated or no update needed
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
metadata_file = package_dir / "metadata.yaml"
|
||||
|
||||
if not metadata_file.exists():
|
||||
logger.warning(f"No metadata.yaml found in {package_dir}")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Load metadata
|
||||
with open(metadata_file, 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
|
||||
package_name = metadata.get('name', package_dir.name)
|
||||
current_version = metadata.get('version')
|
||||
github_repo = metadata.get('github')
|
||||
|
||||
if not github_repo:
|
||||
logger.debug(f"Package {package_name} has no GitHub repo configured")
|
||||
return True
|
||||
|
||||
if not current_version:
|
||||
logger.warning(f"Package {package_name} has no version in metadata")
|
||||
return False
|
||||
|
||||
logger.info(f"Checking {package_name} (current: {current_version}) from {github_repo}")
|
||||
|
||||
# Get latest release from GitHub
|
||||
latest_release = get_github_latest_release(github_repo, github_token)
|
||||
if not latest_release:
|
||||
return False
|
||||
|
||||
latest_version = normalize_version(latest_release.get('tag_name', ''))
|
||||
if not latest_version:
|
||||
logger.warning(f"Could not determine latest version for {package_name}")
|
||||
return False
|
||||
|
||||
# Compare versions
|
||||
if compare_versions(current_version, latest_version):
|
||||
logger.info(f"New version available: {current_version} -> {latest_version}")
|
||||
return update_package_metadata(package_dir, latest_version, dry_run)
|
||||
else:
|
||||
logger.info(f"Package {package_name} is up to date ({current_version})")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check package {package_dir.name}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def find_packages_with_github(rpms_dir: Path) -> List[Path]:
|
||||
"""
|
||||
Find all packages that have GitHub repo configured.
|
||||
|
||||
Args:
|
||||
rpms_dir: Path to rpms directory
|
||||
|
||||
Returns:
|
||||
List of package directories with GitHub repos
|
||||
"""
|
||||
github_packages = []
|
||||
|
||||
for package_dir in rpms_dir.iterdir():
|
||||
if not package_dir.is_dir() or package_dir.name.startswith('.'):
|
||||
continue
|
||||
|
||||
metadata_file = package_dir / "metadata.yaml"
|
||||
if not metadata_file.exists():
|
||||
continue
|
||||
|
||||
try:
|
||||
with open(metadata_file, 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
|
||||
if metadata.get('github'):
|
||||
github_packages.append(package_dir)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return github_packages
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Check GitHub releases and update package metadata',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s --package boilerplate
|
||||
%(prog)s --all
|
||||
%(prog)s --all --dry-run
|
||||
"""
|
||||
)
|
||||
|
||||
# Package selection arguments
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--package', help='Package name to check')
|
||||
group.add_argument('--all', action='store_true', help='Check all packages with GitHub repos')
|
||||
|
||||
# Optional arguments
|
||||
parser.add_argument('--dry-run', action='store_true', help='Show what would be done without making changes')
|
||||
parser.add_argument('--verbose', '-v', action='store_true', help='Enable verbose logging')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
setup_logging(args.verbose)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
# Get root directory and load environment
|
||||
root_dir = Path(__file__).parent.parent
|
||||
rpms_dir = root_dir / "rpms"
|
||||
env_file = root_dir / "env"
|
||||
|
||||
if not rpms_dir.exists():
|
||||
logger.error(f"RPMs directory not found: {rpms_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
# Load environment variables
|
||||
env_vars = load_env_vars(env_file)
|
||||
github_token = env_vars.get('GITHUB_API_TOKEN')
|
||||
|
||||
if not github_token:
|
||||
logger.error("GITHUB_API_TOKEN not found in env file")
|
||||
sys.exit(1)
|
||||
|
||||
success = True
|
||||
|
||||
if args.package:
|
||||
# Check single package
|
||||
package_dir = rpms_dir / args.package
|
||||
if not package_dir.exists():
|
||||
logger.error(f"Package directory not found: {package_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
success = check_package_updates(package_dir, github_token, args.dry_run)
|
||||
else:
|
||||
# Check all packages with GitHub repos
|
||||
github_packages = find_packages_with_github(rpms_dir)
|
||||
|
||||
if not github_packages:
|
||||
logger.info("No packages with GitHub repos found")
|
||||
sys.exit(0)
|
||||
|
||||
logger.info(f"Found {len(github_packages)} packages with GitHub repos")
|
||||
|
||||
updated_count = 0
|
||||
for package_dir in github_packages:
|
||||
if check_package_updates(package_dir, github_token, args.dry_run):
|
||||
updated_count += 1
|
||||
|
||||
logger.info(f"Successfully processed {updated_count}/{len(github_packages)} packages")
|
||||
success = updated_count == len(github_packages)
|
||||
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Update check interrupted by user")
|
||||
sys.exit(130)
|
||||
except Exception as e:
|
||||
logger.error(f"Update check failed: {e}")
|
||||
if args.verbose:
|
||||
logger.exception("Full traceback:")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user