unkin/artifactapi
2
0
Fork 0
Code Issues 5 Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

feat: rename include/index patterns to immutable/mutable with per-remote TTL

Browse Source 
Replace the include_patterns/index_patterns split with a clearer
immutable_patterns/mutable_patterns model:

- immutable_patterns: artifacts cached indefinitely (no TTL)
- mutable_patterns: artifacts that expire and are re-fetched after
  cache.mutable_ttl seconds (replaces cache.index_ttl)

_PACKAGE_INDEX_PATTERNS renamed to _PACKAGE_MUTABLE_PATTERNS; all
built-in package-type index patterns (APKINDEX, repomd, manifests, etc.)
default to the remote's mutable_ttl (default 1 hour).

cache.file_ttl renamed to cache.immutable_ttl for consistency.
Adds github-archive remote to remotes.yaml as a worked example showing
tag archives as immutable and branch archives as mutable (1-day TTL).

docker-compose.yml: fix VERSION=dev → 2.2.2.dev0 (valid PEP 440),
add :z SELinux label to volume mounts.
This commit is contained in:
Ben Vincent
2026-04-27 00:40:13 +10:00
parent 4619ae18d8
commit ce01a94141
8 changed files with 173 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/artifactapi/cache.py
+2 -2
View File
@@ -19,8 +19,8 @@ class RedisCache:
self.client = None
self.available = False
def is_index_file(self, file_path: str, patterns: list[str] | None = None) -> bool:
"""Return True if file_path matches any of the index patterns."""
def is_mutable_file(self, file_path: str, patterns: list[str] | None = None) -> bool:
"""Return True if file_path matches any of the mutable patterns."""
if patterns is None:
patterns = []
return any(re.search(p, file_path) for p in patterns)
src/artifactapi/config.py
+10 -15
View File
@@ -3,7 +3,7 @@ import os
import yaml
_PACKAGE_INDEX_PATTERNS: dict[str, list[str]] = {
_PACKAGE_MUTABLE_PATTERNS: dict[str, list[str]] = {
"alpine": [
r"APKINDEX\.tar\.gz$",
],
@@ -55,26 +55,21 @@ class ConfigManager:
self._check_reload()
return self.config.get("remotes", {}).get(remote_name)
def get_repository_patterns(self, remote_name: str, repo_path: str) -> list:
def get_immutable_patterns(self, remote_name: str, repo_path: str = "") -> list[str]:
remote_config = self.get_remote_config(remote_name)
if not remote_config:
return []
repositories = remote_config.get("repositories", {})
# Handle both dict (GitHub style) and list (Alpine style) repositories
if isinstance(repositories, dict):
repo_config = repositories.get(repo_path)
if repo_config:
patterns = repo_config.get("include_patterns", [])
patterns = repo_config.get("immutable_patterns", [])
else:
patterns = remote_config.get("include_patterns", [])
elif isinstance(repositories, list):
# For Alpine, repositories is just a list of allowed repo names
# Pattern matching is handled by the main include_patterns
patterns = remote_config.get("include_patterns", [])
patterns = remote_config.get("immutable_patterns", [])
else:
patterns = remote_config.get("include_patterns", [])
patterns = remote_config.get("immutable_patterns", [])
return patterns
@@ -129,18 +124,18 @@ class ConfigManager:
db_url = f"postgresql://{db_user}:{db_pass}@{db_host}:{db_port}/{db_name}"
return {"url": db_url}
def get_index_patterns(self, remote_name: str) -> list[str]:
"""Return index-file patterns for a remote.
def get_mutable_patterns(self, remote_name: str) -> list[str]:
"""Return mutable-file patterns for a remote (TTL is configured per-remote in cache.index_ttl).
Merges the package-level defaults with any extra patterns listed under
``index_patterns`` in the remote's config.
``mutable_patterns`` in the remote's config.
"""
remote_config = self.get_remote_config(remote_name)
if not remote_config:
return []
package = remote_config.get("package", "generic")
defaults = _PACKAGE_INDEX_PATTERNS.get(package, [])
extra = remote_config.get("index_patterns", [])
defaults = _PACKAGE_MUTABLE_PATTERNS.get(package, [])
extra = remote_config.get("mutable_patterns", [])
return defaults + [p for p in extra if p not in defaults]
def get_cache_config(self, remote_name: str) -> dict:
src/artifactapi/main.py
+23 -27
View File
@@ -163,13 +163,13 @@ async def construct_remote_url(remote_name: str, path: str) -> str:
async def check_artifact_patterns(remote_name: str, repo_path: str, file_path: str, full_path: str) -> bool:
# First check if this is an index file - always allow index files
index_patterns = config.get_index_patterns(remote_name)
if cache.is_index_file(file_path, index_patterns) or cache.is_index_file(full_path, index_patterns):
# Mutable files (index files) are always allowed through
mutable_patterns = config.get_mutable_patterns(remote_name)
if cache.is_mutable_file(file_path, mutable_patterns) or cache.is_mutable_file(full_path, mutable_patterns):
return True
# Then check basic include patterns
patterns = config.get_repository_patterns(remote_name, repo_path)
# Check immutable include patterns
patterns = config.get_immutable_patterns(remote_name, repo_path)
if not patterns:
return True # Allow all if no patterns configured
@@ -183,7 +183,6 @@ async def check_artifact_patterns(remote_name: str, repo_path: str, file_path: s
if not pattern_matched:
return False
# All remotes now use pattern-based filtering only - no additional checks needed
return True
@@ -297,15 +296,13 @@ async def get_artifact(remote_name: str, path: str):
if not storage.exists(cached_key):
cached_key = None
# For index files, check Redis TTL validity
# For mutable files, check Redis TTL validity
filename = os.path.basename(path)
is_index = cache.is_index_file(path, config.get_index_patterns(remote_name))
is_mutable = cache.is_mutable_file(path, config.get_mutable_patterns(remote_name))
if cached_key and is_index:
# Index file exists, but check if it's still valid
if cached_key and is_mutable:
if not cache.is_index_valid(remote_name, path):
# Index has expired, remove it from S3
logger.info(f"Index EXPIRED: {remote_name}/{path} - removing from cache")
logger.info(f"Mutable file EXPIRED: {remote_name}/{path} - removing from cache")
cache.cleanup_expired_index(storage, remote_name, path)
cached_key = None # Force re-download
@@ -359,13 +356,12 @@ async def get_artifact(remote_name: str, path: str):
logger.error(f"Cache ADD FAILED: {remote_name}/{path} - {result['error']}")
raise HTTPException(status_code=502, detail=f"Failed to fetch artifact: {result['error']}")
# Mark index files as cached in Redis if this was a new download
if result["status"] == "cached" and is_index:
# Get TTL from remote config
# Mark mutable files as cached in Redis with TTL
if result["status"] == "cached" and is_mutable:
cache_config = config.get_cache_config(remote_name)
index_ttl = cache_config.get("index_ttl", 300) # Default 5 minutes
cache.mark_index_cached(remote_name, path, index_ttl)
logger.info(f"Index file cached with TTL: {remote_name}/{path} (ttl: {index_ttl}s)")
mutable_ttl = cache_config.get("mutable_ttl", 3600)
cache.mark_index_cached(remote_name, path, mutable_ttl)
logger.info(f"Mutable file cached with TTL: {remote_name}/{path} (ttl: {mutable_ttl}s)")
# Now return the cached artifact
try:
@@ -424,8 +420,8 @@ async def docker_v2_proxy(request: Request, remote_name: str, path: str):
if remote_config.get("package") != "docker":
raise HTTPException(status_code=400, detail=f"Remote '{remote_name}' is not a docker remote")
# Check include_patterns against the image name (e.g. "library/nginx")
patterns = config.get_repository_patterns(remote_name, "")
# Check immutable_patterns against the image name (e.g. "library/nginx")
patterns = config.get_immutable_patterns(remote_name, "")
if patterns:
path_parts = path.split("/")
image_name = "/".join(path_parts[:2]) if len(path_parts) >= 2 else path
@@ -439,11 +435,11 @@ async def docker_v2_proxy(request: Request, remote_name: str, path: str):
if not storage.exists(cached_key):
cached_key = None
is_index = cache.is_index_file(path, config.get_index_patterns(remote_name))
is_mutable = cache.is_mutable_file(path, config.get_mutable_patterns(remote_name))
if cached_key and is_index:
if cached_key and is_mutable:
if not cache.is_index_valid(remote_name, path):
logger.info(f"Index EXPIRED: {remote_name}/{path} - removing from cache")
logger.info(f"Mutable file EXPIRED: {remote_name}/{path} - removing from cache")
cache.cleanup_expired_index(storage, remote_name, path)
cached_key = None
@@ -452,11 +448,11 @@ async def docker_v2_proxy(request: Request, remote_name: str, path: str):
result = await cache_single_artifact(remote_url, remote_name, path)
if result["status"] == "error":
raise HTTPException(status_code=502, detail=f"Failed to fetch: {result['error']}")
if result["status"] == "cached" and is_index:
if result["status"] == "cached" and is_mutable:
cache_config = config.get_cache_config(remote_name)
index_ttl = cache_config.get("index_ttl", 300)
cache.mark_index_cached(remote_name, path, index_ttl)
logger.info(f"Index file cached with TTL: {remote_name}/{path} (ttl: {index_ttl}s)")
mutable_ttl = cache_config.get("mutable_ttl", 3600)
cache.mark_index_cached(remote_name, path, mutable_ttl)
logger.info(f"Mutable file cached with TTL: {remote_name}/{path} (ttl: {mutable_ttl}s)")
artifact_data = storage.download_object(storage.get_object_key(remote_name, path))