unkin/artifactapi
2
0
Fork 0
Code Issues 5 Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

feat: add test suite, tox, pre-commit, and ruff formatting

Browse Source 
- tests/: 107 unit tests across config, cache, docker_auth, storage,
  and FastAPI routes; all passing under pytest-asyncio auto mode
- tox.ini: runs pytest via uvx --with tox-uv tox (py311)
- .pre-commit-config.yaml: ruff lint + ruff-format at v0.15.12
- pyproject.toml: pytest config (asyncio_mode=auto), ruff config
  (line-length=140), tox/pre-commit added to dev extras
- Makefile: test/tox/pre-commit targets via uvx --python 3.11
- Source files reformatted by ruff-format (no logic changes)
This commit is contained in:
Ben Vincent
2026-04-25 19:21:05 +10:00
parent 2414ddfdd3
commit 2d0e2c64e6
18 changed files with 1296 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.pre-commit-config.yaml
+7
View File
@@ -0,0 +1,7 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.12
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
- id: ruff-format
Makefile
+8 -2
View File
@@ -1,4 +1,4 @@
.PHONY: build install dev clean test lint format docker-build docker-up docker-down docker-logs docker-rebuild docker-clean docker-restart
.PHONY: build install dev clean test lint format pre-commit tox docker-build docker-up docker-down docker-logs docker-rebuild docker-clean docker-restart
build:
docker build --no-cache -t artifactapi:latest .
@@ -17,7 +17,13 @@ clean:
rm -rf *.egg-info/
test:
uv run pytest
uvx --python 3.11 --with tox-uv tox
tox:
uvx --python 3.11 --with tox-uv tox
pre-commit:
uvx --python 3.11 pre-commit run --all-files
lint:
uv run ruff check --fix .
pyproject.toml
+14 -1
View File
@@ -42,5 +42,18 @@ dev = [
"black>=23.9.0",
"isort>=5.12.0",
"mypy>=1.6.0",
"ruff>=0.1.0",
"ruff>=0.4.0",
"tox>=4.0.0",
"pre-commit>=3.0.0",
]
[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
[tool.ruff]
line-length = 140
[tool.ruff.lint]
select = ["E", "F", "I", "UP"]
ignore = ["E501"]
src/artifactapi/cache.py
+6 -5
View File
@@ -1,6 +1,7 @@
import hashlib
import re
import time
import hashlib
import redis
@@ -28,9 +29,7 @@ class RedisCache:
"""Generate cache key for index files"""
return f"index:{remote_name}:{hashlib.sha256(path.encode()).hexdigest()[:16]}"
def is_index_valid(
self, remote_name: str, path: str, ttl_override: int = None
) -> bool:
def is_index_valid(self, remote_name: str, path: str, ttl_override: int = None) -> bool:
"""Check if index file is still valid (not expired)"""
if not self.available:
return False
@@ -59,8 +58,10 @@ class RedisCache:
try:
# Construct the URL the same way as in the main flow
from .config import ConfigManager
import os
from .config import ConfigManager
config_path = os.environ.get("CONFIG_PATH")
if config_path:
config = ConfigManager(config_path)
src/artifactapi/config.py
+11 -12
View File
@@ -1,8 +1,7 @@
import os
import json
import yaml
from typing import Optional
import os
import yaml
_PACKAGE_INDEX_PATTERNS: dict[str, list[str]] = {
"alpine": [
@@ -31,10 +30,8 @@ class ConfigManager:
def _load_config(self) -> dict:
try:
with open(self.config_file, "r") as f:
if self.config_file.endswith(".yaml") or self.config_file.endswith(
".yml"
):
with open(self.config_file) as f:
if self.config_file.endswith(".yaml") or self.config_file.endswith(".yml"):
return yaml.safe_load(f)
else:
return json.load(f)
@@ -54,7 +51,7 @@ class ConfigManager:
except OSError:
pass
def get_remote_config(self, remote_name: str) -> Optional[dict]:
def get_remote_config(self, remote_name: str) -> dict | None:
self._check_reload()
return self.config.get("remotes", {}).get(remote_name)
@@ -111,9 +108,7 @@ class ConfigManager:
if not redis_url:
raise ValueError("REDIS_URL environment variable is required")
return {
"url": redis_url
}
return {"url": redis_url}
def get_database_config(self) -> dict:
"""Get database configuration from environment variables"""
@@ -124,7 +119,11 @@ class ConfigManager:
db_name = os.getenv("DBNAME")
if not all([db_host, db_port, db_user, db_pass, db_name]):
missing = [var for var, val in [("DBHOST", db_host), ("DBPORT", db_port), ("DBUSER", db_user), ("DBPASS", db_pass), ("DBNAME", db_name)] if not val]
missing = [
var
for var, val in [("DBHOST", db_host), ("DBPORT", db_port), ("DBUSER", db_user), ("DBPASS", db_pass), ("DBNAME", db_name)]
if not val
]
raise ValueError(f"All database environment variables are required: {', '.join(missing)}")
db_url = f"postgresql://{db_user}:{db_pass}@{db_host}:{db_port}/{db_name}"
src/artifactapi/database.py
+6 -18
View File
@@ -1,5 +1,3 @@
import os
from typing import Optional
import psycopg2
from psycopg2.extras import RealDictCursor
@@ -54,25 +52,15 @@ class DatabaseManager:
""")
# Create indexes separately
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_s3_key ON artifact_mappings (s3_key)"
)
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_remote_name ON artifact_mappings (remote_name)"
)
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_local_repo_path ON local_files (repository_name, file_path)"
)
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_local_s3_key ON local_files (s3_key)"
)
cursor.execute("CREATE INDEX IF NOT EXISTS idx_s3_key ON artifact_mappings (s3_key)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_remote_name ON artifact_mappings (remote_name)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_local_repo_path ON local_files (repository_name, file_path)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_local_s3_key ON local_files (s3_key)")
print("Database schema initialized")
except Exception as e:
print(f"Error creating schema: {e}")
def record_artifact_mapping(
self, s3_key: str, remote_name: str, file_path: str, size_bytes: int
):
def record_artifact_mapping(self, s3_key: str, remote_name: str, file_path: str, size_bytes: int):
"""Record mapping between S3 key and remote"""
if not self.available:
return
@@ -112,7 +100,7 @@ class DatabaseManager:
print(f"Error getting storage by remote: {e}")
return {}
def get_remote_for_s3_key(self, s3_key: str) -> Optional[str]:
def get_remote_for_s3_key(self, s3_key: str) -> str | None:
"""Get remote name for given S3 key"""
if not self.available:
return None
src/artifactapi/docker_auth.py
+11 -11
View File
@@ -1,7 +1,7 @@
import time
import logging
import re
from typing import Optional
import time
import httpx
logger = logging.getLogger(__name__)
@@ -17,11 +17,11 @@ _WWW_AUTH_RE = re.compile(
)
def _cache_key(realm: str, service: str, scope: str, username: Optional[str]) -> str:
def _cache_key(realm: str, service: str, scope: str, username: str | None) -> str:
return f"{realm}|{service}|{scope}|{username or ''}"
def _get_cached_token(key: str) -> Optional[str]:
def _get_cached_token(key: str) -> str | None:
entry = _token_cache.get(key)
if entry and entry[1] > time.time():
return entry[0]
@@ -38,9 +38,9 @@ async def fetch_token(
realm: str,
service: str,
scope: str,
username: Optional[str] = None,
password: Optional[str] = None,
) -> Optional[str]:
username: str | None = None,
password: str | None = None,
) -> str | None:
"""Fetch a Bearer token from a Docker registry auth server."""
key = _cache_key(realm, service, scope, username)
cached = _get_cached_token(key)
@@ -75,7 +75,7 @@ async def fetch_token(
return token
def parse_www_authenticate(header: str) -> Optional[tuple[str, str, str]]:
def parse_www_authenticate(header: str) -> tuple[str, str, str] | None:
"""Parse WWW-Authenticate: Bearer header. Returns (realm, service, scope) or None."""
m = _WWW_AUTH_RE.search(header)
if not m:
@@ -85,9 +85,9 @@ def parse_www_authenticate(header: str) -> Optional[tuple[str, str, str]]:
async def get_docker_token_for_response(
www_authenticate: str,
username: Optional[str] = None,
password: Optional[str] = None,
) -> Optional[str]:
username: str | None = None,
password: str | None = None,
) -> str | None:
"""Given a WWW-Authenticate header value, fetch and return a Bearer token."""
parsed = parse_www_authenticate(www_authenticate)
if not parsed:
src/artifactapi/main.py
+44 -104
View File
@@ -1,28 +1,30 @@
import hashlib
import json
import logging
import os
import re
import json
import hashlib
import logging
from typing import Dict, Any, Optional
from typing import Any
import httpx
from fastapi import FastAPI, HTTPException, Response, Request, Query, File, UploadFile
from fastapi.responses import PlainTextResponse, JSONResponse
from fastapi import FastAPI, File, HTTPException, Query, Request, Response, UploadFile
from fastapi.responses import JSONResponse, PlainTextResponse
from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
from pydantic import BaseModel
from prometheus_client import generate_latest, CONTENT_TYPE_LATEST
try:
from importlib.metadata import version
__version__ = version("artifactapi")
except ImportError:
# Fallback for development when package isn't installed
__version__ = "dev"
from .cache import RedisCache
from .config import ConfigManager
from .database import DatabaseManager
from .storage import S3Storage
from .cache import RedisCache
from .metrics import MetricsManager
from .docker_auth import get_docker_token_for_response
from .metrics import MetricsManager
from .storage import S3Storage
class ArtifactRequest(BaseModel):
@@ -31,10 +33,7 @@ class ArtifactRequest(BaseModel):
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
app = FastAPI(title="Artifact Storage API", version=__version__)
@@ -75,19 +74,11 @@ def health_check():
@app.put("/cache/flush")
def flush_cache(
remote: str = Query(default=None, description="Specific remote to flush (optional)"),
cache_type: str = Query(default="all", description="Type to flush: 'all', 'index', 'files', 'metrics'")
cache_type: str = Query(default="all", description="Type to flush: 'all', 'index', 'files', 'metrics'"),
):
"""Flush cache entries for specified remote or all remotes"""
try:
result = {
"remote": remote,
"cache_type": cache_type,
"flushed": {
"redis_keys": 0,
"s3_objects": 0,
"operations": []
}
}
result = {"remote": remote, "cache_type": cache_type, "flushed": {"redis_keys": 0, "s3_objects": 0, "operations": []}}
# Flush Redis entries based on cache_type
if cache_type in ["all", "index", "metrics"] and cache.available and cache.client:
@@ -124,8 +115,8 @@ def flush_cache(
list_params["Prefix"] = f"{remote}/"
response = storage.client.list_objects_v2(**list_params)
if 'Contents' in response:
objects_to_delete = [obj['Key'] for obj in response['Contents']]
if "Contents" in response:
objects_to_delete = [obj["Key"] for obj in response["Contents"]]
for key in objects_to_delete:
try:
@@ -156,15 +147,11 @@ def flush_cache(
async def construct_remote_url(remote_name: str, path: str) -> str:
remote_config = config.get_remote_config(remote_name)
if not remote_config:
raise HTTPException(
status_code=404, detail=f"Remote '{remote_name}' not configured"
)
raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
base_url = remote_config.get("base_url")
if not base_url:
raise HTTPException(
status_code=500, detail=f"No base_url configured for remote '{remote_name}'"
)
raise HTTPException(status_code=500, detail=f"No base_url configured for remote '{remote_name}'")
# Handle Docker registry URLs
if remote_config.get("package") == "docker":
@@ -175,9 +162,7 @@ async def construct_remote_url(remote_name: str, path: str) -> str:
return f"{base_url}/{path}"
async def check_artifact_patterns(
remote_name: str, repo_path: str, file_path: str, full_path: str
) -> bool:
async def check_artifact_patterns(remote_name: str, repo_path: str, file_path: str, full_path: str) -> bool:
# First check if this is an index file - always allow index files
index_patterns = config.get_index_patterns(remote_name)
if cache.is_index_file(file_path, index_patterns) or cache.is_index_file(full_path, index_patterns):
@@ -267,9 +252,7 @@ async def get_artifact(remote_name: str, path: str):
# Check if remote is configured
remote_config = config.get_remote_config(remote_name)
if not remote_config:
raise HTTPException(
status_code=404, detail=f"Remote '{remote_name}' not configured"
)
raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
# Check if this is a local repository
if remote_config.get("type") == "local":
@@ -289,9 +272,7 @@ async def get_artifact(remote_name: str, path: str):
return Response(
content=content,
media_type=content_type,
headers={
"Content-Disposition": f"attachment; filename={os.path.basename(path)}"
},
headers={"Content-Disposition": f"attachment; filename={os.path.basename(path)}"},
)
# Extract repository path for pattern checking
@@ -306,9 +287,7 @@ async def get_artifact(remote_name: str, path: str):
# Check if artifact matches configured patterns
if not await check_artifact_patterns(remote_name, repo_path, file_path, path):
logger.info(f"PATTERN BLOCKED: {remote_name}/{path} - not matching include patterns")
raise HTTPException(
status_code=403, detail="Artifact not allowed by configuration patterns"
)
raise HTTPException(status_code=403, detail="Artifact not allowed by configuration patterns")
# Construct the remote URL
remote_url = await construct_remote_url(remote_name, path)
@@ -358,9 +337,7 @@ async def get_artifact(remote_name: str, path: str):
metrics.record_cache_hit(remote_name, len(artifact_data))
# Record artifact mapping in database if not already recorded
database.record_artifact_mapping(
cached_key, remote_name, path, len(artifact_data)
)
database.record_artifact_mapping(cached_key, remote_name, path, len(artifact_data))
return Response(
content=artifact_data,
@@ -372,9 +349,7 @@ async def get_artifact(remote_name: str, path: str):
},
)
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Error retrieving cached artifact: {str(e)}"
)
raise HTTPException(status_code=500, detail=f"Error retrieving cached artifact: {str(e)}")
# Artifact not cached, cache it first
logger.info(f"Cache MISS: {remote_name}/{path} - fetching from remote: {remote_url}")
@@ -382,9 +357,7 @@ async def get_artifact(remote_name: str, path: str):
if result["status"] == "error":
logger.error(f"Cache ADD FAILED: {remote_name}/{path} - {result['error']}")
raise HTTPException(
status_code=502, detail=f"Failed to fetch artifact: {result['error']}"
)
raise HTTPException(status_code=502, detail=f"Failed to fetch artifact: {result['error']}")
# Mark index files as cached in Redis if this was a new download
if result["status"] == "cached" and is_index:
@@ -419,9 +392,7 @@ async def get_artifact(remote_name: str, path: str):
# Record artifact mapping in database
cache_key = storage.get_object_key(remote_name, path)
database.record_artifact_mapping(
cache_key, remote_name, path, len(artifact_data)
)
database.record_artifact_mapping(cache_key, remote_name, path, len(artifact_data))
return Response(
content=artifact_data,
@@ -533,9 +504,7 @@ async def discover_github_releases(remote: str, include_pattern: str) -> list[st
owner, repo = match.groups()
async with httpx.AsyncClient(follow_redirects=True) as client:
response = await client.get(
f"https://api.github.com/repos/{owner}/{repo}/releases"
)
response = await client.get(f"https://api.github.com/repos/{owner}/{repo}/releases")
if response.status_code != 200:
raise HTTPException(
@@ -564,14 +533,10 @@ async def upload_file(remote_name: str, path: str, file: UploadFile = File(...))
# Check if remote is configured and is local
remote_config = config.get_remote_config(remote_name)
if not remote_config:
raise HTTPException(
status_code=404, detail=f"Remote '{remote_name}' not configured"
)
raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
if remote_config.get("type") != "local":
raise HTTPException(
status_code=400, detail="Upload only supported for local repositories"
)
raise HTTPException(status_code=400, detail="Upload only supported for local repositories")
try:
# Read file content
@@ -633,9 +598,7 @@ def check_file_exists(remote_name: str, path: str):
# Check if remote is configured
remote_config = config.get_remote_config(remote_name)
if not remote_config:
raise HTTPException(
status_code=404, detail=f"Remote '{remote_name}' not configured"
)
raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
# Handle local repository
if remote_config.get("type") == "local":
@@ -647,16 +610,10 @@ def check_file_exists(remote_name: str, path: str):
return Response(
headers={
"Content-Length": str(metadata["size_bytes"]),
"Content-Type": metadata.get(
"content_type", "application/octet-stream"
),
"Content-Type": metadata.get("content_type", "application/octet-stream"),
"X-SHA256": metadata["sha256_sum"],
"X-Created-At": metadata["created_at"].isoformat()
if metadata["created_at"]
else "",
"X-Uploaded-At": metadata["uploaded_at"].isoformat()
if metadata["uploaded_at"]
else "",
"X-Created-At": metadata["created_at"].isoformat() if metadata["created_at"] else "",
"X-Uploaded-At": metadata["uploaded_at"].isoformat() if metadata["uploaded_at"] else "",
}
)
except HTTPException:
@@ -665,9 +622,7 @@ def check_file_exists(remote_name: str, path: str):
raise HTTPException(status_code=500, detail=f"Check failed: {str(e)}")
else:
# For remote repositories, just return 405 Method Not Allowed
raise HTTPException(
status_code=405, detail="HEAD method only supported for local repositories"
)
raise HTTPException(status_code=405, detail="HEAD method only supported for local repositories")
@app.delete("/api/v1/remote/{remote_name}/{path:path}")
@@ -676,14 +631,10 @@ def delete_file(remote_name: str, path: str):
# Check if remote is configured and is local
remote_config = config.get_remote_config(remote_name)
if not remote_config:
raise HTTPException(
status_code=404, detail=f"Remote '{remote_name}' not configured"
)
raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
if remote_config.get("type") != "local":
raise HTTPException(
status_code=400, detail="Delete only supported for local repositories"
)
raise HTTPException(status_code=400, detail="Delete only supported for local repositories")
try:
# Get S3 key before deleting from database
@@ -704,11 +655,9 @@ def delete_file(remote_name: str, path: str):
@app.post("/api/v1/artifacts/cache")
async def cache_artifact(request: ArtifactRequest) -> Dict[str, Any]:
async def cache_artifact(request: ArtifactRequest) -> dict[str, Any]:
try:
matching_urls = await discover_artifacts(
request.remote, request.include_pattern
)
matching_urls = await discover_artifacts(request.remote, request.include_pattern)
if not matching_urls:
return {
@@ -723,11 +672,7 @@ async def cache_artifact(request: ArtifactRequest) -> Dict[str, Any]:
result = await cache_single_artifact(url, "", "")
cached_artifacts.append(result)
cached_count = sum(
1
for artifact in cached_artifacts
if artifact["status"] in ["cached", "already_cached"]
)
cached_count = sum(1 for artifact in cached_artifacts if artifact["status"] in ["cached", "already_cached"])
return {
"message": f"Processed {len(matching_urls)} artifacts, {cached_count} successfully cached",
@@ -740,9 +685,7 @@ async def cache_artifact(request: ArtifactRequest) -> Dict[str, Any]:
@app.get("/api/v1/artifacts/{remote:path}")
async def list_cached_artifacts(
remote: str, include_pattern: str = ".*"
) -> Dict[str, Any]:
async def list_cached_artifacts(remote: str, include_pattern: str = ".*") -> dict[str, Any]:
try:
matching_urls = await discover_artifacts(remote, include_pattern)
@@ -750,13 +693,12 @@ async def list_cached_artifacts(
for url in matching_urls:
# Extract path from URL for hierarchical key generation
from urllib.parse import urlparse
parsed = urlparse(url)
path = parsed.path
key = storage.get_object_key(remote, path)
if storage.exists(key):
cached_artifacts.append(
{"url": url, "cached_url": storage.get_url(key), "key": key}
)
cached_artifacts.append({"url": url, "cached_url": storage.get_url(key), "key": key})
return {
"remote": remote,
@@ -772,9 +714,7 @@ async def list_cached_artifacts(
@app.get("/metrics")
def get_metrics(
json: Optional[bool] = Query(
False, description="Return JSON format instead of Prometheus"
),
json: bool | None = Query(False, description="Return JSON format instead of Prometheus"),
):
"""Get comprehensive metrics about the artifact storage system"""
config._check_reload()
src/artifactapi/metrics.py
+18 -51
View File
@@ -1,22 +1,14 @@
from datetime import datetime
from typing import Dict, Any
from typing import Any
from prometheus_client import Counter, Gauge
# Prometheus metrics
request_counter = Counter(
"artifact_requests_total", "Total artifact requests", ["remote", "status"]
)
request_counter = Counter("artifact_requests_total", "Total artifact requests", ["remote", "status"])
cache_hit_counter = Counter("artifact_cache_hits_total", "Total cache hits", ["remote"])
cache_miss_counter = Counter(
"artifact_cache_misses_total", "Total cache misses", ["remote"]
)
bandwidth_saved_counter = Counter(
"artifact_bandwidth_saved_bytes_total", "Total bandwidth saved", ["remote"]
)
storage_size_gauge = Gauge(
"artifact_storage_size_bytes", "Storage size by remote", ["remote"]
)
cache_miss_counter = Counter("artifact_cache_misses_total", "Total cache misses", ["remote"])
bandwidth_saved_counter = Counter("artifact_bandwidth_saved_bytes_total", "Total bandwidth saved", ["remote"])
storage_size_gauge = Gauge("artifact_storage_size_bytes", "Storage size by remote", ["remote"])
redis_keys_gauge = Gauge("artifact_redis_keys_total", "Total Redis keys")
@@ -44,9 +36,7 @@ class MetricsManager:
# Increment per-remote counters
self.redis_client.client.incr(f"metrics:cache_hits:{remote_name}")
self.redis_client.client.incr(f"metrics:total_requests:{remote_name}")
self.redis_client.client.incrby(
f"metrics:bandwidth_saved:{remote_name}", size_bytes
)
self.redis_client.client.incrby(f"metrics:bandwidth_saved:{remote_name}", size_bytes)
except Exception:
pass
@@ -91,7 +81,7 @@ class MetricsManager:
except Exception:
return 0
def get_s3_size_by_remote(self, storage, config_manager) -> Dict[str, int]:
def get_s3_size_by_remote(self, storage, config_manager) -> dict[str, int]:
"""Get size of stored data per remote using database mappings"""
if self.database_manager and self.database_manager.available:
# Get from database if available
@@ -146,7 +136,7 @@ class MetricsManager:
except Exception:
return {}
def get_metrics(self, storage, config_manager) -> Dict[str, Any]:
def get_metrics(self, storage, config_manager) -> dict[str, Any]:
"""Get comprehensive metrics"""
# Update Redis keys gauge
redis_key_count = self.get_redis_key_count()
@@ -173,54 +163,31 @@ class MetricsManager:
if self.redis_client and self.redis_client.available:
try:
# Get global metrics
cache_hits = int(
self.redis_client.client.get("metrics:cache_hits") or 0
)
cache_misses = int(
self.redis_client.client.get("metrics:cache_misses") or 0
)
cache_hits = int(self.redis_client.client.get("metrics:cache_hits") or 0)
cache_misses = int(self.redis_client.client.get("metrics:cache_misses") or 0)
total_requests = cache_hits + cache_misses
bandwidth_saved = int(
self.redis_client.client.get("metrics:bandwidth_saved") or 0
)
bandwidth_saved = int(self.redis_client.client.get("metrics:bandwidth_saved") or 0)
metrics["requests"]["cache_hits"] = cache_hits
metrics["requests"]["cache_misses"] = cache_misses
metrics["requests"]["total_requests"] = total_requests
metrics["requests"]["cache_hit_ratio"] = (
cache_hits / total_requests if total_requests > 0 else 0.0
)
metrics["requests"]["cache_hit_ratio"] = cache_hits / total_requests if total_requests > 0 else 0.0
metrics["bandwidth"]["saved_bytes"] = bandwidth_saved
# Get per-remote metrics
for remote in config_manager.config.get("remotes", {}).keys():
remote_cache_hits = int(
self.redis_client.client.get(f"metrics:cache_hits:{remote}")
or 0
)
remote_cache_misses = int(
self.redis_client.client.get(f"metrics:cache_misses:{remote}")
or 0
)
remote_cache_hits = int(self.redis_client.client.get(f"metrics:cache_hits:{remote}") or 0)
remote_cache_misses = int(self.redis_client.client.get(f"metrics:cache_misses:{remote}") or 0)
remote_total = remote_cache_hits + remote_cache_misses
remote_bandwidth_saved = int(
self.redis_client.client.get(
f"metrics:bandwidth_saved:{remote}"
)
or 0
)
remote_bandwidth_saved = int(self.redis_client.client.get(f"metrics:bandwidth_saved:{remote}") or 0)
metrics["per_remote"][remote] = {
"cache_hits": remote_cache_hits,
"cache_misses": remote_cache_misses,
"total_requests": remote_total,
"cache_hit_ratio": remote_cache_hits / remote_total
if remote_total > 0
else 0.0,
"cache_hit_ratio": remote_cache_hits / remote_total if remote_total > 0 else 0.0,
"bandwidth_saved_bytes": remote_bandwidth_saved,
"storage_size_bytes": metrics["storage"]["size_by_remote"].get(
remote, 0
),
"storage_size_bytes": metrics["storage"]["size_by_remote"].get(remote, 0),
}
except Exception:
src/artifactapi/storage.py
+9 -9
View File
@@ -1,5 +1,6 @@
import os
import hashlib
import os
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
@@ -21,23 +22,22 @@ class S3Storage:
self.bucket = bucket
self.secure = secure
ca_bundle = os.environ.get('REQUESTS_CA_BUNDLE') or os.environ.get('SSL_CERT_FILE')
config_kwargs = {
"request_checksum_calculation": "when_required",
"response_checksum_validation": "when_required"
}
ca_bundle = os.environ.get("REQUESTS_CA_BUNDLE") or os.environ.get("SSL_CERT_FILE")
config_kwargs = {"request_checksum_calculation": "when_required", "response_checksum_validation": "when_required"}
client_kwargs = {
"endpoint_url": f"http{'s' if self.secure else ''}://{self.endpoint}",
"aws_access_key_id": self.access_key,
"aws_secret_access_key": self.secret_key,
"config": Config(**config_kwargs)
"config": Config(**config_kwargs),
}
if ca_bundle and os.path.exists(ca_bundle):
client_kwargs["verify"] = ca_bundle
print(f"Debug: Using CA bundle: {ca_bundle}")
else:
print(f"Debug: No CA bundle found. REQUESTS_CA_BUNDLE={os.environ.get('REQUESTS_CA_BUNDLE')}, SSL_CERT_FILE={os.environ.get('SSL_CERT_FILE')}")
print(
f"Debug: No CA bundle found. REQUESTS_CA_BUNDLE={os.environ.get('REQUESTS_CA_BUNDLE')}, SSL_CERT_FILE={os.environ.get('SSL_CERT_FILE')}"
)
self.client = boto3.client("s3", **client_kwargs)
@@ -56,7 +56,7 @@ class S3Storage:
def get_object_key(self, remote_name: str, path: str) -> str:
# Extract directory path and filename
clean_path = path.lstrip('/')
clean_path = path.lstrip("/")
filename = os.path.basename(clean_path)
directory_path = os.path.dirname(clean_path)
tests/__init__.py
View File
tests/conftest.py
+127
View File
@@ -0,0 +1,127 @@
"""
Pytest configuration and shared fixtures.
Module-level setup (env vars + connection patches) runs before any test
module is imported, so the FastAPI app initialises against mocks rather
than real S3 / Redis / PostgreSQL services.
"""
import os
import tempfile
from unittest.mock import MagicMock, patch
import yaml
# ---------------------------------------------------------------------------
# Test remote configuration
# ---------------------------------------------------------------------------
TEST_REMOTES = {
"remotes": {
"alpine-test": {
"base_url": "https://dl-cdn.alpinelinux.org",
"type": "remote",
"package": "alpine",
"include_patterns": [".*/x86_64/.*\\.apk$"],
"cache": {"file_ttl": 0, "index_ttl": 3600},
},
"rpm-test": {
"base_url": "https://example.com/rpm",
"type": "remote",
"package": "rpm",
"include_patterns": [".*/x86_64/.*\\.rpm$", ".*/repodata/.*$"],
"cache": {"file_ttl": 0, "index_ttl": 3600},
},
"docker-test": {
"base_url": "https://registry.example.com",
"type": "remote",
"package": "docker",
"cache": {"file_ttl": 0, "index_ttl": 300},
},
"docker-restricted": {
"base_url": "https://registry.example.com",
"type": "remote",
"package": "docker",
"include_patterns": ["^library/nginx"],
"cache": {"file_ttl": 0, "index_ttl": 300},
},
"generic-test": {
"base_url": "https://releases.example.com",
"type": "remote",
"package": "generic",
"include_patterns": [".*\\.tar\\.gz$"],
"cache": {"file_ttl": 0, "index_ttl": 0},
},
"custom-index-test": {
"base_url": "https://example.com",
"type": "remote",
"package": "generic",
"index_patterns": ["metadata\\.json$"],
"cache": {"file_ttl": 0, "index_ttl": 600},
},
"local-test": {
"type": "local",
"package": "generic",
"cache": {"file_ttl": 0, "index_ttl": 0},
},
}
}
# ---------------------------------------------------------------------------
# Write temp config and set env vars BEFORE importing the package
# ---------------------------------------------------------------------------
_tmpdir = tempfile.mkdtemp()
_config_path = os.path.join(_tmpdir, "remotes.yaml")
with open(_config_path, "w") as _f:
yaml.dump(TEST_REMOTES, _f)
os.environ.update({
"CONFIG_PATH": _config_path,
"MINIO_ENDPOINT": "localhost:9000",
"MINIO_ACCESS_KEY": "testkey",
"MINIO_SECRET_KEY": "testsecret",
"MINIO_BUCKET": "testbucket",
"REDIS_URL": "redis://localhost:6379/0",
"DBHOST": "localhost",
"DBPORT": "5432",
"DBUSER": "test",
"DBPASS": "test",
"DBNAME": "test",
})
# Patch external service connections before the package is imported.
# These stay active for the whole session (process exits after tests finish).
_boto3_patch = patch("boto3.client", return_value=MagicMock())
_redis_patch = patch("redis.from_url", return_value=MagicMock())
_psycopg2_patch = patch("psycopg2.connect", return_value=MagicMock())
_boto3_patch.start()
_redis_patch.start()
_psycopg2_patch.start()
# ---------------------------------------------------------------------------
# Shared fixtures
# ---------------------------------------------------------------------------
import pytest # noqa: E402
from fastapi.testclient import TestClient # noqa: E402
@pytest.fixture(scope="session")
def app():
from artifactapi.main import app as fastapi_app
return fastapi_app
@pytest.fixture(scope="session")
def client(app):
return TestClient(app)
@pytest.fixture
def config_path():
return _config_path
@pytest.fixture
def test_remotes():
return TEST_REMOTES
tests/test_cache.py
+193
View File
@@ -0,0 +1,193 @@
"""Tests for RedisCache, focusing on is_index_file with configurable patterns."""
from unittest.mock import MagicMock, patch
import pytest
from artifactapi.cache import RedisCache
from artifactapi.config import _PACKAGE_INDEX_PATTERNS
@pytest.fixture
def bare_cache():
"""RedisCache instance bypassing __init__ (no Redis needed for pure-logic tests)."""
return RedisCache.__new__(RedisCache)
@pytest.fixture
def unavailable_cache():
"""RedisCache where Redis is not reachable."""
with patch("redis.from_url", side_effect=Exception("connection refused")):
return RedisCache("redis://localhost:6379/0")
@pytest.fixture
def mock_redis_client():
return MagicMock()
@pytest.fixture
def cache_with_redis(mock_redis_client):
"""RedisCache backed by a MagicMock Redis client."""
with patch("redis.from_url", return_value=mock_redis_client):
c = RedisCache("redis://localhost:6379/0")
c.client = mock_redis_client
c.available = True
return c
# ---------------------------------------------------------------------------
# is_index_file — alpine patterns
# ---------------------------------------------------------------------------
class TestIsIndexFileAlpine:
def test_apkindex_tarball_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
assert bare_cache.is_index_file("alpine/v3.18/x86_64/APKINDEX.tar.gz", patterns)
def test_nested_apkindex_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
assert bare_cache.is_index_file("mirrors/dl-cdn/alpine/v3.19/community/x86_64/APKINDEX.tar.gz", patterns)
def test_apk_package_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
assert not bare_cache.is_index_file("alpine/v3.18/x86_64/musl-1.2.4-r2.apk", patterns)
def test_random_tarball_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
assert not bare_cache.is_index_file("some/path/archive.tar.gz", patterns)
# ---------------------------------------------------------------------------
# is_index_file — rpm patterns
# ---------------------------------------------------------------------------
class TestIsIndexFileRpm:
def test_repomd_xml_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("almalinux/9/x86_64/repomd.xml", patterns)
def test_repodata_primary_xml_gz_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("repo/repodata/primary.xml.gz", patterns)
def test_repodata_sqlite_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("repo/repodata/primary.sqlite", patterns)
def test_repodata_sqlite_bz2_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("repo/repodata/other.sqlite.bz2", patterns)
def test_repodata_yaml_xz_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("repo/repodata/comps.yaml.xz", patterns)
def test_packages_gz_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("debian/dists/stable/main/binary-amd64/Packages.gz", patterns)
def test_rpm_package_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert not bare_cache.is_index_file("almalinux/9/x86_64/Packages/bash-5.1.8.x86_64.rpm", patterns)
def test_arbitrary_xml_outside_repodata_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert not bare_cache.is_index_file("some/path/config.xml", patterns)
# ---------------------------------------------------------------------------
# is_index_file — docker patterns
# ---------------------------------------------------------------------------
class TestIsIndexFileDocker:
def test_tag_manifest_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
assert bare_cache.is_index_file("library/nginx/manifests/latest", patterns)
def test_version_tag_manifest_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
assert bare_cache.is_index_file("library/nginx/manifests/1.25.3", patterns)
def test_digest_manifest_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
digest = "sha256:" + "a" * 64
assert not bare_cache.is_index_file(f"library/nginx/manifests/{digest}", patterns)
def test_tags_list_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
assert bare_cache.is_index_file("library/nginx/tags/list", patterns)
def test_blob_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
assert not bare_cache.is_index_file("library/nginx/blobs/sha256:abc123", patterns)
# ---------------------------------------------------------------------------
# is_index_file — edge cases
# ---------------------------------------------------------------------------
class TestIsIndexFileEdgeCases:
def test_empty_patterns_nothing_is_index(self, bare_cache):
assert not bare_cache.is_index_file("APKINDEX.tar.gz", [])
assert not bare_cache.is_index_file("repomd.xml", [])
assert not bare_cache.is_index_file("library/nginx/manifests/latest", [])
def test_none_patterns_nothing_is_index(self, bare_cache):
assert not bare_cache.is_index_file("APKINDEX.tar.gz", None)
assert not bare_cache.is_index_file("repomd.xml", None)
def test_custom_patterns_match(self, bare_cache):
patterns = [r"metadata\.json$", r"index\.yaml$"]
assert bare_cache.is_index_file("repo/metadata.json", patterns)
assert bare_cache.is_index_file("repo/subdir/index.yaml", patterns)
assert not bare_cache.is_index_file("repo/data.tar.gz", patterns)
def test_custom_pattern_does_not_match_standard_index(self, bare_cache):
patterns = [r"metadata\.json$"]
assert not bare_cache.is_index_file("APKINDEX.tar.gz", patterns)
# ---------------------------------------------------------------------------
# get_index_cache_key
# ---------------------------------------------------------------------------
class TestGetIndexCacheKey:
def test_same_inputs_produce_same_key(self, bare_cache):
k1 = bare_cache.get_index_cache_key("alpine-test", "alpine/v3.18/x86_64/APKINDEX.tar.gz")
k2 = bare_cache.get_index_cache_key("alpine-test", "alpine/v3.18/x86_64/APKINDEX.tar.gz")
assert k1 == k2
def test_different_paths_produce_different_keys(self, bare_cache):
k1 = bare_cache.get_index_cache_key("alpine-test", "alpine/v3.18/x86_64/APKINDEX.tar.gz")
k2 = bare_cache.get_index_cache_key("alpine-test", "alpine/v3.19/x86_64/APKINDEX.tar.gz")
assert k1 != k2
def test_different_remotes_produce_different_keys(self, bare_cache):
k1 = bare_cache.get_index_cache_key("remote-a", "path/to/APKINDEX.tar.gz")
k2 = bare_cache.get_index_cache_key("remote-b", "path/to/APKINDEX.tar.gz")
assert k1 != k2
def test_key_starts_with_index_prefix_and_remote(self, bare_cache):
key = bare_cache.get_index_cache_key("myremote", "some/path")
assert key.startswith("index:myremote:")
# ---------------------------------------------------------------------------
# mark_index_cached / is_index_valid
# ---------------------------------------------------------------------------
class TestIndexValidity:
def test_marked_then_valid(self, cache_with_redis, mock_redis_client):
mock_redis_client.exists.return_value = 1
cache_with_redis.mark_index_cached("remote", "path/APKINDEX.tar.gz", 300)
assert cache_with_redis.is_index_valid("remote", "path/APKINDEX.tar.gz")
def test_missing_key_is_not_valid(self, cache_with_redis, mock_redis_client):
mock_redis_client.exists.return_value = 0
assert not cache_with_redis.is_index_valid("remote", "path/APKINDEX.tar.gz")
def test_unavailable_redis_is_not_valid(self, unavailable_cache):
assert not unavailable_cache.is_index_valid("remote", "some/path")
def test_mark_cached_no_op_when_unavailable(self, unavailable_cache):
unavailable_cache.mark_index_cached("remote", "some/path", 300) # must not raise
tests/test_config.py
+194
View File
@@ -0,0 +1,194 @@
"""Tests for ConfigManager, focusing on get_index_patterns (new logic)."""
import os
import pytest
import yaml
from artifactapi.config import _PACKAGE_INDEX_PATTERNS, ConfigManager
@pytest.fixture
def make_config(tmp_path):
"""Factory: write a remotes dict to a temp YAML and return a ConfigManager."""
def _make(remotes_dict):
cfg_file = tmp_path / "remotes.yaml"
cfg_file.write_text(yaml.dump({"remotes": remotes_dict}))
return ConfigManager(str(cfg_file))
return _make
# ---------------------------------------------------------------------------
# get_index_patterns
# ---------------------------------------------------------------------------
class TestGetIndexPatterns:
def test_alpine_returns_package_defaults(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "alpine", "base_url": "https://x.com"}})
assert cfg.get_index_patterns("r") == _PACKAGE_INDEX_PATTERNS["alpine"]
def test_rpm_returns_package_defaults(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "rpm", "base_url": "https://x.com"}})
assert cfg.get_index_patterns("r") == _PACKAGE_INDEX_PATTERNS["rpm"]
def test_docker_returns_package_defaults(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "docker", "base_url": "https://x.com"}})
assert cfg.get_index_patterns("r") == _PACKAGE_INDEX_PATTERNS["docker"]
def test_generic_returns_empty_list(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}})
assert cfg.get_index_patterns("r") == []
def test_unknown_remote_returns_empty_list(self, make_config):
cfg = make_config({})
assert cfg.get_index_patterns("nonexistent") == []
def test_missing_package_field_defaults_to_generic(self, make_config):
cfg = make_config({"r": {"type": "remote", "base_url": "https://x.com"}})
assert cfg.get_index_patterns("r") == []
def test_extra_patterns_appended_after_defaults(self, make_config):
cfg = make_config({
"r": {
"type": "remote",
"package": "alpine",
"base_url": "https://x.com",
"index_patterns": ["custom\\.json$"],
}
})
patterns = cfg.get_index_patterns("r")
defaults = _PACKAGE_INDEX_PATTERNS["alpine"]
assert patterns[: len(defaults)] == defaults
assert "custom\\.json$" in patterns
def test_duplicate_extra_pattern_not_added_twice(self, make_config):
existing = _PACKAGE_INDEX_PATTERNS["alpine"][0]
cfg = make_config({
"r": {
"type": "remote",
"package": "alpine",
"base_url": "https://x.com",
"index_patterns": [existing],
}
})
patterns = cfg.get_index_patterns("r")
assert patterns.count(existing) == 1
def test_generic_with_only_extra_patterns(self, make_config):
cfg = make_config({
"r": {
"type": "remote",
"package": "generic",
"base_url": "https://x.com",
"index_patterns": ["meta\\.json$", "index\\.yaml$"],
}
})
assert cfg.get_index_patterns("r") == ["meta\\.json$", "index\\.yaml$"]
def test_rpm_extra_patterns_merged(self, make_config):
cfg = make_config({
"r": {
"type": "remote",
"package": "rpm",
"base_url": "https://x.com",
"index_patterns": ["custom-meta\\.xml$"],
}
})
patterns = cfg.get_index_patterns("r")
for default in _PACKAGE_INDEX_PATTERNS["rpm"]:
assert default in patterns
assert "custom-meta\\.xml$" in patterns
# ---------------------------------------------------------------------------
# get_repository_patterns
# ---------------------------------------------------------------------------
class TestGetRepositoryPatterns:
def test_returns_include_patterns(self, make_config):
cfg = make_config({
"r": {
"type": "remote",
"package": "generic",
"base_url": "https://x.com",
"include_patterns": [".*\\.tar\\.gz$"],
}
})
assert cfg.get_repository_patterns("r", "") == [".*\\.tar\\.gz$"]
def test_returns_empty_for_missing_remote(self, make_config):
cfg = make_config({})
assert cfg.get_repository_patterns("nonexistent", "") == []
def test_returns_empty_when_no_patterns_configured(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}})
assert cfg.get_repository_patterns("r", "") == []
def test_multiple_patterns_returned(self, make_config):
patterns = [".*\\.rpm$", ".*/repodata/.*$"]
cfg = make_config({
"r": {
"type": "remote",
"package": "rpm",
"base_url": "https://x.com",
"include_patterns": patterns,
}
})
assert cfg.get_repository_patterns("r", "") == patterns
# ---------------------------------------------------------------------------
# get_cache_config
# ---------------------------------------------------------------------------
class TestGetCacheConfig:
def test_returns_cache_section(self, make_config):
cfg = make_config({
"r": {
"type": "remote",
"package": "generic",
"base_url": "https://x.com",
"cache": {"file_ttl": 0, "index_ttl": 7200},
}
})
result = cfg.get_cache_config("r")
assert result["index_ttl"] == 7200
assert result["file_ttl"] == 0
def test_returns_empty_dict_for_missing_remote(self, make_config):
cfg = make_config({})
assert cfg.get_cache_config("nonexistent") == {}
def test_returns_empty_dict_when_no_cache_key(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}})
assert cfg.get_cache_config("r") == {}
# ---------------------------------------------------------------------------
# Config file reload
# ---------------------------------------------------------------------------
class TestConfigReload:
def test_reloads_when_file_mtime_advances(self, tmp_path):
cfg_file = tmp_path / "remotes.yaml"
cfg_file.write_text(yaml.dump({"remotes": {"repo-a": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}}))
cfg = ConfigManager(str(cfg_file))
assert "repo-a" in cfg.config["remotes"]
cfg_file.write_text(yaml.dump({"remotes": {"repo-b": {"type": "remote", "package": "generic", "base_url": "https://y.com"}}}))
future_mtime = cfg._last_modified + 1
os.utime(str(cfg_file), (future_mtime, future_mtime))
cfg._check_reload()
assert "repo-b" in cfg.config["remotes"]
assert "repo-a" not in cfg.config["remotes"]
def test_no_reload_when_file_unchanged(self, tmp_path):
cfg_file = tmp_path / "remotes.yaml"
cfg_file.write_text(yaml.dump({"remotes": {"repo-a": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}}))
cfg = ConfigManager(str(cfg_file))
# Call check_reload without touching the file → should not reload
cfg._check_reload()
assert "repo-a" in cfg.config["remotes"]
tests/test_docker_auth.py
+223
View File
@@ -0,0 +1,223 @@
"""Tests for docker_auth: WWW-Authenticate parsing and token caching."""
import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artifactapi import docker_auth
from artifactapi.docker_auth import (
_cache_key,
_get_cached_token,
_store_token,
fetch_token,
get_docker_token_for_response,
parse_www_authenticate,
)
@pytest.fixture(autouse=True)
def clear_token_cache():
"""Isolate tests: wipe the module-level token cache before and after each test."""
docker_auth._token_cache.clear()
yield
docker_auth._token_cache.clear()
# ---------------------------------------------------------------------------
# parse_www_authenticate
# ---------------------------------------------------------------------------
class TestParseWwwAuthenticate:
def test_full_bearer_header(self):
header = (
'Bearer realm="https://auth.docker.io/token"'
',service="registry.docker.io"'
',scope="repository:library/nginx:pull"'
)
result = parse_www_authenticate(header)
assert result is not None
realm, service, scope = result
assert realm == "https://auth.docker.io/token"
assert service == "registry.docker.io"
assert scope == "repository:library/nginx:pull"
def test_realm_only(self):
header = 'Bearer realm="https://auth.example.com/token"'
result = parse_www_authenticate(header)
assert result is not None
realm, service, scope = result
assert realm == "https://auth.example.com/token"
assert service == ""
assert scope == ""
def test_realm_and_service_only(self):
header = 'Bearer realm="https://auth.example.com",service="registry.example.com"'
result = parse_www_authenticate(header)
assert result is not None
_, service, scope = result
assert service == "registry.example.com"
assert scope == ""
def test_invalid_scheme_returns_none(self):
assert parse_www_authenticate('Basic realm="example"') is None
def test_empty_header_returns_none(self):
assert parse_www_authenticate("") is None
def test_case_insensitive_bearer(self):
header = 'bearer realm="https://auth.example.com/token"'
assert parse_www_authenticate(header) is not None
# ---------------------------------------------------------------------------
# _cache_key
# ---------------------------------------------------------------------------
class TestCacheKey:
def test_key_contains_all_components(self):
key = _cache_key("https://realm.com", "svc", "scope", "user")
assert "https://realm.com" in key
assert "svc" in key
assert "scope" in key
assert "user" in key
def test_none_username_uses_empty_string(self):
key = _cache_key("https://realm.com", "svc", "scope", None)
assert key.endswith("|")
def test_different_services_give_different_keys(self):
k1 = _cache_key("realm", "svc1", "scope", None)
k2 = _cache_key("realm", "svc2", "scope", None)
assert k1 != k2
def test_different_scopes_give_different_keys(self):
k1 = _cache_key("realm", "svc", "scope:read", None)
k2 = _cache_key("realm", "svc", "scope:write", None)
assert k1 != k2
# ---------------------------------------------------------------------------
# _get_cached_token / _store_token
# ---------------------------------------------------------------------------
class TestTokenCaching:
def test_get_returns_none_when_not_cached(self):
assert _get_cached_token("no-such-key") is None
def test_get_returns_token_when_valid(self):
_store_token("mykey", "tok-abc", 300)
assert _get_cached_token("mykey") == "tok-abc"
def test_get_returns_none_when_expired(self):
docker_auth._token_cache["mykey"] = ("old-token", time.time() - 1)
assert _get_cached_token("mykey") is None
def test_expired_entry_is_removed_from_cache(self):
docker_auth._token_cache["mykey"] = ("old-token", time.time() - 1)
_get_cached_token("mykey")
assert "mykey" not in docker_auth._token_cache
def test_store_expires_30s_before_stated_time(self):
before = time.time()
_store_token("mykey", "tok", 100)
_, expires_at = docker_auth._token_cache["mykey"]
# expires_in - 30 = 70; allow ±2 s clock wiggle
assert before + 68 <= expires_at <= before + 72
def test_store_enforces_minimum_10s_expiry(self):
before = time.time()
_store_token("mykey", "tok", 5) # expires_in - 30 would be negative
_, expires_at = docker_auth._token_cache["mykey"]
assert expires_at >= before + 10
# ---------------------------------------------------------------------------
# fetch_token (async, mocks httpx)
# ---------------------------------------------------------------------------
def _make_mock_http_client(token_payload: dict):
mock_response = MagicMock()
mock_response.raise_for_status = MagicMock()
mock_response.json.return_value = token_payload
mock_client = AsyncMock()
mock_client.get = AsyncMock(return_value=mock_response)
ctx = MagicMock()
ctx.__aenter__ = AsyncMock(return_value=mock_client)
ctx.__aexit__ = AsyncMock(return_value=False)
return ctx, mock_client
class TestFetchToken:
async def test_returns_token_field(self):
ctx, _ = _make_mock_http_client({"token": "bearer-tok", "expires_in": 300})
with patch("httpx.AsyncClient", return_value=ctx):
token = await fetch_token("https://auth.example.com", "svc", "scope")
assert token == "bearer-tok"
async def test_falls_back_to_access_token_field(self):
ctx, _ = _make_mock_http_client({"access_token": "access-tok", "expires_in": 300})
with patch("httpx.AsyncClient", return_value=ctx):
token = await fetch_token("https://auth.example.com", "svc", "scope")
assert token == "access-tok"
async def test_uses_cache_on_second_call_without_http(self):
ctx, mock_client = _make_mock_http_client({"token": "cached-tok", "expires_in": 300})
with patch("httpx.AsyncClient", return_value=ctx):
await fetch_token("https://auth.example.com", "svc", "scope")
mock_client.get.reset_mock()
token = await fetch_token("https://auth.example.com", "svc", "scope")
mock_client.get.assert_not_called()
assert token == "cached-tok"
async def test_returns_none_on_http_error(self):
mock_client = AsyncMock()
mock_client.get = AsyncMock(side_effect=Exception("connection refused"))
ctx = MagicMock()
ctx.__aenter__ = AsyncMock(return_value=mock_client)
ctx.__aexit__ = AsyncMock(return_value=False)
with patch("httpx.AsyncClient", return_value=ctx):
token = await fetch_token("https://auth.example.com", "svc", "scope")
assert token is None
async def test_passes_credentials_as_auth_tuple(self):
ctx, mock_client = _make_mock_http_client({"token": "authed-tok", "expires_in": 300})
with patch("httpx.AsyncClient", return_value=ctx):
await fetch_token("https://auth.example.com", "svc", "scope", "user", "pass")
call_kwargs = mock_client.get.call_args.kwargs
assert call_kwargs.get("auth") == ("user", "pass")
async def test_no_auth_when_no_credentials(self):
ctx, mock_client = _make_mock_http_client({"token": "anon-tok", "expires_in": 300})
with patch("httpx.AsyncClient", return_value=ctx):
await fetch_token("https://auth.example.com", "svc", "scope")
call_kwargs = mock_client.get.call_args.kwargs
assert call_kwargs.get("auth") is None
# ---------------------------------------------------------------------------
# get_docker_token_for_response
# ---------------------------------------------------------------------------
class TestGetDockerTokenForResponse:
async def test_returns_none_for_non_bearer_header(self):
token = await get_docker_token_for_response('Basic realm="example"')
assert token is None
async def test_delegates_to_fetch_token(self):
header = (
'Bearer realm="https://auth.example.com"'
',service="svc"'
',scope="repo:pull"'
)
with patch(
"artifactapi.docker_auth.fetch_token",
new_callable=AsyncMock,
return_value="delegated-tok",
) as mock_fetch:
token = await get_docker_token_for_response(header, "user", "pass")
mock_fetch.assert_called_once_with(
"https://auth.example.com", "svc", "repo:pull", "user", "pass"
)
assert token == "delegated-tok"
tests/test_routes.py
+316
View File
@@ -0,0 +1,316 @@
"""FastAPI route tests using TestClient with mocked service dependencies."""
import hashlib
import json
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Per-test service mocks (replace module-level globals in main.py)
# ---------------------------------------------------------------------------
@pytest.fixture
def mock_storage():
m = MagicMock()
m.get_object_key.return_value = "test-remote/abc123/file.ext"
m.exists.return_value = False
m.download_object.return_value = b"fake content"
m.bucket = "testbucket"
m.client = MagicMock()
return m
@pytest.fixture
def mock_cache():
m = MagicMock()
m.is_index_file.return_value = False
m.is_index_valid.return_value = True
m.available = False
m.client = None
return m
@pytest.fixture
def mock_database():
m = MagicMock()
m.available = False
return m
@pytest.fixture
def mock_metrics():
return MagicMock()
@pytest.fixture
def patched_deps(mock_storage, mock_cache, mock_database, mock_metrics):
"""Swap the module-level service instances in main.py for the duration of a test."""
import artifactapi.main as main_mod
with (
patch.object(main_mod, "storage", mock_storage),
patch.object(main_mod, "cache", mock_cache),
patch.object(main_mod, "database", mock_database),
patch.object(main_mod, "metrics", mock_metrics),
):
yield {
"storage": mock_storage,
"cache": mock_cache,
"database": mock_database,
"metrics": mock_metrics,
}
# ---------------------------------------------------------------------------
# Basic / health endpoints
# ---------------------------------------------------------------------------
class TestBasicEndpoints:
def test_root_returns_remote_list(self, client):
response = client.get("/")
assert response.status_code == 200
data = response.json()
assert "remotes" in data
assert isinstance(data["remotes"], list)
assert len(data["remotes"]) > 0
def test_root_contains_version(self, client):
response = client.get("/")
assert "version" in response.json()
def test_health_check(self, client):
response = client.get("/health")
assert response.status_code == 200
assert response.json()["status"] == "healthy"
def test_docker_v2_ping(self, client):
response = client.get("/v2/")
assert response.status_code == 200
assert response.headers.get("Docker-Distribution-Api-Version") == "registry/2.0"
assert response.json() == {}
# ---------------------------------------------------------------------------
# Docker proxy /v2/{remote}/{path}
# ---------------------------------------------------------------------------
class TestDockerProxy:
def test_unknown_remote_returns_404(self, client, patched_deps):
response = client.get("/v2/no-such-remote/library/nginx/manifests/latest")
assert response.status_code == 404
def test_non_docker_package_returns_400(self, client, patched_deps):
# alpine-test is package: alpine, not docker
response = client.get("/v2/alpine-test/library/nginx/manifests/latest")
assert response.status_code == 400
def test_pattern_blocked_returns_403(self, client, patched_deps):
# docker-restricted allows only "library/nginx"
response = client.get("/v2/docker-restricted/library/ubuntu/manifests/latest")
assert response.status_code == 403
def test_allowed_pattern_proceeds_to_cache(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"layers": [],
}).encode()
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = manifest
deps["cache"].is_index_file.return_value = True
deps["cache"].is_index_valid.return_value = True
response = client.get("/v2/docker-restricted/library/nginx/manifests/latest")
assert response.status_code == 200
def test_cache_hit_manifest_returns_correct_content_type(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"schemaVersion": 2,
"layers": [],
}).encode()
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = manifest
deps["cache"].is_index_file.return_value = True
deps["cache"].is_index_valid.return_value = True
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
assert response.status_code == 200
ct = response.headers["content-type"]
assert ct.startswith("application/vnd.docker.distribution.manifest.v2+json")
def test_cache_hit_sets_docker_content_digest_header(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"layers": [],
}).encode()
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = manifest
deps["cache"].is_index_file.return_value = True
deps["cache"].is_index_valid.return_value = True
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
expected = f"sha256:{hashlib.sha256(manifest).hexdigest()}"
assert response.headers["Docker-Content-Digest"] == expected
def test_head_request_returns_no_body(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"layers": [],
}).encode()
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = manifest
deps["cache"].is_index_file.return_value = False
response = client.head("/v2/docker-test/library/nginx/manifests/latest")
assert response.status_code == 200
assert response.content == b""
def test_cache_miss_calls_upstream_fetch(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"layers": [],
}).encode()
deps["storage"].exists.return_value = False
deps["storage"].download_object.return_value = manifest
deps["cache"].is_index_file.return_value = True
with patch(
"artifactapi.main.cache_single_artifact",
new_callable=AsyncMock,
return_value={"status": "cached"},
) as mock_fetch:
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
mock_fetch.assert_called_once()
assert response.status_code == 200
def test_index_expired_triggers_refetch(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"layers": [],
}).encode()
deps["storage"].exists.return_value = True # cached in S3
deps["cache"].is_index_file.return_value = True
deps["cache"].is_index_valid.return_value = False # but TTL expired
deps["storage"].download_object.return_value = manifest
with patch(
"artifactapi.main.cache_single_artifact",
new_callable=AsyncMock,
return_value={"status": "cached"},
) as mock_fetch:
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
mock_fetch.assert_called_once()
assert response.status_code == 200
# ---------------------------------------------------------------------------
# Generic artifact route /api/v1/remote/{remote}/{path}
# ---------------------------------------------------------------------------
class TestGenericArtifactRoute:
def test_unknown_remote_returns_404(self, client, patched_deps):
response = client.get("/api/v1/remote/nonexistent/path/to/file.tar.gz")
assert response.status_code == 404
def test_pattern_blocked_returns_403(self, client, patched_deps):
# generic-test only allows .tar.gz
response = client.get("/api/v1/remote/generic-test/some/path/file.rpm")
assert response.status_code == 403
def test_cache_hit_returns_200_with_header(self, client, patched_deps):
deps = patched_deps
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = b"tar content"
deps["cache"].is_index_file.return_value = False
response = client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
assert response.status_code == 200
assert response.headers["X-Artifact-Source"] == "cache"
assert response.content == b"tar content"
def test_cache_miss_fetches_upstream_and_returns_200(self, client, patched_deps):
deps = patched_deps
deps["storage"].exists.return_value = False
deps["storage"].download_object.return_value = b"fresh content"
deps["cache"].is_index_file.return_value = False
with patch(
"artifactapi.main.cache_single_artifact",
new_callable=AsyncMock,
return_value={"status": "cached"},
) as mock_fetch:
response = client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
mock_fetch.assert_called_once()
assert response.status_code == 200
assert response.headers["X-Artifact-Source"] == "remote"
def test_upstream_error_returns_502(self, client, patched_deps):
deps = patched_deps
deps["storage"].exists.return_value = False
deps["cache"].is_index_file.return_value = False
with patch(
"artifactapi.main.cache_single_artifact",
new_callable=AsyncMock,
return_value={"status": "error", "error": "upstream unreachable"},
):
response = client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
assert response.status_code == 502
def test_index_file_bypasses_include_patterns(self, client, patched_deps):
"""Index files must be served even when they don't match include_patterns."""
deps = patched_deps
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = b"APKINDEX content"
# Simulate is_index_file returning True (APKINDEX.tar.gz detected as index)
deps["cache"].is_index_file.return_value = True
deps["cache"].is_index_valid.return_value = True
# APKINDEX.tar.gz does not match alpine-test's include_patterns (.*.apk$),
# but since it's an index file it must be allowed through.
response = client.get("/api/v1/remote/alpine-test/alpine/v3.18/x86_64/APKINDEX.tar.gz")
assert response.status_code == 200
def test_local_repo_file_not_found_returns_404(self, client, patched_deps):
deps = patched_deps
deps["database"].get_local_file_metadata.return_value = None
deps["database"].available = True
response = client.get("/api/v1/remote/local-test/path/to/nonexistent.bin")
assert response.status_code == 404
# ---------------------------------------------------------------------------
# Cache flush PUT /cache/flush
# ---------------------------------------------------------------------------
class TestCacheFlushEndpoint:
def test_flush_all_returns_flushed_structure(self, client, patched_deps):
deps = patched_deps
deps["cache"].available = False
deps["storage"].client.list_objects_v2.return_value = {}
response = client.put("/cache/flush")
assert response.status_code == 200
data = response.json()
assert "flushed" in data
assert "redis_keys" in data["flushed"]
assert "s3_objects" in data["flushed"]
def test_flush_specific_remote_echoes_remote(self, client, patched_deps):
deps = patched_deps
deps["cache"].available = False
deps["storage"].client.list_objects_v2.return_value = {}
response = client.put("/cache/flush?remote=alpine-test")
assert response.status_code == 200
assert response.json()["remote"] == "alpine-test"
tests/test_storage.py
+101
View File
@@ -0,0 +1,101 @@
"""Tests for S3Storage, focusing on get_object_key (pure logic, no S3 calls)."""
from unittest.mock import MagicMock, patch
import pytest
from artifactapi.storage import S3Storage
@pytest.fixture
def storage():
"""S3Storage with a mocked boto3 client."""
with patch("boto3.client", return_value=MagicMock()):
s = S3Storage(
endpoint="localhost:9000",
access_key="testkey",
secret_key="testsecret",
bucket="testbucket",
secure=False,
)
s.client = MagicMock()
return s
# ---------------------------------------------------------------------------
# get_object_key
# ---------------------------------------------------------------------------
class TestGetObjectKey:
def test_includes_remote_name_prefix(self, storage):
key = storage.get_object_key("myremote", "some/path/file.rpm")
assert key.startswith("myremote/")
def test_ends_with_filename(self, storage):
key = storage.get_object_key("myremote", "some/path/file.rpm")
assert key.endswith("/file.rpm")
def test_same_path_produces_same_key(self, storage):
k1 = storage.get_object_key("myremote", "some/path/file.rpm")
k2 = storage.get_object_key("myremote", "some/path/file.rpm")
assert k1 == k2
def test_different_remotes_give_different_keys(self, storage):
k1 = storage.get_object_key("remote-a", "path/to/file.rpm")
k2 = storage.get_object_key("remote-b", "path/to/file.rpm")
assert k1 != k2
def test_different_directories_give_different_keys(self, storage):
k1 = storage.get_object_key("myremote", "path/version-1/file.rpm")
k2 = storage.get_object_key("myremote", "path/version-2/file.rpm")
assert k1 != k2
# Same filename, different directory hashes
assert k1.split("/")[-1] == k2.split("/")[-1] == "file.rpm"
def test_leading_slash_stripped(self, storage):
k1 = storage.get_object_key("myremote", "/path/to/file.rpm")
k2 = storage.get_object_key("myremote", "path/to/file.rpm")
assert k1 == k2
def test_file_with_no_directory(self, storage):
key = storage.get_object_key("myremote", "file.rpm")
assert key == "myremote/file.rpm"
def test_docker_blob_uses_digest_path(self, storage):
digest = "abc123def456" * 4
path = f"library/nginx/blobs/sha256:{digest}"
key = storage.get_object_key("dockerhub", path)
assert key == f"dockerhub/blobs/sha256/{digest}"
def test_docker_blob_deduplication_across_images(self, storage):
"""Same blob digest pulled from different images maps to the same S3 key."""
digest = "deadbeef" * 8
k1 = storage.get_object_key("dockerhub", f"library/nginx/blobs/sha256:{digest}")
k2 = storage.get_object_key("dockerhub", f"library/ubuntu/blobs/sha256:{digest}")
assert k1 == k2
def test_docker_blob_different_digests_different_keys(self, storage):
k1 = storage.get_object_key("dockerhub", "library/nginx/blobs/sha256:aaa111")
k2 = storage.get_object_key("dockerhub", "library/nginx/blobs/sha256:bbb222")
assert k1 != k2
def test_docker_blob_different_remotes_different_keys(self, storage):
digest = "abc" * 20
k1 = storage.get_object_key("remote-a", f"library/nginx/blobs/sha256:{digest}")
k2 = storage.get_object_key("remote-b", f"library/nginx/blobs/sha256:{digest}")
assert k1 != k2
# ---------------------------------------------------------------------------
# get_url
# ---------------------------------------------------------------------------
class TestGetUrl:
def test_returns_http_url_for_insecure_endpoint(self, storage):
url = storage.get_url("myremote/abc123/file.rpm")
assert url == "http://localhost:9000/testbucket/myremote/abc123/file.rpm"
def test_url_contains_bucket_and_key(self, storage):
key = "myremote/abc/file.tar.gz"
url = storage.get_url(key)
assert "testbucket" in url
assert key in url
tox.ini
+8
View File
@@ -0,0 +1,8 @@
[tox]
envlist = py311
isolated_build = true
[testenv]
extras = dev
commands =
pytest {posargs:tests}