feat: add test suite, tox, pre-commit, and ruff formatting

- tests/: 107 unit tests across config, cache, docker_auth, storage,
  and FastAPI routes; all passing under pytest-asyncio auto mode
- tox.ini: runs pytest via uvx --with tox-uv tox (py311)
- .pre-commit-config.yaml: ruff lint + ruff-format at v0.15.12
- pyproject.toml: pytest config (asyncio_mode=auto), ruff config
  (line-length=140), tox/pre-commit added to dev extras
- Makefile: test/tox/pre-commit targets via uvx --python 3.11
- Source files reformatted by ruff-format (no logic changes)
This commit is contained in:
2026-04-25 19:21:05 +10:00
parent 2414ddfdd3
commit 2d0e2c64e6
18 changed files with 1296 additions and 213 deletions
+7
View File
@@ -0,0 +1,7 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.12
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
- id: ruff-format
+8 -2
View File
@@ -1,4 +1,4 @@
.PHONY: build install dev clean test lint format docker-build docker-up docker-down docker-logs docker-rebuild docker-clean docker-restart .PHONY: build install dev clean test lint format pre-commit tox docker-build docker-up docker-down docker-logs docker-rebuild docker-clean docker-restart
build: build:
docker build --no-cache -t artifactapi:latest . docker build --no-cache -t artifactapi:latest .
@@ -17,7 +17,13 @@ clean:
rm -rf *.egg-info/ rm -rf *.egg-info/
test: test:
uv run pytest uvx --python 3.11 --with tox-uv tox
tox:
uvx --python 3.11 --with tox-uv tox
pre-commit:
uvx --python 3.11 pre-commit run --all-files
lint: lint:
uv run ruff check --fix . uv run ruff check --fix .
+14 -1
View File
@@ -42,5 +42,18 @@ dev = [
"black>=23.9.0", "black>=23.9.0",
"isort>=5.12.0", "isort>=5.12.0",
"mypy>=1.6.0", "mypy>=1.6.0",
"ruff>=0.1.0", "ruff>=0.4.0",
"tox>=4.0.0",
"pre-commit>=3.0.0",
] ]
[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
[tool.ruff]
line-length = 140
[tool.ruff.lint]
select = ["E", "F", "I", "UP"]
ignore = ["E501"]
+6 -5
View File
@@ -1,6 +1,7 @@
import hashlib
import re import re
import time import time
import hashlib
import redis import redis
@@ -28,9 +29,7 @@ class RedisCache:
"""Generate cache key for index files""" """Generate cache key for index files"""
return f"index:{remote_name}:{hashlib.sha256(path.encode()).hexdigest()[:16]}" return f"index:{remote_name}:{hashlib.sha256(path.encode()).hexdigest()[:16]}"
def is_index_valid( def is_index_valid(self, remote_name: str, path: str, ttl_override: int = None) -> bool:
self, remote_name: str, path: str, ttl_override: int = None
) -> bool:
"""Check if index file is still valid (not expired)""" """Check if index file is still valid (not expired)"""
if not self.available: if not self.available:
return False return False
@@ -59,8 +58,10 @@ class RedisCache:
try: try:
# Construct the URL the same way as in the main flow # Construct the URL the same way as in the main flow
from .config import ConfigManager
import os import os
from .config import ConfigManager
config_path = os.environ.get("CONFIG_PATH") config_path = os.environ.get("CONFIG_PATH")
if config_path: if config_path:
config = ConfigManager(config_path) config = ConfigManager(config_path)
+11 -12
View File
@@ -1,8 +1,7 @@
import os
import json import json
import yaml import os
from typing import Optional
import yaml
_PACKAGE_INDEX_PATTERNS: dict[str, list[str]] = { _PACKAGE_INDEX_PATTERNS: dict[str, list[str]] = {
"alpine": [ "alpine": [
@@ -31,10 +30,8 @@ class ConfigManager:
def _load_config(self) -> dict: def _load_config(self) -> dict:
try: try:
with open(self.config_file, "r") as f: with open(self.config_file) as f:
if self.config_file.endswith(".yaml") or self.config_file.endswith( if self.config_file.endswith(".yaml") or self.config_file.endswith(".yml"):
".yml"
):
return yaml.safe_load(f) return yaml.safe_load(f)
else: else:
return json.load(f) return json.load(f)
@@ -54,7 +51,7 @@ class ConfigManager:
except OSError: except OSError:
pass pass
def get_remote_config(self, remote_name: str) -> Optional[dict]: def get_remote_config(self, remote_name: str) -> dict | None:
self._check_reload() self._check_reload()
return self.config.get("remotes", {}).get(remote_name) return self.config.get("remotes", {}).get(remote_name)
@@ -111,9 +108,7 @@ class ConfigManager:
if not redis_url: if not redis_url:
raise ValueError("REDIS_URL environment variable is required") raise ValueError("REDIS_URL environment variable is required")
return { return {"url": redis_url}
"url": redis_url
}
def get_database_config(self) -> dict: def get_database_config(self) -> dict:
"""Get database configuration from environment variables""" """Get database configuration from environment variables"""
@@ -124,7 +119,11 @@ class ConfigManager:
db_name = os.getenv("DBNAME") db_name = os.getenv("DBNAME")
if not all([db_host, db_port, db_user, db_pass, db_name]): if not all([db_host, db_port, db_user, db_pass, db_name]):
missing = [var for var, val in [("DBHOST", db_host), ("DBPORT", db_port), ("DBUSER", db_user), ("DBPASS", db_pass), ("DBNAME", db_name)] if not val] missing = [
var
for var, val in [("DBHOST", db_host), ("DBPORT", db_port), ("DBUSER", db_user), ("DBPASS", db_pass), ("DBNAME", db_name)]
if not val
]
raise ValueError(f"All database environment variables are required: {', '.join(missing)}") raise ValueError(f"All database environment variables are required: {', '.join(missing)}")
db_url = f"postgresql://{db_user}:{db_pass}@{db_host}:{db_port}/{db_name}" db_url = f"postgresql://{db_user}:{db_pass}@{db_host}:{db_port}/{db_name}"
+6 -18
View File
@@ -1,5 +1,3 @@
import os
from typing import Optional
import psycopg2 import psycopg2
from psycopg2.extras import RealDictCursor from psycopg2.extras import RealDictCursor
@@ -54,25 +52,15 @@ class DatabaseManager:
""") """)
# Create indexes separately # Create indexes separately
cursor.execute( cursor.execute("CREATE INDEX IF NOT EXISTS idx_s3_key ON artifact_mappings (s3_key)")
"CREATE INDEX IF NOT EXISTS idx_s3_key ON artifact_mappings (s3_key)" cursor.execute("CREATE INDEX IF NOT EXISTS idx_remote_name ON artifact_mappings (remote_name)")
) cursor.execute("CREATE INDEX IF NOT EXISTS idx_local_repo_path ON local_files (repository_name, file_path)")
cursor.execute( cursor.execute("CREATE INDEX IF NOT EXISTS idx_local_s3_key ON local_files (s3_key)")
"CREATE INDEX IF NOT EXISTS idx_remote_name ON artifact_mappings (remote_name)"
)
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_local_repo_path ON local_files (repository_name, file_path)"
)
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_local_s3_key ON local_files (s3_key)"
)
print("Database schema initialized") print("Database schema initialized")
except Exception as e: except Exception as e:
print(f"Error creating schema: {e}") print(f"Error creating schema: {e}")
def record_artifact_mapping( def record_artifact_mapping(self, s3_key: str, remote_name: str, file_path: str, size_bytes: int):
self, s3_key: str, remote_name: str, file_path: str, size_bytes: int
):
"""Record mapping between S3 key and remote""" """Record mapping between S3 key and remote"""
if not self.available: if not self.available:
return return
@@ -112,7 +100,7 @@ class DatabaseManager:
print(f"Error getting storage by remote: {e}") print(f"Error getting storage by remote: {e}")
return {} return {}
def get_remote_for_s3_key(self, s3_key: str) -> Optional[str]: def get_remote_for_s3_key(self, s3_key: str) -> str | None:
"""Get remote name for given S3 key""" """Get remote name for given S3 key"""
if not self.available: if not self.available:
return None return None
+11 -11
View File
@@ -1,7 +1,7 @@
import time
import logging import logging
import re import re
from typing import Optional import time
import httpx import httpx
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -17,11 +17,11 @@ _WWW_AUTH_RE = re.compile(
) )
def _cache_key(realm: str, service: str, scope: str, username: Optional[str]) -> str: def _cache_key(realm: str, service: str, scope: str, username: str | None) -> str:
return f"{realm}|{service}|{scope}|{username or ''}" return f"{realm}|{service}|{scope}|{username or ''}"
def _get_cached_token(key: str) -> Optional[str]: def _get_cached_token(key: str) -> str | None:
entry = _token_cache.get(key) entry = _token_cache.get(key)
if entry and entry[1] > time.time(): if entry and entry[1] > time.time():
return entry[0] return entry[0]
@@ -38,9 +38,9 @@ async def fetch_token(
realm: str, realm: str,
service: str, service: str,
scope: str, scope: str,
username: Optional[str] = None, username: str | None = None,
password: Optional[str] = None, password: str | None = None,
) -> Optional[str]: ) -> str | None:
"""Fetch a Bearer token from a Docker registry auth server.""" """Fetch a Bearer token from a Docker registry auth server."""
key = _cache_key(realm, service, scope, username) key = _cache_key(realm, service, scope, username)
cached = _get_cached_token(key) cached = _get_cached_token(key)
@@ -75,7 +75,7 @@ async def fetch_token(
return token return token
def parse_www_authenticate(header: str) -> Optional[tuple[str, str, str]]: def parse_www_authenticate(header: str) -> tuple[str, str, str] | None:
"""Parse WWW-Authenticate: Bearer header. Returns (realm, service, scope) or None.""" """Parse WWW-Authenticate: Bearer header. Returns (realm, service, scope) or None."""
m = _WWW_AUTH_RE.search(header) m = _WWW_AUTH_RE.search(header)
if not m: if not m:
@@ -85,9 +85,9 @@ def parse_www_authenticate(header: str) -> Optional[tuple[str, str, str]]:
async def get_docker_token_for_response( async def get_docker_token_for_response(
www_authenticate: str, www_authenticate: str,
username: Optional[str] = None, username: str | None = None,
password: Optional[str] = None, password: str | None = None,
) -> Optional[str]: ) -> str | None:
"""Given a WWW-Authenticate header value, fetch and return a Bearer token.""" """Given a WWW-Authenticate header value, fetch and return a Bearer token."""
parsed = parse_www_authenticate(www_authenticate) parsed = parse_www_authenticate(www_authenticate)
if not parsed: if not parsed:
+44 -104
View File
@@ -1,28 +1,30 @@
import hashlib
import json
import logging
import os import os
import re import re
import json from typing import Any
import hashlib
import logging
from typing import Dict, Any, Optional
import httpx import httpx
from fastapi import FastAPI, HTTPException, Response, Request, Query, File, UploadFile from fastapi import FastAPI, File, HTTPException, Query, Request, Response, UploadFile
from fastapi.responses import PlainTextResponse, JSONResponse from fastapi.responses import JSONResponse, PlainTextResponse
from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
from pydantic import BaseModel from pydantic import BaseModel
from prometheus_client import generate_latest, CONTENT_TYPE_LATEST
try: try:
from importlib.metadata import version from importlib.metadata import version
__version__ = version("artifactapi") __version__ = version("artifactapi")
except ImportError: except ImportError:
# Fallback for development when package isn't installed # Fallback for development when package isn't installed
__version__ = "dev" __version__ = "dev"
from .cache import RedisCache
from .config import ConfigManager from .config import ConfigManager
from .database import DatabaseManager from .database import DatabaseManager
from .storage import S3Storage
from .cache import RedisCache
from .metrics import MetricsManager
from .docker_auth import get_docker_token_for_response from .docker_auth import get_docker_token_for_response
from .metrics import MetricsManager
from .storage import S3Storage
class ArtifactRequest(BaseModel): class ArtifactRequest(BaseModel):
@@ -31,10 +33,7 @@ class ArtifactRequest(BaseModel):
# Configure logging # Configure logging
logging.basicConfig( logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
app = FastAPI(title="Artifact Storage API", version=__version__) app = FastAPI(title="Artifact Storage API", version=__version__)
@@ -75,19 +74,11 @@ def health_check():
@app.put("/cache/flush") @app.put("/cache/flush")
def flush_cache( def flush_cache(
remote: str = Query(default=None, description="Specific remote to flush (optional)"), remote: str = Query(default=None, description="Specific remote to flush (optional)"),
cache_type: str = Query(default="all", description="Type to flush: 'all', 'index', 'files', 'metrics'") cache_type: str = Query(default="all", description="Type to flush: 'all', 'index', 'files', 'metrics'"),
): ):
"""Flush cache entries for specified remote or all remotes""" """Flush cache entries for specified remote or all remotes"""
try: try:
result = { result = {"remote": remote, "cache_type": cache_type, "flushed": {"redis_keys": 0, "s3_objects": 0, "operations": []}}
"remote": remote,
"cache_type": cache_type,
"flushed": {
"redis_keys": 0,
"s3_objects": 0,
"operations": []
}
}
# Flush Redis entries based on cache_type # Flush Redis entries based on cache_type
if cache_type in ["all", "index", "metrics"] and cache.available and cache.client: if cache_type in ["all", "index", "metrics"] and cache.available and cache.client:
@@ -124,8 +115,8 @@ def flush_cache(
list_params["Prefix"] = f"{remote}/" list_params["Prefix"] = f"{remote}/"
response = storage.client.list_objects_v2(**list_params) response = storage.client.list_objects_v2(**list_params)
if 'Contents' in response: if "Contents" in response:
objects_to_delete = [obj['Key'] for obj in response['Contents']] objects_to_delete = [obj["Key"] for obj in response["Contents"]]
for key in objects_to_delete: for key in objects_to_delete:
try: try:
@@ -156,15 +147,11 @@ def flush_cache(
async def construct_remote_url(remote_name: str, path: str) -> str: async def construct_remote_url(remote_name: str, path: str) -> str:
remote_config = config.get_remote_config(remote_name) remote_config = config.get_remote_config(remote_name)
if not remote_config: if not remote_config:
raise HTTPException( raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
status_code=404, detail=f"Remote '{remote_name}' not configured"
)
base_url = remote_config.get("base_url") base_url = remote_config.get("base_url")
if not base_url: if not base_url:
raise HTTPException( raise HTTPException(status_code=500, detail=f"No base_url configured for remote '{remote_name}'")
status_code=500, detail=f"No base_url configured for remote '{remote_name}'"
)
# Handle Docker registry URLs # Handle Docker registry URLs
if remote_config.get("package") == "docker": if remote_config.get("package") == "docker":
@@ -175,9 +162,7 @@ async def construct_remote_url(remote_name: str, path: str) -> str:
return f"{base_url}/{path}" return f"{base_url}/{path}"
async def check_artifact_patterns( async def check_artifact_patterns(remote_name: str, repo_path: str, file_path: str, full_path: str) -> bool:
remote_name: str, repo_path: str, file_path: str, full_path: str
) -> bool:
# First check if this is an index file - always allow index files # First check if this is an index file - always allow index files
index_patterns = config.get_index_patterns(remote_name) index_patterns = config.get_index_patterns(remote_name)
if cache.is_index_file(file_path, index_patterns) or cache.is_index_file(full_path, index_patterns): if cache.is_index_file(file_path, index_patterns) or cache.is_index_file(full_path, index_patterns):
@@ -267,9 +252,7 @@ async def get_artifact(remote_name: str, path: str):
# Check if remote is configured # Check if remote is configured
remote_config = config.get_remote_config(remote_name) remote_config = config.get_remote_config(remote_name)
if not remote_config: if not remote_config:
raise HTTPException( raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
status_code=404, detail=f"Remote '{remote_name}' not configured"
)
# Check if this is a local repository # Check if this is a local repository
if remote_config.get("type") == "local": if remote_config.get("type") == "local":
@@ -289,9 +272,7 @@ async def get_artifact(remote_name: str, path: str):
return Response( return Response(
content=content, content=content,
media_type=content_type, media_type=content_type,
headers={ headers={"Content-Disposition": f"attachment; filename={os.path.basename(path)}"},
"Content-Disposition": f"attachment; filename={os.path.basename(path)}"
},
) )
# Extract repository path for pattern checking # Extract repository path for pattern checking
@@ -306,9 +287,7 @@ async def get_artifact(remote_name: str, path: str):
# Check if artifact matches configured patterns # Check if artifact matches configured patterns
if not await check_artifact_patterns(remote_name, repo_path, file_path, path): if not await check_artifact_patterns(remote_name, repo_path, file_path, path):
logger.info(f"PATTERN BLOCKED: {remote_name}/{path} - not matching include patterns") logger.info(f"PATTERN BLOCKED: {remote_name}/{path} - not matching include patterns")
raise HTTPException( raise HTTPException(status_code=403, detail="Artifact not allowed by configuration patterns")
status_code=403, detail="Artifact not allowed by configuration patterns"
)
# Construct the remote URL # Construct the remote URL
remote_url = await construct_remote_url(remote_name, path) remote_url = await construct_remote_url(remote_name, path)
@@ -358,9 +337,7 @@ async def get_artifact(remote_name: str, path: str):
metrics.record_cache_hit(remote_name, len(artifact_data)) metrics.record_cache_hit(remote_name, len(artifact_data))
# Record artifact mapping in database if not already recorded # Record artifact mapping in database if not already recorded
database.record_artifact_mapping( database.record_artifact_mapping(cached_key, remote_name, path, len(artifact_data))
cached_key, remote_name, path, len(artifact_data)
)
return Response( return Response(
content=artifact_data, content=artifact_data,
@@ -372,9 +349,7 @@ async def get_artifact(remote_name: str, path: str):
}, },
) )
except Exception as e: except Exception as e:
raise HTTPException( raise HTTPException(status_code=500, detail=f"Error retrieving cached artifact: {str(e)}")
status_code=500, detail=f"Error retrieving cached artifact: {str(e)}"
)
# Artifact not cached, cache it first # Artifact not cached, cache it first
logger.info(f"Cache MISS: {remote_name}/{path} - fetching from remote: {remote_url}") logger.info(f"Cache MISS: {remote_name}/{path} - fetching from remote: {remote_url}")
@@ -382,9 +357,7 @@ async def get_artifact(remote_name: str, path: str):
if result["status"] == "error": if result["status"] == "error":
logger.error(f"Cache ADD FAILED: {remote_name}/{path} - {result['error']}") logger.error(f"Cache ADD FAILED: {remote_name}/{path} - {result['error']}")
raise HTTPException( raise HTTPException(status_code=502, detail=f"Failed to fetch artifact: {result['error']}")
status_code=502, detail=f"Failed to fetch artifact: {result['error']}"
)
# Mark index files as cached in Redis if this was a new download # Mark index files as cached in Redis if this was a new download
if result["status"] == "cached" and is_index: if result["status"] == "cached" and is_index:
@@ -419,9 +392,7 @@ async def get_artifact(remote_name: str, path: str):
# Record artifact mapping in database # Record artifact mapping in database
cache_key = storage.get_object_key(remote_name, path) cache_key = storage.get_object_key(remote_name, path)
database.record_artifact_mapping( database.record_artifact_mapping(cache_key, remote_name, path, len(artifact_data))
cache_key, remote_name, path, len(artifact_data)
)
return Response( return Response(
content=artifact_data, content=artifact_data,
@@ -533,9 +504,7 @@ async def discover_github_releases(remote: str, include_pattern: str) -> list[st
owner, repo = match.groups() owner, repo = match.groups()
async with httpx.AsyncClient(follow_redirects=True) as client: async with httpx.AsyncClient(follow_redirects=True) as client:
response = await client.get( response = await client.get(f"https://api.github.com/repos/{owner}/{repo}/releases")
f"https://api.github.com/repos/{owner}/{repo}/releases"
)
if response.status_code != 200: if response.status_code != 200:
raise HTTPException( raise HTTPException(
@@ -564,14 +533,10 @@ async def upload_file(remote_name: str, path: str, file: UploadFile = File(...))
# Check if remote is configured and is local # Check if remote is configured and is local
remote_config = config.get_remote_config(remote_name) remote_config = config.get_remote_config(remote_name)
if not remote_config: if not remote_config:
raise HTTPException( raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
status_code=404, detail=f"Remote '{remote_name}' not configured"
)
if remote_config.get("type") != "local": if remote_config.get("type") != "local":
raise HTTPException( raise HTTPException(status_code=400, detail="Upload only supported for local repositories")
status_code=400, detail="Upload only supported for local repositories"
)
try: try:
# Read file content # Read file content
@@ -633,9 +598,7 @@ def check_file_exists(remote_name: str, path: str):
# Check if remote is configured # Check if remote is configured
remote_config = config.get_remote_config(remote_name) remote_config = config.get_remote_config(remote_name)
if not remote_config: if not remote_config:
raise HTTPException( raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
status_code=404, detail=f"Remote '{remote_name}' not configured"
)
# Handle local repository # Handle local repository
if remote_config.get("type") == "local": if remote_config.get("type") == "local":
@@ -647,16 +610,10 @@ def check_file_exists(remote_name: str, path: str):
return Response( return Response(
headers={ headers={
"Content-Length": str(metadata["size_bytes"]), "Content-Length": str(metadata["size_bytes"]),
"Content-Type": metadata.get( "Content-Type": metadata.get("content_type", "application/octet-stream"),
"content_type", "application/octet-stream"
),
"X-SHA256": metadata["sha256_sum"], "X-SHA256": metadata["sha256_sum"],
"X-Created-At": metadata["created_at"].isoformat() "X-Created-At": metadata["created_at"].isoformat() if metadata["created_at"] else "",
if metadata["created_at"] "X-Uploaded-At": metadata["uploaded_at"].isoformat() if metadata["uploaded_at"] else "",
else "",
"X-Uploaded-At": metadata["uploaded_at"].isoformat()
if metadata["uploaded_at"]
else "",
} }
) )
except HTTPException: except HTTPException:
@@ -665,9 +622,7 @@ def check_file_exists(remote_name: str, path: str):
raise HTTPException(status_code=500, detail=f"Check failed: {str(e)}") raise HTTPException(status_code=500, detail=f"Check failed: {str(e)}")
else: else:
# For remote repositories, just return 405 Method Not Allowed # For remote repositories, just return 405 Method Not Allowed
raise HTTPException( raise HTTPException(status_code=405, detail="HEAD method only supported for local repositories")
status_code=405, detail="HEAD method only supported for local repositories"
)
@app.delete("/api/v1/remote/{remote_name}/{path:path}") @app.delete("/api/v1/remote/{remote_name}/{path:path}")
@@ -676,14 +631,10 @@ def delete_file(remote_name: str, path: str):
# Check if remote is configured and is local # Check if remote is configured and is local
remote_config = config.get_remote_config(remote_name) remote_config = config.get_remote_config(remote_name)
if not remote_config: if not remote_config:
raise HTTPException( raise HTTPException(status_code=404, detail=f"Remote '{remote_name}' not configured")
status_code=404, detail=f"Remote '{remote_name}' not configured"
)
if remote_config.get("type") != "local": if remote_config.get("type") != "local":
raise HTTPException( raise HTTPException(status_code=400, detail="Delete only supported for local repositories")
status_code=400, detail="Delete only supported for local repositories"
)
try: try:
# Get S3 key before deleting from database # Get S3 key before deleting from database
@@ -704,11 +655,9 @@ def delete_file(remote_name: str, path: str):
@app.post("/api/v1/artifacts/cache") @app.post("/api/v1/artifacts/cache")
async def cache_artifact(request: ArtifactRequest) -> Dict[str, Any]: async def cache_artifact(request: ArtifactRequest) -> dict[str, Any]:
try: try:
matching_urls = await discover_artifacts( matching_urls = await discover_artifacts(request.remote, request.include_pattern)
request.remote, request.include_pattern
)
if not matching_urls: if not matching_urls:
return { return {
@@ -723,11 +672,7 @@ async def cache_artifact(request: ArtifactRequest) -> Dict[str, Any]:
result = await cache_single_artifact(url, "", "") result = await cache_single_artifact(url, "", "")
cached_artifacts.append(result) cached_artifacts.append(result)
cached_count = sum( cached_count = sum(1 for artifact in cached_artifacts if artifact["status"] in ["cached", "already_cached"])
1
for artifact in cached_artifacts
if artifact["status"] in ["cached", "already_cached"]
)
return { return {
"message": f"Processed {len(matching_urls)} artifacts, {cached_count} successfully cached", "message": f"Processed {len(matching_urls)} artifacts, {cached_count} successfully cached",
@@ -740,9 +685,7 @@ async def cache_artifact(request: ArtifactRequest) -> Dict[str, Any]:
@app.get("/api/v1/artifacts/{remote:path}") @app.get("/api/v1/artifacts/{remote:path}")
async def list_cached_artifacts( async def list_cached_artifacts(remote: str, include_pattern: str = ".*") -> dict[str, Any]:
remote: str, include_pattern: str = ".*"
) -> Dict[str, Any]:
try: try:
matching_urls = await discover_artifacts(remote, include_pattern) matching_urls = await discover_artifacts(remote, include_pattern)
@@ -750,13 +693,12 @@ async def list_cached_artifacts(
for url in matching_urls: for url in matching_urls:
# Extract path from URL for hierarchical key generation # Extract path from URL for hierarchical key generation
from urllib.parse import urlparse from urllib.parse import urlparse
parsed = urlparse(url) parsed = urlparse(url)
path = parsed.path path = parsed.path
key = storage.get_object_key(remote, path) key = storage.get_object_key(remote, path)
if storage.exists(key): if storage.exists(key):
cached_artifacts.append( cached_artifacts.append({"url": url, "cached_url": storage.get_url(key), "key": key})
{"url": url, "cached_url": storage.get_url(key), "key": key}
)
return { return {
"remote": remote, "remote": remote,
@@ -772,9 +714,7 @@ async def list_cached_artifacts(
@app.get("/metrics") @app.get("/metrics")
def get_metrics( def get_metrics(
json: Optional[bool] = Query( json: bool | None = Query(False, description="Return JSON format instead of Prometheus"),
False, description="Return JSON format instead of Prometheus"
),
): ):
"""Get comprehensive metrics about the artifact storage system""" """Get comprehensive metrics about the artifact storage system"""
config._check_reload() config._check_reload()
+18 -51
View File
@@ -1,22 +1,14 @@
from datetime import datetime from datetime import datetime
from typing import Dict, Any from typing import Any
from prometheus_client import Counter, Gauge from prometheus_client import Counter, Gauge
# Prometheus metrics # Prometheus metrics
request_counter = Counter( request_counter = Counter("artifact_requests_total", "Total artifact requests", ["remote", "status"])
"artifact_requests_total", "Total artifact requests", ["remote", "status"]
)
cache_hit_counter = Counter("artifact_cache_hits_total", "Total cache hits", ["remote"]) cache_hit_counter = Counter("artifact_cache_hits_total", "Total cache hits", ["remote"])
cache_miss_counter = Counter( cache_miss_counter = Counter("artifact_cache_misses_total", "Total cache misses", ["remote"])
"artifact_cache_misses_total", "Total cache misses", ["remote"] bandwidth_saved_counter = Counter("artifact_bandwidth_saved_bytes_total", "Total bandwidth saved", ["remote"])
) storage_size_gauge = Gauge("artifact_storage_size_bytes", "Storage size by remote", ["remote"])
bandwidth_saved_counter = Counter(
"artifact_bandwidth_saved_bytes_total", "Total bandwidth saved", ["remote"]
)
storage_size_gauge = Gauge(
"artifact_storage_size_bytes", "Storage size by remote", ["remote"]
)
redis_keys_gauge = Gauge("artifact_redis_keys_total", "Total Redis keys") redis_keys_gauge = Gauge("artifact_redis_keys_total", "Total Redis keys")
@@ -44,9 +36,7 @@ class MetricsManager:
# Increment per-remote counters # Increment per-remote counters
self.redis_client.client.incr(f"metrics:cache_hits:{remote_name}") self.redis_client.client.incr(f"metrics:cache_hits:{remote_name}")
self.redis_client.client.incr(f"metrics:total_requests:{remote_name}") self.redis_client.client.incr(f"metrics:total_requests:{remote_name}")
self.redis_client.client.incrby( self.redis_client.client.incrby(f"metrics:bandwidth_saved:{remote_name}", size_bytes)
f"metrics:bandwidth_saved:{remote_name}", size_bytes
)
except Exception: except Exception:
pass pass
@@ -91,7 +81,7 @@ class MetricsManager:
except Exception: except Exception:
return 0 return 0
def get_s3_size_by_remote(self, storage, config_manager) -> Dict[str, int]: def get_s3_size_by_remote(self, storage, config_manager) -> dict[str, int]:
"""Get size of stored data per remote using database mappings""" """Get size of stored data per remote using database mappings"""
if self.database_manager and self.database_manager.available: if self.database_manager and self.database_manager.available:
# Get from database if available # Get from database if available
@@ -146,7 +136,7 @@ class MetricsManager:
except Exception: except Exception:
return {} return {}
def get_metrics(self, storage, config_manager) -> Dict[str, Any]: def get_metrics(self, storage, config_manager) -> dict[str, Any]:
"""Get comprehensive metrics""" """Get comprehensive metrics"""
# Update Redis keys gauge # Update Redis keys gauge
redis_key_count = self.get_redis_key_count() redis_key_count = self.get_redis_key_count()
@@ -173,54 +163,31 @@ class MetricsManager:
if self.redis_client and self.redis_client.available: if self.redis_client and self.redis_client.available:
try: try:
# Get global metrics # Get global metrics
cache_hits = int( cache_hits = int(self.redis_client.client.get("metrics:cache_hits") or 0)
self.redis_client.client.get("metrics:cache_hits") or 0 cache_misses = int(self.redis_client.client.get("metrics:cache_misses") or 0)
)
cache_misses = int(
self.redis_client.client.get("metrics:cache_misses") or 0
)
total_requests = cache_hits + cache_misses total_requests = cache_hits + cache_misses
bandwidth_saved = int( bandwidth_saved = int(self.redis_client.client.get("metrics:bandwidth_saved") or 0)
self.redis_client.client.get("metrics:bandwidth_saved") or 0
)
metrics["requests"]["cache_hits"] = cache_hits metrics["requests"]["cache_hits"] = cache_hits
metrics["requests"]["cache_misses"] = cache_misses metrics["requests"]["cache_misses"] = cache_misses
metrics["requests"]["total_requests"] = total_requests metrics["requests"]["total_requests"] = total_requests
metrics["requests"]["cache_hit_ratio"] = ( metrics["requests"]["cache_hit_ratio"] = cache_hits / total_requests if total_requests > 0 else 0.0
cache_hits / total_requests if total_requests > 0 else 0.0
)
metrics["bandwidth"]["saved_bytes"] = bandwidth_saved metrics["bandwidth"]["saved_bytes"] = bandwidth_saved
# Get per-remote metrics # Get per-remote metrics
for remote in config_manager.config.get("remotes", {}).keys(): for remote in config_manager.config.get("remotes", {}).keys():
remote_cache_hits = int( remote_cache_hits = int(self.redis_client.client.get(f"metrics:cache_hits:{remote}") or 0)
self.redis_client.client.get(f"metrics:cache_hits:{remote}") remote_cache_misses = int(self.redis_client.client.get(f"metrics:cache_misses:{remote}") or 0)
or 0
)
remote_cache_misses = int(
self.redis_client.client.get(f"metrics:cache_misses:{remote}")
or 0
)
remote_total = remote_cache_hits + remote_cache_misses remote_total = remote_cache_hits + remote_cache_misses
remote_bandwidth_saved = int( remote_bandwidth_saved = int(self.redis_client.client.get(f"metrics:bandwidth_saved:{remote}") or 0)
self.redis_client.client.get(
f"metrics:bandwidth_saved:{remote}"
)
or 0
)
metrics["per_remote"][remote] = { metrics["per_remote"][remote] = {
"cache_hits": remote_cache_hits, "cache_hits": remote_cache_hits,
"cache_misses": remote_cache_misses, "cache_misses": remote_cache_misses,
"total_requests": remote_total, "total_requests": remote_total,
"cache_hit_ratio": remote_cache_hits / remote_total "cache_hit_ratio": remote_cache_hits / remote_total if remote_total > 0 else 0.0,
if remote_total > 0
else 0.0,
"bandwidth_saved_bytes": remote_bandwidth_saved, "bandwidth_saved_bytes": remote_bandwidth_saved,
"storage_size_bytes": metrics["storage"]["size_by_remote"].get( "storage_size_bytes": metrics["storage"]["size_by_remote"].get(remote, 0),
remote, 0
),
} }
except Exception: except Exception:
+9 -9
View File
@@ -1,5 +1,6 @@
import os
import hashlib import hashlib
import os
import boto3 import boto3
from botocore.config import Config from botocore.config import Config
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
@@ -21,23 +22,22 @@ class S3Storage:
self.bucket = bucket self.bucket = bucket
self.secure = secure self.secure = secure
ca_bundle = os.environ.get('REQUESTS_CA_BUNDLE') or os.environ.get('SSL_CERT_FILE') ca_bundle = os.environ.get("REQUESTS_CA_BUNDLE") or os.environ.get("SSL_CERT_FILE")
config_kwargs = { config_kwargs = {"request_checksum_calculation": "when_required", "response_checksum_validation": "when_required"}
"request_checksum_calculation": "when_required",
"response_checksum_validation": "when_required"
}
client_kwargs = { client_kwargs = {
"endpoint_url": f"http{'s' if self.secure else ''}://{self.endpoint}", "endpoint_url": f"http{'s' if self.secure else ''}://{self.endpoint}",
"aws_access_key_id": self.access_key, "aws_access_key_id": self.access_key,
"aws_secret_access_key": self.secret_key, "aws_secret_access_key": self.secret_key,
"config": Config(**config_kwargs) "config": Config(**config_kwargs),
} }
if ca_bundle and os.path.exists(ca_bundle): if ca_bundle and os.path.exists(ca_bundle):
client_kwargs["verify"] = ca_bundle client_kwargs["verify"] = ca_bundle
print(f"Debug: Using CA bundle: {ca_bundle}") print(f"Debug: Using CA bundle: {ca_bundle}")
else: else:
print(f"Debug: No CA bundle found. REQUESTS_CA_BUNDLE={os.environ.get('REQUESTS_CA_BUNDLE')}, SSL_CERT_FILE={os.environ.get('SSL_CERT_FILE')}") print(
f"Debug: No CA bundle found. REQUESTS_CA_BUNDLE={os.environ.get('REQUESTS_CA_BUNDLE')}, SSL_CERT_FILE={os.environ.get('SSL_CERT_FILE')}"
)
self.client = boto3.client("s3", **client_kwargs) self.client = boto3.client("s3", **client_kwargs)
@@ -56,7 +56,7 @@ class S3Storage:
def get_object_key(self, remote_name: str, path: str) -> str: def get_object_key(self, remote_name: str, path: str) -> str:
# Extract directory path and filename # Extract directory path and filename
clean_path = path.lstrip('/') clean_path = path.lstrip("/")
filename = os.path.basename(clean_path) filename = os.path.basename(clean_path)
directory_path = os.path.dirname(clean_path) directory_path = os.path.dirname(clean_path)
View File
+127
View File
@@ -0,0 +1,127 @@
"""
Pytest configuration and shared fixtures.
Module-level setup (env vars + connection patches) runs before any test
module is imported, so the FastAPI app initialises against mocks rather
than real S3 / Redis / PostgreSQL services.
"""
import os
import tempfile
from unittest.mock import MagicMock, patch
import yaml
# ---------------------------------------------------------------------------
# Test remote configuration
# ---------------------------------------------------------------------------
TEST_REMOTES = {
"remotes": {
"alpine-test": {
"base_url": "https://dl-cdn.alpinelinux.org",
"type": "remote",
"package": "alpine",
"include_patterns": [".*/x86_64/.*\\.apk$"],
"cache": {"file_ttl": 0, "index_ttl": 3600},
},
"rpm-test": {
"base_url": "https://example.com/rpm",
"type": "remote",
"package": "rpm",
"include_patterns": [".*/x86_64/.*\\.rpm$", ".*/repodata/.*$"],
"cache": {"file_ttl": 0, "index_ttl": 3600},
},
"docker-test": {
"base_url": "https://registry.example.com",
"type": "remote",
"package": "docker",
"cache": {"file_ttl": 0, "index_ttl": 300},
},
"docker-restricted": {
"base_url": "https://registry.example.com",
"type": "remote",
"package": "docker",
"include_patterns": ["^library/nginx"],
"cache": {"file_ttl": 0, "index_ttl": 300},
},
"generic-test": {
"base_url": "https://releases.example.com",
"type": "remote",
"package": "generic",
"include_patterns": [".*\\.tar\\.gz$"],
"cache": {"file_ttl": 0, "index_ttl": 0},
},
"custom-index-test": {
"base_url": "https://example.com",
"type": "remote",
"package": "generic",
"index_patterns": ["metadata\\.json$"],
"cache": {"file_ttl": 0, "index_ttl": 600},
},
"local-test": {
"type": "local",
"package": "generic",
"cache": {"file_ttl": 0, "index_ttl": 0},
},
}
}
# ---------------------------------------------------------------------------
# Write temp config and set env vars BEFORE importing the package
# ---------------------------------------------------------------------------
_tmpdir = tempfile.mkdtemp()
_config_path = os.path.join(_tmpdir, "remotes.yaml")
with open(_config_path, "w") as _f:
yaml.dump(TEST_REMOTES, _f)
os.environ.update({
"CONFIG_PATH": _config_path,
"MINIO_ENDPOINT": "localhost:9000",
"MINIO_ACCESS_KEY": "testkey",
"MINIO_SECRET_KEY": "testsecret",
"MINIO_BUCKET": "testbucket",
"REDIS_URL": "redis://localhost:6379/0",
"DBHOST": "localhost",
"DBPORT": "5432",
"DBUSER": "test",
"DBPASS": "test",
"DBNAME": "test",
})
# Patch external service connections before the package is imported.
# These stay active for the whole session (process exits after tests finish).
_boto3_patch = patch("boto3.client", return_value=MagicMock())
_redis_patch = patch("redis.from_url", return_value=MagicMock())
_psycopg2_patch = patch("psycopg2.connect", return_value=MagicMock())
_boto3_patch.start()
_redis_patch.start()
_psycopg2_patch.start()
# ---------------------------------------------------------------------------
# Shared fixtures
# ---------------------------------------------------------------------------
import pytest # noqa: E402
from fastapi.testclient import TestClient # noqa: E402
@pytest.fixture(scope="session")
def app():
from artifactapi.main import app as fastapi_app
return fastapi_app
@pytest.fixture(scope="session")
def client(app):
return TestClient(app)
@pytest.fixture
def config_path():
return _config_path
@pytest.fixture
def test_remotes():
return TEST_REMOTES
+193
View File
@@ -0,0 +1,193 @@
"""Tests for RedisCache, focusing on is_index_file with configurable patterns."""
from unittest.mock import MagicMock, patch
import pytest
from artifactapi.cache import RedisCache
from artifactapi.config import _PACKAGE_INDEX_PATTERNS
@pytest.fixture
def bare_cache():
"""RedisCache instance bypassing __init__ (no Redis needed for pure-logic tests)."""
return RedisCache.__new__(RedisCache)
@pytest.fixture
def unavailable_cache():
"""RedisCache where Redis is not reachable."""
with patch("redis.from_url", side_effect=Exception("connection refused")):
return RedisCache("redis://localhost:6379/0")
@pytest.fixture
def mock_redis_client():
return MagicMock()
@pytest.fixture
def cache_with_redis(mock_redis_client):
"""RedisCache backed by a MagicMock Redis client."""
with patch("redis.from_url", return_value=mock_redis_client):
c = RedisCache("redis://localhost:6379/0")
c.client = mock_redis_client
c.available = True
return c
# ---------------------------------------------------------------------------
# is_index_file — alpine patterns
# ---------------------------------------------------------------------------
class TestIsIndexFileAlpine:
def test_apkindex_tarball_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
assert bare_cache.is_index_file("alpine/v3.18/x86_64/APKINDEX.tar.gz", patterns)
def test_nested_apkindex_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
assert bare_cache.is_index_file("mirrors/dl-cdn/alpine/v3.19/community/x86_64/APKINDEX.tar.gz", patterns)
def test_apk_package_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
assert not bare_cache.is_index_file("alpine/v3.18/x86_64/musl-1.2.4-r2.apk", patterns)
def test_random_tarball_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["alpine"]
assert not bare_cache.is_index_file("some/path/archive.tar.gz", patterns)
# ---------------------------------------------------------------------------
# is_index_file — rpm patterns
# ---------------------------------------------------------------------------
class TestIsIndexFileRpm:
def test_repomd_xml_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("almalinux/9/x86_64/repomd.xml", patterns)
def test_repodata_primary_xml_gz_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("repo/repodata/primary.xml.gz", patterns)
def test_repodata_sqlite_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("repo/repodata/primary.sqlite", patterns)
def test_repodata_sqlite_bz2_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("repo/repodata/other.sqlite.bz2", patterns)
def test_repodata_yaml_xz_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("repo/repodata/comps.yaml.xz", patterns)
def test_packages_gz_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert bare_cache.is_index_file("debian/dists/stable/main/binary-amd64/Packages.gz", patterns)
def test_rpm_package_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert not bare_cache.is_index_file("almalinux/9/x86_64/Packages/bash-5.1.8.x86_64.rpm", patterns)
def test_arbitrary_xml_outside_repodata_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["rpm"]
assert not bare_cache.is_index_file("some/path/config.xml", patterns)
# ---------------------------------------------------------------------------
# is_index_file — docker patterns
# ---------------------------------------------------------------------------
class TestIsIndexFileDocker:
def test_tag_manifest_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
assert bare_cache.is_index_file("library/nginx/manifests/latest", patterns)
def test_version_tag_manifest_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
assert bare_cache.is_index_file("library/nginx/manifests/1.25.3", patterns)
def test_digest_manifest_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
digest = "sha256:" + "a" * 64
assert not bare_cache.is_index_file(f"library/nginx/manifests/{digest}", patterns)
def test_tags_list_is_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
assert bare_cache.is_index_file("library/nginx/tags/list", patterns)
def test_blob_is_not_index(self, bare_cache):
patterns = _PACKAGE_INDEX_PATTERNS["docker"]
assert not bare_cache.is_index_file("library/nginx/blobs/sha256:abc123", patterns)
# ---------------------------------------------------------------------------
# is_index_file — edge cases
# ---------------------------------------------------------------------------
class TestIsIndexFileEdgeCases:
def test_empty_patterns_nothing_is_index(self, bare_cache):
assert not bare_cache.is_index_file("APKINDEX.tar.gz", [])
assert not bare_cache.is_index_file("repomd.xml", [])
assert not bare_cache.is_index_file("library/nginx/manifests/latest", [])
def test_none_patterns_nothing_is_index(self, bare_cache):
assert not bare_cache.is_index_file("APKINDEX.tar.gz", None)
assert not bare_cache.is_index_file("repomd.xml", None)
def test_custom_patterns_match(self, bare_cache):
patterns = [r"metadata\.json$", r"index\.yaml$"]
assert bare_cache.is_index_file("repo/metadata.json", patterns)
assert bare_cache.is_index_file("repo/subdir/index.yaml", patterns)
assert not bare_cache.is_index_file("repo/data.tar.gz", patterns)
def test_custom_pattern_does_not_match_standard_index(self, bare_cache):
patterns = [r"metadata\.json$"]
assert not bare_cache.is_index_file("APKINDEX.tar.gz", patterns)
# ---------------------------------------------------------------------------
# get_index_cache_key
# ---------------------------------------------------------------------------
class TestGetIndexCacheKey:
def test_same_inputs_produce_same_key(self, bare_cache):
k1 = bare_cache.get_index_cache_key("alpine-test", "alpine/v3.18/x86_64/APKINDEX.tar.gz")
k2 = bare_cache.get_index_cache_key("alpine-test", "alpine/v3.18/x86_64/APKINDEX.tar.gz")
assert k1 == k2
def test_different_paths_produce_different_keys(self, bare_cache):
k1 = bare_cache.get_index_cache_key("alpine-test", "alpine/v3.18/x86_64/APKINDEX.tar.gz")
k2 = bare_cache.get_index_cache_key("alpine-test", "alpine/v3.19/x86_64/APKINDEX.tar.gz")
assert k1 != k2
def test_different_remotes_produce_different_keys(self, bare_cache):
k1 = bare_cache.get_index_cache_key("remote-a", "path/to/APKINDEX.tar.gz")
k2 = bare_cache.get_index_cache_key("remote-b", "path/to/APKINDEX.tar.gz")
assert k1 != k2
def test_key_starts_with_index_prefix_and_remote(self, bare_cache):
key = bare_cache.get_index_cache_key("myremote", "some/path")
assert key.startswith("index:myremote:")
# ---------------------------------------------------------------------------
# mark_index_cached / is_index_valid
# ---------------------------------------------------------------------------
class TestIndexValidity:
def test_marked_then_valid(self, cache_with_redis, mock_redis_client):
mock_redis_client.exists.return_value = 1
cache_with_redis.mark_index_cached("remote", "path/APKINDEX.tar.gz", 300)
assert cache_with_redis.is_index_valid("remote", "path/APKINDEX.tar.gz")
def test_missing_key_is_not_valid(self, cache_with_redis, mock_redis_client):
mock_redis_client.exists.return_value = 0
assert not cache_with_redis.is_index_valid("remote", "path/APKINDEX.tar.gz")
def test_unavailable_redis_is_not_valid(self, unavailable_cache):
assert not unavailable_cache.is_index_valid("remote", "some/path")
def test_mark_cached_no_op_when_unavailable(self, unavailable_cache):
unavailable_cache.mark_index_cached("remote", "some/path", 300) # must not raise
+194
View File
@@ -0,0 +1,194 @@
"""Tests for ConfigManager, focusing on get_index_patterns (new logic)."""
import os
import pytest
import yaml
from artifactapi.config import _PACKAGE_INDEX_PATTERNS, ConfigManager
@pytest.fixture
def make_config(tmp_path):
"""Factory: write a remotes dict to a temp YAML and return a ConfigManager."""
def _make(remotes_dict):
cfg_file = tmp_path / "remotes.yaml"
cfg_file.write_text(yaml.dump({"remotes": remotes_dict}))
return ConfigManager(str(cfg_file))
return _make
# ---------------------------------------------------------------------------
# get_index_patterns
# ---------------------------------------------------------------------------
class TestGetIndexPatterns:
def test_alpine_returns_package_defaults(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "alpine", "base_url": "https://x.com"}})
assert cfg.get_index_patterns("r") == _PACKAGE_INDEX_PATTERNS["alpine"]
def test_rpm_returns_package_defaults(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "rpm", "base_url": "https://x.com"}})
assert cfg.get_index_patterns("r") == _PACKAGE_INDEX_PATTERNS["rpm"]
def test_docker_returns_package_defaults(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "docker", "base_url": "https://x.com"}})
assert cfg.get_index_patterns("r") == _PACKAGE_INDEX_PATTERNS["docker"]
def test_generic_returns_empty_list(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}})
assert cfg.get_index_patterns("r") == []
def test_unknown_remote_returns_empty_list(self, make_config):
cfg = make_config({})
assert cfg.get_index_patterns("nonexistent") == []
def test_missing_package_field_defaults_to_generic(self, make_config):
cfg = make_config({"r": {"type": "remote", "base_url": "https://x.com"}})
assert cfg.get_index_patterns("r") == []
def test_extra_patterns_appended_after_defaults(self, make_config):
cfg = make_config({
"r": {
"type": "remote",
"package": "alpine",
"base_url": "https://x.com",
"index_patterns": ["custom\\.json$"],
}
})
patterns = cfg.get_index_patterns("r")
defaults = _PACKAGE_INDEX_PATTERNS["alpine"]
assert patterns[: len(defaults)] == defaults
assert "custom\\.json$" in patterns
def test_duplicate_extra_pattern_not_added_twice(self, make_config):
existing = _PACKAGE_INDEX_PATTERNS["alpine"][0]
cfg = make_config({
"r": {
"type": "remote",
"package": "alpine",
"base_url": "https://x.com",
"index_patterns": [existing],
}
})
patterns = cfg.get_index_patterns("r")
assert patterns.count(existing) == 1
def test_generic_with_only_extra_patterns(self, make_config):
cfg = make_config({
"r": {
"type": "remote",
"package": "generic",
"base_url": "https://x.com",
"index_patterns": ["meta\\.json$", "index\\.yaml$"],
}
})
assert cfg.get_index_patterns("r") == ["meta\\.json$", "index\\.yaml$"]
def test_rpm_extra_patterns_merged(self, make_config):
cfg = make_config({
"r": {
"type": "remote",
"package": "rpm",
"base_url": "https://x.com",
"index_patterns": ["custom-meta\\.xml$"],
}
})
patterns = cfg.get_index_patterns("r")
for default in _PACKAGE_INDEX_PATTERNS["rpm"]:
assert default in patterns
assert "custom-meta\\.xml$" in patterns
# ---------------------------------------------------------------------------
# get_repository_patterns
# ---------------------------------------------------------------------------
class TestGetRepositoryPatterns:
def test_returns_include_patterns(self, make_config):
cfg = make_config({
"r": {
"type": "remote",
"package": "generic",
"base_url": "https://x.com",
"include_patterns": [".*\\.tar\\.gz$"],
}
})
assert cfg.get_repository_patterns("r", "") == [".*\\.tar\\.gz$"]
def test_returns_empty_for_missing_remote(self, make_config):
cfg = make_config({})
assert cfg.get_repository_patterns("nonexistent", "") == []
def test_returns_empty_when_no_patterns_configured(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}})
assert cfg.get_repository_patterns("r", "") == []
def test_multiple_patterns_returned(self, make_config):
patterns = [".*\\.rpm$", ".*/repodata/.*$"]
cfg = make_config({
"r": {
"type": "remote",
"package": "rpm",
"base_url": "https://x.com",
"include_patterns": patterns,
}
})
assert cfg.get_repository_patterns("r", "") == patterns
# ---------------------------------------------------------------------------
# get_cache_config
# ---------------------------------------------------------------------------
class TestGetCacheConfig:
def test_returns_cache_section(self, make_config):
cfg = make_config({
"r": {
"type": "remote",
"package": "generic",
"base_url": "https://x.com",
"cache": {"file_ttl": 0, "index_ttl": 7200},
}
})
result = cfg.get_cache_config("r")
assert result["index_ttl"] == 7200
assert result["file_ttl"] == 0
def test_returns_empty_dict_for_missing_remote(self, make_config):
cfg = make_config({})
assert cfg.get_cache_config("nonexistent") == {}
def test_returns_empty_dict_when_no_cache_key(self, make_config):
cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}})
assert cfg.get_cache_config("r") == {}
# ---------------------------------------------------------------------------
# Config file reload
# ---------------------------------------------------------------------------
class TestConfigReload:
def test_reloads_when_file_mtime_advances(self, tmp_path):
cfg_file = tmp_path / "remotes.yaml"
cfg_file.write_text(yaml.dump({"remotes": {"repo-a": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}}))
cfg = ConfigManager(str(cfg_file))
assert "repo-a" in cfg.config["remotes"]
cfg_file.write_text(yaml.dump({"remotes": {"repo-b": {"type": "remote", "package": "generic", "base_url": "https://y.com"}}}))
future_mtime = cfg._last_modified + 1
os.utime(str(cfg_file), (future_mtime, future_mtime))
cfg._check_reload()
assert "repo-b" in cfg.config["remotes"]
assert "repo-a" not in cfg.config["remotes"]
def test_no_reload_when_file_unchanged(self, tmp_path):
cfg_file = tmp_path / "remotes.yaml"
cfg_file.write_text(yaml.dump({"remotes": {"repo-a": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}}))
cfg = ConfigManager(str(cfg_file))
# Call check_reload without touching the file → should not reload
cfg._check_reload()
assert "repo-a" in cfg.config["remotes"]
+223
View File
@@ -0,0 +1,223 @@
"""Tests for docker_auth: WWW-Authenticate parsing and token caching."""
import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artifactapi import docker_auth
from artifactapi.docker_auth import (
_cache_key,
_get_cached_token,
_store_token,
fetch_token,
get_docker_token_for_response,
parse_www_authenticate,
)
@pytest.fixture(autouse=True)
def clear_token_cache():
"""Isolate tests: wipe the module-level token cache before and after each test."""
docker_auth._token_cache.clear()
yield
docker_auth._token_cache.clear()
# ---------------------------------------------------------------------------
# parse_www_authenticate
# ---------------------------------------------------------------------------
class TestParseWwwAuthenticate:
def test_full_bearer_header(self):
header = (
'Bearer realm="https://auth.docker.io/token"'
',service="registry.docker.io"'
',scope="repository:library/nginx:pull"'
)
result = parse_www_authenticate(header)
assert result is not None
realm, service, scope = result
assert realm == "https://auth.docker.io/token"
assert service == "registry.docker.io"
assert scope == "repository:library/nginx:pull"
def test_realm_only(self):
header = 'Bearer realm="https://auth.example.com/token"'
result = parse_www_authenticate(header)
assert result is not None
realm, service, scope = result
assert realm == "https://auth.example.com/token"
assert service == ""
assert scope == ""
def test_realm_and_service_only(self):
header = 'Bearer realm="https://auth.example.com",service="registry.example.com"'
result = parse_www_authenticate(header)
assert result is not None
_, service, scope = result
assert service == "registry.example.com"
assert scope == ""
def test_invalid_scheme_returns_none(self):
assert parse_www_authenticate('Basic realm="example"') is None
def test_empty_header_returns_none(self):
assert parse_www_authenticate("") is None
def test_case_insensitive_bearer(self):
header = 'bearer realm="https://auth.example.com/token"'
assert parse_www_authenticate(header) is not None
# ---------------------------------------------------------------------------
# _cache_key
# ---------------------------------------------------------------------------
class TestCacheKey:
def test_key_contains_all_components(self):
key = _cache_key("https://realm.com", "svc", "scope", "user")
assert "https://realm.com" in key
assert "svc" in key
assert "scope" in key
assert "user" in key
def test_none_username_uses_empty_string(self):
key = _cache_key("https://realm.com", "svc", "scope", None)
assert key.endswith("|")
def test_different_services_give_different_keys(self):
k1 = _cache_key("realm", "svc1", "scope", None)
k2 = _cache_key("realm", "svc2", "scope", None)
assert k1 != k2
def test_different_scopes_give_different_keys(self):
k1 = _cache_key("realm", "svc", "scope:read", None)
k2 = _cache_key("realm", "svc", "scope:write", None)
assert k1 != k2
# ---------------------------------------------------------------------------
# _get_cached_token / _store_token
# ---------------------------------------------------------------------------
class TestTokenCaching:
def test_get_returns_none_when_not_cached(self):
assert _get_cached_token("no-such-key") is None
def test_get_returns_token_when_valid(self):
_store_token("mykey", "tok-abc", 300)
assert _get_cached_token("mykey") == "tok-abc"
def test_get_returns_none_when_expired(self):
docker_auth._token_cache["mykey"] = ("old-token", time.time() - 1)
assert _get_cached_token("mykey") is None
def test_expired_entry_is_removed_from_cache(self):
docker_auth._token_cache["mykey"] = ("old-token", time.time() - 1)
_get_cached_token("mykey")
assert "mykey" not in docker_auth._token_cache
def test_store_expires_30s_before_stated_time(self):
before = time.time()
_store_token("mykey", "tok", 100)
_, expires_at = docker_auth._token_cache["mykey"]
# expires_in - 30 = 70; allow ±2 s clock wiggle
assert before + 68 <= expires_at <= before + 72
def test_store_enforces_minimum_10s_expiry(self):
before = time.time()
_store_token("mykey", "tok", 5) # expires_in - 30 would be negative
_, expires_at = docker_auth._token_cache["mykey"]
assert expires_at >= before + 10
# ---------------------------------------------------------------------------
# fetch_token (async, mocks httpx)
# ---------------------------------------------------------------------------
def _make_mock_http_client(token_payload: dict):
mock_response = MagicMock()
mock_response.raise_for_status = MagicMock()
mock_response.json.return_value = token_payload
mock_client = AsyncMock()
mock_client.get = AsyncMock(return_value=mock_response)
ctx = MagicMock()
ctx.__aenter__ = AsyncMock(return_value=mock_client)
ctx.__aexit__ = AsyncMock(return_value=False)
return ctx, mock_client
class TestFetchToken:
async def test_returns_token_field(self):
ctx, _ = _make_mock_http_client({"token": "bearer-tok", "expires_in": 300})
with patch("httpx.AsyncClient", return_value=ctx):
token = await fetch_token("https://auth.example.com", "svc", "scope")
assert token == "bearer-tok"
async def test_falls_back_to_access_token_field(self):
ctx, _ = _make_mock_http_client({"access_token": "access-tok", "expires_in": 300})
with patch("httpx.AsyncClient", return_value=ctx):
token = await fetch_token("https://auth.example.com", "svc", "scope")
assert token == "access-tok"
async def test_uses_cache_on_second_call_without_http(self):
ctx, mock_client = _make_mock_http_client({"token": "cached-tok", "expires_in": 300})
with patch("httpx.AsyncClient", return_value=ctx):
await fetch_token("https://auth.example.com", "svc", "scope")
mock_client.get.reset_mock()
token = await fetch_token("https://auth.example.com", "svc", "scope")
mock_client.get.assert_not_called()
assert token == "cached-tok"
async def test_returns_none_on_http_error(self):
mock_client = AsyncMock()
mock_client.get = AsyncMock(side_effect=Exception("connection refused"))
ctx = MagicMock()
ctx.__aenter__ = AsyncMock(return_value=mock_client)
ctx.__aexit__ = AsyncMock(return_value=False)
with patch("httpx.AsyncClient", return_value=ctx):
token = await fetch_token("https://auth.example.com", "svc", "scope")
assert token is None
async def test_passes_credentials_as_auth_tuple(self):
ctx, mock_client = _make_mock_http_client({"token": "authed-tok", "expires_in": 300})
with patch("httpx.AsyncClient", return_value=ctx):
await fetch_token("https://auth.example.com", "svc", "scope", "user", "pass")
call_kwargs = mock_client.get.call_args.kwargs
assert call_kwargs.get("auth") == ("user", "pass")
async def test_no_auth_when_no_credentials(self):
ctx, mock_client = _make_mock_http_client({"token": "anon-tok", "expires_in": 300})
with patch("httpx.AsyncClient", return_value=ctx):
await fetch_token("https://auth.example.com", "svc", "scope")
call_kwargs = mock_client.get.call_args.kwargs
assert call_kwargs.get("auth") is None
# ---------------------------------------------------------------------------
# get_docker_token_for_response
# ---------------------------------------------------------------------------
class TestGetDockerTokenForResponse:
async def test_returns_none_for_non_bearer_header(self):
token = await get_docker_token_for_response('Basic realm="example"')
assert token is None
async def test_delegates_to_fetch_token(self):
header = (
'Bearer realm="https://auth.example.com"'
',service="svc"'
',scope="repo:pull"'
)
with patch(
"artifactapi.docker_auth.fetch_token",
new_callable=AsyncMock,
return_value="delegated-tok",
) as mock_fetch:
token = await get_docker_token_for_response(header, "user", "pass")
mock_fetch.assert_called_once_with(
"https://auth.example.com", "svc", "repo:pull", "user", "pass"
)
assert token == "delegated-tok"
+316
View File
@@ -0,0 +1,316 @@
"""FastAPI route tests using TestClient with mocked service dependencies."""
import hashlib
import json
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Per-test service mocks (replace module-level globals in main.py)
# ---------------------------------------------------------------------------
@pytest.fixture
def mock_storage():
m = MagicMock()
m.get_object_key.return_value = "test-remote/abc123/file.ext"
m.exists.return_value = False
m.download_object.return_value = b"fake content"
m.bucket = "testbucket"
m.client = MagicMock()
return m
@pytest.fixture
def mock_cache():
m = MagicMock()
m.is_index_file.return_value = False
m.is_index_valid.return_value = True
m.available = False
m.client = None
return m
@pytest.fixture
def mock_database():
m = MagicMock()
m.available = False
return m
@pytest.fixture
def mock_metrics():
return MagicMock()
@pytest.fixture
def patched_deps(mock_storage, mock_cache, mock_database, mock_metrics):
"""Swap the module-level service instances in main.py for the duration of a test."""
import artifactapi.main as main_mod
with (
patch.object(main_mod, "storage", mock_storage),
patch.object(main_mod, "cache", mock_cache),
patch.object(main_mod, "database", mock_database),
patch.object(main_mod, "metrics", mock_metrics),
):
yield {
"storage": mock_storage,
"cache": mock_cache,
"database": mock_database,
"metrics": mock_metrics,
}
# ---------------------------------------------------------------------------
# Basic / health endpoints
# ---------------------------------------------------------------------------
class TestBasicEndpoints:
def test_root_returns_remote_list(self, client):
response = client.get("/")
assert response.status_code == 200
data = response.json()
assert "remotes" in data
assert isinstance(data["remotes"], list)
assert len(data["remotes"]) > 0
def test_root_contains_version(self, client):
response = client.get("/")
assert "version" in response.json()
def test_health_check(self, client):
response = client.get("/health")
assert response.status_code == 200
assert response.json()["status"] == "healthy"
def test_docker_v2_ping(self, client):
response = client.get("/v2/")
assert response.status_code == 200
assert response.headers.get("Docker-Distribution-Api-Version") == "registry/2.0"
assert response.json() == {}
# ---------------------------------------------------------------------------
# Docker proxy /v2/{remote}/{path}
# ---------------------------------------------------------------------------
class TestDockerProxy:
def test_unknown_remote_returns_404(self, client, patched_deps):
response = client.get("/v2/no-such-remote/library/nginx/manifests/latest")
assert response.status_code == 404
def test_non_docker_package_returns_400(self, client, patched_deps):
# alpine-test is package: alpine, not docker
response = client.get("/v2/alpine-test/library/nginx/manifests/latest")
assert response.status_code == 400
def test_pattern_blocked_returns_403(self, client, patched_deps):
# docker-restricted allows only "library/nginx"
response = client.get("/v2/docker-restricted/library/ubuntu/manifests/latest")
assert response.status_code == 403
def test_allowed_pattern_proceeds_to_cache(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"layers": [],
}).encode()
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = manifest
deps["cache"].is_index_file.return_value = True
deps["cache"].is_index_valid.return_value = True
response = client.get("/v2/docker-restricted/library/nginx/manifests/latest")
assert response.status_code == 200
def test_cache_hit_manifest_returns_correct_content_type(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"schemaVersion": 2,
"layers": [],
}).encode()
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = manifest
deps["cache"].is_index_file.return_value = True
deps["cache"].is_index_valid.return_value = True
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
assert response.status_code == 200
ct = response.headers["content-type"]
assert ct.startswith("application/vnd.docker.distribution.manifest.v2+json")
def test_cache_hit_sets_docker_content_digest_header(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"layers": [],
}).encode()
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = manifest
deps["cache"].is_index_file.return_value = True
deps["cache"].is_index_valid.return_value = True
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
expected = f"sha256:{hashlib.sha256(manifest).hexdigest()}"
assert response.headers["Docker-Content-Digest"] == expected
def test_head_request_returns_no_body(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"layers": [],
}).encode()
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = manifest
deps["cache"].is_index_file.return_value = False
response = client.head("/v2/docker-test/library/nginx/manifests/latest")
assert response.status_code == 200
assert response.content == b""
def test_cache_miss_calls_upstream_fetch(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"layers": [],
}).encode()
deps["storage"].exists.return_value = False
deps["storage"].download_object.return_value = manifest
deps["cache"].is_index_file.return_value = True
with patch(
"artifactapi.main.cache_single_artifact",
new_callable=AsyncMock,
return_value={"status": "cached"},
) as mock_fetch:
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
mock_fetch.assert_called_once()
assert response.status_code == 200
def test_index_expired_triggers_refetch(self, client, patched_deps):
deps = patched_deps
manifest = json.dumps({
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"layers": [],
}).encode()
deps["storage"].exists.return_value = True # cached in S3
deps["cache"].is_index_file.return_value = True
deps["cache"].is_index_valid.return_value = False # but TTL expired
deps["storage"].download_object.return_value = manifest
with patch(
"artifactapi.main.cache_single_artifact",
new_callable=AsyncMock,
return_value={"status": "cached"},
) as mock_fetch:
response = client.get("/v2/docker-test/library/nginx/manifests/latest")
mock_fetch.assert_called_once()
assert response.status_code == 200
# ---------------------------------------------------------------------------
# Generic artifact route /api/v1/remote/{remote}/{path}
# ---------------------------------------------------------------------------
class TestGenericArtifactRoute:
def test_unknown_remote_returns_404(self, client, patched_deps):
response = client.get("/api/v1/remote/nonexistent/path/to/file.tar.gz")
assert response.status_code == 404
def test_pattern_blocked_returns_403(self, client, patched_deps):
# generic-test only allows .tar.gz
response = client.get("/api/v1/remote/generic-test/some/path/file.rpm")
assert response.status_code == 403
def test_cache_hit_returns_200_with_header(self, client, patched_deps):
deps = patched_deps
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = b"tar content"
deps["cache"].is_index_file.return_value = False
response = client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
assert response.status_code == 200
assert response.headers["X-Artifact-Source"] == "cache"
assert response.content == b"tar content"
def test_cache_miss_fetches_upstream_and_returns_200(self, client, patched_deps):
deps = patched_deps
deps["storage"].exists.return_value = False
deps["storage"].download_object.return_value = b"fresh content"
deps["cache"].is_index_file.return_value = False
with patch(
"artifactapi.main.cache_single_artifact",
new_callable=AsyncMock,
return_value={"status": "cached"},
) as mock_fetch:
response = client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
mock_fetch.assert_called_once()
assert response.status_code == 200
assert response.headers["X-Artifact-Source"] == "remote"
def test_upstream_error_returns_502(self, client, patched_deps):
deps = patched_deps
deps["storage"].exists.return_value = False
deps["cache"].is_index_file.return_value = False
with patch(
"artifactapi.main.cache_single_artifact",
new_callable=AsyncMock,
return_value={"status": "error", "error": "upstream unreachable"},
):
response = client.get("/api/v1/remote/generic-test/some/path/archive.tar.gz")
assert response.status_code == 502
def test_index_file_bypasses_include_patterns(self, client, patched_deps):
"""Index files must be served even when they don't match include_patterns."""
deps = patched_deps
deps["storage"].exists.return_value = True
deps["storage"].download_object.return_value = b"APKINDEX content"
# Simulate is_index_file returning True (APKINDEX.tar.gz detected as index)
deps["cache"].is_index_file.return_value = True
deps["cache"].is_index_valid.return_value = True
# APKINDEX.tar.gz does not match alpine-test's include_patterns (.*.apk$),
# but since it's an index file it must be allowed through.
response = client.get("/api/v1/remote/alpine-test/alpine/v3.18/x86_64/APKINDEX.tar.gz")
assert response.status_code == 200
def test_local_repo_file_not_found_returns_404(self, client, patched_deps):
deps = patched_deps
deps["database"].get_local_file_metadata.return_value = None
deps["database"].available = True
response = client.get("/api/v1/remote/local-test/path/to/nonexistent.bin")
assert response.status_code == 404
# ---------------------------------------------------------------------------
# Cache flush PUT /cache/flush
# ---------------------------------------------------------------------------
class TestCacheFlushEndpoint:
def test_flush_all_returns_flushed_structure(self, client, patched_deps):
deps = patched_deps
deps["cache"].available = False
deps["storage"].client.list_objects_v2.return_value = {}
response = client.put("/cache/flush")
assert response.status_code == 200
data = response.json()
assert "flushed" in data
assert "redis_keys" in data["flushed"]
assert "s3_objects" in data["flushed"]
def test_flush_specific_remote_echoes_remote(self, client, patched_deps):
deps = patched_deps
deps["cache"].available = False
deps["storage"].client.list_objects_v2.return_value = {}
response = client.put("/cache/flush?remote=alpine-test")
assert response.status_code == 200
assert response.json()["remote"] == "alpine-test"
+101
View File
@@ -0,0 +1,101 @@
"""Tests for S3Storage, focusing on get_object_key (pure logic, no S3 calls)."""
from unittest.mock import MagicMock, patch
import pytest
from artifactapi.storage import S3Storage
@pytest.fixture
def storage():
"""S3Storage with a mocked boto3 client."""
with patch("boto3.client", return_value=MagicMock()):
s = S3Storage(
endpoint="localhost:9000",
access_key="testkey",
secret_key="testsecret",
bucket="testbucket",
secure=False,
)
s.client = MagicMock()
return s
# ---------------------------------------------------------------------------
# get_object_key
# ---------------------------------------------------------------------------
class TestGetObjectKey:
def test_includes_remote_name_prefix(self, storage):
key = storage.get_object_key("myremote", "some/path/file.rpm")
assert key.startswith("myremote/")
def test_ends_with_filename(self, storage):
key = storage.get_object_key("myremote", "some/path/file.rpm")
assert key.endswith("/file.rpm")
def test_same_path_produces_same_key(self, storage):
k1 = storage.get_object_key("myremote", "some/path/file.rpm")
k2 = storage.get_object_key("myremote", "some/path/file.rpm")
assert k1 == k2
def test_different_remotes_give_different_keys(self, storage):
k1 = storage.get_object_key("remote-a", "path/to/file.rpm")
k2 = storage.get_object_key("remote-b", "path/to/file.rpm")
assert k1 != k2
def test_different_directories_give_different_keys(self, storage):
k1 = storage.get_object_key("myremote", "path/version-1/file.rpm")
k2 = storage.get_object_key("myremote", "path/version-2/file.rpm")
assert k1 != k2
# Same filename, different directory hashes
assert k1.split("/")[-1] == k2.split("/")[-1] == "file.rpm"
def test_leading_slash_stripped(self, storage):
k1 = storage.get_object_key("myremote", "/path/to/file.rpm")
k2 = storage.get_object_key("myremote", "path/to/file.rpm")
assert k1 == k2
def test_file_with_no_directory(self, storage):
key = storage.get_object_key("myremote", "file.rpm")
assert key == "myremote/file.rpm"
def test_docker_blob_uses_digest_path(self, storage):
digest = "abc123def456" * 4
path = f"library/nginx/blobs/sha256:{digest}"
key = storage.get_object_key("dockerhub", path)
assert key == f"dockerhub/blobs/sha256/{digest}"
def test_docker_blob_deduplication_across_images(self, storage):
"""Same blob digest pulled from different images maps to the same S3 key."""
digest = "deadbeef" * 8
k1 = storage.get_object_key("dockerhub", f"library/nginx/blobs/sha256:{digest}")
k2 = storage.get_object_key("dockerhub", f"library/ubuntu/blobs/sha256:{digest}")
assert k1 == k2
def test_docker_blob_different_digests_different_keys(self, storage):
k1 = storage.get_object_key("dockerhub", "library/nginx/blobs/sha256:aaa111")
k2 = storage.get_object_key("dockerhub", "library/nginx/blobs/sha256:bbb222")
assert k1 != k2
def test_docker_blob_different_remotes_different_keys(self, storage):
digest = "abc" * 20
k1 = storage.get_object_key("remote-a", f"library/nginx/blobs/sha256:{digest}")
k2 = storage.get_object_key("remote-b", f"library/nginx/blobs/sha256:{digest}")
assert k1 != k2
# ---------------------------------------------------------------------------
# get_url
# ---------------------------------------------------------------------------
class TestGetUrl:
def test_returns_http_url_for_insecure_endpoint(self, storage):
url = storage.get_url("myremote/abc123/file.rpm")
assert url == "http://localhost:9000/testbucket/myremote/abc123/file.rpm"
def test_url_contains_bucket_and_key(self, storage):
key = "myremote/abc/file.tar.gz"
url = storage.get_url(key)
assert "testbucket" in url
assert key in url
+8
View File
@@ -0,0 +1,8 @@
[tox]
envlist = py311
isolated_build = true
[testenv]
extras = dev
commands =
pytest {posargs:tests}