Compare commits
11 Commits
v2.3.0
..
b8bc7f8714
| Author | SHA1 | Date | |
|---|---|---|---|
| b8bc7f8714 | |||
| 0c780c1bd1 | |||
| 173b5d8b10 | |||
| 3352a3e886 | |||
| 8adcbac405 | |||
| 4ca89b9159 | |||
| 25b85ddc92 | |||
| d585ab425c | |||
| 6b1a6c9eb4 | |||
| 5de912db75 | |||
| 8e9d313892 |
@@ -194,6 +194,68 @@ remotes:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 300
|
||||
|
||||
pypi:
|
||||
base_url: "https://files.pythonhosted.org"
|
||||
type: "remote"
|
||||
package: "pypi"
|
||||
description: "Python Package Index — simple index and package files via a single remote"
|
||||
# simple/ requests are transparently fetched from pypi.org; package files come from
|
||||
# files.pythonhosted.org (base_url). URLs in the simple index are rewritten to this remote.
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "packages/.*\\.whl$"
|
||||
- "packages/.*\\.whl\\.metadata$"
|
||||
- "packages/.*\\.tar\\.gz$"
|
||||
- "packages/.*\\.zip$"
|
||||
- "packages/.*\\.egg$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600 # Simple index pages refreshed after 10 minutes
|
||||
|
||||
pypi-gitea:
|
||||
base_url: "https://gitea.example.com/api/packages/myorg/pypi"
|
||||
type: "remote"
|
||||
package: "pypi"
|
||||
description: "Private Gitea PyPI registry — simple index and files at the same host"
|
||||
# username: "your-gitea-username"
|
||||
# password: "your-personal-access-token" # needs package:read scope
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "files/.*\\.whl$"
|
||||
- "files/.*\\.whl\\.metadata$"
|
||||
- "files/.*\\.tar\\.gz$"
|
||||
- "files/.*\\.zip$"
|
||||
- "files/.*\\.egg$"
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600
|
||||
|
||||
npm:
|
||||
base_url: "https://registry.npmjs.org"
|
||||
type: "remote"
|
||||
package: "npm"
|
||||
description: "npm registry — package metadata with tarball URL rewriting"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- \.tgz$
|
||||
mutable_patterns:
|
||||
- ^(?!.*\.tgz$).*
|
||||
cache:
|
||||
immutable_ttl: 0
|
||||
mutable_ttl: 600 # Package metadata refreshed after 10 minutes
|
||||
|
||||
hashicorp-helm:
|
||||
base_url: "https://helm.releases.hashicorp.com"
|
||||
type: "remote"
|
||||
package: "helm"
|
||||
description: "HashiCorp Helm chart repository (Vault, Consul, Nomad, etc.)"
|
||||
check_mutable_updates: true
|
||||
immutable_patterns:
|
||||
- "\\.tgz$"
|
||||
cache:
|
||||
immutable_ttl: 0 # Chart tarballs are versioned — cache forever
|
||||
mutable_ttl: 3600 # index.yaml refreshed after 1 hour
|
||||
|
||||
local-generic:
|
||||
type: "local"
|
||||
package: "generic"
|
||||
|
||||
@@ -18,6 +18,13 @@ _PACKAGE_MUTABLE_PATTERNS: dict[str, list[str]] = {
|
||||
r"/manifests/(?!sha256:)[^/]+$",
|
||||
r"/tags/list$",
|
||||
],
|
||||
"pypi": [
|
||||
r"simple/", # Per-package and top-level simple index pages
|
||||
],
|
||||
"npm": [],
|
||||
"helm": [
|
||||
r"index\.yaml$",
|
||||
],
|
||||
"generic": [],
|
||||
}
|
||||
|
||||
|
||||
+89
-48
@@ -1,3 +1,4 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
@@ -159,12 +160,13 @@ async def construct_remote_url(remote_name: str, path: str) -> str:
|
||||
if not base_url:
|
||||
raise HTTPException(status_code=500, detail=f"No base_url configured for remote '{remote_name}'")
|
||||
|
||||
# Handle Docker registry URLs
|
||||
if remote_config.get("package") == "docker":
|
||||
# Convert Docker paths to v2 API format
|
||||
# e.g., library/nginx/manifests/latest -> v2/library/nginx/manifests/latest
|
||||
return f"{base_url}/v2/{path}"
|
||||
|
||||
# PyPI splits index and files across two hosts; redirect simple/ requests to pypi.org
|
||||
if remote_config.get("package") == "pypi" and base_url.rstrip("/") == "https://files.pythonhosted.org" and "simple/" in path:
|
||||
return f"https://pypi.org/{path}"
|
||||
|
||||
return f"{base_url}/{path}"
|
||||
|
||||
|
||||
@@ -208,8 +210,11 @@ async def cache_single_artifact(url: str, remote_name: str, path: str) -> dict:
|
||||
remote_config = config.get_remote_config(remote_name) or {}
|
||||
is_docker = remote_config.get("package") == "docker" or "/v2/" in url
|
||||
|
||||
# Prepare headers for Docker registry requests
|
||||
# Prepare headers
|
||||
headers = {}
|
||||
username = remote_config.get("username")
|
||||
password = remote_config.get("password")
|
||||
|
||||
if is_docker:
|
||||
if "/manifests/" in url:
|
||||
headers["Accept"] = (
|
||||
@@ -220,6 +225,8 @@ async def cache_single_artifact(url: str, remote_name: str, path: str) -> dict:
|
||||
)
|
||||
elif "/blobs/" in url:
|
||||
headers["Accept"] = "application/octet-stream"
|
||||
elif username and password:
|
||||
headers["Authorization"] = "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
@@ -254,11 +261,20 @@ async def cache_single_artifact(url: str, remote_name: str, path: str) -> dict:
|
||||
return {"url": url, "status": "error", "error": str(e)}
|
||||
|
||||
|
||||
async def _upstream_reachable(url: str) -> bool:
|
||||
def _basic_auth_header(remote_cfg: dict) -> dict[str, str]:
|
||||
username = remote_cfg.get("username")
|
||||
password = remote_cfg.get("password")
|
||||
if username and password:
|
||||
token = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
return {"Authorization": f"Basic {token}"}
|
||||
return {}
|
||||
|
||||
|
||||
async def _upstream_reachable(url: str, auth_headers: dict | None = None) -> bool:
|
||||
"""HEAD with a short timeout. Returns False only on network/timeout errors."""
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
await client.head(url, timeout=10.0)
|
||||
await client.head(url, headers=auth_headers or {}, timeout=10.0)
|
||||
return True
|
||||
except (httpx.NetworkError, httpx.TimeoutException):
|
||||
return False
|
||||
@@ -266,19 +282,19 @@ async def _upstream_reachable(url: str) -> bool:
|
||||
return True # 4xx/5xx means backend is up
|
||||
|
||||
|
||||
async def check_upstream_changed(remote_url: str, remote_name: str, path: str) -> bool:
|
||||
async def check_upstream_changed(remote_url: str, remote_name: str, path: str, auth_headers: dict | None = None) -> bool:
|
||||
"""Conditional HEAD against upstream. Returns False only on a definitive 304.
|
||||
Raises UpstreamUnreachable if the backend cannot be contacted."""
|
||||
meta = cache.get_mutable_meta(remote_name, path)
|
||||
if not meta:
|
||||
return True
|
||||
|
||||
headers = {}
|
||||
headers = dict(auth_headers or {})
|
||||
if meta.get("etag"):
|
||||
headers["If-None-Match"] = meta["etag"]
|
||||
if meta.get("last_modified"):
|
||||
headers["If-Modified-Since"] = meta["last_modified"]
|
||||
if not headers:
|
||||
if not (meta.get("etag") or meta.get("last_modified")):
|
||||
return True
|
||||
|
||||
try:
|
||||
@@ -294,12 +310,13 @@ async def handle_expired_mutable(remote_name: str, path: str, remote_url: str) -
|
||||
mutable_ttl = config.get_cache_config(remote_name).get("mutable_ttl", 3600)
|
||||
|
||||
remote_cfg = config.get_remote_config(remote_name) or {}
|
||||
auth = _basic_auth_header(remote_cfg)
|
||||
check_updates = remote_cfg.get("check_mutable_updates", False)
|
||||
user_mutable = check_updates and cache.is_mutable_file(path, config.get_user_mutable_patterns(remote_name))
|
||||
|
||||
if user_mutable:
|
||||
try:
|
||||
changed = await check_upstream_changed(remote_url, remote_name, path)
|
||||
changed = await check_upstream_changed(remote_url, remote_name, path, auth)
|
||||
except UpstreamUnreachable:
|
||||
cache.mark_index_cached(remote_name, path, mutable_ttl)
|
||||
logger.warning(f"Mutable STALE (backend unreachable): {remote_name}/{path} - TTL extended ({mutable_ttl}s)")
|
||||
@@ -310,7 +327,7 @@ async def handle_expired_mutable(remote_name: str, path: str, remote_url: str) -
|
||||
return True
|
||||
logger.info(f"Mutable file CHANGED: {remote_name}/{path} - re-downloading")
|
||||
else:
|
||||
if not await _upstream_reachable(remote_url):
|
||||
if not await _upstream_reachable(remote_url, auth):
|
||||
cache.mark_index_cached(remote_name, path, mutable_ttl)
|
||||
logger.warning(f"Mutable STALE (backend unreachable): {remote_name}/{path} - TTL extended ({mutable_ttl}s)")
|
||||
return True
|
||||
@@ -320,8 +337,66 @@ async def handle_expired_mutable(remote_name: str, path: str, remote_url: str) -
|
||||
return False
|
||||
|
||||
|
||||
def _get_content_type(filename: str) -> str:
|
||||
if filename.endswith((".tar.gz", ".tgz")):
|
||||
return "application/gzip"
|
||||
if filename.endswith(".zip") or filename.endswith(".whl"):
|
||||
return "application/zip"
|
||||
if filename.endswith(".exe"):
|
||||
return "application/x-msdownload"
|
||||
if filename.endswith(".rpm"):
|
||||
return "application/x-rpm"
|
||||
if filename.endswith(".xml"):
|
||||
return "application/xml"
|
||||
if filename.endswith((".xml.gz", ".xml.bz2", ".xml.xz")):
|
||||
return "application/gzip"
|
||||
if filename.endswith((".yaml", ".yml")):
|
||||
return "text/yaml"
|
||||
return "application/octet-stream"
|
||||
|
||||
|
||||
def _resolve_content(
|
||||
data: bytes,
|
||||
path: str,
|
||||
filename: str,
|
||||
remote_config: dict,
|
||||
request: Request,
|
||||
remote_name: str = "",
|
||||
) -> tuple[bytes, str]:
|
||||
"""Return (possibly-rewritten data, content_type) for a cached artifact."""
|
||||
if remote_config.get("package") == "pypi":
|
||||
immutable = remote_config.get("immutable_patterns", [])
|
||||
if not any(re.search(p, path) for p in immutable):
|
||||
proxy_base = str(request.base_url).rstrip("/")
|
||||
base_url = remote_config.get("base_url", "").rstrip("/")
|
||||
data = data.replace(
|
||||
base_url.encode(),
|
||||
f"{proxy_base}/api/v1/remote/{remote_name}".encode(),
|
||||
)
|
||||
return data, "text/html; charset=utf-8"
|
||||
if remote_config.get("package") == "npm":
|
||||
immutable = remote_config.get("immutable_patterns", [])
|
||||
if not any(re.search(p, path) for p in immutable):
|
||||
proxy_base = str(request.base_url).rstrip("/")
|
||||
base_url = remote_config.get("base_url", "").rstrip("/")
|
||||
data = data.replace(
|
||||
base_url.encode(),
|
||||
f"{proxy_base}/api/v1/remote/{remote_name}".encode(),
|
||||
)
|
||||
return data, "application/json"
|
||||
if remote_config.get("package") == "helm" and filename == "index.yaml":
|
||||
proxy_base = str(request.base_url).rstrip("/")
|
||||
base_url = remote_config.get("base_url", "").rstrip("/")
|
||||
data = data.replace(
|
||||
base_url.encode(),
|
||||
f"{proxy_base}/api/v1/remote/{remote_name}".encode(),
|
||||
)
|
||||
return data, "text/yaml"
|
||||
return data, _get_content_type(filename)
|
||||
|
||||
|
||||
@app.get("/api/v1/remote/{remote_name}/{path:path}")
|
||||
async def get_artifact(remote_name: str, path: str):
|
||||
async def get_artifact(request: Request, remote_name: str, path: str):
|
||||
# Check if remote is configured
|
||||
remote_config = config.get_remote_config(remote_name)
|
||||
if not remote_config:
|
||||
@@ -384,29 +459,11 @@ async def get_artifact(remote_name: str, path: str):
|
||||
try:
|
||||
artifact_data = storage.download_object(cached_key)
|
||||
filename = os.path.basename(path)
|
||||
artifact_data, content_type = _resolve_content(artifact_data, path, filename, remote_config, request, remote_name)
|
||||
|
||||
# Log cache hit
|
||||
logger.info(f"Cache HIT: {remote_name}/{path} (size: {len(artifact_data)} bytes, key: {cached_key})")
|
||||
|
||||
# Determine content type based on file extension
|
||||
content_type = "application/octet-stream"
|
||||
if filename.endswith(".tar.gz"):
|
||||
content_type = "application/gzip"
|
||||
elif filename.endswith(".zip"):
|
||||
content_type = "application/zip"
|
||||
elif filename.endswith(".exe"):
|
||||
content_type = "application/x-msdownload"
|
||||
elif filename.endswith(".rpm"):
|
||||
content_type = "application/x-rpm"
|
||||
elif filename.endswith(".xml"):
|
||||
content_type = "application/xml"
|
||||
elif filename.endswith((".xml.gz", ".xml.bz2", ".xml.xz")):
|
||||
content_type = "application/gzip"
|
||||
|
||||
# Record cache hit metrics
|
||||
metrics.record_cache_hit(remote_name, len(artifact_data))
|
||||
|
||||
# Record artifact mapping in database if not already recorded
|
||||
database.record_artifact_mapping(cached_key, remote_name, path, len(artifact_data))
|
||||
|
||||
return Response(
|
||||
@@ -443,25 +500,9 @@ async def get_artifact(remote_name: str, path: str):
|
||||
cache_key = storage.get_object_key(remote_name, path)
|
||||
artifact_data = storage.download_object(cache_key)
|
||||
filename = os.path.basename(path)
|
||||
artifact_data, content_type = _resolve_content(artifact_data, path, filename, remote_config, request, remote_name)
|
||||
|
||||
content_type = "application/octet-stream"
|
||||
if filename.endswith(".tar.gz"):
|
||||
content_type = "application/gzip"
|
||||
elif filename.endswith(".zip"):
|
||||
content_type = "application/zip"
|
||||
elif filename.endswith(".exe"):
|
||||
content_type = "application/x-msdownload"
|
||||
elif filename.endswith(".rpm"):
|
||||
content_type = "application/x-rpm"
|
||||
elif filename.endswith(".xml"):
|
||||
content_type = "application/xml"
|
||||
elif filename.endswith((".xml.gz", ".xml.bz2", ".xml.xz")):
|
||||
content_type = "application/gzip"
|
||||
|
||||
# Record cache miss metrics
|
||||
metrics.record_cache_miss(remote_name, len(artifact_data))
|
||||
|
||||
# Record artifact mapping in database
|
||||
cache_key = storage.get_object_key(remote_name, path)
|
||||
database.record_artifact_mapping(cache_key, remote_name, path, len(artifact_data))
|
||||
|
||||
|
||||
@@ -72,6 +72,32 @@ TEST_REMOTES = {
|
||||
"package": "generic",
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 0},
|
||||
},
|
||||
"pypi-test": {
|
||||
"base_url": "https://files.pythonhosted.org",
|
||||
"type": "remote",
|
||||
"package": "pypi",
|
||||
"immutable_patterns": [
|
||||
r"packages/.*\.whl$",
|
||||
r"packages/.*\.whl\.metadata$",
|
||||
r"packages/.*\.tar\.gz$",
|
||||
],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 600},
|
||||
},
|
||||
"npm-test": {
|
||||
"base_url": "https://registry.npmjs.org",
|
||||
"type": "remote",
|
||||
"package": "npm",
|
||||
"immutable_patterns": [r"\.tgz$"],
|
||||
"mutable_patterns": [r"^(?!.*\.tgz$).*"],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 600},
|
||||
},
|
||||
"helm-test": {
|
||||
"base_url": "https://helm.releases.hashicorp.com",
|
||||
"type": "remote",
|
||||
"package": "helm",
|
||||
"immutable_patterns": [r"\.tgz$"],
|
||||
"cache": {"immutable_ttl": 0, "mutable_ttl": 3600},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -133,6 +133,58 @@ class TestGetMutablePatterns:
|
||||
assert r"repomd\.xml$" in patterns
|
||||
assert r"custom-meta\.xml$" in patterns
|
||||
|
||||
def test_npm_has_no_package_defaults(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "package": "npm", "base_url": "https://x.com"}})
|
||||
assert cfg.get_mutable_patterns("r") == []
|
||||
|
||||
def test_npm_explicit_mutable_pattern_matches_metadata(self, make_config):
|
||||
import re
|
||||
|
||||
cfg = make_config(
|
||||
{
|
||||
"r": {
|
||||
"type": "remote",
|
||||
"package": "npm",
|
||||
"base_url": "https://x.com",
|
||||
"mutable_patterns": [r"^(?!.*\.tgz$).*"],
|
||||
}
|
||||
}
|
||||
)
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
assert any(re.search(p, "express") for p in patterns)
|
||||
assert any(re.search(p, "@babel/core") for p in patterns)
|
||||
|
||||
def test_helm_returns_index_yaml_as_mutable(self, make_config):
|
||||
cfg = make_config({"r": {"type": "remote", "package": "helm", "base_url": "https://helm.example.com"}})
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
assert r"index\.yaml$" in patterns
|
||||
|
||||
def test_helm_chart_tarballs_not_mutable_by_default(self, make_config):
|
||||
import re
|
||||
|
||||
cfg = make_config({"r": {"type": "remote", "package": "helm", "base_url": "https://helm.example.com"}})
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
# Only index.yaml is mutable; .tgz chart tarballs are not
|
||||
assert not any(re.search(p, "vault-0.29.1.tgz") for p in patterns)
|
||||
assert not any(re.search(p, "consul-1.5.0.tgz") for p in patterns)
|
||||
|
||||
def test_npm_explicit_mutable_pattern_excludes_tarballs(self, make_config):
|
||||
import re
|
||||
|
||||
cfg = make_config(
|
||||
{
|
||||
"r": {
|
||||
"type": "remote",
|
||||
"package": "npm",
|
||||
"base_url": "https://x.com",
|
||||
"mutable_patterns": [r"^(?!.*\.tgz$).*"],
|
||||
}
|
||||
}
|
||||
)
|
||||
patterns = cfg.get_mutable_patterns("r")
|
||||
assert not any(re.search(p, "express-4.18.2.tgz") for p in patterns)
|
||||
assert not any(re.search(p, "express/-/express-4.18.2.tgz") for p in patterns)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_immutable_patterns
|
||||
|
||||
@@ -652,3 +652,275 @@ class TestConfigEndpoint:
|
||||
data = response.json()
|
||||
assert "remotes" in data
|
||||
assert "alpine-test" in data["remotes"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PyPI remote /api/v1/remote/pypi-test/...
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPyPIRemote:
|
||||
def test_simple_index_is_mutable(self, client, patched_deps):
|
||||
"""simple/ paths are detected as mutable (package-type default)."""
|
||||
deps = patched_deps
|
||||
html = b"<html><body><a href='https://files.pythonhosted.org/packages/requests-2.31.0.tar.gz'>...</a></body></html>"
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = html
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/pypi-test/simple/requests/")
|
||||
assert response.status_code == 200
|
||||
deps["cache"].mark_index_cached.assert_not_called()
|
||||
|
||||
def test_simple_index_urls_rewritten_to_proxy(self, client, patched_deps):
|
||||
"""files.pythonhosted.org URLs in a cached simple index are rewritten to our proxy."""
|
||||
deps = patched_deps
|
||||
html = b"<html><body><a href='https://files.pythonhosted.org/packages/requests-2.31.0.tar.gz'>...</a></body></html>"
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = html
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/pypi-test/simple/requests/")
|
||||
assert response.status_code == 200
|
||||
assert b"files.pythonhosted.org" not in response.content
|
||||
assert b"/api/v1/remote/pypi-test/packages/requests-2.31.0.tar.gz" in response.content
|
||||
|
||||
def test_simple_index_content_type_is_html(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"<html></html>"
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/pypi-test/simple/requests/")
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers["content-type"]
|
||||
|
||||
def test_simple_index_cache_miss_fetches_upstream(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
html = b"<html><body><a href='https://files.pythonhosted.org/packages/p-1.0.whl'>...</a></body></html>"
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = html
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
new_callable=AsyncMock,
|
||||
return_value={"status": "cached"},
|
||||
) as mock_fetch:
|
||||
response = client.get("/api/v1/remote/pypi-test/simple/requests/")
|
||||
|
||||
mock_fetch.assert_called_once()
|
||||
assert response.status_code == 200
|
||||
assert b"files.pythonhosted.org" not in response.content
|
||||
|
||||
def test_wheel_file_immutable_returns_correct_content_type(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"PK wheel bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/pypi-test/packages/requests-2.31.0-py3-none-any.whl")
|
||||
assert response.status_code == 200
|
||||
assert "application/zip" in response.headers["content-type"]
|
||||
assert response.headers["X-Artifact-Source"] == "cache"
|
||||
|
||||
def test_sdist_immutable_returns_correct_content_type(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"tar bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/pypi-test/packages/requests-2.31.0.tar.gz")
|
||||
assert response.status_code == 200
|
||||
assert "application/gzip" in response.headers["content-type"]
|
||||
|
||||
def test_unknown_extension_on_pypi_remote_returns_403(self, client, patched_deps):
|
||||
"""Paths that don't match immutable_patterns and aren't mutable are blocked."""
|
||||
response = client.get("/api/v1/remote/pypi-test/packages/requests.unknown")
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# npm remote /api/v1/remote/npm-test/...
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNpmRemote:
|
||||
def test_package_metadata_is_mutable(self, client, patched_deps):
|
||||
"""Top-level package metadata paths are detected as mutable."""
|
||||
deps = patched_deps
|
||||
meta = b'{"name":"express","versions":{}}'
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = meta
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/express")
|
||||
assert response.status_code == 200
|
||||
deps["cache"].mark_index_cached.assert_not_called()
|
||||
|
||||
def test_metadata_tarball_urls_rewritten_to_proxy(self, client, patched_deps):
|
||||
"""registry.npmjs.org tarball URLs in metadata JSON are rewritten to our proxy."""
|
||||
deps = patched_deps
|
||||
meta = b'{"dist":{"tarball":"https://registry.npmjs.org/express/-/express-4.18.2.tgz"}}'
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = meta
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/express")
|
||||
assert response.status_code == 200
|
||||
assert b"registry.npmjs.org" not in response.content
|
||||
assert b"/api/v1/remote/npm-test/express/-/express-4.18.2.tgz" in response.content
|
||||
|
||||
def test_metadata_content_type_is_json(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b'{"name":"express"}'
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/express")
|
||||
assert response.status_code == 200
|
||||
assert "application/json" in response.headers["content-type"]
|
||||
|
||||
def test_scoped_package_metadata_rewritten(self, client, patched_deps):
|
||||
"""@scope/package metadata URLs are also rewritten back to the same npm-test remote."""
|
||||
deps = patched_deps
|
||||
meta = b'{"dist":{"tarball":"https://registry.npmjs.org/@babel/core/-/core-7.21.0.tgz"}}'
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = meta
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/@babel/core")
|
||||
assert response.status_code == 200
|
||||
assert b"registry.npmjs.org" not in response.content
|
||||
assert b"/api/v1/remote/npm-test/@babel/core/-/core-7.21.0.tgz" in response.content
|
||||
|
||||
def test_tarball_not_rewritten(self, client, patched_deps):
|
||||
"""Tarball requests (.tgz) bypass URL rewriting and return binary."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"\x1f\x8b tgz bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/express/-/express-4.18.2.tgz")
|
||||
assert response.status_code == 200
|
||||
assert "application/gzip" in response.headers["content-type"]
|
||||
assert response.headers["X-Artifact-Source"] == "cache"
|
||||
|
||||
def test_metadata_cache_miss_fetches_upstream(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
meta = b'{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"}}'
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = meta
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
new_callable=AsyncMock,
|
||||
return_value={"status": "cached"},
|
||||
) as mock_fetch:
|
||||
response = client.get("/api/v1/remote/npm-test/lodash")
|
||||
|
||||
mock_fetch.assert_called_once()
|
||||
assert response.status_code == 200
|
||||
assert b"registry.npmjs.org" not in response.content
|
||||
|
||||
def test_tarball_immutable_allowed_on_npm_remote(self, client, patched_deps):
|
||||
"""Tarballs (.tgz) match immutable_patterns and are served without rewriting."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"tgz bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/npm-test/express/-/express-4.18.2.tgz")
|
||||
assert response.status_code == 200
|
||||
assert "application/gzip" in response.headers["content-type"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helm remote /api/v1/remote/helm-test/...
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHelmRemote:
|
||||
def test_index_yaml_is_mutable(self, client, patched_deps):
|
||||
"""index.yaml is detected as mutable (package-type default)."""
|
||||
deps = patched_deps
|
||||
index = b"apiVersion: v1\nentries:\n vault:\n - urls:\n - https://helm.releases.hashicorp.com/vault-0.29.1.tgz\n"
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = index
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/helm-test/index.yaml")
|
||||
assert response.status_code == 200
|
||||
deps["cache"].mark_index_cached.assert_not_called()
|
||||
|
||||
def test_index_yaml_urls_rewritten_to_proxy(self, client, patched_deps):
|
||||
"""base_url chart URLs in a cached index.yaml are rewritten to our proxy."""
|
||||
deps = patched_deps
|
||||
index = b"apiVersion: v1\nentries:\n vault:\n - urls:\n - https://helm.releases.hashicorp.com/vault-0.29.1.tgz\n"
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = index
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/helm-test/index.yaml")
|
||||
assert response.status_code == 200
|
||||
assert b"helm.releases.hashicorp.com" not in response.content
|
||||
assert b"/api/v1/remote/helm-test/vault-0.29.1.tgz" in response.content
|
||||
|
||||
def test_index_yaml_content_type_is_yaml(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"apiVersion: v1\nentries: {}\n"
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
deps["cache"].is_index_valid.return_value = True
|
||||
|
||||
response = client.get("/api/v1/remote/helm-test/index.yaml")
|
||||
assert response.status_code == 200
|
||||
assert "text/yaml" in response.headers["content-type"]
|
||||
|
||||
def test_chart_tarball_immutable_returns_gzip_content_type(self, client, patched_deps):
|
||||
"""Versioned chart tarballs match immutable_patterns and are served as binary."""
|
||||
deps = patched_deps
|
||||
deps["storage"].exists.return_value = True
|
||||
deps["storage"].download_object.return_value = b"\x1f\x8b chart bytes"
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/helm-test/vault-0.29.1.tgz")
|
||||
assert response.status_code == 200
|
||||
assert "application/gzip" in response.headers["content-type"]
|
||||
assert response.headers["X-Artifact-Source"] == "cache"
|
||||
|
||||
def test_index_yaml_cache_miss_fetches_upstream(self, client, patched_deps):
|
||||
deps = patched_deps
|
||||
index = b"apiVersion: v1\nentries:\n vault:\n - urls:\n - https://helm.releases.hashicorp.com/vault-0.29.1.tgz\n"
|
||||
deps["storage"].exists.return_value = False
|
||||
deps["storage"].download_object.return_value = index
|
||||
deps["cache"].is_mutable_file.return_value = True
|
||||
|
||||
with patch(
|
||||
"artifactapi.main.cache_single_artifact",
|
||||
new_callable=AsyncMock,
|
||||
return_value={"status": "cached"},
|
||||
) as mock_fetch:
|
||||
response = client.get("/api/v1/remote/helm-test/index.yaml")
|
||||
|
||||
mock_fetch.assert_called_once()
|
||||
assert response.status_code == 200
|
||||
assert b"helm.releases.hashicorp.com" not in response.content
|
||||
|
||||
def test_non_tgz_non_yaml_path_blocked_by_pattern(self, client, patched_deps):
|
||||
"""Paths that don't match immutable_patterns and aren't mutable are blocked."""
|
||||
deps = patched_deps
|
||||
deps["cache"].is_mutable_file.return_value = False
|
||||
|
||||
response = client.get("/api/v1/remote/helm-test/vault.zip")
|
||||
assert response.status_code == 403
|
||||
|
||||
Reference in New Issue
Block a user