Merge pull request 'chore: cleanup the readme' (#23) from benvin/readme-refactor into master

Reviewed-on: #23

remotes.yaml

@@ -195,15 +195,19 @@ remotes:
       mutable_ttl: 300
 
   pypi:
-    base_url: "https://pypi.org"
+    base_url: "https://files.pythonhosted.org"
     type: "remote"
     package: "pypi"
-    description: "Python Package Index — simple repository API"
-    # pypi_files_url: the upstream host used in simple-index hrefs (default: files.pythonhosted.org)
-    # pypi_files_remote: our proxy remote that will serve those files (default: pypi-files)
-    pypi_files_url: "https://files.pythonhosted.org"
-    pypi_files_remote: "pypi-files"
+    description: "Python Package Index — simple index and package files via a single remote"
+    # simple/ requests are transparently fetched from pypi.org; package files come from
+    # files.pythonhosted.org (base_url). URLs in the simple index are rewritten to this remote.
     check_mutable_updates: true
+    immutable_patterns:
+      - "packages/.*\\.whl$"
+      - "packages/.*\\.whl\\.metadata$"
+      - "packages/.*\\.tar\\.gz$"
+      - "packages/.*\\.zip$"
+      - "packages/.*\\.egg$"
     cache:
       immutable_ttl: 0
       mutable_ttl: 600  # Simple index pages refreshed after 10 minutes
@@ -212,12 +216,9 @@ remotes:
     base_url: "https://gitea.example.com/api/packages/myorg/pypi"
     type: "remote"
     package: "pypi"
-    description: "Private Gitea PyPI registry"
+    description: "Private Gitea PyPI registry — simple index and files at the same host"
     # username: "your-gitea-username"
     # password: "your-personal-access-token"  # needs package:read scope
-    # Files are served from the same Gitea instance — rewrite back to this same remote
-    pypi_files_url: "https://gitea.example.com/api/packages/myorg/pypi"
-    pypi_files_remote: "pypi-gitea"
     check_mutable_updates: true
     immutable_patterns:
       - "files/.*\\.whl$"
@@ -229,29 +230,11 @@ remotes:
       immutable_ttl: 0
       mutable_ttl: 600
 
-  pypi-files:
-    base_url: "https://files.pythonhosted.org"
-    type: "remote"
-    package: "generic"
-    description: "Python Package Index — file storage (wheels, sdists)"
-    immutable_patterns:
-      - "packages/.*\\.whl$"
-      - "packages/.*\\.whl\\.metadata$"
-      - "packages/.*\\.tar\\.gz$"
-      - "packages/.*\\.zip$"
-      - "packages/.*\\.egg$"
-    cache:
-      immutable_ttl: 0  # Package files are content-addressed — cache forever
-
   npm:
     base_url: "https://registry.npmjs.org"
     type: "remote"
     package: "npm"
     description: "npm registry — package metadata with tarball URL rewriting"
-    # npm_files_url: the upstream host used in metadata tarball hrefs (default: https://registry.npmjs.org)
-    # npm_files_remote: our proxy remote that will serve those tarballs (default: npm-files)
-    npm_files_url: "https://registry.npmjs.org"
-    npm_files_remote: "npm"
     check_mutable_updates: true
     immutable_patterns:
       - \.tgz$
@@ -261,6 +244,18 @@ remotes:
       immutable_ttl: 0
       mutable_ttl: 600  # Package metadata refreshed after 10 minutes
 
+  hashicorp-helm:
+    base_url: "https://helm.releases.hashicorp.com"
+    type: "remote"
+    package: "helm"
+    description: "HashiCorp Helm chart repository (Vault, Consul, Nomad, etc.)"
+    check_mutable_updates: true
+    immutable_patterns:
+      - "\\.tgz$"
+    cache:
+      immutable_ttl: 0    # Chart tarballs are versioned — cache forever
+      mutable_ttl: 3600   # index.yaml refreshed after 1 hour
+
   local-generic:
     type: "local"
     package: "generic"

src/artifactapi/config.py

@@ -22,6 +22,9 @@ _PACKAGE_MUTABLE_PATTERNS: dict[str, list[str]] = {
         r"simple/",  # Per-package and top-level simple index pages
     ],
     "npm": [],
+    "helm": [
+        r"index\.yaml$",
+    ],
     "generic": [],
 }
 

src/artifactapi/main.py

@@ -160,12 +160,13 @@ async def construct_remote_url(remote_name: str, path: str) -> str:
     if not base_url:
         raise HTTPException(status_code=500, detail=f"No base_url configured for remote '{remote_name}'")
 
-    # Handle Docker registry URLs
     if remote_config.get("package") == "docker":
-        # Convert Docker paths to v2 API format
-        # e.g., library/nginx/manifests/latest -> v2/library/nginx/manifests/latest
         return f"{base_url}/v2/{path}"
 
+    # PyPI splits index and files across two hosts; redirect simple/ requests to pypi.org
+    if remote_config.get("package") == "pypi" and base_url.rstrip("/") == "https://files.pythonhosted.org" and "simple/" in path:
+        return f"https://pypi.org/{path}"
+
     return f"{base_url}/{path}"
 
 
@@ -349,6 +350,8 @@ def _get_content_type(filename: str) -> str:
         return "application/xml"
     if filename.endswith((".xml.gz", ".xml.bz2", ".xml.xz")):
         return "application/gzip"
+    if filename.endswith((".yaml", ".yml")):
+        return "text/yaml"
     return "application/octet-stream"
 
 
@@ -358,26 +361,37 @@ def _resolve_content(
     filename: str,
     remote_config: dict,
     request: Request,
+    remote_name: str = "",
 ) -> tuple[bytes, str]:
     """Return (possibly-rewritten data, content_type) for a cached artifact."""
-    if remote_config.get("package") == "pypi" and "simple/" in path:
-        files_url = remote_config.get("pypi_files_url", "https://files.pythonhosted.org")
-        files_remote = remote_config.get("pypi_files_remote", "pypi-files")
+    if remote_config.get("package") == "pypi":
+        immutable = remote_config.get("immutable_patterns", [])
+        if not any(re.search(p, path) for p in immutable):
+            proxy_base = str(request.base_url).rstrip("/")
+            base_url = remote_config.get("base_url", "").rstrip("/")
+            data = data.replace(
+                base_url.encode(),
+                f"{proxy_base}/api/v1/remote/{remote_name}".encode(),
+            )
+            return data, "text/html; charset=utf-8"
+    if remote_config.get("package") == "npm":
+        immutable = remote_config.get("immutable_patterns", [])
+        if not any(re.search(p, path) for p in immutable):
+            proxy_base = str(request.base_url).rstrip("/")
+            base_url = remote_config.get("base_url", "").rstrip("/")
+            data = data.replace(
+                base_url.encode(),
+                f"{proxy_base}/api/v1/remote/{remote_name}".encode(),
+            )
+            return data, "application/json"
+    if remote_config.get("package") == "helm" and filename == "index.yaml":
         proxy_base = str(request.base_url).rstrip("/")
+        base_url = remote_config.get("base_url", "").rstrip("/")
         data = data.replace(
-            files_url.rstrip("/").encode(),
-            f"{proxy_base}/api/v1/remote/{files_remote}".encode(),
+            base_url.encode(),
+            f"{proxy_base}/api/v1/remote/{remote_name}".encode(),
         )
-        return data, "text/html; charset=utf-8"
-    if remote_config.get("package") == "npm" and not path.endswith(".tgz"):
-        files_url = remote_config.get("npm_files_url", "https://registry.npmjs.org")
-        files_remote = remote_config.get("npm_files_remote", "npm-files")
-        proxy_base = str(request.base_url).rstrip("/")
-        data = data.replace(
-            files_url.rstrip("/").encode(),
-            f"{proxy_base}/api/v1/remote/{files_remote}".encode(),
-        )
-        return data, "application/json"
+        return data, "text/yaml"
     return data, _get_content_type(filename)
 
 
@@ -445,7 +459,7 @@ async def get_artifact(request: Request, remote_name: str, path: str):
         try:
             artifact_data = storage.download_object(cached_key)
             filename = os.path.basename(path)
-            artifact_data, content_type = _resolve_content(artifact_data, path, filename, remote_config, request)
+            artifact_data, content_type = _resolve_content(artifact_data, path, filename, remote_config, request, remote_name)
 
             logger.info(f"Cache HIT: {remote_name}/{path} (size: {len(artifact_data)} bytes, key: {cached_key})")
 
@@ -486,7 +500,7 @@ async def get_artifact(request: Request, remote_name: str, path: str):
         cache_key = storage.get_object_key(remote_name, path)
         artifact_data = storage.download_object(cache_key)
         filename = os.path.basename(path)
-        artifact_data, content_type = _resolve_content(artifact_data, path, filename, remote_config, request)
+        artifact_data, content_type = _resolve_content(artifact_data, path, filename, remote_config, request, remote_name)
 
         metrics.record_cache_miss(remote_name, len(artifact_data))
         cache_key = storage.get_object_key(remote_name, path)

tests/conftest.py

@@ -73,34 +73,31 @@ TEST_REMOTES = {
             "cache": {"immutable_ttl": 0, "mutable_ttl": 0},
         },
         "pypi-test": {
-            "base_url": "https://pypi.org",
-            "type": "remote",
-            "package": "pypi",
-            "pypi_files_url": "https://files.pythonhosted.org",
-            "pypi_files_remote": "pypi-files-test",
-            "cache": {"immutable_ttl": 0, "mutable_ttl": 600},
-        },
-        "pypi-files-test": {
             "base_url": "https://files.pythonhosted.org",
             "type": "remote",
-            "package": "generic",
+            "package": "pypi",
             "immutable_patterns": [
-                "packages/.*\\.whl$",
-                "packages/.*\\.whl\\.metadata$",
-                "packages/.*\\.tar\\.gz$",
+                r"packages/.*\.whl$",
+                r"packages/.*\.whl\.metadata$",
+                r"packages/.*\.tar\.gz$",
             ],
-            "cache": {"immutable_ttl": 0, "mutable_ttl": 0},
+            "cache": {"immutable_ttl": 0, "mutable_ttl": 600},
         },
         "npm-test": {
             "base_url": "https://registry.npmjs.org",
             "type": "remote",
             "package": "npm",
-            "npm_files_url": "https://registry.npmjs.org",
-            "npm_files_remote": "npm-test",
             "immutable_patterns": [r"\.tgz$"],
             "mutable_patterns": [r"^(?!.*\.tgz$).*"],
             "cache": {"immutable_ttl": 0, "mutable_ttl": 600},
         },
+        "helm-test": {
+            "base_url": "https://helm.releases.hashicorp.com",
+            "type": "remote",
+            "package": "helm",
+            "immutable_patterns": [r"\.tgz$"],
+            "cache": {"immutable_ttl": 0, "mutable_ttl": 3600},
+        },
     }
 }
 

tests/test_config.py

@@ -154,6 +154,20 @@ class TestGetMutablePatterns:
         assert any(re.search(p, "express") for p in patterns)
         assert any(re.search(p, "@babel/core") for p in patterns)
 
+    def test_helm_returns_index_yaml_as_mutable(self, make_config):
+        cfg = make_config({"r": {"type": "remote", "package": "helm", "base_url": "https://helm.example.com"}})
+        patterns = cfg.get_mutable_patterns("r")
+        assert r"index\.yaml$" in patterns
+
+    def test_helm_chart_tarballs_not_mutable_by_default(self, make_config):
+        import re
+
+        cfg = make_config({"r": {"type": "remote", "package": "helm", "base_url": "https://helm.example.com"}})
+        patterns = cfg.get_mutable_patterns("r")
+        # Only index.yaml is mutable; .tgz chart tarballs are not
+        assert not any(re.search(p, "vault-0.29.1.tgz") for p in patterns)
+        assert not any(re.search(p, "consul-1.5.0.tgz") for p in patterns)
+
     def test_npm_explicit_mutable_pattern_excludes_tarballs(self, make_config):
         import re
 

tests/test_routes.py

@@ -685,7 +685,7 @@ class TestPyPIRemote:
         response = client.get("/api/v1/remote/pypi-test/simple/requests/")
         assert response.status_code == 200
         assert b"files.pythonhosted.org" not in response.content
-        assert b"/api/v1/remote/pypi-files-test/packages/requests-2.31.0.tar.gz" in response.content
+        assert b"/api/v1/remote/pypi-test/packages/requests-2.31.0.tar.gz" in response.content
 
     def test_simple_index_content_type_is_html(self, client, patched_deps):
         deps = patched_deps
@@ -722,7 +722,7 @@ class TestPyPIRemote:
         deps["storage"].download_object.return_value = b"PK wheel bytes"
         deps["cache"].is_mutable_file.return_value = False
 
-        response = client.get("/api/v1/remote/pypi-files-test/packages/requests-2.31.0-py3-none-any.whl")
+        response = client.get("/api/v1/remote/pypi-test/packages/requests-2.31.0-py3-none-any.whl")
         assert response.status_code == 200
         assert "application/zip" in response.headers["content-type"]
         assert response.headers["X-Artifact-Source"] == "cache"
@@ -733,13 +733,13 @@ class TestPyPIRemote:
         deps["storage"].download_object.return_value = b"tar bytes"
         deps["cache"].is_mutable_file.return_value = False
 
-        response = client.get("/api/v1/remote/pypi-files-test/packages/requests-2.31.0.tar.gz")
+        response = client.get("/api/v1/remote/pypi-test/packages/requests-2.31.0.tar.gz")
         assert response.status_code == 200
         assert "application/gzip" in response.headers["content-type"]
 
-    def test_blocked_path_on_files_remote_returns_403(self, client, patched_deps):
-        """Paths that don't match immutable_patterns on pypi-files-test are blocked."""
-        response = client.get("/api/v1/remote/pypi-files-test/packages/requests.unknown")
+    def test_unknown_extension_on_pypi_remote_returns_403(self, client, patched_deps):
+        """Paths that don't match immutable_patterns and aren't mutable are blocked."""
+        response = client.get("/api/v1/remote/pypi-test/packages/requests.unknown")
         assert response.status_code == 403
 
 
@@ -841,3 +841,86 @@ class TestNpmRemote:
         response = client.get("/api/v1/remote/npm-test/express/-/express-4.18.2.tgz")
         assert response.status_code == 200
         assert "application/gzip" in response.headers["content-type"]
+
+
+# ---------------------------------------------------------------------------
+# Helm remote  /api/v1/remote/helm-test/...
+# ---------------------------------------------------------------------------
+
+
+class TestHelmRemote:
+    def test_index_yaml_is_mutable(self, client, patched_deps):
+        """index.yaml is detected as mutable (package-type default)."""
+        deps = patched_deps
+        index = b"apiVersion: v1\nentries:\n  vault:\n    - urls:\n      - https://helm.releases.hashicorp.com/vault-0.29.1.tgz\n"
+        deps["storage"].exists.return_value = True
+        deps["storage"].download_object.return_value = index
+        deps["cache"].is_mutable_file.return_value = True
+        deps["cache"].is_index_valid.return_value = True
+
+        response = client.get("/api/v1/remote/helm-test/index.yaml")
+        assert response.status_code == 200
+        deps["cache"].mark_index_cached.assert_not_called()
+
+    def test_index_yaml_urls_rewritten_to_proxy(self, client, patched_deps):
+        """base_url chart URLs in a cached index.yaml are rewritten to our proxy."""
+        deps = patched_deps
+        index = b"apiVersion: v1\nentries:\n  vault:\n    - urls:\n      - https://helm.releases.hashicorp.com/vault-0.29.1.tgz\n"
+        deps["storage"].exists.return_value = True
+        deps["storage"].download_object.return_value = index
+        deps["cache"].is_mutable_file.return_value = True
+        deps["cache"].is_index_valid.return_value = True
+
+        response = client.get("/api/v1/remote/helm-test/index.yaml")
+        assert response.status_code == 200
+        assert b"helm.releases.hashicorp.com" not in response.content
+        assert b"/api/v1/remote/helm-test/vault-0.29.1.tgz" in response.content
+
+    def test_index_yaml_content_type_is_yaml(self, client, patched_deps):
+        deps = patched_deps
+        deps["storage"].exists.return_value = True
+        deps["storage"].download_object.return_value = b"apiVersion: v1\nentries: {}\n"
+        deps["cache"].is_mutable_file.return_value = True
+        deps["cache"].is_index_valid.return_value = True
+
+        response = client.get("/api/v1/remote/helm-test/index.yaml")
+        assert response.status_code == 200
+        assert "text/yaml" in response.headers["content-type"]
+
+    def test_chart_tarball_immutable_returns_gzip_content_type(self, client, patched_deps):
+        """Versioned chart tarballs match immutable_patterns and are served as binary."""
+        deps = patched_deps
+        deps["storage"].exists.return_value = True
+        deps["storage"].download_object.return_value = b"\x1f\x8b chart bytes"
+        deps["cache"].is_mutable_file.return_value = False
+
+        response = client.get("/api/v1/remote/helm-test/vault-0.29.1.tgz")
+        assert response.status_code == 200
+        assert "application/gzip" in response.headers["content-type"]
+        assert response.headers["X-Artifact-Source"] == "cache"
+
+    def test_index_yaml_cache_miss_fetches_upstream(self, client, patched_deps):
+        deps = patched_deps
+        index = b"apiVersion: v1\nentries:\n  vault:\n    - urls:\n      - https://helm.releases.hashicorp.com/vault-0.29.1.tgz\n"
+        deps["storage"].exists.return_value = False
+        deps["storage"].download_object.return_value = index
+        deps["cache"].is_mutable_file.return_value = True
+
+        with patch(
+            "artifactapi.main.cache_single_artifact",
+            new_callable=AsyncMock,
+            return_value={"status": "cached"},
+        ) as mock_fetch:
+            response = client.get("/api/v1/remote/helm-test/index.yaml")
+
+        mock_fetch.assert_called_once()
+        assert response.status_code == 200
+        assert b"helm.releases.hashicorp.com" not in response.content
+
+    def test_non_tgz_non_yaml_path_blocked_by_pattern(self, client, patched_deps):
+        """Paths that don't match immutable_patterns and aren't mutable are blocked."""
+        deps = patched_deps
+        deps["cache"].is_mutable_file.return_value = False
+
+        response = client.get("/api/v1/remote/helm-test/vault.zip")
+        assert response.status_code == 403