Merge pull request 'chore: cleanup the readme' (#23) from benvin/readme-refactor into master

Reviewed-on: #23

remotes.yaml

@@ -195,15 +195,19 @@ remotes:
       mutable_ttl: 300
 
   pypi:
-    base_url: "https://pypi.org"
+    base_url: "https://files.pythonhosted.org"
     type: "remote"
     package: "pypi"
-    description: "Python Package Index — simple repository API"
-    # pypi_files_url: the upstream host used in simple-index hrefs (default: files.pythonhosted.org)
-    # pypi_files_remote: our proxy remote that will serve those files (default: pypi-files)
-    pypi_files_url: "https://files.pythonhosted.org"
-    pypi_files_remote: "pypi-files"
+    description: "Python Package Index — simple index and package files via a single remote"
+    # simple/ requests are transparently fetched from pypi.org; package files come from
+    # files.pythonhosted.org (base_url). URLs in the simple index are rewritten to this remote.
     check_mutable_updates: true
+    immutable_patterns:
+      - "packages/.*\\.whl$"
+      - "packages/.*\\.whl\\.metadata$"
+      - "packages/.*\\.tar\\.gz$"
+      - "packages/.*\\.zip$"
+      - "packages/.*\\.egg$"
     cache:
       immutable_ttl: 0
       mutable_ttl: 600  # Simple index pages refreshed after 10 minutes
@@ -212,12 +216,9 @@ remotes:
     base_url: "https://gitea.example.com/api/packages/myorg/pypi"
     type: "remote"
     package: "pypi"
-    description: "Private Gitea PyPI registry"
+    description: "Private Gitea PyPI registry — simple index and files at the same host"
     # username: "your-gitea-username"
     # password: "your-personal-access-token"  # needs package:read scope
-    # Files are served from the same Gitea instance — rewrite back to this same remote
-    pypi_files_url: "https://gitea.example.com/api/packages/myorg/pypi"
-    pypi_files_remote: "pypi-gitea"
     check_mutable_updates: true
     immutable_patterns:
       - "files/.*\\.whl$"
@@ -229,29 +230,11 @@ remotes:
       immutable_ttl: 0
       mutable_ttl: 600
 
-  pypi-files:
-    base_url: "https://files.pythonhosted.org"
-    type: "remote"
-    package: "generic"
-    description: "Python Package Index — file storage (wheels, sdists)"
-    immutable_patterns:
-      - "packages/.*\\.whl$"
-      - "packages/.*\\.whl\\.metadata$"
-      - "packages/.*\\.tar\\.gz$"
-      - "packages/.*\\.zip$"
-      - "packages/.*\\.egg$"
-    cache:
-      immutable_ttl: 0  # Package files are content-addressed — cache forever
-
   npm:
     base_url: "https://registry.npmjs.org"
     type: "remote"
     package: "npm"
     description: "npm registry — package metadata with tarball URL rewriting"
-    # npm_files_url: the upstream host used in metadata tarball hrefs (default: https://registry.npmjs.org)
-    # npm_files_remote: our proxy remote that will serve those tarballs (default: npm-files)
-    npm_files_url: "https://registry.npmjs.org"
-    npm_files_remote: "npm"
     check_mutable_updates: true
     immutable_patterns:
       - \.tgz$

src/artifactapi/main.py

@@ -160,12 +160,13 @@ async def construct_remote_url(remote_name: str, path: str) -> str:
     if not base_url:
         raise HTTPException(status_code=500, detail=f"No base_url configured for remote '{remote_name}'")
 
-    # Handle Docker registry URLs
     if remote_config.get("package") == "docker":
-        # Convert Docker paths to v2 API format
-        # e.g., library/nginx/manifests/latest -> v2/library/nginx/manifests/latest
         return f"{base_url}/v2/{path}"
 
+    # PyPI splits index and files across two hosts; redirect simple/ requests to pypi.org
+    if remote_config.get("package") == "pypi" and base_url.rstrip("/") == "https://files.pythonhosted.org" and "simple/" in path:
+        return f"https://pypi.org/{path}"
+
     return f"{base_url}/{path}"
 
 
@@ -363,24 +364,26 @@ def _resolve_content(
     remote_name: str = "",
 ) -> tuple[bytes, str]:
     """Return (possibly-rewritten data, content_type) for a cached artifact."""
-    if remote_config.get("package") == "pypi" and "simple/" in path:
-        files_url = remote_config.get("pypi_files_url", "https://files.pythonhosted.org")
-        files_remote = remote_config.get("pypi_files_remote", "pypi-files")
-        proxy_base = str(request.base_url).rstrip("/")
-        data = data.replace(
-            files_url.rstrip("/").encode(),
-            f"{proxy_base}/api/v1/remote/{files_remote}".encode(),
-        )
-        return data, "text/html; charset=utf-8"
-    if remote_config.get("package") == "npm" and not path.endswith(".tgz"):
-        files_url = remote_config.get("npm_files_url", "https://registry.npmjs.org")
-        files_remote = remote_config.get("npm_files_remote", "npm-files")
-        proxy_base = str(request.base_url).rstrip("/")
-        data = data.replace(
-            files_url.rstrip("/").encode(),
-            f"{proxy_base}/api/v1/remote/{files_remote}".encode(),
-        )
-        return data, "application/json"
+    if remote_config.get("package") == "pypi":
+        immutable = remote_config.get("immutable_patterns", [])
+        if not any(re.search(p, path) for p in immutable):
+            proxy_base = str(request.base_url).rstrip("/")
+            base_url = remote_config.get("base_url", "").rstrip("/")
+            data = data.replace(
+                base_url.encode(),
+                f"{proxy_base}/api/v1/remote/{remote_name}".encode(),
+            )
+            return data, "text/html; charset=utf-8"
+    if remote_config.get("package") == "npm":
+        immutable = remote_config.get("immutable_patterns", [])
+        if not any(re.search(p, path) for p in immutable):
+            proxy_base = str(request.base_url).rstrip("/")
+            base_url = remote_config.get("base_url", "").rstrip("/")
+            data = data.replace(
+                base_url.encode(),
+                f"{proxy_base}/api/v1/remote/{remote_name}".encode(),
+            )
+            return data, "application/json"
     if remote_config.get("package") == "helm" and filename == "index.yaml":
         proxy_base = str(request.base_url).rstrip("/")
         base_url = remote_config.get("base_url", "").rstrip("/")

tests/conftest.py

@@ -73,30 +73,20 @@ TEST_REMOTES = {
             "cache": {"immutable_ttl": 0, "mutable_ttl": 0},
         },
         "pypi-test": {
-            "base_url": "https://pypi.org",
-            "type": "remote",
-            "package": "pypi",
-            "pypi_files_url": "https://files.pythonhosted.org",
-            "pypi_files_remote": "pypi-files-test",
-            "cache": {"immutable_ttl": 0, "mutable_ttl": 600},
-        },
-        "pypi-files-test": {
             "base_url": "https://files.pythonhosted.org",
             "type": "remote",
-            "package": "generic",
+            "package": "pypi",
             "immutable_patterns": [
-                "packages/.*\\.whl$",
-                "packages/.*\\.whl\\.metadata$",
-                "packages/.*\\.tar\\.gz$",
+                r"packages/.*\.whl$",
+                r"packages/.*\.whl\.metadata$",
+                r"packages/.*\.tar\.gz$",
             ],
-            "cache": {"immutable_ttl": 0, "mutable_ttl": 0},
+            "cache": {"immutable_ttl": 0, "mutable_ttl": 600},
         },
         "npm-test": {
             "base_url": "https://registry.npmjs.org",
             "type": "remote",
             "package": "npm",
-            "npm_files_url": "https://registry.npmjs.org",
-            "npm_files_remote": "npm-test",
             "immutable_patterns": [r"\.tgz$"],
             "mutable_patterns": [r"^(?!.*\.tgz$).*"],
             "cache": {"immutable_ttl": 0, "mutable_ttl": 600},

tests/test_routes.py

@@ -685,7 +685,7 @@ class TestPyPIRemote:
         response = client.get("/api/v1/remote/pypi-test/simple/requests/")
         assert response.status_code == 200
         assert b"files.pythonhosted.org" not in response.content
-        assert b"/api/v1/remote/pypi-files-test/packages/requests-2.31.0.tar.gz" in response.content
+        assert b"/api/v1/remote/pypi-test/packages/requests-2.31.0.tar.gz" in response.content
 
     def test_simple_index_content_type_is_html(self, client, patched_deps):
         deps = patched_deps
@@ -722,7 +722,7 @@ class TestPyPIRemote:
         deps["storage"].download_object.return_value = b"PK wheel bytes"
         deps["cache"].is_mutable_file.return_value = False
 
-        response = client.get("/api/v1/remote/pypi-files-test/packages/requests-2.31.0-py3-none-any.whl")
+        response = client.get("/api/v1/remote/pypi-test/packages/requests-2.31.0-py3-none-any.whl")
         assert response.status_code == 200
         assert "application/zip" in response.headers["content-type"]
         assert response.headers["X-Artifact-Source"] == "cache"
@@ -733,13 +733,13 @@ class TestPyPIRemote:
         deps["storage"].download_object.return_value = b"tar bytes"
         deps["cache"].is_mutable_file.return_value = False
 
-        response = client.get("/api/v1/remote/pypi-files-test/packages/requests-2.31.0.tar.gz")
+        response = client.get("/api/v1/remote/pypi-test/packages/requests-2.31.0.tar.gz")
         assert response.status_code == 200
         assert "application/gzip" in response.headers["content-type"]
 
-    def test_blocked_path_on_files_remote_returns_403(self, client, patched_deps):
-        """Paths that don't match immutable_patterns on pypi-files-test are blocked."""
-        response = client.get("/api/v1/remote/pypi-files-test/packages/requests.unknown")
+    def test_unknown_extension_on_pypi_remote_returns_403(self, client, patched_deps):
+        """Paths that don't match immutable_patterns and aren't mutable are blocked."""
+        response = client.get("/api/v1/remote/pypi-test/packages/requests.unknown")
         assert response.status_code == 403