From 99cc71f56caa1169b1b977ab4b14eb16c9e5b773 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 6 Jun 2026 23:51:52 +1000 Subject: [PATCH] feat: add Terraform/OpenTofu registry remote type (#45) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - New `terraform` package type implementing the [Terraform Registry Protocol](https://developer.hashicorp.com/terraform/internals/provider-registry-protocol) - `construct_url` prepends `/v1/providers/` so paths like `hashicorp/vault/versions` map to `registry.terraform.io/v1/providers/hashicorp/vault/versions` - `resolve_content` rewrites `download_url`, `shasums_url`, and `shasums_signature_url` in per-version download info JSON to route through a companion `releases_remote` (generic remote proxying `releases.hashicorp.com`) - Built-in mutable pattern for `{namespace}/{type}/versions` — version lists expire and are re-fetched; per-version download info is immutable - Client configuration via `.terraformrc` / `.tofurc` host block — no changes to `.tf` provider source addresses needed ## Test plan - [x] 8 unit tests covering mutable detection, URL rewriting, binary pass-through, `construct_url` correctness, and cache miss behaviour - [x] End-to-end: OpenTofu 1.10.3 pulling `hashicorp/vault v4.5.0` through docker-compose stack — `tofu init` succeeded, provider installed and signed - [x] Verified `download_url` / `shasums_url` rewritten to `hashicorp-releases` proxy in cached response - [x] All 339 tests pass Reviewed-on: https://git.unkin.net/unkin/artifactapi/pulls/45 Co-authored-by: Ben Vincent Co-committed-by: Ben Vincent --- README.md | 54 ++++++++++- examples/single-file/remotes.yaml | 31 ++++++ src/artifactapi/artifact/proxy.py | 6 ++ src/artifactapi/config.py | 3 + src/artifactapi/remote/__init__.py | 4 +- src/artifactapi/remote/terraform.py | 36 +++++++ tests/conftest.py | 15 +++ tests/test_routes.py | 144 ++++++++++++++++++++++++++++ 8 files changed, 288 insertions(+), 5 deletions(-) create mode 100644 src/artifactapi/remote/terraform.py diff --git a/README.md b/README.md index 5e34870..a03f363 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ FastAPI caching proxy that downloads and stores files from remote sources in S3- ## Features -- Remote definitions via `remotes.yaml` — generic HTTP, Alpine APK, RPM, Docker, PyPI, npm, Helm, Puppet Forge +- Remote definitions via `remotes.yaml` — generic HTTP, Alpine APK, RPM, Docker, PyPI, npm, Helm, Puppet Forge, Terraform/OpenTofu registry - Virtual repositories — merge multiple remotes of the same package type into a single unified index - Immutable/mutable caching model with per-remote TTLs - Conditional revalidation (`If-None-Match` / `If-Modified-Since`) on TTL expiry @@ -64,7 +64,8 @@ src/artifactapi/ ├── npm.py — npm metadata URL rewriting ├── puppet.py — Puppet Forge JSON URL rewriting ├── python.py — PyPI URL construction + HTML rewriting - └── rpm.py — RPM remotes + ├── rpm.py — RPM remotes + └── terraform.py — Terraform/OpenTofu registry URL construction + download URL rewriting ``` ## API Endpoints @@ -131,7 +132,7 @@ Repositories are declared under three top-level keys matching their type: remotes: # proxy (caching) remotes remote-name: base_url: "https://example.com" - package: "generic" # generic, alpine, rpm, docker, pypi, npm, helm, puppet + package: "generic" # generic, alpine, rpm, docker, pypi, npm, helm, puppet, terraform description: "..." immutable_patterns: # regex — cached forever - ".*\\.tar\\.gz$" @@ -404,6 +405,52 @@ mod 'puppetlabs-stdlib', '9.7.0' mod 'puppetlabs-inifile', '6.2.0' ``` +### terraform + +Proxy for [Terraform](https://registry.terraform.io) / [OpenTofu](https://opentofu.org) provider registries using the [Registry Protocol](https://developer.hashicorp.com/terraform/internals/provider-registry-protocol). Provider version listings are mutable; per-version download info is immutable. + +Two remotes are needed: one for the registry API and one for the release CDN (where the actual `.zip` binaries live): + +```yaml +remotes: + terraform-registry: + base_url: "https://registry.terraform.io" + package: "terraform" + releases_remote: "hashicorp-releases" # name of the CDN remote below + immutable_patterns: + - "[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$" + cache: + immutable_ttl: 0 # per-version download info cached indefinitely + mutable_ttl: 300 # provider version lists refreshed after 5 minutes + + hashicorp-releases: + base_url: "https://releases.hashicorp.com" + package: "generic" + immutable_patterns: + - ".*\\.zip$" + - ".*SHA256SUMS(\\.sig)?$" + cache: + immutable_ttl: 0 + mutable_ttl: 0 +``` + +`{namespace}/{type}/versions` is a built-in mutable pattern — the version list expires after `mutable_ttl` and is re-fetched on the next request. + +**URL rewriting**: the `download_url`, `shasums_url`, and `shasums_signature_url` fields in per-version download info JSON are rewritten from `releases.hashicorp.com` to point at the remote named by `releases_remote`, so Terraform fetches binaries through the proxy. + +**Client configuration**: redirect Terraform's provider registry lookup via `.terraformrc` without changing any provider source addresses in your Terraform code: + +```hcl +# ~/.terraformrc (or /etc/terraform.rc, or TF_CLI_CONFIG_FILE) +host "registry.terraform.io" { + services = { + "providers.v1" = "http://artifacts.example.com/api/v1/remote/terraform-registry/" + } +} +``` + +With this in place, `terraform init` / `tofu init` fetches provider metadata from the proxy and downloads zips from the `hashicorp-releases` remote. No changes to `.tf` files are needed. + ### virtual A virtual repository presents a single unified index built from multiple member remotes of the same package type. Clients configure one endpoint and get access to all member remotes transparently. @@ -501,6 +548,7 @@ Each package type has built-in defaults that are merged with any user-defined `m | `pypi` | `simple/` (per-package and top-level index pages) | | `helm` | `index\.yaml$` | | `puppet` | `^v3/modules/`, `^v3/releases` | +| `terraform` | `[^/]+/[^/]+/versions$` | | `npm` | *(none built-in — define via `mutable_patterns`)* | | `generic` | *(none)* | diff --git a/examples/single-file/remotes.yaml b/examples/single-file/remotes.yaml index c7adb9d..f4aaa06 100644 --- a/examples/single-file/remotes.yaml +++ b/examples/single-file/remotes.yaml @@ -466,6 +466,37 @@ remotes: immutable_ttl: 0 # Module tarballs cached indefinitely mutable_ttl: 600 # Module metadata refreshed after 10 minutes + terraform-registry: + base_url: "https://registry.terraform.io" + package: "terraform" + description: "Terraform/OpenTofu provider registry (Registry Protocol)" + # Provider version lists are mutable by default. + # Point Terraform at this remote via .terraformrc: + # host "registry.terraform.io" { + # services = { + # "providers.v1" = "http://your-proxy/api/v1/remote/terraform-registry/" + # } + # } + # releases_remote must match the name of the hashicorp-releases remote below, + # so download_url / shasums_url in per-version download info are rewritten. + releases_remote: "hashicorp-releases" + immutable_patterns: + - "[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$" + cache: + immutable_ttl: 0 # Per-version download info cached indefinitely + mutable_ttl: 300 # Provider versions list refreshed after 5 minutes + + hashicorp-releases: + base_url: "https://releases.hashicorp.com" + package: "generic" + description: "HashiCorp releases CDN — provider zips, SHA256SUMS, and signatures" + immutable_patterns: + - ".*\\.zip$" + - ".*SHA256SUMS(\\.sig)?$" + cache: + immutable_ttl: 0 # Release artifacts cached indefinitely + mutable_ttl: 0 + virtuals: helm-all: diff --git a/src/artifactapi/artifact/proxy.py b/src/artifactapi/artifact/proxy.py index 6fad465..8d3facd 100644 --- a/src/artifactapi/artifact/proxy.py +++ b/src/artifactapi/artifact/proxy.py @@ -13,6 +13,7 @@ from ..remote import helm as _helm from ..remote import npm as _npm from ..remote import puppet as _puppet from ..remote import python as _pypi +from ..remote import terraform as _terraform from ..remote.base import get_content_type logger = logging.getLogger(__name__) @@ -87,6 +88,9 @@ def _resolve_content( return _helm.resolve_content(data, path, filename, base_url, proxy_base, remote_name) if package == "puppet": return _puppet.resolve_content(data, path, filename, base_url, proxy_base, remote_name) + if package == "terraform": + releases_remote = remote_config.get("releases_remote") + return _terraform.resolve_content(data, path, filename, base_url, proxy_base, remote_name, releases_remote) return data, get_content_type(filename) @@ -96,6 +100,8 @@ def construct_url(remote_config: dict, path: str) -> str: return f"{base_url}/v2/{path}" if remote_config.get("package") == "pypi": return _pypi.construct_url(base_url, path) + if remote_config.get("package") == "terraform": + return _terraform.construct_url(base_url, path) return f"{base_url}/{path}" diff --git a/src/artifactapi/config.py b/src/artifactapi/config.py index 2e5dc14..e1d59f1 100644 --- a/src/artifactapi/config.py +++ b/src/artifactapi/config.py @@ -30,6 +30,9 @@ _PACKAGE_MUTABLE_PATTERNS: dict[str, list[str]] = { r"^v3/modules/", r"^v3/releases", ], + "terraform": [ + r"[^/]+/[^/]+/versions$", + ], "generic": [], } diff --git a/src/artifactapi/remote/__init__.py b/src/artifactapi/remote/__init__.py index f84880f..225f8c5 100644 --- a/src/artifactapi/remote/__init__.py +++ b/src/artifactapi/remote/__init__.py @@ -1,4 +1,4 @@ -from . import generic, helm, npm, puppet, python, rpm +from . import generic, helm, npm, puppet, python, rpm, terraform from .base import get_content_type -__all__ = ["generic", "helm", "npm", "puppet", "python", "rpm", "get_content_type"] +__all__ = ["generic", "helm", "npm", "puppet", "python", "rpm", "terraform", "get_content_type"] diff --git a/src/artifactapi/remote/terraform.py b/src/artifactapi/remote/terraform.py new file mode 100644 index 0000000..ab89ff8 --- /dev/null +++ b/src/artifactapi/remote/terraform.py @@ -0,0 +1,36 @@ +import json +import re +from urllib.parse import urlparse + +from .base import get_content_type + +_DOWNLOAD_PATH = re.compile(r"^[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$") + + +def construct_url(base_url: str, path: str) -> str: + return f"{base_url}/v1/providers/{path}" + + +def resolve_content( + data: bytes, + path: str, + filename: str, + _base_url: str, + proxy_url: str, + _remote_name: str, + releases_remote: str | None = None, +) -> tuple[bytes, str]: + if filename.endswith((".zip", ".sig")): + return data, get_content_type(filename) + if releases_remote and _DOWNLOAD_PATH.match(path): + releases_proxy = f"{proxy_url}/api/v1/remote/{releases_remote}" + try: + obj = json.loads(data) + for field in ("download_url", "shasums_url", "shasums_signature_url"): + if field in obj: + parsed = urlparse(obj[field]) + obj[field] = f"{releases_proxy}{parsed.path}" + data = json.dumps(obj).encode() + except (json.JSONDecodeError, KeyError): + pass + return data, "application/json" diff --git a/tests/conftest.py b/tests/conftest.py index 17b7bf7..2659f14 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -118,6 +118,21 @@ TEST_REMOTES = { "immutable_patterns": [r"^v3/files/.*\.tar\.gz$"], "cache": {"immutable_ttl": 0, "mutable_ttl": 600}, }, + "terraform-registry-test": { + "base_url": "https://registry.terraform.io", + "package": "terraform", + "immutable_patterns": [ + r"[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+$", + ], + "releases_remote": "hashicorp-releases-test", + "cache": {"immutable_ttl": 0, "mutable_ttl": 300}, + }, + "hashicorp-releases-test": { + "base_url": "https://releases.hashicorp.com", + "package": "generic", + "immutable_patterns": [r".*\.zip$", r".*SHA256SUMS(\.sig)?$"], + "cache": {"immutable_ttl": 0, "mutable_ttl": 0}, + }, }, "locals": { "local-test": { diff --git a/tests/test_routes.py b/tests/test_routes.py index 9bb4406..6195e59 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -1234,6 +1234,150 @@ class TestPuppetRemote: assert b'"/v3/files/' not in response.content +# --------------------------------------------------------------------------- +# Terraform registry remote (terraform-registry-test) +# --------------------------------------------------------------------------- + + +class TestTerraformRemote: + def test_versions_path_is_mutable(self, client, patched_deps): + """Provider versions listing is detected as mutable.""" + deps = patched_deps + deps["storage"].exists.return_value = True + deps["storage"].download_object.return_value = b'{"versions":[]}' + deps["cache"].is_mutable_file.return_value = True + deps["cache"].is_index_valid.return_value = True + + response = client.get("/api/v1/remote/terraform-registry-test/hashicorp/vault/versions") + assert response.status_code == 200 + deps["cache"].mark_index_cached.assert_not_called() + + def test_versions_returns_json_content_type(self, client, patched_deps): + deps = patched_deps + deps["storage"].exists.return_value = True + deps["storage"].download_object.return_value = b'{"versions":[]}' + deps["cache"].is_mutable_file.return_value = True + deps["cache"].is_index_valid.return_value = True + + response = client.get("/api/v1/remote/terraform-registry-test/hashicorp/vault/versions") + assert response.status_code == 200 + assert "application/json" in response.headers["content-type"] + + def test_download_info_download_url_rewritten(self, client, patched_deps): + """download_url in download-info JSON is rewritten to point to the releases proxy.""" + deps = patched_deps + download_info = json.dumps( + { + "os": "linux", + "arch": "amd64", + "filename": "terraform-provider-vault_0.28.0_linux_amd64.zip", + "download_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/terraform-provider-vault_0.28.0_linux_amd64.zip", + "shasums_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/terraform-provider-vault_0.28.0_SHA256SUMS", + "shasums_signature_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/terraform-provider-vault_0.28.0_SHA256SUMS.sig", + } + ).encode() + deps["storage"].exists.return_value = True + deps["storage"].download_object.return_value = download_info + deps["cache"].is_mutable_file.return_value = False + + response = client.get("/api/v1/remote/terraform-registry-test/hashicorp/vault/0.28.0/download/linux/amd64") + assert response.status_code == 200 + data = response.json() + assert "releases.hashicorp.com" not in data["download_url"] + assert "/api/v1/remote/hashicorp-releases-test/" in data["download_url"] + + def test_download_info_shasums_url_rewritten(self, client, patched_deps): + """shasums_url is also rewritten to the releases proxy.""" + deps = patched_deps + download_info = json.dumps( + { + "os": "linux", + "arch": "amd64", + "filename": "terraform-provider-vault_0.28.0_linux_amd64.zip", + "download_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/terraform-provider-vault_0.28.0_linux_amd64.zip", + "shasums_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/terraform-provider-vault_0.28.0_SHA256SUMS", + "shasums_signature_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/terraform-provider-vault_0.28.0_SHA256SUMS.sig", + } + ).encode() + deps["storage"].exists.return_value = True + deps["storage"].download_object.return_value = download_info + deps["cache"].is_mutable_file.return_value = False + + response = client.get("/api/v1/remote/terraform-registry-test/hashicorp/vault/0.28.0/download/linux/amd64") + assert response.status_code == 200 + data = response.json() + assert "/api/v1/remote/hashicorp-releases-test/" in data["shasums_url"] + assert "/api/v1/remote/hashicorp-releases-test/" in data["shasums_signature_url"] + assert "releases.hashicorp.com" not in data["shasums_url"] + assert "releases.hashicorp.com" not in data["shasums_signature_url"] + + def test_download_info_path_preserved(self, client, patched_deps): + """The path portion of the upstream URL is preserved when rewriting.""" + deps = patched_deps + zip_path = "/terraform-provider-vault/0.28.0/terraform-provider-vault_0.28.0_linux_amd64.zip" + download_info = json.dumps( + { + "download_url": f"https://releases.hashicorp.com{zip_path}", + "shasums_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/terraform-provider-vault_0.28.0_SHA256SUMS", + "shasums_signature_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/terraform-provider-vault_0.28.0_SHA256SUMS.sig", + } + ).encode() + deps["storage"].exists.return_value = True + deps["storage"].download_object.return_value = download_info + deps["cache"].is_mutable_file.return_value = False + + response = client.get("/api/v1/remote/terraform-registry-test/hashicorp/vault/0.28.0/download/linux/amd64") + assert response.status_code == 200 + data = response.json() + assert data["download_url"].endswith(zip_path) + + def test_zip_served_as_binary(self, client, patched_deps): + """Provider zip files are served as binary without JSON rewriting.""" + deps = patched_deps + deps["storage"].exists.return_value = True + deps["storage"].download_object.return_value = b"PK\x03\x04 zip bytes" + deps["cache"].is_mutable_file.return_value = False + + response = client.get( + "/api/v1/remote/hashicorp-releases-test/terraform-provider-vault/0.28.0/terraform-provider-vault_0.28.0_linux_amd64.zip" + ) + assert response.status_code == 200 + assert response.headers["X-Artifact-Source"] == "cache" + + def test_construct_url_prepends_v1_providers(self, client, patched_deps): + """Upstream URL for the terraform package type prepends /v1/providers/.""" + deps = patched_deps + deps["storage"].exists.return_value = False + + with patch( + "artifactapi.artifact.proxy.cache_single_artifact", + new_callable=AsyncMock, + return_value={"status": "cached"}, + ) as mock_fetch: + deps["storage"].download_object.return_value = b'{"versions":[]}' + deps["cache"].is_mutable_file.return_value = True + client.get("/api/v1/remote/terraform-registry-test/hashicorp/vault/versions") + + called_url = mock_fetch.call_args[0][0] + assert called_url == "https://registry.terraform.io/v1/providers/hashicorp/vault/versions" + + def test_versions_cache_miss_fetches_upstream(self, client, patched_deps): + deps = patched_deps + deps["storage"].exists.return_value = False + deps["storage"].download_object.return_value = b'{"versions":[]}' + deps["cache"].is_mutable_file.return_value = True + + with patch( + "artifactapi.artifact.proxy.cache_single_artifact", + new_callable=AsyncMock, + return_value={"status": "cached"}, + ) as mock_fetch: + response = client.get("/api/v1/remote/terraform-registry-test/hashicorp/vault/versions") + + mock_fetch.assert_called_once() + assert response.status_code == 200 + + # --------------------------------------------------------------------------- # Quarantine (quarantine-test remote: quarantine_new=True, quarantine_days=3) # ---------------------------------------------------------------------------