diff --git a/README.md b/README.md index 03ca7af..9d83124 100644 --- a/README.md +++ b/README.md @@ -122,11 +122,12 @@ remotes: {} # optional base remotes ### remotes.yaml Structure +The top-level key declares the repository type — no `type:` field needed: + ```yaml -remotes: +remote: remote-name: base_url: "https://example.com" - type: "remote" # "remote", "local", or "virtual" package: "generic" # generic, alpine, rpm, docker, pypi, npm, helm description: "..." immutable_patterns: # regex — cached forever @@ -137,6 +138,17 @@ remotes: cache: immutable_ttl: 0 # 0 = indefinitely mutable_ttl: 3600 + +virtual: + virtual-name: + package: "helm" + members: + - remote-name-1 + - remote-name-2 + +local: + local-name: + package: "generic" ``` ## Remote Types @@ -146,10 +158,9 @@ remotes: Arbitrary HTTP file servers — GitHub releases, HashiCorp, custom servers. ```yaml -remotes: +remote: github: base_url: "https://github.com" - type: "remote" package: "generic" immutable_patterns: - "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*" @@ -158,7 +169,6 @@ remotes: github-archive: base_url: "https://github.com" - type: "remote" package: "generic" immutable_patterns: - ".*/archive/refs/tags/.*\\.tar\\.gz$" # tag archives never change @@ -175,10 +185,9 @@ Access: `GET /api/v1/remote/github/owner/repo/releases/download/v1.0/binary.tar. ### alpine ```yaml -remotes: +remote: alpine: base_url: "https://dl-cdn.alpinelinux.org" - type: "remote" package: "alpine" immutable_patterns: - ".*/x86_64/.*\\.apk$" @@ -192,10 +201,9 @@ remotes: ### rpm ```yaml -remotes: +remote: almalinux: base_url: "https://mirror.example.com/almalinux" - type: "remote" package: "rpm" immutable_patterns: - ".*/x86_64/.*\\.rpm$" @@ -210,10 +218,9 @@ remotes: ### docker ```yaml -remotes: +remote: dockerhub: base_url: "https://registry-1.docker.io" - type: "remote" package: "docker" # username / password optional for public images cache: @@ -222,7 +229,6 @@ remotes: ghcr: base_url: "https://ghcr.io" - type: "remote" package: "docker" username: "your-github-username" password: "ghp_your_pat" # read:packages scope @@ -252,10 +258,9 @@ mirrors: ### pypi ```yaml -remotes: +remote: pypi: base_url: "https://files.pythonhosted.org" - type: "remote" package: "pypi" check_mutable_updates: true immutable_patterns: @@ -284,10 +289,9 @@ default = true ### npm ```yaml -remotes: +remote: npm: base_url: "https://registry.npmjs.org" - type: "remote" package: "npm" check_mutable_updates: true immutable_patterns: @@ -311,10 +315,9 @@ registry=https://artifacts.example.com/api/v1/remote/npm/ ### helm ```yaml -remotes: +remote: hashicorp-helm: base_url: "https://helm.releases.hashicorp.com" - type: "remote" package: "helm" check_mutable_updates: true immutable_patterns: @@ -340,10 +343,9 @@ A virtual repository presents a single unified index built from multiple member All members must share the same `package` type as the virtual repo. Currently supported package types: `helm`. ```yaml -remotes: +remote: helm-hashicorp: base_url: "https://helm.releases.hashicorp.com" - type: "remote" package: "helm" immutable_patterns: - "\\.tgz$" @@ -353,7 +355,6 @@ remotes: helm-bitnami: base_url: "https://charts.bitnami.com/bitnami" - type: "remote" package: "helm" immutable_patterns: - "\\.tgz$" @@ -361,8 +362,8 @@ remotes: immutable_ttl: 0 mutable_ttl: 3600 +virtual: helm-all: - type: "virtual" package: "helm" members: - helm-hashicorp # listed first = highest priority @@ -399,9 +400,8 @@ Chart tarball URLs in the merged `index.yaml` are rewritten to point at the indi ### local ```yaml -remotes: +local: local-generic: - type: "local" package: "generic" description: "Local file repository" cache: @@ -411,6 +411,98 @@ remotes: No `base_url`. Files are uploaded via `PUT` and served via `GET`. +## Migration + +### Splitting a single remotes file into per-type files + +The old format used a single `remotes:` map with an explicit `type:` field on each entry. The new format uses top-level type keys (`remote:`, `virtual:`, `local:`) and supports splitting across multiple files via `config_dir`. + +**Before** (`remotes.yaml`): +```yaml +remotes: + dockerhub: + base_url: "https://registry-1.docker.io" + type: "remote" + package: "docker" + cache: + immutable_ttl: 0 + mutable_ttl: 300 + + hashicorp-helm: + base_url: "https://helm.releases.hashicorp.com" + type: "remote" + package: "helm" + immutable_patterns: + - "\\.tgz$" + cache: + immutable_ttl: 0 + mutable_ttl: 3600 + + helm-all: + type: "virtual" + package: "helm" + members: + - hashicorp-helm + + local-files: + type: "local" + package: "generic" +``` + +**After** — one file per type + package type, with a main config pointing at the directory: + +`config.yaml`: +```yaml +config_dir: conf.d +``` + +`conf.d/remote-docker.yaml`: +```yaml +remote: + dockerhub: + base_url: "https://registry-1.docker.io" + package: "docker" + cache: + immutable_ttl: 0 + mutable_ttl: 300 +``` + +`conf.d/remote-helm.yaml`: +```yaml +remote: + hashicorp-helm: + base_url: "https://helm.releases.hashicorp.com" + package: "helm" + immutable_patterns: + - "\\.tgz$" + cache: + immutable_ttl: 0 + mutable_ttl: 3600 +``` + +`conf.d/virtual-helm.yaml`: +```yaml +virtual: + helm-all: + package: "helm" + members: + - hashicorp-helm +``` + +`conf.d/local-generic.yaml`: +```yaml +local: + local-files: + package: "generic" +``` + +Set `CONFIG_PATH` to the main file: +``` +CONFIG_PATH=/etc/artifactapi/config.yaml +``` + +Files in `conf.d/` are merged alphabetically; later files win on conflicts within the same remote name. + ## Caching Model ### Immutable patterns @@ -448,10 +540,9 @@ When a mutable file expires and the upstream is unreachable (connection refused, Set `quarantine_new: true` and `quarantine_days: N` on a remote to block immutable artifacts published within the last N days. Requests return `404` until the quarantine period expires, giving time to detect malicious packages before they are consumed. ```yaml -remotes: +remote: pypi: base_url: "https://files.pythonhosted.org" - type: "remote" package: "pypi" quarantine_new: true quarantine_days: 3 # block packages published in the last 3 days diff --git a/examples/conf.d-method/alpine.yaml b/examples/conf.d-method/alpine.yaml index a8e12f7..2e31d76 100644 --- a/examples/conf.d-method/alpine.yaml +++ b/examples/conf.d-method/alpine.yaml @@ -1,7 +1,6 @@ -remotes: +remote: alpine: base_url: "https://dl-cdn.alpinelinux.org" - type: "remote" package: "alpine" description: "Alpine Linux APK package repository" immutable_patterns: diff --git a/examples/conf.d-method/github.yaml b/examples/conf.d-method/github.yaml index 51aa20a..dd363ba 100644 --- a/examples/conf.d-method/github.yaml +++ b/examples/conf.d-method/github.yaml @@ -1,7 +1,6 @@ -remotes: +remote: github: base_url: "https://github.com" - type: "remote" package: "generic" description: "GitHub releases and files" immutable_patterns: diff --git a/examples/conf.d-method/pypi.yaml b/examples/conf.d-method/pypi.yaml index 4de0058..f459962 100644 --- a/examples/conf.d-method/pypi.yaml +++ b/examples/conf.d-method/pypi.yaml @@ -1,7 +1,6 @@ -remotes: +remote: pypi: base_url: "https://files.pythonhosted.org" - type: "remote" package: "pypi" description: "Python Package Index" check_mutable_updates: true diff --git a/examples/single-file/remotes.yaml b/examples/single-file/remotes.yaml index 6e36b5b..7beb487 100644 --- a/examples/single-file/remotes.yaml +++ b/examples/single-file/remotes.yaml @@ -1,5 +1,10 @@ # Example remotes configuration — copy and adapt for your environment. # +# Top-level keys determine the repository type: +# remote: — proxy to an upstream URL, cache responses in S3 +# virtual: — merge indexes from multiple member remotes into one +# local: — store files uploaded via PUT, serve via GET +# # immutable_patterns: artifacts cached forever (e.g. release binaries, versioned tags). # mutable_patterns: artifacts that expire after cache.mutable_ttl seconds and are # re-fetched from upstream on next request (e.g. index files, @@ -32,10 +37,10 @@ #database: # url: "postgresql://artifacts:artifacts123@localhost:5432/artifacts" # -remotes: + +remote: github: base_url: "https://github.com" - type: "remote" package: "generic" description: "GitHub releases and files" immutable_patterns: @@ -62,12 +67,11 @@ remotes: - "stalwartlabs/stalwart/.*/stalwart-foundationdb-x86_64-unknown-linux-gnu\\.tar\\.gz$" - "stalwartlabs/stalwart/.*/stalwart-x86_64-unknown-linux-gnu\\.tar\\.gz$" cache: - immutable_ttl: 0 # Files cached indefinitely + immutable_ttl: 0 mutable_ttl: 0 github-archive: base_url: "https://github.com" - type: "remote" package: "generic" description: "GitHub repository archive tarballs" immutable_patterns: @@ -82,23 +86,21 @@ remotes: # Only applies to user-defined mutable_patterns, not package-type defaults. check_mutable_updates: true cache: - immutable_ttl: 0 # Tag archives cached indefinitely - mutable_ttl: 86400 # Branch archives refreshed after 1 day + immutable_ttl: 0 # Tag archives cached indefinitely + mutable_ttl: 86400 # Branch archives refreshed after 1 day gitea-dl: base_url: "https://dl.gitea.com" - type: "remote" package: "generic" description: "Gitea download site" immutable_patterns: - "act_runner/.*/act_runner-.*-linux-amd64$" cache: - immutable_ttl: 0 # Files cached indefinitely + immutable_ttl: 0 mutable_ttl: 0 hashicorp-releases: base_url: "https://releases.hashicorp.com" - type: "remote" package: "generic" description: "HashiCorp product releases" immutable_patterns: @@ -114,12 +116,11 @@ remotes: - "nomad/.*/nomad_.*_linux_amd64\\.zip$" - "packer/.*/packer_.*_linux_amd64\\.zip$" cache: - immutable_ttl: 0 # Files cached indefinitely + immutable_ttl: 0 mutable_ttl: 0 alpine: base_url: "https://dl-cdn.alpinelinux.org" - type: "remote" package: "alpine" description: "Alpine Linux APK package repository" immutable_patterns: @@ -128,29 +129,24 @@ remotes: # and is always re-fetched on expiry — conditional checks are skipped for # built-in mutable patterns regardless of this flag. cache: - immutable_ttl: 0 # Files cached indefinitely + immutable_ttl: 0 mutable_ttl: 7200 # Index files (APKINDEX.tar.gz) cached for 2 hours almalinux: base_url: "https://gsl-syd.mm.fcix.net/almalinux" - type: "remote" package: "rpm" description: "AlmaLinux RPM package repository" immutable_patterns: - ".*/x86_64/.*\\.rpm$" - ".*/noarch/.*\\.rpm$" - ".*/repodata/.*$" - - ".*\\.rpm$" # Allow all RPM files - # repomd.xml / repodata are package-type defaults — always re-fetched on - # expiry. check_mutable_updates would only apply to any custom - # mutable_patterns added here. + - ".*\\.rpm$" cache: - immutable_ttl: 0 # Files cached indefinitely + immutable_ttl: 0 mutable_ttl: 7200 # Metadata files cached for 2 hours epel: base_url: "http://mirror.aarnet.edu.au/pub/epel" - type: "remote" package: "rpm" description: "EPEL (Extra Packages for Enterprise Linux)" immutable_patterns: @@ -160,12 +156,11 @@ remotes: - ".*/noarch/.*\\.rpm$" - ".*/repodata/.*$" cache: - immutable_ttl: 0 # Files cached indefinitely - mutable_ttl: 7200 # Metadata files cached for 2 hours + immutable_ttl: 0 + mutable_ttl: 7200 fedora: base_url: "https://gsl-syd.mm.fcix.net/fedora/linux" - type: "remote" package: "rpm" description: "Fedora Linux RPM package repository" immutable_patterns: @@ -175,26 +170,21 @@ remotes: - ".*/noarch/.*\\.rpm$" - "updates/.*/Everything/x86_64/repodata/.*$" cache: - immutable_ttl: 0 # Files cached indefinitely + immutable_ttl: 0 mutable_ttl: 300 # Metadata files cached for 5 minutes ghcr: base_url: "https://ghcr.io" - type: "remote" package: "docker" description: "GitHub Container Registry" # username: "your-github-username" # password: "your-github-pat" # needs read:packages scope - # Docker manifest/tag-list patterns are package-type defaults — always - # re-fetched on expiry. check_mutable_updates only applies to any custom - # mutable_patterns you add (e.g. a metadata endpoint). cache: immutable_ttl: 0 mutable_ttl: 300 dockerhub: base_url: "https://registry-1.docker.io" - type: "remote" package: "docker" description: "Docker Hub registry" cache: @@ -203,15 +193,9 @@ remotes: pypi: base_url: "https://files.pythonhosted.org" - type: "remote" package: "pypi" description: "Python Package Index — simple index and package files via a single remote" - # simple/ requests are transparently fetched from pypi.org; package files come from - # files.pythonhosted.org (base_url). URLs in the simple index are rewritten to this remote. check_mutable_updates: true - # Block packages published within the last 3 days (supply-chain attack mitigation). - # Immutable artifacts (wheel/sdist) newer than quarantine_days return 404 until - # the window passes. Disable by setting quarantine_new: false or removing both keys. quarantine_new: true quarantine_days: 3 immutable_patterns: @@ -226,7 +210,6 @@ remotes: pypi-gitea: base_url: "https://gitea.example.com/api/packages/myorg/pypi" - type: "remote" package: "pypi" description: "Private Gitea PyPI registry — simple index and files at the same host" # username: "your-gitea-username" @@ -244,7 +227,6 @@ remotes: npm: base_url: "https://registry.npmjs.org" - type: "remote" package: "npm" description: "npm registry — package metadata with tarball URL rewriting" check_mutable_updates: true @@ -258,19 +240,17 @@ remotes: hashicorp-helm: base_url: "https://helm.releases.hashicorp.com" - type: "remote" package: "helm" description: "HashiCorp Helm chart repository (Vault, Consul, Nomad, etc.)" check_mutable_updates: true immutable_patterns: - "\\.tgz$" cache: - immutable_ttl: 0 # Chart tarballs are versioned — cache forever - mutable_ttl: 3600 # index.yaml refreshed after 1 hour + immutable_ttl: 0 + mutable_ttl: 3600 metallb: base_url: "https://metallb.github.io/metallb" - type: "remote" package: "helm" description: "MetalLB load balancer Helm charts" check_mutable_updates: true @@ -282,7 +262,6 @@ remotes: jetstack: base_url: "https://charts.jetstack.io" - type: "remote" package: "helm" description: "Jetstack Helm charts (cert-manager)" check_mutable_updates: true @@ -294,7 +273,6 @@ remotes: rancher-stable: base_url: "https://releases.rancher.com/server-charts/stable" - type: "remote" package: "helm" description: "Rancher stable Helm charts" check_mutable_updates: true @@ -306,7 +284,6 @@ remotes: purelb: base_url: "https://gitlab.com/api/v4/projects/20400619/packages/helm/stable" - type: "remote" package: "helm" description: "PureLB load balancer Helm charts" check_mutable_updates: true @@ -318,7 +295,6 @@ remotes: istio: base_url: "https://istio-release.storage.googleapis.com/charts" - type: "remote" package: "helm" description: "Istio service mesh Helm charts" check_mutable_updates: true @@ -330,7 +306,6 @@ remotes: cnpg: base_url: "https://cloudnative-pg.github.io/charts" - type: "remote" package: "helm" description: "CloudNativePG operator Helm charts" check_mutable_updates: true @@ -342,7 +317,6 @@ remotes: ceph-csi: base_url: "https://ceph.github.io/csi-charts" - type: "remote" package: "helm" description: "Ceph CSI driver Helm charts" check_mutable_updates: true @@ -354,7 +328,6 @@ remotes: external-dns: base_url: "https://kubernetes-sigs.github.io/external-dns/" - type: "remote" package: "helm" description: "ExternalDNS Helm charts" check_mutable_updates: true @@ -366,7 +339,6 @@ remotes: intel-helm: base_url: "https://intel.github.io/helm-charts/" - type: "remote" package: "helm" description: "Intel Helm charts" check_mutable_updates: true @@ -378,7 +350,6 @@ remotes: elastic: base_url: "https://helm.elastic.co" - type: "remote" package: "helm" description: "Elastic stack Helm charts" check_mutable_updates: true @@ -390,7 +361,6 @@ remotes: k8up-io: base_url: "https://k8up-io.github.io/k8up" - type: "remote" package: "helm" description: "K8up backup operator Helm charts" check_mutable_updates: true @@ -402,7 +372,6 @@ remotes: victoriametrics: base_url: "https://victoriametrics.github.io/helm-charts/" - type: "remote" package: "helm" description: "VictoriaMetrics observability Helm charts" check_mutable_updates: true @@ -414,7 +383,6 @@ remotes: grafana: base_url: "https://grafana.github.io/helm-charts" - type: "remote" package: "helm" description: "Grafana observability Helm charts" check_mutable_updates: true @@ -426,7 +394,6 @@ remotes: helm-openldap: base_url: "https://jp-gouin.github.io/helm-openldap/" - type: "remote" package: "helm" description: "OpenLDAP Helm charts" check_mutable_updates: true @@ -438,7 +405,6 @@ remotes: woodpecker: base_url: "https://woodpecker-ci.org/" - type: "remote" package: "helm" description: "Woodpecker CI Helm charts" check_mutable_updates: true @@ -450,7 +416,6 @@ remotes: stakater: base_url: "https://stakater.github.io/stakater-charts" - type: "remote" package: "helm" description: "Stakater Helm charts" check_mutable_updates: true @@ -462,7 +427,6 @@ remotes: jfrog: base_url: "https://charts.jfrog.io/" - type: "remote" package: "helm" description: "JFrog Helm charts" check_mutable_updates: true @@ -474,7 +438,6 @@ remotes: openvox: base_url: "https://openvoxproject.github.io/openvox-helm-chart" - type: "remote" package: "helm" description: "OpenVox Helm charts" check_mutable_updates: true @@ -484,8 +447,8 @@ remotes: immutable_ttl: 0 mutable_ttl: 3600 +virtual: helm-all: - type: "virtual" package: "helm" description: "Virtual repository merging all helm remotes — member order is priority order for duplicate chart+version" members: @@ -509,10 +472,10 @@ remotes: - jfrog - openvox +local: local-generic: - type: "local" package: "generic" description: "Local generic file repository" cache: - immutable_ttl: 0 # Files cached indefinitely + immutable_ttl: 0 mutable_ttl: 0 diff --git a/src/artifactapi/config.py b/src/artifactapi/config.py index 6de80e4..e64fd21 100644 --- a/src/artifactapi/config.py +++ b/src/artifactapi/config.py @@ -4,6 +4,21 @@ import os import yaml +_TYPE_KEYS = ("remote", "virtual", "local") + + +def _normalize_loaded(raw: dict) -> dict: + """Convert {remote: {...}, virtual: {...}, local: {...}} into {remotes: {name: {type: ..., ...}}}.""" + remotes = {} + for type_key in _TYPE_KEYS: + for name, cfg in (raw.get(type_key) or {}).items(): + remotes[name] = {"type": type_key, **cfg} + result = {k: v for k, v in raw.items() if k not in _TYPE_KEYS} + if remotes: + result["remotes"] = remotes + return result + + _PACKAGE_MUTABLE_PATTERNS: dict[str, list[str]] = { "alpine": [ r"APKINDEX\.tar\.gz$", @@ -41,8 +56,10 @@ class ConfigManager: try: with open(path) as f: if path.endswith((".yaml", ".yml")): - return yaml.safe_load(f) or {} - return json.load(f) + raw = yaml.safe_load(f) or {} + else: + raw = json.load(f) + return _normalize_loaded(raw) except FileNotFoundError: return {} diff --git a/tests/test_config.py b/tests/test_config.py index a042086..11198d2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -10,11 +10,11 @@ from artifactapi.config import ConfigManager @pytest.fixture def make_config(tmp_path): - """Factory: write a remotes dict to a temp YAML and return a ConfigManager.""" + """Factory: write a remote dict to a temp YAML and return a ConfigManager.""" def _make(remotes_dict): cfg_file = tmp_path / "remotes.yaml" - cfg_file.write_text(yaml.dump({"remotes": remotes_dict})) + cfg_file.write_text(yaml.dump({"remote": remotes_dict})) return ConfigManager(str(cfg_file)) return _make @@ -27,24 +27,24 @@ def make_config(tmp_path): class TestGetMutablePatterns: def test_alpine_returns_package_defaults(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "alpine", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "alpine", "base_url": "https://x.com"}}) patterns = cfg.get_mutable_patterns("r") assert r"APKINDEX\.tar\.gz$" in patterns def test_rpm_returns_package_defaults(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "rpm", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "rpm", "base_url": "https://x.com"}}) patterns = cfg.get_mutable_patterns("r") assert r"repomd\.xml$" in patterns assert any("repodata" in p for p in patterns) def test_docker_returns_package_defaults(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "docker", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "docker", "base_url": "https://x.com"}}) patterns = cfg.get_mutable_patterns("r") assert any("manifests" in p for p in patterns) assert any("tags/list" in p for p in patterns) def test_generic_returns_empty_list(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "generic", "base_url": "https://x.com"}}) assert cfg.get_mutable_patterns("r") == [] def test_unknown_remote_returns_empty_list(self, make_config): @@ -52,19 +52,18 @@ class TestGetMutablePatterns: assert cfg.get_mutable_patterns("nonexistent") == [] def test_missing_package_field_defaults_to_generic(self, make_config): - cfg = make_config({"r": {"type": "remote", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"base_url": "https://x.com"}}) assert cfg.get_mutable_patterns("r") == [] def test_unknown_package_type_returns_empty_list(self, make_config): # A mis-spelled package type silently returns [] — this is a known footgun - cfg = make_config({"r": {"type": "remote", "package": "deb", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "deb", "base_url": "https://x.com"}}) assert cfg.get_mutable_patterns("r") == [] def test_extra_patterns_appended_after_defaults(self, make_config): cfg = make_config( { "r": { - "type": "remote", "package": "alpine", "base_url": "https://x.com", "mutable_patterns": [r"custom\.json$"], @@ -81,7 +80,6 @@ class TestGetMutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "alpine", "base_url": "https://x.com", "mutable_patterns": [], @@ -95,7 +93,6 @@ class TestGetMutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "alpine", "base_url": "https://x.com", "mutable_patterns": [existing], @@ -109,7 +106,6 @@ class TestGetMutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "generic", "base_url": "https://x.com", "mutable_patterns": [r"meta\.json$", r"index\.yaml$"], @@ -122,7 +118,6 @@ class TestGetMutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "rpm", "base_url": "https://x.com", "mutable_patterns": [r"custom-meta\.xml$"], @@ -134,7 +129,7 @@ class TestGetMutablePatterns: assert r"custom-meta\.xml$" in patterns def test_npm_has_no_package_defaults(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "npm", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "npm", "base_url": "https://x.com"}}) assert cfg.get_mutable_patterns("r") == [] def test_npm_explicit_mutable_pattern_matches_metadata(self, make_config): @@ -143,7 +138,6 @@ class TestGetMutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "npm", "base_url": "https://x.com", "mutable_patterns": [r"^(?!.*\.tgz$).*"], @@ -155,14 +149,14 @@ class TestGetMutablePatterns: assert any(re.search(p, "@babel/core") for p in patterns) def test_helm_returns_index_yaml_as_mutable(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "helm", "base_url": "https://helm.example.com"}}) + cfg = make_config({"r": {"package": "helm", "base_url": "https://helm.example.com"}}) patterns = cfg.get_mutable_patterns("r") assert r"index\.yaml$" in patterns def test_helm_chart_tarballs_not_mutable_by_default(self, make_config): import re - cfg = make_config({"r": {"type": "remote", "package": "helm", "base_url": "https://helm.example.com"}}) + cfg = make_config({"r": {"package": "helm", "base_url": "https://helm.example.com"}}) patterns = cfg.get_mutable_patterns("r") # Only index.yaml is mutable; .tgz chart tarballs are not assert not any(re.search(p, "vault-0.29.1.tgz") for p in patterns) @@ -174,7 +168,6 @@ class TestGetMutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "npm", "base_url": "https://x.com", "mutable_patterns": [r"^(?!.*\.tgz$).*"], @@ -196,7 +189,6 @@ class TestGetImmutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "generic", "base_url": "https://x.com", "immutable_patterns": [r".*\.tar\.gz$"], @@ -210,7 +202,7 @@ class TestGetImmutablePatterns: assert cfg.get_immutable_patterns("nonexistent") == [] def test_returns_empty_when_no_patterns_configured(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "generic", "base_url": "https://x.com"}}) assert cfg.get_immutable_patterns("r") == [] def test_multiple_patterns_returned(self, make_config): @@ -218,7 +210,6 @@ class TestGetImmutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "rpm", "base_url": "https://x.com", "immutable_patterns": patterns, @@ -231,7 +222,6 @@ class TestGetImmutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "generic", "base_url": "https://x.com", "immutable_patterns": [r".*\.tar\.gz$"], @@ -247,7 +237,6 @@ class TestGetImmutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "generic", "base_url": "https://x.com", "immutable_patterns": [r".*\.tar\.gz$"], @@ -270,7 +259,6 @@ class TestGetUserMutablePatterns: cfg = make_config( { "r": { - "type": "remote", "package": "alpine", "base_url": "https://x.com", "mutable_patterns": [r"custom\.json$"], @@ -281,7 +269,7 @@ class TestGetUserMutablePatterns: def test_excludes_package_defaults(self, make_config): # Package defaults (APKINDEX etc.) must NOT appear here - cfg = make_config({"r": {"type": "remote", "package": "alpine", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "alpine", "base_url": "https://x.com"}}) assert cfg.get_user_mutable_patterns("r") == [] def test_returns_empty_for_missing_remote(self, make_config): @@ -289,7 +277,7 @@ class TestGetUserMutablePatterns: assert cfg.get_user_mutable_patterns("nonexistent") == [] def test_returns_empty_when_key_absent(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "generic", "base_url": "https://x.com"}}) assert cfg.get_user_mutable_patterns("r") == [] @@ -303,7 +291,6 @@ class TestGetCacheConfig: cfg = make_config( { "r": { - "type": "remote", "package": "generic", "base_url": "https://x.com", "cache": {"immutable_ttl": 0, "mutable_ttl": 7200}, @@ -317,7 +304,7 @@ class TestGetCacheConfig: assert cfg.get_cache_config("nonexistent") == {} def test_returns_empty_dict_when_no_cache_key(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "generic", "base_url": "https://x.com"}}) assert cfg.get_cache_config("r") == {} @@ -329,11 +316,11 @@ class TestGetCacheConfig: class TestConfigReload: def test_reloads_when_file_mtime_advances(self, tmp_path): cfg_file = tmp_path / "remotes.yaml" - cfg_file.write_text(yaml.dump({"remotes": {"repo-a": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}})) + cfg_file.write_text(yaml.dump({"remote": {"repo-a": {"package": "generic", "base_url": "https://x.com"}}})) cfg = ConfigManager(str(cfg_file)) assert "repo-a" in cfg.config["remotes"] - cfg_file.write_text(yaml.dump({"remotes": {"repo-b": {"type": "remote", "package": "generic", "base_url": "https://y.com"}}})) + cfg_file.write_text(yaml.dump({"remote": {"repo-b": {"package": "generic", "base_url": "https://y.com"}}})) future_mtime = cfg._last_modified + 1 os.utime(str(cfg_file), (future_mtime, future_mtime)) @@ -344,7 +331,7 @@ class TestConfigReload: def test_no_reload_when_file_unchanged(self, tmp_path): cfg_file = tmp_path / "remotes.yaml" - cfg_file.write_text(yaml.dump({"remotes": {"repo-a": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}})) + cfg_file.write_text(yaml.dump({"remote": {"repo-a": {"package": "generic", "base_url": "https://x.com"}}})) cfg = ConfigManager(str(cfg_file)) # Call check_reload without touching the file — should not reload @@ -360,7 +347,7 @@ class TestConfigReload: class TestGetQuarantineConfig: def test_returns_false_zero_when_not_configured(self, make_config): - cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}}) + cfg = make_config({"r": {"package": "generic", "base_url": "https://x.com"}}) enabled, days = cfg.get_quarantine_config("r") assert enabled is False assert days == 0 @@ -375,7 +362,6 @@ class TestGetQuarantineConfig: cfg = make_config( { "r": { - "type": "remote", "package": "generic", "base_url": "https://x.com", "quarantine_new": True, @@ -391,7 +377,6 @@ class TestGetQuarantineConfig: cfg = make_config( { "r": { - "type": "remote", "package": "generic", "base_url": "https://x.com", "quarantine_new": False, @@ -407,7 +392,6 @@ class TestGetQuarantineConfig: cfg = make_config( { "r": { - "type": "remote", "package": "generic", "base_url": "https://x.com", "quarantine_new": True, @@ -426,20 +410,20 @@ class TestGetQuarantineConfig: def _remote(base_url: str = "https://x.com") -> dict: - return {"type": "remote", "package": "generic", "base_url": base_url} + return {"package": "generic", "base_url": base_url} class TestConfigDirMode: def test_loads_all_yaml_files(self, tmp_path): - (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}})) - (tmp_path / "b.yaml").write_text(yaml.dump({"remotes": {"repo-b": _remote("https://y.com")}})) + (tmp_path / "a.yaml").write_text(yaml.dump({"remote": {"repo-a": _remote()}})) + (tmp_path / "b.yaml").write_text(yaml.dump({"remote": {"repo-b": _remote("https://y.com")}})) cfg = ConfigManager(str(tmp_path)) assert "repo-a" in cfg.config["remotes"] assert "repo-b" in cfg.config["remotes"] def test_later_file_overrides_earlier_on_same_key(self, tmp_path): - (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"r": _remote("https://first.com")}})) - (tmp_path / "b.yaml").write_text(yaml.dump({"remotes": {"r": _remote("https://second.com")}})) + (tmp_path / "a.yaml").write_text(yaml.dump({"remote": {"r": _remote("https://first.com")}})) + (tmp_path / "b.yaml").write_text(yaml.dump({"remote": {"r": _remote("https://second.com")}})) cfg = ConfigManager(str(tmp_path)) assert cfg.config["remotes"]["r"]["base_url"] == "https://second.com" @@ -449,18 +433,18 @@ class TestConfigDirMode: def test_ignores_non_yaml_files(self, tmp_path): (tmp_path / "notes.txt").write_text("not yaml") - (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}})) + (tmp_path / "a.yaml").write_text(yaml.dump({"remote": {"repo-a": _remote()}})) cfg = ConfigManager(str(tmp_path)) assert list(cfg.config["remotes"].keys()) == ["repo-a"] def test_reload_picks_up_new_file(self, tmp_path): - (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}})) + (tmp_path / "a.yaml").write_text(yaml.dump({"remote": {"repo-a": _remote()}})) cfg = ConfigManager(str(tmp_path)) assert "repo-a" in cfg.config["remotes"] assert "repo-b" not in cfg.config["remotes"] new_file = tmp_path / "b.yaml" - new_file.write_text(yaml.dump({"remotes": {"repo-b": _remote("https://y.com")}})) + new_file.write_text(yaml.dump({"remote": {"repo-b": _remote("https://y.com")}})) future_mtime = cfg._last_modified + 1 os.utime(str(new_file), (future_mtime, future_mtime)) @@ -479,9 +463,9 @@ class TestConfigDirKey: def test_merges_remotes_from_config_dir(self, tmp_path): conf_d = tmp_path / "conf.d" conf_d.mkdir() - (conf_d / "remotes.yaml").write_text(yaml.dump({"remotes": {"repo-extra": _remote("https://extra.com")}})) + (conf_d / "remotes.yaml").write_text(yaml.dump({"remote": {"repo-extra": _remote("https://extra.com")}})) main = tmp_path / "config.yaml" - main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {"repo-main": _remote()}})) + main.write_text(yaml.dump({"config_dir": str(conf_d), "remote": {"repo-main": _remote()}})) cfg = ConfigManager(str(main)) assert "repo-main" in cfg.config["remotes"] assert "repo-extra" in cfg.config["remotes"] @@ -489,9 +473,9 @@ class TestConfigDirKey: def test_relative_config_dir_resolved_from_main_file(self, tmp_path): conf_d = tmp_path / "conf.d" conf_d.mkdir() - (conf_d / "r.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}})) + (conf_d / "r.yaml").write_text(yaml.dump({"remote": {"repo-a": _remote()}})) main = tmp_path / "config.yaml" - main.write_text(yaml.dump({"config_dir": "conf.d", "remotes": {}})) + main.write_text(yaml.dump({"config_dir": "conf.d"})) cfg = ConfigManager(str(main)) assert "repo-a" in cfg.config["remotes"] @@ -499,16 +483,16 @@ class TestConfigDirKey: conf_d = tmp_path / "conf.d" conf_d.mkdir() main = tmp_path / "config.yaml" - main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {}})) + main.write_text(yaml.dump({"config_dir": str(conf_d), "remote": {}})) cfg = ConfigManager(str(main)) assert "config_dir" not in cfg.config def test_dir_remote_overrides_main_file_remote(self, tmp_path): conf_d = tmp_path / "conf.d" conf_d.mkdir() - (conf_d / "override.yaml").write_text(yaml.dump({"remotes": {"r": _remote("https://new.com")}})) + (conf_d / "override.yaml").write_text(yaml.dump({"remote": {"r": _remote("https://new.com")}})) main = tmp_path / "config.yaml" - main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {"r": _remote("https://old.com")}})) + main.write_text(yaml.dump({"config_dir": str(conf_d), "remote": {"r": _remote("https://old.com")}})) cfg = ConfigManager(str(main)) assert cfg.config["remotes"]["r"]["base_url"] == "https://new.com" @@ -516,7 +500,7 @@ class TestConfigDirKey: conf_d = tmp_path / "conf.d" conf_d.mkdir() main = tmp_path / "config.yaml" - main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {"repo-main": _remote()}})) + main.write_text(yaml.dump({"config_dir": str(conf_d), "remote": {"repo-main": _remote()}})) cfg = ConfigManager(str(main)) assert list(cfg.config["remotes"].keys()) == ["repo-main"] @@ -524,13 +508,13 @@ class TestConfigDirKey: conf_d = tmp_path / "conf.d" conf_d.mkdir() dir_file = conf_d / "r.yaml" - dir_file.write_text(yaml.dump({"remotes": {"repo-v1": _remote()}})) + dir_file.write_text(yaml.dump({"remote": {"repo-v1": _remote()}})) main = tmp_path / "config.yaml" - main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {}})) + main.write_text(yaml.dump({"config_dir": str(conf_d), "remote": {}})) cfg = ConfigManager(str(main)) assert "repo-v1" in cfg.config["remotes"] - dir_file.write_text(yaml.dump({"remotes": {"repo-v2": _remote("https://v2.com")}})) + dir_file.write_text(yaml.dump({"remote": {"repo-v2": _remote("https://v2.com")}})) future_mtime = cfg._last_modified + 1 os.utime(str(dir_file), (future_mtime, future_mtime)) @@ -538,3 +522,77 @@ class TestConfigDirKey: assert "repo-v2" in cfg.config["remotes"] assert "repo-v1" not in cfg.config["remotes"] + + +# --------------------------------------------------------------------------- +# YAML format normalisation — top-level type keys +# --------------------------------------------------------------------------- + + +class TestYamlTypeKeys: + def test_remote_key_injects_type_remote(self, tmp_path): + f = tmp_path / "r.yaml" + f.write_text(yaml.dump({"remote": {"my-remote": {"package": "generic", "base_url": "https://x.com"}}})) + cfg = ConfigManager(str(f)) + assert cfg.config["remotes"]["my-remote"]["type"] == "remote" + + def test_virtual_key_injects_type_virtual(self, tmp_path): + f = tmp_path / "r.yaml" + f.write_text(yaml.dump({"virtual": {"my-virtual": {"package": "helm", "members": ["a", "b"]}}})) + cfg = ConfigManager(str(f)) + assert cfg.config["remotes"]["my-virtual"]["type"] == "virtual" + assert cfg.config["remotes"]["my-virtual"]["members"] == ["a", "b"] + + def test_local_key_injects_type_local(self, tmp_path): + f = tmp_path / "r.yaml" + f.write_text(yaml.dump({"local": {"my-local": {"package": "generic"}}})) + cfg = ConfigManager(str(f)) + assert cfg.config["remotes"]["my-local"]["type"] == "local" + + def test_mixed_file_all_three_types(self, tmp_path): + f = tmp_path / "r.yaml" + f.write_text( + yaml.dump( + { + "remote": {"r": {"package": "helm", "base_url": "https://helm.example.com"}}, + "virtual": {"v": {"package": "helm", "members": ["r"]}}, + "local": {"l": {"package": "generic"}}, + } + ) + ) + cfg = ConfigManager(str(f)) + assert cfg.config["remotes"]["r"]["type"] == "remote" + assert cfg.config["remotes"]["v"]["type"] == "virtual" + assert cfg.config["remotes"]["l"]["type"] == "local" + + def test_type_field_not_required_in_yaml(self, tmp_path): + f = tmp_path / "r.yaml" + f.write_text(yaml.dump({"remote": {"r": {"package": "alpine", "base_url": "https://x.com"}}})) + cfg = ConfigManager(str(f)) + raw = cfg.config["remotes"]["r"] + # type is injected by the loader; the original dict had no type key + assert "type" in raw + assert raw["type"] == "remote" + + def test_other_fields_preserved_after_normalisation(self, tmp_path): + f = tmp_path / "r.yaml" + f.write_text( + yaml.dump( + { + "remote": { + "r": { + "package": "helm", + "base_url": "https://helm.example.com", + "immutable_patterns": [r"\.tgz$"], + "cache": {"immutable_ttl": 0, "mutable_ttl": 1800}, + } + } + } + ) + ) + cfg = ConfigManager(str(f)) + remote = cfg.config["remotes"]["r"] + assert remote["package"] == "helm" + assert remote["base_url"] == "https://helm.example.com" + assert remote["cache"] == {"immutable_ttl": 0, "mutable_ttl": 1800} + assert r"\.tgz$" in remote["immutable_patterns"]