From 64266f40e99090f840d36dbddd7d80e235e44d08 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 28 Apr 2026 23:21:02 +1000 Subject: [PATCH 1/2] feat: support config.d directory for split configuration (closes #20) CONFIG_PATH now accepts a directory path (all *.yaml files merged) or a main file with a config_dir key pointing to a drop-in directory. Remotes are merged alphabetically across files; later files win on conflicts. --- README.md | 26 ++++++++- src/artifactapi/config.py | 87 ++++++++++++++++++++++----- tests/test_config.py | 120 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 217 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0e28684..590d9ce 100644 --- a/README.md +++ b/README.md @@ -79,12 +79,13 @@ src/artifactapi/ ## Configuration -Runtime settings come from environment variables; remote definitions live in `remotes.yaml`. +Runtime settings come from environment variables; remote definitions live in one or more YAML files pointed to by `CONFIG_PATH`. ### Environment Variables | Variable | Description | |---|---| +| `CONFIG_PATH` | Path to a config YAML file **or** a directory of YAML files | | `DBHOST`, `DBPORT`, `DBUSER`, `DBPASS`, `DBNAME` | PostgreSQL connection | | `REDIS_URL` | Redis URL (e.g. `redis://localhost:6379`) | | `MINIO_ENDPOINT` | MinIO/S3 endpoint | @@ -93,6 +94,29 @@ Runtime settings come from environment variables; remote definitions live in `re | `MINIO_BUCKET` | S3 bucket name | | `MINIO_SECURE` | Use HTTPS (`true`/`false`) | +### Split configuration + +`CONFIG_PATH` accepts three forms: + +**Single file** (original behaviour): +``` +CONFIG_PATH=/etc/artifactapi/remotes.yaml +``` + +**Directory** — all `*.yaml` / `*.yml` files in the directory are loaded and merged alphabetically. `remotes` keys are merged across files; later files win on conflict: +``` +CONFIG_PATH=/etc/artifactapi/conf.d/ +``` + +**Main file + `config_dir`** — the main file holds global settings and a `config_dir` pointer; each file in that directory contributes its own `remotes`. Relative `config_dir` paths are resolved relative to the main file: +```yaml +# /etc/artifactapi/config.yaml +config_dir: conf.d # or an absolute path + +# s3/redis/database settings go here (or in env vars) +remotes: {} # optional base remotes +``` + ### remotes.yaml Structure ```yaml diff --git a/src/artifactapi/config.py b/src/artifactapi/config.py index 3c83f27..6de80e4 100644 --- a/src/artifactapi/config.py +++ b/src/artifactapi/config.py @@ -1,3 +1,4 @@ +import glob import json import os @@ -30,31 +31,87 @@ _PACKAGE_MUTABLE_PATTERNS: dict[str, list[str]] = { class ConfigManager: - def __init__(self, config_file: str = "remotes.yaml"): - self.config_file = config_file - self._last_modified = 0 + def __init__(self, config_path: str = "remotes.yaml"): + self.config_path = config_path + self._config_dir: str | None = None + self._last_modified: float = 0.0 self.config = self._load_config() - def _load_config(self) -> dict: + def _load_single_file(self, path: str) -> dict: try: - with open(self.config_file) as f: - if self.config_file.endswith(".yaml") or self.config_file.endswith(".yml"): - return yaml.safe_load(f) - else: - return json.load(f) + with open(path) as f: + if path.endswith((".yaml", ".yml")): + return yaml.safe_load(f) or {} + return json.load(f) except FileNotFoundError: + return {} + + @staticmethod + def _merge(base: dict, overlay: dict) -> dict: + result = {**base} + for key, value in overlay.items(): + if key == "remotes" and isinstance(base.get("remotes"), dict) and isinstance(value, dict): + result["remotes"] = {**base.get("remotes", {}), **value} + else: + result[key] = value + return result + + def _load_from_dir(self, dir_path: str) -> dict: + merged: dict = {} + files = sorted(glob.glob(os.path.join(dir_path, "*.yaml")) + glob.glob(os.path.join(dir_path, "*.yml"))) + for path in files: + merged = self._merge(merged, self._load_single_file(path)) + return merged + + def _load_config(self) -> dict: + self._config_dir = None + + if os.path.isdir(self.config_path): + return self._load_from_dir(self.config_path) or {"remotes": {}} + + config = self._load_single_file(self.config_path) + if not config: return {"remotes": {}} - def _check_reload(self) -> None: - """Check if config file has been modified and reload if needed""" - try: - import os + config_dir = config.pop("config_dir", None) + if config_dir: + if not os.path.isabs(config_dir): + config_dir = os.path.join(os.path.dirname(os.path.abspath(self.config_path)), config_dir) + self._config_dir = config_dir + config = self._merge(config, self._load_from_dir(config_dir)) - current_modified = os.path.getmtime(self.config_file) + return config + + def _file_mtimes(self) -> list[float]: + mtimes: list[float] = [] + if os.path.isdir(self.config_path): + for f in glob.glob(os.path.join(self.config_path, "*.yaml")) + glob.glob(os.path.join(self.config_path, "*.yml")): + try: + mtimes.append(os.path.getmtime(f)) + except OSError: + pass + else: + try: + mtimes.append(os.path.getmtime(self.config_path)) + except OSError: + pass + + if self._config_dir and os.path.isdir(self._config_dir): + for f in glob.glob(os.path.join(self._config_dir, "*.yaml")) + glob.glob(os.path.join(self._config_dir, "*.yml")): + try: + mtimes.append(os.path.getmtime(f)) + except OSError: + pass + + return mtimes + + def _check_reload(self) -> None: + try: + current_modified = max(self._file_mtimes(), default=0.0) if current_modified > self._last_modified: self._last_modified = current_modified self.config = self._load_config() - print(f"Config reloaded from {self.config_file}") + print(f"Config reloaded from {self.config_path}") except OSError: pass diff --git a/tests/test_config.py b/tests/test_config.py index 344627c..a042086 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -418,3 +418,123 @@ class TestGetQuarantineConfig: enabled, days = cfg.get_quarantine_config("r") assert enabled is True assert days == 0 + + +# --------------------------------------------------------------------------- +# Directory mode (CONFIG_PATH points to a directory) +# --------------------------------------------------------------------------- + + +def _remote(base_url: str = "https://x.com") -> dict: + return {"type": "remote", "package": "generic", "base_url": base_url} + + +class TestConfigDirMode: + def test_loads_all_yaml_files(self, tmp_path): + (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}})) + (tmp_path / "b.yaml").write_text(yaml.dump({"remotes": {"repo-b": _remote("https://y.com")}})) + cfg = ConfigManager(str(tmp_path)) + assert "repo-a" in cfg.config["remotes"] + assert "repo-b" in cfg.config["remotes"] + + def test_later_file_overrides_earlier_on_same_key(self, tmp_path): + (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"r": _remote("https://first.com")}})) + (tmp_path / "b.yaml").write_text(yaml.dump({"remotes": {"r": _remote("https://second.com")}})) + cfg = ConfigManager(str(tmp_path)) + assert cfg.config["remotes"]["r"]["base_url"] == "https://second.com" + + def test_empty_directory_returns_empty_remotes(self, tmp_path): + cfg = ConfigManager(str(tmp_path)) + assert cfg.config == {"remotes": {}} + + def test_ignores_non_yaml_files(self, tmp_path): + (tmp_path / "notes.txt").write_text("not yaml") + (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}})) + cfg = ConfigManager(str(tmp_path)) + assert list(cfg.config["remotes"].keys()) == ["repo-a"] + + def test_reload_picks_up_new_file(self, tmp_path): + (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}})) + cfg = ConfigManager(str(tmp_path)) + assert "repo-a" in cfg.config["remotes"] + assert "repo-b" not in cfg.config["remotes"] + + new_file = tmp_path / "b.yaml" + new_file.write_text(yaml.dump({"remotes": {"repo-b": _remote("https://y.com")}})) + future_mtime = cfg._last_modified + 1 + os.utime(str(new_file), (future_mtime, future_mtime)) + + cfg._check_reload() + + assert "repo-a" in cfg.config["remotes"] + assert "repo-b" in cfg.config["remotes"] + + +# --------------------------------------------------------------------------- +# config_dir key (main file contains a config_dir pointer) +# --------------------------------------------------------------------------- + + +class TestConfigDirKey: + def test_merges_remotes_from_config_dir(self, tmp_path): + conf_d = tmp_path / "conf.d" + conf_d.mkdir() + (conf_d / "remotes.yaml").write_text(yaml.dump({"remotes": {"repo-extra": _remote("https://extra.com")}})) + main = tmp_path / "config.yaml" + main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {"repo-main": _remote()}})) + cfg = ConfigManager(str(main)) + assert "repo-main" in cfg.config["remotes"] + assert "repo-extra" in cfg.config["remotes"] + + def test_relative_config_dir_resolved_from_main_file(self, tmp_path): + conf_d = tmp_path / "conf.d" + conf_d.mkdir() + (conf_d / "r.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}})) + main = tmp_path / "config.yaml" + main.write_text(yaml.dump({"config_dir": "conf.d", "remotes": {}})) + cfg = ConfigManager(str(main)) + assert "repo-a" in cfg.config["remotes"] + + def test_config_dir_key_not_present_in_loaded_config(self, tmp_path): + conf_d = tmp_path / "conf.d" + conf_d.mkdir() + main = tmp_path / "config.yaml" + main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {}})) + cfg = ConfigManager(str(main)) + assert "config_dir" not in cfg.config + + def test_dir_remote_overrides_main_file_remote(self, tmp_path): + conf_d = tmp_path / "conf.d" + conf_d.mkdir() + (conf_d / "override.yaml").write_text(yaml.dump({"remotes": {"r": _remote("https://new.com")}})) + main = tmp_path / "config.yaml" + main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {"r": _remote("https://old.com")}})) + cfg = ConfigManager(str(main)) + assert cfg.config["remotes"]["r"]["base_url"] == "https://new.com" + + def test_empty_config_dir_uses_main_file_only(self, tmp_path): + conf_d = tmp_path / "conf.d" + conf_d.mkdir() + main = tmp_path / "config.yaml" + main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {"repo-main": _remote()}})) + cfg = ConfigManager(str(main)) + assert list(cfg.config["remotes"].keys()) == ["repo-main"] + + def test_reload_picks_up_changed_dir_file(self, tmp_path): + conf_d = tmp_path / "conf.d" + conf_d.mkdir() + dir_file = conf_d / "r.yaml" + dir_file.write_text(yaml.dump({"remotes": {"repo-v1": _remote()}})) + main = tmp_path / "config.yaml" + main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {}})) + cfg = ConfigManager(str(main)) + assert "repo-v1" in cfg.config["remotes"] + + dir_file.write_text(yaml.dump({"remotes": {"repo-v2": _remote("https://v2.com")}})) + future_mtime = cfg._last_modified + 1 + os.utime(str(dir_file), (future_mtime, future_mtime)) + + cfg._check_reload() + + assert "repo-v2" in cfg.config["remotes"] + assert "repo-v1" not in cfg.config["remotes"] From 3f098df428769dc2ab73439705a33fbf213265a5 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 28 Apr 2026 23:29:41 +1000 Subject: [PATCH 2/2] chore: add conf.d example split-config files Three example files (alpine, github, pypi) demonstrating per-remote YAML files for the conf.d directory mode. --- conf.d/alpine.yaml | 11 +++++++++++ conf.d/github.yaml | 12 ++++++++++++ conf.d/pypi.yaml | 17 +++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 conf.d/alpine.yaml create mode 100644 conf.d/github.yaml create mode 100644 conf.d/pypi.yaml diff --git a/conf.d/alpine.yaml b/conf.d/alpine.yaml new file mode 100644 index 0000000..a8e12f7 --- /dev/null +++ b/conf.d/alpine.yaml @@ -0,0 +1,11 @@ +remotes: + alpine: + base_url: "https://dl-cdn.alpinelinux.org" + type: "remote" + package: "alpine" + description: "Alpine Linux APK package repository" + immutable_patterns: + - ".*/x86_64/.*\\.apk$" + cache: + immutable_ttl: 0 + mutable_ttl: 7200 diff --git a/conf.d/github.yaml b/conf.d/github.yaml new file mode 100644 index 0000000..51aa20a --- /dev/null +++ b/conf.d/github.yaml @@ -0,0 +1,12 @@ +remotes: + github: + base_url: "https://github.com" + type: "remote" + package: "generic" + description: "GitHub releases and files" + immutable_patterns: + - "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*" + - "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$" + cache: + immutable_ttl: 0 + mutable_ttl: 0 diff --git a/conf.d/pypi.yaml b/conf.d/pypi.yaml new file mode 100644 index 0000000..4de0058 --- /dev/null +++ b/conf.d/pypi.yaml @@ -0,0 +1,17 @@ +remotes: + pypi: + base_url: "https://files.pythonhosted.org" + type: "remote" + package: "pypi" + description: "Python Package Index" + check_mutable_updates: true + quarantine_new: true + quarantine_days: 3 + immutable_patterns: + - "packages/.*\\.whl$" + - "packages/.*\\.whl\\.metadata$" + - "packages/.*\\.tar\\.gz$" + - "packages/.*\\.zip$" + cache: + immutable_ttl: 0 + mutable_ttl: 600