Merge pull request 'feat: support config.d directory for split configuration (closes #20)' (#26) from benvin/issue-20-config-dir-split into master

Reviewed-on: #26

README.md

@@ -79,12 +79,13 @@ src/artifactapi/
 
 ## Configuration
 
-Runtime settings come from environment variables; remote definitions live in `remotes.yaml`.
+Runtime settings come from environment variables; remote definitions live in one or more YAML files pointed to by `CONFIG_PATH`.
 
 ### Environment Variables
 
 | Variable | Description |
 |---|---|
+| `CONFIG_PATH` | Path to a config YAML file **or** a directory of YAML files |
 | `DBHOST`, `DBPORT`, `DBUSER`, `DBPASS`, `DBNAME` | PostgreSQL connection |
 | `REDIS_URL` | Redis URL (e.g. `redis://localhost:6379`) |
 | `MINIO_ENDPOINT` | MinIO/S3 endpoint |
@@ -93,6 +94,29 @@ Runtime settings come from environment variables; remote definitions live in `re
 | `MINIO_BUCKET` | S3 bucket name |
 | `MINIO_SECURE` | Use HTTPS (`true`/`false`) |
 
+### Split configuration
+
+`CONFIG_PATH` accepts three forms:
+
+**Single file** (original behaviour):
+```
+CONFIG_PATH=/etc/artifactapi/remotes.yaml
+```
+
+**Directory** — all `*.yaml` / `*.yml` files in the directory are loaded and merged alphabetically. `remotes` keys are merged across files; later files win on conflict:
+```
+CONFIG_PATH=/etc/artifactapi/conf.d/
+```
+
+**Main file + `config_dir`** — the main file holds global settings and a `config_dir` pointer; each file in that directory contributes its own `remotes`. Relative `config_dir` paths are resolved relative to the main file:
+```yaml
+# /etc/artifactapi/config.yaml
+config_dir: conf.d   # or an absolute path
+
+# s3/redis/database settings go here (or in env vars)
+remotes: {}          # optional base remotes
+```
+
 ### remotes.yaml Structure
 
 ```yaml

conf.d/alpine.yaml

@@ -0,0 +1,11 @@
+remotes:
+  alpine:
+    base_url: "https://dl-cdn.alpinelinux.org"
+    type: "remote"
+    package: "alpine"
+    description: "Alpine Linux APK package repository"
+    immutable_patterns:
+      - ".*/x86_64/.*\\.apk$"
+    cache:
+      immutable_ttl: 0
+      mutable_ttl: 7200

conf.d/github.yaml

@@ -0,0 +1,12 @@
+remotes:
+  github:
+    base_url: "https://github.com"
+    type: "remote"
+    package: "generic"
+    description: "GitHub releases and files"
+    immutable_patterns:
+      - "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
+      - "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
+    cache:
+      immutable_ttl: 0
+      mutable_ttl: 0

conf.d/pypi.yaml

@@ -0,0 +1,17 @@
+remotes:
+  pypi:
+    base_url: "https://files.pythonhosted.org"
+    type: "remote"
+    package: "pypi"
+    description: "Python Package Index"
+    check_mutable_updates: true
+    quarantine_new: true
+    quarantine_days: 3
+    immutable_patterns:
+      - "packages/.*\\.whl$"
+      - "packages/.*\\.whl\\.metadata$"
+      - "packages/.*\\.tar\\.gz$"
+      - "packages/.*\\.zip$"
+    cache:
+      immutable_ttl: 0
+      mutable_ttl: 600

src/artifactapi/config.py

@@ -1,3 +1,4 @@
+import glob
 import json
 import os
 
@@ -30,31 +31,87 @@ _PACKAGE_MUTABLE_PATTERNS: dict[str, list[str]] = {
 
 
 class ConfigManager:
-    def __init__(self, config_file: str = "remotes.yaml"):
-        self.config_file = config_file
-        self._last_modified = 0
+    def __init__(self, config_path: str = "remotes.yaml"):
+        self.config_path = config_path
+        self._config_dir: str | None = None
+        self._last_modified: float = 0.0
         self.config = self._load_config()
 
-    def _load_config(self) -> dict:
+    def _load_single_file(self, path: str) -> dict:
         try:
-            with open(self.config_file) as f:
-                if self.config_file.endswith(".yaml") or self.config_file.endswith(".yml"):
-                    return yaml.safe_load(f)
-                else:
-                    return json.load(f)
+            with open(path) as f:
+                if path.endswith((".yaml", ".yml")):
+                    return yaml.safe_load(f) or {}
+                return json.load(f)
         except FileNotFoundError:
+            return {}
+
+    @staticmethod
+    def _merge(base: dict, overlay: dict) -> dict:
+        result = {**base}
+        for key, value in overlay.items():
+            if key == "remotes" and isinstance(base.get("remotes"), dict) and isinstance(value, dict):
+                result["remotes"] = {**base.get("remotes", {}), **value}
+            else:
+                result[key] = value
+        return result
+
+    def _load_from_dir(self, dir_path: str) -> dict:
+        merged: dict = {}
+        files = sorted(glob.glob(os.path.join(dir_path, "*.yaml")) + glob.glob(os.path.join(dir_path, "*.yml")))
+        for path in files:
+            merged = self._merge(merged, self._load_single_file(path))
+        return merged
+
+    def _load_config(self) -> dict:
+        self._config_dir = None
+
+        if os.path.isdir(self.config_path):
+            return self._load_from_dir(self.config_path) or {"remotes": {}}
+
+        config = self._load_single_file(self.config_path)
+        if not config:
             return {"remotes": {}}
 
-    def _check_reload(self) -> None:
-        """Check if config file has been modified and reload if needed"""
-        try:
-            import os
+        config_dir = config.pop("config_dir", None)
+        if config_dir:
+            if not os.path.isabs(config_dir):
+                config_dir = os.path.join(os.path.dirname(os.path.abspath(self.config_path)), config_dir)
+            self._config_dir = config_dir
+            config = self._merge(config, self._load_from_dir(config_dir))
 
-            current_modified = os.path.getmtime(self.config_file)
+        return config
+
+    def _file_mtimes(self) -> list[float]:
+        mtimes: list[float] = []
+        if os.path.isdir(self.config_path):
+            for f in glob.glob(os.path.join(self.config_path, "*.yaml")) + glob.glob(os.path.join(self.config_path, "*.yml")):
+                try:
+                    mtimes.append(os.path.getmtime(f))
+                except OSError:
+                    pass
+        else:
+            try:
+                mtimes.append(os.path.getmtime(self.config_path))
+            except OSError:
+                pass
+
+        if self._config_dir and os.path.isdir(self._config_dir):
+            for f in glob.glob(os.path.join(self._config_dir, "*.yaml")) + glob.glob(os.path.join(self._config_dir, "*.yml")):
+                try:
+                    mtimes.append(os.path.getmtime(f))
+                except OSError:
+                    pass
+
+        return mtimes
+
+    def _check_reload(self) -> None:
+        try:
+            current_modified = max(self._file_mtimes(), default=0.0)
             if current_modified > self._last_modified:
                 self._last_modified = current_modified
                 self.config = self._load_config()
-                print(f"Config reloaded from {self.config_file}")
+                print(f"Config reloaded from {self.config_path}")
         except OSError:
             pass
 

tests/test_config.py

@@ -418,3 +418,123 @@ class TestGetQuarantineConfig:
         enabled, days = cfg.get_quarantine_config("r")
         assert enabled is True
         assert days == 0
+
+
+# ---------------------------------------------------------------------------
+# Directory mode (CONFIG_PATH points to a directory)
+# ---------------------------------------------------------------------------
+
+
+def _remote(base_url: str = "https://x.com") -> dict:
+    return {"type": "remote", "package": "generic", "base_url": base_url}
+
+
+class TestConfigDirMode:
+    def test_loads_all_yaml_files(self, tmp_path):
+        (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}}))
+        (tmp_path / "b.yaml").write_text(yaml.dump({"remotes": {"repo-b": _remote("https://y.com")}}))
+        cfg = ConfigManager(str(tmp_path))
+        assert "repo-a" in cfg.config["remotes"]
+        assert "repo-b" in cfg.config["remotes"]
+
+    def test_later_file_overrides_earlier_on_same_key(self, tmp_path):
+        (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"r": _remote("https://first.com")}}))
+        (tmp_path / "b.yaml").write_text(yaml.dump({"remotes": {"r": _remote("https://second.com")}}))
+        cfg = ConfigManager(str(tmp_path))
+        assert cfg.config["remotes"]["r"]["base_url"] == "https://second.com"
+
+    def test_empty_directory_returns_empty_remotes(self, tmp_path):
+        cfg = ConfigManager(str(tmp_path))
+        assert cfg.config == {"remotes": {}}
+
+    def test_ignores_non_yaml_files(self, tmp_path):
+        (tmp_path / "notes.txt").write_text("not yaml")
+        (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}}))
+        cfg = ConfigManager(str(tmp_path))
+        assert list(cfg.config["remotes"].keys()) == ["repo-a"]
+
+    def test_reload_picks_up_new_file(self, tmp_path):
+        (tmp_path / "a.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}}))
+        cfg = ConfigManager(str(tmp_path))
+        assert "repo-a" in cfg.config["remotes"]
+        assert "repo-b" not in cfg.config["remotes"]
+
+        new_file = tmp_path / "b.yaml"
+        new_file.write_text(yaml.dump({"remotes": {"repo-b": _remote("https://y.com")}}))
+        future_mtime = cfg._last_modified + 1
+        os.utime(str(new_file), (future_mtime, future_mtime))
+
+        cfg._check_reload()
+
+        assert "repo-a" in cfg.config["remotes"]
+        assert "repo-b" in cfg.config["remotes"]
+
+
+# ---------------------------------------------------------------------------
+# config_dir key (main file contains a config_dir pointer)
+# ---------------------------------------------------------------------------
+
+
+class TestConfigDirKey:
+    def test_merges_remotes_from_config_dir(self, tmp_path):
+        conf_d = tmp_path / "conf.d"
+        conf_d.mkdir()
+        (conf_d / "remotes.yaml").write_text(yaml.dump({"remotes": {"repo-extra": _remote("https://extra.com")}}))
+        main = tmp_path / "config.yaml"
+        main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {"repo-main": _remote()}}))
+        cfg = ConfigManager(str(main))
+        assert "repo-main" in cfg.config["remotes"]
+        assert "repo-extra" in cfg.config["remotes"]
+
+    def test_relative_config_dir_resolved_from_main_file(self, tmp_path):
+        conf_d = tmp_path / "conf.d"
+        conf_d.mkdir()
+        (conf_d / "r.yaml").write_text(yaml.dump({"remotes": {"repo-a": _remote()}}))
+        main = tmp_path / "config.yaml"
+        main.write_text(yaml.dump({"config_dir": "conf.d", "remotes": {}}))
+        cfg = ConfigManager(str(main))
+        assert "repo-a" in cfg.config["remotes"]
+
+    def test_config_dir_key_not_present_in_loaded_config(self, tmp_path):
+        conf_d = tmp_path / "conf.d"
+        conf_d.mkdir()
+        main = tmp_path / "config.yaml"
+        main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {}}))
+        cfg = ConfigManager(str(main))
+        assert "config_dir" not in cfg.config
+
+    def test_dir_remote_overrides_main_file_remote(self, tmp_path):
+        conf_d = tmp_path / "conf.d"
+        conf_d.mkdir()
+        (conf_d / "override.yaml").write_text(yaml.dump({"remotes": {"r": _remote("https://new.com")}}))
+        main = tmp_path / "config.yaml"
+        main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {"r": _remote("https://old.com")}}))
+        cfg = ConfigManager(str(main))
+        assert cfg.config["remotes"]["r"]["base_url"] == "https://new.com"
+
+    def test_empty_config_dir_uses_main_file_only(self, tmp_path):
+        conf_d = tmp_path / "conf.d"
+        conf_d.mkdir()
+        main = tmp_path / "config.yaml"
+        main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {"repo-main": _remote()}}))
+        cfg = ConfigManager(str(main))
+        assert list(cfg.config["remotes"].keys()) == ["repo-main"]
+
+    def test_reload_picks_up_changed_dir_file(self, tmp_path):
+        conf_d = tmp_path / "conf.d"
+        conf_d.mkdir()
+        dir_file = conf_d / "r.yaml"
+        dir_file.write_text(yaml.dump({"remotes": {"repo-v1": _remote()}}))
+        main = tmp_path / "config.yaml"
+        main.write_text(yaml.dump({"config_dir": str(conf_d), "remotes": {}}))
+        cfg = ConfigManager(str(main))
+        assert "repo-v1" in cfg.config["remotes"]
+
+        dir_file.write_text(yaml.dump({"remotes": {"repo-v2": _remote("https://v2.com")}}))
+        future_mtime = cfg._last_modified + 1
+        os.utime(str(dir_file), (future_mtime, future_mtime))
+
+        cfg._check_reload()
+
+        assert "repo-v2" in cfg.config["remotes"]
+        assert "repo-v1" not in cfg.config["remotes"]