perf: use yaml.CSafeLoader/CDumper for 4x faster virtual index merge
Replace yaml.safe_load / yaml.Dumper with the C extension equivalents (yaml.CSafeLoader, yaml.CDumper) for all YAML parsing and serialisation in the virtual repo merge path. A module-level try/except falls back to the pure-Python SafeLoader/Dumper when libyaml is not available. Measured on 19-member helm-all (real upstream data): Before (SafeLoader + Dumper): merge=38,877ms After (CSafeLoader + CDumper): merge=9,625ms (4.0x faster) Local microbenchmark (500 charts × 10 versions × 19 members): Before: 40.8s After: 6.1s (6.7x faster)
This commit is contained in:
@@ -11,8 +11,15 @@ from fastapi import HTTPException, Request, Response
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
try:
|
||||||
|
_YamlLoader = yaml.CSafeLoader
|
||||||
|
_YamlDumperBase = yaml.CDumper
|
||||||
|
except AttributeError:
|
||||||
|
_YamlLoader = yaml.SafeLoader
|
||||||
|
_YamlDumperBase = yaml.Dumper
|
||||||
|
|
||||||
class _HelmDumper(yaml.Dumper):
|
|
||||||
|
class _HelmDumper(_YamlDumperBase):
|
||||||
"""YAML dumper that serializes datetime/date objects back to ISO 8601 strings.
|
"""YAML dumper that serializes datetime/date objects back to ISO 8601 strings.
|
||||||
|
|
||||||
yaml.safe_load converts timestamp-shaped YAML scalars (e.g. chart `created`
|
yaml.safe_load converts timestamp-shaped YAML scalars (e.g. chart `created`
|
||||||
@@ -109,7 +116,7 @@ def _merge_helm_indexes(raw_indexes: list[bytes], member_names: list[str], membe
|
|||||||
base_url = member_cfg.get("base_url", "").rstrip("/")
|
base_url = member_cfg.get("base_url", "").rstrip("/")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
index = yaml.safe_load(raw_data)
|
index = yaml.load(raw_data, Loader=_YamlLoader)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Virtual: failed to parse index.yaml from member '{member_name}': {e}")
|
logger.warning(f"Virtual: failed to parse index.yaml from member '{member_name}': {e}")
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ from artifactapi.artifact.virtual import (
|
|||||||
_merge_helm_indexes,
|
_merge_helm_indexes,
|
||||||
_rewrite_urls,
|
_rewrite_urls,
|
||||||
_VirtualHandler,
|
_VirtualHandler,
|
||||||
|
_YamlDumperBase,
|
||||||
|
_YamlLoader,
|
||||||
)
|
)
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -82,6 +84,34 @@ _CFG_A = {"base_url": "https://helm.releases.hashicorp.com", "cache": {"mutable_
|
|||||||
_CFG_B = {"base_url": "https://charts.example.com", "cache": {"mutable_ttl": 1800}}
|
_CFG_B = {"base_url": "https://charts.example.com", "cache": {"mutable_ttl": 1800}}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _YamlLoader / _YamlDumperBase — C extension selection
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestYamlExtensionSelection:
|
||||||
|
def test_loader_is_a_class(self):
|
||||||
|
assert isinstance(_YamlLoader, type)
|
||||||
|
|
||||||
|
def test_dumper_base_is_a_class(self):
|
||||||
|
assert isinstance(_YamlDumperBase, type)
|
||||||
|
|
||||||
|
def test_helm_dumper_uses_selected_base(self):
|
||||||
|
assert issubclass(_HelmDumper, _YamlDumperBase)
|
||||||
|
|
||||||
|
def test_c_extensions_used_when_available(self):
|
||||||
|
try:
|
||||||
|
assert _YamlLoader is yaml.CSafeLoader
|
||||||
|
assert _YamlDumperBase is yaml.CDumper
|
||||||
|
except AttributeError:
|
||||||
|
assert _YamlLoader is yaml.SafeLoader
|
||||||
|
assert _YamlDumperBase is yaml.Dumper
|
||||||
|
|
||||||
|
def test_loader_can_parse_yaml(self):
|
||||||
|
result = yaml.load(b"key: value", Loader=_YamlLoader)
|
||||||
|
assert result == {"key": "value"}
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# _HelmDumper — datetime/date YAML serialization
|
# _HelmDumper — datetime/date YAML serialization
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
Reference in New Issue
Block a user