Compare commits
No commits in common. "master" and "benvin/index_caching" have entirely different histories.
master
...
benvin/ind
@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "artifactapi"
|
name = "artifactapi"
|
||||||
version = "2.0.2"
|
version = "2.0.1"
|
||||||
description = "Generic artifact caching system with support for various package managers"
|
description = "Generic artifact caching system with support for various package managers"
|
||||||
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
|||||||
@ -66,20 +66,8 @@ class RedisCache:
|
|||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Construct the URL the same way as in the main flow
|
# Get the S3 key and remove it
|
||||||
from .config import ConfigManager
|
s3_key = storage.get_object_key_from_path(remote_name, path)
|
||||||
import os
|
|
||||||
config_path = os.environ.get("CONFIG_PATH")
|
|
||||||
if config_path:
|
|
||||||
config = ConfigManager(config_path)
|
|
||||||
remote_config = config.get_remote_config(remote_name)
|
|
||||||
if remote_config:
|
|
||||||
base_url = remote_config.get("base_url")
|
|
||||||
if base_url:
|
|
||||||
# Construct URL the same way as construct_remote_url
|
|
||||||
remote_url = f"{base_url.rstrip('/')}/{path}"
|
|
||||||
# Use URL-based key (same as cache_single_artifact)
|
|
||||||
s3_key = storage.get_object_key(remote_url)
|
|
||||||
if storage.exists(s3_key):
|
if storage.exists(s3_key):
|
||||||
storage.client.delete_object(Bucket=storage.bucket, Key=s3_key)
|
storage.client.delete_object(Bucket=storage.bucket, Key=s3_key)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@ -28,7 +28,7 @@ logging.basicConfig(
|
|||||||
)
|
)
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
app = FastAPI(title="Artifact Storage API", version="2.0.2")
|
app = FastAPI(title="Artifact Storage API", version="2.0.1")
|
||||||
|
|
||||||
# Initialize components using config
|
# Initialize components using config
|
||||||
config_path = os.environ.get("CONFIG_PATH")
|
config_path = os.environ.get("CONFIG_PATH")
|
||||||
@ -63,81 +63,6 @@ def health_check():
|
|||||||
return {"status": "healthy"}
|
return {"status": "healthy"}
|
||||||
|
|
||||||
|
|
||||||
@app.put("/cache/flush")
|
|
||||||
def flush_cache(
|
|
||||||
remote: str = Query(default=None, description="Specific remote to flush (optional)"),
|
|
||||||
cache_type: str = Query(default="all", description="Type to flush: 'all', 'index', 'files', 'metrics'")
|
|
||||||
):
|
|
||||||
"""Flush cache entries for specified remote or all remotes"""
|
|
||||||
try:
|
|
||||||
result = {
|
|
||||||
"remote": remote,
|
|
||||||
"cache_type": cache_type,
|
|
||||||
"flushed": {
|
|
||||||
"redis_keys": 0,
|
|
||||||
"s3_objects": 0,
|
|
||||||
"operations": []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Flush Redis entries based on cache_type
|
|
||||||
if cache_type in ["all", "index", "metrics"] and cache.available and cache.client:
|
|
||||||
patterns = []
|
|
||||||
|
|
||||||
if cache_type in ["all", "index"]:
|
|
||||||
if remote:
|
|
||||||
patterns.append(f"index:{remote}:*")
|
|
||||||
else:
|
|
||||||
patterns.append("index:*")
|
|
||||||
|
|
||||||
if cache_type in ["all", "metrics"]:
|
|
||||||
if remote:
|
|
||||||
patterns.append(f"metrics:*:{remote}")
|
|
||||||
else:
|
|
||||||
patterns.append("metrics:*")
|
|
||||||
|
|
||||||
for pattern in patterns:
|
|
||||||
keys = cache.client.keys(pattern)
|
|
||||||
if keys:
|
|
||||||
cache.client.delete(*keys)
|
|
||||||
result["flushed"]["redis_keys"] += len(keys)
|
|
||||||
logger.info(f"Cache flush: Deleted {len(keys)} Redis keys matching '{pattern}'")
|
|
||||||
|
|
||||||
if result["flushed"]["redis_keys"] > 0:
|
|
||||||
result["flushed"]["operations"].append(f"Deleted {result['flushed']['redis_keys']} Redis keys")
|
|
||||||
|
|
||||||
# Flush S3 objects if requested
|
|
||||||
if cache_type in ["all", "files"]:
|
|
||||||
try:
|
|
||||||
response = storage.client.list_objects_v2(Bucket=storage.bucket)
|
|
||||||
if 'Contents' in response:
|
|
||||||
objects_to_delete = [obj['Key'] for obj in response['Contents']]
|
|
||||||
|
|
||||||
for key in objects_to_delete:
|
|
||||||
try:
|
|
||||||
storage.client.delete_object(Bucket=storage.bucket, Key=key)
|
|
||||||
result["flushed"]["s3_objects"] += 1
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"Failed to delete S3 object {key}: {e}")
|
|
||||||
|
|
||||||
if objects_to_delete:
|
|
||||||
result["flushed"]["operations"].append(f"Deleted {len(objects_to_delete)} S3 objects")
|
|
||||||
logger.info(f"Cache flush: Deleted {len(objects_to_delete)} S3 objects")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
result["flushed"]["operations"].append(f"S3 flush failed: {str(e)}")
|
|
||||||
logger.error(f"Cache flush S3 error: {e}")
|
|
||||||
|
|
||||||
if not result["flushed"]["operations"]:
|
|
||||||
result["flushed"]["operations"].append("No cache entries found to flush")
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Cache flush error: {e}")
|
|
||||||
raise HTTPException(status_code=500, detail=f"Cache flush failed: {str(e)}")
|
|
||||||
|
|
||||||
|
|
||||||
async def construct_remote_url(remote_name: str, path: str) -> str:
|
async def construct_remote_url(remote_name: str, path: str) -> str:
|
||||||
remote_config = config.get_remote_config(remote_name)
|
remote_config = config.get_remote_config(remote_name)
|
||||||
if not remote_config:
|
if not remote_config:
|
||||||
@ -279,7 +204,7 @@ async def get_artifact(remote_name: str, path: str):
|
|||||||
|
|
||||||
# For index files, check Redis TTL validity
|
# For index files, check Redis TTL validity
|
||||||
filename = os.path.basename(path)
|
filename = os.path.basename(path)
|
||||||
is_index = cache.is_index_file(path) # Check full path, not just filename
|
is_index = cache.is_index_file(filename)
|
||||||
|
|
||||||
if cached_key and is_index:
|
if cached_key and is_index:
|
||||||
# Index file exists, but check if it's still valid
|
# Index file exists, but check if it's still valid
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user