feat: add HTTP sessions and retry logic for Gitea/GitHub API calls
Creates two module-level requests.Session objects (_gitea_session, _github_session) with an HTTPAdapter backed by urllib3 Retry: - 3 retries with 0.5s exponential backoff - retries on 429, 500, 502, 503, 504 and connection errors - GET-only to avoid unsafe retries Benefits: - TCP connections are pooled and reused across all package checks, including concurrent builds in ThreadPoolExecutor - Transient network blips no longer cause spurious rebuilds - Per-request timeouts tightened to 10s (Gitea) / 15s (GitHub) so failures surface quickly and the retry budget is actually useful
This commit is contained in:
+45
-15
@@ -23,6 +23,8 @@ import sys
|
||||
import logging
|
||||
import subprocess
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
@@ -33,6 +35,36 @@ import yaml
|
||||
from cerberus import Validator
|
||||
|
||||
|
||||
# ==================== HTTP SESSIONS ====================
|
||||
|
||||
def _make_session(retries: int = 3, backoff_factor: float = 0.5) -> requests.Session:
|
||||
"""
|
||||
Build a requests Session that retries on transient failures.
|
||||
|
||||
Retries up to `retries` times on connection errors, read timeouts, and
|
||||
5xx / 429 responses, with exponential backoff (0s, 0.5s, 1s, 2s …).
|
||||
Only GET requests are retried to stay safe.
|
||||
"""
|
||||
session = requests.Session()
|
||||
retry = Retry(
|
||||
total=retries,
|
||||
backoff_factor=backoff_factor,
|
||||
status_forcelist=[429, 500, 502, 503, 504],
|
||||
allowed_methods=["GET"],
|
||||
raise_on_status=False,
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
return session
|
||||
|
||||
# Shared sessions — one per upstream so connections are pooled and reused
|
||||
# across all package checks (including parallel builds in ThreadPoolExecutor).
|
||||
# requests.Session is thread-safe for concurrent requests.
|
||||
_gitea_session = _make_session()
|
||||
_github_session = _make_session()
|
||||
|
||||
|
||||
# ==================== VALIDATION SCHEMA ====================
|
||||
|
||||
# Cerberus schema for metadata.yaml validation based on PackageMetadata and Build dataclasses
|
||||
@@ -274,15 +306,14 @@ def get_github_latest_release(repo: str) -> Optional[dict]:
|
||||
|
||||
try:
|
||||
github_token = get_github_token()
|
||||
_github_session.headers.update({
|
||||
'Authorization': f'token {github_token}',
|
||||
'Accept': 'application/vnd.github.v3+json',
|
||||
})
|
||||
|
||||
url = f"https://api.github.com/repos/{repo}/releases/latest"
|
||||
headers = {
|
||||
'Authorization': f'token {github_token}',
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
}
|
||||
|
||||
logger.debug(f"Checking GitHub releases: {url}")
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response = _github_session.get(url, timeout=15)
|
||||
|
||||
if response.status_code == 200:
|
||||
release = response.json()
|
||||
@@ -389,15 +420,14 @@ def get_github_releases_by_pattern(repo: str, pattern: str) -> Optional[dict]:
|
||||
|
||||
try:
|
||||
github_token = get_github_token()
|
||||
_github_session.headers.update({
|
||||
'Authorization': f'token {github_token}',
|
||||
'Accept': 'application/vnd.github.v3+json',
|
||||
})
|
||||
|
||||
url = f"https://api.github.com/repos/{repo}/releases"
|
||||
headers = {
|
||||
'Authorization': f'token {github_token}',
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
}
|
||||
|
||||
logger.debug(f"Checking GitHub releases with pattern '{pattern}': {url}")
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response = _github_session.get(url, timeout=15)
|
||||
|
||||
if response.status_code == 200:
|
||||
releases = response.json()
|
||||
@@ -558,10 +588,10 @@ def check_package_exists(package_name: str, version: str, release: str, distro:
|
||||
f"{package_type}/{package_name}/{full_version}"
|
||||
)
|
||||
|
||||
headers = {'Authorization': f'token {gitea_token}'}
|
||||
_gitea_session.headers.update({'Authorization': f'token {gitea_token}'})
|
||||
|
||||
logger.debug(f"Checking package existence: {url}")
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response = _gitea_session.get(url, timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
package_info = response.json()
|
||||
@@ -582,7 +612,7 @@ def check_package_exists(package_name: str, version: str, release: str, distro:
|
||||
|
||||
files_url = f"{base_url}/api/v1/packages/{owner}/{package_type}/{package_name}/{full_version}/files"
|
||||
logger.debug(f"Checking distro-specific files: {files_url}")
|
||||
files_response = requests.get(files_url, headers=headers, timeout=30)
|
||||
files_response = _gitea_session.get(files_url, timeout=10)
|
||||
|
||||
if files_response.status_code == 200:
|
||||
files = files_response.json()
|
||||
|
||||
Reference in New Issue
Block a user