init: api-gateway v2.4.0(Agnes/b.ai/火山方舟 三家供应商网关)

This commit is contained in:
api-gateway committed 2026-09-15 17:05:46 +08:00
commit f136c9ee81
91 files changed
+24653

No files matched your search

+205
View File
@@ -0,0 +1,205 @@
"""供应商级熔断器(Circuit Breaker)。
状态机:CLOSED → OPEN(连续失败达到 threshold)→ HALF_OPEN(cooldown 到期)
→ 试探一次:成功回 CLOSED,失败回 OPEN。
目的:某供应商持续 502/超时期间,直接跳过白等 60s,让 fallback 快速生效。
"""
from __future__ import annotations
import logging
import threading
import time
from collections import deque
log = logging.getLogger("api-gateway.circuit")
class CircuitBreaker:
"""单供应商熔断器。线程安全。
滑动窗口按**次数**生效(最近 window 次调用里失败 ≥ failure_threshold 即熔断),
而非写死时间窗口——window 配置项真正起作用。
"""
CLOSED = "closed"
OPEN = "open"
HALF_OPEN = "half_open"
def __init__(self, provider: str, failure_threshold: int = 3, cooldown: float = 30.0, window: int = 10):
self.provider = provider
self.failure_threshold = max(1, failure_threshold)
self.cooldown = cooldown # 秒
self.window = max(1, window) # 最近 N 次调用作为判定窗口
self._lock = threading.Lock()
self._state: str = self.CLOSED
self._recent: deque[bool] = deque(maxlen=self.window) # 最近 N 次结果(按次数)
self._opened_at: float = 0.0
self._probing: bool = False # HALF_OPEN 期间是否已有试探请求在途
self._probe_at: float = 0.0 # 试探请求发出的时刻(用于探测超时兜底)
# 探测超时:HALF_OPEN 的试探请求可能**永远不回报**(客户端中途断开时
# CancelledError/GeneratorExit 不会走到 record()),若不兜底,breaker 会
# 永久停在 HALF_OPEN 并一直拒绝该供应商,直到重启或改配置。
self.probe_timeout: float = max(30.0, cooldown * 2)
def _probe_expired(self) -> bool:
return self._probing and (time.time() - self._probe_at) > self.probe_timeout
# ── 对外 API ──
def allow(self) -> bool:
"""当前是否允许调用该供应商。"""
with self._lock:
if self._state == self.CLOSED:
return True
if self._state == self.OPEN:
if time.time() - self._opened_at >= self.cooldown:
self._state = self.HALF_OPEN
self._probing = True # 本次调用即唯一试探
self._probe_at = time.time()
log.warning("[%s] 熔断 HALF_OPEN(试探)", self.provider)
return True
return False
# HALF_OPEN:只放行一个试探请求(其余拒绝,避免并发试探打穿下游)
if self._probing:
if self._probe_expired():
# 试探请求超时未回报 → 认定这次探测失败,回 OPEN 等下一轮冷却,
# 避免 breaker 被卡死在 HALF_OPEN(永久拒绝所有请求)。
self._state = self.OPEN
self._opened_at = time.time()
self._probing = False
log.warning("[%s] 试探请求 %.0fs 未回报,判定失败回 OPEN %.0fs",
self.provider, self.probe_timeout, self.cooldown)
return False
self._probing = True
self._probe_at = time.time()
return True
def release_probe(self) -> None:
"""归还 HALF_OPEN 试探令牌(调用方在 `allow()==True` 之后**没真正发请求**时调用)。
第六轮并发审计实测:拿到试探令牌后若在 record() 之前离开(限流跳过、凭证缺失
等),令牌永远不会归还 → 该供应商被半开状态**永久**拒绝。除了下面的探测超时兜底,
主动归还才是正解。
注意(第七轮回归审计):本方法不校验令牌归属,因此**只能由令牌持有者调用**,
且必须满足"`allow()==True` 与本调用之间没有 await"(否则可能把并发请求 B 在
HALF_OPEN 抢到的探测令牌误归还,导致双探)。当前全部调用点均满足该约束
(限流跳过分支、images 的 except 分支为同 tick;videos_create 的 except 分支内
不存在能抛 HTTPException 的源)。如需放宽,请改为 `allow()` 发放 probe token
并由 `release_probe(token)` 校验归属。
"""
with self._lock:
if self._state == self.HALF_OPEN and self._probing:
self._probing = False
self._probe_at = 0.0
def reconfigure(self, failure_threshold: int, cooldown: float, window: int) -> None:
"""热重载更新参数,保留当前状态与最近结果(Bug 4 修复)。"""
with self._lock:
self.failure_threshold = max(1, int(failure_threshold))
self.cooldown = cooldown
new_window = max(1, int(window))
if new_window != self.window:
self._recent = deque(self._recent, maxlen=new_window)
self.window = new_window
self.probe_timeout = max(30.0, cooldown * 2)
def reset(self) -> None:
"""强制复位到 CLOSED 并清空窗口(热重载禁用熔断时用,Bug 7)。
否则一个已 OPEN 的 breaker 即使熔断被关掉,仍会拒绝请求直到冷却结束。
"""
with self._lock:
self._state = self.CLOSED
self._recent.clear()
self._opened_at = 0.0
self._probing = False
self._probe_at = 0.0
def record(self, ok: bool) -> None:
"""记录一次调用结果(最近 window 次按次数滑动),驱动状态机。"""
now = time.time()
with self._lock:
self._recent.append(ok) # deque(maxlen=window) 自动淘汰最旧
recent_fails = sum(1 for okk in self._recent if not okk)
if self._state == self.HALF_OPEN:
self._probing = False
self._probe_at = 0.0
if ok:
self._state = self.CLOSED
self._recent.clear()
log.info("[%s] 熔断恢复 CLOSED", self.provider)
else:
self._state = self.OPEN
self._opened_at = now
log.warning("[%s] 试探失败,回 OPEN %.0fs", self.provider, self.cooldown)
elif self._state == self.CLOSED and not ok and recent_fails >= self.failure_threshold:
self._state = self.OPEN
self._opened_at = now
self._probing = False
log.warning("[%s] 最近 %d 次中失败 %d,达阈值 %d,熔断 OPEN %.0fs",
self.provider, len(self._recent), recent_fails, self.failure_threshold, self.cooldown)
def status(self) -> dict:
with self._lock:
recent_fails = sum(1 for okk in self._recent if not okk)
return {
"provider": self.provider,
"state": self._state,
"recent_failures": recent_fails,
"probing": self._probing,
"window_size": self.window,
"threshold": self.failure_threshold,
"cooldown_seconds": self.cooldown,
}
class CircuitBreakerRegistry:
"""按供应商 key 管理多个熔断器。"""
def __init__(self, default_config: dict | None = None):
self._lock = threading.Lock()
self._breakers: dict[str, CircuitBreaker] = {}
self._cfg = default_config or {}
def _defaults(self, provider: str) -> CircuitBreaker:
enabled = self._cfg.get("enabled", False)
if not enabled:
return CircuitBreaker(provider, failure_threshold=10**6, cooldown=3600)
return CircuitBreaker(
provider,
failure_threshold=self._cfg.get("failure_threshold", 3),
cooldown=self._cfg.get("cooldown", 30),
window=self._cfg.get("window", 10),
)
def get(self, provider: str) -> CircuitBreaker:
with self._lock:
if provider not in self._breakers:
self._breakers[provider] = self._defaults(provider)
return self._breakers[provider]
def update_config(self, cfg: dict) -> None:
"""热重载更新熔断参数(Bug 4 修复):就地更新已有熔断器,**保留其状态**。
之前热重载会整个重建 registry → 已熔断的坏供应商被立刻恢复重试。
"""
with self._lock:
self._cfg = dict(cfg)
enabled = self._cfg.get("enabled", False)
for cb in self._breakers.values():
if not enabled:
cb.reset() # Bug 7: 关掉熔断时复位已 OPEN 的 breaker,立刻恢复放行
cb.reconfigure(10**6, cb.cooldown, cb.window) # 等效禁用
else:
cb.reconfigure(
self._cfg.get("failure_threshold", 3),
self._cfg.get("cooldown", 30),
self._cfg.get("window", 10),
)
def statuses(self) -> list[dict]:
with self._lock:
return [b.status() for b in self._breakers.values()]