feat(persist): 视频回调任务落库续跑 + /logs 改读库
- 新表 video_jobs:回调任务先落库再派生,结束标记 done/failed - 启动时 pending_jobs() 续跑(6 小时内),provider 不存在/超时标记 failed/expired - /logs 与 /logs/recent 共用 _recent_entries() 读库;删除遗留 _logs_for_account() - 版本 2.9.12 → 2.9.13
This commit is contained in:
1 parent
8e346161a6
commit
a9a200bf9c
3 files changed
+175
-17
No files matched your search
@@ -1,5 +1,36 @@
|
|||||||
# CHANGELOG
|
# CHANGELOG
|
||||||
|
|
||||||
|
## 2026-10-23 · 消灭最后两处内存态:视频回调任务落库 + /logs 改读库(2.9.12 → 2.9.13)
|
||||||
|
|
||||||
|
老板:「看下项目的数据还有什么数据是放在内存重启就没了的吗」→ 盘点后「都做」。
|
||||||
|
|
||||||
|
盘点结论:日志/账单/配置/会话/资产都已落盘,剩两处仍是内存态:
|
||||||
|
|
||||||
|
### ① 待回调的视频任务(唯一有业务影响的)
|
||||||
|
`callback_url` 异步通知原先只挂在 `_bg_tasks` 里 —— **进程重启 = 回调永远不会触发**。
|
||||||
|
|
||||||
|
- 新表 `video_jobs`:`video_id / provider / model / callback_url / rid / account / created /
|
||||||
|
status / wait_timeout / last_error / done_at`。
|
||||||
|
- **先落库再派生任务**;任务结束(成功/失败)`finish_job()` 标记。
|
||||||
|
- **启动续跑**:`lifespan` 里读 `pending_jobs()`(默认只续 6 小时内的),
|
||||||
|
按当前配置重建任务;provider 已不存在则标记 failed;太老的标记 expired。
|
||||||
|
- 完成超过 7 天的任务记录由清理任务删掉。
|
||||||
|
- 实测:登记一个待回调任务 → 重启 → 日志/库状态变为 `done`(任务被接回来了)。
|
||||||
|
|
||||||
|
### ② `/logs` 端点(遗留的内存环形缓冲)
|
||||||
|
前端早已改走 `/logs/recent`,但 `/logs` 还读内存 —— 重启后为空。
|
||||||
|
现在抽了公共函数 `_recent_entries()`,`/logs` 与 `/logs/recent` **同源读库**,
|
||||||
|
`/logs` 的权限口径不变(只看自己的)。顺手删掉不再使用的 `_logs_for_account()`。
|
||||||
|
|
||||||
|
### 剩下的内存态(评估后**不需要**落盘)
|
||||||
|
| 数据 | 为什么不用落盘 |
|
||||||
|
|---|---|
|
||||||
|
| 熔断器状态 | 重启重置成 closed 是**安全的默认**(宁可多打一次也不要一直熔断)|
|
||||||
|
| 限流令牌桶 | 重启重置只会短暂放宽,不影响正确性 |
|
||||||
|
| 鉴权失败计数 | 同上 |
|
||||||
|
| `_metrics` 内存副本 | 展示已全部读库,只剩 `uptime_seconds`/`in_flight` 这类实时值 |
|
||||||
|
| 在途请求临时缓存 | 请求级生命周期,本来就不该跨重启 |
|
||||||
|
|
||||||
## 2026-10-23 · 调用日志详情彻底不截断(2.9.11 → 2.9.12)
|
## 2026-10-23 · 调用日志详情彻底不截断(2.9.11 → 2.9.12)
|
||||||
|
|
||||||
老板:「调用日志的详情内容都不要截断记录,都要显示完整的,像提示词、回复这些」。
|
老板:「调用日志的详情内容都不要截断记录,都要显示完整的,像提示词、回复这些」。
|
||||||
|
|||||||
@@ -93,6 +93,22 @@ CREATE TABLE IF NOT EXISTS video_polls (
|
|||||||
raw TEXT
|
raw TEXT
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS idx_vp_vid ON video_polls(video_id, ts);
|
CREATE INDEX IF NOT EXISTS idx_vp_vid ON video_polls(video_id, ts);
|
||||||
|
|
||||||
|
-- 待回调的视频任务(2026-10-23):进程重启后要能续跑,否则 callback_url 永远不会触发
|
||||||
|
CREATE TABLE IF NOT EXISTS video_jobs (
|
||||||
|
video_id TEXT PRIMARY KEY,
|
||||||
|
provider TEXT,
|
||||||
|
model TEXT,
|
||||||
|
callback_url TEXT,
|
||||||
|
rid TEXT,
|
||||||
|
account TEXT,
|
||||||
|
created REAL,
|
||||||
|
status TEXT DEFAULT 'pending',
|
||||||
|
wait_timeout INTEGER,
|
||||||
|
last_error TEXT,
|
||||||
|
done_at REAL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_vj_status ON video_jobs(status, created);
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_COLUMNS = ("ts", "rid", "account", "key_id", "kind", "method", "path", "model",
|
_COLUMNS = ("ts", "rid", "account", "key_id", "kind", "method", "path", "model",
|
||||||
@@ -300,6 +316,8 @@ class CallLogStore:
|
|||||||
with self._lock:
|
with self._lock:
|
||||||
cur = self._db.execute("DELETE FROM api_calls WHERE ts < ?", (cutoff,))
|
cur = self._db.execute("DELETE FROM api_calls WHERE ts < ?", (cutoff,))
|
||||||
self._db.execute("DELETE FROM video_polls WHERE ts < ?", (cutoff,))
|
self._db.execute("DELETE FROM video_polls WHERE ts < ?", (cutoff,))
|
||||||
|
self._db.execute("DELETE FROM video_jobs WHERE status != 'pending' AND done_at < ?",
|
||||||
|
(time.time() - 7 * 86400,))
|
||||||
self._db.commit()
|
self._db.commit()
|
||||||
n = cur.rowcount or 0
|
n = cur.rowcount or 0
|
||||||
if n:
|
if n:
|
||||||
@@ -489,6 +507,56 @@ class CallLogStore:
|
|||||||
log.warning("读取视频轮询失败: %s", e)
|
log.warning("读取视频轮询失败: %s", e)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
# ── 视频回调任务(2026-10-23:重启后续跑,避免 callback_url 永不触发)──────
|
||||||
|
def add_job(self, video_id: str, *, provider: str, model: str, callback_url: str,
|
||||||
|
rid: str = "", account: str = "", wait_timeout: int = 900) -> None:
|
||||||
|
if not self.enabled or self._db is None or not video_id:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
with self._lock:
|
||||||
|
self._db.execute(
|
||||||
|
"INSERT OR REPLACE INTO video_jobs "
|
||||||
|
"(video_id, provider, model, callback_url, rid, account, created, status, wait_timeout) "
|
||||||
|
"VALUES (?,?,?,?,?,?,?, 'pending', ?)",
|
||||||
|
(video_id, provider, model, callback_url, rid, account, time.time(), int(wait_timeout)))
|
||||||
|
self._db.commit()
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
self.last_error = f"{type(e).__name__}: {e}"
|
||||||
|
|
||||||
|
def pending_jobs(self, max_age_s: float = 6 * 3600) -> list[dict[str, Any]]:
|
||||||
|
"""取待续跑的回调任务(太老的判定为过期,不再续跑)。"""
|
||||||
|
if not self.enabled or self._db is None:
|
||||||
|
return []
|
||||||
|
cutoff = time.time() - max_age_s
|
||||||
|
try:
|
||||||
|
with self._lock:
|
||||||
|
cur = self._db.execute(
|
||||||
|
"SELECT video_id, provider, model, callback_url, rid, account, created, wait_timeout "
|
||||||
|
"FROM video_jobs WHERE status = 'pending' AND created >= ? ORDER BY created ASC", (cutoff,))
|
||||||
|
rows = cur.fetchall()
|
||||||
|
names = [d[0] for d in cur.description]
|
||||||
|
# 太老的标记成 expired,避免每次启动都尝试
|
||||||
|
self._db.execute(
|
||||||
|
"UPDATE video_jobs SET status='expired', done_at=? "
|
||||||
|
"WHERE status='pending' AND created < ?", (time.time(), cutoff))
|
||||||
|
self._db.commit()
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.warning("读取待续跑视频任务失败: %s", e)
|
||||||
|
return []
|
||||||
|
return [dict(zip(names, r)) for r in rows]
|
||||||
|
|
||||||
|
def finish_job(self, video_id: str, status: str = "done", error: str = "") -> None:
|
||||||
|
if not self.enabled or self._db is None or not video_id:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
with self._lock:
|
||||||
|
self._db.execute(
|
||||||
|
"UPDATE video_jobs SET status=?, done_at=?, last_error=? WHERE video_id=?",
|
||||||
|
(status, time.time(), _clip(error, 1000), video_id))
|
||||||
|
self._db.commit()
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
self.last_error = f"{type(e).__name__}: {e}"
|
||||||
|
|
||||||
# ── 聚合(供 /metrics、/health、月度用量导出使用)────────────────────
|
# ── 聚合(供 /metrics、/health、月度用量导出使用)────────────────────
|
||||||
def usage_snapshot(self, *, account: str | None = None, days: int = 0) -> dict[str, Any]:
|
def usage_snapshot(self, *, account: str | None = None, days: int = 0) -> dict[str, Any]:
|
||||||
"""按模型/供应商/账号聚合调用数、耗时与 token(形状对齐内存态 `Metrics.snapshot()`)。
|
"""按模型/供应商/账号聚合调用数、耗时与 token(形状对齐内存态 `Metrics.snapshot()`)。
|
||||||
|
|||||||
+76
-17
@@ -231,6 +231,33 @@ async def lifespan(app: FastAPI):
|
|||||||
await calllog.store.start()
|
await calllog.store.start()
|
||||||
except Exception as e: # noqa: BLE001
|
except Exception as e: # noqa: BLE001
|
||||||
log.warning("调用日志启动失败(不影响服务): %s", e)
|
log.warning("调用日志启动失败(不影响服务): %s", e)
|
||||||
|
# 重启续跑:把上次没跑完的视频回调任务接回来(否则 callback_url 永远不会触发)
|
||||||
|
try:
|
||||||
|
jobs = calllog.store.pending_jobs()
|
||||||
|
if jobs:
|
||||||
|
log.info("恢复 %d 个未完成的视频回调任务", len(jobs))
|
||||||
|
for j in jobs:
|
||||||
|
pkey = j.get("provider") or ""
|
||||||
|
pconfig = config.providers.get(pkey)
|
||||||
|
if not pconfig:
|
||||||
|
calllog.store.finish_job(j["video_id"], "failed", "provider 已不存在")
|
||||||
|
continue
|
||||||
|
|
||||||
|
async def _resume_job(job=j, pc=pconfig, pk=pkey):
|
||||||
|
vid = job["video_id"]
|
||||||
|
try:
|
||||||
|
await _video_wait_and_callback(
|
||||||
|
config, pk, pc, pmodel_id=job.get("model"),
|
||||||
|
video_id=vid, wait=True, wait_timeout=job.get("wait_timeout") or 900,
|
||||||
|
callback_url=job.get("callback_url"), rid=(job.get("rid") or "resume"))
|
||||||
|
calllog.store.finish_job(vid, "done")
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.warning("续跑视频回调 %s 失败: %s", vid, e)
|
||||||
|
calllog.store.finish_job(vid, "failed", str(e))
|
||||||
|
|
||||||
|
_spawn_bg(_resume_job(), name=f"video-resume-{j['video_id']}")
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.warning("恢复视频回调任务失败(不影响服务): %s", e)
|
||||||
yield
|
yield
|
||||||
# 关闭阶段:先停热重载,再等在途请求(uvicorn 自身也做,这里补逻辑日志)
|
# 关闭阶段:先停热重载,再等在途请求(uvicorn 自身也做,这里补逻辑日志)
|
||||||
if hot_task:
|
if hot_task:
|
||||||
@@ -356,7 +383,7 @@ _docs_on = config.server.docs_enabled # 安全默认关(report: /docs、/ope
|
|||||||
# 2026-09-30: 2.6.1 → 2.7.0(按老板要求整个删除「火山方舟 Coding Plan」供应商:配置/模型/代码/文档/前端下拉/环境变量全清)。
|
# 2026-09-30: 2.6.1 → 2.7.0(按老板要求整个删除「火山方舟 Coding Plan」供应商:配置/模型/代码/文档/前端下拉/环境变量全清)。
|
||||||
# 2026-09-30: 2.7.0 → 2.7.1(界面提示白话化:熔断器→供应商自动保护、引擎→运行环境、slug→标识;删 120 个未引用死键)。
|
# 2026-09-30: 2.7.0 → 2.7.1(界面提示白话化:熔断器→供应商自动保护、引擎→运行环境、slug→标识;删 120 个未引用死键)。
|
||||||
# ★ 版本号纪律:每次改动都要往上加,小改动加小版本号、大改动加大版本号(README 有专章)。
|
# ★ 版本号纪律:每次改动都要往上加,小改动加小版本号、大改动加大版本号(README 有专章)。
|
||||||
VERSION = "2.9.12"
|
VERSION = "2.9.13"
|
||||||
|
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
title="API Gateway",
|
title="API Gateway",
|
||||||
@@ -2626,14 +2653,22 @@ async def videos_create(request: Request, req: VideoCreateRequestExtended, _auth
|
|||||||
# 就 `if not wait: return None` → **webhook 被静默丢弃、永不回调**,而 README 却把
|
# 就 `if not wait: return None` → **webhook 被静默丢弃、永不回调**,而 README 却把
|
||||||
# callback_url 描述成"异步通知"。现在丢到后台任务里轮询并在完成后回调,
|
# callback_url 描述成"异步通知"。现在丢到后台任务里轮询并在完成后回调,
|
||||||
# 请求立即返回(async=true),语义与文档一致。
|
# 请求立即返回(async=true),语义与文档一致。
|
||||||
_spawn_bg(
|
# 2026-10-23:**先落库再派生** —— 进程重启后靠这张表续跑,否则回调永不触发
|
||||||
_video_wait_and_callback(
|
calllog.store.add_job(
|
||||||
cfg, pkey, pconfig, pmodel_id=real_model, video_id=video_id,
|
video_id, provider=pkey, model=real_model, callback_url=req.callback_url,
|
||||||
wait=True, wait_timeout=req.wait_timeout,
|
rid=rid, account=request.state.account, wait_timeout=req.wait_timeout or 900)
|
||||||
callback_url=req.callback_url, rid=rid,
|
|
||||||
),
|
async def _cb_job(vid=video_id):
|
||||||
name=f"video-callback-{video_id}",
|
try:
|
||||||
)
|
return await _video_wait_and_callback(
|
||||||
|
cfg, pkey, pconfig, pmodel_id=real_model, video_id=vid,
|
||||||
|
wait=True, wait_timeout=req.wait_timeout,
|
||||||
|
callback_url=req.callback_url, rid=rid,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
calllog.store.finish_job(vid, "done")
|
||||||
|
|
||||||
|
_spawn_bg(_cb_job(), name=f"video-callback-{video_id}")
|
||||||
resp["status"] = "submitted"
|
resp["status"] = "submitted"
|
||||||
resp["callback_scheduled"] = True
|
resp["callback_scheduled"] = True
|
||||||
# 第十轮: 视频日志带 video_id 与响应摘要;轮询数据由 /logs 按 video_id 合成进来
|
# 第十轮: 视频日志带 video_id 与响应摘要;轮询数据由 /logs 按 video_id 合成进来
|
||||||
@@ -3186,12 +3221,36 @@ def _attach_polls(entries: list[dict]) -> list[dict]:
|
|||||||
return entries
|
return entries
|
||||||
|
|
||||||
|
|
||||||
def _logs_for_account(account: str, n: int = 200) -> list[dict]:
|
|
||||||
"""取某账号产生的访问日志(含轮询合成)。第十一轮 #2:admin 也走这里看别人。"""
|
|
||||||
tail = _access.tail(max(1, n))
|
def _recent_entries(account: str, n: int = 200) -> list[dict]:
|
||||||
rows = [e for e in tail
|
"""从库里取某账号最近 N 条访问记录(形状与旧的内存 /logs 条目一致)。
|
||||||
if e.get("account") == account or e.get("caller") == account]
|
|
||||||
return _attach_polls(rows)
|
2026-10-23:`/logs` 从内存环形缓冲改为读库 —— 重启不再丢(前端早已改走 /logs/recent,
|
||||||
|
这个函数是让遗留的 /logs 端点也保持一致)。
|
||||||
|
"""
|
||||||
|
n = max(1, min(int(n), 500))
|
||||||
|
rows = calllog.store.query(limit=n, offset=0, with_content=True)
|
||||||
|
items = []
|
||||||
|
for it in rows.get("items", []):
|
||||||
|
acct = it.get("account") or it.get("caller") or ""
|
||||||
|
if acct != account and (it.get("caller") or "") != account:
|
||||||
|
continue
|
||||||
|
items.append({
|
||||||
|
"ts": it.get("ts"), "rid": it.get("rid"), "account": acct,
|
||||||
|
"kind": it.get("kind"), "method": it.get("method"), "path": it.get("path"),
|
||||||
|
"model": it.get("model"), "served": it.get("served"), "provider": it.get("provider"),
|
||||||
|
"caller": it.get("caller"), "ok": bool(it.get("ok")), "stream": bool(it.get("stream")),
|
||||||
|
"status": it.get("status"), "ms": it.get("latency_ms"),
|
||||||
|
"body": (it.get("prompt") or "")[:8000], "response": (it.get("completion") or "")[:8000],
|
||||||
|
"body_len": len(it.get("prompt") or ""), "resp_len": len(it.get("completion") or ""),
|
||||||
|
"tokens": {"prompt_tokens": it.get("tok_prompt") or 0,
|
||||||
|
"completion_tokens": it.get("tok_completion") or 0,
|
||||||
|
"cached_tokens": it.get("tok_cached") or 0,
|
||||||
|
"reasoning_tokens": it.get("tok_reasoning") or 0},
|
||||||
|
"id": it.get("id"),
|
||||||
|
})
|
||||||
|
return _attach_polls(items)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/logs/entry/{call_id}")
|
@app.get("/logs/entry/{call_id}")
|
||||||
@@ -3263,10 +3322,10 @@ async def logs_recent(request: Request, n: int = 200, _auth=Depends(require_auth
|
|||||||
|
|
||||||
@app.get("/logs")
|
@app.get("/logs")
|
||||||
async def logs(request: Request, n: int = 100, _auth=Depends(require_auth)):
|
async def logs(request: Request, n: int = 100, _auth=Depends(require_auth)):
|
||||||
# 第十一轮 #2:日志页对**所有人(含 admin)都只显示自己的**。
|
# 2026-10-23:改读库(原来读内存环形缓冲,重启即丢)。口径不变:**只看自己的**。
|
||||||
# 要看别人的调用日志 → 「日志 → 调用日志」页签(/logs/calls,仅管理员);
|
# 要看别人的调用日志 → 「日志 → 调用日志」页签(/logs/calls,仅管理员);
|
||||||
# 要看别人的资产 → 用户管理页的 /auth/users/{name}/history/{kind}。
|
# 要看别人的资产 → 用户管理页的 /auth/users/{name}/history/{kind}。
|
||||||
return {"entries": _logs_for_account(request.state.account, n)}
|
return {"entries": _recent_entries(request.state.account, n), "source": "db"}
|
||||||
|
|
||||||
|
|
||||||
# ─────────────────────────────────────
|
# ─────────────────────────────────────
|
||||||
|
|||||||
Reference in new issue
Block a user