diff --git a/CHANGELOG.md b/CHANGELOG.md index 435979b..c00fafe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,46 @@ # CHANGELOG +## 2026-09-28 · 修「token 统计恒 0」+ token 细分(缓存命中 / 推理) + +老板报「token 数统计有问题,一直都是 0」。 + +**真因**:`/v1/chat/completions` 的 **managed 流式路径**(改写分支 —— QwenPaw 实际走的就是它) +只把累计出来的 usage **转发给客户端**,**从没写进指标**。上游明明每轮都发 usage +(实测 chunk 里带 `prompt_tokens` / `prompt_tokens_details.cached_tokens`)。 +现场取证:网关记了 **663 次调用、`tokens` 恒为 `{prompt: 0, completion: 0}`**。 + +对照确认:直传路径(messages/responses)与非流式路径本来就有记账 —— 只有 +"流式 + 改写"这一条漏了,而它恰好是最常走的一条。 + +**修复**:该路径补 `_record_usage_metrics(stream_usage)`。 + +**顺带做 token 细分**(上游 usage 本来就有,此前只记总数): + +| 细分 | 来源 | 含义 | +| --- | --- | --- | +| `cached` | `prompt_tokens_details.cached_tokens` | 命中缓存的**输入**量(已含在 prompt 内) | +| `reasoning` | `completion_tokens_details.reasoning_tokens` | 输出里的**思考**量(已含在 completion 内) | + +- `Metrics.record_tokens(prompt, completion, cached=0, reasoning=0)`:全局与按账号两条桶都细分; +- 四处记录点全接上:managed 流式(本轮修的)、managed 非流式、直传非流式、直传流式 + (扫描器补采 `reasoning_tokens`); +- Prometheus 新增 `api_gateway_tokens_cached_total` / `api_gateway_tokens_reasoning_total`(含 HELP/TYPE); +- 前端「指标」页新增两张卡:**缓存命中 Token**(附「占输入 x%」)与**推理 Token**(附「占输出 x%」)。 + +**上线实测**(生产重启后): + +``` +同一瘦请求(managed 流式)→ tokens {prompt: 62, completion: 2, cached: 0, reasoning: 1} +带 4k token 前缀的同一请求发两次: + 第 1 次 cached_tokens=0(未命中);第 2 次 cached_tokens=1920 / prompt_tokens=4076 +网关累计:cached=1920(输入命中率 22.9%) +Prometheus:api_gateway_tokens_cached_total 1920 / api_gateway_tokens_reasoning_total 5 +``` + +**测试**:新增「managed 流式必须记账(含细分)」「细分累加互不干扰」两条回归; +测试数 202 → **204 全过**;`ci_check.sh` 全绿;i18n 中英 759/759 对称。 + + ## 2026-09-24 · 模型接口加入上下文参数(context_window / max_output_tokens) 老板要求:查两家文档有没有写上下文长度,有就改网关、接口加入上下文参数,并纳入每日同步。 diff --git a/app/main.py b/app/main.py index a4096ad..6597b22 100644 --- a/app/main.py +++ b/app/main.py @@ -1337,6 +1337,30 @@ def _usage_counts(protocol: str, data: Any) -> dict[str, int]: "completion_tokens": int(u.get("completion_tokens") or 0)} +def _usage_detail(usage) -> tuple[int, int, int, int]: + """从一份 usage 里抽 (prompt, completion, cached, reasoning),兼容三家字段名。 + + 第三十四轮:cached / reasoning 是**细分项** —— 分别已包含在 prompt / completion 里, + 只用于展示(求解「缓存命中多少、思考烧了多少」),求和时不要再与总数相加。 + """ + if not isinstance(usage, dict): + return (0, 0, 0, 0) + prompt = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0) + completion = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0) + pd = usage.get("prompt_tokens_details") + cd = usage.get("completion_tokens_details") + cached = int((pd or {}).get("cached_tokens") or 0) if isinstance(pd, dict) else 0 + reasoning = int((cd or {}).get("reasoning_tokens") or 0) if isinstance(cd, dict) else 0 + return (prompt, completion, cached, reasoning) + + +def _record_usage_metrics(usage) -> None: + """把一份 usage 记进指标(含缓存命中 / 推理细分);全 0 时不打扰指标。""" + p, c, cached, reasoning = _usage_detail(usage) + if p or c or cached or reasoning: + _metrics.record_tokens(p, c, cached=cached, reasoning=reasoning) + + class _UsageScanner: """流式直传的 usage 扫描器(只读,不改一个字节)。 @@ -1353,6 +1377,7 @@ class _UsageScanner: self.prompt = 0 self.completion = 0 self.cached = 0 + self.reasoning = 0 self.seen = False self._buf = "" @@ -1400,10 +1425,17 @@ class _UsageScanner: self.cached = int(det["cached_tokens"]) except (TypeError, ValueError): pass + # 第三十四轮:输出里的思考量(completion_tokens_details.reasoning_tokens) + cdet = u.get("completion_tokens_details") + if isinstance(cdet, dict) and cdet.get("reasoning_tokens") is not None: + try: + self.reasoning = int(cdet["reasoning_tokens"]) + except (TypeError, ValueError): + pass def counts(self) -> dict[str, int]: return {"prompt_tokens": self.prompt, "completion_tokens": self.completion, - "cached_tokens": self.cached} + "cached_tokens": self.cached, "reasoning_tokens": self.reasoning} def _merge_usage(dst: dict, src: dict) -> dict: @@ -1519,7 +1551,7 @@ async def _passthrough(request: Request, protocol: str): _metrics.record_latency(model_in, time.time() - t0) usage = _usage_counts(protocol, data) if ok: - _metrics.record_tokens(usage["prompt_tokens"], usage["completion_tokens"]) + _record_usage_metrics(data.get("usage") if isinstance(data, dict) else None) _access.add({"method": "POST", "kind": "chat", "path": f"/v1/{protocol}", "rid": rid, "model": model_in, "served": model_in, "provider": pkey, "caller": caller, "ok": ok, "stream": False, @@ -1593,7 +1625,9 @@ async def _passthrough_stream(request, rid, caller, protocol, adapter, cb, _metrics.record_latency(model_in, time.time() - t0) counts = scanner.counts() if ok and scanner.seen: - _metrics.record_tokens(counts["prompt_tokens"], counts["completion_tokens"]) + _metrics.record_tokens(counts["prompt_tokens"], counts["completion_tokens"], + cached=counts["cached_tokens"], + reasoning=counts["reasoning_tokens"]) _access.add({"method": "POST", "kind": "chat", "path": f"/v1/{protocol}", "rid": rid, "model": model_in, "served": model_in, "provider": pkey, "caller": caller, "ok": ok, "stream": True, "response": ""}) @@ -1879,7 +1913,7 @@ async def _chat_once(cfg, rid, caller, request, req, model_id, messages, tools): usage = result.get("usage") or {} _metrics.record_call(model_id, pkey, ok=True) - _metrics.record_tokens(int(usage.get("prompt_tokens", 0)), int(usage.get("completion_tokens", 0))) + _record_usage_metrics(usage) _metrics.record_latency(model_id, time.time() - t0) _access.add({"method": "POST", "kind": "chat", "path": "/v1/chat/completions", "rid": rid, "model": req.model, "served": model_id, "provider": pkey, @@ -2002,6 +2036,11 @@ def _stream_response(request, req, model_id, messages, tools): cb.record(True) _metrics.record_call(model_id, pkey, ok=True) _metrics.record_latency(model_id, time.time() - t0) + # ★ 第三十四轮修复(老板报「token 一直是 0」):这条 managed 流式路径 + # 此前只把 usage **转发给客户端**、从没写进指标 —— 上游明明每轮都发了 + # usage(实测 chunk 里带 prompt_tokens/cached_tokens),但网关的 + # token 统计恒为 0(663 次调用、tokens=0)。这里补上记账(含缓存/推理细分)。 + _record_usage_metrics(stream_usage) _access.add({"method": "POST", "kind": "chat", "path": "/v1/chat/completions", "rid": rid, "model": req.model, "served": model_id, "provider": pkey, "caller": caller, "ok": True, "stream": True, @@ -2869,6 +2908,8 @@ async def metrics_prometheus(request: Request, _auth=Depends(require_auth)): # 快照里已没有 fallback_total 字段,这里再读就是 KeyError(曾被 live_check 抓到 500)。 f'api_gateway_tokens_prompt_total {snap["tokens"]["prompt"]}', f'api_gateway_tokens_completion_total {snap["tokens"]["completion"]}', + f'api_gateway_tokens_cached_total {snap["tokens"].get("cached", 0)}', + f'api_gateway_tokens_reasoning_total {snap["tokens"].get("reasoning", 0)}', # 功能 11: 在途请求 gauge f'api_gateway_in_flight {snap.get("in_flight", 0)}', f'api_gateway_in_flight_peak {snap.get("in_flight_peak", 0)}', @@ -2899,6 +2940,10 @@ async def metrics_prometheus(request: Request, _auth=Depends(require_auth)): "# TYPE api_gateway_tokens_prompt_total counter", "# HELP api_gateway_tokens_completion_total 上游 completion token 累计", "# TYPE api_gateway_tokens_completion_total counter", + "# HELP api_gateway_tokens_cached_total 其中命中缓存的输入 token 累计(含在 prompt 内)", + "# TYPE api_gateway_tokens_cached_total counter", + "# HELP api_gateway_tokens_reasoning_total 其中推理输出 token 累计(含在 completion 内)", + "# TYPE api_gateway_tokens_reasoning_total counter", "# HELP api_gateway_in_flight 在途请求数(进入 → 响应头生成完毕)", "# TYPE api_gateway_in_flight gauge", "# HELP api_gateway_in_flight_peak 在途请求峰值", diff --git a/app/metrics.py b/app/metrics.py index 26002d9..d55534d 100644 --- a/app/metrics.py +++ b/app/metrics.py @@ -65,6 +65,11 @@ class Metrics: self.providers: dict[str, dict[str, int]] = {} # provider -> {ok, fail} self.tokens_prompt: int = 0 self.tokens_completion: int = 0 + # 第三十四轮: token 细分 —— 上游 usage 里本来就有这两项,之前只记了总数: + # cached = prompt_tokens_details.cached_tokens(命中前缀缓存的输入量) + # reasoning = completion_tokens_details.reasoning_tokens(输出里的思考量) + self.tokens_cached: int = 0 + self.tokens_reasoning: int = 0 self.latency_samples: dict[str, deque[float]] = {} # model -> 最近 N 次耗时 # 功能 11: 在途请求计数 self.in_flight: int = 0 @@ -81,7 +86,7 @@ class Metrics: if acc is None: acc = { "calls": {}, "providers": {}, - "tokens": {"prompt": 0, "completion": 0}, + "tokens": {"prompt": 0, "completion": 0, "cached": 0, "reasoning": 0}, "latency": {}, } self.by_account[account] = acc @@ -128,17 +133,29 @@ class Metrics: self._bucket(acc["providers"], provider)["ok" if ok else "fail"] += 1 - def record_tokens(self, prompt: int, completion: int, account: str | None = None) -> None: + def record_tokens(self, prompt: int, completion: int, account: str | None = None, + cached: int = 0, reasoning: int = 0) -> None: + """记一次调用的 token。 + + 第三十四轮新增 cached / reasoning 细分(默认 0,老调用点不受影响): + `cached` 是输入里命中缓存的部分(**已含在 prompt 里**,不是额外的量), + `reasoning` 是输出里的思考部分(**已含在 completion 里**)——两者都只作细分展示, + 求和时不要与 prompt/completion 相加。 + """ if not self.enabled: return a = account or ACCOUNT.get() with self._lock: self.tokens_prompt += prompt self.tokens_completion += completion + self.tokens_cached += cached + self.tokens_reasoning += reasoning if a and a != "anonymous": acc = self._acc(a) acc["tokens"]["prompt"] += prompt acc["tokens"]["completion"] += completion + acc["tokens"]["cached"] += cached + acc["tokens"]["reasoning"] += reasoning def record_latency(self, model: str, seconds: float, account: str | None = None) -> None: if not self.enabled: @@ -177,7 +194,8 @@ class Metrics: return dict(uptime_seconds=round(time.time() - self.started_at, 1), in_flight=self.in_flight, in_flight_peak=self.in_flight_peak, calls={}, providers={}, - tokens={"prompt": 0, "completion": 0}, latency={}) + tokens={"prompt": 0, "completion": 0, "cached": 0, "reasoning": 0}, + latency={}) lat = {} for m, dq in acc["latency"].items(): if dq: @@ -188,7 +206,7 @@ class Metrics: "in_flight_peak": self.in_flight_peak, "calls": {k: dict(v) for k, v in acc["calls"].items()}, "providers": {k: dict(v) for k, v in acc["providers"].items()}, - "tokens": dict(acc["tokens"]), + "tokens": _tokens_out(acc["tokens"]), "latency": lat, } return self._snapshot_global() @@ -207,13 +225,24 @@ class Metrics: "in_flight_peak": self.in_flight_peak, "calls": {k: dict(v) for k, v in self.calls.items()}, "providers": {k: dict(v) for k, v in self.providers.items()}, - "tokens": {"prompt": self.tokens_prompt, "completion": self.tokens_completion}, + "tokens": {"prompt": self.tokens_prompt, "completion": self.tokens_completion, + "cached": self.tokens_cached, "reasoning": self.tokens_reasoning}, "latency": latency_out, } +def _tokens_out(t: dict) -> dict[str, int]: + """token 细分输出的统一形状(老桶可能缺 cached/reasoning,这里补 0)。""" + return { + "prompt": int(t.get("prompt", 0) or 0), + "completion": int(t.get("completion", 0) or 0), + "cached": int(t.get("cached", 0) or 0), + "reasoning": int(t.get("reasoning", 0) or 0), + } + + def _latency_summary(dq: deque) -> dict[str, float]: """把某模型/账号的耗时样本汇总成 avg/p50/p95(nearest-rank)。""" s = sorted(dq) diff --git a/frontend/src/locales/en.ts b/frontend/src/locales/en.ts index 80dc35a..e7f16ce 100644 --- a/frontend/src/locales/en.ts +++ b/frontend/src/locales/en.ts @@ -155,6 +155,10 @@ export default { requests: 'Total requests', failCount: 'Failures', tokenUsage: 'Token usage', + cachedTokens: 'Cached tokens', + cachedNote: '{v}% of input', + reasoningTokens: 'Reasoning tokens', + reasoningNote: '{v}% of output', tokenNote: 'in {p} / out {c}', inflightPeak: 'In flight / peak', failRateNote: 'failure rate {v}%', diff --git a/frontend/src/locales/zh.ts b/frontend/src/locales/zh.ts index bb244ec..ff60aec 100644 --- a/frontend/src/locales/zh.ts +++ b/frontend/src/locales/zh.ts @@ -155,6 +155,10 @@ export default { requests: '请求总数', failCount: '失败数', tokenUsage: 'Token 用量', + cachedTokens: '缓存命中 Token', + cachedNote: '占输入 {v}%', + reasoningTokens: '推理 Token', + reasoningNote: '占输出 {v}%', tokenNote: '入 {p} / 出 {c}', inflightPeak: '在途 / 峰值', failRateNote: '失败率 {v}%', diff --git a/frontend/src/views/Metrics.vue b/frontend/src/views/Metrics.vue index beb8399..8790bf0 100644 --- a/frontend/src/views/Metrics.vue +++ b/frontend/src/views/Metrics.vue @@ -145,6 +145,13 @@ const totals = computed(() => { return { ok, fail, all: ok + fail } }) +/** 占比(分母为 0 时返回 0,避免 NaN 显示) */ +function pct(part: unknown, whole: unknown): string { + const p = Number(part) || 0 + const w = Number(whole) || 0 + return w > 0 ? ((p / w) * 100).toFixed(1) : '0.0' +} + const cards = computed(() => { void resolved.value const tk = data.value.tokens || {} @@ -154,6 +161,9 @@ const cards = computed(() => { { k: t('metrics.requests'), v: all, note: t('dashboard.noteOkFail', { ok, fail }) }, { k: t('metrics.failCount'), v: fail, note: t('metrics.failRateNote', { v: rate.toFixed(1) }) }, { k: t('metrics.tokenUsage'), v: (tk.prompt || 0) + (tk.completion || 0), note: t('metrics.tokenNote', { p: tk.prompt ?? 0, c: tk.completion ?? 0 }) }, + // 第三十四轮: token 细分 —— 缓存命中(含在输入里)与推理(含在输出里),各给一个占比 + { k: t('metrics.cachedTokens'), v: tk.cached || 0, note: t('metrics.cachedNote', { v: pct(tk.cached, tk.prompt) }) }, + { k: t('metrics.reasoningTokens'), v: tk.reasoning || 0, note: t('metrics.reasoningNote', { v: pct(tk.reasoning, tk.completion) }) }, // 第二十七轮: fallback 计数随路由功能删除 → 这张卡片的注脚改成运行时长 { k: t('metrics.inflightPeak'), v: `${data.value.in_flight ?? 0} / ${data.value.in_flight_peak ?? 0}`, note: t('metrics.uptime') + ' ' + fmtUptime.value }, ] diff --git a/tests/test_gateway.py b/tests/test_gateway.py index d4219fc..a615146 100644 --- a/tests/test_gateway.py +++ b/tests/test_gateway.py @@ -5414,6 +5414,65 @@ def test_usage_scanner_collects_cached_tokens(): assert sc2.counts()["cached_tokens"] == 0 +def test_managed_stream_records_tokens_with_breakdown(): + """第三十四轮修复(老板报「token 统计一直是 0」): + + **managed 流式路径**(`/v1/chat/completions` 的改写分支 —— QwenPaw 实际走的就是它) + 此前只把累计 usage **转发给客户端**,从没写进指标;上游明明每轮都发 usage。 + 表现:网关记了 663 次调用、`tokens` 恒为 `{prompt: 0, completion: 0}`。 + + 这里端到端断言:流式一轮结束后指标里真的有数,且**缓存命中 / 推理**两项也细分记下。 + """ + from fastapi.testclient import TestClient + import app.main as m + + class FakeAdapter: + async def chat_stream(self, model=None, messages=None, **kw): + yield {"id": "c1", "object": "chat.completion.chunk", "model": model, + "choices": [{"index": 0, "delta": {"content": "hi"}}]} + yield {"id": "c1", "choices": [], "usage": { + "prompt_tokens": 100, "completion_tokens": 40, + "prompt_tokens_details": {"cached_tokens": 70}, + "completion_tokens_details": {"reasoning_tokens": 12}, + }} + yield {"done": True} + + orig = m._get_adapter + m._get_adapter = lambda model_id, timeout=None, cfg=None: ( + FakeAdapter(), "agnes", m._cfg().providers["agnes"]) + try: + before = m._metrics.snapshot()["tokens"] + client = TestClient(m.app) + with client.stream("POST", "/v1/chat/completions", json={ + "model": "agnes-3.0-flash", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + }) as r: + assert r.status_code == 200 + body = "".join(r.iter_text()) + assert "[DONE]" in body + after = m._metrics.snapshot()["tokens"] + assert after["prompt"] - before["prompt"] == 100, after + assert after["completion"] - before["completion"] == 40, after + assert after["cached"] - before["cached"] == 70, after + assert after["reasoning"] - before["reasoning"] == 12, after + finally: + m._get_adapter = orig + + +def test_metrics_record_tokens_breakdown_accumulates(): + """指标层:cached / reasoning 与 prompt / completion 累加互不干扰(细分项只做展示)。""" + from app.metrics import Metrics + + m = Metrics() + m.record_tokens(100, 40, cached=70, reasoning=12) + m.record_tokens(10, 5) # 老调用点不传细分 → 默认 0 + t = m.snapshot()["tokens"] + assert t == {"prompt": 110, "completion": 45, "cached": 70, "reasoning": 12}, t + # 细分是「已含在总数里」的部分,不能相加 + assert t["cached"] <= t["prompt"] and t["reasoning"] <= t["completion"] + + # 直接运行(非 pytest)—— 必须留在文件末尾,见上方 `test_image_request_goes_to_requested_model` # 之前的旁注:运行器只会收集执行到这一行时已经存在的 globals。 if __name__ == "__main__":