一个能跑的 Agent Demo 与生产系统之间隔着三条鸿沟:并发压力下资源失控、失败之后状态不可恢复、问题发生时链路不可观测。生产级的评判标准不是"能不能跑通",而是"故障发生时能不能收敛、能不能定位、能不能续跑"。本文围绕并发模型、可靠性原语、可观测体系三块,给出一套可直接落地的 Agent 骨架代码。
接入层(背压/准入) → 编排层(Agent Loop + Checkpoint) → 执行层(工具/熔断/重试) → 观测层(Trace/Metric/Log 贯穿)核心约束只有一条:观测不在某一层,而是从入口 trace_id 贯穿到每一次 LLM 调用和每一次工具调用。
生产中必须区分三级并发,分别设限:
层级 | 控制手段 | 目的 |
|---|---|---|
请求级 | 有界池 + 准入拒绝 | 防止雪崩,队列满时快速失败 |
工具级 | asyncio.gather fan-out | 单步内并行调用,压短关键路径 |
单飞 | 相同 key 合并请求 | 同一会话、同一工具不重复打下游 |
超时要分层设置:请求超时 > 单步超时 > 工具超时 > 下游调用超时,每一层都比下一层宽,避免被下游拖死后仍占着上游资源。
Timeout/Connection/RateLimit 这类瞬时错误重试,业务错误直接上抛,避免污染下游。state 落盘,会话恢复从最后一步续跑,而不是从头重放(LLM 调用是有成本的)。此外,工具调用必须携带 dedup_key(如 trace_id + step + tool_name + args_hash),配合幂等存储实现"重试不重放",这是 Agent 在支付、下单类工具上落地的必要条件。优雅关闭也不可省略:SIGTERM 后停止准入、等待活跃步完成、超时强制取消并落 DLQ。
三层数据都要带同一个 trace_id:
agent.run → agent.step → llm.call / tool.xxx,父子关系直出调用链。llm.latency(p50/p99)、tool.err_rate{tool}、circuit_open{tool}、agent.steps 分布、llm.tokens(成本)。其中"步数分布"能最快暴露死循环类问题。# framework.py —— 生产级 Agent 骨架
import asyncio, contextlib, contextvars, json, random, time, uuid
from dataclasses import dataclass, field
from collections import defaultdict
# ---------- 1. 可观测 ----------
@dataclass
class Span:
trace_id: str; span_id: str; name: str; parent_id: str | None
start: float; end: float = 0.0
attrs: dict = field(default_factory=dict); status: str = "ok"
class Tracer:
def __init__(self, sink):
self._ctx = contextvars.ContextVar("span", default=None)
self._sink = sink # 非阻塞写入 OTLP / ClickHouse
@contextlib.asynccontextmanager
async def span(self, name, **attrs):
p = self._ctx.get()
s = Span(p.trace_id if p else uuid.uuid4().hex,
uuid.uuid4().hex[:16], name,
p.span_id if p else None,
time.monotonic(), attrs=dict(attrs))
tok = self._ctx.set(s)
try: yield s
except Exception as e:
s.status = "error"; s.attrs["error"] = repr(e); raise
finally:
s.end = time.monotonic(); self._ctx.reset(tok); self._sink(s)
class Metrics:
def __init__(self):
self._h, self._c = defaultdict(list), defaultdict(int)
def hist(self, k, v): self._h[k].append(v)
def inc(self, k, n=1): self._c[k] += n
def snapshot(self):
def p99(xs):
if not xs: return 0.0
xs = sorted(xs); return xs[min(len(xs)-1, int(len(xs)*.99))]
return {"counters": dict(self._c),
"p99": {k: p99(v) for k, v in self._h.items()}}
# ---------- 2. 有界并发 + 背压 ----------
class BackPressureError(RuntimeError): pass
class BoundedPool:
def __init__(self, max_inflight=64, max_waiting=1024):
self._sem = asyncio.Semaphore(max_inflight)
self._max_waiting, self._waiting = max_waiting, 0
self._active, self._closing = 0, False
async def run(self, fn, *a, timeout=120.0, **kw):
if self._closing: raise RuntimeError("shutting down")
if self._waiting >= self._max_waiting:
raise BackPressureError("queue full")
self._waiting += 1
try: await self._sem.acquire()
finally: self._waiting -= 1
self._active += 1
try: return await asyncio.wait_for(fn(*a, **kw), timeout=timeout)
finally: self._active -= 1; self._sem.release()
def idle(self): return self._active == 0 and self._waiting == 0
# ---------- 3. 重试 + 熔断 ----------
RETRYABLE = (asyncio.TimeoutError, ConnectionError)
async def retry(fn, attempts=3, base=0.2, cap=3.0, retry_on=RETRYABLE):
for i in range(attempts):
try: return await fn()
except retry_on:
if i == attempts - 1: raise
await asyncio.sleep(min(cap, base*2**i) * (0.5 + random.random()))
class CircuitBreaker:
CLOSED, OPEN, HALF = "closed", "open", "half"
def __init__(self, fail_threshold=5, open_secs=30):
self.fail, self.state = 0, self.CLOSED
self.opened_at, self.ft, self.os = 0.0, fail_threshold, open_secs
def allow(self):
if self.state == self.CLOSED: return True
if self.state == self.OPEN:
if time.monotonic() - self.opened_at >= self.os:
self.state = self.HALF; return True
return False
return True # half-open 探测
def record(self, ok):
if ok:
self.fail = 0
if self.state == self.HALF: self.state = self.CLOSED
return
self.fail += 1
if self.state == self.HALF or self.fail >= self.ft:
self.state, self.opened_at = self.OPEN, time.monotonic()
# ---------- 4. Agent 编排 ----------
class Agent:
def __init__(self, llm, tools, tracer, metrics, pool, store):
self.llm, self.tools = llm, tools
self.tracer, self.metrics = tracer, metrics
self.pool, self.store = pool, store
self.breakers = {n: CircuitBreaker() for n in tools}
async def _llm_call(self, messages, schema):
async with self.tracer.span("llm.call", model=self.llm.name):
t0 = time.monotonic()
r = await retry(lambda: self.llm.chat(messages, schema),
retry_on=RETRYABLE + (RateLimitError,))
self.metrics.hist("llm.latency", time.monotonic() - t0)
self.metrics.inc("llm.tokens", r.usage.total_tokens)
return r
async def _call_tool(self, tc):
name, b = tc.function.name, self.breakers[tc.function.name]
if not b.allow():
self.metrics.inc(f"tool.circuit_open.{name}")
return json.dumps({"error": "circuit_open", "tool": name})
async with self.tracer.span(f"tool.{name}", tool=name):
try:
out = await asyncio.wait_for(
self.tools[name](**json.loads(tc.function.arguments)), 30)
b.record(True); self.metrics.inc(f"tool.ok.{name}"); return out
except Exception as e:
b.record(False); self.metrics.inc(f"tool.err.{name}")
return json.dumps({"error": type(e).__name__, "msg": str(e)})
async def _step(self, state):
r = await self._llm_call(state["messages"], state.get("schema"))
if not r.tool_calls:
state["messages"].append({"role": "assistant", "content": r.content})
return "done"
results = await asyncio.gather(*(self._call_tool(tc) for tc in r.tool_calls))
state["messages"].append({"role": "assistant", "tool_calls": r.tool_calls})
for tc, res in zip(r.tool_calls, results):
state["messages"].append(
{"role": "tool", "tool_call_id": tc.id, "content": res})
return "continue"
async def run(self, session_id, user_input):
async with self.tracer.span("agent.run", session=session_id):
state = await self.store.load(session_id) or {
"messages": [{"role": "user", "content": user_input}]}
try:
for _ in range(20):
action = await self.pool.run(self._step, state, timeout=120)
self.metrics.inc("agent.steps")
await self.store.save(session_id, state) # 每步落盘,可断点续跑
if action == "done":
return state["messages"][-1]["content"]
self.metrics.inc("agent.max_steps"); return "超过最大步数,已中止"
except BackPressureError:
self.metrics.inc("agent.rejected"); return "系统繁忙,请稍后重试"
# ---------- 5. 优雅关闭 ----------
async def graceful_shutdown(pool: BoundedPool, deadline=30.0):
pool._closing = True
t0 = time.monotonic()
while time.monotonic() - t0 < deadline:
if pool.idle(): return
await asyncio.sleep(0.2)
# 到点强杀,调用方捕获后落 DLQdedup_key,幂等存储兜底"重试不重放";agent.steps 分布 和 circuit_open 设为告警项,这两个指标先于用户感知到异常。生产级 Agent 的本质是把"不确定性"关进可控的工程边界里:模型可以犯错,但框架必须保证错误可观测、可熔断、可恢复。做到这三点,Agent 才敢真正承接线上流量。
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。