智能成片 Agent 接入

POST /agui/history
Host: smartmedia.vod-qcloud.com
Content-Type: application/json
Authorization: Bearer <TOKEN>

{
	"threadId": "my-thread-001"
}

id: <ID>
data: {"type":"<EventType>","timestamp":<timestamp>,...}

id: RUN_STARTED_1784102011155
data: {"type":"RUN_STARTED","timestamp":1784102011155,"threadId":"thread-001","runId":"run-001"}

id: REASONING_START_1784102012264
data: {"type":"REASONING_START","timestamp":1784102012264,"messageId":"msg-001"}

id: REASONING_MESSAGE_START_1784102012264
data: {"type":"REASONING_MESSAGE_START","timestamp":1784102012264,"messageId":"msg-001","role":"reasoning"}

id: REASONING_MESSAGE_CONTENT_1784102012264
data: {"type":"REASONING_MESSAGE_CONTENT","timestamp":1784102012264,"messageId":"msg-001","delta":"用户想要搜索海浪视频..."}

id: REASONING_MESSAGE_END_1784102013400
data: {"type":"REASONING_MESSAGE_END","timestamp":1784102013400,"messageId":"msg-001"}

id: REASONING_END_1784102013401
data: {"type":"REASONING_END","timestamp":1784102013401,"messageId":"msg-001"}

id: TEXT_MESSAGE_START_1784102019000
data: {"type":"TEXT_MESSAGE_START","timestamp":1784102019000,"messageId":"msg-002"}

id: TEXT_MESSAGE_CONTENT_1784102019050
data: {"type":"TEXT_MESSAGE_CONTENT","timestamp":1784102019050,"messageId":"msg-002","delta":"我来帮您搜索。"}

id: TEXT_MESSAGE_END_1784102024858
data: {"type":"TEXT_MESSAGE_END","timestamp":1784102024858,"messageId":"msg-002"}

id: TOOL_CALL_START_1784102126631
data: {"type":"TOOL_CALL_START","timestamp":1784102126631,"toolCallId":"call-001","toolCallName":"search_media_by_semantics","parentMessageId":"msg-002"}

id: TOOL_CALL_ARGS_1784102126631
data: {"type":"TOOL_CALL_ARGS","timestamp":1784102126631,"toolCallId":"call-001","delta":"{\\"query\\":\\"海浪\\"}"}

id: TOOL_CALL_END_1784102126631
data: {"type":"TOOL_CALL_END","timestamp":1784102126631,"toolCallId":"call-001"}

id: TOOL_CALL_RESULT_1784102126886
data: {"type":"TOOL_CALL_RESULT","timestamp":1784102126886,"messageId":"msg-003","toolCallId":"call-001","content":"{\\"Recall\\":[]}","role":"tool"}

id: RUN_FINISHED_1784102025004
data: {"type":"RUN_FINISHED","timestamp":1784102025004,"threadId":"thread-001","runId":"run-001"}

{
  "threadId": "my-thread-001",
  "runId": "run-002",
  "messages": [{"role": "user", "content": ""}],
  "forwardedProps": {"approval_mode": "level:low", "scenario_name": "video-mixcut"},
  "resume": [
    {"interruptId": "interruptId-001", "status": "resolved"},
    {"interruptId": "interruptId-002", "status": "resolved"}
  ]
}


{
  "threadId": "my-thread-001",
  "runId": "run-002",
  "messages": [{"role": "user", "content": ""}],
  "forwardedProps": {"approval_mode": "level:low", "scenario_name": "video-mixcut"},
  "resume": [
    {"interruptId": "interruptId-001", "status": "cancelled", "payload": {"feedback": "不需要裁剪"}},
    {"interruptId": "interruptId-002", "status": "cancelled", "payload": {"feedback": "不需要水印"}}
  ]
}

{
  "threadId": "my-thread-001",
  "runId": "run-002",
  "messages": [{"role": "user", "content": ""}],
  "forwardedProps": {"approval_mode": "level:low", "scenario_name": "video-mixcut"},
  "resume": [
    {"interruptId": "interruptId-001", "status": "resolved"},
    {"interruptId": "interruptId-002", "status": "cancelled", "payload": {"feedback": "不要添加水印"}}
  ]
}

{
  "threadId": "thread-001",
  "runId": "run-001",
  "messages": [{"role": "user", "content": "搜索海浪视频"}],
  "forwardedProps": {"scenario_name": "video-mixcut"}
}

{
  "threadId": "thread-001",
  "runId": "run-001",
  "messages": [
    {
      "role": "user",
      "content": [
        {
          "type": "text",
          "text": "帮我混剪这些素材"
        },
        {
          "type": "mixcut_assets",
          "metadata": {
            "attachments": [
              {"url": " http://example.com/video-a.mp4"},
              {"fileId": "vod-file-001"}
            ]
          }
        }
      ]
    }
  ]
}

{
  "name": "<工具名>",
  "description": "<工具用途描述，LLM 据此决策何时调用>",
  "parameters": <工具参数的JSON schema定义>
}

{
  "name": "clarify",
  "description": "Ask the user a question when you need clarification...",
  "parameters": {
    "type": "object",
    "properties": {
      "question": {
        "type": "string",
        "description": "The question itself. Do NOT embed options here."
      },
      "choices": {
        "type": "array",
        "items": { "type": "string" },
        "maxItems": 4,
        "description": "Selectable options (up to 4). Omit for free-text."
      }
    },
    "required": ["question"]
  }
}

请求 → LLM 决策调用外部工具
  ↓  SSE 事件流
  TOOL_CALL_START   { toolCallId, toolCallName }
  TOOL_CALL_ARGS    { delta (JSON fragment) }
  TOOL_CALL_END     { toolCallId }
  ↓  工具执行被挂起，Agent结束当前轮次
  RUN_FINISHED

[客户端执行外部工具，填入结果]
  ↓
  发送新请求（/agui/chat）
  {
    "threadId": "...",
    "runId": "...",
    "messages": [ { "role": "tool", "content": "<工具执行结果>", "toolCallId": "<id>" } ],
    "tools": [ ... ]
  }
  → Agent 继续执行

id: TOOL_CALL_START_xxx
data: {"type":"TOOL_CALL_START","timestamp":...,"toolCallId":"call-ext-001","toolCallName":"external_search","parentMessageId":"msg-001"}

id: TOOL_CALL_ARGS_xxx
data: {"type":"TOOL_CALL_ARGS","timestamp":...,"toolCallId":"call-ext-001","delta":"{\\"query\\":\\"VOD 架构文档\\"}"}

id: TOOL_CALL_END_xxx
data: {"type":"TOOL_CALL_END","timestamp":...,"toolCallId":"call-ext-001"}

id: RUN_FINISHED_xxx
data: {"type":"RUN_FINISHED","timestamp":...,"threadId":"thread-001","runId":"run-001"}

{
  "threadId": "my-thread-001",
  "runId": "run-002",
  "messages": [
    {
      "role": "tool",
      "content": "<工具执行结果（字符串，LLM 会解读）>",
      "toolCallId": "call-ext-001"
    }
  ],
  "tools": [
    {
      "name": "clarify",
      "description": "...",
      "parameters": {"type":"object","properties":{}}
    }
  ]
}

from ag_ui.client import HttpAgent
from ag_ui.core import RunAgentInput, Message, ResumeEntry

BASE_URL = "https://smartmedia.vod-qcloud.com/agui"
API_KEY = "<your-token>"

def run_agent(thread_id, run_id, message, resume=None, forwarded_props=None):
    agent = HttpAgent(
        base_url=BASE_URL,
        api_key=API_KEY,
    )

    input_params = RunAgentInput(
        thread_id=thread_id,
        run_id=run_id,
        messages=[Message(role="user", content=message)],
    )
    if forwarded_props:
        input_params.forwarded_props = forwarded_props
    if resume:
        input_params.resume = resume

    interrupts = []
    for event in agent.run(input_params):
        etype = event.type
        if etype == "TEXT_MESSAGE_CONTENT":
            print(event.delta, end="", flush=True)
        elif etype == "TOOL_CALL_START":
            print(f"\\n[工具] {event.tool_call_name}")
        elif etype == "RUN_FINISHED":
            outcome = getattr(event, "outcome", None)
            if outcome and outcome.type == "interrupt":
                interrupts = outcome.interrupts
                for i, intr in enumerate(interrupts):
                    print(f"\\n[中断 {i + 1}] {intr.message} (id: {intr.id})")
        elif etype == "RUN_ERROR":
            print(f"\\n[错误] {event.message}")
    return interrupts

# 1. 首次对话
interrupts = run_agent(
    thread_id="thread-001",
    run_id="run-001",
    message="搜索海浪视频并裁剪前10秒",
    forwarded_props={
        "approval_mode": "level:low",
        "scenario_name": "media_management",
    },
)

# 2. 每工具独立决策 — 批准第一个，拒绝第二个
if interrupts:
    print(f"\\n共 {len(interrupts)} 个工具需要确认")
    resume = []
    for i, intr in enumerate(interrupts):
        if i == 0:
            resume.append(ResumeEntry(
                interrupt_id=intr.id,
                status="resolved",
            ))
        else:
            resume.append(ResumeEntry(
                interrupt_id=intr.id,
                status="cancelled",
                payload={"feedback": "不需要此操作"},
            ))

    # 3. 恢复（需携带相同的 approval_mode）
    run_agent(
        "thread-001", "run-002", "",
        resume=resume,
        forwarded_props={"approval_mode": "level:low"},
    )

import { HttpAgent } from "@ag-ui/client";
import type { RunAgentInput, Interrupt } from "@ag-ui/core";

const BASE_URL = "https://smartmedia.vod-qcloud.com/agui";
const API_KEY = "<your-token>";

async function runAgent(params: {
  threadId: string;
  runId: string;
  message: string;
  forwardedProps?: Record<string, unknown>;
  resume?: Array<{
    interruptId: string;
    status: "resolved" | "cancelled";
    payload?: Record<string, unknown>;
  }>;
}): Promise<Interrupt[]> {
  const agent = new HttpAgent({
    serverUrl: BASE_URL,
    apiKey: API_KEY,
  });

  const input: RunAgentInput = {
    threadId: params.threadId,
    runId: params.runId,
    messages: [{ role: "user", content: params.message }],
    forwardedProps: params.forwardedProps,
    resume: params.resume,
  };

  const interrupts: Interrupt[] = [];

  // 订阅事件流
  const unsubscribe = agent.subscribe((event: any) => {
    switch (event.type) {
      case "TEXT_MESSAGE_CONTENT":
        process.stdout.write(event.delta || "");
        break;
      case "TOOL_CALL_START":
        console.log(`\\n[Tool] ${event.toolCallName}`);
        break;
      case "RUN_FINISHED":
        if (event.outcome?.type === "interrupt") {
          interrupts.push(...event.outcome.interrupts);
          interrupts.forEach((intr, i) =>
            console.log(`\\n[Interrupt ${i + 1}] ${intr.message} (id: ${intr.id})`)
          );
        }
        break;
      case "RUN_ERROR":
        console.error(`\\n[Error] ${event.message}`);
        break;
    }
  });

  // 运行 Agent
  await agent.run(input);
  unsubscribe();
  return interrupts;
}

// 使用示例：每工具独立审批
async function main() {
  // 1. 首次对话
  const interrupts = await runAgent({
    threadId: "thread-001",
    runId: "run-001",
    message: "搜索海浪视频并裁剪前10秒",
    forwardedProps: {
      approval_mode: "level:low",
      scenario_name: "media_management",
    },
  });

  // 2. 每工具独立决策 — 批准第一个，拒绝第二个
  if (interrupts.length > 0) {
    console.log(`\\n共 ${interrupts.length} 个工具需要确认`);
    await runAgent({
      threadId: "thread-001",
      runId: "run-002",
      message: "",
      forwardedProps: { approval_mode: "level:low" },
      resume: interrupts.map((intr, idx) => ({
        interruptId: intr.id,
        status: idx === 0 ? "resolved" as const : "cancelled" as const,
        payload: idx === 0 ? undefined : { feedback: "不需要此操作" },
      })),
    });
  }
}

main();