导入和部署自定义 LLM 大模型（平台内置推理镜像）

!pip install modelscope

from modelscope import snapshot_download
#qwen/Qwen2-7B-Instruct为需要下载的模型名称，cache_dir为下载模型保存的地址，这里'./'表示将下载模型保存在CFS的根目录中
model_dir = snapshot_download('qwen/Qwen2-7B-Instruct', cache_dir='./')

{"messages":[{"role": "user", "content": "你是谁"}]}

# 公网访问地址
SERVER_URL = https://ms-gp6rjk2jj-**********.gw.ap-shanghai.ti.tencentcs.com/ms-gp6rjk2j

# 非流式调用
curl -H "content-type: application/json" ${SERVER_URL}/v1/chat/completions -d '{"messages":[{"role": "user", "content": "你好"}], "temperature": 0.0}'
# 流式调用
curl -H "content-type: application/json" ${SERVER_URL}/v1/chat/completions -d '{"messages":[{"role": "user", "content": "你好"}], "temperature": 0.0, "stream": true}'

{"id":"chatcmpl-4aeRgYwnaYe4RzmmcyKtYs","object":"chat.completion","created":1698291242,"model":"baichuan-13b-chat","choices":[{"index":0,"message":{"role":"assistant","content":"你好！有什么我能帮到你的吗？"},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"total_tokens":16,"completion_tokens":12}}

data: {"id": "chatcmpl-hn5mCVt4szVVZBa4fVFZWF", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": null}]}

data: {"id": "chatcmpl-hn5mCVt4szVVZBa4fVFZWF", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": " 你"}, "finish_reason": null}], "usage": {"prompt_tokens": 4, "total_tokens": 5, "completion_tokens": 1}}

data: {"id": "chatcmpl-hn5mCVt4szVVZBa4fVFZWF", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "好"}, "finish_reason": null}], "usage": {"prompt_tokens": 4, "total_tokens": 6, "completion_tokens": 2}}

..........此处省略中间结果..............

data: {"id": "chatcmpl-hn5mCVt4szVVZBa4fVFZWF", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "？"}, "finish_reason": null}], "usage": {"prompt_tokens": 4, "total_tokens": 15, "completion_tokens": 11}}

data: {"id": "chatcmpl-hn5mCVt4szVVZBa4fVFZWF", "object": "chat.completion.chunk", "created": 1714120317, "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 4, "total_tokens": 16, "completion_tokens": 12}}

data: [DONE]

import argparse
import requests
import json


def chat(messages):
    data = {
        "messages": messages,
        "temperature": args.temperature,
        "max_tokens": args.max_tokens,
        "top_p": args.top_p,
        "stream": True,  # 开启流式输出
    }
    header = {
        "Content-Type": "application/json",
    }
    if args.token:
        header["Authorization"] = f"Bearer {args.token}"

    response = requests.post(f"{args.server}/v1/chat/completions", json=data, headers=header, stream=True)  # 设置 stream=True 参数获取实时数据流
    if response.status_code != 200:
        print(response.json())
        exit()

    result = ""
    print("Assistant: ", end = "", flush = True)
    for part in response.iter_lines():
        if part:
            if "content" in part.decode("utf-8"):
                content = json.loads(part.decode("utf-8")[5:])["choices"][0]["delta"]["content"]  # 字符串过滤 data: 之后转成 json 格式再取文本
                result += content
                print(content, end = "", flush = True)
    print("\\n")

    return result


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Chat CLI Demo.")    
    # --server替换为实际的调用地址
    parser.add_argument("--server", type=str, default="https://ms-gp6rjk2j-*******.gw.ap-shanghai.ti.tencentcs.com/ms-gp6rjk2j")
    parser.add_argument("--max-tokens", type=int, default=512)
    parser.add_argument("--temperature", type=float, default=0.7)
    parser.add_argument("--top_p", type=float, default=1.0)
    parser.add_argument("--system", type=str, default=None)
    parser.add_argument("--token", type=str, default=None)
    args = parser.parse_args()

    messages = []
    if args.system:
        messages.append({"role": "system", "content": args.system})

    while True:
        user_input = input("User: ")
        messages.append({"role": "user", "content": user_input})
        response = chat(messages)
        messages.append({"role": "assistant", "content": response})