【腾讯云 HAI域探秘】利用HAI+ChatGLM-6B轻松创作个人专属的知识宇宙-基于腾讯云CloudStudio和HAI

cd ./ChatGLM2-6B
python api.py

import requests

# 定义测试数据，以及FastAPI服务器的地址和端口
server_url = "http://0.0.0.0:8000"  # 请确保将地址和端口更改为您的API服务器的实际地址和端口
test_data = {
    "prompt": "'你好，发热了怎么办？'",
    "history": [],
    "max_length": 50,
    "top_p": 0.7,
    "temperature": 0.95
}

# 发送HTTP POST请求
response = requests.post(server_url, json=test_data)

# 处理响应
if response.status_code == 200:
    result = response.json()
    print("Response:", result["response"])
    print("History:", result["history"])
    print("Status:", result["status"])
    print("Time:", result["time"])
else:
    print("Failed to get a valid response. Status code:", response.status_code)

# -*-coding:utf-8-*-
'''
File Name:chatglm2-6b-stream-api.py
Author:Luofan
Time:2023/6/26 13:33
'''

import os
import sys
import json
import torch
import uvicorn
import logging
import argparse
from fastapi import FastAPI
from transformers import AutoTokenizer, AutoModel
from fastapi.middleware.cors import CORSMiddleware
from sse_starlette.sse import ServerSentEvent, EventSourceResponse


def getLogger(name, file_name, use_formatter=True):
    logger = logging.getLogger(name)
    logger.setLevel(logging.INFO)
    console_handler = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter('%(asctime)s    %(message)s')
    console_handler.setFormatter(formatter)
    console_handler.setLevel(logging.INFO)
    logger.addHandler(console_handler)
    if file_name:
        handler = logging.FileHandler(file_name, encoding='utf8')
        handler.setLevel(logging.INFO)
        if use_formatter:
            formatter = logging.Formatter('%(asctime)s - %(name)s - %(message)s')
            handler.setFormatter(formatter)
        logger.addHandler(handler)
    return logger


logger = getLogger('ChatGLM', 'chatlog.log')

MAX_HISTORY = 3


class ChatGLM():
    def __init__(self) -> None:
        logger.info("Start initialize model...")
        self.tokenizer =  AutoTokenizer.from_pretrained("THUDM/chatglm2-6b", revision="v1.0", trust_remote_code=True)
        self.model = AutoModel.from_pretrained("THUDM/chatglm2-6b", revision="v1.0", trust_remote_code=True).cuda()
        self.model.eval()
        logger.info("Model initialization finished.")

    def clear(self) -> None:
        if torch.cuda.is_available():
            with torch.cuda.device(f"cuda:{args.device}"):
                torch.cuda.empty_cache()
                torch.cuda.ipc_collect()

    def answer(self, query: str, history):
        response, history = self.model.chat(self.tokenizer, query, history=history)
        history = [list(h) for h in history]
        return response, history

    def stream(self, query, history):
        if query is None or history is None:
            yield {"query": "", "response": "", "history": [], "finished": True}
        size = 0
        response = ""
        for response, history in self.model.stream_chat(self.tokenizer, query, history):
            this_response = response[size:]
            history = [list(h) for h in history]
            size = len(response)
            yield {"delta": this_response, "response": response, "finished": False}
        logger.info("Answer - {}".format(response))
        yield {"query": query, "delta": "[EOS]", "response": response, "history": history, "finished": True}


def start_server(http_address: str, port: int, gpu_id: str):
    os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
    os.environ['CUDA_VISIBLE_DEVICES'] = gpu_id

    bot = ChatGLM()

    app = FastAPI()
    app.add_middleware(CORSMiddleware,
                       allow_origins=["*"],
                       allow_credentials=True,
                       allow_methods=["*"],
                       allow_headers=["*"]
                       )

    @app.get("/")
    def index():
        return {'message': 'started', 'success': True}

    @app.post("/chat")
    async def answer_question(arg_dict: dict):
        result = {"query": "", "response": "", "success": False}
        try:
            text = arg_dict["query"]
            ori_history = arg_dict["history"]
            logger.info("Query - {}".format(text))
            if len(ori_history) > 0:
                logger.info("History - {}".format(ori_history))
            history = ori_history[-MAX_HISTORY:]
            history = [tuple(h) for h in history]
            response, history = bot.answer(text, history)
            logger.info("Answer - {}".format(response))
            ori_history.append((text, response))
            result = {"query": text, "response": response,
                      "history": ori_history, "success": True}
        except Exception as e:
            logger.error(f"error: {e}")
        return result

    @app.post("/stream")
    def answer_question_stream(arg_dict: dict):
        def decorate(generator):
            for item in generator:
                #yield ServerSentEvent(json.dumps(item, ensure_ascii=False), event='delta')
                yield ServerSentEvent(json.dumps(item, ensure_ascii=False))

        try:
            text = arg_dict["query"]
            ori_history = arg_dict["history"]
            logger.info("Query - {}".format(text))
            if len(ori_history) > 0:
                logger.info("History - {}".format(ori_history))
            history = ori_history[-MAX_HISTORY:]
            history = [tuple(h) for h in history]
            return EventSourceResponse(decorate(bot.stream(text, history)))
        except Exception as e:
            logger.error(f"error: {e}")
            return EventSourceResponse(decorate(bot.stream(None, None)))

    @app.get("/free_gc")
    def free_gpu_cache():
        try:
            bot.clear()
            return {"success": True}
        except Exception as e:
            logger.error(f"error: {e}")
            return {"success": False}

    logger.info("starting server...")
    uvicorn.run(app=app, host=http_address, port=port, workers=1)


if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Stream API Service for ChatGLM2-6B')
    parser.add_argument('--device', '-d', help='device，-1 means cpu, other means gpu ids', default='0')
    parser.add_argument('--host', '-H', help='host to listen', default='0.0.0.0')
    parser.add_argument('--port', '-P', help='port of this service', default=8000)
    args = parser.parse_args()
    start_server(args.host, int(args.port), args.device)

python chatglm2-6b-stream-api.py

import requests
import json

# 设置服务器地址和端口
server_address = "http://0.0.0.0"  # 请将地址和端口更改为实际的服务器地址和端口
server_port = 8000

# 构造请求数据
request_data = {
    "query": "你好，发热了怎么办？",
    "history": []
}

# 发送HTTP POST请求到聊天端点
response = requests.post(f"{server_address}:{server_port}/chat", json=request_data)

# 处理响应
if response.status_code == 200:
    result = response.json()
    if result["success"]:
        print("Response:", result["response"])
        print("History:", result["history"])
    else:
        print("Failed to get a valid response.")
else:
    print("Failed to connect to the server. Status code:", response.status_code)

python use_chatglm2-6b-stream-api.py

import aiohttp  
import json  
import asyncio  
  
async def main():  
    async with aiohttp.ClientSession() as session:  
        server_address = "http://0.0.0.0"  # 请将地址更改为实际的服务器地址  
        server_port = 8000  
        endpoint = f"{server_address}:{server_port}/stream"  # 流式处理端点  
  
        request_data = {  
            "query": "你好，发热怎么办？",  
            "history": []  
        }  
  
        try:  
            async with session.post(endpoint, json=request_data, timeout=None) as response:  
                if response.status == 200:  
                    async for line in response.content.iter_any():  
                        data = line.decode('utf-8')  
                        data = data.replace('event: delta\r\ndata: ','')
                        data = data.replace('\r\n','')
                        data = data.replace('data: ','')
                        try:  
                            result = json.loads(data)  
                            if result.get("finished"):  
                                    print("Response:", result["response"])  
                                    print('\r\n')
                                    print("History:", result["history"])  
                            #else:  
                            #    print("Delta:", result["delta"])  
                        except json.JSONDecodeError:  
                            print("Failed to parse JSON:")  
                else:  
                    print(f"Failed to connect to the server. Status code: {response.status}")  
        except aiohttp.ClientError as e:  
            print(f"Client error occurred: {e}")  
            await asyncio.sleep(1)  # Avoid continuous failures, wait a bit before retrying  
            print(f"Unexpected error occurred: {e}")  
            await asyncio.sleep(1)  # Avoid continuous failures, wait a bit before retrying  
  
if __name__ == '__main__':  
    asyncio.run(main())

pip install aiohttp 

python use_stream_chatglm2-6b-stream-api.py

# coding=utf-8
# Implements API for ChatGLM2-6B in OpenAI's format. (https://platform.openai.com/docs/api-reference/chat)
# Usage: python openai_api.py
# Visit http://localhost:8000/docs for documents.

import time
import torch
import uvicorn
from pydantic import BaseModel, Field
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from contextlib import asynccontextmanager
from typing import Any, Dict, List, Literal, Optional, Union
from transformers import AutoTokenizer, AutoModel
from sse_starlette.sse import ServerSentEvent, EventSourceResponse


@asynccontextmanager
async def lifespan(app: FastAPI): # collects GPU memory
    yield
    if torch.cuda.is_available():
        torch.cuda.empty_cache()
        torch.cuda.ipc_collect()


app = FastAPI(lifespan=lifespan)

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

class ModelCard(BaseModel):
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
    owned_by: str = "owner"
    root: Optional[str] = None
    parent: Optional[str] = None
    permission: Optional[list] = None


class ModelList(BaseModel):
    object: str = "list"
    data: List[ModelCard] = []


class ChatMessage(BaseModel):
    role: Literal["user", "assistant", "system"]
    content: str


class DeltaMessage(BaseModel):
    role: Optional[Literal["user", "assistant", "system"]] = None
    content: Optional[str] = None


class ChatCompletionRequest(BaseModel):
    model: str
    messages: List[ChatMessage]
    temperature: Optional[float] = None
    top_p: Optional[float] = None
    max_length: Optional[int] = None
    stream: Optional[bool] = False


class ChatCompletionResponseChoice(BaseModel):
    index: int
    message: ChatMessage
    finish_reason: Literal["stop", "length"]


class ChatCompletionResponseStreamChoice(BaseModel):
    index: int
    delta: DeltaMessage
    finish_reason: Optional[Literal["stop", "length"]]


class ChatCompletionResponse(BaseModel):
    model: str
    object: Literal["chat.completion", "chat.completion.chunk"]
    choices: List[Union[ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice]]
    created: Optional[int] = Field(default_factory=lambda: int(time.time()))


@app.get("/v1/models", response_model=ModelList)
async def list_models():
    global model_args
    model_card = ModelCard(id="gpt-3.5-turbo")
    return ModelList(data=[model_card])


@app.post("/v1/chat/completions", response_model=ChatCompletionResponse)
async def create_chat_completion(request: ChatCompletionRequest):
    global model, tokenizer

    if request.messages[-1].role != "user":
        raise HTTPException(status_code=400, detail="Invalid request")
    query = request.messages[-1].content

    prev_messages = request.messages[:-1]
    if len(prev_messages) > 0 and prev_messages[0].role == "system":
        query = prev_messages.pop(0).content + query

    history = []
    if len(prev_messages) % 2 == 0:
        for i in range(0, len(prev_messages), 2):
            if prev_messages[i].role == "user" and prev_messages[i+1].role == "assistant":
                history.append([prev_messages[i].content, prev_messages[i+1].content])

    if request.stream:
        generate = predict(query, history, request.model)
        return EventSourceResponse(generate, media_type="text/event-stream")

    response, _ = model.chat(tokenizer, query, history=history)
    choice_data = ChatCompletionResponseChoice(
        index=0,
        message=ChatMessage(role="assistant", content=response),
        finish_reason="stop"
    )

    return ChatCompletionResponse(model=request.model, choices=[choice_data], object="chat.completion")


async def predict(query: str, history: List[List[str]], model_id: str):
    global model, tokenizer

    choice_data = ChatCompletionResponseStreamChoice(
        index=0,
        delta=DeltaMessage(role="assistant"),
        finish_reason=None
    )
    chunk = ChatCompletionResponse(model=model_id, choices=[choice_data], object="chat.completion.chunk")
    #yield "{}".format(chunk.json(exclude_unset=True, ensure_ascii=False))
    yield "{}".format(chunk.model_dump_json(exclude_unset=True))

    current_length = 0

    for new_response, _ in model.stream_chat(tokenizer, query, history):
        if len(new_response) == current_length:
            continue

        new_text = new_response[current_length:]
        current_length = len(new_response)

        choice_data = ChatCompletionResponseStreamChoice(
            index=0,
            delta=DeltaMessage(content=new_text),
            finish_reason=None
        )
        chunk = ChatCompletionResponse(model=model_id, choices=[choice_data], object="chat.completion.chunk")
        #yield "{}".format(chunk.json(exclude_unset=True, ensure_ascii=False))
        yield "{}".format(chunk.model_dump_json(exclude_unset=True))

    choice_data = ChatCompletionResponseStreamChoice(
        index=0,
        delta=DeltaMessage(),
        finish_reason="stop"
    )
    chunk = ChatCompletionResponse(model=model_id, choices=[choice_data], object="chat.completion.chunk")
    #yield "{}".format(chunk.json(exclude_unset=True, ensure_ascii=False))
    yield "{}".format(chunk.model_dump_json(exclude_unset=True))
    yield '[DONE]'



if __name__ == "__main__":
    tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm2-6b", revision="v1.0", trust_remote_code=True)
    model = AutoModel.from_pretrained("THUDM/chatglm2-6b", revision="v1.0", trust_remote_code=True).cuda()
    # 多显卡支持，使用下面两行代替上面一行，将num_gpus改为你实际的显卡数量
    # from utils import load_model_on_gpus
    # model = load_model_on_gpus("THUDM/chatglm2-6b", num_gpus=2)
    model.eval()

    uvicorn.run(app, host='0.0.0.0', port=8000, workers=1)

python openai_api.py

npm install 

yarn dev

cd ./ChatGLM2-6B
git clone https://gitee.com/dadac/chatglm2-6b-int4.git
# 输入你的gitee账号密码
cd chatglm2-6b-int4

ssh-keygen -t ed25519 -C ""  #双引号里面填ssh key，一般填自己邮箱
# 回车一直默认即可，会在家目录生成 ~/.ssh 文件夹，存放密钥信息
cat ~/.ssh/id_ed25519.pub  # 查看生成的公钥

在Jupyter终端中依次执行以下命令
git clone git@gitee.com:dadac/chatglm2-6b-int4.git 
# 输入yes 回车确认

rm pytorch_model.bin #删除旧文件
wget http://xxxxxx   # 复制的pytorch_model.bin 文件链接
rm tokenizer.model   #删除旧文件
wget http://xxxxxx   # 复制的tokenizer.model 文件链接

tokenizer = AutoTokenizer.from_pretrained("./chatglm2-6b-int4", revision="v1.0", trust_remote_code=True)
model = AutoModel.from_pretrained("./chatglm2-6b-int4", revision="v1.0", trust_remote_code=True).half().cuda()

apt-get update && apt-get install sudo
sudo apt-get update  
sudo apt-get install psmisc
sudo fuser -k 6889/tcp #执行这条命令将关闭 HAI提供的 chatglm2_gradio webui功能

cd /root/ChatGLM2-6B
python web_demo.py --listen --port 8000

[	
	{
        "content": "你好，你是谁",
        "summary": "你好，我是腾讯云高性能应用HAI的助手小hai。"
    },
	{
        "content": "你是谁",
        "summary": "你好，我是HAI的助手小hai。"
    },
    {
        "content": "HAI是谁",
        "summary": "HAI是为开发者量身打造的澎湃算力平台。无需复杂配置，便可享受即开即用的GPU云服务体验。在HAI中，根据应用智能匹配并推选出最适合的GPU算力资源，以确保您在数据科学、LLM、AI作画等高性能应用中获得最佳性价比。"
    }
]

[	
	{
        "content": "你好，你是谁",
        "summary": "你好，我是腾讯云高性能应用HAI的助手小hai。"
    },
	{
        "content": "你是谁",
        "summary": "你好，我是HAI的助手小hai。"
    },
    {
        "content": "HAI是谁",
        "summary": "HAI是为开发者量身打造的澎湃算力平台。无需复杂配置，便可享受即开即用的GPU云服务体验。在HAI中，根据应用智能匹配并推选出最适合的GPU算力资源，以确保您在数据科学、LLM、AI作画等高性能应用中获得最佳性价比。"
    }
]

PRE_SEQ_LEN=128
LR=1e-2
NUM_GPUS=1

torchrun --standalone --nnodes=1 --nproc-per-node=$NUM_GPUS main.py \
    --do_train \
    --train_file train.json \
    --validation_file verify.json \
    --preprocessing_num_workers 10 \
    --prompt_column content \
    --response_column summary \
    --overwrite_cache \
    --model_name_or_path /root/ChatGLM2-6B/chatglm2-6b-int4 \
    --output_dir /root/ChatGLM2-6B/output/chatglm2-6b-int4−$LR \
    --overwrite_output_dir \
    --max_source_length 512 \
    --max_target_length 512 \
    --per_device_train_batch_size 1 \
    --per_device_eval_batch_size 1 \
    --gradient_accumulation_steps 16 \
    --predict_with_generate \
    --max_steps 3000 \
    --logging_steps 10 \
    --save_steps 1000 \
    --learning_rate $LR \
    --pre_seq_len $PRE_SEQ_LEN \
    --quantization_bit 4

pip install -i https://mirrors.cloud.tencent.com/pypi/simple rouge_chinese nltk jieba datasets

cd ChatGLM2-6B/ptuning
sh train_chat.sh

from transformers import AutoModel, AutoTokenizer,AutoConfig
import gradio as gr
import mdtex2html
import torch
import os
from utils import load_model_on_gpus


MODEL_PATH = "./chatglm2-6b-int4"
CHECKPOINT_PATH = "./output/chatglm2-6b-int4−1e-2/checkpoint-3000"
 
# 载入Tokenizer
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
 
config = AutoConfig.from_pretrained(MODEL_PATH, trust_remote_code=True, pre_seq_len=128)
model = AutoModel.from_pretrained(MODEL_PATH, config=config, trust_remote_code=True).cuda()
 
prefix_state_dict = torch.load(os.path.join(CHECKPOINT_PATH, "pytorch_model.bin"))
new_prefix_state_dict = {}
 
for k, v in prefix_state_dict.items():
    if k.startswith("transformer.prefix_encoder."):
        new_prefix_state_dict[k[len("transformer.prefix_encoder."):]] = v
model.transformer.prefix_encoder.load_state_dict(new_prefix_state_dict)
 
print(f"Quantized to 4 bit")
model = model.quantize(4)
model = model.half().cuda()
model.transformer.prefix_encoder.float()
model = model.eval()

#tokenizer = AutoTokenizer.from_pretrained("./chatglm2-6b-int4", revision="v1.0", trust_remote_code=True)
#model = AutoModel.from_pretrained("./chatglm2-6b-int4", revision="v1.0", trust_remote_code=True).half().cuda()

# 多显卡支持，使用下面两行代替上面一行，将num_gpus改为你实际的显卡数量
# from utils import load_model_on_gpus
# model = load_model_on_gpus("THUDM/chatglm2-6B", num_gpus=2)
#model = model.eval()

"""Override Chatbot.postprocess"""


def postprocess(self, y):
    if y is None:
        return []
    for i, (message, response) in enumerate(y):
        y[i] = (
            None if message is None else mdtex2html.convert((message)),
            None if response is None else mdtex2html.convert(response),
        )
    return y


gr.Chatbot.postprocess = postprocess


def parse_text(text):
    """copy from https://github.com/GaiZhenbiao/ChuanhuChatGPT/"""
    lines = text.split("\n")
    lines = [line for line in lines if line != ""]
    count = 0
    for i, line in enumerate(lines):
        if "```" in line:
            count += 1
            items = line.split('`')
            if count % 2 == 1:
                lines[i] = f'<pre><code class="language-{items[-1]}">'
            else:
                lines[i] = f'<br></code></pre>'
        else:
            if i > 0:
                if count % 2 == 1:
                    line = line.replace("`", "\`")
                    line = line.replace("<", "&lt;")
                    line = line.replace(">", "&gt;")
                    line = line.replace(" ", "&nbsp;")
                    line = line.replace("*", "&ast;")
                    line = line.replace("_", "&lowbar;")
                    line = line.replace("-", "&#45;")
                    line = line.replace(".", "&#46;")
                    line = line.replace("!", "&#33;")
                    line = line.replace("(", "&#40;")
                    line = line.replace(")", "&#41;")
                    line = line.replace("$", "&#36;")
                lines[i] = "<br>"+line
    text = "".join(lines)
    return text


def predict(input, chatbot, max_length, top_p, temperature, history, past_key_values):
    chatbot.append((parse_text(input), ""))
    for response, history, past_key_values in model.stream_chat(tokenizer, input, history, past_key_values=past_key_values,
                                                                return_past_key_values=True,
                                                                max_length=max_length, top_p=top_p,
                                                                temperature=temperature):
        chatbot[-1] = (parse_text(input), parse_text(response))

        yield chatbot, history, past_key_values


def reset_user_input():
    return gr.update(value='')


def reset_state():
    return [], [], None


if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--listen", action='store_true',
                        help="launch gradio with 0.0.0.0 as server name, allowing to respond to network requests")
    parser.add_argument("--port", type=int,
                        help="launch gradio with given server port, you need root/admin rights for ports < 1024, defaults to 7860 if available",
                        default=None)
    args = parser.parse_args()
    server_name = "0.0.0.0" if args.listen else None
    server_port = args.port

    with gr.Blocks() as demo:
        gr.HTML("""<h1 align="center">ChatGLM2-6B</h1>""")

        chatbot = gr.Chatbot()
        with gr.Row():
            with gr.Column(scale=4):
                with gr.Column(scale=12):
                    user_input = gr.Textbox(show_label=False, placeholder="Input...", lines=10).style(
                        container=False)
                with gr.Column(min_width=32, scale=1):
                    submitBtn = gr.Button("Submit", variant="primary")
            with gr.Column(scale=1):
                emptyBtn = gr.Button("Clear History")
                max_length = gr.Slider(0, 32768, value=8192, step=1.0, label="Maximum length", interactive=True)
                top_p = gr.Slider(0, 1, value=0.8, step=0.01, label="Top P", interactive=True)
                temperature = gr.Slider(0, 1, value=0.95, step=0.01, label="Temperature", interactive=True)

        history = gr.State([])
        past_key_values = gr.State(None)

        submitBtn.click(predict, [user_input, chatbot, max_length, top_p, temperature, history, past_key_values],
                        [chatbot, history, past_key_values], show_progress=True)
        submitBtn.click(reset_user_input, [], [user_input])

        emptyBtn.click(reset_state, outputs=[chatbot, history, past_key_values], show_progress=True)

        demo.queue().launch(share=False, inbrowser=True, server_name=server_name, server_port=server_port)

python web_demo.py --listen --port 8000