Baichuan

docker run -it --name HFTGI_baichuan_13b_chat --gpus all -e HF_ENDPOINT="https://hf-mirror.com" -e HF_HUB_OFFLINE=1 -p 8080:80 -v $PWD/data:/data ghcr.io/huggingface/text-generation-inference:latest --model-id baichuan-inc/Baichuan-13B-Chat --trust-remote-code

...
INFO hf_hub: Token file not found "/root/.cache/huggingface/token"    
INFO text_generation_launcher: Default `max_input_tokens` to 4095
INFO text_generation_launcher: Default `max_total_tokens` to 4096
INFO text_generation_launcher: Default `max_batch_prefill_tokens` to 4145
INFO text_generation_launcher: Using default cuda graphs [1, 2, 4, 8, 16, 32]
WARN text_generation_launcher: `trust_remote_code` is set. Trusting that model `baichuan-inc/Baichuan-13B-Chat` do not contain malicious code.
INFO download: text_generation_launcher: Starting check and download process for baichuan-inc/Baichuan-13B-Chat
INFO text_generation_launcher: Detected system cuda
WARN text_generation_launcher: No safetensors weights found for model baichuan-inc/Baichuan-13B-Chat at revision None. Converting PyTorch weights to safetensors.
INFO text_generation_launcher: Convert: [1/3] -- Took: 0:00:10.643016
INFO text_generation_launcher: Convert: [2/3] -- Took: 0:00:09.686510
INFO text_generation_launcher: Convert: [3/3] -- Took: 0:00:06.149757
INFO download: text_generation_launcher: Successfully downloaded weights for baichuan-inc/Baichuan-13B-Chat
INFO shard-manager: text_generation_launcher: Starting shard rank=0
INFO text_generation_launcher: Detected system cuda
INFO shard-manager: text_generation_launcher: Waiting for shard to be ready... rank=0
INFO text_generation_launcher: Server started at unix:///tmp/text-generation-server-0
INFO shard-manager: text_generation_launcher: Shard ready in 12.109051544s rank=0
INFO text_generation_launcher: Starting Webserver
WARN text_generation_router: router/src/main.rs:218: Offline mode active using cache defaults
INFO text_generation_router: router/src/main.rs:349: Using config Some(Baichuan)
WARN text_generation_router: router/src/main.rs:351: Could not find a fast tokenizer implementation for baichuan-inc/Baichuan-13B-Chat
WARN text_generation_router: router/src/main.rs:352: Rust input length validation and truncation is disabled
WARN text_generation_router: router/src/main.rs:358: no pipeline tag found for model baichuan-inc/Baichuan-13B-Chat
WARN text_generation_router: router/src/main.rs:376: Invalid hostname, defaulting to 0.0.0.0
INFO text_generation_router::server: router/src/server.rs:1577: Warming up model
INFO text_generation_launcher: Cuda Graphs are enabled for sizes [32, 16, 8, 4, 2, 1]
INFO text_generation_router::server: router/src/server.rs:1604: Using scheduler V3
INFO text_generation_router::server: router/src/server.rs:1656: Setting max batch total tokens to 21872
INFO text_generation_router::server: router/src/server.rs:1894: Connected

#重新启动容器
docker start HFTGI_baichuan_13b_chat

#实时查看Web Server的运行状态
docker logs -f HFTGI_baichuan_13b_chat

curl -v http://localhost:8080/generate -X POST -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":68}}' -H 'Content-Type: application/json'

#将pip换成清华源加快下载速度
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple

#安装text-generation包
pip install text-generation

from text_generation import Client


# Generate
client = Client("http://localhost:8080")
output = client.generate("Why is the sky blue?", max_new_tokens=92).generated_text
print(f"Generate Output: {output}")


# Generate stream
text = ""
for response in client.generate_stream("Why is the sky blue?", max_new_tokens=92):
    if not response.token.special:
        text += response.token.text
print(f"Generate stream Output: {text}")

python TGI_Client.py

#将pip换成清华源加快下载速度
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple

#安装必要的包
pip install langchain transformers langchain_community

# Wrapper to TGI client with langchain


from langchain.llms import HuggingFaceTextGenInference
from langchain import PromptTemplate, LLMChain


inference_server_url_local = "http://localhost:8080"


llm_local = HuggingFaceTextGenInference(
    inference_server_url=inference_server_url_local,
    max_new_tokens=400,
    top_k=10,
    top_p=0.95,
    typical_p=0.95,
    temperature=0.7,
    repetition_penalty=1.03,
)


question = "whats 2 * (1 + 2)"
template = """Question: {question}
Answer: Let's think step by step."""


prompt = PromptTemplate(
    template=template, 
    input_variables= ["question"]
)


llm_chain_local = LLMChain(prompt=prompt, llm=llm_local)
output = llm_chain_local({question})
print(output)

python LangChain.py