LLaMA

docker run -it --name HFTGI_llama_7b --gpus all -e HF_ENDPOINT="https://hf-mirror.com" -e HF_HUB_OFFLINE=1 -p 8080:80 -v $PWD/data:/data ghcr.io/huggingface/text-generation-inference:latest --model-id openlm-research/open_llama_7b --trust-remote-code

...
INFO hf_hub: Token file not found "/root/.cache/huggingface/token"    
INFO text_generation_launcher: Default `max_input_tokens` to 4095
INFO text_generation_launcher: Default `max_total_tokens` to 4096
INFO text_generation_launcher: Default `max_batch_prefill_tokens` to 4145
INFO text_generation_launcher: Using default cuda graphs [1, 2, 4, 8, 16, 32]
INFO download: text_generation_launcher: Starting check and download process for openlm-research/open_llama_7b
INFO text_generation_launcher: Detected system cuda
INFO text_generation_launcher: Files are already present on the host. Skipping download.
INFO download: text_generation_launcher: Successfully downloaded weights for openlm-research/open_llama_7b
INFO shard-manager: text_generation_launcher: Starting shard rank=0
INFO text_generation_launcher: Detected system cuda
INFO text_generation_launcher: Server started at unix:///tmp/text-generation-server-0
INFO shard-manager: text_generation_launcher: Shard ready in 5.904282524s rank=0
INFO text_generation_launcher: Starting Webserver
WARN text_generation_router: router/src/main.rs:218: Offline mode active using cache defaults
INFO text_generation_router: router/src/main.rs:349: Using config Some(Llama)
WARN text_generation_router: router/src/main.rs:351: Could not find a fast tokenizer implementation for openlm-research/open_llama_7b
WARN text_generation_router: router/src/main.rs:352: Rust input length validation and truncation is disabled
WARN text_generation_router: router/src/main.rs:358: no pipeline tag found for model openlm-research/open_llama_7b
WARN text_generation_router: router/src/main.rs:376: Invalid hostname, defaulting to 0.0.0.0
INFO text_generation_router::server: router/src/server.rs:1577: Warming up model
INFO text_generation_launcher: Cuda Graphs are enabled for sizes [32, 16, 8, 4, 2, 1]
INFO text_generation_router::server: router/src/server.rs:1604: Using scheduler V3
INFO text_generation_router::server: router/src/server.rs:1656: Setting max batch total tokens to 58928
INFO text_generation_router::server: router/src/server.rs:1894: Connected

#重新启动容器
docker start HFTGI_llama_7b

#实时查看Web Server的运行状态
docker logs -f HFTGI_llama_7b

curl -v http://localhost:8080/generate -X POST -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' -H 'Content-Type: application/json'

#将pip换成清华源加快下载速度
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple

#安装text-generation包
pip install text-generation

from text_generation import Client


# Generate
client = Client("http://localhost:8080")
output = client.generate("Why is the sky blue?", max_new_tokens=92).generated_text
print(f"Generate Output: {output}")


# Generate stream
text = ""
for response in client.generate_stream("Why is the sky blue?", max_new_tokens=92):
    if not response.token.special:
        text += response.token.text
print(f"Generate stream Output: {text}")

python TGI_Client.py

Generate Output: 
The sky is blue because of the scattering of light by the atmosphere.
The scattering of light by the atmosphere is caused by the presence of molecules in the atmosphere.
The molecules in the atmosphere are made up of atoms.
The atoms in the atmosphere are made up of protons, neutrons, and electrons.
The protons, neutrons, and electrons in the atmosphere are arranged in a way that allows them to scatter light.

Generate stream Output: 
The sky is blue because of the scattering of light by the atmosphere.
The scattering of light by the atmosphere is caused by the presence of molecules in the atmosphere.
The molecules in the atmosphere are made up of atoms.
The atoms in the atmosphere are made up of protons, neutrons, and electrons.
The protons, neutrons, and electrons in the atmosphere are arranged in a way that allows them to scatter light.

#将pip换成清华源加快下载速度
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple

#安装必要的包
pip install langchain transformers langchain_community

# Wrapper to TGI client with langchain


from langchain.llms import HuggingFaceTextGenInference
from langchain import PromptTemplate, LLMChain


inference_server_url_local = "http://localhost:8080"


llm_local = HuggingFaceTextGenInference(
    inference_server_url=inference_server_url_local,
    max_new_tokens=400,
    top_k=10,
    top_p=0.95,
    typical_p=0.95,
    temperature=0.7,
    repetition_penalty=1.03,
)


question = "whats 2 * (1 + 2)"
template = """Question: {question}
Answer: Let's think step by step."""


prompt = PromptTemplate(
    template=template, 
    input_variables= ["question"]
)


llm_chain_local = LLMChain(prompt=prompt, llm=llm_local)
output = llm_chain_local({question})
print(output)

python LangChain.py