5分钟玩转PDF聊天机器人!超简单的Langchain+ChatGPT实现攻略

# # 创建虚拟环境
$ python3 -m venv llm_app_env 

# # 在 MacOS 或 Linux 上
$ source llm_app_env/bin/activate 

# # 在 Window 上
$ llm_app_env\Scripts\activate.bat 

# # 然后安装以下库。
openai[embeddings]==0.27.6 
langchain==0.0.155 
pypdf==3.8.1 
tiktoken==0.3.3 
faiss-cpu==1.7.4
unstructured==0.6.2 
chromadb==0.3.21 
llama-index ==0.6.1 
jupyterlab

import logging
import sys
import os

os.environ["OPENAI_API_KEY"] = "<YOUR_OPENAI_API_KEY>"

## load the PDF using pypdf
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter

# load the data
loader = PyPDFLoader('../notebooks/documents/Apple-Financial-Report-Q1-2022.pdf')

# the 10k financial report are huge, we will need to split the doc into multiple chunk.
# This text splitter is the recommended one for generic text. It is parameterized by a list of characters. 
# It tries to split on them in order until the chunks are small enough.
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
data = loader.load()
texts = text_splitter.split_documents(data)

# view the first chunk
texts[0]

# import Chroma and OpenAIEmbeddings
from langchain.vectorstores import Chroma
from langchain.embeddings.openai import OpenAIEmbeddings

# initialize OpenAIEmbedding
embeddings = OpenAIEmbeddings(model='text-embedding-ada-002')

# use Chroma to create in-memory embedding database from the doc
docsearch = Chroma.from_documents(texts, embeddings,  metadatas=[{"source": str(i)} for i in range(len(texts))])

## perform search based on the question
query = "What is the operating income?"
docs = docsearch.similarity_search(query)

## importing necessary framework
from langchain.chains.question_answering import load_qa_chain
from langchain.chains.qa_with_sources import load_qa_with_sources_chain
from langchain.chains import RetrievalQA
from langchain.chains import RetrievalQAWithSourcesChain

from langchain.chat_models import ChatOpenAI

## use LLM to get answering
chain = load_qa_chain(ChatOpenAI(temperature=0.2,model_name='gpt-3.5-turbo'), 
                      chain_type="stuff")
query = "What is the operating income?"
chain.run(input_documents=docs, question=query)

chain = load_qa_with_sources_chain(ChatOpenAI(temperature=0.2,model_name='gpt-3.5-turbo'), 
                                   chain_type="stuff")
query = "What is the operating income?"
chain({"input_documents": docs, "question": query}, return_only_outputs=True)

qa=RetrievalQA.from_chain_type(llm=ChatOpenAI(temperature=0.2,model_name='gpt-3.5-turbo'), chain_type="stuff", 
                                                retriever=docsearch.as_retriever())
query = "What is the operating income?"
qa.run(query)

chain=RetrievalQAWithSourcesChain.from_chain_type(ChatOpenAI(temperature=0.2,model_name='gpt-3.5-turbo'), chain_type="stuff", 
                                                    retriever=docsearch.as_retriever())
chain({"question": "What is the operating income?"}, return_only_outputs=True)