【RAG入门教程04】Langchian的文档切分

from langchain_text_splitters import CharacterTextSplitter

text_splitter = CharacterTextSplitter(
    separator="\n\n",
    chunk_size=1000,
    chunk_overlap=200,
    length_function=len,
    is_separator_regex=False,
)

from langchain_community.document_loaders import PyPDFLoader

loader = PyPDFLoader("book.pdf")
pages = loader.load_and_split()

from langchain_text_splitters import CharacterTextSplitter

text_splitter = CharacterTextSplitter(
    separator="\n",
    chunk_size=1000,
    chunk_overlap=200,
    length_function=len,
    is_separator_regex=False,
)

texts = text_splitter.split_text(pages[0].page_content)
print(len(texts))

# 4

texts[0]

"""
'Our goal with this book is to provide the guidance and framework for you,
 the reader, to grow on \nthe path to being a truly excellent database 
reliability engineer (DBRE). When naming the book we \nchose to use the
 words reliability engineer , rather than administrator.  \nBen Treynor, 
VP of Engineering at Google, says the following about reliability engi‐ 
neering:  \nfundamentally doing work that has historically been done by an 
operations team, but using engineers with software \nexpertise, and banking
 on the fact that these engineers are inherently both predisposed to, and 
have the ability to, \nsubstitute automation for  human labor.  \nToday’s 
database professionals must be engineers, not administrators. 
We build things. We create \nthings. As engineers practicing devops, 
we are all in this together, and nothing is someone else’s \nproblem.
 As engineers, we apply repeatable processes, establ ished knowledge, 
and expert judgment'
"""

texts[1]

"""
'things. As engineers practicing devops, we are all in this together, and nothing is someone else’s \nproblem. As engineers, we apply repeatable processes, establ ished knowledge, and expert judgment \nto design, build, and operate production data stores and the data structures within. As database \nreliability engineers, we must take the operational principles and the depth of database expertise \nthat we possess one ste p further.  \nIf you look at the non -storage components of today’s infrastructures, you will see sys‐ tems that are \neasily built, run, and destroyed via programmatic and often automatic means. The lifetimes of these \ncomponents can be measured in days, and sometimes even  hours or minutes. When one goes away, \nthere is any number of others to step in and keep the quality of service at expected levels.  \nOur next goal is that you gain a framework of principles and practices for the design, building, and'
"""

from langchain_text_splitters import RecursiveCharacterTextSplitter

text_splitter = RecursiveCharacterTextSplitter(
    # Set a really small chunk size, just to show.
    separators=["\n\n", "\n", " ", ""],
    chunk_size=50,
    chunk_overlap=40,
    length_function=len,
    is_separator_regex=False,
)
texts = text_splitter.split_text(pages[0].page_content)
print(len(texts))

texts[2]

"""
'book is to provide the guidance and framework for'
"""


texts[3]

"""
'provide the guidance and framework for you, the'
"""

from langchain_text_splitters import TokenTextSplitter

text_splitter = TokenTextSplitter(chunk_size=10, chunk_overlap=1)

texts = text_splitter.split_text(pages[0].page_content)

texts[0]

"""
'Our goal with this book is to provide the guidance'
"""

texts[1]

"""
' guidance and framework for you, the reader, to'
"""

from langchain_text_splitters import SpacyTextSplitter

text_splitter = SpacyTextSplitter(chunk_size=1000)

texts = text_splitter.split_text(pages[0].page_content)

from langchain_text_splitters import NLTKTextSplitter

text_splitter = NLTKTextSplitter(chunk_size=1000)

texts = text_splitter.split_text(pages[0].page_content)

from transformers import GPT2TokenizerFast

tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")

text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(
    tokenizer, chunk_size=100, chunk_overlap=10
)
texts = text_splitter.split_text(pages[0].page_content)

from langchain_text_splitters import HTMLHeaderTextSplitter

html_string = """
<!DOCTYPE html>
<html>
<body>
    <div>
        <h1>Foo</h1>
        <p>Some intro text about Foo.</p>
        <div>
            <h2>Bar main section</h2>
            <p>Some intro text about Bar.</p>
            <h3>Bar subsection 1</h3>
            <p>Some text about the first subtopic of Bar.</p>
            <h3>Bar subsection 2</h3>
            <p>Some text about the second subtopic of Bar.</p>
        </div>
        <div>
            <h2>Baz</h2>
            <p>Some text about Baz</p>
        </div>
        <br>
        <p>Some concluding text about Foo</p>
    </div>
</body>
</html>
"""

headers_to_split_on = [
    ("h1", "Header 1"),
    ("h2", "Header 2"),
    ("h3", "Header 3"),
]

html_splitter = HTMLHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
html_header_splits = html_splitter.split_text(html_string)
html_header_splits

"""
[Document(page_content='Foo'),
 Document(page_content='Some intro text about Foo.  \nBar main section Bar subsection 1 Bar subsection 2', metadata={'Header 1': 'Foo'}),
 Document(page_content='Some intro text about Bar.', metadata={'Header 1': 'Foo', 'Header 2': 'Bar main section'}),
 Document(page_content='Some text about the first subtopic of Bar.', metadata={'Header 1': 'Foo', 'Header 2': 'Bar main section', 'Header 3': 'Bar subsection 1'}),
 Document(page_content='Some text about the second subtopic of Bar.', metadata={'Header 1': 'Foo', 'Header 2': 'Bar main section', 'Header 3': 'Bar subsection 2'}),
 Document(page_content='Baz', metadata={'Header 1': 'Foo'}),
 Document(page_content='Some text about Baz', metadata={'Header 1': 'Foo', 'Header 2': 'Baz'}),
 Document(page_content='Some concluding text about Foo', metadata={'Header 1': 'Foo'})]
"""

from langchain_text_splitters import MarkdownHeaderTextSplitter

markdown_document = "# Foo\n\n    ## Bar\n\nHi this is Jim\n\nHi this is Joe\n\n ### Boo \n\n Hi this is Lance \n\n ## Baz\n\n Hi this is Molly"

headers_to_split_on = [
    ("#", "Header 1"),
    ("##", "Header 2"),
    ("###", "Header 3"),
]

markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
md_header_splits = markdown_splitter.split_text(markdown_document)
md_header_splits

"""
[Document(page_content='Hi this is Jim  \nHi this is Joe', metadata={'Header 1': 'Foo', 'Header 2': 'Bar'}),
 Document(page_content='Hi this is Lance', metadata={'Header 1': 'Foo', 'Header 2': 'Bar', 'Header 3': 'Boo'}),
 Document(page_content='Hi this is Molly', metadata={'Header 1': 'Foo', 'Header 2': 'Baz'})]
"""

import requests

# This is a large nested json object and will be loaded as a python dict
json_data = requests.get("https://api.smith.langchain.com/openapi.json").json()

from langchain_text_splitters import RecursiveJsonSplitter

splitter = RecursiveJsonSplitter(max_chunk_size=300)

# Recursively split json data - If you need to access/manipulate the smaller json chunks
json_chunks = splitter.split_json(json_data=json_data)

json_chunks
"""
{'openapi': '3.0.2',
  'info': {'title': 'LangSmith', 'version': '0.1.0'},
  'paths': {'/api/v1/sessions/{session_id}': {'get': {'tags': ['tracer-sessions'],
     'summary': 'Read Tracer Session',
     'description': 'Get a specific session.'}}}},
 {'paths': {'/api/v1/sessions/{session_id}': {'get': {'operationId': 'read_tracer_session_api_v1_sessions__session_id__get'}}}},
 {'paths': {'/api/v1/sessions/{session_id}': {'get': {'parameters': [{'required': True,
       'schema': {'title': 'Session Id', 'type': 'string', 'format': 'uuid'},
       'name': 'session_id',
       'in': 'path'},
      {'required': False,
       'schema': {'title': 'Include Stats',
        'type': 'boolean',
        'default': False},
       'name': 'include_stats',
       'in': 'query'},
      {'required': False,
       'schema': {'title': 'Accept', 'type': 'string'},
       'name': 'accept',
       'in': 'header'}]}}}},
 {'paths': {'/api/v1/sessions/{session_id}': {'get': {'responses': {'200': {'description': 'Successful Response',
       'content': {'application/json': {'schema': {'$ref': '#/components/schemas/TracerSession'}}}}}}}}},
 {'paths': {'/api/v1/sessions/{session_id}': {'get': {'responses': {'422': {'description': 'Validation Error',
       'content': {'application/json': {'schema': {'$ref': '#/components/schemas/HTTPValidationError'}}}}},
     'security': [{'API Key': []}, {'Tenant ID': []}, {'Bearer Auth': []}]}}}},
...
 {'components': {'securitySchemes': {'API Key': {'type': 'apiKey',
     'in': 'header',
     'name': 'X-API-Key'},
    'Tenant ID': {'type': 'apiKey', 'in': 'header', 'name': 'X-Tenant-Id'},
    'Bearer Auth': {'type': 'http', 'scheme': 'bearer'}}}}]
"""

from langchain_text_splitters import Language

[e.value for e in Language]

"""
['cpp',
 'go',
 'java',
 'kotlin',
 'js',
 'ts',
 'php',
 'proto',
 'python',
 'rst',
 'ruby',
 'rust',
 'scala',
 'swift',
 'markdown',
 'latex',
 'html',
 'sol',
 'csharp',
 'cobol',
 'c',
 'lua',
 'perl']
"""

from langchain_text_splitters import (
    Language,
    RecursiveCharacterTextSplitter,
)

PYTHON_CODE = """
def hello_world():
    print("Hello, World!")

# Call the function
hello_world()
"""
python_splitter = RecursiveCharacterTextSplitter.from_language(
    language=Language.PYTHON, chunk_size=50, chunk_overlap=0
)
python_docs = python_splitter.create_documents([PYTHON_CODE])
python_docs

"""
[Document(page_content='def hello_world():\n    print("Hello, World!")'),
 Document(page_content='# Call the function\nhello_world()')]
"""
JS_CODE = """
function helloWorld() {
  console.log("Hello, World!");
}

// Call the function
helloWorld();
"""

js_splitter = RecursiveCharacterTextSplitter.from_language(
    language=Language.JS, chunk_size=60, chunk_overlap=0
)
js_docs = js_splitter.create_documents([JS_CODE])
js_docs

"""
[Document(page_content='function helloWorld() {\n  console.log("Hello, World!");\n}'),
 Document(page_content='// Call the function\nhelloWorld();')]
"""