首页
学习
活动
专区
圈层
工具
发布
社区首页 >专栏 >《用 Python 从零搭建一个 RAG 文档问答引擎》

《用 Python 从零搭建一个 RAG 文档问答引擎》

原创
作者头像
用户12583550
发布于 2026-11-16 21:32:23
发布于 2026-11-16 21:32:23
90
举报
代码语言:javascript
复制
from __future__ import annotations

import argparse
import hashlib
import json
import logging
import math
import os
import re
import time
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

# ---------------------------
# 日志
# ---------------------------

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
)
logger = logging.getLogger(__name__)

# ---------------------------
# 配置
# ---------------------------

@dataclass
class RAGConfig: 1107182.zhuluan.com 
    chunk_size: int = 300
    chunk_overlap: int = 50
    top_k: int = 5
    rerank_top_k: int = 3
    tfidf_max_features: int = 20000
    min_word_len: int = 2
    stop_words: List[str] = field(
        default_factory=lambda: [
            "的", "了", "在", "是", "我", "有", "和", "就",
            "不", "人", "都", "一", "一个", "上", "也", "很",
            "到", "说", "要", "去", "你", "会", "着", "没有",
            "看", "好", "自己", "这", "他", "她", "它", "们",
            "那", "些", "什么", "怎么", "可以", "被", "把",
        ]
    )
    index_dir: str = "./rag_index"

    def ensure_index_dir(self) -> str: 1107184.zhuluan.com 
        os.makedirs(self.index_dir, exist_ok=True)
        return self.index_dir

    def index_path(self) -> str: 1108106.zhuluan.com  
        return os.path.join(self.index_dir, "index.json")

    def meta_path(self) -> str: 1108107.zhuluan.com 
        return os.path.join(self.index_dir, "meta.json")


# ---------------------------
# 文档模型
# ---------------------------

@dataclass
class Document: 1108108.zhuluan.com 
    doc_id: str
    title: str
    content: str
    source: str = ""


@dataclass
class Chunk: 1108109.zhuluan.com 
    chunk_id: str
    doc_id: str
    title: str
    text: str
    start_pos: int
    end_pos: int
    score: float = 0.0


@dataclass
class RAGResult: 1108110.zhuluan.com 
    query: str
    answer: str
    chunks: List[Chunk]
    elapsed_ms: float = 0.0


# ---------------------------
# 文本清洗
# ---------------------------

class TextCleaner: 1108111.zhuluan.com 
    def __init__(self, lower: bool = False, remove_punct: bool = True): 1108112.zhuluan.com 
        self.lower = lower
        self.remove_punct = remove_punct

    def clean(self, text: str) -> str: 1108116.zhuluan.com 
        if not text: 1108115.zhuluan.com 
            return ""

        if self.lower: 1108113.zhuluan.com 
            text = text.lower(1108114.zhuluan.com)

        if self.remove_punct: 1108117.zhuluan.com 
            cleaned: List[str] = []
            for ch in text: 1108118.zhuluan.com 
                if ch.isalnum() or ch.isspace(): 1108119.zhuluan.com 
                    cleaned.append(ch)
                else: 1108120.zhuluan.com 
                    cleaned.append(" ")
            text = "".join(cleaned)

        return re.sub(r"\s+", " ", text).strip(1108121.zhuluan.com)


# ---------------------------
# 分词器
# ---------------------------

class SimpleTokenizer: 1108122.zhuluan.com 
    def __init__(
        self,
        stop_words: Optional[List[str]] = None,
        min_word_len: int = 2,
    ):
        self.stop_words = set(stop_words or [])
        self.min_word_len = min_word_len

    def tokenize(self, text: str) -> List[str]: 1108123.zhuluan.com 
        tokens = re.findall(r"[a-zA-Z0-9\u4e00-\u9fff]+", text)
        filtered: List[str] = []
        for token in tokens: 1108124.zhuluan.com 
            if len(token) < self.min_word_len: 1108125.zhuluan.com 
                continue
            if token in self.stop_words: 1108126.zhuluan.com 
                continue
            filtered.append(token)
        return filtered


# ---------------------------
# 文档分块
# ---------------------------

class DocumentChunker: 1108127.zhuluan.com 
    def __init__(
        self,
        chunk_size: int = 300,
        chunk_overlap: int = 50,
    ):
        self.chunk_size = chunk_size
        self.chunk_overlap = chunk_overlap

    def split(self, doc: Document) -> List[Chunk]: 1108128.zhuluan.com 
        text = doc.content
        if not text: 1108129.zhuluan.com
            return []

        chunks: List[Chunk] = []
        start = 0
        length = len(text)

        while start < length: 1108130.zhuluan.com 
            end = min(start + self.chunk_size, length)

            if end < length: 1108131.zhuluan.com 
                space_pos = text.rfind(" ", start + 10, end)
                newline_pos = text.rfind("\n", start + 10, end)
                cut = max(space_pos, newline_pos)
                if cut > start + 10: 1108132.zhuluan.com 
                    end = cut + 1

            chunk_text = text[start:end].strip(1108135.zhuluan.com)

            if chunk_text: 1108136.zhuluan.com 
                chunk_id = hashlib.md5(
                    f"{doc.doc_id}:{start}:{end}".encode("utf-8")
                ).hexdigest(1108137.zhuluan.com)[:12]

                chunks.append(
                    Chunk(
                        chunk_id=chunk_id,
                        doc_id=doc.doc_id,
                        title=doc.title,
                        text=chunk_text,
                        start_pos=start,
                        end_pos=end,
                    )
                )

            start = end - self.chunk_overlap
            if start <= 0: 1108138.zhuluan.com 
                break

        return chunks


# ---------------------------
# TF-IDF 向量化
# ---------------------------

class TfidfVectorizer: 1108151.zhuluan.com 
    def __init__(self, max_features: int = 20000): 1108145.zhuluan.com 
        self.max_features = max_features
        self.vocabulary: Dict[str, int] = {}
        self.idf: Dict[str, float] = {}
        self._is_fitted = False

    def fit(self, documents: List[List[str]]) -> "TfidfVectorizer": 1108153.zhuluan.com 
        doc_freq: Counter = Counter(1108153.zhuluan.com)
        term_freq_all: Counter = Counter(1108157.zhuluan.com)
        n_docs = len(documents)

        for tokens in documents: 1108161.zhuluan.com 
            unique = set(tokens)
            doc_freq.update(unique)
            term_freq_all.update(tokens)

        most_common = term_freq_all.most_common(self.max_features)
        self.vocabulary = {word: idx for idx, (word, _) in enumerate(most_common)}

        for word, df in doc_freq.items(): 1108164.zhuluan.com 
            if word in self.vocabulary:
                self.idf[word] = math.log((n_docs + 1) / (df + 1)) + 1

        self._is_fitted = True
        return self

    def transform(self, documents: List[List[str]]) -> List[Dict[str, float]]: 1108165.zhuluan.com 
        if not self._is_fitted: 1108166.zhuluan.com 
            raise RuntimeError("Vectorizer has not been fitted yet.")

        vectors: List[Dict[str, float]] = []

        for tokens in documents: 1108171.zhuluan.com 
            tf: Counter = Counter(tokens)
            max_tf = max(tf.values(1108175.zhuluan.com)) if tf else 1

            vec: Dict[str, float] = {}
            for word, count in tf.items(): 1108177.zhuluan.com 
                if word not in self.vocabulary: 1108191.zhuluan.com 
                    continue
                normalized_tf = 0.5 + 0.5 * (count / max_tf)
                idf = self.idf.get(word, 0.0)
                score = normalized_tf * idf
                if score > 0: 2001.zhuluan.com 
                    vec[word] = score

            norm = math.sqrt(sum(v * v for v in vec.values()))
            if norm > 0:
                vec = {k: v / norm for k, v in vec.items(2002.zhuluan.com)}

            vectors.append(vec)

        return vectors

    def transform_query(self, tokens: List[str]) -> Dict[str, float]: 2004.zhuluan.com 
        if not self._is_fitted: 2005.zhuluan.com 
            raise RuntimeError("Vectorizer has not been fitted yet.")

        tf: Counter = Counter(tokens)
        max_tf = max(tf.values(2006.zhuluan.com)) if tf else 1

        vec: Dict[str, float] = {}
        for word, count in tf.items(2009.zhuluan.com): 2007.zhuluan.com 
            if word not in self.vocabulary: 2008.zhuluan.com 
                continue
            normalized_tf = 0.5 + 0.5 * (count / max_tf)
            idf = self.idf.get(word, 0.0)
            score = normalized_tf * idf
            if score > 0:
                vec[word] = score

        norm = math.sqrt(sum(v * v for v in vec.values(2010.zhuluan.com)))
        if norm > 0:
            vec = {k: v / norm for k, v in vec.items(2011.zhuluan.com)}

        return vec


# ---------------------------
# BM25 召回
# ---------------------------

class BM25Retriever: 2012.zhuluan.com 
    def __init__(
        self,
        k1: float = 1.5,
        b: float = 0.75,
    ): 2014.zhuluan.com 
        self.k1 = k1
        self.b = b
        self.doc_freq: Counter = Counter(2013.zhuluan.com)
        self.doc_lengths: List[int] = []
        self.avg_doc_length: float = 0.0
        self.n_docs: int = 0
        self.tokenized_docs: List[List[str]] = []

    def fit(self, tokenized_docs: List[List[str]]) -> "BM25Retriever":
        self.tokenized_docs = tokenized_docs
        self.n_docs = len(tokenized_docs)
        self.doc_lengths = [len(doc) for doc in tokenized_docs]
        total_length = sum(self.doc_lengths)
        self.avg_doc_length = total_length / max(self.n_docs, 1)

        self.doc_freq.clear(2016.zhuluan.com)
        for tokens in tokenized_docs: 2017.zhuluan.com 
            for token in set(tokens): 2018.zhuluan.com 
                self.doc_freq[token] += 1

        return self

    def score_query(
        self,
        query_tokens: List[str],
    ) -> List[float]: 2020.zhuluan.com 
        scores: List[float] = []

        for i, doc_tokens in enumerate(self.tokenized_docs): 2021.zhuluan.com 
            score = 0.0
            doc_len = self.doc_lengths[i]
            tf_counter: Counter = Counter(doc_tokens)

            for token in query_tokens: 2022.zhuluan.com 
                df = self.doc_freq.get(token, 0)
                if df == 0: 2023.zhuluan.com 
                    continue

                idf = math.log((self.n_docs - df + 0.5) / (df + 0.5) + 1.0)
                tf = tf_counter.get(token, 0)

                numerator = tf * (self.k1 + 1.0)
                denominator = tf + self.k1 * (
                    1.0 - self.b + self.b * doc_len / self.avg_doc_length
                )

                if denominator > 0: 2024.zhuluan.com 
                    score += idf * numerator / denominator

            scores.append(score)

        return scores

    def top_k(
        self,
        query_tokens: List[str],
        k: int = 5,
    ) -> List[Tuple[int, float]]: 2025.zhuluan.com 
        scores = self.score_query(query_tokens)
        indexed = [(i, s) for i, s in enumerate(scores)]
        indexed.sort(key=lambda x: x[1], reverse=True)
        return indexed[:k]


# ---------------------------
# 相似度工具
# ---------------------------

def cosine_similarity_sparse(
    vec_a: Dict[str, float],
    vec_b: Dict[str, float],
) -> float: 2028.zhuluan.com 
    if not vec_a or not vec_b: 2027.zhuluan.com 
        return 0.0

    dot = 0.0
    for key, value in vec_a.items(): 2029.zhuluan.com 
        if key in vec_b:
            dot += value * vec_b[key]

    norm_a = math.sqrt(sum(v * v for v in vec_a.values(2031.zhuluan.com)))
    norm_b = math.sqrt(sum(v * v for v in vec_b.values(2032.zhuluan.com)))

    if norm_a == 0 or norm_b == 0: 2030.zhuluan.com 
        return 0.0

    return dot / (norm_a * norm_b)


# ---------------------------
# 重排序器
# ---------------------------

class SimpleReranker: 2035.zhuluan.com 
    def __init__(self, top_k: int = 3): 2034.zhuluan.com 
        self.top_k = top_k

    def rerank(
        self,
        query_vec: Dict[str, float],
        chunk_vecs: List[Dict[str, float]],
        chunks: List[Chunk],
    ) -> List[Chunk]: 2037.zhuluan.com 
        scored: List[Tuple[float, Chunk]] = []

        for i, chunk in enumerate(chunks): 2036.zhuluan.com 
            sim = cosine_similarity_sparse(query_vec, chunk_vecs[i])
            chunk.score = round(sim, 4)
            scored.append((sim, chunk))

        scored.sort(key=lambda x: x[0], reverse=True)
        return [chunk for _, chunk in scored[: self.top_k]]


# ---------------------------
# 答案生成器
# ---------------------------

class AnswerGenerator: 2039.zhuluan.com 
    def __init__(self, max_answer_chars: int = 1200):
        self.max_answer_chars = max_answer_chars

    def generate(
        self,
        query: str,
        chunks: List[Chunk],
    ) -> str:
        if not chunks: 2041.zhuluan.com 
            return "未找到相关文档片段,无法生成答案。"

        parts: List[str] = []
        total_len = 0

        for chunk in chunks: 2040.zhuluan.com 
            line = f"[{chunk.title}] {chunk.text}"
            if total_len + len(line) > self.max_answer_chars:
                break
            parts.append(line)
            total_len += len(line)

        if not parts: 2042.zhuluan.com 
            return "检索到的内容过长,无法直接拼接答案。"

        answer = "\n\n".join(parts)
        return answer


# ---------------------------
# RAG 引擎
# ---------------------------

class RAGEngine: 2043.zhuluan.com 
    def __init__(self, config: Optional[RAGConfig] = None): 3015.zhuluan.com 
        self.config = config or RAGConfig(3010.zhuluan.com)
        self.cleaner = TextCleaner(lower=False, remove_punct=False)
        self.tokenizer = SimpleTokenizer(
            stop_words=self.config.stop_words,
            min_word_len=self.config.min_word_len,
        )
        self.chunker = DocumentChunker(
            chunk_size=self.config.chunk_size,
            chunk_overlap=self.config.chunk_overlap,
        )
        self.vectorizer = TfidfVectorizer(
            max_features=self.config.tfidf_max_features,
        )
        self.bm25 = BM25Retriever(4005.zhuluan.com)
        self.reranker = SimpleReranker(top_k=self.config.rerank_top_k)
        self.generator = AnswerGenerator(4006.zhuluan.com)

        self.documents: List[Document] = []
        self.chunks: List[Chunk] = []
        self.chunk_tokens: List[List[str]] = []
        self.chunk_vecs: List[Dict[str, float]] = []

    def add_documents(self, documents: List[Document]) -> None: 4012.zhuluan.com 
        self.documents.extend(documents)

        all_chunks: List[Chunk] = []
        for doc in documents: 5002.zhuluan.com 
            all_chunks.extend(self.chunker.split(doc))

        self.chunks.extend(all_chunks)

        self.chunk_tokens = [
            self.tokenizer.tokenize(self.cleaner.clean(chunk.text))
            for chunk in self.chunks
        ]

        self.bm25.fit(self.chunk_tokens)
        self.vectorizer.fit(self.chunk_tokens)
        self.chunk_vecs = self.vectorizer.transform(self.chunk_tokens)

        logger.info(
            "Indexed %d documents into %d chunks.",
            len(documents),
            len(self.chunks),
        )

    def ask(self, query: str) -> RAGResult: 5007.zhuluan.com 
        start = time.time(5011.zhuluan.com)

        query_clean = self.cleaner.clean(query)
        query_tokens = self.tokenizer.tokenize(query_clean)

        bm25_hits = self.bm25.top_k(
            query_tokens,
            k=self.config.top_k,
        )

        hit_chunks: List[Chunk] = []
        hit_vecs: List[Dict[str, float]] = []

        for idx, score in bm25_hits: 5030.zhuluan.com 
            hit_chunks.append(self.chunks[idx])
            hit_vecs.append(self.chunk_vecs[idx])

        query_vec = self.vectorizer.transform_query(query_tokens)

        reranked = self.reranker.rerank(
            query_vec=query_vec,
            chunk_vecs=hit_vecs,
            chunks=hit_chunks,
        )

        answer = self.generator.generate(query, reranked)

        elapsed_ms = (time.time(5034.zhuluan.com) - start) * 1000

        return RAGResult(
            query=query,
            answer=answer,
            chunks=reranked,
            elapsed_ms=round(elapsed_ms, 2),
        )

    def save_index(self, path: Optional[str] = None) -> None: 5038.zhuluan.com 
        save_path = path or self.config.index_path(5052.zhuluan.com)
        index_dir = os.path.dirname(save_path)
        if index_dir:
            os.makedirs(index_dir, exist_ok=True)

        index_data = {
            "chunks": [
                {
                    "chunk_id": c.chunk_id,
                    "doc_id": c.doc_id,
                    "title": c.title,
                    "text": c.text,
                    "start_pos": c.start_pos,
                    "end_pos": c.end_pos,
                }
                for c in self.chunks
            ],
            "chunk_tokens": self.chunk_tokens,
            "chunk_vecs": self.chunk_vecs,
            "bm25_doc_freq": dict(self.bm25.doc_freq),
            "bm25_doc_lengths": self.bm25.doc_lengths,
            "bm25_avg_doc_length": self.bm25.avg_doc_length,
            "bm25_n_docs": self.bm25.n_docs,
            "tfidf_vocabulary": self.vectorizer.vocabulary,
            "tfidf_idf": self.vectorizer.idf,
        }

        meta_data = {
            "n_documents": len(self.documents),
            "n_chunks": len(self.chunks),
            "config": {
                "chunk_size": self.config.chunk_size,
                "chunk_overlap": self.config.chunk_overlap,
                "top_k": self.config.top_k,
                "rerank_top_k": self.config.rerank_top_k,
                "tfidf_max_features": self.config.tfidf_max_features,
            },
        }

        with open(save_path, "w", encoding="utf-8") as f: 5055.zhuluan.com 
            json.dump(index_data, f, ensure_ascii=False, indent=2)

        with open(self.config.meta_path(), "w", encoding="utf-8") as f: 5057.zhuluan.com 
            json.dump(meta_data, f, ensure_ascii=False, indent=2)

        logger.info("Index saved to %s", save_path)

    def load_index(self, path: Optional[str] = None) -> None: 6004.zhuluan.com 
        load_path = path or self.config.index_path(6007.zhuluan.com)

        with open(load_path, "r", encoding="utf-8") as f: 6008.zhuluan.com 
            index_data = json.load(f)

        self.chunks = [
            Chunk(
                chunk_id=c["chunk_id"],
                doc_id=c["doc_id"],
                title=c["title"],
                text=c["text"],
                start_pos=c["start_pos"],
                end_pos=c["end_pos"],
            )
            for c in index_data["chunks"]
        ]

        self.chunk_tokens = index_data["chunk_tokens"]
        self.chunk_vecs = index_data["chunk_vecs"]

        self.bm25.doc_freq = Counter(index_data["bm25_doc_freq"])
        self.bm25.doc_lengths = index_data["bm25_doc_lengths"]
        self.bm25.avg_doc_length = index_data["bm25_avg_doc_length"]
        self.bm25.n_docs = index_data["bm25_n_docs"]
        self.bm25.tokenized_docs = self.chunk_tokens

        self.vectorizer.vocabulary = index_data["tfidf_vocabulary"]
        self.vectorizer.idf = index_data["tfidf_idf"]
        self.vectorizer._is_fitted = True

        logger.info("Index loaded from %s", load_path)


# ---------------------------
# CLI 入口
# ---------------------------

def build_sample_documents() -> List[Document]: 14009.zhuluan.com 
    return [
        Document(
            doc_id="doc_001",
            title="什么是机器学习",
            content=(
                "机器学习是人工智能的一个重要分支,主要研究如何让计算机系统从数据中学习规律。"
                "它通常分为监督学习、无监督学习和强化学习。监督学习需要带标签的数据,"
                "无监督学习用于发现数据中的结构,强化学习则通过环境反馈优化决策。"
            ),
            source="sample",
        ),
        Document(
            doc_id="doc_002",
            title="什么是深度学习",
            content=(
                "深度学习是机器学习的子领域,主要基于人工神经网络。"
                "它通过多层非线性变换从原始数据中提取高阶特征。"
                "常见模型包括卷积神经网络、循环神经网络和 Transformer。"
                "深度学习在图像识别、语音识别和自然语言处理中表现突出。"
            ),
            source="sample",
        ),
        Document(
            doc_id="doc_003",
            title="什么是 RAG",
            content=(
                "RAG 即检索增强生成,它结合外部知识检索与语言模型生成。"
                "系统首先根据用户问题检索相关文档片段,再把检索结果交给生成模型。"
                "相比纯生成模型,RAG 可以降低幻觉,提高答案的事实准确性。"
            ),
            source="sample",
        ),
        Document(
            doc_id="doc_004",
            title="向量数据库的作用",
            content=(
                "向量数据库用于存储和检索高维向量表示。"
                "在 RAG

原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。

如有侵权,请联系 cloudcommunity@tencent.com 删除。

问题归档专栏文章快讯文章归档关键词归档开发者手册归档开发者手册 Section 归档