from __future__ import annotations
import argparse
import hashlib
import json
import logging
import math
import os
import re
import time
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
# ---------------------------
# 日志
# ---------------------------
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
)
logger = logging.getLogger(__name__)
# ---------------------------
# 配置
# ---------------------------
@dataclass
class RAGConfig: 1107182.zhuluan.com
chunk_size: int = 300
chunk_overlap: int = 50
top_k: int = 5
rerank_top_k: int = 3
tfidf_max_features: int = 20000
min_word_len: int = 2
stop_words: List[str] = field(
default_factory=lambda: [
"的", "了", "在", "是", "我", "有", "和", "就",
"不", "人", "都", "一", "一个", "上", "也", "很",
"到", "说", "要", "去", "你", "会", "着", "没有",
"看", "好", "自己", "这", "他", "她", "它", "们",
"那", "些", "什么", "怎么", "可以", "被", "把",
]
)
index_dir: str = "./rag_index"
def ensure_index_dir(self) -> str: 1107184.zhuluan.com
os.makedirs(self.index_dir, exist_ok=True)
return self.index_dir
def index_path(self) -> str: 1108106.zhuluan.com
return os.path.join(self.index_dir, "index.json")
def meta_path(self) -> str: 1108107.zhuluan.com
return os.path.join(self.index_dir, "meta.json")
# ---------------------------
# 文档模型
# ---------------------------
@dataclass
class Document: 1108108.zhuluan.com
doc_id: str
title: str
content: str
source: str = ""
@dataclass
class Chunk: 1108109.zhuluan.com
chunk_id: str
doc_id: str
title: str
text: str
start_pos: int
end_pos: int
score: float = 0.0
@dataclass
class RAGResult: 1108110.zhuluan.com
query: str
answer: str
chunks: List[Chunk]
elapsed_ms: float = 0.0
# ---------------------------
# 文本清洗
# ---------------------------
class TextCleaner: 1108111.zhuluan.com
def __init__(self, lower: bool = False, remove_punct: bool = True): 1108112.zhuluan.com
self.lower = lower
self.remove_punct = remove_punct
def clean(self, text: str) -> str: 1108116.zhuluan.com
if not text: 1108115.zhuluan.com
return ""
if self.lower: 1108113.zhuluan.com
text = text.lower(1108114.zhuluan.com)
if self.remove_punct: 1108117.zhuluan.com
cleaned: List[str] = []
for ch in text: 1108118.zhuluan.com
if ch.isalnum() or ch.isspace(): 1108119.zhuluan.com
cleaned.append(ch)
else: 1108120.zhuluan.com
cleaned.append(" ")
text = "".join(cleaned)
return re.sub(r"\s+", " ", text).strip(1108121.zhuluan.com)
# ---------------------------
# 分词器
# ---------------------------
class SimpleTokenizer: 1108122.zhuluan.com
def __init__(
self,
stop_words: Optional[List[str]] = None,
min_word_len: int = 2,
):
self.stop_words = set(stop_words or [])
self.min_word_len = min_word_len
def tokenize(self, text: str) -> List[str]: 1108123.zhuluan.com
tokens = re.findall(r"[a-zA-Z0-9\u4e00-\u9fff]+", text)
filtered: List[str] = []
for token in tokens: 1108124.zhuluan.com
if len(token) < self.min_word_len: 1108125.zhuluan.com
continue
if token in self.stop_words: 1108126.zhuluan.com
continue
filtered.append(token)
return filtered
# ---------------------------
# 文档分块
# ---------------------------
class DocumentChunker: 1108127.zhuluan.com
def __init__(
self,
chunk_size: int = 300,
chunk_overlap: int = 50,
):
self.chunk_size = chunk_size
self.chunk_overlap = chunk_overlap
def split(self, doc: Document) -> List[Chunk]: 1108128.zhuluan.com
text = doc.content
if not text: 1108129.zhuluan.com
return []
chunks: List[Chunk] = []
start = 0
length = len(text)
while start < length: 1108130.zhuluan.com
end = min(start + self.chunk_size, length)
if end < length: 1108131.zhuluan.com
space_pos = text.rfind(" ", start + 10, end)
newline_pos = text.rfind("\n", start + 10, end)
cut = max(space_pos, newline_pos)
if cut > start + 10: 1108132.zhuluan.com
end = cut + 1
chunk_text = text[start:end].strip(1108135.zhuluan.com)
if chunk_text: 1108136.zhuluan.com
chunk_id = hashlib.md5(
f"{doc.doc_id}:{start}:{end}".encode("utf-8")
).hexdigest(1108137.zhuluan.com)[:12]
chunks.append(
Chunk(
chunk_id=chunk_id,
doc_id=doc.doc_id,
title=doc.title,
text=chunk_text,
start_pos=start,
end_pos=end,
)
)
start = end - self.chunk_overlap
if start <= 0: 1108138.zhuluan.com
break
return chunks
# ---------------------------
# TF-IDF 向量化
# ---------------------------
class TfidfVectorizer: 1108151.zhuluan.com
def __init__(self, max_features: int = 20000): 1108145.zhuluan.com
self.max_features = max_features
self.vocabulary: Dict[str, int] = {}
self.idf: Dict[str, float] = {}
self._is_fitted = False
def fit(self, documents: List[List[str]]) -> "TfidfVectorizer": 1108153.zhuluan.com
doc_freq: Counter = Counter(1108153.zhuluan.com)
term_freq_all: Counter = Counter(1108157.zhuluan.com)
n_docs = len(documents)
for tokens in documents: 1108161.zhuluan.com
unique = set(tokens)
doc_freq.update(unique)
term_freq_all.update(tokens)
most_common = term_freq_all.most_common(self.max_features)
self.vocabulary = {word: idx for idx, (word, _) in enumerate(most_common)}
for word, df in doc_freq.items(): 1108164.zhuluan.com
if word in self.vocabulary:
self.idf[word] = math.log((n_docs + 1) / (df + 1)) + 1
self._is_fitted = True
return self
def transform(self, documents: List[List[str]]) -> List[Dict[str, float]]: 1108165.zhuluan.com
if not self._is_fitted: 1108166.zhuluan.com
raise RuntimeError("Vectorizer has not been fitted yet.")
vectors: List[Dict[str, float]] = []
for tokens in documents: 1108171.zhuluan.com
tf: Counter = Counter(tokens)
max_tf = max(tf.values(1108175.zhuluan.com)) if tf else 1
vec: Dict[str, float] = {}
for word, count in tf.items(): 1108177.zhuluan.com
if word not in self.vocabulary: 1108191.zhuluan.com
continue
normalized_tf = 0.5 + 0.5 * (count / max_tf)
idf = self.idf.get(word, 0.0)
score = normalized_tf * idf
if score > 0: 2001.zhuluan.com
vec[word] = score
norm = math.sqrt(sum(v * v for v in vec.values()))
if norm > 0:
vec = {k: v / norm for k, v in vec.items(2002.zhuluan.com)}
vectors.append(vec)
return vectors
def transform_query(self, tokens: List[str]) -> Dict[str, float]: 2004.zhuluan.com
if not self._is_fitted: 2005.zhuluan.com
raise RuntimeError("Vectorizer has not been fitted yet.")
tf: Counter = Counter(tokens)
max_tf = max(tf.values(2006.zhuluan.com)) if tf else 1
vec: Dict[str, float] = {}
for word, count in tf.items(2009.zhuluan.com): 2007.zhuluan.com
if word not in self.vocabulary: 2008.zhuluan.com
continue
normalized_tf = 0.5 + 0.5 * (count / max_tf)
idf = self.idf.get(word, 0.0)
score = normalized_tf * idf
if score > 0:
vec[word] = score
norm = math.sqrt(sum(v * v for v in vec.values(2010.zhuluan.com)))
if norm > 0:
vec = {k: v / norm for k, v in vec.items(2011.zhuluan.com)}
return vec
# ---------------------------
# BM25 召回
# ---------------------------
class BM25Retriever: 2012.zhuluan.com
def __init__(
self,
k1: float = 1.5,
b: float = 0.75,
): 2014.zhuluan.com
self.k1 = k1
self.b = b
self.doc_freq: Counter = Counter(2013.zhuluan.com)
self.doc_lengths: List[int] = []
self.avg_doc_length: float = 0.0
self.n_docs: int = 0
self.tokenized_docs: List[List[str]] = []
def fit(self, tokenized_docs: List[List[str]]) -> "BM25Retriever":
self.tokenized_docs = tokenized_docs
self.n_docs = len(tokenized_docs)
self.doc_lengths = [len(doc) for doc in tokenized_docs]
total_length = sum(self.doc_lengths)
self.avg_doc_length = total_length / max(self.n_docs, 1)
self.doc_freq.clear(2016.zhuluan.com)
for tokens in tokenized_docs: 2017.zhuluan.com
for token in set(tokens): 2018.zhuluan.com
self.doc_freq[token] += 1
return self
def score_query(
self,
query_tokens: List[str],
) -> List[float]: 2020.zhuluan.com
scores: List[float] = []
for i, doc_tokens in enumerate(self.tokenized_docs): 2021.zhuluan.com
score = 0.0
doc_len = self.doc_lengths[i]
tf_counter: Counter = Counter(doc_tokens)
for token in query_tokens: 2022.zhuluan.com
df = self.doc_freq.get(token, 0)
if df == 0: 2023.zhuluan.com
continue
idf = math.log((self.n_docs - df + 0.5) / (df + 0.5) + 1.0)
tf = tf_counter.get(token, 0)
numerator = tf * (self.k1 + 1.0)
denominator = tf + self.k1 * (
1.0 - self.b + self.b * doc_len / self.avg_doc_length
)
if denominator > 0: 2024.zhuluan.com
score += idf * numerator / denominator
scores.append(score)
return scores
def top_k(
self,
query_tokens: List[str],
k: int = 5,
) -> List[Tuple[int, float]]: 2025.zhuluan.com
scores = self.score_query(query_tokens)
indexed = [(i, s) for i, s in enumerate(scores)]
indexed.sort(key=lambda x: x[1], reverse=True)
return indexed[:k]
# ---------------------------
# 相似度工具
# ---------------------------
def cosine_similarity_sparse(
vec_a: Dict[str, float],
vec_b: Dict[str, float],
) -> float: 2028.zhuluan.com
if not vec_a or not vec_b: 2027.zhuluan.com
return 0.0
dot = 0.0
for key, value in vec_a.items(): 2029.zhuluan.com
if key in vec_b:
dot += value * vec_b[key]
norm_a = math.sqrt(sum(v * v for v in vec_a.values(2031.zhuluan.com)))
norm_b = math.sqrt(sum(v * v for v in vec_b.values(2032.zhuluan.com)))
if norm_a == 0 or norm_b == 0: 2030.zhuluan.com
return 0.0
return dot / (norm_a * norm_b)
# ---------------------------
# 重排序器
# ---------------------------
class SimpleReranker: 2035.zhuluan.com
def __init__(self, top_k: int = 3): 2034.zhuluan.com
self.top_k = top_k
def rerank(
self,
query_vec: Dict[str, float],
chunk_vecs: List[Dict[str, float]],
chunks: List[Chunk],
) -> List[Chunk]: 2037.zhuluan.com
scored: List[Tuple[float, Chunk]] = []
for i, chunk in enumerate(chunks): 2036.zhuluan.com
sim = cosine_similarity_sparse(query_vec, chunk_vecs[i])
chunk.score = round(sim, 4)
scored.append((sim, chunk))
scored.sort(key=lambda x: x[0], reverse=True)
return [chunk for _, chunk in scored[: self.top_k]]
# ---------------------------
# 答案生成器
# ---------------------------
class AnswerGenerator: 2039.zhuluan.com
def __init__(self, max_answer_chars: int = 1200):
self.max_answer_chars = max_answer_chars
def generate(
self,
query: str,
chunks: List[Chunk],
) -> str:
if not chunks: 2041.zhuluan.com
return "未找到相关文档片段,无法生成答案。"
parts: List[str] = []
total_len = 0
for chunk in chunks: 2040.zhuluan.com
line = f"[{chunk.title}] {chunk.text}"
if total_len + len(line) > self.max_answer_chars:
break
parts.append(line)
total_len += len(line)
if not parts: 2042.zhuluan.com
return "检索到的内容过长,无法直接拼接答案。"
answer = "\n\n".join(parts)
return answer
# ---------------------------
# RAG 引擎
# ---------------------------
class RAGEngine: 2043.zhuluan.com
def __init__(self, config: Optional[RAGConfig] = None): 3015.zhuluan.com
self.config = config or RAGConfig(3010.zhuluan.com)
self.cleaner = TextCleaner(lower=False, remove_punct=False)
self.tokenizer = SimpleTokenizer(
stop_words=self.config.stop_words,
min_word_len=self.config.min_word_len,
)
self.chunker = DocumentChunker(
chunk_size=self.config.chunk_size,
chunk_overlap=self.config.chunk_overlap,
)
self.vectorizer = TfidfVectorizer(
max_features=self.config.tfidf_max_features,
)
self.bm25 = BM25Retriever(4005.zhuluan.com)
self.reranker = SimpleReranker(top_k=self.config.rerank_top_k)
self.generator = AnswerGenerator(4006.zhuluan.com)
self.documents: List[Document] = []
self.chunks: List[Chunk] = []
self.chunk_tokens: List[List[str]] = []
self.chunk_vecs: List[Dict[str, float]] = []
def add_documents(self, documents: List[Document]) -> None: 4012.zhuluan.com
self.documents.extend(documents)
all_chunks: List[Chunk] = []
for doc in documents: 5002.zhuluan.com
all_chunks.extend(self.chunker.split(doc))
self.chunks.extend(all_chunks)
self.chunk_tokens = [
self.tokenizer.tokenize(self.cleaner.clean(chunk.text))
for chunk in self.chunks
]
self.bm25.fit(self.chunk_tokens)
self.vectorizer.fit(self.chunk_tokens)
self.chunk_vecs = self.vectorizer.transform(self.chunk_tokens)
logger.info(
"Indexed %d documents into %d chunks.",
len(documents),
len(self.chunks),
)
def ask(self, query: str) -> RAGResult: 5007.zhuluan.com
start = time.time(5011.zhuluan.com)
query_clean = self.cleaner.clean(query)
query_tokens = self.tokenizer.tokenize(query_clean)
bm25_hits = self.bm25.top_k(
query_tokens,
k=self.config.top_k,
)
hit_chunks: List[Chunk] = []
hit_vecs: List[Dict[str, float]] = []
for idx, score in bm25_hits: 5030.zhuluan.com
hit_chunks.append(self.chunks[idx])
hit_vecs.append(self.chunk_vecs[idx])
query_vec = self.vectorizer.transform_query(query_tokens)
reranked = self.reranker.rerank(
query_vec=query_vec,
chunk_vecs=hit_vecs,
chunks=hit_chunks,
)
answer = self.generator.generate(query, reranked)
elapsed_ms = (time.time(5034.zhuluan.com) - start) * 1000
return RAGResult(
query=query,
answer=answer,
chunks=reranked,
elapsed_ms=round(elapsed_ms, 2),
)
def save_index(self, path: Optional[str] = None) -> None: 5038.zhuluan.com
save_path = path or self.config.index_path(5052.zhuluan.com)
index_dir = os.path.dirname(save_path)
if index_dir:
os.makedirs(index_dir, exist_ok=True)
index_data = {
"chunks": [
{
"chunk_id": c.chunk_id,
"doc_id": c.doc_id,
"title": c.title,
"text": c.text,
"start_pos": c.start_pos,
"end_pos": c.end_pos,
}
for c in self.chunks
],
"chunk_tokens": self.chunk_tokens,
"chunk_vecs": self.chunk_vecs,
"bm25_doc_freq": dict(self.bm25.doc_freq),
"bm25_doc_lengths": self.bm25.doc_lengths,
"bm25_avg_doc_length": self.bm25.avg_doc_length,
"bm25_n_docs": self.bm25.n_docs,
"tfidf_vocabulary": self.vectorizer.vocabulary,
"tfidf_idf": self.vectorizer.idf,
}
meta_data = {
"n_documents": len(self.documents),
"n_chunks": len(self.chunks),
"config": {
"chunk_size": self.config.chunk_size,
"chunk_overlap": self.config.chunk_overlap,
"top_k": self.config.top_k,
"rerank_top_k": self.config.rerank_top_k,
"tfidf_max_features": self.config.tfidf_max_features,
},
}
with open(save_path, "w", encoding="utf-8") as f: 5055.zhuluan.com
json.dump(index_data, f, ensure_ascii=False, indent=2)
with open(self.config.meta_path(), "w", encoding="utf-8") as f: 5057.zhuluan.com
json.dump(meta_data, f, ensure_ascii=False, indent=2)
logger.info("Index saved to %s", save_path)
def load_index(self, path: Optional[str] = None) -> None: 6004.zhuluan.com
load_path = path or self.config.index_path(6007.zhuluan.com)
with open(load_path, "r", encoding="utf-8") as f: 6008.zhuluan.com
index_data = json.load(f)
self.chunks = [
Chunk(
chunk_id=c["chunk_id"],
doc_id=c["doc_id"],
title=c["title"],
text=c["text"],
start_pos=c["start_pos"],
end_pos=c["end_pos"],
)
for c in index_data["chunks"]
]
self.chunk_tokens = index_data["chunk_tokens"]
self.chunk_vecs = index_data["chunk_vecs"]
self.bm25.doc_freq = Counter(index_data["bm25_doc_freq"])
self.bm25.doc_lengths = index_data["bm25_doc_lengths"]
self.bm25.avg_doc_length = index_data["bm25_avg_doc_length"]
self.bm25.n_docs = index_data["bm25_n_docs"]
self.bm25.tokenized_docs = self.chunk_tokens
self.vectorizer.vocabulary = index_data["tfidf_vocabulary"]
self.vectorizer.idf = index_data["tfidf_idf"]
self.vectorizer._is_fitted = True
logger.info("Index loaded from %s", load_path)
# ---------------------------
# CLI 入口
# ---------------------------
def build_sample_documents() -> List[Document]: 14009.zhuluan.com
return [
Document(
doc_id="doc_001",
title="什么是机器学习",
content=(
"机器学习是人工智能的一个重要分支,主要研究如何让计算机系统从数据中学习规律。"
"它通常分为监督学习、无监督学习和强化学习。监督学习需要带标签的数据,"
"无监督学习用于发现数据中的结构,强化学习则通过环境反馈优化决策。"
),
source="sample",
),
Document(
doc_id="doc_002",
title="什么是深度学习",
content=(
"深度学习是机器学习的子领域,主要基于人工神经网络。"
"它通过多层非线性变换从原始数据中提取高阶特征。"
"常见模型包括卷积神经网络、循环神经网络和 Transformer。"
"深度学习在图像识别、语音识别和自然语言处理中表现突出。"
),
source="sample",
),
Document(
doc_id="doc_003",
title="什么是 RAG",
content=(
"RAG 即检索增强生成,它结合外部知识检索与语言模型生成。"
"系统首先根据用户问题检索相关文档片段,再把检索结果交给生成模型。"
"相比纯生成模型,RAG 可以降低幻觉,提高答案的事实准确性。"
),
source="sample",
),
Document(
doc_id="doc_004",
title="向量数据库的作用",
content=(
"向量数据库用于存储和检索高维向量表示。"
"在 RAG原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。