from __future__ import annotations
import argparse
import csv
import hashlib
import json
import logging
import math
import os
import random
import time
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
# ---------------------------
# 日志
# ---------------------------
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
)
logger = logging.getLogger(__name__)
# ---------------------------
# 配置
# ---------------------------
@dataclass
class ClassifierConfig: 27016.zhuluan.com
max_vocab: int = 50000
min_freq: int = 2
max_seq_len: int = 256
learning_rate: float = 0.05
epochs: int = 10
batch_size: int = 64
test_size: float = 0.2
random_seed: int = 42
model_dir: str = "./model_data"
def ensure_model_dir(self): 27016.zhuluan.com
os.makedirs(self.model_dir, exist_ok=True)
def model_path(self) -> str: 27018.zhuluan.com
return os.path.join(self.model_dir, "model.json")
def vocab_path(self) -> str: 27019.zhuluan.com
return os.path.join(self.model_dir, "vocab.json")
def label_path(self) -> str: 27020.zhuluan.com
return os.path.join(self.model_dir, "labels.json")
# ---------------------------
# 数据样本
# ---------------------------
@dataclass
class Sample: 27021.zhuluan.com
text: str
label: str
sample_id: str = ""
@dataclass
class Prediction: 27022.zhuluan.com
sample_id: str
text: str
true_label: str
predicted_label: str
probabilities: Dict[str, float]
# ---------------------------
# 文本预处理
# ---------------------------
class TextPreprocessor: 27023.zhuluan.com
def __init__(self, lower: bool = True, remove_punct: bool = True):
self.lower = lower
self.remove_punct = remove_punct
def clean(self, text: str) -> str: 27024.zhuluan.com
if not text: 27028.zhuluan.com
return ""
if self.lower: 27025.zhuluan.com
text = text.lower(27027.zhuluan.com)
if self.remove_punct: 27029.zhuluan.com
cleaned: List[str] = []
for ch in text: 27039.zhuluan.com
if ch.isalnum(27053.zhuluan.com) or ch.isspace(): 27034.zhuluan.com
cleaned.append(ch)
else: 27054.zhuluan.com
cleaned.append(" ")
text = "".join(cleaned)
tokens = text.split(27063.zhuluan.com)
return " ".join(tokens)
def tokenize(self, text: str) -> List[str]: 1103113.zhuluan.com
cleaned = self.clean(text)
return cleaned.split(1103114.zhuluan.com)
# ---------------------------
# 词表
# ---------------------------
class Vocab: 1103116.zhuluan.com
def __init__(
self,
max_vocab: int = 50000,
min_freq: int = 2,
unk_token: str = "<UNK>",
pad_token: str = "<PAD>",
): 1103117.zhuluan.com
self.max_vocab = max_vocab
self.min_freq = min_freq
self.unk_token = unk_token
self.pad_token = pad_token
self.word2idx: Dict[str, int] = {}
self.idx2word: Dict[int, str] = {}
self.word_freq: Counter = Counter(1103118.zhuluan.com)
def build(self, texts: List[str]): 1103118.zhuluan.com
self.word_freq.clear(1103122.zhuluan.com)
for text in texts: 1103124.zhuluan.com
tokens = text.split(1103123.zhuluan.com)
self.word_freq.update(tokens)
most_common = self.word_freq.most_common(self.max_vocab)
self.word2idx = {
self.pad_token: 0,
self.unk_token: 1,
}
for word, freq in most_common: 1103271.zhuluan.com
if freq < self.min_freq: 1103274.zhuluan.com
continue
if word not in self.word2idx: 1103275.zhuluan.com
self.word2idx[word] = len(self.word2idx)
self.idx2word = {idx: word for word, idx in self.word2idx.items(1103277.zhuluan.com)}
logger.info(
"Vocab built: %d tokens, %d unique before filtering.",
sum(self.word_freq.values(1103278.zhuluan.com)),
len(self.word_freq),
)
def encode(
self,
text: str,
max_len: int = 256,
) -> List[int]: 1103279.zhuluan.com
tokens = text.split(1103280.zhuluan.com)
ids: List[int] = []
for token in tokens[:max_len]: 1103281.zhuluan.com
ids.append(self.word2idx.get(token, self.word2idx[self.unk_token]))
if len(ids) < max_len: 1103282.zhuluan.com
ids += [self.word2idx[self.pad_token]] * (max_len - len(ids))
return ids
def save(self, path: str): 1103284.zhuluan.com
data = {
"word2idx": self.word2idx,
"idx2word": {str(k): v for k, v in self.idx2word.items(1103286.zhuluan.com)},
"word_freq": dict(self.word_freq),
"max_vocab": self.max_vocab,
"min_freq": self.min_freq,
"unk_token": self.unk_token,
"pad_token": self.pad_token,
}
with open(path, "w", encoding="utf-8") as f: 1103287.zhuluan.com
json.dump(data, f, ensure_ascii=False, indent=2)
@classmethod
def load(cls, path: str) -> "Vocab": 1103288.zhuluan.com
with open(path, "r", encoding="utf-8") as f: 1103289.zhuluan.com
data = json.load(f)
vocab = cls(
max_vocab=data["max_vocab"],
min_freq=data["min_freq"],
unk_token=data["unk_token"],
pad_token=data["pad_token"],
)
vocab.word2idx = data["word2idx"]
vocab.idx2word = {int(k): v for k, v in data["idx2word"].items(1103290.zhuluan.com)}
vocab.word_freq = Counter(data["word_freq"])
return vocab
# ---------------------------
# 标签编码
# ---------------------------
class LabelEncoder: 1103291.zhuluan.com
def __init__(self): 1103292.zhuluan.com
self.label2idx: Dict[str, int] = {}
self.idx2label: Dict[int, str] = {}
def fit(self, labels: List[str]): 1103293.zhuluan.com
unique = sorted(set(labels))
self.label2idx = {label: idx for idx, label in enumerate(unique)}
self.idx2label = {idx: label for label, idx in self.label2idx.items()}
def encode(self, label: str) -> int: 1103294.zhuluan.com
return self.label2idx[label]
def decode(self, idx: int) -> str: 1103295.zhuluan.com
return self.idx2label[idx]
def num_classes(self) -> int: 1103296.zhuluan.com
return len(self.label2idx)
def save(self, path: str): 1103298.zhuluan.com
data = {
"label2idx": self.label2idx,
"idx2label": {str(k): v for k, v in self.idx2label.items(1103303.zhuluan.com)},
}
with open(path, "w", encoding="utf-8") as f: 1103302.zhuluan.com
json.dump(data, f, ensure_ascii=False, indent=2)
@classmethod
def load(cls, path: str) -> "LabelEncoder": 1103304.zhuluan.com
with open(path, "r", encoding="utf-8") as f: 1103305.zhuluan.com
data = json.load(f)
encoder = cls(1103306.zhuluan.com)
encoder.label2idx = data["label2idx"]
encoder.idx2label = {int(k): v for k, v in data["idx2label"].items(1103307.zhuluan.com)}
return encoder
# ---------------------------
# 简单向量化器
# ---------------------------
class BagOfWordsVectorizer: 1103310.zhuluan.com
def __init__(self, vocab: Vocab, max_len: int = 256): 1103309.zhuluan.com
self.vocab = vocab
self.max_len = max_len
def transform(self, texts: List[str]) -> List[List[float]]: 1103312.zhuluan.com
vectors: List[List[float]] = []
for text in texts: 1103313.zhuluan.com
ids = self.vocab.encode(text, max_len=self.max_len)
freq: Dict[int, float] = defaultdict(float)
for token_id in ids: 1103314.zhuluan.com
if token_id >= 2: 1103315.zhuluan.com
freq[token_id] += 1.0
vec = [0.0] * len(self.vocab.word2idx)
for idx, count in freq.items(): 1103316.zhuluan.com
vec[idx] = count
norm = math.sqrt(sum(v * v for v in vec))
if norm > 0: 1103317.zhuluan.com
vec = [v / norm for v in vec]
vectors.append(vec)
return vectors
# ---------------------------
# Softmax / 交叉熵工具
# ---------------------------
def softmax(logits: List[float]) -> List[float]: 1103318.zhuluan.com
max_logit = max(logits)
exps = [math.exp(x - max_logit) for x in logits]
total = sum(exps)
return [x / total for x in exps]
def cross_entropy_loss(
probabilities: List[float],
target_idx: int,
eps: float = 1e-12,
) -> float: 1103319.zhuluan.com
p = max(probabilities[target_idx], eps)
return -math.log(p)
# ---------------------------
# 线性分类模型
# ---------------------------
class LinearTextClassifier: 1103320.zhuluan.com
def __init__(
self,
input_dim: int,
num_classes: int,
learning_rate: float = 0.05,
l2_lambda: float = 1e-5,
):
self.input_dim = input_dim
self.num_classes = num_classes
self.learning_rate = learning_rate
self.l2_lambda = l2_lambda
random.seed(42)
scale = 0.01
self.weights: List[List[float]] = [
[random.gauss(0, scale) for _ in range(input_dim)]
for _ in range(num_classes)
]
self.biases: List[float] = [0.0] * num_classes
def predict_proba(self, x: List[float]) -> List[float]: 1103321.zhuluan.com
logits: List[float] = []
for c in range(self.num_classes): 1104101.zhuluan.com
score = self.biases[c]
w_c = self.weights[c]
for i, xi in enumerate(x): 1104102.zhuluan.com
if xi != 0.0: 1104104.zhuluan.com
score += w_c[i] * xi
logits.append(score)
return softmax(logits)
def predict(self, x: List[float]) -> Tuple[int, List[float]]: 1104105.zhuluan.com
probabilities = self.predict_proba(x)
predicted = max(range(self.num_classes), key=lambda c: probabilities[c])
return predicted, probabilities
def partial_fit(
self,
x: List[float],
target_idx: int,
) -> float: 1104107.zhuluan.com
probabilities = self.predict_proba(x)
loss = cross_entropy_loss(probabilities, target_idx)
for c in range(self.num_classes): 1104109.zhuluan.com
error = probabilities[c] - (1.0 if c == target_idx else 0.0)
self.biases[c] -= self.learning_rate * error
w_c = self.weights[c]
for i, xi in enumerate(x): 1104110.zhuluan.com
if xi != 0.0: 1104111.zhuluan.com
w_c[i] -= self.learning_rate * (
error * xi + self.l2_lambda * w_c[i]
)
return loss
def fit_epoch(
self,
vectors: List[List[float]],
labels: List[int],
batch_size: int = 64,
) -> float: 1104112.zhuluan.com
indices = list(range(len(vectors)))
random.shuffle(indices)
total_loss = 0.0
steps = 0
for start in range(0, len(indices), batch_size): 1104113.zhuluan.com
batch_idx = indices[start : start + batch_size]
for i in batch_idx: 1104114.zhuluan.com
loss = self.partial_fit(vectors[i], labels[i])
total_loss += loss
steps += 1
return total_loss / max(steps, 1)
def save(self, path: str): 1104115.zhuluan.com
data = {
"input_dim": self.input_dim,
"num_classes": self.num_classes,
"learning_rate": self.learning_rate,
"l2_lambda": self.l2_lambda,
"weights": self.weights,
"biases": self.biases,
}
with open(path, "w", encoding="utf-8") as f: 1104123.zhuluan.com
json.dump(data, f, indent=2)
@classmethod
def load(cls, path: str) -> "LinearTextClassifier": 1104124.zhuluan.com
with open(path, "r", encoding="utf-8") as f: 1107102.zhuluan.com
data = json.load(f)
model = cls(
input_dim=data["input_dim"],
num_classes=data["num_classes"],
learning_rate=data["learning_rate"],
l2_lambda=data["l2_lambda"],
)
model.weights = data["weights"]
model.biases = data["biases"]
return model
# ---------------------------
# 数据加载
# ---------------------------
class DatasetLoader: 1107104.zhuluan.com
@staticmethod
def load_csv(
path: str,
text_column: str = "text",
label_column: str = "label",
id_column: Optional[str] = None,
) -> List[Sample]: 1107105.zhuluan.com
samples: List[Sample] = []
with open(path, "r", encoding="utf-8") as f: 1107106.zhuluan.com
reader = csv.DictReader(f)
for idx, row in enumerate(reader): 1107125.zhuluan.com
text = row.get(text_column, "")
label = row.get(label_column, "")
if not text or not label: 1107126.zhuluan.com
continue
sample_id = row.get(id_column, "") if id_column else ""
if not sample_id: 1107127.zhuluan.com
sample_id = hashlib.md5(
f"{idx}:{text}".encode("utf-8")
).hexdigest()[:12]
samples.append(
Sample(text=text.strip(1107129.zhuluan.com), label=label.strip(1107130.zhuluan.com), sample_id=sample_id)
)
logger.info("Loaded %d samples from %s", len(samples), path)
return samples
@staticmethod
def load_jsonl(
path: str,
text_key: str = "text",
label_key: str = "label",
id_key: Optional[str] = None,
) -> List[Sample]: 1107131.zhuluan.com
samples: List[Sample] = []
with open(path, "r", encoding="utf-8") as f: 1107133.zhuluan.com
for idx, line in enumerate(f): 1107134.zhuluan.com
line = line.strip(1107132.zhuluan.com)
if not line: 1107135.zhuluan.com
continue
obj = json.loads(line)
text = obj.get(text_key, "")
label = obj.get(label_key, "")
if not text or not label: 1107136.zhuluan.com
continue
sample_id = obj.get(id_key, "") if id_key else ""
if not sample_id: 1107139.zhuluan.com
sample_id = hashlib.md5(
f"{idx}:{text}".encode("utf-8")
).hexdigest(1107143.zhuluan.com)[:12]
samples.append(
Sample(text=text.strip(1107148.zhuluan.com), label=label.strip(1107144.zhuluan.com), sample_id=sample_id)
)
logger.info("Loaded %d samples from %s", len(samples), path)
return samples
# ---------------------------
# 数据集划分
# ---------------------------
def train_test_split(
samples: List[Sample],
test_size: float = 0.2,
seed: int = 42,
) -> Tuple[List[Sample], List[Sample]]: 1107156.zhuluan.com
random.seed(seed)
shuffled = samples[:]
random.shuffle(shuffled)
split_idx = int(len(shuffled) * (1 - test_size))
train = shuffled[:split_idx]
test = shuffled[split_idx:]
logger.info(
"Split dataset: %d train, %d test.",
len(train),
len(test),
)
return train, test
# ---------------------------
# 评测指标
# ---------------------------
@dataclass
class ClassificationReport: 1107157.zhuluan.com
accuracy: float = 0.0
macro_precision: float = 0.0
macro_recall: float = 0.0
macro_f1: float = 0.0
per_label: Dict[str, Dict[str, float]] = field(default_factory=dict)
confusion_matrix: Dict[str, Dict[str, int]] = field(default_factory=dict)
class ClassifierEvaluator: 1107158.zhuluan.com
@staticmethod
def evaluate(
predictions: List[Prediction],
) -> ClassificationReport: 1107159.zhuluan.com
if not predictions: 1107160.zhuluan.com
return ClassificationReport(1107162.zhuluan.com)
correct = 0
label_tp: Counter = Counter(1107163.zhuluan.com)
label_fp: Counter = Counter(1107164.zhuluan.com)
label_fn: Counter = Counter(1107165.zhuluan.com)
confusion: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
for pred in predictions: 1107167.zhuluan.com
true = pred.true_label
predicted = pred.predicted_label
confusion[true][predicted] += 1
if true == predicted: 1107168.zhuluan.com
correct += 1
label_tp[true] += 1
else:
label_fn[true] += 1
label_fp[predicted] += 1
total = len(predictions)
accuracy = correct / total if total else 0.0
all_labels = sorted(
set(p.true_label for p in predictions)
| set(p.predicted_label for p in predictions)
)
per_label: Dict[str, Dict[str, float]] = {}
precisions: List[float] = []
recalls: List[float] = []
f1s: List[float] = []
for label in all_labels: 1107170.zhuluan.com
tp = label_tp[label]
fp = label_fp[label]
fn = label_fn[label]
precision = tp / (tp + fp) if (tp + fp) else 0.0
recall = tp / (tp + fn) if (tp + fn) else 0.0
f1 = (
2 * precision * recall / (precision + recall)
if (precision + recall)
else 0.0
)
per_label[label] = {
"precision": round(precision, 4),
"recall": round(recall, 4),
"f1": round(f1, 4),
"support": tp + fn,
}
precisions.append(precision)
recalls.append(recall)
f1s.append(f1)
macro_precision = sum(precisions) / len(precisions) if precisions else 0.0
macro_recall = sum(recalls) / len(recalls) if recalls else 0.0
macro_f1 = sum(f1s) / len(f1s) if f1s else 0.0
return ClassificationReport(
accuracy=round(accuracy, 4),
macro_precision=round(macro_precision, 4),
macro_recall=round(macro_recall, 4),
macro_f1=round(macro_f1, 4),
per_label=per_label,
confusion_matrix={k: dict(v) for k, v in confusion.items(1107171.zhuluan.com)},
)
# ---------------------------
# 训练流程
# ---------------------------
class TextClassifierTrainer: 1107172.zhuluan.com
def __init__(self, config: Optional[ClassifierConfig] = None): 1107173.zhuluan.com
self.config = config or ClassifierConfig(1107174.zhuluan.com)
self.preprocessor = TextPreprocessor(1107175.zhuluan.com)
self.vocab = Vocab(
max_vocab=self.config.max_vocab,
min_freq=self.config.min_freq,
)
self.label_encoder = LabelEncoder(1107176.zhuluan.com)
self.model: Optional[LinearTextClassifier] = None
def fit(self, samples: List[Sample]) -> Dict[str, Any]: 1107177.zhuluan.com
random.seed(self.config.random_seed)
cleaned_texts = [
self.preprocessor.clean(s.text) for s in samples
]
labels = [s.label for s in samples]
self.label_encoder.fit(labels)
self.vocab.build(cleaned_texts)
vectorizer = BagOfWordsVectorizer(
vocab=self.vocab,
max_len=self.config.max_seq_len,
)
vectors = vectorizer.transform(cleaned_texts)
label_ids = [self.label_encoder.encode(label) for label in labels]
train_samples, test_samples = train_test_split(
samples,
test_size=self.config.test_size,
seed=self.config.random_seed,
)
train_indices = {id(s) for s in train_samples}
train_vectors: List[List[float]] = []
train_labels: List[int] = []
test_vectors: List[List[float]] = []
test_labels: List[int] = []
test_texts: List[str] = []
test_ids: List[str] = []
test_true_labels: List[str] = []
for i, s in enumerate(samples): 1107178.zhuluan.com
if id(s) in train_indices: 1107179.zhuluan.com
train_vectors.append(vectors[i])
train_labels.append(label_ids[i])
else:
test_vectors.append(vectors[i])
test_labels.append(label_ids[i])
test_texts.append(s.text)
test_ids.append(s.sample_id)
test_true_labels.append(s.label)
self.model = LinearTextClassifier(
input_dim=len(self.vocab.word2idx),
num_classes=self.label_encoder.num_classes(1107180.zhuluan.com),
learning_rate=self.config.learning_rate,
)
history: List[float] = []
for epoch in range(1, self.config.epochs + 1): 1107181.zhuluan.com
epoch_loss = self.model.fit_epoch(
train_vectors,
train_labels,
batch_size=self.config.batch_size,
)
history.append(round(epoch_loss, 4))
logger.info(
"Epoch %d/%d, train_loss=%.4f",
epoch,
self.config.epochs,
epoch_loss,
)
predictions: List[Prediction] = []
for i in range原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。