首页
学习
活动
专区
圈层
工具
发布
社区首页 >专栏 >《用 Python 搭建一个轻量级 AI 文本分类与评测系统》

《用 Python 搭建一个轻量级 AI 文本分类与评测系统》

原创
作者头像
用户12583550
发布于 2026-11-16 21:21:03
发布于 2026-11-16 21:21:03
10
举报
代码语言:javascript
复制
from __future__ import annotations

import argparse
import csv
import hashlib
import json
import logging
import math
import os
import random
import time
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

# ---------------------------
# 日志
# ---------------------------

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
)
logger = logging.getLogger(__name__)

# ---------------------------
# 配置
# ---------------------------

@dataclass
class ClassifierConfig: 27016.zhuluan.com 
    max_vocab: int = 50000
    min_freq: int = 2
    max_seq_len: int = 256
    learning_rate: float = 0.05
    epochs: int = 10
    batch_size: int = 64
    test_size: float = 0.2
    random_seed: int = 42
    model_dir: str = "./model_data"

    def ensure_model_dir(self): 27016.zhuluan.com 
        os.makedirs(self.model_dir, exist_ok=True)

    def model_path(self) -> str: 27018.zhuluan.com 
        return os.path.join(self.model_dir, "model.json")

    def vocab_path(self) -> str: 27019.zhuluan.com 
        return os.path.join(self.model_dir, "vocab.json")

    def label_path(self) -> str: 27020.zhuluan.com 
        return os.path.join(self.model_dir, "labels.json")


# ---------------------------
# 数据样本
# ---------------------------

@dataclass
class Sample: 27021.zhuluan.com 
    text: str
    label: str
    sample_id: str = ""


@dataclass
class Prediction: 27022.zhuluan.com 
    sample_id: str
    text: str
    true_label: str
    predicted_label: str
    probabilities: Dict[str, float]


# ---------------------------
# 文本预处理
# ---------------------------

class TextPreprocessor: 27023.zhuluan.com 
    def __init__(self, lower: bool = True, remove_punct: bool = True):
        self.lower = lower
        self.remove_punct = remove_punct

    def clean(self, text: str) -> str: 27024.zhuluan.com 
        if not text: 27028.zhuluan.com 
            return ""

        if self.lower: 27025.zhuluan.com
 
            text = text.lower(27027.zhuluan.com)

        if self.remove_punct: 27029.zhuluan.com 
            cleaned: List[str] = []
            for ch in text: 27039.zhuluan.com 
                if ch.isalnum(27053.zhuluan.com) or ch.isspace(): 27034.zhuluan.com 
                    cleaned.append(ch)
                else: 27054.zhuluan.com 
                    cleaned.append(" ")
            text = "".join(cleaned)

        tokens = text.split(27063.zhuluan.com)
        return " ".join(tokens)

    def tokenize(self, text: str) -> List[str]: 1103113.zhuluan.com 
        cleaned = self.clean(text)
        return cleaned.split(1103114.zhuluan.com)


# ---------------------------
# 词表
# ---------------------------

class Vocab: 1103116.zhuluan.com 
    def __init__(
        self,
        max_vocab: int = 50000,
        min_freq: int = 2,
        unk_token: str = "<UNK>",
        pad_token: str = "<PAD>",
    ): 1103117.zhuluan.com 
        self.max_vocab = max_vocab
        self.min_freq = min_freq
        self.unk_token = unk_token
        self.pad_token = pad_token

        self.word2idx: Dict[str, int] = {}
        self.idx2word: Dict[int, str] = {}
        self.word_freq: Counter = Counter(1103118.zhuluan.com)

    def build(self, texts: List[str]): 1103118.zhuluan.com 
        self.word_freq.clear(1103122.zhuluan.com)

        for text in texts: 1103124.zhuluan.com 
            tokens = text.split(1103123.zhuluan.com)
            self.word_freq.update(tokens)

        most_common = self.word_freq.most_common(self.max_vocab)

        self.word2idx = {
            self.pad_token: 0,
            self.unk_token: 1,
        }

        for word, freq in most_common: 1103271.zhuluan.com 
            if freq < self.min_freq: 1103274.zhuluan.com 
                continue
            if word not in self.word2idx: 1103275.zhuluan.com 
                self.word2idx[word] = len(self.word2idx)

        self.idx2word = {idx: word for word, idx in self.word2idx.items(1103277.zhuluan.com)}

        logger.info(
            "Vocab built: %d tokens, %d unique before filtering.",
            sum(self.word_freq.values(1103278.zhuluan.com)),
            len(self.word_freq),
        )

    def encode(
        self,
        text: str,
        max_len: int = 256,
    ) -> List[int]: 1103279.zhuluan.com 
        tokens = text.split(1103280.zhuluan.com)
        ids: List[int] = []

        for token in tokens[:max_len]: 1103281.zhuluan.com 
            ids.append(self.word2idx.get(token, self.word2idx[self.unk_token]))

        if len(ids) < max_len: 1103282.zhuluan.com 
            ids += [self.word2idx[self.pad_token]] * (max_len - len(ids))

        return ids

    def save(self, path: str): 1103284.zhuluan.com 
        data = {
            "word2idx": self.word2idx,
            "idx2word": {str(k): v for k, v in self.idx2word.items(1103286.zhuluan.com)},
            "word_freq": dict(self.word_freq),
            "max_vocab": self.max_vocab,
            "min_freq": self.min_freq,
            "unk_token": self.unk_token,
            "pad_token": self.pad_token,
        }
        with open(path, "w", encoding="utf-8") as f: 1103287.zhuluan.com 
            json.dump(data, f, ensure_ascii=False, indent=2)

    @classmethod
    def load(cls, path: str) -> "Vocab": 1103288.zhuluan.com 
        with open(path, "r", encoding="utf-8") as f: 1103289.zhuluan.com 
            data = json.load(f)

        vocab = cls(
            max_vocab=data["max_vocab"],
            min_freq=data["min_freq"],
            unk_token=data["unk_token"],
            pad_token=data["pad_token"],
        )
        vocab.word2idx = data["word2idx"]
        vocab.idx2word = {int(k): v for k, v in data["idx2word"].items(1103290.zhuluan.com)}
        vocab.word_freq = Counter(data["word_freq"])
        return vocab


# ---------------------------
# 标签编码
# ---------------------------

class LabelEncoder: 1103291.zhuluan.com 
    def __init__(self): 1103292.zhuluan.com 
        self.label2idx: Dict[str, int] = {}
        self.idx2label: Dict[int, str] = {}

    def fit(self, labels: List[str]): 1103293.zhuluan.com 
        unique = sorted(set(labels))
        self.label2idx = {label: idx for idx, label in enumerate(unique)}
        self.idx2label = {idx: label for label, idx in self.label2idx.items()}

    def encode(self, label: str) -> int: 1103294.zhuluan.com 
        return self.label2idx[label]

    def decode(self, idx: int) -> str: 1103295.zhuluan.com 
        return self.idx2label[idx]

    def num_classes(self) -> int: 1103296.zhuluan.com 
        return len(self.label2idx)

    def save(self, path: str): 1103298.zhuluan.com 
        data = {
            "label2idx": self.label2idx,
            "idx2label": {str(k): v for k, v in self.idx2label.items(1103303.zhuluan.com)},
        }
        with open(path, "w", encoding="utf-8") as f: 1103302.zhuluan.com 
            json.dump(data, f, ensure_ascii=False, indent=2)

    @classmethod
    def load(cls, path: str) -> "LabelEncoder": 1103304.zhuluan.com 
        with open(path, "r", encoding="utf-8") as f: 1103305.zhuluan.com 
            data = json.load(f)

        encoder = cls(1103306.zhuluan.com)
        encoder.label2idx = data["label2idx"]
        encoder.idx2label = {int(k): v for k, v in data["idx2label"].items(1103307.zhuluan.com)}
        return encoder


# ---------------------------
# 简单向量化器
# ---------------------------

class BagOfWordsVectorizer: 1103310.zhuluan.com 
    def __init__(self, vocab: Vocab, max_len: int = 256): 1103309.zhuluan.com 
        self.vocab = vocab
        self.max_len = max_len

    def transform(self, texts: List[str]) -> List[List[float]]: 1103312.zhuluan.com 
        vectors: List[List[float]] = []

        for text in texts: 1103313.zhuluan.com 
            ids = self.vocab.encode(text, max_len=self.max_len)
            freq: Dict[int, float] = defaultdict(float)

            for token_id in ids: 1103314.zhuluan.com 
                if token_id >= 2: 1103315.zhuluan.com 
                    freq[token_id] += 1.0

            vec = [0.0] * len(self.vocab.word2idx)
            for idx, count in freq.items(): 1103316.zhuluan.com 
                vec[idx] = count

            norm = math.sqrt(sum(v * v for v in vec))
            if norm > 0: 1103317.zhuluan.com 
                vec = [v / norm for v in vec]

            vectors.append(vec)

        return vectors


# ---------------------------
# Softmax / 交叉熵工具
# ---------------------------

def softmax(logits: List[float]) -> List[float]: 1103318.zhuluan.com 
    max_logit = max(logits)
    exps = [math.exp(x - max_logit) for x in logits]
    total = sum(exps)
    return [x / total for x in exps]


def cross_entropy_loss(
    probabilities: List[float],
    target_idx: int,
    eps: float = 1e-12,
) -> float: 1103319.zhuluan.com 
    p = max(probabilities[target_idx], eps)
    return -math.log(p)


# ---------------------------
# 线性分类模型
# ---------------------------

class LinearTextClassifier: 1103320.zhuluan.com 
    def __init__(
        self,
        input_dim: int,
        num_classes: int,
        learning_rate: float = 0.05,
        l2_lambda: float = 1e-5,
    ):
        self.input_dim = input_dim
        self.num_classes = num_classes
        self.learning_rate = learning_rate
        self.l2_lambda = l2_lambda

        random.seed(42)
        scale = 0.01
        self.weights: List[List[float]] = [
            [random.gauss(0, scale) for _ in range(input_dim)]
            for _ in range(num_classes)
        ]
        self.biases: List[float] = [0.0] * num_classes

    def predict_proba(self, x: List[float]) -> List[float]: 1103321.zhuluan.com 
        logits: List[float] = []

        for c in range(self.num_classes): 1104101.zhuluan.com 
            score = self.biases[c]
            w_c = self.weights[c]
            for i, xi in enumerate(x): 1104102.zhuluan.com 
                if xi != 0.0: 1104104.zhuluan.com 
                    score += w_c[i] * xi
            logits.append(score)

        return softmax(logits)

    def predict(self, x: List[float]) -> Tuple[int, List[float]]: 1104105.zhuluan.com 
        probabilities = self.predict_proba(x)
        predicted = max(range(self.num_classes), key=lambda c: probabilities[c])
        return predicted, probabilities

    def partial_fit(
        self,
        x: List[float],
        target_idx: int,
    ) -> float: 1104107.zhuluan.com 
        probabilities = self.predict_proba(x)
        loss = cross_entropy_loss(probabilities, target_idx)

        for c in range(self.num_classes): 1104109.zhuluan.com 
            error = probabilities[c] - (1.0 if c == target_idx else 0.0)

            self.biases[c] -= self.learning_rate * error

            w_c = self.weights[c]
            for i, xi in enumerate(x): 1104110.zhuluan.com 
                if xi != 0.0: 1104111.zhuluan.com 
                    w_c[i] -= self.learning_rate * (
                        error * xi + self.l2_lambda * w_c[i]
                    )

        return loss

    def fit_epoch(
        self,
        vectors: List[List[float]],
        labels: List[int],
        batch_size: int = 64,
    ) -> float: 1104112.zhuluan.com 
        indices = list(range(len(vectors)))
        random.shuffle(indices)

        total_loss = 0.0
        steps = 0

        for start in range(0, len(indices), batch_size): 1104113.zhuluan.com 
            batch_idx = indices[start : start + batch_size]

            for i in batch_idx: 1104114.zhuluan.com 
                loss = self.partial_fit(vectors[i], labels[i])
                total_loss += loss
                steps += 1

        return total_loss / max(steps, 1)

    def save(self, path: str): 1104115.zhuluan.com 
        data = { 
            "input_dim": self.input_dim,
            "num_classes": self.num_classes,
            "learning_rate": self.learning_rate,
            "l2_lambda": self.l2_lambda,
            "weights": self.weights,
            "biases": self.biases,
        }
        with open(path, "w", encoding="utf-8") as f: 1104123.zhuluan.com 
            json.dump(data, f, indent=2)

    @classmethod
    def load(cls, path: str) -> "LinearTextClassifier": 1104124.zhuluan.com 
        with open(path, "r", encoding="utf-8") as f: 1107102.zhuluan.com 
            data = json.load(f)

        model = cls(
            input_dim=data["input_dim"],
            num_classes=data["num_classes"],
            learning_rate=data["learning_rate"],
            l2_lambda=data["l2_lambda"],
        )
        model.weights = data["weights"]
        model.biases = data["biases"]
        return model


# ---------------------------
# 数据加载
# ---------------------------

class DatasetLoader: 1107104.zhuluan.com 
    @staticmethod
    def load_csv(
        path: str,
        text_column: str = "text",
        label_column: str = "label",
        id_column: Optional[str] = None,
    ) -> List[Sample]: 1107105.zhuluan.com 
        samples: List[Sample] = []

        with open(path, "r", encoding="utf-8") as f: 1107106.zhuluan.com 
            reader = csv.DictReader(f)
            for idx, row in enumerate(reader): 1107125.zhuluan.com 
                text = row.get(text_column, "")
                label = row.get(label_column, "")

                if not text or not label: 1107126.zhuluan.com 
                    continue

                sample_id = row.get(id_column, "") if id_column else ""
                if not sample_id: 1107127.zhuluan.com 
                    sample_id = hashlib.md5(
                        f"{idx}:{text}".encode("utf-8")
                    ).hexdigest()[:12]

                samples.append(
                    Sample(text=text.strip(1107129.zhuluan.com), label=label.strip(1107130.zhuluan.com), sample_id=sample_id)
                )

        logger.info("Loaded %d samples from %s", len(samples), path)
        return samples

    @staticmethod
    def load_jsonl(
        path: str,
        text_key: str = "text",
        label_key: str = "label",
        id_key: Optional[str] = None,
    ) -> List[Sample]: 1107131.zhuluan.com 
        samples: List[Sample] = []

        with open(path, "r", encoding="utf-8") as f: 1107133.zhuluan.com 
            for idx, line in enumerate(f): 1107134.zhuluan.com 
                line = line.strip(1107132.zhuluan.com)
                if not line: 1107135.zhuluan.com 
                    continue

                obj = json.loads(line)
                text = obj.get(text_key, "")
                label = obj.get(label_key, "")

                if not text or not label: 1107136.zhuluan.com 
                    continue

                sample_id = obj.get(id_key, "") if id_key else ""
                if not sample_id: 1107139.zhuluan.com 
                    sample_id = hashlib.md5(
                        f"{idx}:{text}".encode("utf-8")
                    ).hexdigest(1107143.zhuluan.com)[:12]

                samples.append(
                    Sample(text=text.strip(1107148.zhuluan.com), label=label.strip(1107144.zhuluan.com), sample_id=sample_id)
                )

        logger.info("Loaded %d samples from %s", len(samples), path)
        return samples


# ---------------------------
# 数据集划分
# ---------------------------

def train_test_split(
    samples: List[Sample],
    test_size: float = 0.2,
    seed: int = 42,
) -> Tuple[List[Sample], List[Sample]]: 1107156.zhuluan.com 
    random.seed(seed)
    shuffled = samples[:]
    random.shuffle(shuffled)

    split_idx = int(len(shuffled) * (1 - test_size))
    train = shuffled[:split_idx]
    test = shuffled[split_idx:]

    logger.info(
        "Split dataset: %d train, %d test.",
        len(train),
        len(test),
    )
    return train, test


# ---------------------------
# 评测指标
# ---------------------------

@dataclass
class ClassificationReport: 1107157.zhuluan.com 
    accuracy: float = 0.0
    macro_precision: float = 0.0
    macro_recall: float = 0.0
    macro_f1: float = 0.0
    per_label: Dict[str, Dict[str, float]] = field(default_factory=dict)
    confusion_matrix: Dict[str, Dict[str, int]] = field(default_factory=dict)


class ClassifierEvaluator: 1107158.zhuluan.com 
    @staticmethod
    def evaluate(
        predictions: List[Prediction],
    ) -> ClassificationReport: 1107159.zhuluan.com 
        if not predictions: 1107160.zhuluan.com 
            return ClassificationReport(1107162.zhuluan.com)

        correct = 0
        label_tp: Counter = Counter(1107163.zhuluan.com)
        label_fp: Counter = Counter(1107164.zhuluan.com)
        label_fn: Counter = Counter(1107165.zhuluan.com)

        confusion: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))

        for pred in predictions: 1107167.zhuluan.com 
            true = pred.true_label
            predicted = pred.predicted_label

            confusion[true][predicted] += 1

            if true == predicted: 1107168.zhuluan.com 
                correct += 1
                label_tp[true] += 1
            else:
                label_fn[true] += 1
                label_fp[predicted] += 1

        total = len(predictions)
        accuracy = correct / total if total else 0.0

        all_labels = sorted(
            set(p.true_label for p in predictions)
            | set(p.predicted_label for p in predictions)
        )

        per_label: Dict[str, Dict[str, float]] = {}
        precisions: List[float] = []
        recalls: List[float] = []
        f1s: List[float] = []

        for label in all_labels: 1107170.zhuluan.com 
            tp = label_tp[label]
            fp = label_fp[label]
            fn = label_fn[label]

            precision = tp / (tp + fp) if (tp + fp) else 0.0
            recall = tp / (tp + fn) if (tp + fn) else 0.0
            f1 = (
                2 * precision * recall / (precision + recall)
                if (precision + recall)
                else 0.0
            )

            per_label[label] = {
                "precision": round(precision, 4),
                "recall": round(recall, 4),
                "f1": round(f1, 4),
                "support": tp + fn,
            }

            precisions.append(precision)
            recalls.append(recall)
            f1s.append(f1)

        macro_precision = sum(precisions) / len(precisions) if precisions else 0.0
        macro_recall = sum(recalls) / len(recalls) if recalls else 0.0
        macro_f1 = sum(f1s) / len(f1s) if f1s else 0.0

        return ClassificationReport(
            accuracy=round(accuracy, 4),
            macro_precision=round(macro_precision, 4),
            macro_recall=round(macro_recall, 4),
            macro_f1=round(macro_f1, 4),
            per_label=per_label,
            confusion_matrix={k: dict(v) for k, v in confusion.items(1107171.zhuluan.com)},
        )


# ---------------------------
# 训练流程
# ---------------------------

class TextClassifierTrainer: 1107172.zhuluan.com 
    def __init__(self, config: Optional[ClassifierConfig] = None): 1107173.zhuluan.com 
        self.config = config or ClassifierConfig(1107174.zhuluan.com)
        self.preprocessor = TextPreprocessor(1107175.zhuluan.com)
        self.vocab = Vocab(
            max_vocab=self.config.max_vocab,
            min_freq=self.config.min_freq,
        )
        self.label_encoder = LabelEncoder(1107176.zhuluan.com)
        self.model: Optional[LinearTextClassifier] = None

    def fit(self, samples: List[Sample]) -> Dict[str, Any]: 1107177.zhuluan.com 
        random.seed(self.config.random_seed)

        cleaned_texts = [
            self.preprocessor.clean(s.text) for s in samples
        ]

        labels = [s.label for s in samples]
        self.label_encoder.fit(labels)

        self.vocab.build(cleaned_texts)

        vectorizer = BagOfWordsVectorizer(
            vocab=self.vocab,
            max_len=self.config.max_seq_len,
        )

        vectors = vectorizer.transform(cleaned_texts)
        label_ids = [self.label_encoder.encode(label) for label in labels]

        train_samples, test_samples = train_test_split(
            samples,
            test_size=self.config.test_size,
            seed=self.config.random_seed,
        )

        train_indices = {id(s) for s in train_samples}
        train_vectors: List[List[float]] = []
        train_labels: List[int] = []

        test_vectors: List[List[float]] = []
        test_labels: List[int] = []
        test_texts: List[str] = []
        test_ids: List[str] = []
        test_true_labels: List[str] = []

        for i, s in enumerate(samples): 1107178.zhuluan.com 
            if id(s) in train_indices: 1107179.zhuluan.com 
                train_vectors.append(vectors[i])
                train_labels.append(label_ids[i])
            else:
                test_vectors.append(vectors[i])
                test_labels.append(label_ids[i])
                test_texts.append(s.text)
                test_ids.append(s.sample_id)
                test_true_labels.append(s.label)

        self.model = LinearTextClassifier(
            input_dim=len(self.vocab.word2idx),
            num_classes=self.label_encoder.num_classes(1107180.zhuluan.com),
            learning_rate=self.config.learning_rate,
        )

        history: List[float] = []

        for epoch in range(1, self.config.epochs + 1): 1107181.zhuluan.com 
            epoch_loss = self.model.fit_epoch(
                train_vectors,
                train_labels,
                batch_size=self.config.batch_size,
            )
            history.append(round(epoch_loss, 4))
            logger.info(
                "Epoch %d/%d, train_loss=%.4f",
                epoch,
                self.config.epochs,
                epoch_loss,
            )

        predictions: List[Prediction] = []

        for i in range

原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。

如有侵权,请联系 cloudcommunity@tencent.com 删除。

问题归档专栏文章快讯文章归档关键词归档开发者手册归档开发者手册 Section 归档