海量文档分类算法选择与实现

# 方案一：构建 TF-log(1 + IDF) 向量并使用余弦相似度进行分类
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
​
vectorizer = TfidfVectorizer()
X_train = vectorizer.fit_transform(train_documents)
X_test = vectorizer.transform(test_documents)
​
category_vectors = []
for category in categories:
    category_documents = [doc for doc in train_documents if doc.category == category]
    category_vector = vectorizer.fit_transform(category_documents)
    category_vectors.append(category_vector)
​
for test_document in X_test:
    similarities = cosine_similarity(test_document, category_vectors)
    predicted_category = categories[np.argmax(similarities)]
    print(f"Predicted category for document {test_document}: {predicted_category}")
​
# 方案二：使用具有 L1 惩罚项的逻辑回归模型进行分类
from sklearn.linear_model import LogisticRegression
​
classifier = LogisticRegression(penalty='l1')
classifier.fit(X_train, y_train)
​
y_pred = classifier.predict(X_test)
print(classification_report(y_test, y_pred))
​
# 方案三：使用多分类 SVM 进行分类
from sklearn.svm import SVC
​
classifier = SVC(kernel='linear')
classifier.fit(X_train, y_train)
​
y_pred = classifier.predict(X_test)
print(classification_report(y_test, y_pred))
​
# 方案四：使用 "无类别" 选项进行分类
from sklearn.preprocessing import LabelEncoder
​
le = LabelEncoder()
y_train = le.fit_transform(y_train)
y_test = le.transform(y_test)
​
classifier = LogisticRegression(penalty='l1')
classifier.fit(X_train, y_train)
​
y_pred = classifier.predict(X_test)
y_pred[y_pred == -1] = 'no_category'
y_test[y_test == -1] = 'no_category'
​
print(classification_report(y_test, y_pred))