基于TF-IDF和KNN的模糊字符串匹配优化

import pandas as pd
import numpy as np
from fuzzywuzzy import fuzz, process
import re
import itertools
from typing import Union, List, Tuple
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.neighbors import NearestNeighbors
import jieba
import pickle
import time


def preprocess_string(s):
    s = re.sub(r'(?<=\b\w)\s*[ &]\s*(?=\w\b)', '', s)
    return s

def build_vectorizer( clean: pd.Series,  analyzer: str = 'char', ngram_range: Tuple[int, int] = (1, 4),    n_neighbors: int = 1,    **kwargs    ) -> Tuple:
    vectorizer = TfidfVectorizer(analyzer = analyzer, ngram_range = ngram_range, **kwargs)
    X = vectorizer.fit_transform(clean.values.astype('U'))

    nbrs = NearestNeighbors(n_neighbors = n_neighbors, metric = 'cosine').fit(X)
    return vectorizer, nbrs

def tfidf_nn(
    messy,
    clean,
    n_neighbors = 1,
    **kwargs
    ):
    # Fit clean data and transform messy data
    vectorizer, nbrs = build_vectorizer(clean, n_neighbors = n_neighbors, **kwargs)
    input_vec = vectorizer.transform(messy)

    # Determine best possible matches
    distances, indices = nbrs.kneighbors(input_vec, n_neighbors = n_neighbors)
    nearest_values = np.array(clean)[indices]
    return nearest_values, distances

# String matching - match fuzzy
def find_matches_fuzzy(
    row,
    match_candidates,
    limit = 5
    ):
    row_matches = process.extract(
        row, dict(enumerate(match_candidates)),
        scorer = fuzz.token_sort_ratio,
        limit = limit
        )
    result = [(row, match[0], match[1]) for match in row_matches]
    return result

# String matching - TF-IDF
def fuzzy_nn_match(
    messy,
    clean,
    column,
    col,
    n_neighbors = 100,
    limit = 5, **kwargs):
    nearest_values, _ = tfidf_nn(messy, clean, n_neighbors, **kwargs)
    results = [find_matches_fuzzy(row, nearest_values[i], limit) for i, row in enumerate(messy)]
    df = pd.DataFrame(itertools.chain.from_iterable(results),
        columns = [column, col, 'Ratio']
        )
    return df


def fuzzy_tf_idf( df: pd.DataFrame,  column: str, clean: pd.Series,
    mapping_df: pd.DataFrame,    col: str,    analyzer: str = 'char',    ngram_range: Tuple[int, int] = (1, 3)   ) -> pd.Series:
    clean = clean.drop_duplicates().reset_index(drop = True)
    messy_prep = df[column].drop_duplicates().dropna().reset_index(drop = True).astype(str)
    messy = messy_prep.apply(preprocess_string)
    result = fuzzy_nn_match(messy = messy, clean = clean, column = column, col = col, n_neighbors = 1)
    return result

# Run the fuzzy string matching algorithm
start = time.time()
df_result = (df.pipe(fuzzy_tf_idf, # Function and messy data
                     column = 'Expedia', # Messy column in data
                     clean = df['Booking.com'], # Master data (list)
                     mapping_df = df, # Master data
                     col = 'Result') # Can be customized
            )
end = time.time()
print('Fuzzy string matching in {} seconds'.format(end - start))
df_result.head()

def stringMatching( df: pd.DataFrame,    column: str, clean: pd.Series,  mapping_df: pd.DataFrame,    col: str    ):
    categoryClean = clean.drop_duplicates().reset_index(drop = True)
    categoryMessy = df[column].drop_duplicates().dropna().reset_index(drop = True).astype(str)

    categoryFuzzy = {}
    ratioFuzzy = {}
    for i in range(len(categoryMessy)):
        resultFuzzy = process.extractOne(categoryMessy[i].lower(), categoryClean)
       catFuzzy = {categoryMessy[i]:resultFuzzy[0]}
        ratFuzzy = {categoryMessy[i]:resultFuzzy[1]}
        categoryFuzzy.update(catFuzzy)
        ratioFuzzy.update(ratFuzzy)

    catCol = col
    ratCol = 'Ratio'
    df[catCol] = df[column]
    df[ratCol] = df[column]

    df[catCol] = df[catCol].map(categoryFuzzy)
    df[ratCol] = df[ratCol].map(ratioFuzzy)
    return df

# Run the fuzzy string matching algorithm
start = time.time()
df_result = (df.pipe(stringMatching, # Function and messy data
                     column = 'Expedia', # Messy column in data
                     clean = df['Booking.com'], # Master data (list)
                     mapping_df = df, # Master data
                     col = 'Result') # Can be customized
            )
end = time.time()
print('Fuzzy string matching in {} seconds'.format(end - start))
df_result.head()