使用spacy对Pandas Dataframe中已解析的html文本列进行词法分析

import spacy
from spacy.lang.en import English
import pandas as pd

nlp = English()

def perform_lexical_analysis(text):
    doc = nlp(text)
    tokens = [token.text for token in doc]
    return tokens

df['lexical_tokens'] = df['html_text_column'].apply(perform_lexical_analysis)