python中不使用NLTK的Tokenizer方法

import re

def tokenize_text(text):
    # 使用正则表达式将文本分割成单词
    tokens = re.findall(r'\b\w+\b', text)
    return tokens

# 示例用法
text = "This is a sample sentence."
tokens = tokenize_text(text)
print(tokens)

import spacy

def tokenize_text(text):
    # 加载英文分词模型
    nlp = spacy.load("en_core_web_sm")
    # 对文本进行分词
    doc = nlp(text)
    tokens = [token.text for token in doc]
    return tokens

# 示例用法
text = "This is a sample sentence."
tokens = tokenize_text(text)
print(tokens)