使用正则表达式将句子拆分成标记，去掉所有必要的标点符号，但不包括作为单词一部分的标点符号

import re

def tokenize_sentence(sentence):
    # 定义需要去掉的标点符号
    punctuation_to_remove = r'[!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~]'
    
    # 使用正则表达式去掉不必要的标点符号
    cleaned_sentence = re.sub(punctuation_to_remove, '', sentence)
    
    # 使用正则表达式拆分句子成标记
    tokens = re.findall(r'\b\w+\b', cleaned_sentence)
    
    return tokens

# 示例句子
sentence = "Hello, world! This is a test-sentence with e-mail addresses like test@example.com."

# 拆分成标记
tokens = tokenize_sentence(sentence)
print(tokens)

sentence = "Hello, world! This is a test-sentence with e-mail addresses like test@example.com."

['Hello', 'world', 'This', 'is', 'a', 'testsentence', 'with', 'email', 'addresses', 'like', 'testexamplecom']

def tokenize_sentence(sentence):
    # 定义需要去掉的标点符号，保留连字符和点
    punctuation_to_remove = r'[!"#$%&\'()*+,/:;<=>?@[\\]^_`{|}~]'
    
    # 使用正则表达式去掉不必要的标点符号
    cleaned_sentence = re.sub(punctuation_to_remove, '', sentence)
    
    # 使用正则表达式拆分句子成标记
    tokens = re.findall(r'\b[\w.-]+\b', cleaned_sentence)
    
    return tokens

# 示例句子
sentence = "Hello, world! This is a test-sentence with e-mail addresses like test@example.com."

# 拆分成标记
tokens = tokenize_sentence(sentence)
print(tokens)

['Hello', 'world', 'This', 'is', 'a', 'test-sentence', 'with', 'e-mail', 'addresses', 'like', 'test@example.com']