字符串中的Pyspark双字符替换避免未映射到pandas或rdd的特定单词

from pyspark.sql import SparkSession
from pyspark.sql.functions import regexp_replace

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建示例数据集
data = [("Hello, World! This is a test.",), ("Another example with double characters.",)]
df = spark.createDataFrame(data, ["text"])

# 定义要替换的双字符序列和替换后的内容
replacement_map = {
    "is": "was",
    "th": "zz",
}

# 使用正则表达式替换双字符序列
for pattern, replacement in replacement_map.items():
    df = df.withColumn("text", regexp_replace("text", pattern, replacement))

# 显示替换后的结果
df.show(truncate=False)

+----------------------------------+
|text                              |
+----------------------------------+
|Hello, World! Tzz zz a test.       |
|Another example wz double characters.|
+----------------------------------+