Pyspark检查hdfs文件夹修改日期

from pyspark import SparkContext, SparkConf
from py4j.java_gateway import java_import

# 创建Spark配置
conf = SparkConf().setAppName("HDFS Modification Date").setMaster("local")
sc = SparkContext(conf=conf)

# 导入Hadoop API
java_import(sc._gateway.jvm, "org.apache.hadoop.fs.FileSystem")
java_import(sc._gateway.jvm, "org.apache.hadoop.fs.Path")
java_import(sc._gateway.jvm, "org.apache.hadoop.fs.FileStatus")

# HDFS文件夹路径
folder_path = "hdfs://<namenode>:<port>/path/to/folder"

# 获取Hadoop文件系统对象
fs = sc._gateway.jvm.FileSystem.get(sc._jsc.hadoopConfiguration())

# 获取文件夹路径
folder = sc._gateway.jvm.Path(folder_path)

# 获取文件夹下的所有文件
file_statuses = fs.listStatus(folder)

# 遍历文件状态并打印修改日期
for file_status in file_statuses:
    modification_time = file_status.getModificationTime()
    print("File: {}, Modification Date: {}".format(file_status.getPath(), modification_time))

# 关闭Spark上下文
sc.stop()