始终从spark中的s3存储桶中读取最新文件夹

import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.conf.Configuration

val accessKeyId = "your_access_key_id"
val secretAccessKey = "your_secret_access_key"
val bucketName = "your_bucket_name"
val folderPath = "your_folder_path"

val conf = new Configuration()
conf.set("fs.s3a.access.key", accessKeyId)
conf.set("fs.s3a.secret.key", secretAccessKey)

val fs = FileSystem.get(conf)
val path = new Path(s"s3a://$bucketName/$folderPath")

val folders = fs.listStatus(path)
val latestFolder = folders
  .filter(_.isDirectory)
  .sortBy(_.getModificationTime)
  .last

val latestFolderPath = latestFolder.getPath.toString
val data = spark.read.textFile(latestFolderPath)