spark读取HDFS中zip文件的内容

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.input.PortableDataStream
import java.util.zip.ZipInputStream
import java.io.BufferedReader
import java.io.InputStreamReader

val conf = new SparkConf().setAppName("ReadZipFile").setMaster("local")
val sc = new SparkContext(conf)

val zipFile = "hdfs://path/to/your/zipfile.zip"
val zipData = sc.binaryFiles(zipFile)
val fileContents = zipData.flatMap { case (filePath, fileData) =>
  val zipStream = new ZipInputStream(fileData.open())
  Stream.continually(zipStream.getNextEntry)
    .takeWhile(_ != null)
    .flatMap { entry =>
      val br = new BufferedReader(new InputStreamReader(zipStream))
      Stream.continually(br.readLine()).takeWhile(_ != null)
    }
}

fileContents.foreach(println)