使用HCFS读取JSON-newline文件

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.io.IOUtils;

public class HCFSReader {
    public static void main(String[] args) {
        String hdfsUri = "hdfs://your-hdfs-uri";
        String filePath = "/path/to/json-newline-file";

        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", hdfsUri);

        try {
            FileSystem fs = FileSystem.get(conf);
            Path file = new Path(filePath);

            if (fs.exists(file)) {
                FileStatus[] statuses = fs.listStatus(file);
                for (FileStatus status : statuses) {
                    Path currentPath = status.getPath();
                    FSDataInputStream inputStream = fs.open(currentPath);

                    // 读取JSON-newline文件内容
                    String line;
                    while ((line = inputStream.readLine()) != null) {
                        // 处理每一行的JSON对象
                        System.out.println(line);
                    }

                    IOUtils.closeStream(inputStream);
                }
            } else {
                System.out.println("File does not exist: " + filePath);
            }

            fs.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}