《HDFS的使用教程》---大数据系列

//启动hdfs
$ sbin/start-dfs.sh
//hdfs的shell操作：
    hdfs dfs -ls /                 --- 查看根目录下的文件
    hdfs dfs -put hello.txt /         --- 将本地的hello.txt提交到hdfs根目录下
    hdfs dfs -text /hello.txt         --- 查看hdfs目录下的hello.txt中的内容
    hdfs dfs -mkdir /test            --- 在hdfs中新建一个文件夹
    hdfs dfs -mkdir -p /test/a/b                    --- 在hdfs中递归地新建文件夹
    hdfs dfs -ls -R /               --- 递归地查看根目录下的所有文件
    hdfs dfs -copyFromLocal hello.txt /test/a/b/h.txt   --- 将本地的hello.txt提交到hdfs根目录下
    hdfs dfs -get /test/a/b/h.txt      --- 从hdfs根目录获取h.txt到本地
    hdfs dfs                    --- 查看帮助，基本跟Linux的命令操作一样。

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.zero</groupId>
  <artifactId>learnHdfs</artifactId>
  <version>1.0</version>
  <name>learnHdfs</name>
  <!-- FIXME change it to the project's website -->
  <url>http://www.example.com</url>
  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
    <maven.compiler.source>1.7</maven.compiler.source>
    <maven.compiler.target>1.7</maven.compiler.target>
  </properties>
  <repositories>
    <repository>
      <id>cloudera</id>
      <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
    </repository>
  </repositories>
  <dependencies>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client</artifactId>
      <version>${hadoop.version}</version>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.12</version>
      <scope>test</scope>
    </dependency>
  </dependencies>
</project>

package com.zero.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;

public class HDFSApp {
    public static  final  String HDFS_PATH = "hdfs://192.168.11.133:8020";
    FileSystem fileSystem =  null;
    Configuration configuration = null;

    @Before
    public void setUp() throws  Exception{
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration,"root");
        System.out.println("HDFSApp.setUp");
    }
    /**
     * 创建文件夹
     * @throws Exception
     */
   @Test
    public void mkdir() throws  Exception{
        fileSystem.mkdirs(new Path("/hdfsdat/test"));
    }
    /**
     * 新建文件
     * @throws Exception
     */
    @Test
    public void  create() throws  Exception{
        FSDataOutputStream output = fileSystem.create(new Path("/hdfsdat/test/a.txt"));
        output.write("hello baby".getBytes());
        output.flush();
        output.close();
    }
    /**
     * 查看hdfs文件的内容
     * @throws Exception
     */
    @Test
    public void cat() throws Exception{
        FSDataInputStream in = fileSystem.open(new Path("/hdfsdat/test/a.txt"));
        IOUtils.copyBytes(in,System.out,1024);
        in.close();
    }
    /**
     * 重命名
     * @throws Exception
     */
    @Test
    public void rename() throws Exception{
        Path oldPath = new Path("/hdfsdat/test/a.txt");
        Path newPath = new Path("/hdfsdat/test/b.txt");
        fileSystem.rename(oldPath,newPath);
    }
    /**
     * 上传文件到hdfs
     * @throws Exception
     */
    @Test
    public void copyFromLocalFile() throws  Exception{
        Path localPath = new Path("D:/data/h.txt");
        Path hdfsPath = new Path("/hdfsdat/test");
        fileSystem.copyFromLocalFile(localPath,hdfsPath);
    }
    /**
     * 上传大文件到hdfs，带进度条
     * @throws Exception
     */
    @Test
    public void copyFromLocalFileWithProgress() throws  Exception{
        InputStream in = new BufferedInputStream(new FileInputStream(
                new File("D:/downloads/spark-2.1.0-bin-2.6.0-cdh5.7.0.tgz")
        ));
        FSDataOutputStream output = fileSystem.create(new Path("/hdfsdat/test/spark-2.1.0-bin-2.6.0-cdh5.7.0.tgz"),
                new Progressable() {
                    @Override
                    public void progress() {
                        System.out.print("*");//进度提醒
                    }
                });
        IOUtils.copyBytes(in,output,4096);
    }
    /**
     * 下载文件到本地
     * @throws Exception
     */
    @Test
    public void copyTolocalFile() throws Exception{
        Path localPath = new Path("D:/data/h.txt");
        Path hdfsPath = new Path("/hdfsdat/test/h.txt");
//        fileSystem.copyToLocalFile(hdfsPath,localPath);//会报空指针的
        fileSystem.copyToLocalFile(false,hdfsPath,localPath,true);
    }
    @Test
    public void listFiles() throws Exception {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/hdfsdat/test"));
        for (FileStatus fileStatus: fileStatuses) {
            String isDir = fileStatus.isDirectory()?"文件夹":"文件";
            //几个副本
            short replication = fileStatus.getReplication();
            //文件的大小
            long len = fileStatus.getLen();
            String path = fileStatus.getPath().toString();
            System.out.println(isDir+"\t"+replication+"\t"+len+"\t"+path);
        }
    }
    @Test
    public void  delete() throws Exception{
        fileSystem.delete(new Path("/hdfsdat/test/a.txt"),false);//第二个参数指是否递归删除
    }
    @After
    public void tearDown() throws  Exception{
        configuration = null;
        fileSystem = null;
        System.out.println("HDFSApp.tearDown");
    }
}