SpringBoot整合HBase将数据写入Docker中的HBase

gzq大数据

发布于 2020-12-25 10:47:29

1.5K0

发布于 2020-12-25 10:47:29

文章被收录于专栏：大数据那些事大数据那些事

在之前的项目里，docker容器中已经运行了HBase，现将API操作HBase实现数据的增删改查通过SpringBoot整合Hbase是一个很好的选择首先打开IDEA，创建项目（project）时选择Spring Initializer

选择如下三种来创建工程

创建完后的pom文件如下（修改了spring-boot-starter-parent版本），并且加入了HBase的客户端依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.1.0.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.gzq</groupId>
    <artifactId>springboot-database</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>springboot-database</name>
    <description>Demo project for Spring Boot</description>

    <properties>
        <java.version>1.8</java.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-devtools</artifactId>
            <scope>runtime</scope>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>1.3.1</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <configuration>
                    <excludes>
                        <exclude>
                            <groupId>org.projectlombok</groupId>
                            <artifactId>lombok</artifactId>
                        </exclude>
                    </excludes>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>

随后写好java代码：

package com.gzq.springbootdatabase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;

import java.io.IOException;


public class TestHbase {


    @Test
    public void testCreate() throws IOException {
        Configuration configuration = HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.quorum", "xxx");
        Connection connection = ConnectionFactory.createConnection(configuration);

        Admin admin = connection.getAdmin();

        HTableDescriptor test1 = new HTableDescriptor(TableName.valueOf("xx:xxx"));
        HColumnDescriptor firstColumnDescriptor = new HColumnDescriptor("xxx".getBytes());
        HColumnDescriptor secondColumnDescriptor = new HColumnDescriptor("xxx".getBytes());
        HColumnDescriptor thirdColumnDescriptor = new HColumnDescriptor("xxx".getBytes());
        test1.addFamily(firstColumnDescriptor);
        test1.addFamily(secondColumnDescriptor);
        test1.addFamily(thirdColumnDescriptor);

        admin.createTable(test1);
        admin.close();
        connection.close();
    }

    @Test
    public void testCreateNameSpace() throws IOException {
        Configuration configuration = HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.quorum", "xxx");
        Connection connection = ConnectionFactory.createConnection(configuration);

        Admin admin = connection.getAdmin();

        NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("xxx").build();
        admin.createNamespace(namespaceDescriptor);

        admin.close();
        connection.close();
    }

    @Test
    public void testPutData() throws IOException {
        Configuration configuration = HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.quorum", "xxxx");
        Connection connection = ConnectionFactory.createConnection(configuration);


        //获取表对象
        Table table = connection.getTable(TableName.valueOf("xx:xxx"));

        Put put = new Put(Bytes.toBytes("xxx" + System.currentTimeMillis()));

        put.addColumn(Bytes.toBytes("xxx"), Bytes.toBytes("xx"), Bytes.toBytes("xx"));

        table.put(put);

        table.close();
        connection.close();


    }
}

ps：因为是在云服务器上进行操作（如果是在本地操作不需要看如下内容），所以为了安全，在云服务器上开启了防火墙，如果直接执行程序就会报错，无法连接，所以通过管道的方式安全连接，我用的是mobaSSHTunnel（MobaXterm工具下的插件），随后开启相应的端口，并且我的docker也映射了云服务器上的端口：

随后在mobaSSHTunnel开启已经配置好的端口映射即可安全的访问，我映射的端口有2181,16000,16020（这三个端口保证都打开很重要）这几个关键的端口，并且有一个坑：在 configuration.set(“hbase.zookeeper.quorum”, “xxx”);这行代码里后面的xxx是你的主机名称，我的HBase里的hbase-site.xml里面的配置对应的是cdata01，那么这个xxx必须是cdata01，但是通过你的管道访问时要连接端口必须通过2181连接，并且在mobaSSHTunnel里的对应的访问域名必须设为cdata01，而这个cdata01在你的windows上的hosts文件里必须映射的是127.0.0.1,（切记不要将你的hosts文件里的cdata01改成云服务器的地址，如果改成就直接访问云服务器了，但是云服务器开了防火墙，你必定连接不上，你唯一的通道是通过Tunnel连接，所以必须将此处的windows的hosts文件里设置为127.0.0.1），在mobaSSHTunnel对应的2181端口映射为下图：