什么是Avro?Hadoop首选串行化系统——Avro简介及详细使用

{
    "namespace": "com.czxy.hdfs.avro",
    "type": "record",
    "name": "User",
    "fields": [
        {"name": "name", "type": "string"},
        {"name": "id", "type": "int"},
        {"name": "salary", "type": "int"},
        {"name": "age", "type": "int"},
        {"name": "address", "type": "string"}
    ]
}

 <dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
        <dependency>
            <groupId>org.apache.avro</groupId>
            <artifactId>avro</artifactId>
            <version>1.9.2</version>
        </dependency>


        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13</version>
        </dependency>

        <dependency>
            <groupId>org.junit.jupiter</groupId>
            <artifactId>junit-jupiter-api</artifactId>
            <version>RELEASE</version>
            <scope>compile</scope>
        </dependency>


    </dependencies>

    /**
     * 使用编译的方式,实现序列化avro文件
     * @throws Exception
     */
    @Test
   void test01() throws Exception {

        // 声明并初始化User对象
        // 方式1
        User user1 = new User();
        user1.setName("zhangsan");
        user1.setId(1);
        user1.setSalary(1000);
        user1.setAge(20);
        user1.setAddress("beijing");

        // 方式2 使用构造函数
        User user2 = new User("wangwu", 2, 1000, 19, "guangzhou");

        // 方式3 使用Build方式
        User user3 = User.newBuilder()
                .setName("lisi")
                .setId(3)
                .setAge(21)
                .setSalary(2000)
                .setAddress("shenzhen")
                .build();

        // avro文件存放目录
        String path = "G:\\2020干货\\avro\\user.avro";

        // 创建write对象[创建一个写入器]
        DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);

        // 写入文件[创建一个数据文件写入器,对写入器进行包装]
        DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);

        // 串行化数据到磁盘[schema 就是我们写的模式文件经过转义后的内容]
        dataFileWriter.create(user1.getSchema(), new File(path));

        // 把生成的user对象写入到avro文件
        dataFileWriter.append(user1);
        dataFileWriter.append(user2);
        dataFileWriter.append(user3);

        // 关闭流
        dataFileWriter.close();

    }

/**
     * 使用编译的方式,实现avro的反序列化
     * @throws Exception
     */
    @Test
    void show02() throws Exception {

        DatumReader<User> reader = new SpecificDatumReader<User>(User.class);
        DataFileReader<User> dataFileReader = new DataFileReader<User>(new File("G:\\2020干货\\avro\\user.avro"),reader);
        User user = null;
    
        // 此处采用迭代器遍历
        while (dataFileReader.hasNext()){

            user= dataFileReader.next();

            System.out.println(user);

        }
        
        // 关闭流
        dataFileReader.close();
    }

 /**
     *  直接使用schema文件进行写,不需要编译
     */
    @Test
    void show03() throws Exception {

        // 指定定义的avsc文件[加载]
        Schema schema = new Schema.Parser().parse(new File("G:\\2020干货\\avro\\User.avsc"));

        // 创建GenericRecord,相当于 User1
        GenericRecord user1 = new GenericData.Record(schema);

        //  设置javabean的属性
        user1.put("name","zhaoliu");
        user1.put("id",1);
        user1.put("salary",3000);
        user1.put("age",18);
        user1.put("address","shanghai");

        GenericRecord user2 = new GenericData.Record(schema);

        // 设置javabean的属性
        user2.put("id",2);
        user2.put("name","maqi");
        user2.put("salary",3000);
        user2.put("age",28);
        user2.put("address","nanjing");

        // 数据写入
        DatumWriter datumWriter = new SpecificDatumWriter(schema);

        DataFileWriter userfileWrite = new DataFileWriter(datumWriter);

        userfileWrite.create(schema,new File("G:\\2020干货\\avro\\user2.avro"));

        userfileWrite.append(user1);
        userfileWrite.append(user2);

        // 关闭流
        userfileWrite.close();

    }

 /**
     * 直接使用schema文件进行读,不需要编译
     * 反串行化avro数据
     * @throws Exception
     */
    @Test
    void show04() throws Exception{

        // 指定定义的avsc文件
        Schema schema = new Schema.Parser().parse(new File("G:\\2020干货\\avro\\User.avsc"));

        DatumReader r1 = new SpecificDatumReader(schema);
        DataFileReader r2 = new DataFileReader(new File("G:\\2020干货\\avro\\user2.avro"),r1);

        // 创建GenericRecord,相当于 User1
        GenericRecord user = null;

        while (r2.hasNext()){
           user = (GenericRecord) r2.next();
            System.out.println(user);
        }
    }