使用Distcp和HMS-Mirror同步Hive到CDP

[root@ccycloud hive-testbench]# hdfs dfsadmin -allowSnapshot /user/hive/warehouse/test_db.db
Allowing snaphot on /user/hive/warehouse/test_db.db succeeded
[root@ccycloud hive-testbench]#

[root@ccycloud hive-testbench]# hdfs dfs -createSnapshot /user/hive/warehouse/test_db.db test_db_snapshot
Created snapshot /user/hive/warehouse/test_db.db/.snapshot/test_db_snapshot
[root@ccycloud hive-testbench]#

[root@ccycloud hive-testbench]# hdfs dfs -ls /user/hive/warehouse/test_db.db/.snapshot
Found 1 items
drwxrwxrwt   - admin hive          0 2022-01-18 03:28 /user/hive/warehouse/test_db.db/.snapshot/test_db_snapshot
[root@ccycloud hive-testbench]#

hadoop distcp -D ipc.client.fallback-to-simple-auth-allowed=true  hdfs://172.27.38.73:8020/user/hive/warehouse/test_db.db hdfs:// 172.27.116.128/user/hive/warehouse/test_db.db

[root@ccycloud 79-hive_on_tez-HIVESERVER2]# hdfs dfs -ls /user/hive/warehouse/test_db.db
WARNING: HADOOP_PREFIX has been replaced by HADOOP_HOME. Using value of HADOOP_PREFIX.
Found 4 items
drwxr-xr-x   - hive hive          0 2022-01-23 02:00 /user/hive/warehouse/test_db.db/supplier_avro
drwxr-xr-x   - hive hive          0 2022-01-23 02:00 /user/hive/warehouse/test_db.db/supplier_orc
drwxr-xr-x   - hive hive          0 2022-01-23 02:00 /user/hive/warehouse/test_db.db/supplier_parquet
drwxr-xr-x   - hive hive          0 2022-01-23 02:00 /user/hive/warehouse/test_db.db/supplier_text
[root@ccycloud 79-hive_on_tez-HIVESERVER2]# hdfs dfs -du -h /user/hive/warehouse/test_db.db
WARNING: HADOOP_PREFIX has been replaced by HADOOP_HOME. Using value of HADOOP_PREFIX.
2.7 M     2.7 M     /user/hive/warehouse/test_db.db/supplier_avro
1003.2 K  1003.2 K  /user/hive/warehouse/test_db.db/supplier_orc
2.9 M     2.9 M     /user/hive/warehouse/test_db.db/supplier_parquet
2.7 M     2.7 M     /user/hive/warehouse/test_db.db/supplier_text
[root@ccycloud 79-hive_on_tez-HIVESERVER2]#

https://github.com/dstreev/hms-mirror/releases/download/1.4.0.2-SNAPSHOT/hms-mirror-dist.tar.gz
tar zxvf hms-mirror-dist.tar.gz
hms-mirror-install/setup.sh

[root@ccycloud hms-test]# cat default-template.yaml
# Copyright 2021 Cloudera, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


transfer:
# Optional (default: 4)
concurrency: 10
# Optional (default: 'transfer_')
transferPrefix: "transfer_"
# This directory is appended to the 'clusters:...:hcfsNamespace' value to store the transfer package for hive export/import.
# Optional (default: '/apps/hive/warehouse/export_')
exportBaseDirPrefix: "/apps/hive/warehouse/export_"
clusters:
LEFT:
# Set for Hive 1/2 environments
legacyHive: true
# Is the 'Hadoop COMPATIBLE File System' used to prefix data locations for this cluster.
# It is mainly used as the transfer location for metadata (export)
# If the primary storage for this cluster is 'hdfs' than use 'hdfs://...'
# If the primary storage for this action is cloud storage, use the
#    cloud storage prefix. IE: s3a://my_bucket
hcfsNamespace: "hdfs://172.27.38.73:8020"
hiveServer2:
# URI is the Hive JDBC URL in the form of:
# jdbc:hive2://<server>:<port>
# See docs for restrictions
uri: "jdbc:hive2://172.27.38.73:10000"
connectionProperties:
user: "*****"
password: "*****"
# Standalone jar file used to connect via JDBC to the LEFT environment Hive Server 2
# NOTE: Hive 3 jars will NOT work against Hive 1.  The protocol isn't compatible.
jarFile: "/root/hms-test/aux_libs/hive-jdbc-1.1.0-cdh5.16.2-standalone.jar"
RIGHT:
legacyHive: false
# Is the 'Hadoop COMPATIBLE File System' used to prefix data locations for this cluster.
# It is mainly used to as a baseline for where "DATA" will be transfered in the
# STORAGE stage.  The data location in the source location will be move to this
# base location + the extended path where it existed in the source system.
# The intent is to keep the data in the same relative location for this new cluster
# as the old cluster.
# If the LEFT and RIGHT clusters are share the same cloud storage, then use the same
# hcfs base location as the LEFT cluster.
hcfsNamespace: "hdfs://172.27.116.128:8020"
hiveServer2:
# URI is the Hive JDBC URL in the form of:
# jdbc:hive2://<server>:<port>
# See docs for restrictions
uri: "jdbc:hive2://172.27.116.128:10000/default;principal=hive/ccycloud.xfwangcdp.root.hwx.site@CLOUDERA.COM"
connectionProperties:
user: "*****"
password: "*****"
# Standalone jar file used to connect via JDBC to the LEFT environment Hive Server 2
# NOTE: Hive 3 jars will NOT work against Hive 1.  The protocol isn't compatible.
#jarFile: "<environment-specific-jdbc-standalone-driver>"
partitionDiscovery:
# Addition HMS configuration needed for this "discover.partitions"="true"
auto: true
# When a table is created, run MSCK when there are partitions.
initMSCK: true

[root@ccycloud ~]# tree ~/.hms-mirror/
/root/.hms-mirror/
|-- aux_libs
|   `-- hive-jdbc-3.1.3000.7.1.7.0-551-standalone.jar
|-- cfg
|   `-- default.yaml
`-- retry


3 directories, 2 files
[root@ccycloud ~]#

[root@ccycloud ~]# tree hms-test/
hms-test/
|-- aux_libs
|   `-- hive-jdbc-1.1.0-cdh5.16.2-standalone.jar
`-- default-template.yaml


1 directory, 2 files
[root@ccycloud ~]#

hms-mirror -cfg /root/hms-test/default-template.yaml  -db test_db -o temp

:1,$s/ccycloud.cdh5.root.hwx.site/ccycloud.xfwangcdp.root.hwx.site/g

beeline -f test_db_RIGHT_execute.sql

hdfs dfs -ls /user/hive/warehouse/test_db.db
hdfs dfs -du -h /user/hive/warehouse/test_db.db