chunjun同步两个带Kerberos认证的CDH集群数据
2024-09-18 16:26:19
最近在项目中遇到同步两个 CDH 数据文件的需求，调研了一圈，发现 datax 不支持两个 cdh 同时进行 kerberos 认证，因此切换到 chunjun，chunjun 在 kerberos 认证的配置这块，官网也没有详细的介绍案例，在查看源代码后，确定了配置项及各配置项的值，供大家参考
1 {
2 "job": {
3 "content": [
4 {
5 "reader" : {
6 "parameter" : {
7 "path" : "hdfs://cdh-test-namenode05:8020/user/hive/warehouse/testuser_db.db/testuser_user/t4.txt",
8 "hadoopConfig" : {
9 "properties.hadoop.user.name": "testuser",
10 "properties.dfs.ha.namenodes.nameservice1": "namenode27,namenode29",
11 "properties.fs.defaultFS": "hdfs://nameservice1",
12 "properties.dfs.namenode.rpc-address.nameservice1.namenode27": "cdh-test-namenode04:8020",
13 "properties.dfs.client.failover.proxy.provider.nameservice1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
14 "properties.dfs.namenode.rpc-address.nameservice1.namenode29": "cdh-test-namenode05:8020",
15 "properties.dfs.nameservices": "nameservice1",
16 "properties.fs.hdfs.impl.disable.cache": "true",
17 "properties.dfs.client.use.datanode.hostname":"true",
18 "dfs.client.use.datanode.hostname":"true",
19 "properties.fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
20 "hadoop.security.authentication": "Kerberos",
21 "dfs.namenode.kerberos.principal": "hdfs/_HOST@CDHDEV.COM",
22 "dfs.datanode.kerberos.principal": "hdfs/_HOST@CDHDEV.COM",
23 "yarn.resourcemanager.principal": "rm/_HOST@CDHDEV.COM",
24 "dfs.namenode.kerberos.internal.spnego.principal": "HTTP/_HOST@CDHDEV.COM",
25 "hadoop.security.authorization": "true",
26 "dfs.namenode.keytab.file": "/app/data/CDH6.3.2/kerberos/testuser.keytab",
27 "java.security.krb5.conf": "/app/data/CDH6.3.2/krb5/krb5.conf",
28 "useLocalFile": "true",
29 "principalFile": "/app/data/CDH6.3.2/kerberos/testuser.keytab",
30 "principal": "testuser/cdh-test-namenode05"
31 },
32
33 "column": [
34 {
35 "name": "id",
36 "type": "int"
37 },
38 {
39 "name": "name",
40 "type": "string"
41 }
42 ],
43 "fieldDelimiter" : ",",
44 "encoding" : "utf-8",
45 "defaultFS": "hdfs://cdh-test-namenode05:8020",
46 "fileType" : "text"
47 },
48 "name" : "hdfsreader"
49 },
50 "writer": {
51 "name": "hdfswriter",
52 "parameter": {
53 "path": "/user/hive/warehouse/test_system_user.db/testuser",
54 "defaultFS": "hdfs://test01:8020",
55 "column": [
56 {
57 "name": "id",
58 "type": "int"
59 },
60 {
61 "name": "name",
62 "type": "string"
63 }
64 ],
65 "fileType": "text",
66 "fieldDelimiter": ",",
67 "compress":"gzip",
68 "encoding": "utf-8",
69 "fileName": "20220705",
70 "writeMode": "overwrite",
71 "hadoopConfig": {
72 "properties.hadoop.user.name": "test_system_user",
73 "properties.dfs.ha.namenodes.nameservice1": "namenode37,namenode51",
74 "properties.fs.defaultFS": "hdfs://nameservice1",
75 "properties.dfs.namenode.rpc-address.nameservice1.namenode51": "test02:8020",
76 "properties.dfs.client.failover.proxy.provider.nameservice1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
77 "properties.dfs.namenode.rpc-address.nameservice1.namenode37": "test01:8020",
78 "properties.dfs.nameservices": "nameservice1",
79 "properties.dfs.client.use.datanode.hostname":"true",
80 "dfs.client.use.datanode.hostname":"true",
81 "properties.fs.hdfs.impl.disable.cache": "true",
82 "properties.fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
83 "hadoop.security.authentication": "Kerberos",
84 "dfs.namenode.kerberos.principal": "hdfs/_HOST@SINOSIG.COM",
85 "dfs.datanode.kerberos.principal": "hdfs/_HOST@SINOSIG.COM",
86 "yarn.resourcemanager.principal": "rm/_HOST@SINOSIG.COM",
87 "dfs.namenode.kerberos.internal.spnego.principal": "HTTP/_HOST@SINOSIG.COM",
88 "hadoop.security.authorization": "true",
89 "dfs.namenode.keytab.file": "/app/data/cloudera/CDH-6.3.2/kerberos/test_system_user.keytab",
90 "java.security.krb5.conf": "/app/data/cloudera/CDH-6.3.2/krb5/krb5.conf",
91 "useLocalFile": "true",
92 "principalFile": "/app/data/cloudera/CDH-6.3.2/kerberos/test_system_user.keytab",
93 "principal": "test_system_user/test02@SINOSIG.COM"
94 }
95 }
96 }
97 }
98 ],
99 "setting": {
100 "speed": {
101 "channel": 4,
102 "bytes": 0
103 }
104 }
105 }
106 }
最新文章
- 赵文豪 GDB调试汇编堆栈过程分析
- No module named migrate.versioning
- Mac 连接阿里云服务器
- WPF学习之路（二） XAML
- POJ2635The Embarrassed Cryptographer（大数取余+素数筛选+好题）
- 爬虫神器xpath的用法（二）
- es6新特性学习
- Spring Controller 获取请求参数的几种方法
- WORD中如何让前两页不显示页码
- lintcode ：Ugly Numbers 丑数
- Linux 命令 - cp: 拷贝文件和目录
- RabbitMQ RPC问题
- hadoop面试时可能遇到的问题
- 使用PDO持久化连接
- js立即执行函数: (function ( ){...})( ) 与 (function ( ){...}( ))
- Go终端读写
- luogu P4389 付公主的背包
- Django MTV 开发模式 + 数据库配置
- 锁、C#中Monitor和Lock以及区别
- 08 IO库