4 cdh 5.12 centos 6.10三节点安装
2024-10-20 01:34:29
4 cdh 5.12 centos 6.10 三节点安装
[root@hadoop1 opt]# cat /etc/redhat-release
CentOS release 6.10 (Final)
# service iptables stop
# chkconfig iptables off
# sed -i 's/=enforcing/=disabled/' /etc/selinux/config
# cat >> /etc/profile << EOF
export HISTTIMEFORMAT='%F %T '
EOF
# sed -i '$a vm.swappiness = 0' /etc/sysctl.conf
# sed -i '$a echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag' /etc/rc.local
# sed -i '$a echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled' /etc/rc.local
# vim /etc/ntp.conf ##注释掉所有的 server 开头内容
# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
server cn.pool.ntp.org
# service ntpd start
# chkconfig ntpd on # rpm -qa | grep java-1
java-1.5.0-gcj-1.5.0.0-29.1.el6.x86_64
java-1.6.0-openjdk-1.6.0.41-1.13.13.1.el6_8.x86_64
java-1.7.0-openjdk-1.7.0.181-2.6.14.10.el6.x86_64
# rpm -e --nodeps ` rpm -qa | grep java-1`
# tar -zxvf jdk-8u191-linux-x64.tar.gz
# ln -s jdk1.8.0_191 jdk
# alternatives --install /usr/bin/java java /opt/jdk/bin/java 100
# alternatives --install /usr/bin/javac javac /opt/jdk/bin/javac 100
# cat >> /etc/profile << EOF
export JAVA_HOME=/opt/jdk
export PATH=$PATH:$JAVA_HOME/bin
EOF
[root@hadoop1 opt]# source /etc/profile
[root@hadoop1 opt]# java -version
java version "1.8.0_191"
Java(TM) SE Runtime Environment (build 1.8.0_191-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.191-b12, mixed mode)
[root@hadoop1 opt]# javac -version
javac 1.8.0_191
# cat >> /etc/hosts << EOF
* hadoop1
* hadoop2
* hadoop3
EOF
[root@hadoop1 opt]# id hadoop
id: hadoop: No such user
[root@hadoop1 opt]# useradd hadoop
[root@hadoop1 opt]# vim /etc/sudoers
hadoop ALL=NOPASSWD:ALL chown -R hadoop:hadoop /opt
[root@hadoop1 opt]# ./test_hadoop_env.sh
######### 1. iptables: #####################
current iptables status: iptables: Firewall is not running.
chkconfig status： iptables 0:off 1:off 2:off 3:off 4:off 5:off 6:off
######### 2. Selinux: ######################
current selinux status: Disabled
config of selinux: SELINUX=disabled
######### 3. THP: ##########################
defrag status:[always] madvise never
enabled status:[always] madvise never
######### 4. Swappiness => 0 : ###################
current swapness setting: 0
######### 5.ntp: ###################
remote refid st t when poll reach delay offset jitter
==============================================================================
*ntp1.flashdance * 2 u 977 1024 375 296.330 15.505 30.880
######### 6. JDK: ###################
current java version:
javac 1.8.0_191
######### 7. hosts: ###################
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
* hadoop1
* hadoop2
* hadoop3
[root@hadoop1 opt]# chmox +x test_hadoop_env.sh
[root@hadoop1 opt]# pssh -h cluster -i "/opt/test_hadoop_env.sh"
[root@hadoop1 opt]# prsync -h cluster /etc/yum.repos.d/cm512.repo /etc/yum.repos.d/. [root@hadoop1 opt]# ./mysql_auto_glibc_5724.sh
[mysql@hadoop1 scripts]$ vim cdh521.sql
[mysql@hadoop1 scripts]$ mysql -uroot -p -S /data/mysqldata/3306/mysql.sock
(root@localhost:mysql.sock) [(none)]> source /data/mysqldata/scripts/cdh521.sql
http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.12.1/RPMS/x86_64/
# cd /var/www/html/
# mkdir cm512
# cd cm512/
[root@hadoop1 cm512]# ll
-rw-r--r-- 1 root root 9676820 Feb 21 2018 cloudera-manager-agent-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm
-rw-r--r-- 1 root root 709818804 May 24 11:49 cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm
-rw-r--r-- 1 root root 8692 May 24 11:18 cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm
-rw-r--r-- 1 root root 10600 May 24 11:18 cloudera-manager-server-db-2-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm
-rw-r--r-- 1 root root 31918952 May 24 11:19 enterprise-debuginfo-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm
-rw-r--r-- 1 root root 71204325 May 24 11:20 jdk-6u31-linux-amd64.rpm
-rw-r--r-- 1 root root 142039186 May 24 11:22 oracle-j2sdk1.7-1.7.0+update67-1.x86_64.rpm
# createrepo .
# w3m http://hadoop1/cm512
[root@hadoop1 yum.repos.d]# vim cm512.repo
# yum clean all
# yum repolist
http://archive.cloudera.com/cdh5/parcels/5.12.1/
CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel
CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha1
manifest.json
# mkdir -p /var/www/html/cdh512
[root@hadoop1 cdh512]# ll
-rw-r--r-- 1 root root 1620405492 May 24 11:22 CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel
-rw-r--r-- 1 root root 41 May 24 10:47 CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha1
-rw-r--r-- 1 root root 72312 Feb 23 2018 manifest.json
--通过yum安装ClouderaManager Server
[root@hadoop1 ~]# yum -y install cloudera-manager-server
(1/2): cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm | 677 MB 00:13
(2/2): cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm | 8.5 kB 00:00
---------------------------------------------------------------------------------------------------------------------------------------------------------------
Total 49 MB/s | 677 MB 00:13
warning: rpmts_HdrFromFdno: Header V4 DSA/SHA1 Signature, key ID e8f86acd: NOKEY Public key for cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm is not installed # yum -y install cloudera-manager-server --nogpgcheck
Installing : cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64 1/2
Installing : cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64 2/2
Verifying : cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64 1/2
Verifying : cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64 2/2 Installed:
cloudera-manager-server.x86_64 0:5.12.1-1.cm5121.p0.6.el6 Dependency Installed:
cloudera-manager-daemons.x86_64 0:5.12.1-1.cm5121.p0.6.el6
--初始化CM数据库
[root@hadoop1 ~]# /usr/share/cmf/schema/scm_prepare_database.sh mysql cm cm password
2019-05-24 14:50:31,604 [main] INFO com.cloudera.enterprise.dbutil.DbCommandExecutor - Unable to find JDBC driver for database type: MySQL
2019-05-24 14:50:31,606 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor - JDBC Driver com.mysql.jdbc.Driver not found.
2019-05-24 14:50:31,606 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor - Exiting with exit code 3
--下载mysql驱动
https://dev.mysql.com/downloads/connector/j/
[root@hadoop1 opt]# cp mysql-connector-java-5.1.47-bin.jar /usr/share/cmf/lib/. [root@hadoop1 opt]# /usr/share/cmf/schema/scm_prepare_database.sh mysql -h * --scm-host * cm cm password
JAVA_HOME=/opt/jdk
Verifying that we can write to /etc/cloudera-scm-server
Creating SCM configuration file in /etc/cloudera-scm-server
Executing: /opt/jdk/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/cmf/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db.
Fri May 24 15:11:10 CST 2019 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
2019-05-24 15:11:10,656 [main] INFO com.cloudera.enterprise.dbutil.DbCommandExecutor - Successfully connected to database.
2019-05-24 15:11:10,687 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor - Unable to create/drop a table.
java.sql.SQLException: Statement violates GTID consistency: CREATE TABLE ... SELECT.
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:965)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3978)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3914)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2530)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2683)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2491)
at com.mysql.jdbc.StatementImpl.executeUpdateInternal(StatementImpl.java:1552)
at com.mysql.jdbc.StatementImpl.executeLargeUpdate(StatementImpl.java:2607)
at com.mysql.jdbc.StatementImpl.executeUpdate(StatementImpl.java:1480)
at com.cloudera.enterprise.dbutil.DbCommandExecutor.runDdlStatementPair(DbCommandExecutor.java:357)
at com.cloudera.enterprise.dbutil.DbCommandExecutor.testIfUserCanCreateTable(DbCommandExecutor.java:333)
at com.cloudera.enterprise.dbutil.DbCommandExecutor.testIfUserCanRunDdl(DbCommandExecutor.java:299)
at com.cloudera.enterprise.dbutil.DbCommandExecutor.testDbConnection(DbCommandExecutor.java:258)
at com.cloudera.enterprise.dbutil.DbCommandExecutor.main(DbCommandExecutor.java:138)
2019-05-24 15:11:10,696 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor - Exiting with exit code 9
--> Error 9, giving up (use --force if you wish to ignore the error)
--注释vim /data/mysqldata/3306/my.cnf文件中的GTID，要关闭gtid模式
[root@hadoop1 opt]# /usr/share/cmf/schema/scm_prepare_database.sh mysql -h * --scm-host * cm cm password
JAVA_HOME=/opt/jdk
Verifying that we can write to /etc/cloudera-scm-server
Creating SCM configuration file in /etc/cloudera-scm-server
Executing: /opt/jdk/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/cmf/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db.
Fri May 24 15:46:09 CST 2019 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
2019-05-24 15:46:09,582 [main] INFO com.cloudera.enterprise.dbutil.DbCommandExecutor - Successfully connected to database.
All done, your SCM database is configured correctly! --启动ClouderaManager Server
[root@hadoop1 opt]# service cloudera-scm-server start
Starting cloudera-scm-server: [FAILED] [root@hadoop1 cloudera-scm-server]# cat /var/log/cloudera-scm-server/cloudera-scm-server.out
+======================================================================+
| Error: JAVA_HOME is not set and Java could not be found
[root@hadoop1 cloudera-scm-server]# echo $JAVA_HOME
/opt/jdk
[root@hadoop1 cloudera-scm-server]# find / -name 'cloudera-scm-server'
[root@hadoop1 cloudera-scm-server]# vim /etc/rc.d/init.d/cloudera-scm-server export JAVA_HOME=/opt/jdk
[root@hadoop1 cloudera-scm-server]# service cloudera-scm-server start
Starting cloudera-scm-server: [ OK ]
[root@hadoop1 cloudera-scm-server]# netstat -lnt|grep 7180
tcp 0 0 0.0.0.0:7180 0.0.0.0:* LISTEN
[root@hadoop1 cdh512]# mv CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha1 CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha --访问控制台
http://*:7180/cmf/login
最新文章
- Android 中关于Fragment嵌套Fragment的问题
- 2.Java异常学习
- BZOJ2597 [Wc2007]剪刀石头布（最小费用最大流）
- spring bean实例化方式
- linux命名管道通信过程
- STL笔记之【map之移除元素】
- Django: 之数据库导入、迁移和联用
- angular中路由的实现（针对新手）
- Python3 标准库概览
- 学习pthreads，管理线程的栈
- Atlas实现MySQL大表部署读写分离
- Android-Java-普通类与抽象类(覆盖)&方法重载
- Fiddler修改请求、返回数据
- Flask 系列之 Pagination
- javascript之奇淫技巧
- 安装Harbor
- JAVA多线程17个问题
- Eclipse Java注释模板设置
- TCxGrid 把列移上移下。
- linux中的查找命令