说明
大数据集群自动化部署两种方式：Ambari + hdp 和 Cloudera Manger + CDH
Cloudera Data Platform(CDP) ，CDP 包括各种组件，例如 Apache HDFS、Apache Hive 3、Apache HBase 和 Apache Impala，以及许多其他用于特殊工作负载的组件。
软件下载
集群主机
| IP | 主机名 | 角色 |
|---|---|---|
| 192.168.1.171 | cdp01 | manager |
| 192.168.1.139 | cdp02 | master |
| 192.168.1.65 | cdp03 | worker |
安装
1. 初始化主机
systemctl stop firewalld.service && systemctl disable firewalld.service
sudo sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
cat >> /etc/hosts <<EOF
192.168.1.171 cdp01
192.168.1.139 cdp02
192.168.1.65 cdp03
EOF
yum install -y ntp ntpdate
systemctl start ntpd
systemctl enable ntpd
# 设置swap
cat >> /etc/sysctl.conf <<EOF
vm.swappiness=1
EOF
sysctl vm.swappiness=1
# 关闭透明大页
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.d/rc.local
echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.d/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
vi /etc/rc.d/rc.local
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi
chmod +x /etc/rc.d/rc.local
# 设置 limits
cat >> /etc/security/limits.conf <<EOF
hdfs - nofile 32768
mapred - nofile 32768
hbase - nofile 32768
hdfs - noproc 32768
mapred - noproc 32768
hbase - noproc 32768
EOF
echo "* soft nofile 65535" >> /etc/security/limits.conf
echo "* hard nofile 65535" >> /etc/security/limits.conf
设置ssh免密登录
ssh-copy-id root@cdp01
ssh-copy-id root@cdp02
ssh-copy-id root@cdp03
2. 安装基础软件
三台机器安装
安装jdk
使用rpm包安装
rpm -ivh jdk-8u191-linux-x64.rpm
自己安装的jdk要设置软连接
mkdir -p /usr/java
ln -s /opt/jdk1.8.0_161 /usr/java/default
vi /etc/profile
export JAVA_HOME=/opt/jdk1.8.0_161
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/jre/lib/rt.jar
export PATH=$PATH:$JAVA_HOME/bin
source /etc/profile
节点1安装MySQL
yum localinstall -y mysql-*.rpm
初始化mysql
更改mysql数据库所属于用户及其所属于组
mysqld --initialize
chown mysql:mysql /var/lib/mysql -R
启动mysql数据库
[root@test mysql]# systemctl start mysqld.service
[root@test ~]# grep 'password' /var/log/mysqld.log
2023-04-14T06:57:30.646293Z 1 [Note] A temporary password is generated for root@localhost: kdfYnW87&:lm
修改数据库密码
[root@hadoop log]# mysql -u root -p
mysql中执行
set password=password('EuuWwnZK3frG');
flush privileges;
grant all privileges on *.* to root@'%' identified by 'EuuWwnZK3frG' with grant option;
flush privileges;
设置字符编码和表名不区分大小写
vi /etc/my.cnf
增加
[mysqld]
character-set-server=utf8
lower_case_table_names=1
systemctl restart mysqld.service
systemctl enable mysqld.service
所有机器上传mysql连接驱动
mkdir -p /usr/share/java
上传mysql-connector-java-5.1.46.jar
mv mysql-connector-java-5.1.46.jar mysql-connector-java.jar
3. 配置本地yum源
节点一
yum install httpd -y
systemctl start httpd.service
systemctl enable httpd.service
yum install -y createrepo
mkdir -p /var/www/html/cm7.1.4
cd /var/www/html/cm7.1.4
上传安装包
createrepo .
mkdir -p /var/www/html/cdh7.1.7
cd /var/www/html/cdh7.1.7
上传安装包
createrepo .
三台机器都设置软件源
cat >> /etc/yum.repos.d/cm7.repo <<EOF
[cm]
name=cm,Version 7.1.4
baseurl=http://192.168.1.171/cm7.1.4/
enable=1
gpgcheck=0
[cdh]
name=cdh
baseurl=http://192.168.1.171/cdh7.1.7/
enable=1
gpgcheck=0
EOF
yum clean all
yum makecache
yum list |grep cloudera
4. 初始化数据库
mysql -u root -p
set global validate_password_policy=LOW;
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'EuuWwnZK3frG';
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'EuuWwnZK3frG';
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'EuuWwnZK3frG';
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'EuuWwnZK3frG';
CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hive.* TO 'hive'@'%' IDENTIFIED BY 'EuuWwnZK3frG';
CREATE DATABASE ranger DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON ranger.* TO 'rangeradmin'@'%' IDENTIFIED BY 'EuuWwnZK3frG';
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'EuuWwnZK3frG';
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'EuuWwnZK3frG';
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'EuuWwnZK3frG';
flush privileges;
5. 节点一安装Cloudera Manager Server
yum -y install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
初始化mysql
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm EuuWwnZK3frG
报错：javax.net.ssl.SSLHandshakeException: No appropriate protocol (protocol is disabled or cipher suites are inappropriate)
vi /usr/java/jdk1.8.0_361-amd64/jre/lib/security/java.security
删除 jdk.tls.disabledAlgorithms 中的 SSLv3, TLSv1, TLSv1.1,
# JDK的软连接
mkdir -p /usr/java
ln -s /opt/jdk1.8.0_161 /usr/java/default
# 启动服务
systemctl start cloudera-scm-server.service
systemctl status cloudera-scm-server.service
systemctl enable cloudera-scm-server.service
# 检查服务
sudo tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
netstat -nltp | grep 7180
# 访问
http://192.168.1.171:7180/cmf/login
admin/admin
以下是图形界面安装步骤:
设置对应的数据库
最终报错，命令执行失败，web网页无法访问