大数据平台Cloudera Manger+CDH集群安装(安装失败)

mtain 2023年04月14日 652次浏览

说明

大数据集群自动化部署两种方式:Ambari + hdp 和 Cloudera Manger + CDH

Cloudera Data Platform(CDP) ,CDP 包括各种组件,例如 Apache HDFS、Apache Hive 3、Apache HBase 和 Apache Impala,以及许多其他用于特殊工作负载的组件。

软件下载

集群主机

IP主机名角色
192.168.1.171cdp01manager
192.168.1.139cdp02master
192.168.1.65cdp03worker

安装

1. 初始化主机

systemctl stop firewalld.service && systemctl disable firewalld.service

sudo sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

cat >> /etc/hosts <<EOF
192.168.1.171    cdp01
192.168.1.139    cdp02
192.168.1.65     cdp03
EOF

yum install -y ntp ntpdate
systemctl start ntpd
systemctl enable ntpd


# 设置swap
cat >> /etc/sysctl.conf <<EOF
vm.swappiness=1
EOF

sysctl vm.swappiness=1


# 关闭透明大页
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.d/rc.local
echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.d/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag

vi /etc/rc.d/rc.local

if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
        echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
        echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi


chmod +x /etc/rc.d/rc.local


# 设置 limits
cat >> /etc/security/limits.conf <<EOF
hdfs - nofile 32768
mapred - nofile 32768
hbase - nofile 32768
hdfs - noproc 32768
mapred - noproc 32768
hbase - noproc 32768
EOF

echo "* soft nofile 65535" >> /etc/security/limits.conf
echo "* hard nofile 65535" >> /etc/security/limits.conf

设置ssh免密登录
ssh-copy-id root@cdp01
ssh-copy-id root@cdp02
ssh-copy-id root@cdp03

2. 安装基础软件

三台机器安装

安装jdk
使用rpm包安装
rpm -ivh jdk-8u191-linux-x64.rpm


自己安装的jdk要设置软连接
mkdir -p /usr/java
ln -s /opt/jdk1.8.0_161  /usr/java/default

vi /etc/profile
export JAVA_HOME=/opt/jdk1.8.0_161
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/jre/lib/rt.jar
export PATH=$PATH:$JAVA_HOME/bin

source /etc/profile

节点1安装MySQL

yum localinstall -y  mysql-*.rpm


初始化mysql
更改mysql数据库所属于用户及其所属于组
mysqld --initialize
chown mysql:mysql /var/lib/mysql -R


启动mysql数据库
[root@test mysql]# systemctl start mysqld.service
[root@test ~]# grep 'password' /var/log/mysqld.log 
2023-04-14T06:57:30.646293Z 1 [Note] A temporary password is generated for root@localhost: kdfYnW87&:lm


修改数据库密码
[root@hadoop log]# mysql -u root -p

mysql中执行
set password=password('EuuWwnZK3frG');
flush privileges;
grant all privileges on *.* to root@'%' identified by 'EuuWwnZK3frG' with grant option;
flush privileges;


设置字符编码和表名不区分大小写
vi /etc/my.cnf
增加
[mysqld]
character-set-server=utf8
lower_case_table_names=1

systemctl restart mysqld.service
systemctl enable mysqld.service

所有机器上传mysql连接驱动

mkdir -p /usr/share/java
上传mysql-connector-java-5.1.46.jar
mv mysql-connector-java-5.1.46.jar mysql-connector-java.jar

3. 配置本地yum源

节点一

yum install httpd -y
systemctl start httpd.service
systemctl enable httpd.service

yum install -y createrepo

mkdir -p /var/www/html/cm7.1.4
cd /var/www/html/cm7.1.4
上传安装包
createrepo .

mkdir -p /var/www/html/cdh7.1.7
cd /var/www/html/cdh7.1.7
上传安装包
createrepo .

三台机器都设置软件源

cat >> /etc/yum.repos.d/cm7.repo <<EOF
[cm]
name=cm,Version 7.1.4
baseurl=http://192.168.1.171/cm7.1.4/
enable=1
gpgcheck=0
[cdh]
name=cdh
baseurl=http://192.168.1.171/cdh7.1.7/
enable=1
gpgcheck=0
EOF

yum clean all
yum makecache
yum list |grep cloudera

4. 初始化数据库

mysql -u root -p

set global validate_password_policy=LOW;
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'EuuWwnZK3frG';   
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'EuuWwnZK3frG'; 
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'EuuWwnZK3frG';   
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'EuuWwnZK3frG'; 
CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hive.* TO 'hive'@'%' IDENTIFIED BY 'EuuWwnZK3frG'; 
CREATE DATABASE ranger DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON ranger.* TO 'rangeradmin'@'%' IDENTIFIED BY 'EuuWwnZK3frG';  
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'EuuWwnZK3frG'; 
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'EuuWwnZK3frG';
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'EuuWwnZK3frG';  
flush privileges;

5. 节点一安装Cloudera Manager Server

yum -y install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server

初始化mysql
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm EuuWwnZK3frG

报错:javax.net.ssl.SSLHandshakeException: No appropriate protocol (protocol is disabled or cipher suites are inappropriate)
vi /usr/java/jdk1.8.0_361-amd64/jre/lib/security/java.security
删除 jdk.tls.disabledAlgorithms 中的 SSLv3, TLSv1, TLSv1.1,

# JDK的软连接
mkdir -p /usr/java
ln -s /opt/jdk1.8.0_161  /usr/java/default

# 启动服务
systemctl start cloudera-scm-server.service
systemctl status cloudera-scm-server.service
systemctl enable cloudera-scm-server.service

# 检查服务
sudo tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
netstat -nltp | grep 7180


# 访问
http://192.168.1.171:7180/cmf/login
admin/admin

以下是图形界面安装步骤:
image.png

image.png

image.png

image.png

image.png

image.png

image.png

image.png

image.png

image.png

设置对应的数据库
image.png

最终报错,命令执行失败,web网页无法访问