yum 安装cm
主机名
/etc/hosts
hostnamectl set-hostname foo-1.dataexa.com hostnamectl set-hostname foo-2.dataexa.com hostnamectl set-hostname foo-2.dataexa.com hostnamectl set-hostname cdh-1.dataexa.com hostnamectl set-hostname cdh-2.dataexa.com cat >> /etc/hosts <<EOF 192.168.2.150 cdh-1.dataexa.com cdh-1 192.168.2.117 cdh-2.dataexa.com cdh-2 EOF 192.168.1.142 foo-1.dataexa.com foo-1 192.168.1.143 foo-2.dataexa.com foo-2 swappiness /proc/sys/vm/swappiness 设置为 10 echo never > /sys/kernel/mm/transparent_hugepage/defrag
selinux
vi /etc/selinux/config sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/confi getenforce
防火墙
systemctl stop firewalld systemctl disable firewalld systemctl status firewalld
ssh免密
ssh-keygen -t rsa cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys chmod 600 ~/.ssh/authorized_keys 在一台机器上自己ssh 自己，成功后将.ssh目录拷贝到其他机器
repo本地源
cat > /etc/yum.repos.d/local.repo <<EOF [localrepo] name=localrepo baseurl=file:///cm-5.13.3/ #gpgkey=http://192.168.2.113/cm-5.13.3/RPM-GPG-KEY-cloudera enabled = 1 gpgcheck = 0 EOF
远程源
rm -rf /etc/yum.repos.d/* cat > /etc/yum.repos.d/cloudera-manager.repo <<EOF [clouderamanager] name=Cloudera Manager baseurl=http://192.168.2.113/cm-5.13.3/ gpgkey=http://192.168.2.113/cm-5.13.3/RPM-GPG-KEY-cloudera enabled = 1 gpgcheck = 0 EOF yum clean all && yum makecache
ntp
主节点
yum -y install ntp
grep -v "^#" /etc/ntp.conf | grep -v "^$"
cat > /etc/ntp.conf <<EOF
driftfile /var/lib/ntp/drift
restrict default nomodify notrap nopeer noquery
restrict 127.0.0.1
restrict ::1
restrict 192.168.2.0 mask 255.255.255.0 nomodify notrap
server 127.127.1.0
Fudge 127.127.1.0 stratum 1
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor
EOF
客户端
grep -v "^#" /etc/ntp.conf | grep -v "^$"
cat > /etc/ntp.conf <<EOF
driftfile /var/lib/ntp/drift
restrict default nomodify notrap nopeer noquery
####restrict 192.168.1.135 nomodify notrap noquery 这个不知道要不要
restrict 127.0.0.1
restrict ::1
restrict 192.168.2.0 mask 255.255.255.0 nomodify notrap
server 192.168.2.150 profer
Fudge 192.168.2.150 stratum 10
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor
EOF
开机自动启动
systemctl enable ntpd.service
systemctl start ntpd.service
手动同步
ntpdate -u cdh-1
ntpq -p
查看ntp运行状态
ntpstat
jdk（所有节点）
卸载自带的jdk-open rpm -qa | grep java rpm -qa | grep jdk - yum -y remove xxjdk #删除所有的jdk yum remove -y *jdk* yum remove -y *java* rpm -ivh jdk-8u202-linux-x64.rpm yum install java -y
数据库安装
卸载原有的mariadb
rpm -qa | grep mariadb #查看安装信息
rpm --nodeps -e `rpm -qa | grep mariadb`
安装数据库顺序 最好解压安装
tar -xf mysql-5.7.21-1.el7.x86_64.rpm-bundle.tar
首先安装（其他节点也需要安装，同时也安装 mysql-community-libs-compat-5.7.21-1.el7.x86_64.rpm ）
yum install net-tools -y #安装net-tools包
yum install perl -y #安装perl包
安装顺序
rpm -ivh mysql-community-common-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-server-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-compat-5.7.21-1.el7.x86_64.rpm(compat解决hue数据库连接不上) 比较重要(其他节点都要安装)
rpm -ivh mysql-community-devel-5.7.21-1.el7.x86_64.rpm #安装mysqlclient需要用到
yum install mysql-community-common -y
yum install mysql-community-libs -y
yum install mysql-community-client -y
yum install mysql-community-libs-compat -y
yum install mysql-community-server -y
yum install mysql-community-devel -y
数据库默认配置
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0
key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1
max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M
#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log
#In later versions of MySQL, if you enable the binary log and do not set
#a server_id, MySQL will not start. The server_id must be unique within
#the replicating group.
server_id=1
binlog_format = mixed
read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M
# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M
[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
sql_mode=STRICT_ALL_TABLES
启动mysql
systemctl start mysqld
systemctl enable mysqld
systemctl daemon-reload
查看root初始密码
cat /var/log/mysqld.log |grep password
#vi /etc/my.cnf
在[mysqld]的段中加上一句：skip-grant-tables
例如：
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
skip-grant-tables
USE mysql 修改密码;
ALTER USER 'root'@'localhost' IDENTIFIED BY 'xx.com';
FLUSH PRIVILEGES;
mysql -uroot -pxx.com
登录mysql数据库
mysql -uroot –p密码 # 密码为上一步看到的
首先，修改validate_password_policy参数的值
set global validate_password_policy=0;
set global validate_password_length=2;
set global validate_password_special_char_count=0;
set global validate_password_mixed_case_count=0;
set global validate_password_number_count=0;
set password = password('daxxx.com');
查看密码策略:
SHOW VARIABLES LIKE 'validate_password%';
授权用户远程连接
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'xxx.com' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON *.* TO 'cdh'@'%' IDENTIFIED BY 'xxx.com' WITH GRANT OPTION;
set global validate_password_policy=0;
set global validate_password_length=2;
set global validate_password_special_char_count=0;
set global validate_password_mixed_case_count=0;
set global validate_password_number_count=0;
grant all privileges on *.* to 'scm'@'localhost' identified by 'scm' with grant option;
grant all privileges on *.* to 'scm'@'127.0.0.1' identified by 'scm' with grant option;
grant all privileges on *.* to 'scm'@'%' identified by 'scm' with grant option;
grant all privileges on *.* to 'root'@'localhost' identified by 'xxxx.com' with grant option;
grant all privileges on *.* to 'root'@'127.0.0.1' identified by 'dxx.com' with grant option;
grant all privileges on *.* to 'root'@'%' identified by 'xxa.com' with grant option;
grant all privileges on *.* to 'cdh'@'localhost' identified by 'dxx.com' with grant option;
grant all privileges on *.* to 'cdh'@'127.0.0.1' identified by 'xx.com' with grant option;
grant all privileges on *.* to 'cdh'@'%' identified by 'xxxa.com' with grant option;
flush privileges;
创建需要的数据库：
create database metastore default character set utf8 DEFAULT COLLATE utf8_general_ci;
create database scm default character set utf8 DEFAULT COLLATE utf8_general_ci;
create database amon default character set utf8 DEFAULT COLLATE utf8_general_ci;
create database rmon default character set utf8 DEFAULT COLLATE utf8_general_ci;
create database hue default character set utf8 DEFAULT COLLATE utf8_general_ci;
create database sentry default character set utf8 DEFAULT COLLATE utf8_general_ci;
create database nav default character set utf8 DEFAULT COLLATE utf8_general_ci;
create database navms default character set utf8 DEFAULT COLLATE utf8_general_ci;
create database oozie default character set utf8 DEFAULT COLLATE utf8_general_ci;
create database hive default character set utf8 DEFAULT COLLATE utf8_general_ci;
CM安装
[root@foo-1 yum.repos.d]# cat cloudera-manager.repo [clouderamanager] name=Cloudera Manager baseurl=http://192.168.2.113/cm-5.13.3/ gpgkey=http://192.168.2.113/cm-5.13.3/RPM-GPG-KEY-cloudera enabled = 1 gpgcheck = 0
install the Cloudera Manager packages.
在管理节点安装cloudera-manager-daemons cloudera-manager-server
#RHEL, if you have a yum repo configured
sudo yum install cloudera-manager-daemons cloudera-manager-server
sudo yum install cloudera-manager-agent
#On all hosts, run the following command to install the Cloudera Manager agent:
在子节点，安装管理客户端
sudo yum install cloudera-manager-daemons cloudera-manager-agent
*** 在所有节点修改管理代理连接管理服务器的主机名
On all hosts, configure the Cloudera Manager Agent to point to the Cloudera Manager Server by setting the following properties in the /etc/cloudera-scm-agent/config.ini configuration file:
server_host Name of the host where Cloudera Manager Server is running.
server_port Port on the host where Cloudera Manager Server is running.
sed -i 's/server_host=localhost/server_host=cdh-1/' /etc/cloudera-scm-agent/config.ini
数据库连接程序
初始化 yum安装的scm_prepare_database.sh 位置
mkdir /usr/share/java/ && cd /usr/share/java/
curl -O http://192.168.2.113/mysql-connector-java/mysql-connector-java-5.1.46.tar.gz
mkdir /usr/share/java/
cp mysql-connector-java-5.1.46-bin.jar mysql-connector-java.jar
chmod 755 mysql-connector-java-5.1.46-bin.jar mysql-connector-java.jar
cp mysql-connector-java-5.1.46-bin.jar mysql-connector-java.jar /usr/share/cmf/lib/
mkdir /usr/share/java/
初始化数据库：
/usr/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -pxxxx.com --scm-host cdh-1 scm scm scm
所有节点运行管理代理
管理节点
systemctl start cloudera-scm-server
systemctl start cloudera-scm-agent
判断是否启动成功
find / -name cloudera-scm-server.log
tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
grep "Started Jetty server" cloudera-scm-server.log
子节点
systemctl start cloudera-scm-agent
安装后节点也需要安装
cd /usr/share/cmf/lib/
mv /usr/share/java/mysql-connector-java-5.1.46-bin.jar .
cdh 也需要驱动
scp mysql-connector-java-5.1.46-bin.jar foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib
hive 数据库也需要
scp mysql-connector-java-5.1.46-bin.jar foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
scp mysql-connector-java-5.1.46-bin.jar foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib
scp mysql-connector-java-5.1.46-bin.jar foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
scp mysql-connector-java-5.1.46-bin.jar foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/oozie/lib
scp mysql-connector-java-5.1.46-bin.jar foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hue/lib
scp mysql-connector-java-5.1.46-bin.jar foo-2:/var/lib/oozie
scp mysql-connector-java-5.1.46-bin.jar foo-2:/var/lib/hue
http://192.168.2.150:7180/cmf/clusters/1/express-add-services/index#step=commandDetailsStep