CDH6.2安装

Wesley13
• 阅读 717

一、准备工作:
1、普通用户root权限准备(sudo,此处用aiot用户,sudoer)
groupadd -r test
useradd -d /home/test/ -m -s /bin/bash -c "user" -g test -p 2019 -r test
#另外特别注意:aiot及密码用户在cloudera manager server中会配置进去,作为集群管理【host添加、agent安装&卸载、parcels管理os user】,
所以不能轻易在os修改aiot用户密码,如果需要修改,运维同学需要通知cdh集群管理员同步修改cms保存的密码)
passwd test
root: vim /etc/sudoers
test ALL=(ALL) NOPASSWD: ALL
2.必备工具
sudo yum install yum-utils createrepo -y
sudo yum install ansible -y
sudo yum install httpd -y
sudo yum install systemctl -y
sudo yum install ntp -y
sudo yum install hostnamectl -y
~~~~~~~~~~~~~~~~~~~

3.集群 /etc/hosts设置
sudo vim /etc/hosts
cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.0.3 cdh3
192.168.0.4   cdh4
192.168.0.5  cdh5

#修改主机名
sudo vim /etc/sysconfig/network:
NETWORKING=yes
HOSTNAME=cdh3

reboot

4.关闭防火墙
getenforce
sudo systemctl stop firewalld
sudo systemctl disable firewalld.service
sudo vim /etc/selinux/config 

-------------------------------

SELINUX=disabled

-------------------------------

5 ssh 互信 (所有节点)test用户
$ mkdir .ssh
$ chmod 750 .ssh
$ ssh-keygen -t rsa
##192.168.0.3操作
$cat /home/aiot/.ssh/id_rsa.pub >> /home/aiot/.ssh/authorized_keys
$ chmod 600 ~/.ssh/authorized_keys

所有节点测试互信测试

6.配置NTP服务(所有节点)两种方法二选一

sudo yum install -y ntp
方法一时间同步主节点

----------------------------------------

主节点配置(192.168.0.3 )
sudo vim /etc/ntp.conf
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
server ntp.aliyun.com
从节点配置(192.168.0.4 192.168.0.5)
sudo vim /etc/ntp.conf
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
server 192.168.0.3  cdh3

----------------------------------------------

sudo systemctl enable ntpd
sudo systemctl start ntpd

6.所有节点禁用大透明页 和vm.swappiness设置
sysctl vm.swappiness=10
vim /etc/sysctl.conf

vm.swappiness=10

echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
上面两条加入/etc/rc.local开机自启
vim /etc/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
chmod +x /etc/rc.d/rc.local

7.JDK安装(所有节点)
sudo rpm -qa | grep java # 查询已安装的java
sudo yum remove java* # 卸载
sudo mkdir -p /usr/java
sudo tar -zxf /tmp/installer/jdk-8u181-linux-x64.tar.gz -C /usr/java/
sudo ln -s /usr/java/jdk1.8.0_181 /usr/java/jdk
sudo vim /etc/profile

export JAVA_HOME=/usr/java/jdk
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib:$CLASSPATH

sudo source /etc/profile
#如果不是root用户 需要添加
~/.bashrc
~/.bash_profile

检查环境变量 echo $JAVA_HOME

所有节点以下操作
sudo mkdir -p /usr/share/java
sudo cp /data/mysql-connector-java-5.1.46.jar /usr/share/java/mysql-connector-java.jar

二#mysql5.7.20安装
sudo rpm -qa | grep mariadb
sudo rpm -e mariadb-libs-5.5.60-1.el7_5.x86_64 --nodeps
sudo yum -y install libaio
#官方下载
#sudo wget https://dev.mysql.com/get/Downloads/MySQL-5.7/mysql-5.7.20-1.el7.x86_64.rpm-bundle.tar
#解压
#sudo tar -xvf mysql-5.7.20-1.el7.x86_64.rpm-bundle.tar
#本地rpm安装
sudo rpm -ivh mysql-community-common-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-libs-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-libs-compat-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-client-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-server-5.7.20-1.el7.x86_64.rpm
#如果指定了mysql用户需要授权给mysql
没有/data/mysqldata/mysql目录
需要
sudo mkdir -p /data/mysqldata/mysql
sudo vim /etc/my.cnf:
--------------------------------
[mysqld]
datadir=/data/mysqldata/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation=READ-COMMITTED
symbolic-links=0
key_buffer_size=32M
max_allowed_packet=32M
thread_stack=256K
thread_cache_size=64
query_cache_limit=8M
query_cache_size=64M
query_cache_type=1
max_connections=550
log_bin=/data/mysqldata/mysql/mysql_binary_log
server_id=1
binlog_format=mixed
read_buffer_size=2M
read_rnd_buffer_size=16M
sort_buffer_size=8M
join_buffer_size=8M
innodb_file_per_table=1
innodb_flush_log_at_trx_commit=2
innodb_log_buffer_size=64M
innodb_buffer_pool_size=4G
innodb_thread_concurrency=8
innodb_flush_method=O_DIRECT
innodb_log_file_size=512M
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
sql_mode=STRICT_ALL_TABLES
-----------------------------
sudo systemctl enable mysqld
#sudo systemctl stop mysqld
sudo systemctl start mysqld
sudo systemctl status mysqld -l
sudo grep 'temporary password' /var/log/mysqld.log
#过滤出初始密码 然后登陆
mysql -u root -p
#修改用户密码用下面这条
ALTER USER USER() IDENTIFIED BY '2019';
#必要的库和用户、权限
grant all privileges on *.* to 'root'@'%' identified by '2019' with grant option;
create database scm DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database nas DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database actmonitor DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database rptmgn DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database navmetas DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database sentry DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

grant all privileges on *.* to 'admin'@'%' identified by '2019' with grant option;

grant all privileges on scm.* to 'scm'@'%' identified by '2019' with grant option;
grant all privileges on hive.* to 'hive'@'%' identified by '2019' with grant option;
grant all privileges on oozie.* to 'oozie'@'%' identified by '2019' with grant option;
grant all privileges on hue.* to 'hue'@'%' identified by '2019' with grant option;
grant all privileges on nas.* to 'nas'@'%' identified by '2019' with grant option;
grant all privileges on actmonitor.* to 'actmonitor'@'%' identified by '2019' with grant option;
grant all privileges on rptmgn.* to 'rptmgn'@'%' identified by '2019' with grant option;
grant all privileges on navmetas.* to 'navmetas'@'%' identified by '2019' with grant option;
grant all privileges on sentry.* to 'sentry'@'%' identified by '2019' with grant option;
flush privileges;

安装CMS 主节点 192.168.0.3
准备: CMS、CDH-Parcel internal REPO搭建 [6.2.0为例]
1)sudo yum install httpd
2)sudo vim /etc/httpd/conf/httpd.conf添加如下:
AddType application/x-gzip .gz .tgz .parcel
3)sudo systemctl start httpd
sudo systemctl enable httpd
#创建本地yum源仓库 实现本地yum安装
4)sudo mkdir -p /var/www/html/cloudera-repos
####5-10步骤没有加速特别慢 自己下的时候 发现是在dump网站 建议拷贝 可以删除里面其它不用的包
###换为下面的单独下载包5)sudo proxychains4 wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/ -P /var/www/html/cloudera-repos
####5 centos选择CDH-6.2.0-1.cdh6.2.0.p0.967373-el7开头的三个下载即可

sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha1 -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha256 -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/manifest.json -P /var/www/html/cloudera-repos

6)sudo proxychains4 wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/gplextras6/6.2.0/parcels/ -P /var/www/html/cloudera-repos
######6 centos选择GPLEXTRAS-6.2.0-1.gplextras6.2.0.p0.967373-el7开头的三个下载即可
7)sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cm6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
8)sudo wget https://archive.cloudera.com/cm6/6.2.0/allkeys.asc -P /var/www/html/cloudera-repos/cm6/6.2.0/
9)sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/kafka/parcels/4/ -P /var/www/html/cloudera-repos
10)sudo wget --recursive --no-parent --no-host-directories http://archive.cloudera.com/sqoop-connectors/parcels/latest/ -P /var/www/html/cloudera-repos
11)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cdh6
12)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/gplextras6
13)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cm6
14)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/kafka
15)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/sqoop-connectors
16)curl http://localhost/cloudera-repos/
17)sudo vim /etc/yum.repos.d/cloudera-manager6.2.0.repo

[cloudera-manager]
name=Cloudera Manager 6.2.0
baseurl=http://本机IP或者本地hosts解析/cloudera-repos/cm6/6.2.0/redhat7/yum/
gpgkey=http://本机IP或者本地hosts解析/cloudera-repos/cm6/6.2.0/redhat7/yum/RPM-GPG-KEY-cloudera
gpgcheck=1
enabled=1
autorefresh=0
type=rpm-md

17.2)sudo vim /etc/yum.repos.d/cloudera-manager-gplextras6.2.0.repo

#两模块不要一样
[cloudera-manager-gplextras]
name=Cloudera Manager gplextras 6.2.0
baseurl=http://本机IP或者本地hosts解析/cloudera-repos/gplextras6/6.2.0/redhat7/yum/
gpgkey=http://本机IP或者本地hosts解析/cloudera-repos/gplextras6/6.2.0/redhat7/yum/RPM-GPG-KEY-cloudera
gpgcheck=1
enabled=1
autorefresh=0
type=rpm-md

18)sudo vim /etc/httpd/conf/httpd.conf

<Directory "/var/www/cgi-bin">
AllowOverride None
Options FollowSymLinks
Order deny,allow
allow from all

19)sudo systemctl restart httpd
20)sudo yum clean all
21)sudo yum makecache
#执行 yum makecache 报错 缺少/repodata/repomd.xml: [Errno 14] HTTP Error 404 - Not Found
#解决办法
#proxychains4 wget --recursive --no-parent --no-host-directories http://archive.cloudera.com/gplextras6/6.2.0/redhat7/yum/repodata/ -P /var/www/html/cloudera-repos
22)sudo yum update
23)sudo yum -y search cloudera-manager
24)sudo systemctl enable httpd

三)6.2.0 internal repo
#如果要求全部内网下载可以把主节点的repo直接ansible推过去[主节点一定要执行从节点可选吧 查资料上说为避免尽可能的减少出错 三节点都提前安装好但只需要主节点启动server端]

sudo yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
echo $JAVA_HOME

#只需要主节点执行------
sudo JAVA_HOME=/usr/java/jdk /opt/cloudera/cm-agent/bin/certmanager --location /opt/cloudera/CMCA setup --configure-services
sudo -E /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm '2019'

点赞
收藏
评论区
推荐文章
blmius blmius
3年前
MySQL:[Err] 1292 - Incorrect datetime value: ‘0000-00-00 00:00:00‘ for column ‘CREATE_TIME‘ at row 1
文章目录问题用navicat导入数据时,报错:原因这是因为当前的MySQL不支持datetime为0的情况。解决修改sql\mode:sql\mode:SQLMode定义了MySQL应支持的SQL语法、数据校验等,这样可以更容易地在不同的环境中使用MySQL。全局s
皕杰报表之UUID
​在我们用皕杰报表工具设计填报报表时,如何在新增行里自动增加id呢?能新增整数排序id吗?目前可以在新增行里自动增加id,但只能用uuid函数增加UUID编码,不能新增整数排序id。uuid函数说明:获取一个UUID,可以在填报表中用来创建数据ID语法:uuid()或uuid(sep)参数说明:sep布尔值,生成的uuid中是否包含分隔符'',缺省为
Jacquelyn38 Jacquelyn38
3年前
2020年前端实用代码段,为你的工作保驾护航
有空的时候,自己总结了几个代码段,在开发中也经常使用,谢谢。1、使用解构获取json数据let jsonData  id: 1,status: "OK",data: 'a', 'b';let  id, status, data: number   jsonData;console.log(id, status, number )
Stella981 Stella981
2年前
List的Select 和Select().tolist()
List<PersondelpnewList<Person{newPerson{Id1,Name"小明1",Age11,Sign0},newPerson{Id2,Name"小明2",Age12,
Wesley13 Wesley13
2年前
mysql设置时区
mysql设置时区mysql\_query("SETtime\_zone'8:00'")ordie('时区设置失败,请联系管理员!');中国在东8区所以加8方法二:selectcount(user\_id)asdevice,CONVERT\_TZ(FROM\_UNIXTIME(reg\_time),'08:00','0
Wesley13 Wesley13
2年前
Java日期时间API系列36
  十二时辰,古代劳动人民把一昼夜划分成十二个时段,每一个时段叫一个时辰。二十四小时和十二时辰对照表:时辰时间24时制子时深夜11:00凌晨01:0023:0001:00丑时上午01:00上午03:0001:0003:00寅时上午03:00上午0
Wesley13 Wesley13
2年前
00:Java简单了解
浅谈Java之概述Java是SUN(StanfordUniversityNetwork),斯坦福大学网络公司)1995年推出的一门高级编程语言。Java是一种面向Internet的编程语言。随着Java技术在web方面的不断成熟,已经成为Web应用程序的首选开发语言。Java是简单易学,完全面向对象,安全可靠,与平台无关的编程语言。
Stella981 Stella981
2年前
Django中Admin中的一些参数配置
设置在列表中显示的字段,id为django模型默认的主键list_display('id','name','sex','profession','email','qq','phone','status','create_time')设置在列表可编辑字段list_editable
Wesley13 Wesley13
2年前
MySQL部分从库上面因为大量的临时表tmp_table造成慢查询
背景描述Time:20190124T00:08:14.70572408:00User@Host:@Id:Schema:sentrymetaLast_errno:0Killed:0Query_time:0.315758Lock_
Python进阶者 Python进阶者
8个月前
Excel中这日期老是出来00:00:00,怎么用Pandas把这个去除
大家好,我是皮皮。一、前言前几天在Python白银交流群【上海新年人】问了一个Pandas数据筛选的问题。问题如下:这日期老是出来00:00:00,怎么把这个去除。二、实现过程后来【论草莓如何成为冻干莓】给了一个思路和代码如下:pd.toexcel之前把这