开源大数据集群部署（十一）Ranger 集成Hadoop集群

BitCosmicRover
• 阅读 223

作者：櫰木
1、节点选择
部署在两个namenode节点

cd /opt/bigdata

tar -xzvf ranger-2.3.0-hdfs-plugin.tar.gz -C /opt/

cd /opt/ranger-2.3.0-hdfs-plugin

vim install.properties

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
 
#
# Location of Policy Manager URL  
#
# Example:
# POLICY_MGR_URL=http://policymanager.xasecure.net:6080
#
POLICY_MGR_URL=http://hd1.dtstack.com:6080/
 
#
# This is the repository name created within policy manager
#
# Example:
# REPOSITORY_NAME=hadoopdev
#
REPOSITORY_NAME=hadoopdev
 
#
# Set hadoop home when hadoop program and Ranger HDFS Plugin are not in the
# same path.
#
COMPONENT_INSTALL_DIR_NAME=/opt/hadoop
 
# AUDIT configuration with V3 properties
# Enable audit logs to Solr
#Example
#XAAUDIT.SOLR.ENABLE=true
#XAAUDIT.SOLR.URL=http://localhost:6083/solr/ranger_audits
#XAAUDIT.SOLR.ZOOKEEPER=
#XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hadoop/hdfs/audit/solr/spool
 
XAAUDIT.SOLR.ENABLE=false
XAAUDIT.SOLR.URL=NONE
XAAUDIT.SOLR.USER=NONE
XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hadoop/hdfs/audit/solr/spool
 
# Enable audit logs to ElasticSearch
#Example
#XAAUDIT.ELASTICSEARCH.ENABLE=true
#XAAUDIT.ELASTICSEARCH.URL=localhost
#XAAUDIT.ELASTICSEARCH.INDEX=audit
 
XAAUDIT.ELASTICSEARCH.ENABLE=false
XAAUDIT.ELASTICSEARCH.URL=NONE
XAAUDIT.ELASTICSEARCH.USER=NONE
XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
XAAUDIT.ELASTICSEARCH.INDEX=NONE
XAAUDIT.ELASTICSEARCH.PORT=NONE
XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
 
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
#XAAUDIT.HDFS.HDFS_DIR=hdfs://node-1.example.com:8020/ranger/audit
#XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/hdfs/audit/hdfs/spool
#  If using Azure Blob Storage
#XAAUDIT.HDFS.HDFS_DIR=wasb[s]://<containername>@<accountname>.blob.core.windows.net/<path>
#XAAUDIT.HDFS.HDFS_DIR=wasb://ranger_audit_container@my-azure-account.blob.core.windows.net/ranger/audit
 
XAAUDIT.HDFS.ENABLE=false
XAAUDIT.HDFS.HDFS_DIR=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit
XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/hdfs/audit/hdfs/spool
 
# Following additional propertis are needed When auditing to Azure Blob Storage via HDFS
# Get these values from your /etc/hadoop/conf/core-site.xml
#XAAUDIT.HDFS.HDFS_DIR=wasb[s]://<containername>@<accountname>.blob.core.windows.net/<path>
XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY
XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER
XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER
 
#Log4j Audit Provider
XAAUDIT.LOG4J.ENABLE=false
XAAUDIT.LOG4J.IS_ASYNC=false
XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
XAAUDIT.LOG4J.DESTINATION.LOG4J=true
XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
 
# Enable audit logs to Amazon CloudWatch Logs
#Example
#XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=true
#XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=ranger_audits
#XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM={instance_id}
#XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=/var/log/hive/audit/amazon_cloudwatch/spool
 
XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
 
# End of V3 properties
 
#
#  Audit to HDFS Configuration
#
# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens
# that start with __REPLACE__ with appropriate values
#  XAAUDIT.HDFS.IS_ENABLED=true
#  XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%
#  XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hadoop/%app-type%/audit
#  XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive
#
# Example:
#  XAAUDIT.HDFS.IS_ENABLED=true
#  XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd%
#  XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hadoop/%app-type%/audit
#  XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hadoop/%app-type%/audit/archive
#
XAAUDIT.HDFS.IS_ENABLED=false
XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hadoop/%app-type%/audit
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive
 
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
 
#Solr Audit Provider
XAAUDIT.SOLR.IS_ENABLED=false
XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
 
# End of V2 properties
 
#
# SSL Client Certificate Information
#
# Example:
# SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
# SSL_KEYSTORE_PASSWORD=none
# SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks
# SSL_TRUSTSTORE_PASSWORD=none
#
# You do not need use SSL between agent and security admin tool, please leave these sample value as it is.
#
SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
SSL_KEYSTORE_PASSWORD=myKeyFilePassword
SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks
SSL_TRUSTSTORE_PASSWORD=changeit
 
# Custom component user
# CUSTOM_COMPONENT_USER=<custom-user>
# keep blank if component user is default
CUSTOM_USER=hdfs
# Custom component group
# CUSTOM_COMPONENT_GROUP=<custom-group>
# keep blank if component group is default
CUSTOM_GROUP=hadoop

ranger hdfs初始化

cd /opt/ranger-2.3.0-hdfs-plugin
./enable-hdfs-plugin.sh

初始化完成后会在/opt/hadoop/etc/hadoop目录下生成3个文件

开源大数据集群部署（十一）Ranger 集成Hadoop集群
在rangeradmin页面上进行配置

开源大数据集群部署（十一）Ranger 集成Hadoop集群
servicename必须和install.properties中的REPOSITORY_NAME填写一致
开源大数据集群部署（十一）Ranger 集成Hadoop集群
以下内容参考hdfs-site.xml和core-site.xml参考填写
开源大数据集群部署（十一）Ranger 集成Hadoop集群
添加以下配置
开源大数据集群部署（十一）Ranger 集成Hadoop集群

tag.download.auth.users  hdfs
policy.download.auth.users hdfs

测试是否连接成功
开源大数据集群部署（十一）Ranger 集成Hadoop集群

更多技术信息请查看云掣官网https://yunche.pro/?t=yrgw

大数据
点赞
收藏
评论区
推荐文章
待兔 待兔
•
1年前
手写Java HashMap源码
HashMap的使用教程HashMap的使用教程HashMap的使用教程HashMap的使用教程HashMap的使用教程22
3382
0
0
Wesley13 Wesley13
•
4年前
RAC环境单实例启动数据库收到ORA
     在RAC环境中，如果你在没有启动节点的集群服务的情况下单实例启动数据库，将收到类似如下的报错：\oracle@rhel1u01\$sqlSQL\Plus:Release10.2.0.5.0ProductiononTueApr215:00:272013Copyright(
1231
0
0
Wesley13 Wesley13
•
4年前
05.kafka提前准备工作：搭建zookeeper集群环境
总体参考：http://www.cnblogs.com/zhangs1986/p/6564839.html搭建之间同步下spark01、02、03的环境复制/opt/flume这个文件夹到spark02的/opt目录下，同理03scpr/opt/flume/spark02:/opt/scpr/opt/kafka/spar
1178
0
0
Wesley13 Wesley13
•
4年前
MySQL 5.6 Generic Binary安装与配置
安装软件包\mkdirp/opt/mysql\mvmysql5.6.17linuxglibc2.5x86\_64.tar.gz/opt/mysql\cd/opt/mysql\tarxvfzmysql5.6.17linuxglibc2.5x86\_64.tar.gz\创建数据库
1096
0
0
Easter79 Easter79
•
4年前
TiDB 压力测试报告
TiDB压力测试报告（转载自公众号DBATech）一、测试环境1、tidb集群架构：测试使用最基本的TiDB架构。即3个tidbserver节点3个tikv节点3个pd节点。2、tidb集群的部署环境(混合部署)：192.168.xx.A1\server1\PD1\tikv192.
1389
0
0
Stella981 Stella981
•
4年前
Ambari在离线环境中安装Hadoop集群
(WJW)Ambari在离线环境中安装Hadoop集群    如果手工安装Hadoop集群,在管理和后继部署中，越来越多的问题不断出现，主要如下：各集群节点的配置同步新应用的部署：比如为集群添加hbase应用，需要很细心复杂的配置才能完成，而且要是操作错误，还可能导致正常的集
1665
0
0
Stella981 Stella981
•
4年前
Kubernetes学习之路（三）之Mater节点二进制部署
K8SMater节点部署1、部署KubernetesAPI服务部署apiserver提供集群管理的RESTAPI接口，包括认证授权、数据校验以及集群状态变更等。只有APIServer才能直接操作etcd；其他模块通过APIServer查
1058
0
0
Stella981 Stella981
•
4年前
Hadoop2.7.3完全分布式集群安装过程
需要安装的软件Hadoop包含HDFS集群和YARN集群。部署Hadoop就是部署HDFS和YARN集群。机器数量、角色4台。NameNode1台、DataNode3台主机名IP角色amaster192.168.37.143NameNode:9000ResourceManag
1233
0
0
Stella981 Stella981
•
4年前
Consul 启动命令，Web UI
一、服务端：nohupconsulagentserverbootstrapexpect1configdir/etc/consul.d/datadir/var/opt/consulbind125.72.214.83/var/opt/consul/consul.log
1545
0
0
Stella981 Stella981
•
4年前
Docker搭建Nexus（Maven私库）
0、镜像的查找：dockersearchnexus1、拉取官方镜像：dockerpullsonatype/nexus32、创建了自己的目录(/opt/nexus/nexusdata)　　mkdir /opt/nexus/nexusdata&&chownR200 /opt/nexus/nexusdata3、运行容器
1139
0
0
Stella981 Stella981
•
4年前
Quartz依赖数据库表
Quartz的集群部署方案在架构上是分布式的，没有负责集中管理的节点，而是利用数据库锁的方式来实现集群环境下进行并发控制。BTW，分布式部署时需要保证各个节点的系统时间一致。Quartz数据库核心表如下：tabledescriptionQRTZ\_CALENDARS存储Quartz的Calendar信息QRTZ\_CR
1079
0
0
BitCosmicRover
BitCosmicRover
Lv1
门有车马客，驾言发故乡。
文章
5
粉丝
0
获赞
0
热门文章
【C】 42_内存操作经典问题分析 二
技术分享 | innodb-cluster 扫盲-安装篇
并发编程
前端阶段性总结（一）： HTTP 协议，从1.0到2.0