This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch ranger-2.4 in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 258e74c81b4da0c845ba42aea1bf03286e7d190d Author: Madhan Neethiraj <[email protected]> AuthorDate: Sat Dec 3 16:01:42 2022 -0800 RANGER-3978: Docker setup fixes - HBase and Knox version updated, incorrect mysql path in KMS, incorrect service-config, updated README.md - #3 (cherry picked from commit 88c4bdacc785d8056bb641b6ff37a19a272f692a) --- dev-support/ranger-docker/.env | 5 ++- dev-support/ranger-docker/Dockerfile.ranger-kms | 2 +- dev-support/ranger-docker/README.md | 52 +++++++++++++--------- .../ranger-docker/docker-compose.ranger-build.yml | 1 + .../scripts/create-ranger-services.py | 12 ++--- 5 files changed, 43 insertions(+), 29 deletions(-) diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index 7bc0daf14..540e1e70d 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -2,12 +2,13 @@ BUILD_HOST_SRC=true SKIPTESTS=true GIT_URL=https://github.com/apache/ranger.git BRANCH=ranger-2.4 +PROFILE= RANGER_VERSION=2.4.0-SNAPSHOT HADOOP_VERSION=3.3.0 -HBASE_VERSION=2.2.6 +HBASE_VERSION=2.4.6 HIVE_VERSION=3.1.2 HIVE_HADOOP_VERSION=3.1.1 -KAFKA_VERSION=2.5.0 +KAFKA_VERSION=2.8.1 KNOX_VERSION=1.4.0 diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kms b/dev-support/ranger-docker/Dockerfile.ranger-kms index 4cf8b9d73..96fcecff9 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kms +++ b/dev-support/ranger-docker/Dockerfile.ranger-kms @@ -52,7 +52,7 @@ FROM ranger-kms AS ranger_mysql COPY ./downloads/mysql-connector-java-8.0.28.jar /home/ranger/dist/ COPY ./downloads/log4jdbc-1.2.jar /home/ranger/dist/ RUN mv /home/ranger/dist/mysql-connector-java-8.0.28.jar /usr/share/java/mysql-connector.jar && \ - mv /home/ranger/dist/log4jdbc-1.2.jar ${RANGER_HOME}/admin/ews/webapp/WEB-INF/lib/log4jdbc-1.2.jar + mv /home/ranger/dist/log4jdbc-1.2.jar ${RANGER_HOME}/kms/ews/webapp/WEB-INF/lib/log4jdbc-1.2.jar FROM ranger_${RANGER_DB_TYPE} diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md index 3e441c1ce..d6337329e 100644 --- a/dev-support/ranger-docker/README.md +++ b/dev-support/ranger-docker/README.md @@ -25,22 +25,31 @@ Docker files in this folder create docker images and run them to build Apache Ra 1. Ensure that you have recent version of Docker installed from [docker.io](http://www.docker.io) (as of this writing: Engine 20.10.5, Compose 1.28.5). Make sure to configure docker with at least 6gb of memory. - + 2. Update environment variables in ```.env``` file, if necessary 3. Set ```dev-support/ranger-docker``` as your working directory. -4. Execute this command to download necessary archives to set up Ranger/HDFS/Hive/HBase/Kafka/Knox services: - - ```chmod +x download-archives.sh && ./download-archives.sh``` +4. Execute following command to download necessary archives to setup Ranger/HDFS/Hive/HBase/Kafka/Knox services: + ~~~ + chmod +x download-archives.sh && ./download-archives.sh + ~~~ -5. Build Apache Ranger in containers using docker-compose +5. Execute following commands to set environment variables to build Apache Ranger docker containers: + ~~~ + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 + export RANGER_DB_TYPE=postgres + ~~~ - 1. Execute following command to build Apache Ranger: +6. Build Apache Ranger in containers using docker-compose - ```docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger-build.yml up``` + 1. Execute following command to build Apache Ranger: + ~~~ + docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger-build.yml up + ~~~ - Time taken to complete the build might vary (upto an hour), depending on status of ```${HOME}/.m2``` directory cache. + Time taken to complete the build might vary (upto an hour), depending on status of ```${HOME}/.m2``` directory cache. 2. Alternatively, the following commands can be executed from the parent directory 1. To generate tarballs:```mvn clean package -DskipTests``` @@ -52,17 +61,20 @@ Docker files in this folder create docker images and run them to build Apache Ra ~~~ 3. Build the ranger-base image: - ```docker-compose -f docker-compose.ranger-base.yml build --no-cache``` + ~~~ + docker-compose -f docker-compose.ranger-base.yml build --no-cache + ~~~ -6. Execute following command to start Ranger, Ranger enabled HDFS/YARN/HBase/Hive/Kafka/Knox and dependent services (Solr, DB) in containers: - ~~~ - export RANGER_DB_TYPE=postgres - docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml -f docker-compose.ranger-usersync.yml -f docker-compose.ranger-tagsync.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml -f docker-compose.ranger-hive.yml -f docker-compose.ranger-knox.yml up -d - ~~~ - - valid values for ```RANGER_DB_TYPE```: mysql or postgres +7. Execute following command to start Ranger, Ranger enabled HDFS/YARN/HBase/Hive/Kafka/Knox and dependent services (Solr, DB) in containers: + ~~~ + docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml -f docker-compose.ranger-usersync.yml -f docker-compose.ranger-tagsync.yml -f docker-compose.ranger-kms.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml -f docker-compose.ranger-hive.yml -f docker-compose.ranger-knox.yml up -d + ~~~ -7. To rebuild specific images and start containers with the new image, use following command: - ~~~ - docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-usersync.yml -f docker-compose.ranger-tagsync.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml -f docker-compose.ranger-hive.yml -f docker-compose.ranger-knox.yml up -d --no-deps --force-recreate --build <service-1> <service-2> - ~~~ -8. Ranger Admin can be accessed at http://localhost:6080 (admin/rangerR0cks!) + - valid values for RANGER_DB_TYPE: mysql or postgres + +8. To rebuild specific images and start containers with the new image, use following command: + ~~~ + docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-usersync.yml -f docker-compose.ranger-tagsync.yml -f docker-compose.ranger-kms.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml -f docker-compose.ranger-hive.yml -f docker-compose.ranger-knox.yml up -d --no-deps --force-recreate --build <service-1> <service-2> + ~~~ + +9. Ranger Admin can be accessed at http://localhost:6080 (admin/rangerR0cks!) diff --git a/dev-support/ranger-docker/docker-compose.ranger-build.yml b/dev-support/ranger-docker/docker-compose.ranger-build.yml index f0b5b05e0..48396a92d 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-build.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-build.yml @@ -20,6 +20,7 @@ services: environment: - BRANCH - BUILD_HOST_SRC + - PROFILE - GIT_URL - RANGER_VERSION - SKIPTESTS diff --git a/dev-support/ranger-docker/scripts/create-ranger-services.py b/dev-support/ranger-docker/scripts/create-ranger-services.py index a3df61fba..c9ab3263c 100644 --- a/dev-support/ranger-docker/scripts/create-ranger-services.py +++ b/dev-support/ranger-docker/scripts/create-ranger-services.py @@ -7,10 +7,10 @@ ranger_client = RangerClient('http://ranger:6080', ('admin', 'rangerR0cks!')) def service_not_exists(service): try: - ranger_client.get_service(service.name) + svc = ranger_client.get_service(service.name) except JSONDecodeError: return 1 - return 0 + return 0 if svc is not None else 1 hdfs = RangerService({'name': 'dev_hdfs', 'type': 'hdfs', @@ -22,7 +22,7 @@ hdfs = RangerService({'name': 'dev_hdfs', 'type': 'hdfs', hive = RangerService({'name': 'dev_hive', 'type': 'hive', 'configs': {'username': 'hive', 'password': 'hive', 'jdbc.driverClassName': 'org.apache.hive.jdbc.HiveDriver', - 'jdbc.url': 'jdfb:hive2://ranger-hadoop:10000', + 'jdbc.url': 'jdbc:hive2://ranger-hive:10000', 'hadoop.security.authorization': 'true'}}) kafka = RangerService({'name': 'dev_kafka', 'type': 'kafka', @@ -30,7 +30,7 @@ kafka = RangerService({'name': 'dev_kafka', 'type': 'kafka', 'zookeeper.connect': 'ranger-zk.example.com:2181'}}) knox = RangerService({'name': 'dev_knox', 'type': 'knox', - 'configs': {'username': 'knox', 'password': 'knox', 'knox.url': 'http://ranger-hadoop:8088'}}) + 'configs': {'username': 'knox', 'password': 'knox', 'knox.url': 'https://ranger-knox:8443'}}) yarn = RangerService({'name': 'dev_yarn', 'type': 'yarn', 'configs': {'username': 'yarn', 'password': 'yarn', @@ -41,8 +41,8 @@ hbase = RangerService({'name': 'dev_hbase', 'type': 'hbase', 'hadoop.security.authentication': 'simple', 'hbase.security.authentication': 'simple', 'hadoop.security.authorization': 'true', - 'hbase.zookeeper.property.clientPort': '16181', - 'hbase.zookeeper.quorum': 'ranger-hbase', + 'hbase.zookeeper.property.clientPort': '2181', + 'hbase.zookeeper.quorum': 'ranger-zk', 'zookeeper.znode.parent': '/hbase'}}) kms = RangerService({'name': 'dev_kms', 'type': 'kms',
