This is an automated email from the ASF dual-hosted git repository. abhi pushed a commit to branch debug_hdp_ci in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 083e0a71b8895e3bf5a25ca74988eed574b3f3d2 Author: Abhishek Kumar <[email protected]> AuthorDate: Sun Nov 9 14:04:27 2025 -0800 RANGER-5395: Split docker builds across GitHub runners with ranger-service and ranger-plugin builds --- .github/workflows/ci.yml | 98 +++++++++++++++++++--- .../ranger-docker/docker-compose.ranger-hadoop.yml | 6 +- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a0e1546a..c0a9139ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,7 @@ jobs: name: target-11 path: target/* - docker-build: + services-docker-build: needs: - build-8 runs-on: ubuntu-24.04 # bumping os version for more disk space @@ -115,7 +115,10 @@ jobs: - name: Copy artifacts for docker build run: | - cp ranger-*.tar.gz dev-support/ranger-docker/dist + cp ranger-*-admin.tar.gz dev-support/ranger-docker/dist + cp ranger-*-usersync.tar.gz dev-support/ranger-docker/dist + cp ranger-*-tagsync.tar.gz dev-support/ranger-docker/dist + cp ranger-*-kms.tar.gz dev-support/ranger-docker/dist cp version dev-support/ranger-docker/dist - name: Cache downloaded archives @@ -126,10 +129,10 @@ jobs: restore-keys: | ${{ runner.os }}-ranger-downloads- - - name: Run download-archives.sh + - name: Download DB and Log4J connectors run: | cd dev-support/ranger-docker - ./download-archives.sh hadoop hive hbase kafka knox ozone + ./download-archives.sh none - name: Clean up Docker space run: docker system prune --all --force --volumes @@ -142,7 +145,81 @@ jobs: -f docker-compose.ranger.yml \ -f docker-compose.ranger-usersync.yml \ -f docker-compose.ranger-tagsync.yml \ - -f docker-compose.ranger-kms.yml \ + -f docker-compose.ranger-kms.yml build + + - name: Bring up ranger-service containers + run: | + cd dev-support/ranger-docker + export RANGER_DB_TYPE=postgres + docker compose \ + -f docker-compose.ranger.yml \ + -f docker-compose.ranger-usersync.yml \ + -f docker-compose.ranger-tagsync.yml \ + -f docker-compose.ranger-kms.yml up -d + + - name: Check status of containers and remove them + run: | + sleep 60 + containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms); + flag=true; + for container in "${containers[@]}"; do + if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then + echo "Container $container is running!"; + else + flag=false; + echo "Container $container is NOT running! Status: $(docker inspect -f '{{.State.Status}}' $container 2>/dev/null)"; + echo "docker logs $container:"; + docker logs $container + fi + done + + if [[ $flag == true ]]; then + echo "All required containers are up and running"; + docker stop $(docker ps -q) && docker rm $(docker ps -aq); + else + docker stop $(docker ps -q) && docker rm $(docker ps -aq); + exit 1; + fi + + plugins-docker-build: + needs: + - build-8 + runs-on: ubuntu-24.04 # bumping os version for more disk space + timeout-minutes: 60 # bumping timeout to allow containers to come up + steps: + - uses: actions/checkout@v4 + - name: Download build-8 artifacts + uses: actions/download-artifact@v4 + with: + name: target-8 + + - name: Copy artifacts for docker build + run: | + cp ranger-*-plugin.tar.gz dev-support/ranger-docker/dist + cp version dev-support/ranger-docker/dist + + - name: Cache downloaded archives + uses: actions/cache@v4 + with: + path: dev-support/ranger-docker/downloads + key: ${{ runner.os }}-ranger-downloads-${{ hashFiles('dev-support/ranger-docker/.env') }} + restore-keys: | + ${{ runner.os }}-ranger-downloads- + + - name: Run download-archives.sh + run: | + cd dev-support/ranger-docker + ./download-archives.sh hadoop hive hbase kafka knox ozone + + - name: Clean up Docker space + run: docker system prune --all --force --volumes + + - name: Build all ranger-plugin images + run: | + cd dev-support/ranger-docker + export RANGER_DB_TYPE=postgres + docker compose \ + -f docker-compose.ranger.yml \ -f docker-compose.ranger-hadoop.yml \ -f docker-compose.ranger-hbase.yml \ -f docker-compose.ranger-kafka.yml \ @@ -150,16 +227,13 @@ jobs: -f docker-compose.ranger-knox.yml \ -f docker-compose.ranger-ozone.yml build - - name: Bring up containers + - name: Bring up ranger-plugin containers run: | cd dev-support/ranger-docker ./scripts/ozone/ozone-plugin-docker-setup.sh export RANGER_DB_TYPE=postgres docker compose \ -f docker-compose.ranger.yml \ - -f docker-compose.ranger-usersync.yml \ - -f docker-compose.ranger-tagsync.yml \ - -f docker-compose.ranger-kms.yml \ -f docker-compose.ranger-hadoop.yml \ -f docker-compose.ranger-hbase.yml \ -f docker-compose.ranger-kafka.yml \ @@ -170,7 +244,7 @@ jobs: - name: Check status of containers and remove them run: | sleep 60 - containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); + containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); flag=true; for container in "${containers[@]}"; do if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then @@ -182,11 +256,11 @@ jobs: docker logs $container fi done - + if [[ $flag == true ]]; then echo "All required containers are up and running"; docker stop $(docker ps -q) && docker rm $(docker ps -aq); else docker stop $(docker ps -q) && docker rm $(docker ps -aq); exit 1; - fi + fi \ No newline at end of file diff --git a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml index 0e02b5b78..71e1d40fd 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml @@ -27,10 +27,10 @@ services: ranger: condition: service_started healthcheck: - test: 'su -c "/home/ranger/scripts/ranger-hadoop-healthcheck.sh" healthcheck' - interval: 15s + test: ["CMD-SHELL", "/home/ranger/scripts/ranger-hadoop-healthcheck.sh"] + interval: 20s timeout: 10s - retries: 30 + retries: 40 start_period: 40s environment: - HADOOP_VERSION
