This is an automated email from the ASF dual-hosted git repository. tuhaihe pushed a commit to branch REL_2_STABLE in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
commit 42fdc41b6c8e5db61bd4f78dcb38bd07b0e744b7 Author: liuxiaoyu <[email protected]> AuthorDate: Thu Mar 26 20:53:27 2026 +0800 ci: add Rocky 9 support for CI and Docker build environment (#78) - Unify CI scripts from ubuntu/ to common/, supporting both Ubuntu and Rocky/RHEL - via auto-detection of package manager (apt-get vs dnf) - Add Rocky 9 Docker Compose and RPM build script (rocky9/) - Parameterize singlecluster Dockerfile with ARG BASE_IMAGE for multi-distro builds - Add Rocky 9 singlecluster image build job to GitHub Actions workflow - Cross-platform JAVA_HOME auto-detection in build_pxf.sh --- .github/workflows/pxf-ci.yml | 402 +++++++++++++++++---- .../{ubuntu => common}/script/build_cloudberrry.sh | 128 +++++-- .../{ubuntu => common}/script/build_pxf.sh | 25 +- .../{ubuntu => common}/script/entrypoint.sh | 275 +++++++++----- .../{ubuntu => common}/script/pxf-env.sh | 33 +- .../{ubuntu => common}/script/pxf-test.sh | 0 .../{ubuntu => common}/script/run_tests.sh | 0 .../{ubuntu => common}/script/start_minio.bash | 0 .../{ubuntu => common}/script/utils.sh | 15 +- ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml | 46 +++ .../rocky9/script/build_cloudberry_rpm.sh | 110 ++++++ .../ubuntu/script/build_cloudberry_deb.sh | 34 +- ci/singlecluster/Dockerfile | 67 ++-- 13 files changed, 842 insertions(+), 293 deletions(-) diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml index 6e2b9fc2..86be5b58 100644 --- a/.github/workflows/pxf-ci.yml +++ b/.github/workflows/pxf-ci.yml @@ -91,8 +91,54 @@ jobs: path: workspace/cloudberry-source.tar.gz retention-days: 7 + build-cloudberry-rpm: + name: Build Cloudberry RPM Package + runs-on: ubuntu-latest + container: + image: apache/incubator-cloudberry:cbdb-build-rocky9-latest + options: --user root + steps: + - name: Checkout Cloudberry source + uses: actions/checkout@v4 + with: + repository: apache/cloudberry + ref: ${{ env.CLOUDBERRY_VERSION }} + path: workspace/cloudberry + submodules: true + + - name: Checkout PXF source (for build script) + uses: actions/checkout@v4 + with: + path: cloudberry-pxf + + - name: Build Cloudberry RPM + run: | + export WORKSPACE=$PWD/workspace + export CLOUDBERRY_VERSION=99.0.0 + export CLOUDBERRY_BUILD=1 + bash cloudberry-pxf/ci/docker/pxf-cbdb-dev/rocky9/script/build_cloudberry_rpm.sh + + - name: Package Cloudberry source + run: | + cd workspace + tar czf cloudberry-source-rocky9.tar.gz cloudberry/ + + - name: Upload RPM artifact + uses: actions/upload-artifact@v4 + with: + name: cloudberry-rpm + path: workspace/cloudberry-rpm/*.rpm + retention-days: 7 + + - name: Upload Cloudberry source artifact (Rocky 9) + uses: actions/upload-artifact@v4 + with: + name: cloudberry-source-rocky9 + path: workspace/cloudberry-source-rocky9.tar.gz + retention-days: 7 + build-docker-images: - name: Build Docker Images + name: Build Docker Images (Ubuntu) runs-on: ubuntu-latest steps: - name: Checkout PXF source @@ -130,6 +176,28 @@ jobs: working-directory: ./server run: make test + build-docker-images-rocky9: + name: Build Docker Images (Rocky 9) + runs-on: ubuntu-latest + steps: + - name: Checkout PXF source + uses: actions/checkout@v4 + with: + path: cloudberry-pxf + + - name: Build singlecluster Rocky 9 image + run: | + cd cloudberry-pxf/ci/singlecluster + docker build --build-arg BASE_IMAGE=apache/incubator-cloudberry:cbdb-build-rocky9-latest -t pxf/singlecluster-rocky9:3 . + docker save pxf/singlecluster-rocky9:3 > /tmp/singlecluster-rocky9-image.tar + + - name: Upload singlecluster Rocky 9 image + uses: actions/upload-artifact@v4 + with: + name: singlecluster-rocky9-image + path: /tmp/singlecluster-rocky9-image.tar + retention-days: 1 + # Stage 2: Parallel test jobs using matrix strategy pxf-test: name: Test PXF - ${{ matrix.test_group }} @@ -214,14 +282,14 @@ jobs: docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/workspace/cloudberry docker cp /tmp/*.deb pxf-cbdb-dev:/tmp/ docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.deb - docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/ci/docker/pxf-cbdb-dev/ubuntu && ./script/entrypoint.sh" + docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf && ./ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh" - name: Run Test - ${{ matrix.test_group }} id: run_test continue-on-error: true timeout-minutes: 120 run: | - docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/automation && source ../ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh && ../ci/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh ${{ matrix.test_group }}" + docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/automation && source ../ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh && ../ci/docker/pxf-cbdb-dev/common/script/run_tests.sh ${{ matrix.test_group }}" - name: Collect artifacts and generate stats if: always() @@ -301,10 +369,182 @@ jobs: exit 1 fi + # Stage 2b: Rocky 9 parallel test jobs + pxf-test-rocky9: + name: Test PXF Rocky9 - ${{ matrix.test_group }} + needs: [build-cloudberry-rpm, build-docker-images-rocky9] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test_group: + - cli + - external-table + - fdw + - server + - sanity + - smoke + - hdfs + - hcatalog + - hcfs + - hive + - hbase + - profile + - jdbc + - proxy + - unused + - s3 + - features + - gpdb + - gpdb_fdw + - load + - pxf_extension + steps: + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/hostedtoolcache + sudo docker system prune -af + df -h + + - name: Checkout PXF source + uses: actions/checkout@v4 + with: + fetch-depth: 1 + path: cloudberry-pxf + submodules: true + + - name: Download Cloudberry RPM + uses: actions/download-artifact@v4 + with: + name: cloudberry-rpm + path: /tmp + + - name: Download Cloudberry source (Rocky 9) + uses: actions/download-artifact@v4 + with: + name: cloudberry-source-rocky9 + path: /tmp + + - name: Download singlecluster Rocky 9 image + uses: actions/download-artifact@v4 + with: + name: singlecluster-rocky9-image + path: /tmp + + - name: Load singlecluster Rocky 9 image + run: | + docker load < /tmp/singlecluster-rocky9-image.tar + + - name: Prepare Cloudberry source + run: | + tar xzf /tmp/cloudberry-source-rocky9.tar.gz + chmod -R u+rwX,go+rX cloudberry + + - name: Start Services + id: start_services + run: | + cd cloudberry-pxf + docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml down -v || true + docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml build + docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml up -d + docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/workspace/cloudberry + docker cp /tmp/*.rpm pxf-cbdb-dev:/tmp/ + docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.rpm + docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf && ./ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh" + + - name: Run Test - ${{ matrix.test_group }} + id: run_test + continue-on-error: true + timeout-minutes: 120 + run: | + docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/automation && source ../ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh && ../ci/docker/pxf-cbdb-dev/common/script/run_tests.sh ${{ matrix.test_group }}" + + - name: Collect artifacts and generate stats + if: always() + id: collect_artifacts + run: | + mkdir -p artifacts/logs + TEST_GROUP="${{ matrix.test_group }}" + TEST_RESULT="${{ steps.run_test.outcome }}" + + # Initialize counters + TOTAL=0 + PASSED=0 + FAILED=0 + SKIPPED=0 + + # Copy test artifacts + cp -r cloudberry-pxf/automation/test_artifacts/* artifacts/ 2>/dev/null || true + docker exec pxf-cbdb-dev bash -c "cp -r /usr/local/pxf/logs/* /tmp/pxf-logs/ 2>/dev/null || true" || true + docker cp pxf-cbdb-dev:/tmp/pxf-logs artifacts/logs/ 2>/dev/null || true + + # Parse surefire reports for automation tests + if [[ "$TEST_GROUP" != "cli" && "$TEST_GROUP" != "server" ]]; then + for xml in cloudberry-pxf/automation/target/surefire-reports/TEST-*.xml; do + if [ -f "$xml" ]; then + tests=$(grep -oP 'tests="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0") + failures=$(grep -oP 'failures="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0") + errors=$(grep -oP 'errors="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0") + skipped=$(grep -oP 'skipped="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0") + + TOTAL=$((TOTAL + tests)) + FAILED=$((FAILED + failures + errors)) + SKIPPED=$((SKIPPED + skipped)) + fi + done + PASSED=$((TOTAL - FAILED - SKIPPED)) + fi + + # Generate stats JSON + cat > artifacts/test_stats.json <<EOF + { + "group": "$TEST_GROUP", + "result": "$TEST_RESULT", + "total": $TOTAL, + "passed": $PASSED, + "failed": $FAILED, + "skipped": $SKIPPED + } + EOF + + echo "failed_count=$FAILED" >> $GITHUB_OUTPUT + echo "skipped_count=$SKIPPED" >> $GITHUB_OUTPUT + echo "Test stats for $TEST_GROUP (Rocky 9): total=$TOTAL, passed=$PASSED, failed=$FAILED, skipped=$SKIPPED" + + - name: Cleanup containers + if: always() + run: | + cd cloudberry-pxf + docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml down -v || true + + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-rocky9-${{ matrix.test_group }} + path: artifacts/** + if-no-files-found: ignore + retention-days: 7 + + - name: Check test result + if: always() + run: | + FAILED_COUNT="${{ steps.collect_artifacts.outputs.failed_count || 0 }}" + SKIPPED_COUNT="${{ steps.collect_artifacts.outputs.skipped_count || 0 }}" + + if [ "${{ steps.run_test.outcome }}" == "failure" ] || [ "$FAILED_COUNT" -gt 0 ]; then + echo "Test group ${{ matrix.test_group }} (Rocky 9) failed (Failures: $FAILED_COUNT, Skipped: $SKIPPED_COUNT)" + exit 1 + fi + # Stage 3: Summary job test-summary: name: Test Summary - needs: [pxf-test] + needs: [pxf-test, pxf-test-rocky9] if: always() runs-on: ubuntu-latest steps: @@ -318,79 +558,89 @@ jobs: run: | echo "## PXF Test Results Summary" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - # Overall counters - OVERALL_TOTAL=0 - OVERALL_PASSED=0 - OVERALL_FAILED=0 - OVERALL_SKIPPED=0 - GROUPS_PASSED=0 - GROUPS_FAILED=0 - FAILED_GROUP_NAMES="" - - # Collect all test stats - declare -A GROUP_STATS - - for dir in all-artifacts/test-results-*; do - if [ -d "$dir" ] && [ -f "$dir/test_stats.json" ]; then - group=$(cat "$dir/test_stats.json" | grep -oP '"group":\s*"\K[^"]+' || basename "$dir" | sed 's/test-results-//') - result=$(cat "$dir/test_stats.json" | grep -oP '"result":\s*"\K[^"]+' || echo "unknown") - total=$(cat "$dir/test_stats.json" | grep -oP '"total":\s*\K\d+' || echo "0") - passed=$(cat "$dir/test_stats.json" | grep -oP '"passed":\s*\K\d+' || echo "0") - failed=$(cat "$dir/test_stats.json" | grep -oP '"failed":\s*\K\d+' || echo "0") - skipped=$(cat "$dir/test_stats.json" | grep -oP '"skipped":\s*\K\d+' || echo "0") - - GROUP_STATS[$group]="$result,$total,$passed,$failed,$skipped" - - OVERALL_TOTAL=$((OVERALL_TOTAL + total)) - OVERALL_PASSED=$((OVERALL_PASSED + passed)) - OVERALL_FAILED=$((OVERALL_FAILED + failed)) - OVERALL_SKIPPED=$((OVERALL_SKIPPED + skipped)) - + + generate_os_summary() { + local os_label="$1" + local pattern="$2" + + # OS-level counters + local OVERALL_TOTAL=0 + local OVERALL_PASSED=0 + local OVERALL_FAILED=0 + local OVERALL_SKIPPED=0 + local GROUPS_PASSED=0 + local GROUPS_FAILED=0 + local FAILED_GROUP_NAMES="" + + declare -A GROUP_STATS + + for dir in all-artifacts/${pattern}; do + if [ -d "$dir" ] && [ -f "$dir/test_stats.json" ]; then + group=$(cat "$dir/test_stats.json" | grep -oP '"group":\s*"\K[^"]+' || basename "$dir" | sed "s/${pattern%\*}//") + result=$(cat "$dir/test_stats.json" | grep -oP '"result":\s*"\K[^"]+' || echo "unknown") + total=$(cat "$dir/test_stats.json" | grep -oP '"total":\s*\K\d+' || echo "0") + passed=$(cat "$dir/test_stats.json" | grep -oP '"passed":\s*\K\d+' || echo "0") + failed=$(cat "$dir/test_stats.json" | grep -oP '"failed":\s*\K\d+' || echo "0") + skipped=$(cat "$dir/test_stats.json" | grep -oP '"skipped":\s*\K\d+' || echo "0") + + GROUP_STATS[$group]="$result,$total,$passed,$failed,$skipped" + + OVERALL_TOTAL=$((OVERALL_TOTAL + total)) + OVERALL_PASSED=$((OVERALL_PASSED + passed)) + OVERALL_FAILED=$((OVERALL_FAILED + failed)) + OVERALL_SKIPPED=$((OVERALL_SKIPPED + skipped)) + + if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then + GROUPS_PASSED=$((GROUPS_PASSED + 1)) + else + GROUPS_FAILED=$((GROUPS_FAILED + 1)) + FAILED_GROUP_NAMES="${FAILED_GROUP_NAMES}${group} " + fi + fi + done + + echo "### ${os_label}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [ $GROUPS_FAILED -eq 0 ]; then + echo "All ${GROUPS_PASSED} test groups passed" >> $GITHUB_STEP_SUMMARY + else + echo "${GROUPS_FAILED} of $((GROUPS_PASSED + GROUPS_FAILED)) test groups failed" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "- Total Tests: $OVERALL_TOTAL" >> $GITHUB_STEP_SUMMARY + echo "- Passed: $OVERALL_PASSED" >> $GITHUB_STEP_SUMMARY + echo "- Failed: $OVERALL_FAILED" >> $GITHUB_STEP_SUMMARY + echo "- Skipped: $OVERALL_SKIPPED" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "| Test Group | Status | Passed | Failed | Skipped | Total |" >> $GITHUB_STEP_SUMMARY + echo "|------------|--------|-------:|-------:|--------:|------:|" >> $GITHUB_STEP_SUMMARY + + for group in $(echo "${!GROUP_STATS[@]}" | tr ' ' '\n' | sort); do + IFS=',' read -r result total passed failed skipped <<< "${GROUP_STATS[$group]}" if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then - GROUPS_PASSED=$((GROUPS_PASSED + 1)) + status="PASS" else - GROUPS_FAILED=$((GROUPS_FAILED + 1)) - FAILED_GROUP_NAMES="${FAILED_GROUP_NAMES}${group} " + status="FAIL" fi + echo "| $group | $status | $passed | $failed | $skipped | $total |" >> $GITHUB_STEP_SUMMARY + done + + echo "" >> $GITHUB_STEP_SUMMARY + + if [ $GROUPS_FAILED -gt 0 ]; then + echo "::error::${os_label}: ${GROUPS_FAILED} test group(s) failed: ${FAILED_GROUP_NAMES}" + return 1 fi - done - - # Overall summary - echo "### Overall Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - if [ $GROUPS_FAILED -eq 0 ]; then - echo "✅ **All ${GROUPS_PASSED} test groups passed**" >> $GITHUB_STEP_SUMMARY - else - echo "❌ **${GROUPS_FAILED} of $((GROUPS_PASSED + GROUPS_FAILED)) test groups failed**" >> $GITHUB_STEP_SUMMARY - fi - echo "" >> $GITHUB_STEP_SUMMARY - echo "- Total Tests: $OVERALL_TOTAL" >> $GITHUB_STEP_SUMMARY - echo "- Passed: $OVERALL_PASSED" >> $GITHUB_STEP_SUMMARY - echo "- Failed: $OVERALL_FAILED" >> $GITHUB_STEP_SUMMARY - echo "- Skipped: $OVERALL_SKIPPED" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Detailed table - echo "### Test Results by Group" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "| Test Group | Status | Passed | Failed | Skipped | Total |" >> $GITHUB_STEP_SUMMARY - echo "|------------|--------|-------:|-------:|--------:|------:|" >> $GITHUB_STEP_SUMMARY - - for group in $(echo "${!GROUP_STATS[@]}" | tr ' ' '\n' | sort); do - IFS=',' read -r result total passed failed skipped <<< "${GROUP_STATS[$group]}" - if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then - status="✅ PASS" - else - status="❌ FAIL" - fi - echo "| $group | $status | $passed | $failed | $skipped | $total |" >> $GITHUB_STEP_SUMMARY - done - - echo "" >> $GITHUB_STEP_SUMMARY - - # Check if any group failed - if [ $GROUPS_FAILED -gt 0 ]; then - echo "::error::${GROUPS_FAILED} test group(s) failed: ${FAILED_GROUP_NAMES}" - exit 1 - fi + return 0 + } + + EXIT_CODE=0 + + # Ubuntu results (test-results-<group>, excluding test-results-rocky9-*) + generate_os_summary "Ubuntu 22.04" "test-results-[!r]*" || EXIT_CODE=1 + + # Rocky 9 results + generate_os_summary "Rocky 9" "test-results-rocky9-*" || EXIT_CODE=1 + + exit $EXIT_CODE diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberrry.sh b/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh similarity index 69% rename from ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberrry.sh rename to ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh index e1ae6e26..cd4c4758 100755 --- a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberrry.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh @@ -1,3 +1,4 @@ +#!/bin/bash # -------------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,8 +18,14 @@ # permissions and limitations under the License. # # -------------------------------------------------------------------- +# Build Cloudberry from source — works on both Ubuntu and Rocky/RHEL + # Install sudo & git -sudo apt update && sudo apt install -y sudo git +if command -v apt-get >/dev/null 2>&1; then + sudo apt update && sudo apt install -y sudo git +elif command -v dnf >/dev/null 2>&1; then + sudo dnf install -y sudo git +fi # Required configuration ## Add Cloudberry environment setup to .bashrc @@ -57,44 +64,85 @@ EOF ulimit -a # Install basic system packages -sudo apt update -sudo apt install -y bison \ - bzip2 \ - cmake \ - curl \ - flex \ - gcc \ - g++ \ - iproute2 \ - iputils-ping \ - language-pack-en \ - locales \ - libapr1-dev \ - libbz2-dev \ - libcurl4-gnutls-dev \ - libevent-dev \ - libkrb5-dev \ - libipc-run-perl \ - libldap2-dev \ - libpam0g-dev \ - libprotobuf-dev \ - libreadline-dev \ - libssl-dev \ - libuv1-dev \ - liblz4-dev \ - libxerces-c-dev \ - libxml2-dev \ - libyaml-dev \ - libzstd-dev \ - libperl-dev \ - make \ - pkg-config \ - protobuf-compiler \ - python3-dev \ - python3-pip \ - python3-setuptools \ - rsync \ - libsnappy-dev +if command -v apt-get >/dev/null 2>&1; then + sudo apt update + sudo apt install -y bison \ + bzip2 \ + cmake \ + curl \ + flex \ + gcc \ + g++ \ + iproute2 \ + iputils-ping \ + language-pack-en \ + locales \ + libapr1-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libevent-dev \ + libkrb5-dev \ + libipc-run-perl \ + libldap2-dev \ + libpam0g-dev \ + libprotobuf-dev \ + libreadline-dev \ + libssl-dev \ + libuv1-dev \ + liblz4-dev \ + libxerces-c-dev \ + libxml2-dev \ + libyaml-dev \ + libzstd-dev \ + libperl-dev \ + make \ + pkg-config \ + protobuf-compiler \ + python3-dev \ + python3-pip \ + python3-setuptools \ + rsync \ + libsnappy-dev +elif command -v dnf >/dev/null 2>&1; then + sudo dnf install -y \ + bison \ + bzip2 \ + cmake \ + curl \ + flex \ + gcc \ + gcc-c++ \ + iproute \ + iputils \ + glibc-langpack-en \ + glibc-locale-source \ + apr-devel \ + bzip2-devel \ + libcurl-devel \ + libevent-devel \ + krb5-devel \ + perl-IPC-Run \ + openldap-devel \ + pam-devel \ + protobuf-devel \ + readline-devel \ + openssl-devel \ + libuv-devel \ + lz4-devel \ + xerces-c-devel \ + libxml2-devel \ + libyaml-devel \ + libzstd-devel \ + perl-devel \ + make \ + pkgconfig \ + protobuf-compiler \ + python3-devel \ + python3-pip \ + python3-setuptools \ + rsync \ + snappy-devel +fi # Continue as gpadmin user @@ -149,4 +197,4 @@ source /usr/local/cloudberry-db/cloudberry-env.sh make create-demo-cluster -C ~/workspace/cloudberry source ~/workspace/cloudberry/gpAux/gpdemo/gpdemo-env.sh psql -P pager=off template1 -c 'SELECT * from gp_segment_configuration' -psql template1 -c 'SELECT version()' \ No newline at end of file +psql template1 -c 'SELECT version()' diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh b/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh similarity index 80% rename from ci/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh rename to ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh index b48cbc9e..08f7c765 100755 --- a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh @@ -1,3 +1,4 @@ +#!/bin/bash # -------------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,18 +18,28 @@ # permissions and limitations under the License. # # -------------------------------------------------------------------- -case "$(uname -m)" in - aarch64|arm64) JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-arm64} ;; - x86_64|amd64) JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-amd64} ;; - *) JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-amd64} ;; -esac +# Build and install PXF — works on both Ubuntu and Rocky/RHEL + +# Auto-detect Java 11 path +if [ -d /usr/lib/jvm/java-11-openjdk-amd64 ]; then + JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-amd64} +elif [ -d /usr/lib/jvm/java-11-openjdk-arm64 ]; then + JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-arm64} +else + JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk} +fi export PATH=$JAVA_HOME/bin:$PATH export GPHOME=/usr/local/cloudberry-db source /usr/local/cloudberry-db/cloudberry-env.sh export PATH=$GPHOME/bin:$PATH -sudo apt update -sudo apt install -y openjdk-11-jdk maven +# Install Java 11 JDK and Maven +if command -v apt-get >/dev/null 2>&1; then + sudo apt update + sudo apt install -y openjdk-11-jdk maven +elif command -v dnf >/dev/null 2>&1; then + sudo dnf install -y java-11-openjdk-devel maven +fi cd /home/gpadmin/workspace/cloudberry-pxf diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh similarity index 66% rename from ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh rename to ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh index f3ca1bee..832e5067 100755 --- a/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh @@ -26,41 +26,66 @@ die() { log "ERROR $*"; exit 1; } ROOT_DIR=/home/gpadmin/workspace REPO_DIR=${ROOT_DIR}/cloudberry-pxf GPHD_ROOT=${ROOT_DIR}/singlecluster -PXF_SCRIPTS=${REPO_DIR}/ci/docker/pxf-cbdb-dev/ubuntu/script -source "${PXF_SCRIPTS}/utils.sh" +COMMON_SCRIPTS=${REPO_DIR}/ci/docker/pxf-cbdb-dev/common/script +source "${COMMON_SCRIPTS}/utils.sh" HADOOP_ROOT=${GPHD_ROOT}/hadoop HIVE_ROOT=${GPHD_ROOT}/hive HBASE_ROOT=${GPHD_ROOT}/hbase ZOOKEEPER_ROOT=${GPHD_ROOT}/zookeeper -JAVA_11_ARM=/usr/lib/jvm/java-11-openjdk-arm64 -JAVA_11_AMD=/usr/lib/jvm/java-11-openjdk-amd64 -JAVA_8_ARM=/usr/lib/jvm/java-8-openjdk-arm64 -JAVA_8_AMD=/usr/lib/jvm/java-8-openjdk-amd64 +# -------------------------------------------------------------------- +# OS detection: "deb" (Ubuntu/Debian) or "rpm" (Rocky/RHEL/CentOS) +# -------------------------------------------------------------------- +if command -v apt-get >/dev/null 2>&1; then + OS_FAMILY="deb" +else + OS_FAMILY="rpm" +fi detect_java_paths() { - case "$(uname -m)" in - aarch64|arm64) JAVA_BUILD=${JAVA_11_ARM}; JAVA_HADOOP=${JAVA_8_ARM} ;; - x86_64|amd64) JAVA_BUILD=${JAVA_11_AMD}; JAVA_HADOOP=${JAVA_8_AMD} ;; - *) JAVA_BUILD=${JAVA_11_ARM}; JAVA_HADOOP=${JAVA_8_ARM} ;; - esac + if [ "$OS_FAMILY" = "deb" ]; then + case "$(uname -m)" in + aarch64|arm64) JAVA_BUILD=/usr/lib/jvm/java-11-openjdk-arm64; JAVA_HADOOP=/usr/lib/jvm/java-8-openjdk-arm64 ;; + *) JAVA_BUILD=/usr/lib/jvm/java-11-openjdk-amd64; JAVA_HADOOP=/usr/lib/jvm/java-8-openjdk-amd64 ;; + esac + else + JAVA_BUILD=/usr/lib/jvm/java-11-openjdk + JAVA_HADOOP=/usr/lib/jvm/java-1.8.0-openjdk + fi export JAVA_BUILD JAVA_HADOOP } setup_locale_and_packages() { log "install base packages and locales" - sudo apt-get update - sudo apt-get install -y wget lsb-release locales maven unzip openssh-server iproute2 sudo \ - openjdk-11-jre-headless openjdk-8-jre-headless - sudo locale-gen en_US.UTF-8 ru_RU.CP1251 ru_RU.UTF-8 - sudo update-locale LANG=en_US.UTF-8 + if [ "$OS_FAMILY" = "deb" ]; then + sudo apt-get update + sudo apt-get install -y wget lsb-release locales maven unzip openssh-server iproute2 sudo \ + openjdk-11-jre-headless openjdk-8-jre-headless + sudo locale-gen en_US.UTF-8 ru_RU.CP1251 ru_RU.UTF-8 + sudo update-locale LANG=en_US.UTF-8 + else + sudo dnf install -y wget maven unzip openssh-server iproute sudo \ + java-11-openjdk-headless java-1.8.0-openjdk-headless \ + glibc-langpack-en glibc-locale-source + sudo localedef -c -i en_US -f UTF-8 en_US.UTF-8 || true + sudo localedef -c -i ru_RU -f UTF-8 ru_RU.UTF-8 || true + fi sudo localedef -c -i ru_RU -f CP1251 ru_RU.CP1251 || true export LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 } setup_ssh() { log "configure ssh" + # Rocky 9 / RHEL 9 enforces system-wide crypto-policies that override sshd_config + # settings for algorithm negotiation. The automation test framework uses the + # Ganymed SSH-2 (ch.ethz.ssh2) library which only supports older KEX algorithms + # (diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, etc.). + # Downgrade to the LEGACY crypto policy so sshd accepts these algorithms. + if [ "$OS_FAMILY" = "rpm" ] && command -v update-crypto-policies >/dev/null 2>&1; then + log "setting LEGACY crypto policy for SSH compatibility" + sudo update-crypto-policies --set LEGACY 2>/dev/null || true + fi sudo ssh-keygen -A sudo bash -c 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config' sudo mkdir -p /etc/ssh/sshd_config.d @@ -69,7 +94,11 @@ KexAlgorithms +diffie-hellman-group-exchange-sha1,diffie-hellman-group14-sha1,di HostKeyAlgorithms +ssh-rsa,ssh-dss PubkeyAcceptedAlgorithms +ssh-rsa,ssh-dss EOF' - sudo usermod -a -G sudo gpadmin + if [ "$OS_FAMILY" = "deb" ]; then + sudo usermod -a -G sudo gpadmin + else + sudo usermod -a -G wheel gpadmin 2>/dev/null || true + fi echo "gpadmin:cbdb@123" | sudo chpasswd echo "gpadmin ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers >/dev/null echo "root ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers >/dev/null @@ -84,7 +113,21 @@ EOF' ssh-keyscan -t rsa mdw cdw localhost 2>/dev/null > /home/gpadmin/.ssh/known_hosts || true sudo rm -rf /run/nologin sudo mkdir -p /var/run/sshd && sudo chmod 0755 /var/run/sshd - sudo /usr/sbin/sshd || die "Failed to start sshd" + # Ensure privilege separation user exists (required by Rocky 9 sshd) + id sshd &>/dev/null || sudo useradd -r -d /var/empty/sshd -s /sbin/nologin sshd 2>/dev/null || true + sudo mkdir -p /var/empty/sshd && sudo chmod 0755 /var/empty/sshd + sudo /usr/sbin/sshd -E /tmp/sshd.log || die "Failed to start sshd, check /tmp/sshd.log" + sleep 1 + if ! ss -tlnp | grep -q ':22 '; then + log "ERROR: sshd is not listening on port 22" + cat /tmp/sshd.log 2>/dev/null || true + sudo /usr/sbin/sshd -D -e & + sleep 1 + if ! ss -tlnp | grep -q ':22 '; then + die "sshd failed to bind to port 22" + fi + fi + log "sshd is running on port 22" } relax_pg_hba() { @@ -101,105 +144,81 @@ EOF fi } -install_cloudberry_from_deb() { - log "installing Cloudberry from .deb package" - local deb_file=$(find /tmp -name "apache-cloudberry-db*.deb" 2>/dev/null | head -1) - if [ -z "$deb_file" ]; then - die "No .deb package found in /tmp" +install_build_deps() { + if [ "$OS_FAMILY" = "deb" ]; then + sudo apt update && sudo apt install -y sudo git + sudo apt update + sudo apt install -y bison bzip2 cmake curl flex gcc g++ iproute2 iputils-ping \ + language-pack-en locales libapr1-dev libbz2-dev libcurl4-gnutls-dev libevent-dev \ + libkrb5-dev libipc-run-perl libldap2-dev libpam0g-dev libprotobuf-dev libreadline-dev \ + libssl-dev libuv1-dev liblz4-dev libxerces-c-dev libxml2-dev libyaml-dev libzstd-dev \ + libperl-dev make pkg-config protobuf-compiler python3-dev python3-pip python3-setuptools \ + rsync libsnappy-dev + else + sudo dnf install -y sudo git + sudo dnf install -y --allowerasing bison bzip2 cmake curl flex gcc gcc-c++ iproute iputils \ + glibc-langpack-en glibc-locale-source apr-devel bzip2-devel libcurl-devel libevent-devel \ + krb5-devel perl-IPC-Run openldap-devel pam-devel protobuf-devel readline-devel \ + openssl-devel libuv-devel lz4-devel libxml2-devel libyaml-devel \ + libzstd-devel perl-devel make pkgconfig protobuf-compiler python3-devel python3-pip \ + python3-setuptools rsync snappy-devel fi +} - # Install sudo & git - sudo apt update && sudo apt install -y sudo git +install_cloudberry_from_package() { + log "installing Cloudberry from package" + + local pkg_file="" + if [ "$OS_FAMILY" = "deb" ]; then + pkg_file=$(find /tmp -name "apache-cloudberry-db*.deb" 2>/dev/null | head -1) + [ -z "$pkg_file" ] && die "No .deb package found in /tmp" + else + pkg_file=$(find /tmp -name "apache-cloudberry-db*.rpm" 2>/dev/null | head -1) + [ -z "$pkg_file" ] && die "No .rpm package found in /tmp" + fi + + install_build_deps # Required configuration - ## Add Cloudberry environment setup to .bashrc echo -e '\n# Add Cloudberry entries if [ -f /usr/local/cloudberry-db/cloudberry-env.sh ]; then source /usr/local/cloudberry-db/cloudberry-env.sh fi - ## US English with UTF-8 character encoding export LANG=en_US.UTF-8 ' >> /home/gpadmin/.bashrc - ## Set up SSH for passwordless access + mkdir -p /home/gpadmin/.ssh if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then ssh-keygen -t rsa -b 2048 -C 'apache-cloudberry-dev' -f /home/gpadmin/.ssh/id_rsa -N "" fi cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys - ## Set proper SSH directory permissions chmod 700 /home/gpadmin/.ssh chmod 600 /home/gpadmin/.ssh/authorized_keys chmod 644 /home/gpadmin/.ssh/id_rsa.pub -# Configure system settings sudo tee /etc/security/limits.d/90-db-limits.conf << 'EOF' -## Core dump file size limits for gpadmin gpadmin soft core unlimited gpadmin hard core unlimited -## Open file limits for gpadmin gpadmin soft nofile 524288 gpadmin hard nofile 524288 -## Process limits for gpadmin gpadmin soft nproc 131072 gpadmin hard nproc 131072 EOF - # Verify resource limits ulimit -a - # Install basic system packages - sudo apt update - sudo apt install -y bison \ - bzip2 \ - cmake \ - curl \ - flex \ - gcc \ - g++ \ - iproute2 \ - iputils-ping \ - language-pack-en \ - locales \ - libapr1-dev \ - libbz2-dev \ - libcurl4-gnutls-dev \ - libevent-dev \ - libkrb5-dev \ - libipc-run-perl \ - libldap2-dev \ - libpam0g-dev \ - libprotobuf-dev \ - libreadline-dev \ - libssl-dev \ - libuv1-dev \ - liblz4-dev \ - libxerces-c-dev \ - libxml2-dev \ - libyaml-dev \ - libzstd-dev \ - libperl-dev \ - make \ - pkg-config \ - protobuf-compiler \ - python3-dev \ - python3-pip \ - python3-setuptools \ - rsync \ - libsnappy-dev - - - # Continue as gpadmin user - - - # Prepare the build environment for Apache Cloudberry sudo rm -rf /usr/local/cloudberry-db sudo chmod a+w /usr/local mkdir -p /usr/local/cloudberry-db sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db - sudo dpkg -i "$deb_file" || sudo apt-get install -f -y - log "Cloudberry installed from $deb_file" - + if [ "$OS_FAMILY" = "deb" ]; then + sudo dpkg -i "$pkg_file" || sudo apt-get install -f -y + else + sudo rpm -Uvh --force "$pkg_file" || sudo dnf install -y "$pkg_file" + fi + log "Cloudberry installed from $pkg_file" + # Initialize and start Cloudberry cluster source /usr/local/cloudberry-db/cloudberry-env.sh make create-demo-cluster -C ~/workspace/cloudberry || { @@ -219,30 +238,30 @@ build_cloudberry() { rm -rf /home/gpadmin/workspace/cloudberry/gpAux/gpdemo/datadirs rm -f /tmp/.s.PGSQL.700* find "${ROOT_DIR}" -not -path '*/.git/*' -exec sudo chown gpadmin:gpadmin {} + 2>/dev/null || true - "${PXF_SCRIPTS}/build_cloudberrry.sh" + "${COMMON_SCRIPTS}/build_cloudberrry.sh" } setup_cloudberry() { - # Auto-detect: if deb exists, install it; otherwise build from source - if [ -f /tmp/apache-cloudberry-db*.deb ]; then - log "detected .deb package, using fast install" - install_cloudberry_from_deb - elif [ "${CLOUDBERRY_USE_DEB:-}" = "true" ]; then - die "CLOUDBERRY_USE_DEB=true but no .deb found in /tmp" + # Auto-detect: if package exists, install it; otherwise build from source + if ls /tmp/apache-cloudberry-db*.deb 1>/dev/null 2>&1 || ls /tmp/apache-cloudberry-db*.rpm 1>/dev/null 2>&1; then + log "detected package, using fast install" + install_cloudberry_from_package + elif [ "${CLOUDBERRY_USE_PKG:-}" = "true" ]; then + die "CLOUDBERRY_USE_PKG=true but no .deb/.rpm found in /tmp" else - log "no .deb found, building from source (local dev mode)" + log "no package found, building from source (local dev mode)" build_cloudberry fi } build_pxf() { log "build PXF" - "${PXF_SCRIPTS}/build_pxf.sh" + "${COMMON_SCRIPTS}/build_pxf.sh" } configure_pxf() { log "configure PXF" - source "${PXF_SCRIPTS}/pxf-env.sh" + source "${COMMON_SCRIPTS}/pxf-env.sh" export PATH="$PXF_HOME/bin:$PATH" export PXF_JVM_OPTS="-Xmx512m -Xms256m" export PXF_HOST=localhost @@ -318,7 +337,7 @@ EOF # Configure S3 settings mkdir -p "$PXF_BASE/servers/s3" "$PXF_HOME/servers/s3" - + for s3_site in "$PXF_BASE/servers/s3/s3-site.xml" "$PXF_BASE/servers/default/s3-site.xml" "$PXF_HOME/servers/s3/s3-site.xml"; do mkdir -p "$(dirname "$s3_site")" cat > "$s3_site" <<'EOF' @@ -364,6 +383,63 @@ EOF } +wait_for_datanode() { + log "waiting for HDFS DataNode to become available..." + local max_attempts=2 + for _attempt in $(seq 1 ${max_attempts}); do + local dn_ready=false + # Wait up to 90s (45 tries * 2s) for DataNode to register + for _dn_try in $(seq 1 45); do + if hdfs dfsadmin -report 2>/dev/null | grep -q "Live datanodes.*[1-9]"; then + dn_ready=true + break + fi + sleep 2 + done + + if [ "${dn_ready}" = "true" ]; then + log "HDFS DataNode is available" + return 0 + fi + + # DataNode didn't come up; diagnose and attempt restart + log "DataNode not available after 90s (attempt ${_attempt}/${max_attempts})" + log "--- DataNode diagnostic info ---" + # Check if DataNode process is alive + if command -v jps >/dev/null 2>&1; then + log "JPS output: $(jps 2>&1)" + fi + if pgrep -f "proc_datanode" >/dev/null 2>&1 || pgrep -f "datanode" >/dev/null 2>&1; then + log "DataNode process exists but not yet registered with NameNode" + else + log "DataNode process is NOT running" + fi + # Show DataNode logs if available + local dn_log + dn_log=$(ls -t "${GPHD_ROOT}"/storage/logs/*datanode*.log 2>/dev/null | head -1) + if [ -n "${dn_log}" ]; then + log "Last 30 lines of DataNode log (${dn_log}):" + tail -30 "${dn_log}" 2>/dev/null || true + fi + log "HDFS report:" + hdfs dfsadmin -report 2>&1 | head -20 || true + log "--- end diagnostic ---" + + if [ "${_attempt}" -lt "${max_attempts}" ]; then + log "Attempting to restart DataNode..." + # Stop any zombie DataNode processes + pkill -f "proc_datanode" 2>/dev/null || true + sleep 2 + # Restart DataNode via the singlecluster script + "${GPHD_ROOT}/bin/hadoop-datanode.sh" start 0 2>&1 || true + "${HADOOP_ROOT}/sbin/hadoop-daemon.sh" --config "${GPHD_ROOT}/storage/hadoop/datanode0/etc/hadoop" start datanode 2>&1 || true + log "DataNode restart issued, waiting again..." + fi + done + + die "HDFS DataNode failed to start after ${max_attempts} attempts. Tez upload will fail without a running DataNode." +} + prepare_hadoop_stack() { log "prepare Hadoop/Hive/HBase stack" export JAVA_HOME="${JAVA_HADOOP}" @@ -386,12 +462,19 @@ prepare_hadoop_stack() { if pgrep -f HiveServer2 >/dev/null 2>&1; then "${GPHD_ROOT}/bin/hive-service.sh" hiveserver2 stop || true fi + log "JAVA_HOME=${JAVA_HOME} JAVA_HADOOP=${JAVA_HADOOP}" + log "java check: $(ls -la ${JAVA_HOME}/bin/java 2>&1)" if [ ! -d "${GPHD_ROOT}/storage/hadoop/dfs/name/current" ]; then - ${GPHD_ROOT}/bin/init-gphd.sh + log "initializing HDFS namenode..." + ${GPHD_ROOT}/bin/init-gphd.sh 2>&1 || log "init-gphd.sh failed with exit code $?" fi - if ! ${GPHD_ROOT}/bin/start-gphd.sh; then + log "starting HDFS/YARN/HBase via start-gphd.sh..." + if ! ${GPHD_ROOT}/bin/start-gphd.sh 2>&1; then log "start-gphd.sh returned non-zero (services may already be running), continue" fi + # Wait for HDFS DataNode to be ready before proceeding; Tez upload in + # start_hive_services will fail if no DataNode is accepting writes. + wait_for_datanode if ! ${GPHD_ROOT}/bin/start-zookeeper.sh; then log "start-zookeeper.sh returned non-zero (may already be running)" fi @@ -487,7 +570,7 @@ start_hive_services() { deploy_minio() { log "deploying MinIO" - bash "${PXF_SCRIPTS}/start_minio.bash" + bash "${COMMON_SCRIPTS}/start_minio.bash" } main() { diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh b/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh similarity index 81% rename from ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh rename to ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh index bd785ea1..81b23556 100755 --- a/ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh @@ -21,22 +21,25 @@ # Centralized environment for Cloudberry + PXF + Hadoop stack # -------------------------------------------------------------------- -# Architecture-aware Java selections +# Architecture-aware Java selections (auto-detect OS) # -------------------------------------------------------------------- -case "$(uname -m)" in - aarch64|arm64) - JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk-arm64} - JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk-arm64} - ;; - x86_64|amd64) - JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk-amd64} - JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk-amd64} - ;; - *) - JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk} - JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk} - ;; -esac +if [ -d /usr/lib/jvm/java-11-openjdk-amd64 ] || [ -d /usr/lib/jvm/java-11-openjdk-arm64 ]; then + # Debian/Ubuntu: paths include architecture suffix + case "$(uname -m)" in + aarch64|arm64) + JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk-arm64} + JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk-arm64} + ;; + *) + JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk-amd64} + JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk-amd64} + ;; + esac +else + # RHEL/Rocky: architecture-independent symlinks + JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk} + JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-1.8.0-openjdk} +fi # -------------------------------------------------------------------- # Core paths diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-test.sh b/ci/docker/pxf-cbdb-dev/common/script/pxf-test.sh similarity index 100% rename from ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-test.sh rename to ci/docker/pxf-cbdb-dev/common/script/pxf-test.sh diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh similarity index 100% rename from ci/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh rename to ci/docker/pxf-cbdb-dev/common/script/run_tests.sh diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/start_minio.bash b/ci/docker/pxf-cbdb-dev/common/script/start_minio.bash similarity index 100% rename from ci/docker/pxf-cbdb-dev/ubuntu/script/start_minio.bash rename to ci/docker/pxf-cbdb-dev/common/script/start_minio.bash diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/utils.sh b/ci/docker/pxf-cbdb-dev/common/script/utils.sh old mode 100644 new mode 100755 similarity index 89% rename from ci/docker/pxf-cbdb-dev/ubuntu/script/utils.sh rename to ci/docker/pxf-cbdb-dev/common/script/utils.sh index 786017fc..c055dd25 --- a/ci/docker/pxf-cbdb-dev/ubuntu/script/utils.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/utils.sh @@ -107,13 +107,26 @@ check_pxf() { fi } +# Auto-detect default Java 8 path based on OS +_detect_java8_default() { + if [ -d /usr/lib/jvm/java-8-openjdk-amd64 ]; then + echo "/usr/lib/jvm/java-8-openjdk-amd64" + elif [ -d /usr/lib/jvm/java-8-openjdk-arm64 ]; then + echo "/usr/lib/jvm/java-8-openjdk-arm64" + elif [ -d /usr/lib/jvm/java-1.8.0-openjdk ]; then + echo "/usr/lib/jvm/java-1.8.0-openjdk" + else + echo "/usr/lib/jvm/java-8-openjdk" + fi +} + health_check() { log "sanity check Hadoop/Hive/HBase/PXF" GPHD_ROOT=${GPHD_ROOT:-/home/gpadmin/workspace/singlecluster} HADOOP_ROOT=${HADOOP_ROOT:-${GPHD_ROOT}/hadoop} HBASE_ROOT=${HBASE_ROOT:-${GPHD_ROOT}/hbase} HIVE_ROOT=${HIVE_ROOT:-${GPHD_ROOT}/hive} - JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk-amd64} + JAVA_HADOOP=${JAVA_HADOOP:-$(_detect_java8_default)} export JAVA_HOME="${JAVA_HADOOP}" export PATH="$JAVA_HOME/bin:$HADOOP_ROOT/bin:$HIVE_ROOT/bin:$HBASE_ROOT/bin:$PATH" diff --git a/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml b/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml new file mode 100644 index 00000000..37738078 --- /dev/null +++ b/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml @@ -0,0 +1,46 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +services: +# hadoop + singlecluster: + build: + context: ../../../singlecluster + args: + BASE_IMAGE: apache/incubator-cloudberry:cbdb-build-rocky9-latest + image: pxf/singlecluster-rocky9:3 + container_name: pxf_singlecluster_rocky9 + hostname: cdw + + pxf-cbdb-dev: + image: pxf/singlecluster-rocky9:3 + container_name: pxf-cbdb-dev + hostname: mdw + depends_on: + - singlecluster + ports: + - "2222:22" + volumes: + - ../../../../../cloudberry-pxf:/home/gpadmin/workspace/cloudberry-pxf + - ../../../../../cloudberry:/home/gpadmin/workspace/cloudberry + command: ["tail", "-f", "/dev/null"] + +networks: + default: + name: pxf-cbdb-ci diff --git a/ci/docker/pxf-cbdb-dev/rocky9/script/build_cloudberry_rpm.sh b/ci/docker/pxf-cbdb-dev/rocky9/script/build_cloudberry_rpm.sh new file mode 100755 index 00000000..1d3e6888 --- /dev/null +++ b/ci/docker/pxf-cbdb-dev/rocky9/script/build_cloudberry_rpm.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +set -euo pipefail + +# Cloudberry RPM Package Build Script for Rocky 9 +CLOUDBERRY_VERSION="${CLOUDBERRY_VERSION:-99.0.0}" +CLOUDBERRY_BUILD="${CLOUDBERRY_BUILD:-1}" +INSTALL_PREFIX="${INSTALL_PREFIX:-/usr/local/cloudberry-db}" +WORKSPACE="${WORKSPACE:-$HOME/workspace}" +CLOUDBERRY_SRC="${WORKSPACE}/cloudberry" + +echo "=== Cloudberry RPM Package Build ===" +echo "Version: ${CLOUDBERRY_VERSION}" +echo "Build: ${CLOUDBERRY_BUILD}" +echo "Install Prefix: ${INSTALL_PREFIX}" +echo "Source: ${CLOUDBERRY_SRC}" + +# Clean previous installation +rm -rf "${INSTALL_PREFIX}" +mkdir -p "${INSTALL_PREFIX}" + +# Copy xerces-c shared libraries required by ORCA +mkdir -p "${INSTALL_PREFIX}/lib" +cp -v /usr/local/xerces-c/lib/libxerces-c.so \ + /usr/local/xerces-c/lib/libxerces-c-3.*.so \ + "${INSTALL_PREFIX}/lib/" + +# Build Cloudberry using official build scripts +export SRC_DIR="${CLOUDBERRY_SRC}" +export CPPFLAGS="${CPPFLAGS:-} -I/usr/local/xerces-c/include" +export LDFLAGS="${LDFLAGS:-} -L${INSTALL_PREFIX}/lib" +mkdir -p "${SRC_DIR}/build-logs" +cd "${CLOUDBERRY_SRC}" +./devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +./devops/build/automation/cloudberry/scripts/build-cloudberry.sh + +# Copy LICENSE +cp LICENSE "${INSTALL_PREFIX}/" + +# Create RPM build structure +RPM_BUILD_DIR="${WORKSPACE}/cloudberry-rpm" +mkdir -p "${RPM_BUILD_DIR}"/{BUILD,RPMS,SOURCES,SPECS,SRPMS} +RPM_INSTALL_ROOT="${RPM_BUILD_DIR}/BUILDROOT/apache-cloudberry-db-${CLOUDBERRY_VERSION}-${CLOUDBERRY_BUILD}.x86_64" +mkdir -p "${RPM_INSTALL_ROOT}${INSTALL_PREFIX}" + +# Copy installed files +cp -a "${INSTALL_PREFIX}"/* "${RPM_INSTALL_ROOT}${INSTALL_PREFIX}/" + +# Create spec file +cat > "${RPM_BUILD_DIR}/SPECS/cloudberry-db.spec" << EOF +Name: apache-cloudberry-db +Version: ${CLOUDBERRY_VERSION} +Release: ${CLOUDBERRY_BUILD}%{?dist} +Summary: Apache Cloudberry Database +License: Apache-2.0 +Group: Applications/Databases +AutoReqProv: no + +%description +Apache Cloudberry is a massively parallel processing (MPP) database +built on PostgreSQL for analytics and data warehousing. + +%install +mkdir -p %{buildroot}${INSTALL_PREFIX} +cp -a ${RPM_INSTALL_ROOT}${INSTALL_PREFIX}/* %{buildroot}${INSTALL_PREFIX}/ + +%files +${INSTALL_PREFIX} + +%post +if ! id -u gpadmin >/dev/null 2>&1; then + useradd -m -s /bin/bash gpadmin +fi +chown -R gpadmin:gpadmin ${INSTALL_PREFIX} +echo "Apache Cloudberry Database installed successfully" + +%clean +rm -rf %{buildroot} +EOF + +# Build RPM package +rpmbuild --define "_topdir ${RPM_BUILD_DIR}" -bb "${RPM_BUILD_DIR}/SPECS/cloudberry-db.spec" + +RPM_FILE=$(find "${RPM_BUILD_DIR}/RPMS" -name "*.rpm" | head -1) +echo "=== RPM Package Created ===" +ls -lh "${RPM_FILE}" +rpm -qpi "${RPM_FILE}" + +# Copy RPM to output directory +cp "${RPM_FILE}" "${RPM_BUILD_DIR}/" + +echo "=== Build Complete ===" diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh b/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh index fc117e88..991cb648 100755 --- a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh +++ b/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh @@ -37,36 +37,12 @@ echo "Source: ${CLOUDBERRY_SRC}" rm -rf "${INSTALL_PREFIX}" mkdir -p "${INSTALL_PREFIX}" -# Configure Cloudberry +# Build Cloudberry using official build scripts +export SRC_DIR="${CLOUDBERRY_SRC}" +mkdir -p "${SRC_DIR}/build-logs" cd "${CLOUDBERRY_SRC}" -./configure --prefix="${INSTALL_PREFIX}" \ - --disable-external-fts \ - --enable-gpcloud \ - --enable-ic-proxy \ - --enable-mapreduce \ - --enable-orafce \ - --enable-orca \ - --disable-pax \ - --disable-pxf \ - --enable-tap-tests \ - --with-gssapi \ - --with-ldap \ - --with-libxml \ - --with-lz4 \ - --with-pam \ - --with-perl \ - --with-pgport=5432 \ - --with-python \ - --with-pythonsrc-ext \ - --with-ssl=openssl \ - --with-uuid=e2fs \ - --with-includes=/usr/include/xercesc - -# Build and install -make -j$(nproc) -make -j$(nproc) -C contrib -make install -make install -C contrib +./devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +./devops/build/automation/cloudberry/scripts/build-cloudberry.sh # Copy LICENSE cp LICENSE "${INSTALL_PREFIX}/" diff --git a/ci/singlecluster/Dockerfile b/ci/singlecluster/Dockerfile index accf1d51..4d6bb655 100644 --- a/ci/singlecluster/Dockerfile +++ b/ci/singlecluster/Dockerfile @@ -17,15 +17,24 @@ # permissions and limitations under the License. # # -------------------------------------------------------------------- -FROM apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest - -ENV DEBIAN_FRONTEND noninteractive - -RUN sudo apt-get update && \ - sudo apt-get install -y --no-install-recommends \ - curl ca-certificates \ - openjdk-8-jdk-headless \ - openjdk-11-jdk-headless +ARG BASE_IMAGE=apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest +FROM ${BASE_IMAGE} + +# Install Java 8 & 11: auto-detect OS package manager +RUN if command -v apt-get >/dev/null 2>&1; then \ + export DEBIAN_FRONTEND=noninteractive && \ + sudo apt-get update && \ + sudo apt-get install -y --no-install-recommends \ + curl ca-certificates \ + openjdk-8-jdk-headless \ + openjdk-11-jdk-headless; \ + elif command -v dnf >/dev/null 2>&1; then \ + sudo dnf install -y --allowerasing \ + curl ca-certificates \ + java-1.8.0-openjdk-devel \ + java-11-openjdk-devel && \ + sudo dnf clean all; \ + fi # TODO: update hive to support java 11+ ENV HADOOP_VERSION=3.1.2 @@ -60,36 +69,36 @@ ENV ZOOKEEPER_ROOT=$GPHD_ROOT/zookeeper ENV TEZ_ROOT=$GPHD_ROOT/tez RUN mkdir -p $HADOOP_ROOT && \ - curl -fSL "$HADOOP_URL" -o hadoop.tar.gz && \ - echo "$HADOOP_SHA512 hadoop.tar.gz" | sha512sum -c && \ - tar xvf hadoop.tar.gz -C $HADOOP_ROOT --strip-components 1 --exclude="share/doc/*" --exclude="*-sources.jar" && \ - rm hadoop.tar.gz && \ + curl -fSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz && \ + echo "$HADOOP_SHA512 /tmp/hadoop.tar.gz" | sha512sum -c && \ + tar xvf /tmp/hadoop.tar.gz -C $HADOOP_ROOT --strip-components 1 --exclude="share/doc/*" --exclude="*-sources.jar" && \ + rm /tmp/hadoop.tar.gz && \ curl -fSL "https://repo1.maven.org/maven2/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar"; \ -o $HADOOP_ROOT/share/hadoop/common/lib/javax.activation-api-1.2.0.jar RUN mkdir -p $HIVE_ROOT && \ - curl -fSL $HIVE_URL -o hive.tar.gz && \ - echo "$HIVE_SHA256 hive.tar.gz" | sha256sum -c && \ - tar xvf hive.tar.gz -C $HIVE_ROOT --strip-components 1 && \ - rm hive.tar.gz + curl -fSL $HIVE_URL -o /tmp/hive.tar.gz && \ + echo "$HIVE_SHA256 /tmp/hive.tar.gz" | sha256sum -c && \ + tar xvf /tmp/hive.tar.gz -C $HIVE_ROOT --strip-components 1 && \ + rm /tmp/hive.tar.gz RUN mkdir -p $ZOOKEEPER_ROOT && \ - curl -fSL $ZOOKEEPER_URL -o zookeeper.tar.gz && \ - echo "$ZOOKEEPER_SHA512 zookeeper.tar.gz" | sha512sum -c && \ - tar xvf zookeeper.tar.gz -C $ZOOKEEPER_ROOT --strip-components 1 --exclude="docs/*" && \ - rm zookeeper.tar.gz + curl -fSL $ZOOKEEPER_URL -o /tmp/zookeeper.tar.gz && \ + echo "$ZOOKEEPER_SHA512 /tmp/zookeeper.tar.gz" | sha512sum -c && \ + tar xvf /tmp/zookeeper.tar.gz -C $ZOOKEEPER_ROOT --strip-components 1 --exclude="docs/*" && \ + rm /tmp/zookeeper.tar.gz RUN mkdir -p $HBASE_ROOT && \ - curl -fSL "$HBASE_URL" -o hbase.tar.gz && \ - echo "$HBASE_SHA512 hbase.tar.gz" | sha512sum -c && \ - tar xvf hbase.tar.gz -C $HBASE_ROOT --strip-components 1 --exclude="docs/*" --exclude="lib/*-tests.jar" --exclude="lib/shaded-clients" && \ - rm hbase.tar.gz + curl -fSL "$HBASE_URL" -o /tmp/hbase.tar.gz && \ + echo "$HBASE_SHA512 /tmp/hbase.tar.gz" | sha512sum -c && \ + tar xvf /tmp/hbase.tar.gz -C $HBASE_ROOT --strip-components 1 --exclude="docs/*" --exclude="lib/*-tests.jar" --exclude="lib/shaded-clients" && \ + rm /tmp/hbase.tar.gz RUN mkdir -p $TEZ_ROOT && \ - curl -fSL "$TEZ_URL" -o tez.tar.gz && \ - echo "$TEZ_SHA512 tez.tar.gz" | sha512sum -c && \ - tar xvf tez.tar.gz -C $TEZ_ROOT --strip-components 1 && \ - rm tez.tar.gz + curl -fSL "$TEZ_URL" -o /tmp/tez.tar.gz && \ + echo "$TEZ_SHA512 /tmp/tez.tar.gz" | sha512sum -c && \ + tar xvf /tmp/tez.tar.gz -C $TEZ_ROOT --strip-components 1 && \ + rm /tmp/tez.tar.gz # Install Go (required by PXF). Pick archive based on architecture (amd64/arm64). ARG TARGETARCH --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
