This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4c1c6cf630 HDDS-12714. Create acceptance test framework for debug and
repair tools (#8510)
4c1c6cf630 is described below
commit 4c1c6cf6302afddc996bffccd4b0cf3a0b2faa07
Author: Zita Dombi <[email protected]>
AuthorDate: Tue May 27 19:47:55 2025 +0200
HDDS-12714. Create acceptance test framework for debug and repair tools
(#8510)
---
.../dist/src/main/compose/ozonesecure-ha/.env | 1 +
.../main/compose/ozonesecure-ha/debug-tools.yaml | 162 +++++++++++++++++++++
.../compose/ozonesecure-ha/test-debug-tools.sh | 55 +++++++
.../smoketest/debug/ozone-debug-tests-ec3-2.robot | 9 +-
.../smoketest/debug/ozone-debug-tests-ec6-3.robot | 5 +-
.../main/smoketest/debug/ozone-debug-tests.robot | 9 +-
.../src/main/smoketest/debug/ozone-debug.robot | 5 +-
7 files changed, 235 insertions(+), 11 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
index c3a2c5329a..df30e9989a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
@@ -20,4 +20,5 @@ HADOOP_VERSION=${hadoop.version}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
+OZONE_VOLUME=./data
OZONE_OPTS=
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml
new file mode 100644
index 0000000000..e07b41ad89
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml
@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+x-debug-tools-config:
+ &common-env-file
+ dns_search: .
+ image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+ env_file:
+ - ./docker-config
+
+x-volumes:
+ - &keytabs ../_keytabs:/etc/security/keytabs
+ - &krb5conf ./krb5.conf:/etc/krb5.conf
+ - &ozone-dir ../..:/opt/hadoop
+ - &transformation
../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+
+services:
+ kdc:
+ volumes:
+ - *keytabs
+ - *ozone-dir
+ kms:
+ volumes:
+ - ${OZONE_VOLUME}/kms:/data
+ - *keytabs
+ - *krb5conf
+ - *transformation
+ datanode1:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/dn1:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ datanode2:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/dn2:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ datanode3:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/dn3:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ datanode4:
+ <<: *common-env-file
+ ports:
+ - 9870:9999
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ extra_hosts:
+ - "scm1.org=172.25.0.116"
+ - "scm2.org=172.25.0.117"
+ - "scm3.org=172.25.0.118"
+ - "recon=172.25.0.115"
+ environment:
+ WAITFOR: scm3.org:9894
+ OZONE_OPTS:
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.105
+ volumes:
+ - ${OZONE_VOLUME}/dn4:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ datanode5:
+ <<: *common-env-file
+ ports:
+ - 9872:9999
+ command: [ "/opt/hadoop/bin/ozone","datanode" ]
+ extra_hosts:
+ - "scm1.org=172.25.0.116"
+ - "scm2.org=172.25.0.117"
+ - "scm3.org=172.25.0.118"
+ - "recon=172.25.0.115"
+ environment:
+ WAITFOR: scm3.org:9894
+ OZONE_OPTS:
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.106
+ volumes:
+ - ${OZONE_VOLUME}/dn5:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ om1:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/om1:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ om2:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/om2:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ om3:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/om3:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ s3g:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/s3g:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ scm1.org:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/scm1:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ scm2.org:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/scm2:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ scm3.org:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/scm3:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+ recon:
+ <<: *common-env-file
+ volumes:
+ - ${OZONE_VOLUME}/recon:/data
+ - *keytabs
+ - *krb5conf
+ - *ozone-dir
+
+
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh
new file mode 100644
index 0000000000..c54c17425b
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:HA-secure
+
+set -u -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=true
+export OM_SERVICE_ID=omservice
+export SCM=scm1.org
+export COMPOSE_FILE=docker-compose.yaml:debug-tools.yaml
+export OZONE_DIR=/opt/hadoop
+
+: "${OZONE_VOLUME_OWNER:=}"
+: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
+
+export OZONE_VOLUME
+
+# Clean up saved internal state from each container's volume for the next run.
+rm -rf "${OZONE_VOLUME}"
+mkdir -p
"${OZONE_VOLUME}"/{dn1,dn2,dn3,dn4,dn5,om1,om2,om3,scm1,scm2,scm3,recon,s3g,kms}
+
+if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
+ current_user=$(whoami)
+ if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then
+ chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \
+ || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}"
+ fi
+fi
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+execute_robot_test ${SCM} kinit.robot
+
+source "$COMPOSE_DIR/../common/replicas-test.sh"
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot
index 0c310f62e8..e10e03443d 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot
@@ -29,17 +29,18 @@ ${BUCKET} cli-debug-bucket
${TESTFILE} testfile
${EC_DATA} 3
${EC_PARITY} 2
+${OM_SERVICE_ID} %{OM_SERVICE_ID}
*** Keywords ***
Create Volume Bucket
- Execute ozone sh volume create o3://om/${VOLUME}
- Execute ozone sh bucket create o3://om/${VOLUME}/${BUCKET}
+ Execute ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME}
+ Execute ozone sh bucket create
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}
Create EC key
[arguments] ${bs} ${count}
Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs}
count=${count}
- Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/testfile
${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
+ Execute ozone sh key put
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r
rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
*** Test Cases ***
0 data block
@@ -87,5 +88,5 @@ Create EC key
Test ozone debug replicas chunk-info
Create EC key 1048576 3
- ${count} = Execute ozone debug replicas chunk-info
o3://om/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file |
test("\\\\.block$")) | .file] | length'
+ ${count} = Execute ozone debug replicas chunk-info
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] |
select(.file | test("\\\\.block$")) | .file] | length'
Should Be Equal As Integers ${count} 5
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot
index 9c83cad697..c3b8c7ad1a 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot
@@ -28,13 +28,14 @@ ${BUCKET} cli-debug-bucket
${TESTFILE} testfile
${EC_DATA} 6
${EC_PARITY} 3
+${OM_SERVICE_ID} %{OM_SERVICE_ID}
*** Keywords ***
Create EC key
[arguments] ${bs} ${count}
Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs}
count=${count}
- Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/testfile
${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
+ Execute ozone sh key put
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r
rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
*** Test Cases ***
0 data block
@@ -95,5 +96,5 @@ Create EC key
Test ozone debug replicas chunk-info
Create EC key 1048576 6
- ${count} = Execute ozone debug replicas chunk-info
o3://om/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file |
test("\\\\.block$")) | .file] | length'
+ ${count} = Execute ozone debug replicas chunk-info
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] |
select(.file | test("\\\\.block$")) | .file] | length'
Should Be Equal As Integers ${count} 9
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
index 97fe36d2c5..532f931bb5 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
@@ -27,17 +27,18 @@ ${VOLUME} cli-debug-volume${PREFIX}
${BUCKET} cli-debug-bucket
${DEBUGKEY} debugKey
${TESTFILE} testfile
+${OM_SERVICE_ID} %{OM_SERVICE_ID}
*** Keywords ***
Write keys
- Execute ozone sh volume create o3://om/${VOLUME} --space-quota
100TB --namespace-quota 100
- Execute ozone sh bucket create o3://om/${VOLUME}/${BUCKET}
--space-quota 1TB
+ Execute ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME}
--space-quota 100TB --namespace-quota 100
+ Execute ozone sh bucket create
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET} --space-quota 1TB
Execute dd if=/dev/urandom of=${TEMP_DIR}/${TESTFILE}
bs=100000 count=15
- Execute ozone sh key put
o3://om/${VOLUME}/${BUCKET}/${TESTFILE} ${TEMP_DIR}/${TESTFILE}
+ Execute ozone sh key put
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} ${TEMP_DIR}/${TESTFILE}
*** Test Cases ***
Test ozone debug replicas verify checksums
- ${output} = Execute ozone debug replicas verify --checksums
o3://om/${VOLUME}/${BUCKET}/${TESTFILE} --output-dir ${TEMP_DIR}
+ ${output} = Execute ozone debug replicas verify --checksums
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} --output-dir ${TEMP_DIR}
${json} = Evaluate json.loads('''${output}''') json
# 'keys' array should be empty if all keys and their replicas passed
checksum verification
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
index b042f606cf..c8eb089247 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
@@ -18,9 +18,12 @@ Documentation Keyword definitions for Ozone Debug CLI
tests
Library Collections
Resource ../lib/os.robot
+*** Variables ***
+${OM_SERVICE_ID} %{OM_SERVICE_ID}
+
*** Keywords ***
Execute replicas verify checksums CLI tool
- Execute ozone debug
-Dozone.network.topology.aware.read=true replicas verify --checksums
--output-dir ${TEMP_DIR} o3://om/${VOLUME}/${BUCKET}/${TESTFILE}
+ Execute ozone debug
-Dozone.network.topology.aware.read=true replicas verify --checksums
--output-dir ${TEMP_DIR} o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE}
${directory} = Execute ls -d
${TEMP_DIR}/${VOLUME}_${BUCKET}_${TESTFILE}_*/ | tail -n 1
Directory Should Exist ${directory}
File Should Exist ${directory}/${TESTFILE}_manifest
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
