This is an automated email from the ASF dual-hosted git repository.
tejaskriya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new bb3d2872e9f HDDS-12890. Add acceptance robot tests for ozone debug
replicas verify (#8613)
bb3d2872e9f is described below
commit bb3d2872e9f7c25480e5407d7d9cbcee3de349dd
Author: Sarveksha Yeshavantha Raju
<[email protected]>
AuthorDate: Fri Jun 27 10:13:23 2025 +0530
HDDS-12890. Add acceptance robot tests for ozone debug replicas verify
(#8613)
---
.../dist/src/main/compose/common/replicas-test.sh | 64 +++++++------
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 1 -
.../src/main/compose/ozonesecure-ha/docker-config | 2 +
...-datanode.robot => block-existence-check.robot} | 17 ++--
...pt-block.robot => corrupt-block-checksum.robot} | 30 +++----
.../smoketest/debug/ozone-debug-keywords.robot | 63 +++++++++++++
.../debug/ozone-debug-stale-datanode.robot | 46 ----------
.../smoketest/debug/ozone-debug-tests-ec3-2.robot | 49 +---------
.../smoketest/debug/ozone-debug-tests-ec6-3.robot | 100 ---------------------
.../main/smoketest/debug/ozone-debug-tests.robot | 8 +-
.../src/main/smoketest/debug/ozone-debug.robot | 98 --------------------
...atanode.robot => stale-datanode-checksum.robot} | 25 +++---
12 files changed, 142 insertions(+), 361 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh
b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh
index b875bfcafc8..520ab8c4790 100755
--- a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh
@@ -21,33 +21,45 @@ volume="cli-debug-volume${prefix}"
bucket="cli-debug-bucket"
key="testfile"
+dn_container="ozonesecure-ha-datanode1-1"
+container_db_path="/data/hdds/hdds/"
+local_db_backup_path="${COMPOSE_DIR}/container_db_backup"
+mkdir -p "${local_db_backup_path}"
+
+echo "Taking a backup of container.db"
+docker exec "${dn_container}" find "${container_db_path}" -name "container.db"
| while read -r db; do
+ docker cp "${dn_container}:${db}" "${local_db_backup_path}/container.db"
+done
+
execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests.robot
-#TODO HDDS-12890: Add acceptance robot tests for ozone debug replicas verify
# get block locations for key
-#chunkinfo="${key}-blocks-${prefix}"
-#docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info
${volume}/${bucket}/${key}" > "$chunkinfo"
-#host="$(jq -r '.KeyLocations[0][0]["Datanode-HostName"]' ${chunkinfo})"
-#container="${host%%.*}"
-#
-## corrupt the first block of key on one of the datanodes
-#datafile="$(jq -r '.KeyLocations[0][0].Locations.files[0]' ${chunkinfo})"
-#docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}"
-#
-#execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}"
debug/ozone-debug-corrupt-block.robot
-#
-#docker stop "${container}"
-#
-#wait_for_datanode "${container}" STALE 60
-#execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}"
debug/ozone-debug-stale-datanode.robot
-#
-#wait_for_datanode "${container}" DEAD 60
-#execute_robot_test ${SCM} -v "PREFIX:${prefix}"
debug/ozone-debug-dead-datanode.robot
-#
-#docker start "${container}"
-#
-#wait_for_datanode "${container}" HEALTHY 60
+chunkinfo="${key}-blocks-${prefix}"
+docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info
${volume}/${bucket}/${key}" > "$chunkinfo"
+host="$(jq -r '.keyLocations[0][0].datanode["hostname"]' ${chunkinfo})"
+container="${host%%.*}"
+
+# corrupt the first block of key on one of the datanodes
+datafile="$(jq -r '.keyLocations[0][0].file' ${chunkinfo})"
+docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}"
+
+execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}"
debug/corrupt-block-checksum.robot
+
+echo "Overwriting container.db with the backup db"
+target_container_dir=$(docker exec "${container}" find "${container_db_path}"
-name "container.db" | xargs dirname)
+docker cp "${local_db_backup_path}/container.db"
"${container}:${target_container_dir}/"
+docker exec "${container}" sudo chown -R hadoop:hadoop
"${target_container_dir}"
+
+docker stop "${container}"
+
+wait_for_datanode "${container}" STALE 60
+execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}"
debug/stale-datanode-checksum.robot
+
+docker start "${container}"
+
+wait_for_datanode "${container}" HEALTHY 60
+
+execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "DATANODE:${host}"
debug/block-existence-check.robot
-#start_docker_env 9
-#execute_robot_test ${SCM} -v "PREFIX:${prefix}"
debug/ozone-debug-tests-ec3-2.robot
-#execute_robot_test ${SCM} -v "PREFIX:${prefix}"
debug/ozone-debug-tests-ec6-3.robot
+execute_robot_test ${OM} kinit.robot
+execute_robot_test ${OM} -v "PREFIX:${prefix}"
debug/ozone-debug-tests-ec3-2.robot
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
index 929632a07d0..653a0aaf766 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
@@ -49,7 +49,6 @@ execute_robot_test scm cli
execute_robot_test scm admincli
execute_robot_test scm -v USERNAME:httpfs httpfs
-source "$COMPOSE_DIR/../common/replicas-test.sh"
execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N
ozonefs-o3fs-bucket ozonefs/ozonefs.robot
execute_robot_test scm -v SCHEME:ofs -N ozonefs-obs ozonefs/ozonefs-obs.robot
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
index 9e2e7cbf0a0..c5ab0bf9443 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
@@ -47,6 +47,8 @@ OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.scm.stale.node.interval=30s
+OZONE-SITE.XML_hdds.heartbeat.interval=5s
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/block-existence-check.robot
similarity index 69%
copy from
hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot
copy to hadoop-ozone/dist/src/main/smoketest/debug/block-existence-check.robot
index 33a2b78c3df..4c319f169d3 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/block-existence-check.robot
@@ -14,21 +14,22 @@
# limitations under the License.
*** Settings ***
-Documentation Test checksums in case of one datanode is dead
+Documentation Test existence of a block on a datanode
Library OperatingSystem
Resource ../lib/os.robot
-Resource ozone-debug.robot
+Resource ozone-debug-keywords.robot
Test Timeout 5 minute
+
*** Variables ***
${PREFIX} ${EMPTY}
+${DATANODE} ${EMPTY}
${VOLUME} cli-debug-volume${PREFIX}
${BUCKET} cli-debug-bucket
${TESTFILE} testfile
+${CHECK_TYPE} blockExistence
*** Test Cases ***
-Test ozone debug checksums with one datanode DEAD
- ${directory} = Execute replicas verify checksums CLI tool
- Set Test Variable ${DIR} ${directory}
-
- ${count_files} = Count Files In Directory ${directory}
- Should Be Equal As Integers ${count_files} 1
+Test block existence with a block missing on a replica
+ ${output} = Execute replicas verify block existence debug tool
+ ${json} = Parse replicas verify JSON output ${output}
+ Check to Verify Replicas ${json} ${CHECK_TYPE} ${DATANODE} Unable to
find the block
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/corrupt-block-checksum.robot
similarity index 52%
rename from
hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot
rename to
hadoop-ozone/dist/src/main/smoketest/debug/corrupt-block-checksum.robot
index ba17ca9f085..4bd06e1e5f9 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/corrupt-block-checksum.robot
@@ -14,33 +14,23 @@
# limitations under the License.
*** Settings ***
-Documentation Test read-replicas in case of a corrupt replica
+Documentation Test checksums on a corrupt block replica
Library OperatingSystem
Resource ../lib/os.robot
-Resource ozone-debug.robot
+Resource ozone-debug-keywords.robot
Test Timeout 5 minute
+
*** Variables ***
${PREFIX} ${EMPTY}
+${CORRUPT_DATANODE} ${EMPTY}
${VOLUME} cli-debug-volume${PREFIX}
${BUCKET} cli-debug-bucket
${TESTFILE} testfile
-${CORRUPT_DATANODE} ozone_datanode_1.ozone_default
-
-*** Test Cases ***
-Test ozone debug checksums with corrupt block replica
- ${directory} = Execute replicas verify checksums CLI
tool
- Set Test Variable ${DIR} ${directory}
+${CHECK_TYPE} checksum
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
- ${json} = Read Replicas Manifest
- ${md5sum} = Execute md5sum
${TEMP_DIR}/${TESTFILE} | awk '{print $1}'
-
- FOR ${replica} IN RANGE 3
- ${datanode} = Set Variable
${json}[blocks][0][replicas][${replica}][hostname]
-
- IF '${datanode}' == '${CORRUPT_DATANODE}'
- Should Contain
${json}[blocks][0][replicas][${replica}][exception] Checksum mismatch
- END
- END
+*** Test Cases ***
+Test checksums with a corrupt block replica
+ ${output} = Execute replicas verify checksums debug tool
+ ${json} = Parse replicas verify JSON output ${output}
+ Check to Verify Replicas ${json} ${CHECK_TYPE} ${CORRUPT_DATANODE}
Checksum mismatched
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot
new file mode 100644
index 00000000000..19257a62c72
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Keyword definitions for Ozone Debug CLI tests
+Library Collections
+Resource ../lib/os.robot
+
+*** Variables ***
+${OM_SERVICE_ID} %{OM_SERVICE_ID}
+
+*** Keywords ***
+Execute replicas verify checksums debug tool
+ ${output} Execute ozone debug replicas verify --checksums
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} --all-results
+ [Return] ${output}
+
+Execute replicas verify block existence debug tool
+ ${output} Execute ozone debug replicas verify
--block-existence o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE}
--all-results
+ [Return] ${output}
+
+Parse replicas verify JSON output
+ [Arguments] ${output}
+ ${json} = Evaluate json.loads('''${output}''') json
+ [Return] ${json}
+
+Check to Verify Replicas
+ [Arguments] ${json} ${check_type} ${faulty_datanode}
${expected_message}
+ ${replicas} = Get From Dictionary ${json['keys'][0]['blocks'][0]}
replicas
+ FOR ${replica} IN @{replicas}
+ ${datanode} = Get From Dictionary ${replica} datanode
+ ${hostname} = Get From Dictionary ${datanode} hostname
+ Run Keyword If '${hostname}' == '${faulty_datanode}' Check
Replica Failed ${replica} ${check_type} ${expected_message}
+ Run Keyword If '${hostname}' != '${faulty_datanode}' Check
Replica Passed ${replica} ${check_type}
+ END
+
+Check Replica Failed
+ [Arguments] ${replica} ${check_type} ${expected_message}
+ ${checks} = Get From Dictionary ${replica} checks
+ ${check} = Get From List ${checks} 0
+ Should Be Equal ${check['type']} ${check_type}
+ Should Be Equal ${check['pass']} ${False}
+ Should Contain ${check['failures'][0]['message']}
${expected_message}
+
+Check Replica Passed
+ [Arguments] ${replica} ${check_type}
+ ${checks} = Get From Dictionary ${replica} checks
+ ${check} = Get From List ${checks} 0
+ Should Be Equal ${check['type']} ${check_type}
+ Should Be True ${check['completed']}
+ Should Be True ${check['pass']}
+ Should Be Empty ${check['failures']}
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot
deleted file mode 100644
index 69448f21e6b..00000000000
---
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot
+++ /dev/null
@@ -1,46 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Test checksums in case of one datanode is stale
-Library OperatingSystem
-Resource ../lib/os.robot
-Resource ozone-debug.robot
-Test Timeout 5 minute
-*** Variables ***
-${PREFIX} ${EMPTY}
-${VOLUME} cli-debug-volume${PREFIX}
-${BUCKET} cli-debug-bucket
-${TESTFILE} testfile
-${STALE_DATANODE} ozone_datanode_1.ozone_default
-
-*** Test Cases ***
-Test ozone debug checksums with one datanode STALE
- ${directory} = Execute replicas verify checksums CLI tool
- Set Test Variable ${DIR} ${directory}
-
- ${count_files} = Count Files In Directory ${directory}
- Should Be Equal As Integers ${count_files} 1
-
- ${json} = Read Replicas Manifest
- ${md5sum} = Execute md5sum ${TEMP_DIR}/${TESTFILE}
| awk '{print $1}'
-
- FOR ${replica} IN RANGE 3
- ${datanode} = Set Variable
${json}[blocks][0][replicas][${replica}][hostname]
-
- IF '${datanode}' == '${STALE_DATANODE}'
- Verify Stale Replica ${json} ${replica}
- END
- END
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot
index e10e03443d6..7b88f97254c 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot
@@ -18,14 +18,14 @@ Documentation Test ozone Debug CLI for EC(3,2)
replicated keys
Library OperatingSystem
Library Process
Resource ../lib/os.robot
-Resource ozone-debug.robot
+Resource ozone-debug-keywords.robot
Test Timeout 5 minute
Suite Setup Create Volume Bucket
*** Variables ***
${PREFIX} ${EMPTY}
-${VOLUME} cli-debug-volume${PREFIX}
-${BUCKET} cli-debug-bucket
+${VOLUME} cli-debug-ec-volume${PREFIX}
+${BUCKET} cli-debug-ec-bucket
${TESTFILE} testfile
${EC_DATA} 3
${EC_PARITY} 2
@@ -43,49 +43,6 @@ Create EC key
Execute ozone sh key put
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r
rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
*** Test Cases ***
-0 data block
- Create EC key 1000 0
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-1 data block
- Create EC key 1048576 1
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-2 data blocks
- Create EC key 1048576 2
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-3 data blocks
- Create EC key 1048576 3
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-3 data blocks and partial stripe
- Create EC key 1000000 4
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- ${sum_size_last_stripe} = Evaluate ((1000000 * 4) % 1048576)
* 3
- Should Be Equal As Integers ${count_files} 1
-
-4 data blocks and partial stripe
- Create EC key 1000000 5
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-6 data blocks
- Create EC key 1048576 6
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
Test ozone debug replicas chunk-info
Create EC key 1048576 3
${count} = Execute ozone debug replicas chunk-info
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] |
select(.file | test("\\\\.block$")) | .file] | length'
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot
deleted file mode 100644
index c3b8c7ad1a3..00000000000
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot
+++ /dev/null
@@ -1,100 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Test ozone Debug CLI for EC(6,3) replicated key
-Library OperatingSystem
-Library Process
-Resource ../lib/os.robot
-Resource ozone-debug.robot
-Test Timeout 5 minute
-
-*** Variables ***
-${PREFIX} ${EMPTY}
-${VOLUME} cli-debug-volume${PREFIX}
-${BUCKET} cli-debug-bucket
-${TESTFILE} testfile
-${EC_DATA} 6
-${EC_PARITY} 3
-${OM_SERVICE_ID} %{OM_SERVICE_ID}
-
-*** Keywords ***
-Create EC key
- [arguments] ${bs} ${count}
-
- Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs}
count=${count}
- Execute ozone sh key put
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r
rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
-
-*** Test Cases ***
-0 data block
- Create EC key 1048576 0
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-1 data block
- Create EC key 1048576 1
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-2 data blocks
- Create EC key 1048576 2
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-3 data blocks
- Create EC key 1048576 3
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-4 data blocks
- Create EC key 1048576 4
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-5 data blocks
- Create EC key 1048576 5
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-6 data blocks
- Create EC key 1048576 6
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- Should Be Equal As Integers ${count_files} 1
-
-6 data blocks and partial stripe
- Create EC key 1000000 7
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- ${sum_size_last_stripe} = Evaluate ((1000000 * 7) % 1048576)
* 4
- Should Be Equal As Integers ${count_files} 1
-
-7 data blocks and partial stripe
- Create EC key 1000000 8
- ${directory} = Execute replicas verify checksums CLI
tool
- ${count_files} = Count Files In Directory
${directory}
- ${sum_size_last_stripe} = Evaluate 1048576 * 4 + ((1000000 *
8) % 1048576)
- Should Be Equal As Integers ${count_files} 1
-
-Test ozone debug replicas chunk-info
- Create EC key 1048576 6
- ${count} = Execute ozone debug replicas chunk-info
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] |
select(.file | test("\\\\.block$")) | .file] | length'
- Should Be Equal As Integers ${count} 9
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
index 9364c98a6ce..60e7b69a2ce 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
@@ -14,10 +14,10 @@
# limitations under the License.
*** Settings ***
-Documentation Test ozone Debug CLI
+Documentation Test ozone debug CLI
Library OperatingSystem
Resource ../lib/os.robot
-Resource ozone-debug.robot
+Resource ozone-debug-keywords.robot
Test Timeout 5 minute
Suite Setup Write keys
@@ -38,9 +38,9 @@ Write keys
*** Test Cases ***
Test ozone debug replicas verify checksums
${output} = Execute ozone debug replicas verify --checksums
--block-existence --container-state
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE}
- ${json} = Evaluate json.loads('''${output}''') json
+ ${json} = Parse replicas verify JSON output ${output}
- # 'keys' array should be empty if all keys and their replicas passed
checksum verification
+ # 'keys' array should be empty if all keys and their replicas passed
Should Be Empty ${json}[keys]
Should Be True ${json}[pass] ${True}
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
deleted file mode 100644
index c3cb8d2cf15..00000000000
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
+++ /dev/null
@@ -1,98 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Keyword definitions for Ozone Debug CLI tests
-Library Collections
-Resource ../lib/os.robot
-
-*** Variables ***
-${OM_SERVICE_ID} %{OM_SERVICE_ID}
-
-*** Keywords ***
-Execute replicas verify checksums CLI tool
- Execute ozone debug
-Dozone.network.topology.aware.read=true replicas verify --checksums
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE}
- ${directory} = Execute ls -d
${TEMP_DIR}/${VOLUME}_${BUCKET}_${TESTFILE}_*/ | tail -n 1
- Directory Should Exist ${directory}
- File Should Exist ${directory}/${TESTFILE}_manifest
- [Return] ${directory}
-
-Read Replicas Manifest
- ${manifest} = Get File ${DIR}/${TESTFILE}_manifest
- ${json} = Evaluate json.loads('''${manifest}''')
json
- Validate JSON ${json}
- [return] ${json}
-
-Validate JSON
- [arguments] ${json}
- Should Be Equal ${json}[filename]
${VOLUME}/${BUCKET}/${TESTFILE}
- ${file_size} = Get File Size
${TEMP_DIR}/${TESTFILE}
- Should Be Equal ${json}[datasize]
${file_size}
- Should Be Equal As Integers ${json}[blocks][0][blockIndex] 1
- Should Not Be Empty Convert To String
${json}[blocks][0][containerId]
- Should Not Be Empty Convert To String
${json}[blocks][0][localId]
- Should Be Equal As Integers ${json}[blocks][0][length] 1048576
- Should Not Be Empty Convert To String
${json}[blocks][0][offset]
- Should Be Equal As Integers ${json}[blocks][1][blockIndex] 2
- Should Not Be Empty Convert To String
${json}[blocks][1][containerId]
- Should Not Be Empty Convert To String
${json}[blocks][1][localId]
- Should Be Equal As Integers ${json}[blocks][1][length] 451424
- Should Not Be Empty Convert To String
${json}[blocks][1][offset]
-
-Get Replica Filenames
- [arguments] ${json} ${replica}
-
- ${list} = Create List
-
- FOR ${block} IN RANGE 2
- ${datanode} = Set Variable
${json}[blocks][${block}][replicas][${replica}][hostname]
- ${n} = Evaluate ${block} + 1
- Append To List ${list} ${DIR}/${TESTFILE}_block${n}_${datanode}
- END
-
- ${filenames} = Catenate @{list}
-
- [return] ${filenames}
-
-Verify Healthy Replica
- [arguments] ${json} ${replica} ${expected_md5sum}
-
- ${block_filenames} = Get Replica Filenames ${json} ${replica}
- ${md5sum} = Execute cat ${block_filenames} | md5sum | awk
'{print $1}'
- Should Be Equal ${md5sum} ${expected_md5sum}
-
-Verify Healthy EC Replica
- [arguments] ${directory} ${block} ${expected_block_size}
-
- ${block_size} = Execute ls -l ${directory} | grep
"testfile_block${block}_ozone-datanode-.*\.ozone_default" | awk '{sum += $5}
END {print sum}'
- Should Be Equal As Integers ${block_size} ${expected_block_size}
-
-Verify Corrupt Replica
- [arguments] ${json} ${replica} ${valid_md5sum}
-
- ${block_filenames} = Get Replica Filenames ${json} ${replica}
- ${md5sum} = Execute cat ${block_filenames} | md5sum | awk
'{print $1}'
- Should Not Be Equal ${md5sum} ${valid_md5sum}
-
-Verify Stale Replica
- [arguments] ${json} ${replica}
-
- FOR ${block} IN RANGE 2
- ${datanode} = Set Variable
${json}[blocks][${block}][replicas][${replica}][hostname]
-
- IF '${datanode}' == '${STALE_DATANODE}'
- Should Contain
${json}[blocks][${block}][replicas][${replica}][exception] UNAVAILABLE
- END
- END
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/stale-datanode-checksum.robot
similarity index 61%
rename from
hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot
rename to
hadoop-ozone/dist/src/main/smoketest/debug/stale-datanode-checksum.robot
index 33a2b78c3df..56f2133cd0f 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/stale-datanode-checksum.robot
@@ -14,21 +14,22 @@
# limitations under the License.
*** Settings ***
-Documentation Test checksums in case of one datanode is dead
+Documentation Test checksums in case of a stale datanode
Library OperatingSystem
Resource ../lib/os.robot
-Resource ozone-debug.robot
+Resource ozone-debug-keywords.robot
Test Timeout 5 minute
+
*** Variables ***
-${PREFIX} ${EMPTY}
-${VOLUME} cli-debug-volume${PREFIX}
-${BUCKET} cli-debug-bucket
-${TESTFILE} testfile
+${PREFIX} ${EMPTY}
+${STALE_DATANODE} ${EMPTY}
+${VOLUME} cli-debug-volume${PREFIX}
+${BUCKET} cli-debug-bucket
+${TESTFILE} testfile
+${CHECK_TYPE} checksum
*** Test Cases ***
-Test ozone debug checksums with one datanode DEAD
- ${directory} = Execute replicas verify checksums CLI tool
- Set Test Variable ${DIR} ${directory}
-
- ${count_files} = Count Files In Directory ${directory}
- Should Be Equal As Integers ${count_files} 1
+Test checksums with a stale datanode
+ ${output} = Execute replicas verify checksums debug tool
+ ${json} = Parse replicas verify JSON output ${output}
+ Check to Verify Replicas ${json} ${CHECK_TYPE} ${STALE_DATANODE}
UNAVAILABLE
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
