This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch ozone-2.1
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/ozone-2.1 by this push:
new 80f51977920 Revert "HDDS-13594. Use a different endpoint for fetching
the OM checkpoint tarball. (#8963)"
80f51977920 is described below
commit 80f51977920caf7ea634adb91ad503e5374540ac
Author: Wei-Chiu Chuang <[email protected]>
AuthorDate: Wed Nov 12 09:26:48 2025 -0800
Revert "HDDS-13594. Use a different endpoint for fetching the OM checkpoint
tarball. (#8963)"
This reverts commit bc731ea6af706fd51eedbc31bd1eeea0bd75c4ee.
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 -
.../hadoop/hdds/utils/DBCheckpointServlet.java | 14 +--
.../hadoop/ozone/om/helpers/OMNodeDetails.java | 4 +-
hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 38 -------
.../main/smoketest/compatibility/checkpoint.robot | 110 ---------------------
.../ozone/recon/TestReconWithOzoneManagerHA.java | 4 +-
.../om/OMDBCheckpointServletInodeBasedXfer.java | 2 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 3 -
.../spi/impl/OzoneManagerServiceProviderImpl.java | 8 +-
9 files changed, 10 insertions(+), 175 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index cb4490c2c1d..b2f6ed8af60 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -75,8 +75,6 @@ public final class OzoneConsts {
"/serviceList";
public static final String OZONE_DB_CHECKPOINT_HTTP_ENDPOINT =
"/dbCheckpoint";
- public static final String OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2 =
- "/v2/dbCheckpoint";
// Ozone File System scheme
public static final String OZONE_URI_SCHEME = "o3fs";
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
index dae2df9e5c3..26c93cd1792 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
@@ -20,7 +20,6 @@
import static
org.apache.hadoop.hdds.utils.HddsServerUtil.writeDBCheckpointToStream;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST;
-import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX;
import com.google.common.annotations.VisibleForTesting;
import java.io.File;
@@ -278,18 +277,7 @@ public void
processMetadataSnapshotRequest(HttpServletRequest request, HttpServl
}
}
- protected static Set<String> extractSstFilesToExclude(String[]
filesInExclusionParam) {
- Set<String> sstFilesToExclude = new HashSet<>();
- if (filesInExclusionParam != null) {
- sstFilesToExclude.addAll(
- Arrays.stream(filesInExclusionParam).filter(s ->
s.endsWith(ROCKSDB_SST_SUFFIX))
- .distinct().collect(Collectors.toList()));
- logSstFileList(sstFilesToExclude, "Received list of {} SST files to be
excluded{}: {}", 5);
- }
- return sstFilesToExclude;
- }
-
- protected static Set<String> extractFilesToExclude(String[] sstParam) {
+ protected static Set<String> extractSstFilesToExclude(String[] sstParam) {
Set<String> receivedSstFiles = new HashSet<>();
if (sstParam != null) {
receivedSstFiles.addAll(
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
index ff51cfa7adc..adcf07d2cb1 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
@@ -17,7 +17,7 @@
package org.apache.hadoop.ozone.om.helpers;
-import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
@@ -189,7 +189,7 @@ public URL getOMDBCheckpointEndpointUrl(boolean isHttp,
boolean flush)
URIBuilder urlBuilder = new URIBuilder().
setScheme(isHttp ? "http" : "https").
setHost(isHttp ? getHttpAddress() : getHttpsAddress()).
- setPath(OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2).
+ setPath(OZONE_DB_CHECKPOINT_HTTP_ENDPOINT).
addParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA, "true").
addParameter(OZONE_DB_CHECKPOINT_REQUEST_FLUSH,
flush ? "true" : "false");
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/lib.sh
b/hadoop-ozone/dist/src/main/compose/xcompat/lib.sh
index 80bcbd24e2a..5e53515b6f5 100755
--- a/hadoop-ozone/dist/src/main/compose/xcompat/lib.sh
+++ b/hadoop-ozone/dist/src/main/compose/xcompat/lib.sh
@@ -73,15 +73,6 @@ _read() {
compatibility/read.robot
}
-_test_checkpoint_compatibility() {
- _kinit
- execute_robot_test ${container} -N
"xcompat-cluster-${cluster_version}-client-${client_version}-checkpoint" \
- -v CLIENT_VERSION:${client_version} \
- -v CLUSTER_VERSION:${cluster_version} \
- -v TEST_DATA_DIR:/testdata \
- compatibility/checkpoint.robot
-}
-
test_cross_compatibility() {
echo "Starting ${cluster_version} cluster with COMPOSE_FILE=${COMPOSE_FILE}"
@@ -116,35 +107,6 @@ test_cross_compatibility() {
done
done
- # Add checkpoint compatibility tests (only for clusters that support
checkpoint endpoints)
- # Skip checkpoint tests for very old clusters that don't have the endpoints
- if [[ "${cluster_version}" < "2.0.0" ]]; then
- echo "Skipping checkpoint compatibility tests for cluster
${cluster_version} (checkpoint endpoints not available)"
- else
- echo ""
- echo "=========================================="
- echo "Running checkpoint compatibility tests"
- echo "=========================================="
-
- # Test 2.0.0 client (if available)
- for client_version in "$@"; do
- if [[ "${client_version}" == "2.0.0" ]]; then
- echo "Testing 2.0.0 client against ${cluster_version} cluster"
- client _test_checkpoint_compatibility
- break # Only test 2.0 once
- fi
- done
-
- # Test current client (if different from 2.0.0 and available)
- for client_version in "$@"; do
- if [[ "${client_version}" == "${current_version}" ]]; then
- echo "Testing ${current_version} client against ${cluster_version}
cluster"
- client _test_checkpoint_compatibility
- break # Only test current version once
- fi
- done
- fi
-
KEEP_RUNNING=false stop_docker_env
}
diff --git
a/hadoop-ozone/dist/src/main/smoketest/compatibility/checkpoint.robot
b/hadoop-ozone/dist/src/main/smoketest/compatibility/checkpoint.robot
deleted file mode 100644
index e1776ef1a4b..00000000000
--- a/hadoop-ozone/dist/src/main/smoketest/compatibility/checkpoint.robot
+++ /dev/null
@@ -1,110 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Checkpoint Compatibility
-Resource ../ozone-lib/shell.robot
-Resource setup.robot
-Test Timeout 5 minutes
-
-*** Variables ***
-${CHECKPOINT_V2_VERSION} 2.1.0
-${OM_HOST} om
-${OM_PORT} 9874
-
-*** Keywords ***
-Download Checkpoint V1
- [Documentation] Download checkpoint using v1 endpoint (/dbCheckpoint)
- [Arguments] ${expected_result}
-
- Log Testing v1 checkpoint endpoint with authentication
-
- # Try different keytabs based on client version/container
- ${download_file} = Set Variable
/tmp/checkpoint_v1_${CLIENT_VERSION}.tar.gz
-
- # Debug: Check keytab availability first
- ${keytab_check} = Execute ls -la /etc/security/keytabs/ 2>&1 | head
-5 || echo "No keytabs directory"
- Log Keytab directory: ${keytab_check}
-
- # Combine kinit and curl in a single command to preserve Kerberos session
- ${combined_cmd} = Set Variable kinit -k -t
/etc/security/keytabs/testuser.keytab testuser/[email protected] && curl -f
--negotiate -u : --connect-timeout 10 --max-time 30 -o ${download_file}
http://${OM_HOST}:${OM_PORT}/dbCheckpoint
-
- Log Executing: ${combined_cmd}
- ${result} = Execute and checkrc ${combined_cmd}
${expected_result}
-
- IF ${expected_result} == 0
- # If we expect success, verify the file was created and has content
- ${file_check} = Execute ls -la ${download_file} 2>/dev/null ||
echo "File not found"
- Should Not Contain ${file_check} File not found
- Should Contain ${file_check}
checkpoint_v1_${CLIENT_VERSION}.tar.gz
- Log Successfully downloaded checkpoint via v1
endpoint: ${file_check}
- ELSE
- Log v1 endpoint failed as expected for this version
combination
- END
-
-Download Checkpoint V2
- [Documentation] Download checkpoint using v2 endpoint (/dbCheckpointv2)
- [Arguments] ${expected_result}
-
- Log Testing v2 checkpoint endpoint with authentication
-
- # Debug: Check keytab availability first (reuse from V1 if already checked)
- ${keytab_check} = Execute ls -la /etc/security/keytabs/ 2>&1 | head
-5 || echo "No keytabs directory"
- Log Keytab directory: ${keytab_check}
-
- # Combine kinit and curl in a single command to preserve Kerberos session
- ${download_file} = Set Variable
/tmp/checkpoint_v2_${CLIENT_VERSION}.tar.gz
- ${combined_cmd} = Set Variable kinit -k -t
/etc/security/keytabs/testuser.keytab testuser/[email protected] && curl -f
--negotiate -u : --connect-timeout 10 --max-time 30 -o ${download_file}
http://${OM_HOST}:${OM_PORT}/v2/dbCheckpoint
-
- Log Executing: ${combined_cmd}
- ${result} = Execute and checkrc ${combined_cmd}
${expected_result}
-
- IF ${expected_result} == 0
- # If we expect success, verify the file was created and has content
- ${file_check} = Execute ls -la ${download_file} 2>/dev/null ||
echo "File not found"
- Should Not Contain ${file_check} File not found
- Should Contain ${file_check}
checkpoint_v2_${CLIENT_VERSION}.tar.gz
- Log Successfully downloaded checkpoint via v2
endpoint: ${file_check}
- ELSE
- Log v2 endpoint failed as expected for this version
combination
- END
-
-*** Test Cases ***
-Checkpoint V1 Endpoint Compatibility
- [Documentation] Test v1 checkpoint endpoint (/dbCheckpoint) - should
work for all versions (backward compatibility)
-
- Log Testing v1 checkpoint endpoint: CLIENT=${CLIENT_VERSION},
CLUSTER=${CLUSTER_VERSION}
-
- # Both old and new clusters should serve v1 endpoint for backward
compatibility
- Download Checkpoint V1 0
-
-Checkpoint V2 Endpoint Compatibility
- [Documentation] Test v2 checkpoint endpoint (/v2/dbCheckpoint) - should
only work with new cluster
-
- Log Testing v2 checkpoint endpoint: CLIENT=${CLIENT_VERSION},
CLUSTER=${CLUSTER_VERSION}
-
- IF '${CLUSTER_VERSION}' < '${CHECKPOINT_V2_VERSION}'
- # Old cluster doesn't have v2 endpoint - should fail with any non-zero
exit code
- ${result} = Run Keyword And Return Status Download Checkpoint V2
0
- IF not ${result}
- Log v2 endpoint correctly failed on old cluster
${CLUSTER_VERSION} (expected failure)
- ELSE
- Fail v2 endpoint unexpectedly succeeded on old cluster
${CLUSTER_VERSION}
- END
- ELSE
- # New cluster has v2 endpoint - should succeed
- Download Checkpoint V2 0
- Log v2 endpoint correctly succeeded on new cluster
${CLUSTER_VERSION}
- END
diff --git
a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
index 254a9f39256..790932f2568 100644
---
a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
+++
b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.recon;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -119,7 +119,7 @@ public void testReconGetsSnapshotFromLeader() throws
Exception {
String expectedUrl = "http://"; +
(hostname.equals("0.0.0.0") ? "localhost" : hostname) + ":" +
ozoneManager.get().getHttpServer().getHttpAddress().getPort() +
- OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2;
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
String snapshotUrl = impl.getOzoneManagerSnapshotUrl();
assertEquals(expectedUrl, snapshotUrl);
// Write some data
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
index 28769f75409..31d83c6b4cf 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
@@ -139,7 +139,7 @@ public void
processMetadataSnapshotRequest(HttpServletRequest request, HttpServl
String[] sstParam = isFormData ?
parseFormDataParameters(request) : request.getParameterValues(
OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST);
- Set<String> receivedSstFiles = extractFilesToExclude(sstParam);
+ Set<String> receivedSstFiles = extractSstFilesToExclude(sstParam);
Path tmpdir = null;
try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) {
tmpdir = Files.createTempDirectory(getBootstrapTempData().toPath(),
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
index 9c2688de812..8836324410b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.om;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
-import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT;
import java.io.IOException;
@@ -37,8 +36,6 @@ public OzoneManagerHttpServer(MutableConfigurationSource conf,
addServlet("serviceList", OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT,
ServiceListJSONServlet.class);
addServlet("dbCheckpoint", OZONE_DB_CHECKPOINT_HTTP_ENDPOINT,
- OMDBCheckpointServlet.class);
- addServlet("dbCheckpointv2", OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2,
OMDBCheckpointServletInodeBasedXfer.class);
getWebAppContext().setAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE, om);
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
index 3cd785e53a5..5409309516a 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.recon.spi.impl;
import static
org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DB_DIRS_PERMISSIONS_DEFAULT;
-import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE;
import static
org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_SNAPSHOT_DB;
@@ -197,11 +197,11 @@ public OzoneManagerServiceProviderImpl(
HttpConfig.Policy policy = HttpConfig.getHttpPolicy(configuration);
omDBSnapshotUrl = "http://"; + ozoneManagerHttpAddress +
- OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2;
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
if (policy.isHttpsEnabled()) {
omDBSnapshotUrl = "https://"; + ozoneManagerHttpsAddress +
- OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2;
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
}
boolean flushParam = configuration.getBoolean(
@@ -414,7 +414,7 @@ public String getOzoneManagerSnapshotUrl() throws
IOException {
omLeaderUrl = (policy.isHttpsEnabled() ?
"https://"; + info.getServiceAddress(Type.HTTPS) :
"http://"; + info.getServiceAddress(Type.HTTP)) +
- OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2;
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
