This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 93613c1cae HDDS-12734. Enable native lib in CI checks (#8190)
93613c1cae is described below
commit 93613c1cae2b1da823bdc9c57ee636bac622c47e
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Thu Apr 17 13:23:02 2025 +0200
HDDS-12734. Enable native lib in CI checks (#8190)
---
.github/workflows/ci.yml | 21 ++----------
dev-support/ci/categorize_basic_checks.sh | 2 +-
dev-support/ci/selective_ci_checks.bats | 26 +++++++--------
dev-support/ci/selective_ci_checks.sh | 38 ----------------------
hadoop-hdds/rocks-native/pom.xml | 1 -
.../apache/hadoop/hdds/utils/NativeConstants.java | 1 +
.../utils/db/managed/ManagedRawSSTFileReader.java | 9 +++++
.../hadoop/hdds/utils/TestNativeLibraryLoader.java | 5 +--
.../db/managed/TestManagedRawSSTFileIterator.java | 8 ++---
.../ozone/rocksdb/util/TestSstFileSetReader.java | 11 +++----
.../java/org/apache/ozone/test/tag/Native.java | 37 ---------------------
hadoop-ozone/dev-support/checks/native.sh | 23 -------------
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 5 +++
.../snapshot/TestOmSnapshotFsoWithNativeLib.java | 6 ++--
...mSnapshotFsoWithNativeLibWithLinkedBuckets.java | 6 ++--
.../apache/hadoop/ozone/debug/TestCheckNative.java | 8 +++--
pom.xml | 34 ++-----------------
17 files changed, 57 insertions(+), 184 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f68ae51111..f0b6f47da5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,7 +27,7 @@ on:
default: ''
required: false
env:
- BUILD_ARGS: "-Pdist -Psrc -Dmaven.javadoc.skip=true"
+ BUILD_ARGS: "-Pdist -Psrc -Dmaven.javadoc.skip=true -Drocks_tools_native"
# Minimum required Java version for running Ozone is defined in pom.xml
(javac.version).
TEST_JAVA_VERSION: 21 # JDK version used by CI build and tests; should match
the JDK version in apache/ozone-runner image
# MAVEN_ARGS and MAVEN_OPTS are duplicated in check.yml, please keep in sync
@@ -44,7 +44,6 @@ jobs:
acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }}
integration-suites: ${{ steps.integration-suites.outputs.suites }}
needs-basic-check: ${{
steps.categorize-basic-checks.outputs.needs-basic-check }}
- needs-native-check: ${{
steps.categorize-basic-checks.outputs.needs-native-check }}
basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }}
needs-build: ${{ steps.selective-checks.outputs.needs-build }}
needs-compile: ${{ steps.selective-checks.outputs.needs-compile }}
@@ -174,21 +173,6 @@ jobs:
check: ${{ fromJson(needs.build-info.outputs.basic-checks) }}
fail-fast: false
- native:
- needs:
- - build-info
- - basic
- if: needs.build-info.outputs.needs-native-check == 'true'
- uses: ./.github/workflows/check.yml
- secrets: inherit
- with:
- java-version: ${{ needs.build-info.outputs.java-version }}
- ratis-args: ${{ inputs.ratis_args }}
- script: native
- sha: ${{ needs.build-info.outputs.sha }}
- timeout-minutes: 150
- with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
-
dependency:
needs:
- build-info
@@ -289,7 +273,7 @@ jobs:
java-version: ${{ needs.build-info.outputs.java-version }}
ratis-args: ${{ inputs.ratis_args }}
script: integration
- script-args: -Ptest-${{ matrix.profile }}
+ script-args: -Ptest-${{ matrix.profile }} -Drocks_tools_native
sha: ${{ needs.build-info.outputs.sha }}
split: ${{ matrix.profile }}
timeout-minutes: 150
@@ -307,7 +291,6 @@ jobs:
- build-info
- acceptance
- integration
- - native
steps:
- name: Checkout project
uses: actions/checkout@v4
diff --git a/dev-support/ci/categorize_basic_checks.sh
b/dev-support/ci/categorize_basic_checks.sh
index 9b4adf2640..23dc38dfbe 100755
--- a/dev-support/ci/categorize_basic_checks.sh
+++ b/dev-support/ci/categorize_basic_checks.sh
@@ -30,7 +30,7 @@ if [[ -n "${SPACE_DELIMITED_ALL_CHECKS}" ]]; then
# add framing blanks
SPACE_DELIMITED_ALL_CHECKS=" ${SPACE_DELIMITED_ALL_CHECKS[*]} "
- for check in basic native; do
+ for check in basic; do
CHECKS=$(grep -lr "^#checks:${check}$" hadoop-ozone/dev-support/checks \
| sort -u \
| xargs -n1 basename \
diff --git a/dev-support/ci/selective_ci_checks.bats
b/dev-support/ci/selective_ci_checks.bats
index e1d5ee7578..12600c065f 100644
--- a/dev-support/ci/selective_ci_checks.bats
+++ b/dev-support/ci/selective_ci_checks.bats
@@ -135,7 +135,7 @@ load bats-assert/load.bash
@test "script change including junit.sh" {
run dev-support/ci/selective_ci_checks.sh 66093e52c6
- assert_output -p
'basic-checks=["rat","bats","checkstyle","findbugs","native"]'
+ assert_output -p 'basic-checks=["rat","bats","checkstyle","findbugs"]'
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=false
@@ -179,22 +179,22 @@ load bats-assert/load.bash
@test "native only" {
run dev-support/ci/selective_ci_checks.sh 5b1319a8c2
- assert_output -p
'basic-checks=["rat","author","checkstyle","findbugs","pmd","native"]'
+ assert_output -p
'basic-checks=["rat","author","checkstyle","findbugs","pmd"]'
assert_output -p needs-build=true
assert_output -p needs-compile=true
- assert_output -p needs-compose-tests=false
- assert_output -p needs-integration-tests=false
- assert_output -p needs-kubernetes-tests=false
+ assert_output -p needs-compose-tests=true
+ assert_output -p needs-integration-tests=true
+ assert_output -p needs-kubernetes-tests=true
}
@test "native test in other module" {
run dev-support/ci/selective_ci_checks.sh 822c0dee1a
- assert_output -p
'basic-checks=["rat","author","checkstyle","findbugs","pmd","native"]'
+ assert_output -p
'basic-checks=["rat","author","checkstyle","findbugs","pmd"]'
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=false
- assert_output -p needs-integration-tests=false
+ assert_output -p needs-integration-tests=true
assert_output -p needs-kubernetes-tests=false
}
@@ -245,7 +245,7 @@ load bats-assert/load.bash
@test "java and compose change" {
run dev-support/ci/selective_ci_checks.sh d0f0f806e
- assert_output -p
'basic-checks=["rat","author","checkstyle","findbugs","pmd","native"]'
+ assert_output -p
'basic-checks=["rat","author","checkstyle","findbugs","pmd"]'
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=true
@@ -267,7 +267,7 @@ load bats-assert/load.bash
@test "pom change" {
run dev-support/ci/selective_ci_checks.sh 9129424a9
- assert_output -p
'basic-checks=["rat","checkstyle","findbugs","pmd","native"]'
+ assert_output -p 'basic-checks=["rat","checkstyle","findbugs","pmd"]'
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=true
@@ -278,7 +278,7 @@ load bats-assert/load.bash
@test "CI lib change" {
run dev-support/ci/selective_ci_checks.sh ceb79acaa
- assert_output -p
'basic-checks=["author","bats","checkstyle","docs","findbugs","native","pmd","rat"]'
+ assert_output -p
'basic-checks=["author","bats","checkstyle","docs","findbugs","pmd","rat"]'
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=true
@@ -289,7 +289,7 @@ load bats-assert/load.bash
@test "CI workflow change" {
run dev-support/ci/selective_ci_checks.sh 90a8d7c01
- assert_output -p
'basic-checks=["author","bats","checkstyle","docs","findbugs","native","pmd","rat"]'
+ assert_output -p
'basic-checks=["author","bats","checkstyle","docs","findbugs","pmd","rat"]'
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=true
@@ -312,7 +312,7 @@ load bats-assert/load.bash
@test "CI workflow change (ci.yaml)" {
run dev-support/ci/selective_ci_checks.sh 90fd5f2adc
- assert_output -p
'basic-checks=["author","bats","checkstyle","docs","findbugs","native","pmd","rat"]'
+ assert_output -p
'basic-checks=["author","bats","checkstyle","docs","findbugs","pmd","rat"]'
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=true
@@ -411,7 +411,7 @@ load bats-assert/load.bash
@test "properties file in resources" {
run dev-support/ci/selective_ci_checks.sh
71b8bdd8becf72d6f7d4e7986895504b8259b3e5
- assert_output -p 'basic-checks=["rat","checkstyle","native"]'
+ assert_output -p 'basic-checks=["rat","checkstyle"]'
assert_output -p needs-build=false
assert_output -p needs-compile=false
assert_output -p needs-compose-tests=false
diff --git a/dev-support/ci/selective_ci_checks.sh
b/dev-support/ci/selective_ci_checks.sh
index f46d8fb498..10c6044e31 100755
--- a/dev-support/ci/selective_ci_checks.sh
+++ b/dev-support/ci/selective_ci_checks.sh
@@ -266,9 +266,6 @@ function get_count_integration_files() {
"src/test/java"
"src/test/resources"
)
- local ignore_array=(
- $(grep -Flr 'org.apache.ozone.test.tag.Native'
hadoop-ozone/integration-test)
- )
filter_changed_files true
COUNT_INTEGRATION_CHANGED_FILES=${match_count}
readonly COUNT_INTEGRATION_CHANGED_FILES
@@ -440,40 +437,6 @@ function check_needs_pmd() {
start_end::group_end
}
-function check_needs_native() {
- start_end::group_start "Check if native is needed"
- local pattern_array=(
- "^hadoop-ozone/dev-support/checks/native.sh"
- "^hadoop-hdds/rocks-native"
- # include tests tagged as @Native in any module
- $(grep -Flr 'org.apache.ozone.test.tag.Native'
hadoop-*/*/src/test/java)
- )
- filter_changed_files true
-
- if [[ ${match_count} != "0" ]]; then
- add_basic_check native
- else
- local pattern_array=(
- "^hadoop-ozone/dev-support/checks/junit.sh"
- # dependencies
- "^hadoop-hdds/annotations"
- "^hadoop-hdds/common"
- "^hadoop-hdds/config"
- "^hadoop-hdds/hadoop-dependency-client"
- "^hadoop-hdds/managed-rocksdb"
- "^hadoop-hdds/test-utils"
- "^pom.xml"
- )
- filter_changed_files
-
- if [[ ${match_count} != "0" ]]; then
- add_basic_check native
- fi
- fi
-
- start_end::group_end
-}
-
# Counts other files which do not need to trigger any functional test
# (i.e. no compose/integration/kubernetes)
function get_count_misc_files() {
@@ -609,6 +572,5 @@ check_needs_checkstyle
check_needs_docs
check_needs_findbugs
check_needs_pmd
-check_needs_native
calculate_test_types_to_run
set_outputs
diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml
index 8f0da5a368..5c3eeb8689 100644
--- a/hadoop-hdds/rocks-native/pom.xml
+++ b/hadoop-hdds/rocks-native/pom.xml
@@ -336,7 +336,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
- <groups>native</groups>
<argLine>${maven-surefire-plugin.argLine} @{argLine}
-Djava.library.path=${project.build.directory}/native/rocksdb</argLine>
</configuration>
</plugin>
diff --git
a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java
b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java
index eb89264af4..7d49eb20d7 100644
---
a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java
+++
b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java
@@ -26,4 +26,5 @@ private NativeConstants() {
}
public static final String ROCKS_TOOLS_NATIVE_LIBRARY_NAME =
"ozone_rocksdb_tools";
+ public static final String ROCKS_TOOLS_NATIVE_PROPERTY =
"rocks_tools_native";
}
diff --git
a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java
b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java
index b441784c4d..eadf1afcb8 100644
---
a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java
+++
b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java
@@ -32,6 +32,15 @@
*/
public class ManagedRawSSTFileReader<T> implements Closeable {
+ public static boolean tryLoadLibrary() {
+ try {
+ loadLibrary();
+ return true;
+ } catch (NativeLibraryNotLoadedException ignored) {
+ return false;
+ }
+ }
+
public static boolean loadLibrary() throws NativeLibraryNotLoadedException {
ManagedRocksObjectUtils.loadRocksDBLibrary();
if
(!NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME,
Arrays.asList(
diff --git
a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
index 4f0b0ae9c7..3dbbdc3551 100644
---
a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
+++
b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.utils;
import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
+import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
import static
org.apache.hadoop.hdds.utils.NativeLibraryLoader.NATIVE_LIB_TMP_DIR;
import static
org.apache.hadoop.hdds.utils.NativeLibraryLoader.getJniLibraryFileName;
import static org.assertj.core.api.Assertions.assertThat;
@@ -37,7 +38,7 @@
import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
-import org.apache.ozone.test.tag.Native;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@@ -55,7 +56,7 @@ private static Stream<String>
nativeLibraryDirectoryLocations() {
return Stream.of("", tempDir.toAbsolutePath().toString(), null);
}
- @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
+ @EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches =
"true")
@ParameterizedTest
@MethodSource("nativeLibraryDirectoryLocations")
public void testNativeLibraryLoader(String nativeLibraryDirectoryLocation)
throws NativeLibraryNotLoadedException {
diff --git
a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileIterator.java
b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileIterator.java
index 6c7a5a3b8b..98f8708e21 100644
---
a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileIterator.java
+++
b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileIterator.java
@@ -17,7 +17,7 @@
package org.apache.hadoop.hdds.utils.db.managed;
-import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
+import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -37,9 +37,9 @@
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException;
import org.apache.hadoop.hdds.utils.TestUtils;
-import org.apache.ozone.test.tag.Native;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Named;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
@@ -48,7 +48,7 @@
/**
* Test for ManagedRawSSTFileReaderIterator.
*/
-@Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
+@EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true")
class TestManagedRawSSTFileIterator {
@TempDir
@@ -74,7 +74,7 @@ private File createSSTFileWithKeys(
return file;
}
- private static Stream<? extends Arguments> keyValueFormatArgs() {
+ private static Stream<Arguments> keyValueFormatArgs() {
return Stream.of(Arguments.of(Named.of("Key starting with a single quote",
"'key%1$d=>"),
Named.of("Value starting with a number ending with a single
quote", "%1$dvalue'")),
Arguments.of(Named.of("Key ending with a number", "key%1$d"),
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java
index 7172798cb1..bb5c243aaf 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java
@@ -17,7 +17,7 @@
package org.apache.ozone.rocksdb.util;
-import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
+import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -37,13 +37,12 @@
import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.StringUtils;
-import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException;
import org.apache.hadoop.hdds.utils.TestUtils;
import org.apache.hadoop.hdds.utils.db.managed.ManagedEnvOptions;
import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter;
-import org.apache.ozone.test.tag.Native;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
@@ -153,12 +152,12 @@ public void testGetKeyStream(int numberOfFiles)
}
}
- @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
+ @EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches =
"true")
@ParameterizedTest
@ValueSource(ints = {0, 1, 2, 3, 7, 10})
public void testGetKeyStreamWithTombstone(int numberOfFiles)
- throws RocksDBException, IOException, NativeLibraryNotLoadedException {
- assumeTrue(ManagedRawSSTFileReader.loadLibrary());
+ throws RocksDBException, IOException {
+ assumeTrue(ManagedRawSSTFileReader.tryLoadLibrary());
Pair<SortedMap<String, Integer>, List<String>> data =
createDummyData(numberOfFiles);
List<String> files = data.getRight();
diff --git
a/hadoop-hdds/test-utils/src/test/java/org/apache/ozone/test/tag/Native.java
b/hadoop-hdds/test-utils/src/test/java/org/apache/ozone/test/tag/Native.java
deleted file mode 100644
index 0e9dd8f227..0000000000
--- a/hadoop-hdds/test-utils/src/test/java/org/apache/ozone/test/tag/Native.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ozone.test.tag;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import org.junit.jupiter.api.Tag;
-
-/**
- * Annotation to mark JUnit5 test classes that require native libraries.
- */
-@Target({ ElementType.TYPE, ElementType.METHOD })
-@Retention(RetentionPolicy.RUNTIME)
-@Tag("native")
-public @interface Native {
- /**
- * Native Library being used.
- */
- String value();
-}
diff --git a/hadoop-ozone/dev-support/checks/native.sh
b/hadoop-ozone/dev-support/checks/native.sh
deleted file mode 100755
index f9d1c1b23a..0000000000
--- a/hadoop-ozone/dev-support/checks/native.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#checks:native
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-CHECK=native
-
-source "${DIR}/junit.sh" -Pnative-tests -Drocks_tools_native \
- "$@"
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
index d8b05683ff..056040e3eb 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
@@ -94,6 +94,7 @@
import org.apache.hadoop.hdds.utils.db.DBProfile;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
@@ -192,6 +193,10 @@ public TestOmSnapshot(BucketLayout newBucketLayout,
this.counter = new AtomicInteger();
this.createLinkedBucket = createLinkedBucket;
init();
+
+ if (!disableNativeDiff) {
+ assumeTrue(ManagedRawSSTFileReader.tryLoadLibrary());
+ }
}
/**
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java
index 0d672add7d..d8d2411905 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java
@@ -17,16 +17,16 @@
package org.apache.hadoop.ozone.om.snapshot;
-import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
+import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
import static
org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
-import org.apache.ozone.test.tag.Native;
import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
/**
* Test OmSnapshot for FSO bucket type when native lib is enabled.
*/
-@Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
+@EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true")
@Timeout(300)
class TestOmSnapshotFsoWithNativeLib extends TestOmSnapshot {
TestOmSnapshotFsoWithNativeLib() throws Exception {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithLinkedBuckets.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithLinkedBuckets.java
index f7672002d8..0b1cf3abbf 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithLinkedBuckets.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithLinkedBuckets.java
@@ -17,16 +17,16 @@
package org.apache.hadoop.ozone.om.snapshot;
-import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
+import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
import static
org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
-import org.apache.ozone.test.tag.Native;
import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
/**
* Test OmSnapshot for FSO bucket type when native lib is enabled.
*/
-@Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
+@EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true")
@Timeout(300)
class TestOmSnapshotFsoWithNativeLibWithLinkedBuckets extends TestOmSnapshot {
TestOmSnapshotFsoWithNativeLibWithLinkedBuckets() throws Exception {
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java
index eb6009a686..3550389e21 100644
---
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestCheckNative.java
@@ -17,15 +17,16 @@
package org.apache.hadoop.ozone.debug;
-import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
+import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
import static org.assertj.core.api.Assertions.assertThat;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ozone.test.tag.Native;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
/**
* Tests for {@link CheckNative}.
@@ -39,13 +40,14 @@ void init() {
out = GenericTestUtils.captureOut();
}
+ @DisabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches =
"true")
@Test
void testCheckNativeNotLoaded() {
executeCheckNative();
assertOutput(false);
}
- @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
+ @EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches =
"true")
@Test
void testCheckNativeRocksToolsLoaded() {
executeCheckNative();
diff --git a/pom.xml b/pom.xml
index 166fb5e885..54c847998f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -89,7 +89,7 @@
<enforced.maven.version>[3.3.0,)</enforced.maven.version>
<errorprone-annotations.version>2.29.2</errorprone-annotations.version>
<!-- test groups excluded by default (without any manual profile
activation) -->
- <excluded-test-groups>native | unhealthy</excluded-test-groups>
+ <excluded-test-groups>unhealthy</excluded-test-groups>
<exec-maven-plugin.version>3.5.0</exec-maven-plugin.version>
<failIfNoTests>false</failIfNoTests>
<frontend-maven-plugin.version>1.15.1</frontend-maven-plugin.version>
@@ -224,8 +224,8 @@
<test.build.dir>${project.build.directory}/test-dir</test.build.dir>
<!-- number of threads/forks to use when running tests in parallel, see
parallel-tests profile -->
<testsThreadCount>4</testsThreadCount>
- <!-- test groups excluded in CI (except in dedicated profiles for flaky
and native) -->
- <unstable-test-groups>flaky | native | slow |
unhealthy</unstable-test-groups>
+ <!-- test groups excluded in CI (except in dedicated profiles for flaky)
-->
+ <unstable-test-groups>flaky | slow | unhealthy</unstable-test-groups>
<vault.driver.version>5.1.0</vault.driver.version>
<weld-servlet.version>3.1.9.Final</weld-servlet.version>
<woodstox.version>5.4.0</woodstox.version>
@@ -2289,18 +2289,6 @@
<skip.npx>true</skip.npx>
</properties>
</profile>
- <profile>
- <!-- @Native tests are excluded by default. This profile removes the
exclusion if native build is enabled. -->
- <id>rocks-native-tests</id>
- <activation>
- <property>
- <name>rocks_tools_native</name>
- </property>
- </activation>
- <properties>
- <excluded-test-groups>unhealthy</excluded-test-groups>
- </properties>
- </profile>
<!-- The following profiles define test splits for CI -->
<profile>
<id>test-client</id>
@@ -2498,22 +2486,6 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<groups>flaky</groups>
- <excludedGroups>native | slow | unhealthy</excludedGroups>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
- <!-- Test split dedicated for running @Native tests -->
- <id>native-tests</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <groups>native</groups>
<excludedGroups>slow | unhealthy</excludedGroups>
</configuration>
</plugin>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
