This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 31735012d1 HDDS-12462. Use exclude rules for defining shaded
filesystem jar contents (#8008)
31735012d1 is described below
commit 31735012d155699f6287d594761a22c299bb0f35
Author: XiChen <[email protected]>
AuthorDate: Sat Mar 22 05:23:20 2025 +0800
HDDS-12462. Use exclude rules for defining shaded filesystem jar contents
(#8008)
---
hadoop-ozone/ozonefs-shaded/pom.xml | 254 ++++++++++++++++++++++++++++++++----
pom.xml | 6 +
2 files changed, 231 insertions(+), 29 deletions(-)
diff --git a/hadoop-ozone/ozonefs-shaded/pom.xml
b/hadoop-ozone/ozonefs-shaded/pom.xml
index 9d29f59dd6..8ade37ce20 100644
--- a/hadoop-ozone/ozonefs-shaded/pom.xml
+++ b/hadoop-ozone/ozonefs-shaded/pom.xml
@@ -27,7 +27,10 @@
<properties>
<!-- no tests in this module so far -->
<maven.test.skip>true</maven.test.skip>
- <shaded.prefix>org.apache.hadoop.ozone.shaded</shaded.prefix>
+
<ozone.shaded.native.prefix>org_apache_ozone_shaded</ozone.shaded.native.prefix>
+ <!-- refer to ratis thirdparty ratis.thirdparty.shaded.native.prefix -->
+
<ratis.thirdparty.shaded.native.prefix>org_apache_ratis_thirdparty_</ratis.thirdparty.shaded.native.prefix>
+ <shaded.prefix>org.apache.ozone.shaded</shaded.prefix>
</properties>
<dependencies>
@@ -35,6 +38,30 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-resolver-dns-native-macos</artifactId>
+ <classifier>osx-x86_64</classifier>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ <classifier>linux-aarch_64</classifier>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ <classifier>linux-x86_64</classifier>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-kqueue</artifactId>
+ <classifier>osx-x86_64</classifier>
+ </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>ozone-filesystem-common</artifactId>
@@ -75,6 +102,10 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.junit</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>*</artifactId>
@@ -102,6 +133,52 @@
<phase>package</phase>
<configuration>
<skip>${maven.shade.skip}</skip>
+ <artifactSet>
+ <excludes>
+ <exclude>com.sun.javadoc:*</exclude>
+ <exclude>com.sun.jndi:*</exclude>
+ <exclude>com.sun.management:*</exclude>
+ <exclude>com.sun.security:*</exclude>
+ <exclude>com.sun.tools:*</exclude>
+ <exclude>org.ietf.jgss:*</exclude>
+ <exclude>org.omg:*</exclude>
+ <exclude>org.w3c.dom:*</exclude>
+ <exclude>org.xerial.snappy:*</exclude>
+ <exclude>org.xml.sax:*</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <!-- Exclude android (dependency from io.grpc:grpc-core)
to reduce unnecessary dependencies and avoid potential conflicts -->
+ <exclude>android/**</exclude>
+ <!-- Hadoop env provides these dependencies.
+ To prevent conflicts, remove them from Ozone release
package.-->
+ <exclude>javax/annotation/**</exclude>
+ <!-- Filter ratis thirdparty shaded netty_tcnative libs -->
+
<exclude>META-INF/native/liborg_apache_ratis_thirdparty_netty_resolver_dns_native_macos_x86_64.jnilib</exclude>
+
<exclude>META-INF/native/liborg_apache_ratis_thirdparty_netty_tcnative_linux_aarch_64.so</exclude>
+
<exclude>META-INF/native/liborg_apache_ratis_thirdparty_netty_tcnative_linux_x86_64.so</exclude>
+
<exclude>META-INF/native/liborg_apache_ratis_thirdparty_netty_tcnative_osx_aarch_64.jnilib</exclude>
+
<exclude>META-INF/native/liborg_apache_ratis_thirdparty_netty_tcnative_osx_x86_64.jnilib</exclude>
+
<exclude>META-INF/native/liborg_apache_ratis_thirdparty_netty_transport_native_epoll_aarch_64.so</exclude>
+
<exclude>META-INF/native/liborg_apache_ratis_thirdparty_netty_transport_native_epoll_x86_64.so</exclude>
+
<exclude>META-INF/native/liborg_apache_ratis_thirdparty_netty_transport_native_kqueue_x86_64.jnilib</exclude>
+
<exclude>META-INF/native/org_apache_ratis_thirdparty_netty_tcnative_windows_x86_64.dll</exclude>
+ <!-- Filter unshaded netty_tcnative libs -->
+
<exclude>META-INF/native/libnetty_resolver_dns_native_macos_x86_64.jnilib</exclude>
+
<exclude>META-INF/native/libnetty_tcnative_linux_aarch_64.so</exclude>
+
<exclude>META-INF/native/libnetty_tcnative_linux_x86_64.so</exclude>
+
<exclude>META-INF/native/libnetty_tcnative_osx_aarch_64.jnilib</exclude>
+
<exclude>META-INF/native/libnetty_tcnative_osx_x86_64.jnilib</exclude>
+
<exclude>META-INF/native/libnetty_transport_native_epoll_aarch_64.so</exclude>
+
<exclude>META-INF/native/libnetty_transport_native_epoll_x86_64.so</exclude>
+
<exclude>META-INF/native/libnetty_transport_native_kqueue_x86_64.jnilib</exclude>
+
<exclude>META-INF/native/netty_tcnative_windows_x86_64.dll</exclude>
+ </excludes>
+ </filter>
+ </filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources>
@@ -120,38 +197,66 @@
<relocation>
<pattern>org</pattern>
<shadedPattern>${shaded.prefix}.org</shadedPattern>
- <includes>
- <include>org.yaml.**.*</include>
- <include>org.sqlite.**.*</include>
- <include>org.tukaani.**.*</include>
- <include>org.bouncycastle.**.*</include>
- <include>org.rocksdb.**.*</include>
- <include>org.apache.commons.cli.**.*</include>
- <include>org.apache.commons.compress.**.*</include>
- <include>org.apache.commons.codec.**.*</include>
- <include>org.apache.commons.beanutils.**.*</include>
- <include>org.apache.commons.collections.**.*</include>
- <include>org.apache.commons.digester.**.*</include>
- <include>org.apache.commons.io.**.*</include>
- <include>org.apache.commons.logging.**.*</include>
- <include>org.apache.commons.validator.**.*</include>
- <include>org.apache.commons.lang3.**.*</include>
- <include>org.sqlite.**.*</include>
- <include>org.apache.thrift.**.*</include>
- </includes>
+ <excludes>
+ <exclude>org.apache.hadoop.**</exclude>
+ <exclude>org.apache.log4j.**</exclude>
+ <exclude>org.apache.ozone.**</exclude>
+ <exclude>org.apache.ratis.**</exclude>
+ <!-- These packages are the ones that are not shaded in
the hadoop release package.
+ To prevent conflicts, we do not include these packages
in the ozone client release package.
+ Ozone will use hadoop / JVM to provide these
dependencies. -->
+ <exclude>org.ietf.jgss.*</exclude>
+ <exclude>org.omg.**</exclude>
+ <exclude>org.slf4j.**</exclude>
+ <exclude>org.w3c.dom.**</exclude>
+ <exclude>org.xerial.snappy.**</exclude>
+ <exclude>org.xml.sax.**</exclude>
+ <exclude>org.wildfly.**</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.ratis</pattern>
+
<shadedPattern>${shaded.prefix}.org.apache.ratis</shadedPattern>
</relocation>
<relocation>
<pattern>com</pattern>
<shadedPattern>${shaded.prefix}.com</shadedPattern>
- <includes>
- <include>com.google.common.**.*</include>
- <include>com.google.gson.**.*</include>
- <include>com.codahale.**.*</include>
- <include>com.fasterxml.**.*</include>
- <include>com.lmax.**.*</include>
- <include>com.github.joshelser.**.*</include>
- <include>com.twitter.**.*</include>
- </includes>
+ <excludes>
+ <exclude>com.google.protobuf.**</exclude>
+ <!-- These packages are the ones that are not shaded in
the hadoop release package.
+ To prevent conflicts, we do not include these packages in
the ozone client release package.
+ Ozone will use hadoop / JVM to provide these dependencies.
-->
+ <exclude>com.sun.javadoc.**</exclude>
+ <exclude>com.sun.jndi.**</exclude>
+ <exclude>com.sun.management.**</exclude>
+ <exclude>com.sun.security.**</exclude>
+ <exclude>com.sun.tools.**</exclude>
+ <exclude>com.ibm.security.**</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>google</pattern>
+ <shadedPattern>${shaded.prefix}.google</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>net.jcip</pattern>
+ <shadedPattern>${shaded.prefix}.net.jcip</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javassist</pattern>
+ <shadedPattern>${shaded.prefix}.javassist</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax.xml.bind</pattern>
+
<shadedPattern>${shaded.prefix}.javax.xml.bind</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax.activation</pattern>
+
<shadedPattern>${shaded.prefix}.javax.activation</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>jakarta.annotation</pattern>
+
<shadedPattern>${shaded.prefix}.jakarta.annotation</shadedPattern>
</relocation>
<relocation>
<pattern>kotlin</pattern>
@@ -170,6 +275,18 @@
<shadedPattern>${shaded.prefix}.io</shadedPattern>
<excludes>
<exclude>io!netty!*</exclude>
+ <!-- Exclude config keys for Hadoop that look like package
names -->
+ <!-- These packages are the ones that are not shaded in
the hadoop release package.
+ To prevent conflicts, we do not include these
packages in the ozone client release package.
+ Ozone will use hadoop / JVM to provide these
dependencies. -->
+ <exclude>io.compression.**</exclude>
+ <exclude>io.file.buffer.size</exclude>
+ <exclude>io.mapfile.**</exclude>
+ <exclude>io.map.index</exclude>
+ <exclude>io.seqfile.**</exclude>
+ <exclude>io.serializations</exclude>
+ <exclude>io.skip.checksum.errors</exclude>
+ <exclude>io.sort.**</exclude>
</excludes>
</relocation>
@@ -187,6 +304,85 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>unpack-dependencies</id>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <phase>validate</phase>
+ <configuration>
+ <includeGroupIds>io.netty</includeGroupIds>
+ <includes>**/META-INF/native/*</includes>
+ <includeArtifactIds>netty-resolver-dns-native-macos,
+ netty-tcnative-boringssl-static,
+ netty-transport-native-epoll,
+ netty-transport-native-kqueue</includeArtifactIds>
+
<outputDirectory>${project.build.directory}/classes/</outputDirectory>
+ <overWriteReleases>true</overWriteReleases>
+ <overWriteSnapshots>true</overWriteSnapshots>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- The native libraries need to be named in the same way we relocate
the Netty classes via the shade-plugin -->
+ <!-- We shaded ratis thirdparty, so the dynamic library name here
includes both ratis thirdparty shaded and ozone shaded-->
+ <plugin>
+ <groupId>com.coderplus.maven.plugins</groupId>
+ <artifactId>copy-rename-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>rename-file</id>
+ <goals>
+ <goal>rename</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <fileSets>
+ <fileSet>
+
<sourceFile>${project.build.directory}/classes/META-INF/native/libnetty_resolver_dns_native_macos_x86_64.jnilib</sourceFile>
+
<destinationFile>${project.build.directory}/classes/META-INF/native/lib${ozone.shaded.native.prefix}_${ratis.thirdparty.shaded.native.prefix}netty_resolver_dns_native_macos_x86_64.jnilib</destinationFile>
+ </fileSet>
+ <fileSet>
+
<sourceFile>${project.build.directory}/classes/META-INF/native/libnetty_transport_native_epoll_aarch_64.so</sourceFile>
+
<destinationFile>${project.build.directory}/classes/META-INF/native/lib${ozone.shaded.native.prefix}_${ratis.thirdparty.shaded.native.prefix}netty_transport_native_epoll_aarch_64.so</destinationFile>
+ </fileSet>
+ <fileSet>
+
<sourceFile>${project.build.directory}/classes/META-INF/native/libnetty_transport_native_epoll_x86_64.so</sourceFile>
+
<destinationFile>${project.build.directory}/classes/META-INF/native/lib${ozone.shaded.native.prefix}_${ratis.thirdparty.shaded.native.prefix}netty_transport_native_epoll_x86_64.so</destinationFile>
+ </fileSet>
+ <fileSet>
+
<sourceFile>${project.build.directory}/classes/META-INF/native/libnetty_transport_native_kqueue_x86_64.jnilib</sourceFile>
+
<destinationFile>${project.build.directory}/classes/META-INF/native/lib${ozone.shaded.native.prefix}_${ratis.thirdparty.shaded.native.prefix}netty_transport_native_kqueue_x86_64.jnilib</destinationFile>
+ </fileSet>
+ <fileSet>
+
<sourceFile>${project.build.directory}/classes/META-INF/native/libnetty_tcnative_linux_aarch_64.so</sourceFile>
+
<destinationFile>${project.build.directory}/classes/META-INF/native/lib${ozone.shaded.native.prefix}_${ratis.thirdparty.shaded.native.prefix}netty_tcnative_linux_aarch_64.so</destinationFile>
+ </fileSet>
+ <fileSet>
+
<sourceFile>${project.build.directory}/classes/META-INF/native/libnetty_tcnative_linux_x86_64.so</sourceFile>
+
<destinationFile>${project.build.directory}/classes/META-INF/native/lib${ozone.shaded.native.prefix}_${ratis.thirdparty.shaded.native.prefix}netty_tcnative_linux_x86_64.so</destinationFile>
+ </fileSet>
+ <fileSet>
+
<sourceFile>${project.build.directory}/classes/META-INF/native/libnetty_tcnative_osx_aarch_64.jnilib</sourceFile>
+
<destinationFile>${project.build.directory}/classes/META-INF/native/lib${ozone.shaded.native.prefix}_${ratis.thirdparty.shaded.native.prefix}netty_tcnative_osx_aarch_64.jnilib</destinationFile>
+ </fileSet>
+ <fileSet>
+
<sourceFile>${project.build.directory}/classes/META-INF/native/libnetty_tcnative_osx_x86_64.jnilib</sourceFile>
+
<destinationFile>${project.build.directory}/classes/META-INF/native/lib${ozone.shaded.native.prefix}_${ratis.thirdparty.shaded.native.prefix}libnetty_tcnative_osx_x86_64.jnilib</destinationFile>
+ </fileSet>
+ <fileSet>
+
<sourceFile>${project.build.directory}/classes/META-INF/native/netty_tcnative_windows_x86_64.dll</sourceFile>
+
<destinationFile>${project.build.directory}/classes/META-INF/native/${ozone.shaded.native.prefix}_${ratis.thirdparty.shaded.native.prefix}netty_tcnative_windows_x86_64.dll</destinationFile>
+ </fileSet>
+ </fileSets>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/pom.xml b/pom.xml
index 47fa1b83ba..7db1fd60bf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,6 +62,7 @@
<commons-text.version>1.12.0</commons-text.version>
<commons-validator.version>1.6</commons-validator.version>
<compile-testing.version>0.21.0</compile-testing.version>
+ <copy-rename-maven-plugin.version>1.0</copy-rename-maven-plugin.version>
<curator.version>4.2.0</curator.version>
<cyclonedx.version>2.9.1</cyclonedx.version>
<!-- the version of Hadoop declared in the version resources; can be
overridden
@@ -1777,6 +1778,11 @@
<includeTests>true</includeTests>
</configuration>
</plugin>
+ <plugin>
+ <groupId>com.coderplus.maven.plugins</groupId>
+ <artifactId>copy-rename-maven-plugin</artifactId>
+ <version>${copy-rename-maven-plugin.version}</version>
+ </plugin>
</plugins>
</pluginManagement>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
