This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new fbbfa798698 HDDS-5410. Improve protobuf compilation (#9081)
fbbfa798698 is described below
commit fbbfa798698caca5bf450630f69b6402a7ee6197
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Mon Oct 6 17:35:40 2025 +0200
HDDS-5410. Improve protobuf compilation (#9081)
---
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 4 +-
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 16 +-------
.../main/resources/hdds-version-info.properties | 4 +-
hadoop-hdds/interface-admin/pom.xml | 9 ++---
hadoop-hdds/interface-client/pom.xml | 45 ++++++++++------------
hadoop-hdds/interface-server/pom.xml | 26 ++++++-------
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 4 +-
.../main/resources/ozone-version-info.properties | 4 +-
hadoop-ozone/csi/pom.xml | 23 +++++------
hadoop-ozone/interface-client/pom.xml | 36 ++++++++---------
hadoop-ozone/interface-storage/pom.xml | 10 ++---
hadoop-ozone/ozonefs-hadoop2/pom.xml | 2 +-
hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 2 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 2 +-
pom.xml | 25 ++++++++----
15 files changed, 92 insertions(+), 120 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/HddsVersionInfo.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/HddsVersionInfo.java
index 5ca2576387c..8e91db8b617 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/HddsVersionInfo.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/HddsVersionInfo.java
@@ -44,9 +44,7 @@ public static void main(String[] args) {
"Source code repository " + HDDS_VERSION_INFO.getUrl() + " -r " +
HDDS_VERSION_INFO.getRevision());
System.out.println(
- "Compiled with protoc " + HDDS_VERSION_INFO.getHadoopProtoc2Version() +
- ", " + HDDS_VERSION_INFO.getGrpcProtocVersion() +
- " and " + HDDS_VERSION_INFO.getHadoopProtoc3Version());
+ "Compiled with protoc " + HDDS_VERSION_INFO.getProtoVersions());
System.out.println(
"From source with checksum " + HDDS_VERSION_INFO.getSrcChecksum());
System.out.println(
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/VersionInfo.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/VersionInfo.java
index a0d703a4873..e34a4885f03 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/VersionInfo.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/VersionInfo.java
@@ -71,20 +71,8 @@ public String getSrcChecksum() {
return info.getProperty("srcChecksum", "Unknown");
}
- public String getHadoopProtoc2Version() {
- return info.getProperty("hadoopProtoc2Version", "Unknown");
- }
-
- public String getHadoopProtocVersion() {
- return getHadoopProtoc2Version();
- }
-
- public String getHadoopProtoc3Version() {
- return info.getProperty("hadoopProtoc3Version", "Unknown");
- }
-
- public String getGrpcProtocVersion() {
- return info.getProperty("grpcProtocVersion", "Unknown");
+ public String getProtoVersions() {
+ return info.getProperty("protoVersions", "Unknown");
}
public String getCompilePlatform() {
diff --git a/hadoop-hdds/common/src/main/resources/hdds-version-info.properties
b/hadoop-hdds/common/src/main/resources/hdds-version-info.properties
index 3ba2c2cbfa2..38ce15bf2d9 100644
--- a/hadoop-hdds/common/src/main/resources/hdds-version-info.properties
+++ b/hadoop-hdds/common/src/main/resources/hdds-version-info.properties
@@ -20,7 +20,5 @@ version=${declared.hdds.version}
revision=${version-info.scm.commit}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}
-hadoopProtoc2Version=${proto2.hadooprpc.protobuf.version}
-hadoopProtoc3Version=${proto3.hadooprpc.protobuf.version}
-grpcProtocVersion=${grpc.protobuf-compile.version}
+protoVersions=${protobuf2.version}, ${protobuf3.version},
${hadoop-thirdparty.protobuf.version} (Hadoop),
${ratis-thirdparty.protobuf.version} (Ratis)
compilePlatform=${os.detected.classifier}
diff --git a/hadoop-hdds/interface-admin/pom.xml
b/hadoop-hdds/interface-admin/pom.xml
index 2cee06431b2..0d0dedb35c5 100644
--- a/hadoop-hdds/interface-admin/pom.xml
+++ b/hadoop-hdds/interface-admin/pom.xml
@@ -28,6 +28,7 @@
<properties>
<!-- no testable code in this module -->
<maven.test.skip>true</maven.test.skip>
+ <protobuf.version>${protobuf2.version}</protobuf.version>
<!-- only generated code in this module -->
<spotbugs.skip>true</spotbugs.skip>
</properties>
@@ -59,19 +60,15 @@
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
- <version>${protobuf-maven-plugin.version}</version>
- <extensions>true</extensions>
<executions>
<execution>
- <id>compile-protoc-2</id>
+ <id>compile-proto-${protobuf.version}</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
</goals>
<configuration>
-
<protocArtifact>com.google.protobuf:protoc:${proto2.hadooprpc.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
- <outputDirectory>target/generated-sources/java</outputDirectory>
+
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
<clearOutputDirectory>false</clearOutputDirectory>
</configuration>
</execution>
diff --git a/hadoop-hdds/interface-client/pom.xml
b/hadoop-hdds/interface-client/pom.xml
index 1e876c1199b..18d02e29225 100644
--- a/hadoop-hdds/interface-client/pom.xml
+++ b/hadoop-hdds/interface-client/pom.xml
@@ -39,7 +39,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop.thirdparty</groupId>
- <artifactId>hadoop-shaded-protobuf_3_25</artifactId>
+ <artifactId>${hadoop-thirdparty.protobuf.artifact}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.ratis</groupId>
@@ -68,11 +68,9 @@
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
- <version>${protobuf-maven-plugin.version}</version>
- <extensions>true</extensions>
<executions>
<execution>
- <id>compile-protoc-grpc</id>
+ <id>compile-proto-for-ratis</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
@@ -80,49 +78,45 @@
<goal>test-compile-custom</goal>
</goals>
<configuration>
-
<protocArtifact>com.google.protobuf:protoc:${grpc.protobuf-compile.version}:exe:${os.detected.classifier}</protocArtifact>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+
<protocArtifact>com.google.protobuf:protoc:${ratis-thirdparty.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
<includes>
<include>DatanodeClientProtocol.proto</include>
- <include>SCMClientProtocol.proto</include>
</includes>
- <outputDirectory>target/generated-sources/java</outputDirectory>
+
<outputDirectory>target/generated-sources/proto-java-for-ratis</outputDirectory>
<clearOutputDirectory>false</clearOutputDirectory>
<pluginId>grpc-java</pluginId>
-
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${io.grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${ratis-thirdparty.grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
</configuration>
</execution>
<execution>
- <id>compile-protoc-2</id>
+ <id>compile-proto-${protobuf2.version}</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
</goals>
<configuration>
-
<protocArtifact>com.google.protobuf:protoc:${proto2.hadooprpc.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+
<protocArtifact>com.google.protobuf:protoc:${protobuf2.version}:exe:${os.detected.classifier}</protocArtifact>
<includes>
<include>hdds.proto</include>
<include>ReconfigureProtocol.proto</include>
</includes>
- <outputDirectory>target/generated-sources/java</outputDirectory>
+
<outputDirectory>target/generated-sources/proto-java-for-protobuf-${protobuf2.version}</outputDirectory>
<clearOutputDirectory>false</clearOutputDirectory>
</configuration>
</execution>
<execution>
- <id>compile-protoc-3</id>
+ <id>compile-proto-for-hadoop</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
</goals>
<configuration>
-
<protocArtifact>com.google.protobuf:protoc:${proto3.hadooprpc.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+
<protocArtifact>com.google.protobuf:protoc:${hadoop-thirdparty.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
<includes>
<include>hdds.proto</include>
<include>ReconfigureProtocol.proto</include>
</includes>
-
<outputDirectory>target/generated-sources/java/proto3</outputDirectory>
+
<outputDirectory>target/generated-sources/proto-java-for-hadoop</outputDirectory>
<clearOutputDirectory>false</clearOutputDirectory>
</configuration>
</execution>
@@ -139,13 +133,16 @@
<phase>generate-sources</phase>
<configuration>
<target>
- <replace
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto"
token="com.google.protobuf"
value="org.apache.ratis.thirdparty.com.google.protobuf" />
- <replace
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto"
token="io.grpc" value="org.apache.ratis.thirdparty.io.grpc" />
- <replace
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto"
token="com.google.common"
value="org.apache.ratis.thirdparty.com.google.common" />
- <replace
dir="target/generated-sources/java/proto3/org/apache/hadoop/hdds/protocol/proto"
token="org.apache.hadoop.hdds.protocol.proto"
value="org.apache.hadoop.hdds.protocol.proto3" />
- <replace
dir="target/generated-sources/java/proto3/org/apache/hadoop/hdds/protocol/proto"
token="com.google.protobuf" value="org.apache.hadoop.thirdparty.protobuf" />
- <move
file="target/generated-sources/java/proto3/org/apache/hadoop/hdds/protocol/proto"
tofile="target/generated-sources/java/proto3/org/apache/hadoop/hdds/protocol/proto3"
/>
- <move file="target/generated-sources/java/proto3"
tofile="target/generated-sources/java" />
+ <!-- use grpc, guava, protobuf from ratis-thirdparty -->
+ <replace dir="target/generated-sources/proto-java-for-ratis"
token="com.google.common" value="org.apache.ratis.thirdparty.com.google.common"
/>
+ <replace dir="target/generated-sources/proto-java-for-ratis"
token="com.google.protobuf"
value="org.apache.ratis.thirdparty.com.google.protobuf" />
+ <replace dir="target/generated-sources/proto-java-for-ratis"
token="io.grpc" value="org.apache.ratis.thirdparty.io.grpc" />
+
+ <!-- use protobuf from hadoop-thirdparty -->
+ <replace dir="target/generated-sources/proto-java-for-hadoop"
token="com.google.protobuf" value="org.apache.hadoop.thirdparty.protobuf" />
+ <!-- rename package to avoid conflict -->
+ <replace dir="target/generated-sources/proto-java-for-hadoop"
token="org.apache.hadoop.hdds.protocol.proto"
value="org.apache.hadoop.hdds.protocol.hadoop_proto" />
+ <move
file="target/generated-sources/proto-java-for-hadoop/org/apache/hadoop/hdds/protocol/proto"
tofile="target/generated-sources/proto-java-for-hadoop/org/apache/hadoop/hdds/protocol/hadoop_proto"
/>
</target>
</configuration>
</execution>
diff --git a/hadoop-hdds/interface-server/pom.xml
b/hadoop-hdds/interface-server/pom.xml
index cee021923a3..90c462658d8 100644
--- a/hadoop-hdds/interface-server/pom.xml
+++ b/hadoop-hdds/interface-server/pom.xml
@@ -28,6 +28,7 @@
<properties>
<!-- no testable code in this module -->
<maven.test.skip>true</maven.test.skip>
+ <protobuf.version>${protobuf2.version}</protobuf.version>
<!-- only generated code in this module -->
<spotbugs.skip>true</spotbugs.skip>
</properties>
@@ -69,11 +70,9 @@
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
- <version>${protobuf-maven-plugin.version}</version>
- <extensions>true</extensions>
<executions>
<execution>
- <id>compile-protoc-3</id>
+ <id>compile-proto-for-ratis</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
@@ -81,32 +80,30 @@
<goal>test-compile-custom</goal>
</goals>
<configuration>
-
<protocArtifact>com.google.protobuf:protoc:${grpc.protobuf-compile.version}:exe:${os.detected.classifier}</protocArtifact>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+
<protocArtifact>com.google.protobuf:protoc:${ratis-thirdparty.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
<includes>
<include>InterSCMProtocol.proto</include>
<include>SCMUpdateProtocol.proto</include>
</includes>
- <outputDirectory>target/generated-sources/java</outputDirectory>
+
<outputDirectory>target/generated-sources/proto-java-for-ratis</outputDirectory>
<clearOutputDirectory>false</clearOutputDirectory>
<pluginId>grpc-java</pluginId>
-
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${io.grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${ratis-thirdparty.grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
</configuration>
</execution>
<execution>
- <id>compile-protoc-2</id>
+ <id>compile-proto-${protobuf.version}</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
</goals>
<configuration>
-
<protocArtifact>com.google.protobuf:protoc:${proto2.hadooprpc.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
<excludes>
<exclude>InterSCMProtocol.proto</exclude>
<exclude>SCMUpdateProtocol.proto</exclude>
</excludes>
- <outputDirectory>target/generated-sources/java</outputDirectory>
+
<outputDirectory>target/generated-sources/proto-java-for-protobuf-${protobuf.version}</outputDirectory>
<clearOutputDirectory>false</clearOutputDirectory>
</configuration>
</execution>
@@ -123,9 +120,10 @@
<phase>generate-sources</phase>
<configuration>
<target>
- <replace
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/scm/proto"
token="com.google.protobuf"
value="org.apache.ratis.thirdparty.com.google.protobuf" />
- <replace
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/scm/proto"
token="io.grpc" value="org.apache.ratis.thirdparty.io.grpc" />
- <replace
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/scm/proto"
token="com.google.common" value="org.apache.ratis.thirdparty.com.google.common"
/>
+ <!-- use grpc, guava, protobuf from ratis-thirdparty -->
+ <replace dir="target/generated-sources/proto-java-for-ratis"
token="com.google.common" value="org.apache.ratis.thirdparty.com.google.common"
/>
+ <replace dir="target/generated-sources/proto-java-for-ratis"
token="com.google.protobuf"
value="org.apache.ratis.thirdparty.com.google.protobuf" />
+ <replace dir="target/generated-sources/proto-java-for-ratis"
token="io.grpc" value="org.apache.ratis.thirdparty.io.grpc" />
</target>
</configuration>
</execution>
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/OzoneVersionInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/OzoneVersionInfo.java
index ad4e3429a0e..b3391ebbd61 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/OzoneVersionInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/OzoneVersionInfo.java
@@ -82,9 +82,7 @@ public static void main(String[] args) {
"Source code repository " + OZONE_VERSION_INFO.getUrl() + " -r " +
OZONE_VERSION_INFO.getRevision());
System.out.println(
- "Compiled with protoc " + OZONE_VERSION_INFO.getHadoopProtoc2Version()
+
- ", " + OZONE_VERSION_INFO.getGrpcProtocVersion() +
- " and " + OZONE_VERSION_INFO.getHadoopProtoc3Version());
+ "Compiled with protoc " + OZONE_VERSION_INFO.getProtoVersions());
System.out.println(
"From source with checksum " + OZONE_VERSION_INFO.getSrcChecksum());
System.out.println(
diff --git
a/hadoop-ozone/common/src/main/resources/ozone-version-info.properties
b/hadoop-ozone/common/src/main/resources/ozone-version-info.properties
index 73f02760d6f..02d76c04cd7 100644
--- a/hadoop-ozone/common/src/main/resources/ozone-version-info.properties
+++ b/hadoop-ozone/common/src/main/resources/ozone-version-info.properties
@@ -21,7 +21,5 @@ release=${ozone.release}
revision=${version-info.scm.commit}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}
-hadoopProtoc2Version=${proto2.hadooprpc.protobuf.version}
-hadoopProtoc3Version=${proto3.hadooprpc.protobuf.version}
-grpcProtocVersion=${grpc.protobuf-compile.version}
+protoVersions=${protobuf2.version}, ${protobuf3.version},
${hadoop-thirdparty.protobuf.version} (Hadoop),
${ratis-thirdparty.protobuf.version} (Ratis)
compilePlatform=${os.detected.classifier}
diff --git a/hadoop-ozone/csi/pom.xml b/hadoop-ozone/csi/pom.xml
index df81c787fb4..511c9b08cea 100644
--- a/hadoop-ozone/csi/pom.xml
+++ b/hadoop-ozone/csi/pom.xml
@@ -30,6 +30,7 @@
<maven.javadoc.skip>true</maven.javadoc.skip>
<!-- no tests in this module so far -->
<maven.test.skip>true</maven.test.skip>
+ <protobuf.version>${protobuf3.version}</protobuf.version>
</properties>
<dependencies>
@@ -40,7 +41,7 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>${grpc.protobuf-compile.version}</version>
+ <version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
@@ -155,7 +156,7 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
- <version>${grpc.protobuf-compile.version}</version>
+ <version>${protobuf.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -221,20 +222,9 @@
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
- <version>${protobuf-maven-plugin.version}</version>
- <extensions>true</extensions>
- <configuration>
-
<protocArtifact>com.google.protobuf:protoc:${grpc.protobuf-compile.version}:exe:${os.detected.classifier}</protocArtifact>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
- <includes>
- <include>csi.proto</include>
- </includes>
- <outputDirectory>target/generated-sources/java</outputDirectory>
- <clearOutputDirectory>false</clearOutputDirectory>
- </configuration>
<executions>
<execution>
- <id>compile-protoc</id>
+ <id>compile-proto-${protobuf.version}</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
@@ -244,6 +234,11 @@
<configuration>
<pluginId>grpc-java</pluginId>
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${io.grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+ <includes>
+ <include>csi.proto</include>
+ </includes>
+ <clearOutputDirectory>false</clearOutputDirectory>
</configuration>
</execution>
</executions>
diff --git a/hadoop-ozone/interface-client/pom.xml
b/hadoop-ozone/interface-client/pom.xml
index 894e1209301..cd1aabff531 100644
--- a/hadoop-ozone/interface-client/pom.xml
+++ b/hadoop-ozone/interface-client/pom.xml
@@ -67,7 +67,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop.thirdparty</groupId>
- <artifactId>hadoop-shaded-protobuf_3_25</artifactId>
+ <artifactId>${hadoop-thirdparty.protobuf.artifact}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
@@ -106,11 +106,9 @@
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
- <version>${protobuf-maven-plugin.version}</version>
- <extensions>true</extensions>
<executions>
<execution>
- <id>compile-protoc-OmGrpc</id>
+ <id>compile-proto-${protobuf2.version}</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
@@ -118,24 +116,22 @@
<goal>test-compile-custom</goal>
</goals>
<configuration>
-
<protocArtifact>com.google.protobuf:protoc:${proto2.hadooprpc.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
-
<outputDirectory>target/generated-sources/protobuf/java</outputDirectory>
+
<protocArtifact>com.google.protobuf:protoc:${protobuf2.version}:exe:${os.detected.classifier}</protocArtifact>
+
<outputDirectory>target/generated-sources/proto-java-protobuf-${protobuf2.version}</outputDirectory>
<clearOutputDirectory>false</clearOutputDirectory>
<pluginId>grpc-java</pluginId>
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${io.grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
</configuration>
</execution>
<execution>
- <id>compile-protoc3</id>
+ <id>compile-proto-for-hadoop</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
</goals>
<configuration>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
-
<protocArtifact>com.google.protobuf:protoc:${proto3.hadooprpc.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
-
<outputDirectory>target/generated-sources/protobuf/java/proto3</outputDirectory>
+
<protocArtifact>com.google.protobuf:protoc:${hadoop-thirdparty.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+
<outputDirectory>target/generated-sources/proto-java-for-hadoop</outputDirectory>
<clearOutputDirectory>false</clearOutputDirectory>
</configuration>
</execution>
@@ -152,15 +148,15 @@
<phase>generate-sources</phase>
<configuration>
<target>
- <replace
dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"
token="com.google.protobuf" value="org.apache.hadoop.thirdparty.protobuf" />
- <replace
dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto"
token="com.google.protobuf" value="org.apache.hadoop.thirdparty.protobuf" />
- <replace
dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"
token="org.apache.hadoop.ozone.protocol.proto"
value="org.apache.hadoop.ozone.protocol.proto3" />
- <replace
dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto"
token="org.apache.hadoop.ozone.security.proto"
value="org.apache.hadoop.ozone.security.proto3" />
- <replace
dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"
token="org.apache.hadoop.hdds.protocol.proto"
value="org.apache.hadoop.hdds.protocol.proto3" />
- <replace
dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"
token="org.apache.hadoop.ozone.security.proto"
value="org.apache.hadoop.ozone.security.proto3" />
- <move
file="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"
tofile="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto3"
/>
- <move
file="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto"
tofile="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto3"
/>
- <move file="target/generated-sources/protobuf/java/proto3"
tofile="target/generated-sources/protobuf/java/" />
+ <!-- use protobuf from hadoop-thirdparty -->
+ <replace dir="target/generated-sources/proto-java-for-hadoop"
token="com.google.protobuf" value="org.apache.hadoop.thirdparty.protobuf" />
+ <!-- use matching hdds proto -->
+ <replace dir="target/generated-sources/proto-java-for-hadoop"
token="org.apache.hadoop.hdds.protocol.proto"
value="org.apache.hadoop.hdds.protocol.hadoop_proto" />
+ <!-- rename packages to avoid conflict -->
+ <replace dir="target/generated-sources/proto-java-for-hadoop"
token="org.apache.hadoop.ozone.protocol.proto"
value="org.apache.hadoop.ozone.protocol.hadoop_proto" />
+ <replace dir="target/generated-sources/proto-java-for-hadoop"
token="org.apache.hadoop.ozone.security.proto"
value="org.apache.hadoop.ozone.security.hadoop_proto" />
+ <move
file="target/generated-sources/proto-java-for-hadoop/org/apache/hadoop/ozone/protocol/proto"
tofile="target/generated-sources/proto-java-for-hadoop/org/apache/hadoop/ozone/protocol/hadoop_proto"
/>
+ <move
file="target/generated-sources/proto-java-for-hadoop/org/apache/hadoop/ozone/security/proto"
tofile="target/generated-sources/proto-java-for-hadoop/org/apache/hadoop/ozone/security/hadoop_proto"
/>
</target>
</configuration>
</execution>
diff --git a/hadoop-ozone/interface-storage/pom.xml
b/hadoop-ozone/interface-storage/pom.xml
index 5d1b4cc137e..ae54089416b 100644
--- a/hadoop-ozone/interface-storage/pom.xml
+++ b/hadoop-ozone/interface-storage/pom.xml
@@ -24,6 +24,9 @@
<packaging>jar</packaging>
<name>Apache Ozone Storage Interface</name>
<description>Apache Ozone Storage Interface</description>
+ <properties>
+ <protobuf.version>${protobuf2.version}</protobuf.version>
+ </properties>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
@@ -93,18 +96,15 @@
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
- <version>${protobuf-maven-plugin.version}</version>
- <extensions>true</extensions>
<executions>
<execution>
- <id>compile-protoc</id>
+ <id>compile-proto-${protobuf.version}</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
</goals>
<configuration>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
-
<protocArtifact>com.google.protobuf:protoc:${proto2.hadooprpc.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
</configuration>
</execution>
</executions>
diff --git a/hadoop-ozone/ozonefs-hadoop2/pom.xml
b/hadoop-ozone/ozonefs-hadoop2/pom.xml
index ccc072e7fc7..2e1ea5ae9f2 100644
--- a/hadoop-ozone/ozonefs-hadoop2/pom.xml
+++ b/hadoop-ozone/ozonefs-hadoop2/pom.xml
@@ -33,7 +33,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop.thirdparty</groupId>
- <artifactId>hadoop-shaded-protobuf_3_25</artifactId>
+ <artifactId>${hadoop-thirdparty.protobuf.artifact}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
diff --git a/hadoop-ozone/ozonefs-hadoop3-client/pom.xml
b/hadoop-ozone/ozonefs-hadoop3-client/pom.xml
index fdbff700610..2871f512245 100644
--- a/hadoop-ozone/ozonefs-hadoop3-client/pom.xml
+++ b/hadoop-ozone/ozonefs-hadoop3-client/pom.xml
@@ -50,7 +50,7 @@
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
- <artifactId>hadoop-shaded-protobuf_3_25</artifactId>
+ <artifactId>${hadoop-thirdparty.protobuf.artifact}</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
diff --git a/hadoop-ozone/ozonefs-hadoop3/pom.xml
b/hadoop-ozone/ozonefs-hadoop3/pom.xml
index 1d26aafc2d9..32a21fb4e16 100644
--- a/hadoop-ozone/ozonefs-hadoop3/pom.xml
+++ b/hadoop-ozone/ozonefs-hadoop3/pom.xml
@@ -31,7 +31,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop.thirdparty</groupId>
- <artifactId>hadoop-shaded-protobuf_3_25</artifactId>
+ <artifactId>${hadoop-thirdparty.protobuf.artifact}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
diff --git a/pom.xml b/pom.xml
index 09f916ec61e..5d17a1736bd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -92,10 +92,12 @@
<exec-maven-plugin.version>3.5.1</exec-maven-plugin.version>
<failIfNoTests>false</failIfNoTests>
<frontend-maven-plugin.version>1.15.4</frontend-maven-plugin.version>
- <grpc.protobuf-compile.version>3.19.6</grpc.protobuf-compile.version>
<gson.version>2.13.1</gson.version>
<guava.version>33.5.0-jre</guava.version>
<guice.version>6.0.0</guice.version>
+ <!-- versions included in hadoop-thirdparty, update in sync (+ jar-report
and LICENSE) -->
+
<hadoop-thirdparty.protobuf.artifact>hadoop-shaded-protobuf_3_25</hadoop-thirdparty.protobuf.artifact>
+
<hadoop-thirdparty.protobuf.version>3.25.5</hadoop-thirdparty.protobuf.version>
<hadoop-thirdparty.version>1.4.0</hadoop-thirdparty.version>
<hadoop.version>3.4.2</hadoop.version>
<hadoop2.version>2.10.2</hadoop2.version>
@@ -192,13 +194,14 @@
<prometheus.version>0.16.0</prometheus.version>
<properties.maven.plugin.version>1.2.1</properties.maven.plugin.version>
<proto-backwards-compatibility.version>1.0.7</proto-backwards-compatibility.version>
- <!-- ProtocolBuffer version, used to verify the protoc version and -->
- <!-- define the protobuf JAR version -->
-
<proto2.hadooprpc.protobuf.version>2.5.0</proto2.hadooprpc.protobuf.version>
-
<proto3.hadooprpc.protobuf.version>3.25.5</proto3.hadooprpc.protobuf.version>
- <!-- Maven protoc compiler -->
<protobuf-maven-plugin.version>0.6.1</protobuf-maven-plugin.version>
+ <protobuf2.version>2.5.0</protobuf2.version>
+ <protobuf3.version>3.25.8</protobuf3.version>
<ranger.version>2.6.0</ranger.version>
+ <!-- versions included in ratis-thirdparty, update in sync -->
+ <ratis-thirdparty.grpc.version>1.71.0</ratis-thirdparty.grpc.version>
+
<ratis-thirdparty.netty.version>4.1.119.Final</ratis-thirdparty.netty.version>
+
<ratis-thirdparty.protobuf.version>3.25.5</ratis-thirdparty.protobuf.version>
<ratis.thirdparty.version>1.0.9</ratis.thirdparty.version>
<ratis.version>3.2.0</ratis.version>
<re2j.version>1.7</re2j.version>
@@ -410,7 +413,7 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>${proto2.hadooprpc.protobuf.version}</version>
+ <version>${protobuf2.version}</version>
</dependency>
<dependency>
<groupId>com.google.re2j</groupId>
@@ -956,7 +959,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop.thirdparty</groupId>
- <artifactId>hadoop-shaded-protobuf_3_25</artifactId>
+ <artifactId>${hadoop-thirdparty.protobuf.artifact}</artifactId>
<version>${hadoop-thirdparty.version}</version>
</dependency>
<dependency>
@@ -2216,6 +2219,12 @@
<includeTests>true</includeTests>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <version>${protobuf-maven-plugin.version}</version>
+ <extensions>true</extensions>
+ </plugin>
</plugins>
</pluginManagement>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
