This is an automated email from the ASF dual-hosted git repository.
MartijnVisser pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 687ef8c4363 [FLINK-37672][build] Migrate protobuf code generation to
ascopes plugin (#29434)
687ef8c4363 is described below
commit 687ef8c43630a1691ba2d87c6430635adce31b86
Author: Martijn Visser <[email protected]>
AuthorDate: Fri Oct 9 12:09:31 2026 +0200
[FLINK-37672][build] Migrate protobuf code generation to ascopes plugin
(#29434)
* [FLINK-37672][build] Migrate protobuf code generation to ascopes plugin
protoc-jar-maven-plugin (flink-python, flink-protobuf) and the xolstice
protobuf-maven-plugin (flink-parquet) are no longer maintained. Replace
both with io.github.ascopes:protobuf-maven-plugin, configured in the
root pom. It resolves protoc per platform and registers the generated
sources, so os-maven-plugin and the build-helper executions go.
It also makes protoc executable on every run and detects stale sources
by digest, which removes the workaround from FLINK-11427 and
FLINK-19616 (checkStaleness and the proto timestamps in
unpack_build_artifact.sh).
4.0.0 and later require Java 17, so stay on 3.10.3 while Flink builds
on JDK 11. Platforms without a protoc on Maven Central, such as
riscv64, can build with -Dprotobuf.compiler.version=PATH.
Generated-by: Claude Code (Claude Opus 5.5)
* [FLINK-37672][parquet] Move flink-parquet test protos to src/test/proto
Use the same layout as flink-protobuf. The protos no longer end up in
the test resources, and with a license header they no longer need a
rat exclude.
Generated-by: Claude Code (Claude Opus 5.5)
---
flink-formats/flink-parquet/pom.xml | 52 +++-------------------
.../src/test/proto/simple_record.proto | 26 +++++++++++
.../flink-parquet/src/test/proto/test_proto2.proto | 27 +++++++++++
.../test/resources/protobuf/simple_record.proto | 8 ----
.../src/test/resources/protobuf/test_proto2.proto | 9 ----
flink-formats/flink-protobuf/pom.xml | 49 ++++++--------------
flink-python/pom.xml | 16 +++----
pom.xml | 14 +++++-
tools/ci/unpack_build_artifact.sh | 6 +--
9 files changed, 94 insertions(+), 113 deletions(-)
diff --git a/flink-formats/flink-parquet/pom.xml
b/flink-formats/flink-parquet/pom.xml
index 09e9bb316ff..a65251b861d 100644
--- a/flink-formats/flink-parquet/pom.xml
+++ b/flink-formats/flink-parquet/pom.xml
@@ -302,14 +302,6 @@ under the License.
</dependencyManagement>
<build>
- <extensions>
- <extension>
- <groupId>kr.motd.maven</groupId>
- <artifactId>os-maven-plugin</artifactId>
- <version>1.7.1</version>
- </extension>
- </extensions>
-
<plugins>
<!-- Generate Test class from avro schema -->
<plugin>
@@ -330,50 +322,20 @@ under the License.
</executions>
</plugin>
- <!-- Geneate Test class from protobuf schema -->
<plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
+ <groupId>io.github.ascopes</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
- <version>0.5.1</version>
- <extensions>true</extensions>
- <configuration>
- <!-- Currently Flink azure test
pipeline would first pre-compile and then upload the compiled
- directory, then it download the
directory and run the corresponding tests. However, the protoc
- executable under the target directory
would lost the execution permission bit after downloading.
- To solve this issue we would skip
generating the target files if they already exist after
- downloading. Meanwhile, since the time
might be not consistent between the pre-compile and
- the actual execution, we need to
adjust the timestamp manually, see unpack_build_artifact.sh-->
- <checkStaleness>true</checkStaleness>
-
<protoTestSourceRoot>${project.basedir}/src/test/resources/protobuf</protoTestSourceRoot>
- <!-- Generates classes into a separate
directory since the generator always removes existing files. -->
-
<outputDirectory>${project.build.directory}/generated-test-sources/protobuf/java</outputDirectory>
-
<protocArtifact>com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier}</protocArtifact>
- </configuration>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
-
<goal>test-compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <!-- Adding protobuf generated classes to test build
path -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
-
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
- <id>add-test-source</id>
- <phase>generate-sources</phase>
<goals>
-
<goal>add-test-source</goal>
+
<goal>generate-test</goal>
</goals>
<configuration>
- <sources>
-
<source>${project.build.directory}/generated-test-sources/protobuf/java</source>
- </sources>
+ <sourceDirectories>
+
<sourceDirectory>${project.basedir}/src/test/proto</sourceDirectory>
+ </sourceDirectories>
+ <!-- Same path as
before, so stale xolstice output under avro's test root cannot compile twice.
-->
+
<outputDirectory>${project.build.directory}/generated-test-sources/protobuf/java</outputDirectory>
</configuration>
</execution>
</executions>
diff --git a/flink-formats/flink-parquet/src/test/proto/simple_record.proto
b/flink-formats/flink-parquet/src/test/proto/simple_record.proto
new file mode 100644
index 00000000000..51132f8a80e
--- /dev/null
+++ b/flink-formats/flink-parquet/src/test/proto/simple_record.proto
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+package org.apache.flink.formats.parquet.protobuf;
+message SimpleProtoRecord {
+ string foo = 1;
+ string bar = 2;
+ int32 num = 3;
+}
diff --git a/flink-formats/flink-parquet/src/test/proto/test_proto2.proto
b/flink-formats/flink-parquet/src/test/proto/test_proto2.proto
new file mode 100644
index 00000000000..fcaa2fb2d6f
--- /dev/null
+++ b/flink-formats/flink-parquet/src/test/proto/test_proto2.proto
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto2";
+
+package org.apache.flink.formats.parquet.protobuf;
+
+message TestProto2Record {
+ optional string name = 1;
+ optional int32 value = 2;
+ optional bool flag = 3;
+}
diff --git
a/flink-formats/flink-parquet/src/test/resources/protobuf/simple_record.proto
b/flink-formats/flink-parquet/src/test/resources/protobuf/simple_record.proto
deleted file mode 100644
index f96d13cb714..00000000000
---
a/flink-formats/flink-parquet/src/test/resources/protobuf/simple_record.proto
+++ /dev/null
@@ -1,8 +0,0 @@
-syntax = "proto3";
-
-package org.apache.flink.formats.parquet.protobuf;
-message SimpleProtoRecord {
- string foo = 1;
- string bar = 2;
- int32 num = 3;
-}
diff --git
a/flink-formats/flink-parquet/src/test/resources/protobuf/test_proto2.proto
b/flink-formats/flink-parquet/src/test/resources/protobuf/test_proto2.proto
deleted file mode 100644
index d580e93076c..00000000000
--- a/flink-formats/flink-parquet/src/test/resources/protobuf/test_proto2.proto
+++ /dev/null
@@ -1,9 +0,0 @@
-syntax = "proto2";
-
-package org.apache.flink.formats.parquet.protobuf;
-
-message TestProto2Record {
- optional string name = 1;
- optional int32 value = 2;
- optional bool flag = 3;
-}
diff --git a/flink-formats/flink-protobuf/pom.xml
b/flink-formats/flink-protobuf/pom.xml
index 5e8f9341882..5cac1008d36 100644
--- a/flink-formats/flink-protobuf/pom.xml
+++ b/flink-formats/flink-protobuf/pom.xml
@@ -108,46 +108,25 @@ under the License.
<build>
<plugins>
<plugin>
- <groupId>com.github.os72</groupId>
- <artifactId>protoc-jar-maven-plugin</artifactId>
- <version>3.11.4</version>
+ <groupId>io.github.ascopes</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
<executions>
<execution>
- <phase>generate-sources</phase>
<goals>
- <goal>run</goal>
+
<goal>generate-test</goal>
</goals>
<configuration>
-
<protocArtifact>com.google.protobuf:protoc:${protoc.version}</protocArtifact>
-
<includeMavenTypes>direct</includeMavenTypes>
- <inputDirectories>
-
<include>src/test/proto</include>
- </inputDirectories>
- <outputTargets>
- <outputTarget>
-
<type>java</type>
-
<addSources>none</addSources>
-
<outputDirectory>target/test-proto-sources</outputDirectory>
- </outputTarget>
- </outputTargets>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
-
<artifactId>build-helper-maven-plugin</artifactId>
- <version>3.0.0</version>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
-
<goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
-
<source>target/test-proto-sources</source>
- </sources>
+ <sourceDirectories>
+
<sourceDirectory>${project.basedir}/src/test/proto</sourceDirectory>
+ </sourceDirectories>
+ <!-- Provides
google/protobuf/timestamp.proto. -->
+ <importDependencies>
+
<importDependency>
+
<groupId>com.google.protobuf</groupId>
+
<artifactId>protobuf-java</artifactId>
+
<version>${protoc.version}</version>
+
</importDependency>
+ </importDependencies>
</configuration>
</execution>
</executions>
diff --git a/flink-python/pom.xml b/flink-python/pom.xml
index ca08c75951c..c5000d8aff5 100644
--- a/flink-python/pom.xml
+++ b/flink-python/pom.xml
@@ -871,21 +871,17 @@ under the License.
</executions>
</plugin>
<plugin>
- <groupId>com.github.os72</groupId>
- <artifactId>protoc-jar-maven-plugin</artifactId>
- <version>3.11.4</version>
+ <groupId>io.github.ascopes</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
<executions>
<execution>
- <phase>generate-sources</phase>
<goals>
- <goal>run</goal>
+ <goal>generate</goal>
</goals>
<configuration>
-
<protocArtifact>com.google.protobuf:protoc:${protoc.version}
- </protocArtifact>
- <inputDirectories>
-
<include>pyflink/proto</include>
- </inputDirectories>
+ <sourceDirectories>
+
<sourceDirectory>${project.basedir}/pyflink/proto</sourceDirectory>
+ </sourceDirectories>
</configuration>
</execution>
</executions>
diff --git a/pom.xml b/pom.xml
index 097d288e759..e2e87141081 100644
--- a/pom.xml
+++ b/pom.xml
@@ -166,6 +166,8 @@ under the License.
<py4j.version>0.10.9.7</py4j.version>
<beam.version>2.75.0</beam.version>
<protoc.version>4.32.1</protoc.version>
+ <!-- protoc used by protobuf-maven-plugin; set to PATH to use a
local protoc, e.g. on riscv64. -->
+
<protobuf.compiler.version>${protoc.version}</protobuf.compiler.version>
<okhttp.version>3.14.9</okhttp.version>
<testcontainers.version>2.0.5</testcontainers.version>
<lz4.version>1.10.3</lz4.version>
@@ -1704,7 +1706,6 @@ under the License.
<inputExclude>flink-formats/flink-json/src/test/resources/*.txt</inputExclude>
<inputExclude>flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/generated/*.java</inputExclude>
<inputExclude>flink-formats/flink-parquet/src/test/resources/avro/**</inputExclude>
-
<inputExclude>flink-formats/flink-parquet/src/test/resources/protobuf/**</inputExclude>
<inputExclude>flink-table/flink-sql-gateway/src/test/resources/*.txt</inputExclude>
<!-- netty test file, still
Apache License 2.0 but with a different header -->
<inputExclude>flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/AbstractByteBufTest.java</inputExclude>
@@ -2318,6 +2319,17 @@ under the License.
<version>3.6.0</version>
</plugin>
+ <plugin>
+ <groupId>io.github.ascopes</groupId>
+
<artifactId>protobuf-maven-plugin</artifactId>
+ <!-- 4.x and later require Java 17;
stay on 3.x while Flink builds on JDK 11. -->
+ <version>3.10.3</version>
+ <configuration>
+ <!-- Only put dependency protos
on the import path where a module asks for them. -->
+
<ignoreProjectDependencies>true</ignoreProjectDependencies>
+ </configuration>
+ </plugin>
+
<plugin>
<!-- Inherited from Apache parent, but
not actually used. Disable to reduce noise. -->
<groupId>org.apache.maven.plugins</groupId>
diff --git a/tools/ci/unpack_build_artifact.sh
b/tools/ci/unpack_build_artifact.sh
index ac55c2a5135..fe959db5d7f 100755
--- a/tools/ci/unpack_build_artifact.sh
+++ b/tools/ci/unpack_build_artifact.sh
@@ -34,14 +34,10 @@ else
fi
echo "Adjusting timestamps"
-# Set timestamps: proto < source < compiled, so Maven skips recompilation
+# Set timestamps: source < compiled, so Maven skips recompilation
# 5-second gaps ensure reliable ordering on filesystems with low resolution
BASE_TIME=$(date +%s)
-# T+0: proto files (oldest - won't trigger regeneration)
-PROTO_TIME=$(date -d "@$BASE_TIME" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r
"$BASE_TIME" '+%Y-%m-%d %H:%M:%S')
-find . -type f -name '*.proto' -exec touch -d "$PROTO_TIME" {} +
-
# T+5: source files
SOURCE_TIME=$(date -d "@$((BASE_TIME + 5))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null
|| date -r "$((BASE_TIME + 5))" '+%Y-%m-%d %H:%M:%S')
find . -type f \( -name '*.java' -o -name '*.scala' \) -exec touch -d
"$SOURCE_TIME" {} +
