This is an automated email from the ASF dual-hosted git repository. czy006 pushed a commit to branch codex/upgrade-thrift-0.25.0 in repository https://gitbox.apache.org/repos/asf/amoro-shade.git
commit 2d63b66aff868c6522530aee105feb216e6e929f Author: chenzhengyu <[email protected]> AuthorDate: Thu Oct 15 11:54:41 2026 +0800 build: upgrade Thrift to 0.25.0 and validate shaded artifacts on Java 11 --- .github/workflows/ci.yaml | 4 +- README.md | 32 +++++ amoro-shade-thrift/pom.xml | 43 +++++- .../src/main/resources/META-INF/NOTICE | 4 +- .../apache/amoro/shade/thrift/ThriftShadeIT.java | 150 +++++++++++++++++++++ .../src/test/resources/ThriftSmokeTest.java | 146 ++++++++++++++++++++ pom.xml | 10 +- 7 files changed, 382 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a75ab90..463a9da 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Java 8 Build +name: Java 11 Build on: [push, pull_request] @@ -31,7 +31,7 @@ jobs: - name: Set JDK uses: actions/setup-java@v5 with: - java-version: 8 + java-version: 11 distribution: 'temurin' cache: 'maven' diff --git a/README.md b/README.md index 6503925..4b8238d 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,38 @@ Shaded dependencies contained here do not expose any transitive dependencies. Th When using these dependencies it is recommended to work directly against the shaded namespaces. +## 编译与验证 + +默认使用 **JDK 11**，Java 编译目标为 11，CI 使用 Temurin 11。构建会拒绝 JDK 8； +Maven 使用当前 `JAVA_HOME` 指向的 JDK，开始前请通过 `mvn -version` 确认。 +建议使用 Maven 3.6.3 或更新版本。 + +macOS 可以在当前终端选择 JDK 11： + +```bash +export JAVA_HOME="$(/usr/libexec/java_home -v 11)" +export PATH="$JAVA_HOME/bin:$PATH" +mvn -version +mvn clean verify +``` + +Linux/Windows 请将 `JAVA_HOME` 指向本机 JDK 11 安装目录。仅验证 Thrift 模块及其父项目： + +```bash +mvn -pl amoro-shade-thrift -am clean verify +``` + +Thrift 依赖版本为 **0.25.0**，当前产物坐标为 +`org.apache.amoro:amoro-shade-thrift:0.25.0-0.9-SNAPSHOT`，输出到 +`amoro-shade-thrift/target/amoro-shade-thrift-0.25.0-0.9-SNAPSHOT.jar`。 +包名仍使用 `org.apache.amoro.shade.thrift.org.apache.thrift`。 +SLF4J API 保持由使用方提供，不打入该 JAR。 + +`verify` 阶段会编译并运行仅依赖成品 shaded JAR 与 SLF4J API 的消费端测试， +覆盖 Binary/Compact/JSON 协议、连续帧传输、两种 HTTP 客户端、Servlet/Annotation 重定位， +并检查类文件的 Java 11 兼容性。`mvn test` 不执行这些打包后的集成测试。 +Thrift 编译器和 Amoro 主仓库的生成代码及服务联调需要在使用方另外验证。 + ## Sources We currently do not release jars containing the shaded sources due to the unanswered legal questions raised [here](https://github.com/apache/flink-shaded/issues/25). diff --git a/amoro-shade-thrift/pom.xml b/amoro-shade-thrift/pom.xml index 1c235f3..51c2b9c 100644 --- a/amoro-shade-thrift/pom.xml +++ b/amoro-shade-thrift/pom.xml @@ -41,10 +41,40 @@ under the License. </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <version>5.11.4</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>1.7.36</version> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <version>3.5.4</version> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + </execution> + </executions> + <configuration> + <systemPropertyVariables> + <shadedJar>${project.build.directory}/${project.build.finalName}.jar</shadedJar> + </systemPropertyVariables> + </configuration> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> @@ -58,6 +88,17 @@ under the License. <configuration> <createDependencyReducedPom>true</createDependencyReducedPom> <dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation> + <filters combine.children="append"> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/maven/**</exclude> + <!-- Original module descriptors refer to unshaded packages. --> + <exclude>module-info.class</exclude> + <exclude>META-INF/versions/**/module-info.class</exclude> + </excludes> + </filter> + </filters> <artifactSet> <includes> <include>*</include> @@ -86,4 +127,4 @@ under the License. </plugins> </build> -</project> \ No newline at end of file +</project> diff --git a/amoro-shade-thrift/src/main/resources/META-INF/NOTICE b/amoro-shade-thrift/src/main/resources/META-INF/NOTICE index 5e0e77e..7f9d414 100644 --- a/amoro-shade-thrift/src/main/resources/META-INF/NOTICE +++ b/amoro-shade-thrift/src/main/resources/META-INF/NOTICE @@ -8,7 +8,9 @@ This project bundles the following dependencies under the Apache Software Licens - org.apache.httpcomponents.client5:httpclient5:5.2.1 - org.apache.httpcomponents.core5:httpcore5:5.2 -- org.apache.commons:commons-lang3:3.12.0 +- org.apache.httpcomponents.core5:httpcore5-h2:5.2 +- org.apache.thrift:libthrift:0.25.0 +- org.apache.commons:commons-lang3:3.18.0 This project bundles the following dependencies under Eclipse Public License v. 2.0 (http://www.eclipse.org/legal/epl-2.0) - jakarta.servlet:jakarta.servlet-api:5.0.0 diff --git a/amoro-shade-thrift/src/test/java/org/apache/amoro/shade/thrift/ThriftShadeIT.java b/amoro-shade-thrift/src/test/java/org/apache/amoro/shade/thrift/ThriftShadeIT.java new file mode 100644 index 0000000..1c17fde --- /dev/null +++ b/amoro-shade-thrift/src/test/java/org/apache/amoro/shade/thrift/ThriftShadeIT.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.amoro.shade.thrift; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.File; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Enumeration; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.LoggerFactory; + +class ThriftShadeIT { + @TempDir static Path classes; + + private static Path shadedJar; + private static URLClassLoader loader; + + @BeforeAll + static void compileConsumer() throws Exception { + shadedJar = Path.of(System.getProperty("shadedJar")); + assertTrue(Files.isRegularFile(shadedJar), "Run verify to build the shaded JAR first"); + URL slf4j = LoggerFactory.class.getProtectionDomain().getCodeSource().getLocation(); + Path source = Path.of(ThriftShadeIT.class.getResource("/ThriftSmokeTest.java").toURI()); + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + assertNotNull(compiler, "The integration tests require a JDK"); + ByteArrayOutputStream errors = new ByteArrayOutputStream(); + int result = + compiler.run( + null, + errors, + errors, + "--release", + "11", + "-encoding", + "UTF-8", + "-proc:none", + "-classpath", + shadedJar + File.pathSeparator + Path.of(slf4j.toURI()), + "-d", + classes.toString(), + source.toString()); + assertEquals(0, result, errors.toString("UTF-8")); + // Only the packaged JAR and the intentionally external logging API are visible. + loader = + new URLClassLoader( + new URL[] {classes.toUri().toURL(), shadedJar.toUri().toURL(), slf4j}, + ClassLoader.getPlatformClassLoader()); + } + + @AfterAll + static void closeConsumer() throws Exception { + if (loader != null) { + loader.close(); + } + } + + @Test + void containsOnlyRelocatedClassesCompatibleWithJava11() throws Exception { + try (JarFile jar = new JarFile(shadedJar.toFile())) { + assertNotNull( + jar.getJarEntry("org/apache/amoro/shade/thrift/org/apache/thrift/TException.class")); + Enumeration<JarEntry> entries = jar.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String name = entry.getName(); + assertFalse(name.startsWith("META-INF/maven/"), name); + if (!name.endsWith(".class")) { + continue; + } + assertFalse( + name.endsWith("module-info.class"), + "Original module descriptors do not describe the relocated JAR: " + name); + assertTrue(name.startsWith("org/apache/amoro/shade/thrift/"), name); + try (DataInputStream input = new DataInputStream(jar.getInputStream(entry))) { + assertEquals(0xCAFEBABE, input.readInt(), name); + input.readUnsignedShort(); + assertTrue(input.readUnsignedShort() <= 55, "Requires a newer JDK: " + name); + } + } + assertNotNull(jar.getJarEntry("META-INF/LICENSE")); + assertNotNull(jar.getJarEntry("META-INF/NOTICE")); + } + } + + @Test + void binaryProtocolRoundTrip() throws Exception { + runConsumer("binaryProtocol"); + } + + @Test + void compactProtocolRoundTrip() throws Exception { + runConsumer("compactProtocol"); + } + + @Test + void jsonProtocolRoundTrip() throws Exception { + runConsumer("jsonProtocol"); + } + + @Test + void framedTransportRoundTrip() throws Exception { + runConsumer("framedTransport"); + } + + @Test + void httpClientRoundTrip() throws Exception { + runConsumer("httpClient"); + } + + @Test + void servletAndAnnotationDependenciesAreRelocated() throws Exception { + runConsumer("servletAndAnnotations"); + } + + private static void runConsumer(String method) throws Exception { + loader.loadClass("ThriftSmokeTest").getMethod(method).invoke(null); + } +} diff --git a/amoro-shade-thrift/src/test/resources/ThriftSmokeTest.java b/amoro-shade-thrift/src/test/resources/ThriftSmokeTest.java new file mode 100644 index 0000000..54a9320 --- /dev/null +++ b/amoro-shade-thrift/src/test/resources/ThriftSmokeTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.sun.net.httpserver.HttpServer; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import org.apache.amoro.shade.thrift.jakarta.annotation.PostConstruct; +import org.apache.amoro.shade.thrift.jakarta.servlet.http.HttpServlet; +import org.apache.amoro.shade.thrift.org.apache.commons.lang3.Validate; +import org.apache.amoro.shade.thrift.org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.amoro.shade.thrift.org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TCompactProtocol; +import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TField; +import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TJSONProtocol; +import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TProtocol; +import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TProtocolFactory; +import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TStruct; +import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TType; +import org.apache.amoro.shade.thrift.org.apache.thrift.server.TServlet; +import org.apache.amoro.shade.thrift.org.apache.thrift.transport.THttpClient; +import org.apache.amoro.shade.thrift.org.apache.thrift.transport.TMemoryBuffer; +import org.apache.amoro.shade.thrift.org.apache.thrift.transport.layered.TFramedTransport; + +/** Compiled against the final shaded JAR, without the original dependency classes. */ +public class ThriftSmokeTest { + public static void binaryProtocol() throws Exception { + protocolRoundTrip(new TBinaryProtocol.Factory()); + } + + public static void compactProtocol() throws Exception { + protocolRoundTrip(new TCompactProtocol.Factory()); + } + + public static void jsonProtocol() throws Exception { + protocolRoundTrip(new TJSONProtocol.Factory()); + } + + private static void protocolRoundTrip(TProtocolFactory factory) throws Exception { + TMemoryBuffer memory = new TMemoryBuffer(256); + TProtocol writer = factory.getProtocol(memory); + writer.writeStructBegin(new TStruct("SmokeTest")); + writer.writeFieldBegin(new TField("message", TType.STRING, (short) 1)); + writer.writeString("Amoro Thrift 中文"); + writer.writeFieldEnd(); + writer.writeFieldBegin(new TField("value", TType.I32, (short) 2)); + writer.writeI32(250); + writer.writeFieldEnd(); + writer.writeFieldStop(); + writer.writeStructEnd(); + + TProtocol reader = factory.getProtocol(memory); + reader.readStructBegin(); + TField message = reader.readFieldBegin(); + check(message.id == 1 && message.type == TType.STRING, "String field header"); + check("Amoro Thrift 中文".equals(reader.readString()), "UTF-8 round trip"); + reader.readFieldEnd(); + TField value = reader.readFieldBegin(); + check(value.id == 2 && value.type == TType.I32, "Integer field header"); + check(reader.readI32() == 250, "Integer round trip"); + reader.readFieldEnd(); + check(reader.readFieldBegin().type == TType.STOP, "End of struct"); + reader.readStructEnd(); + } + + public static void framedTransport() throws Exception { + TMemoryBuffer memory = new TMemoryBuffer(256); + TFramedTransport transport = new TFramedTransport(memory); + // Two consecutive frames exercise the frame/message accounting changed in 0.25.0. + for (String message : new String[] {"first frame", "second frame 中文"}) { + byte[] expected = message.getBytes(StandardCharsets.UTF_8); + transport.write(expected); + transport.flush(); + byte[] actual = new byte[expected.length]; + transport.readAll(actual, 0, actual.length); + check(Arrays.equals(expected, actual), "Framed transport round trip"); + } + transport.close(); + } + + public static void httpClient() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/thrift", + exchange -> { + try { + byte[] request = exchange.getRequestBody().readAllBytes(); + exchange.getResponseHeaders().set("Content-Type", "application/x-thrift"); + exchange.sendResponseHeaders(200, request.length); + exchange.getResponseBody().write(request); + } finally { + exchange.close(); + } + }); + server.start(); + try (CloseableHttpClient http = HttpClients.createDefault()) { + String url = "http://127.0.0.1:"; + server.getAddress().getPort() + "/thrift"; + // Cover both the JDK URLConnection path and the relocated HttpComponents path. + for (THttpClient client : + new THttpClient[] {new THttpClient(url), new THttpClient(url, http)}) { + try { + client.setConnectTimeout(5000); + client.setReadTimeout(5000); + byte[] expected = "Amoro HTTP 中文".getBytes(StandardCharsets.UTF_8); + client.write(expected); + client.flush(); + byte[] actual = new byte[expected.length]; + client.readAll(actual, 0, actual.length); + check(Arrays.equals(expected, actual), "HTTP transport round trip"); + } finally { + client.close(); + } + } + } finally { + server.stop(0); + } + } + + public static void servletAndAnnotations() { + check(TServlet.class.getSuperclass() == HttpServlet.class, "Relocated servlet superclass"); + check(PostConstruct.class.isAnnotation(), "Relocated annotation API"); + Validate.isTrue(true, "Relocated Commons Lang"); + } + + private static void check(boolean condition, String message) { + if (!condition) { + throw new AssertionError(message); + } + } +} diff --git a/pom.xml b/pom.xml index d975986..ce8a3e3 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,8 @@ under the License. <properties> <shading.prefix>org.apache.amoro.shade</shading.prefix> <guava.version>32.1.1-jre</guava.version> - <thrift.version>0.20.0</thrift.version> + <thrift.version>0.25.0</thrift.version> + <maven.compiler.release>11</maven.compiler.release> </properties> <profiles> @@ -101,10 +102,10 @@ under the License. <configuration> <rules> <requireJavaVersion> - <version>1.8.0</version> + <version>[11,)</version> </requireJavaVersion> <requireMavenVersion> - <version>(,3.3)</version> + <version>[3.6.3,)</version> </requireMavenVersion> </rules> </configuration> @@ -212,6 +213,9 @@ under the License. </goals> <configuration> <rules> + <requireJavaVersion> + <version>[11,)</version> + </requireJavaVersion> <dependencyConvergence/> </rules> </configuration>
