This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
The following commit(s) were added to refs/heads/master by this push:
new e1f94a028 [SUREFIRE-2276] JUnit5's TestTemplate failures treated as
flakes with retries
e1f94a028 is described below
commit e1f94a0281f0db1f9169e06b7ad149a27afd3c46
Author: Hugo G <hgrzeskow...@atlassian.com>
AuthorDate: Thu Sep 26 21:27:04 2024 +1000
[SUREFIRE-2276] JUnit5's TestTemplate failures treated as flakes with
retries
This closes #788
---
.../maven/surefire/its/JUnitPlatformEnginesIT.java | 39 ++++++++++++++
.../test/resources/junit5-testtemplate-bug/pom.xml | 55 +++++++++++++++++++
.../src/test/java/pkg/FieldSettingTest.java | 63 ++++++++++++++++++++++
.../src/test/java/pkg/ParamsContextTest.java | 62 +++++++++++++++++++++
.../surefire/junitplatform/RunListenerAdapter.java | 4 +-
5 files changed, 222 insertions(+), 1 deletion(-)
diff --git
a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformEnginesIT.java
b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformEnginesIT.java
index ba79ccb9f..255aea291 100644
---
a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformEnginesIT.java
+++
b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformEnginesIT.java
@@ -296,6 +296,45 @@ public class JUnitPlatformEnginesIT extends
SurefireJUnit4IntegrationTestCase {
.assertContainsText("Encountered failure in TestTemplate
provideTestTemplateInvocationContexts()");
}
+ @Test
+ public void testJupiterEngineWithTestTemplateNotClassifiedAsFlake() {
+ unpack("junit5-testtemplate-bug", "-" + jupiter)
+ .setTestToRun("FieldSettingTest")
+ .sysProp("junit5.version", jupiter)
+ .maven()
+ .withFailure()
+ .executeTest()
+ .verifyTextInLog("AssertionFailedError")
+ .assertTestSuiteResults(2, 0, 1, 0, 0);
+
+ unpack("junit5-testtemplate-bug", "-" + jupiter)
+ .debugLogging()
+ .setTestToRun("FieldSettingTest")
+ .sysProp("junit5.version", jupiter)
+ // The tests are failing deterministically, so rerunning them
should not change the result
+ .sysProp("surefire.rerunFailingTestsCount", "1")
+ .maven()
+ .withFailure()
+ .executeTest()
+ .verifyTextInLog("AssertionFailedError")
+ .assertTestSuiteResults(2, 0, 1, 0, 0);
+ }
+
+ @Test
+ public void testJupiterEngineWithParameterizedTestsNotClassifiedAsFlake() {
+ unpack("junit5-testtemplate-bug", "-" + jupiter)
+ .debugLogging()
+ .setTestToRun("ParamsContextTest")
+ .sysProp("junit5.version", jupiter)
+ // The tests are failing deterministically, so rerunning them
should not change the result
+ .sysProp("surefire.rerunFailingTestsCount", "1")
+ .maven()
+ .withFailure()
+ .executeTest()
+ .verifyTextInLog("AssertionFailedError")
+ .assertTestSuiteResults(2, 0, 1, 0, 0);
+ }
+
@Test
public void testJupiterEngineWithAssertionsFailNoParameters() {
// `Assertions.fail()` not supported until 5.2.0
diff --git a/surefire-its/src/test/resources/junit5-testtemplate-bug/pom.xml
b/surefire-its/src/test/resources/junit5-testtemplate-bug/pom.xml
new file mode 100644
index 000000000..335ba72b5
--- /dev/null
+++ b/surefire-its/src/test/resources/junit5-testtemplate-bug/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.maven.plugins.surefire</groupId>
+ <artifactId>surefire-junit-testtemplate-retry-bug</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <name>Test for JUnit 5 TestTemplate with retries</name>
+
+ <properties>
+
<maven.compiler.source>${java.specification.version}</maven.compiler.source>
+
<maven.compiler.target>${java.specification.version}</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <version>${junit5.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <version>${junit5.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <version>${junit5.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.1</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.version}</version>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git
a/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/FieldSettingTest.java
b/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/FieldSettingTest.java
new file mode 100644
index 000000000..61f5fa5c5
--- /dev/null
+++
b/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/FieldSettingTest.java
@@ -0,0 +1,63 @@
+package pkg;
+
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class FieldSettingTest {
+ private int testValue = 42;
+
+ // We're calling this in the provider underneath
+ public void setTestValue(int testValue) {
+ this.testValue = testValue;
+ }
+
+ @TestTemplate
+ @ExtendWith(FieldSettingContextProvider.class)
+ public void testTemplatePartiallyFails() {
+ assertEquals(42, testValue);
+ }
+}
+
+
+class FieldSettingContextProvider implements
TestTemplateInvocationContextProvider {
+ @Override
+ public boolean supportsTestTemplate(ExtensionContext extensionContext) {
+ return true;
+ }
+
+ @Override
+ public Stream<TestTemplateInvocationContext>
provideTestTemplateInvocationContexts(ExtensionContext extensionContext) {
+ return Stream.of(context(0), context(42));
+ }
+
+ private TestTemplateInvocationContext context(int parameter) {
+ return new TestTemplateInvocationContext() {
+ @Override
+ public String getDisplayName(int invocationIndex) {
+ return String.format("[%d] %s", invocationIndex, parameter);
+ }
+
+ @Override
+ public List<Extension> getAdditionalExtensions() {
+ return getBeforeEachCallbacks(parameter);
+ }
+ };
+ }
+
+ private List<Extension> getBeforeEachCallbacks(int value) {
+ return Arrays.asList((BeforeEachCallback) ctx ->
+ ((FieldSettingTest)
ctx.getRequiredTestInstance()).setTestValue(value)
+ );
+ }
+}
diff --git
a/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/ParamsContextTest.java
b/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/ParamsContextTest.java
new file mode 100644
index 000000000..4022e2cbb
--- /dev/null
+++
b/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/ParamsContextTest.java
@@ -0,0 +1,62 @@
+package pkg;
+
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class ParamsContextTest {
+
+ @TestTemplate
+ @ExtendWith(ParamsContextProvider.class)
+ void testTemplatePartiallyFails(int value) {
+ assertEquals(42, value);
+ }
+}
+
+class ParamsContextProvider implements TestTemplateInvocationContextProvider {
+
+ @Override
+ public boolean supportsTestTemplate(ExtensionContext context) {
+ return true;
+ }
+
+ @Override
+ public Stream<TestTemplateInvocationContext>
provideTestTemplateInvocationContexts(ExtensionContext context) {
+ return Stream.of(invocationContext(0), invocationContext(42));
+ }
+
+ private TestTemplateInvocationContext invocationContext(int parameter) {
+ return new TestTemplateInvocationContext() {
+ @Override
+ public String getDisplayName(int invocationIndex) {
+ return String.format("[%d] %s", invocationIndex, parameter);
+ }
+
+ @Override
+ public List<Extension> getAdditionalExtensions() {
+ return Collections.singletonList(new ParameterResolver() {
+ @Override
+ public boolean supportsParameter(ParameterContext
parameterContext, ExtensionContext extensionContext) {
+ return true;
+ }
+
+ @Override
+ public Object resolveParameter(ParameterContext
parameterContext, ExtensionContext extensionContext) {
+ return parameter;
+ }
+ });
+ }
+ };
+ }
+}
diff --git
a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
index 8c274fd1c..7ae8cc79d 100644
---
a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
+++
b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
@@ -317,12 +317,14 @@ final class RunListenerAdapter implements
TestExecutionListener, TestOutputRecei
boolean needsSpaceSeparator = isNotBlank(parentDisplay) &&
!display.startsWith("[");
String methodDisplay = parentDisplay + (needsSpaceSeparator ? " "
: "") + display;
+ boolean isParameterized =
isNotBlank(methodSource.getMethodParameterTypes());
boolean hasParameterizedParent =
collectAllTestIdentifiersInHierarchy(testIdentifier)
.filter(identifier -> !identifier.getSource().isPresent())
.map(TestIdentifier::getLegacyReportingName)
.anyMatch(legacyReportingName ->
legacyReportingName.matches("^\\[.+]$"));
+ boolean isTestTemplate =
testIdentifier.getLegacyReportingName().matches("^.*\\[\\d+]$");
- boolean parameterized =
isNotBlank(methodSource.getMethodParameterTypes()) || hasParameterizedParent;
+ boolean parameterized = isParameterized || hasParameterizedParent
|| isTestTemplate;
String methodName = methodSource.getMethodName();
String description = testIdentifier.getLegacyReportingName();
boolean equalDescriptions = methodDisplay.equals(description);
