This is an automated email from the ASF dual-hosted git repository.
tibordigana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
The following commit(s) were added to refs/heads/master by this push:
new b4a53de9b Properly parse flake failures in beforeAll stages
b4a53de9b is described below
commit b4a53de9b3a9ce02f306f02b064f4f58070678f4
Author: Jakub Stejskal <[email protected]>
AuthorDate: Wed Oct 8 16:48:44 2025 +0200
Properly parse flake failures in beforeAll stages
Signed-off-by: Jakub Stejskal <[email protected]>
# Conflicts:
#
maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
#
maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
---
.../surefire/report/DefaultReporterFactory.java | 102 +++++++++++++++++++
.../surefire/report/StatelessXmlReporter.java | 112 ++++++++++++++++++---
.../report/DefaultReporterFactoryTest.java | 48 +++++++--
.../its/JUnitPlatformFailingBeforeAllRerunIT.java | 64 ++++++++++++
.../pom.xml | 93 +++++++++++++++++
.../test/java/junitplatform/AlwaysFailingTest.java | 22 ++++
.../java/junitplatform/FlakyFirstTimeTest.java | 57 +++++++++++
.../src/test/java/junitplatform/PassingTest.java | 19 ++++
8 files changed, 496 insertions(+), 21 deletions(-)
diff --git
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
index f95c061ab..54ed44813 100644
---
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
+++
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
@@ -41,6 +41,7 @@
import
org.apache.maven.surefire.extensions.StatelessTestsetInfoConsoleReportEventListener;
import
org.apache.maven.surefire.extensions.StatelessTestsetInfoFileReportEventListener;
import org.apache.maven.surefire.report.RunStatistics;
+import org.apache.maven.surefire.shared.utils.StringUtils;
import org.apache.maven.surefire.shared.utils.logging.MessageBuilder;
import static org.apache.maven.plugin.surefire.log.api.Level.resolveLevel;
@@ -72,6 +73,9 @@ public class DefaultReporterFactory implements
ReporterFactory, ReportsMerger {
private RunStatistics globalStats = new RunStatistics();
+ // from "<testclass>.<testmethod>" -> statistics about all the runs for
success tests
+ private Map<String, List<TestMethodStats>> successTests;
+
// from "<testclass>.<testmethod>" -> statistics about all the runs for
flaky tests
private Map<String, List<TestMethodStats>> flakyTests;
@@ -243,6 +247,7 @@ static TestResultType
getTestResultType(List<ReportEntryType> reportEntries, int
*/
private void mergeTestHistoryResult() {
globalStats = new RunStatistics();
+ successTests = new TreeMap<>();
flakyTests = new TreeMap<>();
failedTests = new TreeMap<>();
errorTests = new TreeMap<>();
@@ -265,10 +270,40 @@ private void mergeTestHistoryResult() {
// Update globalStatistics by iterating through mergedTestHistoryResult
int completedCount = 0, skipped = 0;
+ Map<String, List<TestMethodStats>> beforeAllFailures = new HashMap<>();
for (Map.Entry<String, List<TestMethodStats>> entry :
mergedTestHistoryResult.entrySet()) {
List<TestMethodStats> testMethodStats = entry.getValue();
String testClassMethodName = entry.getKey();
+
+ // Handle @BeforeAll failures (null, empty, or ends with ".null"
method names)
+ // But only if they actually failed (ERROR or FAILURE), not if
they were skipped
+ if ((StringUtils.isBlank(testClassMethodName) ||
testClassMethodName.endsWith(".null"))
+ && (testClassMethodName == null ||
!testClassMethodName.contains("$"))) {
+
+ // Check if this is actually a failure/error (not skipped or
success)
+ boolean isActualFailure = testMethodStats.stream()
+ .anyMatch(stat -> stat.getResultType() ==
ReportEntryType.ERROR
+ || stat.getResultType() ==
ReportEntryType.FAILURE);
+
+ if (isActualFailure) {
+ // Extract class name from the test class method name
+ String className =
extractClassNameFromMethodName(testClassMethodName);
+ if (className != null) {
+ if (beforeAllFailures.containsKey(className)) {
+ List<TestMethodStats> previousMethodStats =
beforeAllFailures.get(className);
+ previousMethodStats.addAll(testMethodStats);
+ beforeAllFailures.put(className,
previousMethodStats);
+ } else {
+ beforeAllFailures.put(className, new
ArrayList<>(testMethodStats));
+ }
+ }
+ // Skip normal processing of @BeforeAll failures because
it needs special care
+ continue;
+ }
+ // If it's skipped or success with null method name, fall
through to normal processing
+ }
+
completedCount++;
List<ReportEntryType> resultTypes = new ArrayList<>();
@@ -286,6 +321,7 @@ private void mergeTestHistoryResult() {
}
}
completedCount += successCount - 1;
+ successTests.put(testClassMethodName, testMethodStats);
break;
case SKIPPED:
skipped++;
@@ -304,6 +340,37 @@ private void mergeTestHistoryResult() {
}
}
+ // Loop over all success tests and find those that are passed flakes
for beforeAll failures
+ for (Map.Entry<String, List<TestMethodStats>> entry :
successTests.entrySet()) {
+ List<TestMethodStats> testMethodStats = entry.getValue();
+ String testClassMethodName = entry.getKey();
+ // If current test belong to class that failed during beforeAll
store that info to proper log info
+ String className =
extractClassNameFromMethodName(testClassMethodName);
+ if (beforeAllFailures.containsKey(className)) {
+ List<TestMethodStats> previousMethodStats =
beforeAllFailures.get(className);
+ previousMethodStats.addAll(testMethodStats);
+ beforeAllFailures.put(className, previousMethodStats);
+ }
+ }
+
+ // Process @BeforeAll failures after we know which classes have
successful tests
+ for (Map.Entry<String, List<TestMethodStats>> entry :
beforeAllFailures.entrySet()) {
+ String className = entry.getKey();
+ List<TestMethodStats> testMethodStats = entry.getValue();
+ String classNameKey = className + ".<beforeAll>";
+
+ if (reportConfiguration.getRerunFailingTestsCount() > 0
+ && testMethodStats.stream()
+ .anyMatch(methodStats ->
methodStats.getTestClassMethodName() != null
+ &&
!methodStats.getTestClassMethodName().isEmpty()
+ &&
methodStats.getResultType().equals(ReportEntryType.SUCCESS))) {
+ flakyTests.put(classNameKey, testMethodStats);
+ } else {
+ errorTests.put(classNameKey, testMethodStats);
+ completedCount++;
+ }
+ }
+
globalStats.set(completedCount, errorTests.size(), failedTests.size(),
skipped, flakyTests.size());
}
@@ -362,6 +429,41 @@ boolean printTestFailures(TestResultType type) {
return printed;
}
+ /**
+ * Extract class name from test class method name like
"com.example.TestClass.methodName"
+ */
+ private static String extractClassNameFromMethodName(String
testClassMethodName) {
+ if (StringUtils.isBlank(testClassMethodName)) {
+ return null;
+ }
+ int lastDotIndex = testClassMethodName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ return testClassMethodName.substring(0, lastDotIndex);
+ }
+ return null;
+ }
+
+ /**
+ * Extract class name from stack trace when method name is null (e.g.,
@BeforeAll failures)
+ */
+ private static String extractClassNameFromStackTrace(TestMethodStats
stats) {
+ if (stats.getStackTraceWriter() == null) {
+ return null;
+ }
+ String stackTrace =
stats.getStackTraceWriter().smartTrimmedStackTrace();
+ if (stackTrace == null || stackTrace.isEmpty()) {
+ return null;
+ }
+
+ // Strip everything after the first whitespace character
+ int firstWhitespace = stackTrace.indexOf(' ');
+ if (firstWhitespace > 0) {
+ stackTrace = stackTrace.substring(0, firstWhitespace);
+ }
+
+ return extractClassNameFromMethodName(stackTrace);
+ }
+
// Describe the result of a given test
enum TestResultType {
ERROR("Errors: "),
diff --git
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
index 05b5059de..f925a7c25 100644
---
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
+++
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
@@ -161,7 +161,7 @@ public void testSetCompleted(WrappedReportEntry
testSetReportEntry, TestSetStats
OutputStreamWriter fw = getWriter(outputStream)) {
XMLWriter ppw = new PrettyPrintXMLWriter(new PrintWriter(fw),
XML_INDENT, XML_NL, UTF_8.name(), null);
- createTestSuiteElement(ppw, testSetReportEntry, testSetStats); //
TestSuite
+ createTestSuiteElement(ppw, testSetReportEntry,
classMethodStatistics); // TestSuite
if (enablePropertiesElement) {
showProperties(ppw, testSetReportEntry.getSystemProperties());
@@ -186,9 +186,9 @@ public void testSetCompleted(WrappedReportEntry
testSetReportEntry, TestSetStats
}
for (Entry<String, Map<String, List<WrappedReportEntry>>>
statistics : classMethodStatistics.entrySet()) {
- for (Entry<String, List<WrappedReportEntry>> thisMethodRuns :
- statistics.getValue().entrySet()) {
- serializeTestClass(outputStream, fw, ppw,
thisMethodRuns.getValue());
+ Map<String, List<WrappedReportEntry>> methodStatistics =
statistics.getValue();
+ for (Entry<String, List<WrappedReportEntry>> thisMethodRuns :
methodStatistics.entrySet()) {
+ serializeTestClass(outputStream, fw, ppw,
thisMethodRuns.getValue(), methodStatistics);
}
}
@@ -224,10 +224,14 @@ private Deque<WrappedReportEntry>
aggregateCacheFromMultipleReruns(
}
private void serializeTestClass(
- OutputStream outputStream, OutputStreamWriter fw, XMLWriter ppw,
List<WrappedReportEntry> methodEntries)
+ OutputStream outputStream,
+ OutputStreamWriter fw,
+ XMLWriter ppw,
+ List<WrappedReportEntry> methodEntries,
+ Map<String, List<WrappedReportEntry>> methodStatistics)
throws IOException {
if (rerunFailingTestsCount > 0) {
- serializeTestClassWithRerun(outputStream, fw, ppw, methodEntries);
+ serializeTestClassWithRerun(outputStream, fw, ppw, methodEntries,
methodStatistics);
} else {
// rerunFailingTestsCount is smaller than 1, but for some reasons
a test could be run
// for more than once
@@ -258,10 +262,18 @@ private void serializeTestClassWithoutRerun(
}
private void serializeTestClassWithRerun(
- OutputStream outputStream, OutputStreamWriter fw, XMLWriter ppw,
List<WrappedReportEntry> methodEntries)
+ OutputStream outputStream,
+ OutputStreamWriter fw,
+ XMLWriter ppw,
+ List<WrappedReportEntry> methodEntries,
+ Map<String, List<WrappedReportEntry>> methodStatistics)
throws IOException {
WrappedReportEntry firstMethodEntry = methodEntries.get(0);
- switch (getTestResultType(methodEntries)) {
+
+ TestResultType resultType =
+
getTestResultTypeWithBeforeAllHandling(firstMethodEntry.getName(),
methodEntries, methodStatistics);
+
+ switch (resultType) {
case SUCCESS:
for (WrappedReportEntry methodEntry : methodEntries) {
if (methodEntry.getReportEntryType() == SUCCESS) {
@@ -370,6 +382,40 @@ private TestResultType
getTestResultType(List<WrappedReportEntry> methodEntryLis
return DefaultReporterFactory.getTestResultType(testResultTypeList,
rerunFailingTestsCount);
}
+ /**
+ * Determines the final result type for a test method, applying special
handling for @BeforeAll failures.
+ * If a @BeforeAll fails but any actual test methods succeed, it's
classified as a FLAKE.
+ *
+ * @param methodName the name of the test method (null or "null" for
@BeforeAll)
+ * @param methodRuns the list of runs for this method
+ * @param methodStatistics all method statistics for the test class
+ * @return the final TestResultType
+ */
+ private TestResultType getTestResultTypeWithBeforeAllHandling(
+ String methodName,
+ List<WrappedReportEntry> methodRuns,
+ Map<String, List<WrappedReportEntry>> methodStatistics) {
+ TestResultType resultType = getTestResultType(methodRuns);
+
+ // Special handling for @BeforeAll failures (null method name or
method name is "null")
+ // If @BeforeAll failed but any actual test methods succeeded, treat
it as a flake
+ if ((methodName == null || methodName.equals("null"))
+ && (resultType == TestResultType.ERROR || resultType ==
TestResultType.FAILURE)) {
+ // Check if any actual test methods (non-null and not "null"
names) succeeded
+ boolean hasSuccessfulTestMethods =
methodStatistics.entrySet().stream()
+ .filter(entry ->
+ entry.getKey() != null &&
!entry.getKey().equals("null")) // Only actual test methods
+ .anyMatch(entry -> entry.getValue().stream()
+ .anyMatch(reportEntry ->
reportEntry.getReportEntryType() == SUCCESS));
+
+ if (hasSuccessfulTestMethods) {
+ resultType = TestResultType.FLAKE;
+ }
+ }
+
+ return resultType;
+ }
+
private Deque<WrappedReportEntry> getAddMethodRunHistoryMap(String
testClassName) {
Deque<WrappedReportEntry> methodRunHistory =
testClassMethodRunHistoryMap.get(testClassName);
if (methodRunHistory == null) {
@@ -420,7 +466,10 @@ private void startTestElement(XMLWriter ppw,
WrappedReportEntry report) throws I
}
}
- private void createTestSuiteElement(XMLWriter ppw, WrappedReportEntry
report, TestSetStats testSetStats)
+ private void createTestSuiteElement(
+ XMLWriter ppw,
+ WrappedReportEntry report,
+ Map<String, Map<String, List<WrappedReportEntry>>>
classMethodStatistics)
throws IOException {
ppw.startElement("testsuite");
@@ -441,13 +490,46 @@ private void createTestSuiteElement(XMLWriter ppw,
WrappedReportEntry report, Te
ppw.addAttribute("time", String.valueOf(report.getElapsed() /
ONE_SECOND));
}
- ppw.addAttribute("tests",
String.valueOf(testSetStats.getCompletedCount()));
-
- ppw.addAttribute("errors", String.valueOf(testSetStats.getErrors()));
-
- ppw.addAttribute("skipped", String.valueOf(testSetStats.getSkipped()));
+ // Count actual unique test methods and their final results from
classMethodStatistics (accumulated across
+ // reruns)
+ int actualTestCount = 0;
+ int errors = 0;
+ int failures = 0;
+ int skipped = 0;
+ int flakes = 0;
+
+ for (Map<String, List<WrappedReportEntry>> methodStats :
classMethodStatistics.values()) {
+ actualTestCount += methodStats.size();
+ for (Map.Entry<String, List<WrappedReportEntry>> methodEntry :
methodStats.entrySet()) {
+ String methodName = methodEntry.getKey();
+ List<WrappedReportEntry> methodRuns = methodEntry.getValue();
+ TestResultType resultType =
getTestResultTypeWithBeforeAllHandling(methodName, methodRuns, methodStats);
+
+ switch (resultType) {
+ case ERROR:
+ errors++;
+ break;
+ case FAILURE:
+ failures++;
+ break;
+ case SKIPPED:
+ skipped++;
+ break;
+ case FLAKE:
+ flakes++;
+ break;
+ case SUCCESS:
+ default:
+ break;
+ }
+ }
+ }
- ppw.addAttribute("failures",
String.valueOf(testSetStats.getFailures()));
+ ppw.addAttribute("tests", String.valueOf(actualTestCount));
+ ppw.addAttribute("errors", String.valueOf(errors));
+ ppw.addAttribute("skipped", String.valueOf(skipped));
+ ppw.addAttribute("failures", String.valueOf(failures));
+ ppw.addAttribute("flakes", String.valueOf(flakes));
}
private static void getTestProblems(
diff --git
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
index 86c23c82a..12ad8db8c 100644
---
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
+++
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
@@ -68,6 +68,12 @@ public class DefaultReporterFactoryTest extends TestCase {
private static final String ERROR = "error";
+ private static final String TEST_BEFORE_ALL_FLAKE =
"com.example.FlakyClass";
+
+ private static final String TEST_BEFORE_ALL_ERROR =
"com.example.AlwaysFailClass";
+
+ private static final String TEST_ERROR_SUFFIX = "-- Time elapsed: 292.2 s
<<< ERROR!";
+
public void testMergeTestHistoryResult() throws Exception {
MessageUtils.setColorEnabled(false);
File target = new File(System.getProperty("user.dir"), "target");
@@ -97,7 +103,7 @@ public void testMergeTestHistoryResult() throws Exception {
DefaultReporterFactory factory = new
DefaultReporterFactory(reportConfig, reporter);
- // First run, four tests failed and one passed
+ // First run, four tests failed and one passed, plus @BeforeAll failure
Queue<TestMethodStats> firstRunStats = new ArrayDeque<>();
firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR,
new DummyStackTraceWriter(ERROR)));
firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR,
new DummyStackTraceWriter(ERROR)));
@@ -106,6 +112,11 @@ public void testMergeTestHistoryResult() throws Exception {
firstRunStats.add(
new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new
DummyStackTraceWriter(ASSERTION_FAIL)));
firstRunStats.add(new TestMethodStats(TEST_FIVE,
ReportEntryType.SUCCESS, null));
+ // @BeforeAll failure for a test class that will eventually succeed
+ firstRunStats.add(new TestMethodStats(
+ TEST_BEFORE_ALL_FLAKE + ".null",
+ ReportEntryType.ERROR,
+ new DummyStackTraceWriter(TEST_BEFORE_ALL_FLAKE + ".null " +
TEST_ERROR_SUFFIX)));
// Second run, two tests passed
Queue<TestMethodStats> secondRunStats = new ArrayDeque<>();
@@ -114,11 +125,23 @@ public void testMergeTestHistoryResult() throws Exception
{
secondRunStats.add(new TestMethodStats(TEST_TWO,
ReportEntryType.SUCCESS, null));
secondRunStats.add(new TestMethodStats(TEST_THREE,
ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
secondRunStats.add(new TestMethodStats(TEST_FOUR,
ReportEntryType.SUCCESS, null));
+ // Successful test from the class that had @BeforeAll failure
+ secondRunStats.add(new TestMethodStats(TEST_BEFORE_ALL_FLAKE +
".testSucceed", ReportEntryType.SUCCESS, null));
+ // @BeforeAll failure for a different class that will stay as error
+ secondRunStats.add(new TestMethodStats(
+ TEST_BEFORE_ALL_ERROR + ".null",
+ ReportEntryType.ERROR,
+ new DummyStackTraceWriter(TEST_BEFORE_ALL_ERROR + ".null " +
TEST_ERROR_SUFFIX)));
// Third run, another test passed
Queue<TestMethodStats> thirdRunStats = new ArrayDeque<>();
thirdRunStats.add(new TestMethodStats(TEST_ONE,
ReportEntryType.SUCCESS, null));
thirdRunStats.add(new TestMethodStats(TEST_THREE,
ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
+ // Another @BeforeAll failure for the always-failing class
+ thirdRunStats.add(new TestMethodStats(
+ TEST_BEFORE_ALL_ERROR + ".null",
+ ReportEntryType.ERROR,
+ new DummyStackTraceWriter(TEST_BEFORE_ALL_ERROR + ".null")));
TestSetRunListener firstRunListener = mock(TestSetRunListener.class);
TestSetRunListener secondRunListener = mock(TestSetRunListener.class);
@@ -134,17 +157,21 @@ public void testMergeTestHistoryResult() throws Exception
{
invokeMethod(factory, "mergeTestHistoryResult");
RunStatistics mergedStatistics = factory.getGlobalRunStatistics();
- // Only TEST_THREE is a failing test, other three are flaky tests
- assertEquals(5, mergedStatistics.getCompletedCount());
- assertEquals(1, mergedStatistics.getErrors());
+ // TEST_THREE and AlwaysFailClass.null are failing tests, regular
tests + FlakyClass.null are flaky
+ assertEquals(7, mergedStatistics.getCompletedCount());
+ assertEquals(2, mergedStatistics.getErrors());
assertEquals(0, mergedStatistics.getFailures());
- assertEquals(3, mergedStatistics.getFlakes());
+ assertEquals(4, mergedStatistics.getFlakes());
assertEquals(0, mergedStatistics.getSkipped());
// Now test the result will be printed out correctly
factory.printTestFailures(TestResultType.FLAKE);
String[] expectedFlakeOutput = {
"Flakes: ",
+ TEST_BEFORE_ALL_FLAKE + ".<beforeAll>",
+ " Run 1: " + TEST_BEFORE_ALL_FLAKE + ".null " + TEST_ERROR_SUFFIX,
+ " Run 2: PASS",
+ "",
TEST_FOUR,
" Run 1: " + ASSERTION_FAIL,
" Run 2: PASS",
@@ -164,7 +191,16 @@ public void testMergeTestHistoryResult() throws Exception {
reporter.reset();
factory.printTestFailures(TestResultType.ERROR);
String[] expectedFailureOutput = {
- "Errors: ", TEST_THREE, " Run 1: " + ASSERTION_FAIL, " Run 2: "
+ ERROR, " Run 3: " + ERROR, ""
+ "Errors: ",
+ TEST_BEFORE_ALL_ERROR + ".<beforeAll>",
+ " Run 1: " + TEST_BEFORE_ALL_ERROR + ".null " + TEST_ERROR_SUFFIX,
+ " Run 2: " + TEST_BEFORE_ALL_ERROR + ".null",
+ "",
+ TEST_THREE,
+ " Run 1: " + ASSERTION_FAIL,
+ " Run 2: " + ERROR,
+ " Run 3: " + ERROR,
+ ""
};
assertEquals(asList(expectedFailureOutput), reporter.getMessages());
diff --git
a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformFailingBeforeAllRerunIT.java
b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformFailingBeforeAllRerunIT.java
new file mode 100644
index 000000000..c2a049f3f
--- /dev/null
+++
b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformFailingBeforeAllRerunIT.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.maven.surefire.its;
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+/**
+ * Integration tests for JUnit Platform @BeforeAll failures with rerun
functionality.
+ * Tests various scenarios where @BeforeAll lifecycle methods fail and are
rerun.
+ */
+public class JUnitPlatformFailingBeforeAllRerunIT extends
SurefireJUnit4IntegrationTestCase {
+ private static final String VERSION = "5.9.1";
+
+ private static final String TEST_PROJECT_BASE =
"junit-platform-rerun-failing-before-all";
+
+ @Test
+ public void testBeforeAllFailures() {
+ // Test that @BeforeAll failures are properly handled when they
succeed on rerun
+ OutputValidator outputValidator = unpack(TEST_PROJECT_BASE)
+ .setJUnitVersion(VERSION)
+ .maven()
+ .debugLogging()
+ .addGoal("-Dsurefire.rerunFailingTestsCount=3")
+ .withFailure()
+ .executeTest()
+ .assertTestSuiteResults(7, 1, 0, 0, 4);
+
+ // Verify the @BeforeAll is reported as a flake with proper formatting
+
outputValidator.verifyTextInLog("junitplatform.FlakyFirstTimeTest.<beforeAll>");
+ outputValidator.verifyTextInLog("Run 1: FlakyFirstTimeTest.setup:53
IllegalArgument");
+ outputValidator.verifyTextInLog("Run 2: PASS");
+
+ // Verify XML report doesn't contain error testcase with empty name
+ outputValidator
+
.getSurefireReportsXmlFile("TEST-junitplatform.FlakyFirstTimeTest.xml")
+ .assertContainsText("tests=\"4\" errors=\"0\"")
+ .assertContainsText("name=\"testFailingTestOne\"")
+ .assertContainsText("name=\"testErrorTestOne\"")
+ .assertContainsText("name=\"testPassingTest\"");
+
+ // Verify @BeforeAll is reported as error
+ outputValidator.verifyTextInLog("Errors:");
+
outputValidator.verifyTextInLog("junitplatform.AlwaysFailingTest.<beforeAll>");
+ outputValidator.verifyTextInLog("Run 3: AlwaysFailingTest.setup:15
IllegalArgument BeforeAll always fails");
+ }
+}
diff --git
a/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/pom.xml
b/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/pom.xml
new file mode 100644
index 000000000..702d04f4f
--- /dev/null
+++
b/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/pom.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.maven.plugins.surefire</groupId>
+ <artifactId>junit-platform-rerun-failing-before-all</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <name>Test for rerun failing tests with BeforeAll in JUnit
5/Platform</name>
+
+ <properties>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.version}</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/ParametersTest.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <profiles>
+ <profile>
+ <id>parameters</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/PassingTest.java</exclude>
+ <exclude>**/FlakyFirstTimeTest.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+</project>
diff --git
a/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/src/test/java/junitplatform/AlwaysFailingTest.java
b/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/src/test/java/junitplatform/AlwaysFailingTest.java
new file mode 100644
index 000000000..511026a39
--- /dev/null
+++
b/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/src/test/java/junitplatform/AlwaysFailingTest.java
@@ -0,0 +1,22 @@
+package junitplatform;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test class with @BeforeAll that always fails.
+ * Used to test scenario where @BeforeAll never succeeds even with reruns.
+ */
+public class AlwaysFailingTest {
+
+ @BeforeAll
+ static void setup() {
+ System.out.println("Error beforeAll in AlwaysFailingTest");
+ throw new IllegalArgumentException("BeforeAll always fails");
+ }
+
+ @Test
+ public void testThatNeverRuns() {
+ System.out.println("This test never runs because beforeAll always
fails");
+ }
+}
\ No newline at end of file
diff --git
a/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/src/test/java/junitplatform/FlakyFirstTimeTest.java
b/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/src/test/java/junitplatform/FlakyFirstTimeTest.java
new file mode 100644
index 000000000..fe2e48e79
--- /dev/null
+++
b/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/src/test/java/junitplatform/FlakyFirstTimeTest.java
@@ -0,0 +1,57 @@
+package junitplatform;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+public class FlakyFirstTimeTest
+{
+ private static int failingCount = 0;
+
+ private static int errorCount = 0;
+ private static int errorBeforeAllCount = 0;
+
+
+ @Test
+ public void testFailingTestOne()
+ {
+ System.out.println( "Failing test" );
+ // This test will fail with only one retry, but will pass with two
+ if ( failingCount < 1 )
+ {
+ failingCount++;
+ fail( "Failing test" );
+ }
+ }
+
+ @Test
+ public void testErrorTestOne() throws Exception
+ {
+ System.out.println( "Error test" );
+ // This test will error out with only one retry, but will pass with two
+ if ( errorCount < 2 )
+ {
+ errorCount++;
+ throw new IllegalArgumentException( "..." );
+ }
+ }
+
+ @Test
+ public void testPassingTest() throws Exception
+ {
+ System.out.println( "Passing test" );
+ }
+
+ @BeforeAll
+ static void setup() {
+ if ( errorBeforeAllCount < 1 )
+ {
+ System.out.println( "Error beforeAll" );
+ errorBeforeAllCount++;
+ throw new IllegalArgumentException( "..." );
+ }
+ System.out.println( "Passing beforeAll" );
+ }
+}
diff --git
a/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/src/test/java/junitplatform/PassingTest.java
b/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/src/test/java/junitplatform/PassingTest.java
new file mode 100644
index 000000000..cc48ef4fb
--- /dev/null
+++
b/surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/src/test/java/junitplatform/PassingTest.java
@@ -0,0 +1,19 @@
+package junitplatform;
+
+import org.junit.jupiter.api.Test;
+
+
+public class PassingTest
+{
+ @Test
+ public void testPassingTestOne()
+ {
+ System.out.println( "Passing test one" );
+ }
+
+ @Test
+ public void testPassingTestTwo() throws Exception
+ {
+ System.out.println( "Passing test two" );
+ }
+}
