[
https://issues.apache.org/jira/browse/SUREFIRE-2148?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Pavlo Shevchenko updated SUREFIRE-2148:
---------------------------------------
Description:
*Summary*
If a JUnit5 test class fails during setup and succeeds on retry, then the
surefire test goal and entire build will fail. On the other hand, if the
failure occurs in a test method which succeeds on retry, then the goal and the
build will succeed.
*Reproducer*
Test class with flaky setup:
{code:java}
package example;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
public class FlakyClassTest {
@BeforeAll
public static void setup() throws IOException {
String testSetupMarkerFile = "testSetupMarker.txt";
if (!new File(testSetupMarkerFile).exists()) {
System.out.println("I'm failing!");
Files.write(Paths.get(testSetupMarkerFile), "Hello".getBytes());
throw new RuntimeException("I'm failing!");
} else {
System.out.println("I'm passing!");
}
}
@Test
public void test() {
}
} {code}
Test class with flaky method:
{code:java}
package example;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
public class FlakyMethodTest {
@Test
public void test() throws IOException {
String testMethodMarkerFile = "testMethodMarker.txt";
if (!new File(testMethodMarkerFile).exists()) {
System.out.println("I'm failing!");
Files.write(Paths.get(testMethodMarkerFile), "Hello".getBytes());
throw new RuntimeException("I'm failing!");
} else {
System.out.println("I'm passing!");
}
}
}
{code}
Surefire config
{code:java}
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M8</version>
<configuration>
<rerunFailingTestsCount>1</rerunFailingTestsCount>
</configuration>
</plugin>{code}
*Actual behavior*
1. mvn test -Dtest={*}FlakyMethod{*} succeeds
2. mvn test -Dtest={*}FlakyClass{*} fails
*Expected behavior*
1. Both executions succed
was:
*Summary*
If a JUnit5 test class fails during setup and succeeds on retry, then the
surefire test goal and entire build will fail. On the other hand, if the
failure occurs in a test method which succeeds on retry, then the goal and the
build will succeed.
*Reproducer*
Test class with flaky setup:
{code:java}
package example;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
public class FlakyClassTest {
@BeforeAll
public static void setup() throws IOException {
String testSetupMarkerFile = "testSetupMarker.txt";
if (!new File(testSetupMarkerFile).exists()) {
System.out.println("I'm failing!");
Files.write(Paths.get(testSetupMarkerFile), "Hello".getBytes());
throw new RuntimeException("I'm failing!");
} else {
System.out.println("I'm passing!");
}
}
@Test
public void test() {
}
} {code}
Test class with flaky method:
{code:java}
package example;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
public class FlakyMethodTest {
@Test
public void test() throws IOException {
String testMethodMarkerFile = "testMethodMarker.txt";
if (!new File(testMethodMarkerFile).exists()) {
System.out.println("I'm failing!");
Files.write(Paths.get(testMethodMarkerFile), "Hello".getBytes());
throw new RuntimeException("I'm failing!");
} else {
System.out.println("I'm passing!");
}
}
}
{code}
Surefire config
{code:java}
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M8</version>
<configuration>
<rerunFailingTestsCount>1</rerunFailingTestsCount>
</configuration>
</plugin>{code}
*Actual behavior*
1. mvn test -Dtest=*FlakyMethod* succeeds
2. mvn test -Dtest=*FlakyClass* fails
*Expected behavior*
1. Both executions succed
> Build fails if retried test classes failed during setup
> -------------------------------------------------------
>
> Key: SUREFIRE-2148
> URL: https://issues.apache.org/jira/browse/SUREFIRE-2148
> Project: Maven Surefire
> Issue Type: Bug
> Components: JUnit 5.x support
> Affects Versions: 3.0.0-M8
> Reporter: Pavlo Shevchenko
> Priority: Major
>
> *Summary*
> If a JUnit5 test class fails during setup and succeeds on retry, then the
> surefire test goal and entire build will fail. On the other hand, if the
> failure occurs in a test method which succeeds on retry, then the goal and
> the build will succeed.
>
> *Reproducer*
> Test class with flaky setup:
>
> {code:java}
> package example;
> import org.junit.jupiter.api.BeforeAll;
> import org.junit.jupiter.api.Test;
> import java.io.File;
> import java.io.IOException;
> import java.nio.file.Files;
> import java.nio.file.Paths;
> public class FlakyClassTest {
> @BeforeAll
> public static void setup() throws IOException {
> String testSetupMarkerFile = "testSetupMarker.txt";
> if (!new File(testSetupMarkerFile).exists()) {
> System.out.println("I'm failing!");
> Files.write(Paths.get(testSetupMarkerFile), "Hello".getBytes());
> throw new RuntimeException("I'm failing!");
> } else {
> System.out.println("I'm passing!");
> }
> }
> @Test
> public void test() {
> }
> } {code}
> Test class with flaky method:
>
>
> {code:java}
> package example;
> import org.junit.jupiter.api.Test;
> import java.io.File;
> import java.io.IOException;
> import java.nio.file.Files;
> import java.nio.file.Paths;
> public class FlakyMethodTest {
> @Test
> public void test() throws IOException {
> String testMethodMarkerFile = "testMethodMarker.txt";
> if (!new File(testMethodMarkerFile).exists()) {
> System.out.println("I'm failing!");
> Files.write(Paths.get(testMethodMarkerFile), "Hello".getBytes());
> throw new RuntimeException("I'm failing!");
> } else {
> System.out.println("I'm passing!");
> }
> }
> }
> {code}
> Surefire config
>
>
> {code:java}
> <plugin>
> <artifactId>maven-surefire-plugin</artifactId>
> <version>3.0.0-M8</version>
> <configuration>
> <rerunFailingTestsCount>1</rerunFailingTestsCount>
> </configuration>
> </plugin>{code}
>
> *Actual behavior*
> 1. mvn test -Dtest={*}FlakyMethod{*} succeeds
> 2. mvn test -Dtest={*}FlakyClass{*} fails
>
> *Expected behavior*
> 1. Both executions succed
>
>
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
