This is an automated email from the ASF dual-hosted git repository. tibordigana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
commit 75239c925dc3686072c755267b375665d905ccb0 Author: Stefan Oehme <[email protected]> AuthorDate: Thu Jan 28 12:11:30 2021 +0100 [SUREFIRE-1878] Add failOnFlakeCount option --- .../apache/maven/plugin/failsafe/VerifyMojo.java | 24 +++++++++++ .../plugin/surefire/AbstractSurefireMojo.java | 6 +++ .../maven/plugin/surefire/SurefireHelper.java | 30 ++++++++++++- .../plugin/surefire/SurefireReportParameters.java | 4 ++ .../plugin/surefire/AbstractSurefireMojoTest.java | 13 ++++++ .../maven/plugin/surefire/SurefireHelperTest.java | 49 +++++++++++++++++++--- .../maven/plugin/surefire/SurefirePlugin.java | 35 ++++++++++++++++ .../maven/plugin/surefire/SurefirePluginTest.java | 13 ++++++ .../its/JUnitPlatformRerunFailingTestsIT.java | 13 ++++++ 9 files changed, 180 insertions(+), 7 deletions(-) diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/VerifyMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/VerifyMojo.java index dd8950d..b9309a3 100644 --- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/VerifyMojo.java +++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/VerifyMojo.java @@ -140,6 +140,13 @@ public class VerifyMojo private Boolean failIfNoTests; /** + * Set this to a value greater than 0 to fail the whole test set if the cumulative number of flakes reaches + * this threshold. Set to 0 to allow an unlimited number of flakes. + */ + @Parameter( property = "failsafe.failOnFlakeCount", defaultValue = "0" ) + private int failOnFlakeCount; + + /** * The character encoding scheme to be applied. * Deprecated since 2.20.1 and used encoding UTF-8 in <tt>failsafe-summary.xml</tt>. * @@ -236,6 +243,11 @@ public class VerifyMojo return false; } + if ( failOnFlakeCount < 0 ) + { + throw new MojoFailureException( "Parameter \"failOnFlakeCount\" should not be negative." ); + } + return true; } @@ -357,6 +369,18 @@ public class VerifyMojo this.failIfNoTests = failIfNoTests; } + @Override + public int getFailOnFlakeCount() + { + return failOnFlakeCount; + } + + @Override + public void setFailOnFlakeCount( int failOnFlakeCount ) + { + this.failOnFlakeCount = failOnFlakeCount; + } + private boolean existsSummaryFile() { return summaryFile != null && summaryFile.isFile(); diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java index ecceea6..7ae8486 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java @@ -1133,6 +1133,7 @@ public abstract class AbstractSurefireMojo warnIfNotApplicableSkipAfterFailureCount(); warnIfIllegalTempDir(); warnIfForkCountIsZero(); + warnIfIllegalFailOnFlakeCount(); printDefaultSeedIfNecessary(); } return true; @@ -3055,6 +3056,11 @@ public abstract class AbstractSurefireMojo } } + protected void warnIfIllegalFailOnFlakeCount() throws MojoFailureException + { + + } + private void printDefaultSeedIfNecessary() { if ( getRunOrderRandomSeed() == null && getRunOrder().equals( RunOrder.RANDOM.name() ) ) diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/SurefireHelper.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/SurefireHelper.java index 9149946..d9a6918 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/SurefireHelper.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/SurefireHelper.java @@ -142,7 +142,9 @@ public final class SurefireHelper PluginConsoleLogger log, Exception firstForkException ) throws MojoFailureException, MojoExecutionException { - if ( firstForkException == null && !result.isTimeout() && result.isErrorFree() ) + boolean isError = firstForkException != null || result.isTimeout() || !result.isErrorFree(); + boolean isTooFlaky = isTooFlaky( result, reportParameters ); + if ( !isError && !isTooFlaky ) { if ( result.getCompletedCount() == 0 && failIfNoTests( reportParameters ) ) { @@ -286,7 +288,25 @@ public final class SurefireHelper } else { - msg.append( "There are test failures.\n\nPlease refer to " ) + if ( result.getFailures() > 0 ) + { + msg.append( "There are test failures." ); + } + if ( isTooFlaky( result, reportParameters ) ) + { + if ( result.getFailures() > 0 ) + { + msg.append( "\n" ); + } + msg.append( "There" ) + .append( result.getFlakes() == 1 ? " is " : " are " ) + .append( result.getFlakes() ) + .append( result.getFlakes() == 1 ? " flake " : " flakes " ) + .append( "and failOnFlakeCount is set to " ) + .append( reportParameters.getFailOnFlakeCount() ) + .append( "." ); + } + msg.append( "\n\nPlease refer to " ) .append( reportParameters.getReportsDirectory() ) .append( " for the individual test results." ) .append( '\n' ) @@ -314,4 +334,10 @@ public final class SurefireHelper return msg.toString(); } + private static boolean isTooFlaky( RunResult result, SurefireReportParameters reportParameters ) + { + int failOnFlakeCount = reportParameters.getFailOnFlakeCount(); + return failOnFlakeCount > 0 && result.getFlakes() >= failOnFlakeCount; + } + } diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/SurefireReportParameters.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/SurefireReportParameters.java index f345b46..8b964e7 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/SurefireReportParameters.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/SurefireReportParameters.java @@ -59,4 +59,8 @@ public interface SurefireReportParameters Boolean getFailIfNoTests(); void setFailIfNoTests( boolean failIfNoTests ); + + int getFailOnFlakeCount(); + + void setFailOnFlakeCount( int failOnFlakeCount ); } diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java index 06e52be..53e7fba 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java @@ -2026,6 +2026,7 @@ public class AbstractSurefireMojoTest private File mainBuildPath; private File testClassesDirectory; private boolean useModulePath; + private int failOnFlakeCount; private JUnitPlatformProviderInfo createJUnitPlatformProviderInfo( Artifact junitPlatformArtifact, TestClassPath testClasspathWrapper ) @@ -2469,6 +2470,18 @@ public class AbstractSurefireMojoTest { setInternalState( this, "jvm", jvm ); } + + @Override + public int getFailOnFlakeCount() + { + return failOnFlakeCount; + } + + @Override + public void setFailOnFlakeCount( int failOnFlakeCount ) + { + this.failOnFlakeCount = failOnFlakeCount; + } } private static File mockFile( String absolutePath ) diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/SurefireHelperTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/SurefireHelperTest.java index 19fe288..fb4033d 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/SurefireHelperTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/SurefireHelperTest.java @@ -22,7 +22,9 @@ package org.apache.maven.plugin.surefire; import org.apache.maven.plugin.MojoFailureException; import org.apache.maven.plugin.surefire.AbstractSurefireMojoTest.Mojo; import org.apache.maven.surefire.api.suite.RunResult; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.io.File; import java.util.ArrayList; @@ -42,6 +44,10 @@ import static org.junit.Assume.assumeTrue; */ public class SurefireHelperTest { + + @Rule + public ExpectedException e = ExpectedException.none(); + @Test public void shouldReplaceForkNumberPath() { @@ -142,6 +148,10 @@ public class SurefireHelperTest try { reportExecution( new Mojo(), summary, null, null ); + fail( "Expected MojoFailureException with message " + + "'There are test failures.\n\nPlease refer to null " + + "for the individual test results.\nPlease refer to dump files (if any exist) " + + "[date].dump, [date]-jvmRun[N].dump and [date].dumpstream.'" ); } catch ( MojoFailureException e ) { @@ -149,11 +159,40 @@ public class SurefireHelperTest .isEqualTo( "There are test failures.\n\nPlease refer to null " + "for the individual test results.\nPlease refer to dump files (if any exist) " + "[date].dump, [date]-jvmRun[N].dump and [date].dumpstream." ); - return; } - fail( "Expected MojoFailureException with message " - + "'There are test failures.\n\nPlease refer to null " - + "for the individual test results.\nPlease refer to dump files (if any exist) " - + "[date].dump, [date]-jvmRun[N].dump and [date].dumpstream.'" ); + } + + @Test + public void failsIfThereAreTooManyFlakes() throws Exception + { + RunResult summary = new RunResult( 1, 0, 0, 0, 1 ); + Mojo reportParameters = new Mojo(); + reportParameters.setFailOnFlakeCount( 1 ); + e.expect( MojoFailureException.class ); + e.expectMessage( "There is 1 flake and failOnFlakeCount is set to 1.\n\nPlease refer to null " + + "for the individual test results.\nPlease refer to dump files (if any exist) " + + "[date].dump, [date]-jvmRun[N].dump and [date].dumpstream." ); + reportExecution( reportParameters, summary, null, null ); + } + + @Test + public void reportsFailuresAndFlakes() throws Exception + { + RunResult summary = new RunResult( 1, 0, 1, 0, 2 ); + Mojo reportParameters = new Mojo(); + reportParameters.setFailOnFlakeCount( 1 ); + e.expect( MojoFailureException.class ); + e.expectMessage( "There are test failures.\nThere are 2 flakes and failOnFlakeCount is set to 1." + + "\n\nPlease refer to null " + + "for the individual test results.\nPlease refer to dump files (if any exist) " + + "[date].dump, [date]-jvmRun[N].dump and [date].dumpstream." ); + reportExecution( reportParameters, summary, null, null ); + } + + @Test + public void passesIfFlakesAreWithinThreshold() throws Exception + { + RunResult summary = new RunResult( 1, 0, 0, 0 , 1 ); + reportExecution( new Mojo(), summary, null, null ); } } diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java index cd260c1..5126a62 100644 --- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java +++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.List; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugin.surefire.booterclient.ChecksumCalculator; import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; @@ -254,6 +255,13 @@ public class SurefirePlugin private int rerunFailingTestsCount; /** + * Set this to a value greater than 0 to fail the whole test set if the cumulative number of flakes reaches + * this threshold. Set to 0 to allow an unlimited number of flakes. + */ + @Parameter( property = "surefire.failOnFlakeCount", defaultValue = "0" ) + private int failOnFlakeCount; + + /** * (TestNG) List of <suiteXmlFile> elements specifying TestNG suite xml file locations. Note that * {@code suiteXmlFiles} is incompatible with several other parameters of this plugin, like * {@code includes} and {@code excludes}.<br> @@ -463,6 +471,18 @@ public class SurefirePlugin } @Override + public int getFailOnFlakeCount() + { + return failOnFlakeCount; + } + + @Override + public void setFailOnFlakeCount( int failOnFlakeCount ) + { + this.failOnFlakeCount = failOnFlakeCount; + } + + @Override protected void handleSummary( RunResult summary, Exception firstForkException ) throws MojoExecutionException, MojoFailureException { @@ -880,4 +900,19 @@ public class SurefirePlugin { return forkNode; } + + @Override + protected void warnIfIllegalFailOnFlakeCount() throws MojoFailureException + { + if ( failOnFlakeCount < 0 ) + { + throw new MojoFailureException( "Parameter \"failOnFlakeCount\" should not be negative." ); + } + } + + @Override + protected void addPluginSpecificChecksumItems( ChecksumCalculator checksum ) + { + checksum.add( skipAfterFailureCount ); + } } diff --git a/maven-surefire-plugin/src/test/java/org/apache/maven/plugin/surefire/SurefirePluginTest.java b/maven-surefire-plugin/src/test/java/org/apache/maven/plugin/surefire/SurefirePluginTest.java index 71ad8e3..e4297f4 100644 --- a/maven-surefire-plugin/src/test/java/org/apache/maven/plugin/surefire/SurefirePluginTest.java +++ b/maven-surefire-plugin/src/test/java/org/apache/maven/plugin/surefire/SurefirePluginTest.java @@ -22,6 +22,8 @@ package org.apache.maven.plugin.surefire; import junit.framework.TestCase; import org.apache.maven.plugin.MojoFailureException; import org.apache.maven.surefire.api.suite.RunResult; +import org.junit.Rule; +import org.junit.rules.ExpectedException; import java.io.File; @@ -32,6 +34,9 @@ import static org.fest.assertions.Assertions.assertThat; */ public class SurefirePluginTest extends TestCase { + @Rule + public final ExpectedException e = ExpectedException.none(); + public void testDefaultIncludes() { assertThat( new SurefirePlugin().getDefaultIncludes() ) @@ -114,4 +119,12 @@ public class SurefirePluginTest extends TestCase assertThat( plugin.getSystemPropertiesFile() ) .isEqualTo( new File( "testShouldGetPropertyFile" ) ); } + + public void testFailOnFlakeCountVerification() + { + SurefirePlugin plugin = new SurefirePlugin(); + plugin.setFailOnFlakeCount( -1 ); + e.expect( MojoFailureException.class ); + e.expectMessage( "Parameter \"failOnFlakeCount\" should not be negative." ); + } } diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java index 2b2b3b4..146e325 100644 --- a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java @@ -204,6 +204,19 @@ public class JUnitPlatformRerunFailingTestsIT extends SurefireJUnit4IntegrationT } @Test + public void testFailOnTooManyFlakes() + { + OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging() + .addGoal( "-Dsurefire.rerunFailingTestsCount=2" ) + .addGoal( "-Dsurefire.failOnFlakeCount=1" ) + .withFailure() + .executeTest() + .assertTestSuiteResults( 5, 0, 0, 0, 4 ); + + outputValidator.verifyTextInLog( "There are 2 flakes and failOnFlakeCount is set to 1" ); + } + + @Test public void testParameterizedTest() { unpack()
