Repository: maven-surefire Updated Branches: refs/heads/master 1cdf49dc2 -> 2bdeeaf1a
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java b/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java index f3fcc0b..3c2fa86 100644 --- a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java +++ b/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; @@ -38,6 +39,7 @@ import org.apache.maven.surefire.group.parse.ParseException; import org.junit.experimental.categories.Category; import org.junit.runner.Description; import org.junit.runner.manipulation.Filter; +import sun.security.krb5.internal.crypto.Des; /** * @author Todd Lipcon @@ -102,6 +104,11 @@ public class FilterFactory return new MethodFilter( requestedTestMethod ); } + public Filter createFailingMethodFilter( Map<Class<?>, Set<String>> failingClassMethodMap ) + { + return new FailingMethodFilter( failingClassMethodMap ); + } + public Filter and( Filter filter1, Filter filter2 ) { return new AndFilter( filter1, filter2 ); @@ -144,6 +151,56 @@ public class FilterFactory } } + // Only run test methods in the given input map, indexed by test class + private static class FailingMethodFilter + extends Filter + { + // Map from Class -> List of method names. Are the method names hashed to include the signature? + private final Map<Class<?>, Set<String>> failingClassMethodMap; + + public FailingMethodFilter( Map<Class<?>, Set<String>> failingClassMethodMap ) + { + this.failingClassMethodMap = failingClassMethodMap; + } + + @Override + public boolean shouldRun( Description description ) + { + return isDescriptionMatch( description ); + } + + private boolean isDescriptionMatch( Description description ) + { + if ( description.getTestClass() == null || description.getMethodName() == null ) + { + for ( Description childrenDescription : description.getChildren() ) + { + if ( isDescriptionMatch( childrenDescription ) ) + { + return true; + } + } + return false; + } + + Set<String> testMethods = failingClassMethodMap.get( description.getTestClass() ); + if ( testMethods == null ) + { + return false; + } + else + { + return testMethods.contains( description.getMethodName() ); + } + } + + @Override + public String describe() + { + return "By failing class method"; + } + } + private static class GroupMatcherCategoryFilter extends Filter { http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java index a03c7d2..03012bd 100644 --- a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java +++ b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java @@ -19,14 +19,12 @@ package org.apache.maven.surefire.junit4; * under the License. */ -import java.lang.reflect.Method; -import java.util.Iterator; -import java.util.List; - import org.apache.maven.shared.utils.io.SelectorUtils; +import org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil; import org.apache.maven.surefire.common.junit4.JUnit4RunListener; import org.apache.maven.surefire.common.junit4.JUnit4RunListenerFactory; import org.apache.maven.surefire.common.junit4.JUnit4TestChecker; +import org.apache.maven.surefire.common.junit4.JUnitTestFailureListener; import org.apache.maven.surefire.providerapi.AbstractProvider; import org.apache.maven.surefire.providerapi.ProviderParameters; import org.apache.maven.surefire.report.ConsoleOutputCapture; @@ -43,12 +41,19 @@ import org.apache.maven.surefire.util.RunOrderCalculator; import org.apache.maven.surefire.util.ScanResult; import org.apache.maven.surefire.util.TestsToRun; import org.apache.maven.surefire.util.internal.StringUtils; - import org.junit.runner.Request; import org.junit.runner.Result; import org.junit.runner.Runner; +import org.junit.runner.notification.Failure; import org.junit.runner.notification.RunNotifier; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * @author Kristian Rosenvold */ @@ -71,6 +76,8 @@ public class JUnit4Provider private final ScanResult scanResult; + private final int rerunFailingTestsCount; + public JUnit4Provider( ProviderParameters booterParameters ) { @@ -82,7 +89,7 @@ public class JUnit4Provider createCustomListeners( booterParameters.getProviderProperties().getProperty( "listener" ) ); jUnit4TestChecker = new JUnit4TestChecker( testClassLoader ); requestedTestMethod = booterParameters.getTestRequest().getRequestedTestMethod(); - + rerunFailingTestsCount = booterParameters.getTestRequest().getRerunFailingTestsCount(); } public RunResult invoke( Object forkTestSet ) @@ -108,7 +115,7 @@ public class JUnit4Provider final ReporterFactory reporterFactory = providerParameters.getReporterFactory(); - final RunListener reporter = reporterFactory.createReporter(); + RunListener reporter = reporterFactory.createReporter(); ConsoleOutputCapture.startCapture( (ConsoleOutputReceiver) reporter ); @@ -121,7 +128,7 @@ public class JUnit4Provider for ( Class aTestsToRun : testsToRun ) { - executeTestSet( aTestsToRun, reporter, runNotifer ); + executeTestSet( aTestsToRun, reporter, runNotifer, null ); } runNotifer.fireTestRunFinished( result ); @@ -129,28 +136,30 @@ public class JUnit4Provider JUnit4RunListener.rethrowAnyTestMechanismFailures( result ); closeRunNotifer( jUnit4TestSetReporter, customRunListeners ); - return reporterFactory.close(); } - private void executeTestSet( Class<?> clazz, RunListener reporter, RunNotifier listeners ) + private void executeTestSet( Class<?> clazz, RunListener reporter, RunNotifier listeners, + String[] failingTestMethods ) throws ReporterException, TestSetFailedException { final ReportEntry report = new SimpleReportEntry( this.getClass().getName(), clazz.getName() ); reporter.testSetStarting( report ); + + try { if ( !StringUtils.isBlank( this.requestedTestMethod ) ) { String actualTestMethod = getMethod( clazz, this.requestedTestMethod );//add by rainLee String[] testMethods = StringUtils.split( actualTestMethod, "+" ); - execute( clazz, listeners, testMethods ); + executeWithRerun( clazz, listeners, testMethods ); } else {//the original way - execute( clazz, listeners, null ); + executeWithRerun( clazz, listeners, failingTestMethods ); } } catch ( TestSetFailedException e ) @@ -169,6 +178,34 @@ public class JUnit4Provider } } + private void executeWithRerun( Class<?> clazz, RunNotifier listeners, String[] testMethods ) + throws TestSetFailedException + { + JUnitTestFailureListener failureListener = new JUnitTestFailureListener(); + listeners.addListener( failureListener ); + + execute( clazz, listeners, testMethods ); + + // Rerun failing tests if rerunFailingTestsCount is larger than 0 + int rerunCount = this.rerunFailingTestsCount; + if ( rerunCount > 0 ) + { + for ( int i = 0; i < rerunCount; i++ ) + { + List<Failure> failures = failureListener.getAllFailures(); + if ( failures.size() == 0 ) + { + break; + } + + Set<String> methodsSet = JUnit4ProviderUtil.generateFailingTests( failureListener.getAllFailures() ); + String[] methods = methodsSet.toArray(new String[methodsSet.size()]); + failureListener.reset(); + execute( clazz, listeners, methods ); + } + } + } + private RunNotifier getRunNotifer( org.junit.runner.notification.RunListener main, Result result, List<org.junit.runner.notification.RunListener> others ) { http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderTest.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderTest.java b/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderTest.java index 147a5c1..bbca2d3 100644 --- a/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderTest.java +++ b/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderTest.java @@ -19,11 +19,18 @@ package org.apache.maven.surefire.junit4; * under the License. */ -import java.util.Properties; +import junit.framework.TestCase; import org.apache.maven.surefire.booter.BaseProviderFactory; import org.apache.maven.surefire.testset.TestRequest; +import org.apache.maven.surefire.util.TestsToRun; +import org.junit.runner.Description; +import org.junit.runner.notification.Failure; -import junit.framework.TestCase; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; /** * @author Kristian Rosenvold @@ -33,10 +40,15 @@ public class JUnit4ProviderTest { public void testCreateProvider() { + assertNotNull( getJUnit4Provider() ); + } + + private JUnit4Provider getJUnit4Provider() + { BaseProviderFactory providerParameters = new BaseProviderFactory( null, Boolean.TRUE ); providerParameters.setProviderProperties( new Properties() ); providerParameters.setClassLoaders( this.getClass().getClassLoader() ); providerParameters.setTestRequest( new TestRequest( null, null, null ) ); - assertNotNull( new JUnit4Provider( providerParameters ) ); + return new JUnit4Provider( providerParameters ); } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java index 04f1673..8cbf0cd 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java @@ -19,12 +19,17 @@ package org.apache.maven.surefire.junitcore; * under the License. */ +import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil; import org.apache.maven.surefire.common.junit4.JUnit4RunListenerFactory; +import org.apache.maven.surefire.common.junit4.JUnitTestFailureListener; import org.apache.maven.surefire.common.junit48.FilterFactory; import org.apache.maven.surefire.common.junit48.JUnit48Reflector; import org.apache.maven.surefire.common.junit48.JUnit48TestChecker; @@ -43,7 +48,11 @@ import org.apache.maven.surefire.util.ScanResult; import org.apache.maven.surefire.util.ScannerFilter; import org.apache.maven.surefire.util.TestsToRun; import org.apache.maven.surefire.util.internal.StringUtils; +import org.junit.internal.runners.statements.Fail; import org.junit.runner.manipulation.Filter; +import org.junit.runner.notification.Failure; + +import static org.apache.maven.surefire.common.junit4.JUnit4RunListener.isFailureInsideJUnitItself; /** * @author Kristian Rosenvold @@ -72,6 +81,8 @@ public class JUnitCoreProvider private final ScanResult scanResult; + private final int rerunFailingTestsCount; + public JUnitCoreProvider( ProviderParameters providerParameters ) { this.providerParameters = providerParameters; @@ -81,6 +92,7 @@ public class JUnitCoreProvider this.jUnitCoreParameters = new JUnitCoreParameters( providerParameters.getProviderProperties() ); this.scannerFilter = new JUnit48TestChecker( testClassLoader ); this.requestedTestMethod = providerParameters.getTestRequest().getRequestedTestMethod(); + this.rerunFailingTestsCount = providerParameters.getTestRequest().getRerunFailingTestsCount(); customRunListeners = JUnit4RunListenerFactory.createCustomListeners( providerParameters.getProviderProperties().getProperty( "listener" ) ); @@ -131,7 +143,33 @@ public class JUnitCoreProvider org.junit.runner.notification.RunListener jUnit4RunListener = getRunListener( reporterFactory, consoleLogger ); customRunListeners.add( 0, jUnit4RunListener ); + + // Add test failure listener + JUnitTestFailureListener testFailureListener = new JUnitTestFailureListener(); + customRunListeners.add( 0, testFailureListener ); + JUnitCoreWrapper.execute( testsToRun, jUnitCoreParameters, customRunListeners, filter ); + + // Rerun failing tests if rerunFailingTestsCount is larger than 0 + int rerunCount = this.rerunFailingTestsCount; + if ( rerunCount > 0 ) + { + for ( int i = 0; i < rerunCount; i++ ) + { + List<Failure> failures = testFailureListener.getAllFailures(); + if ( failures.size() == 0 ) + { + break; + } + Map<Class<?>, Set<String>> failingTests = + JUnit4ProviderUtil.generateFailingTests( failures, testsToRun ); + testFailureListener.reset(); + final FilterFactory filterFactory = new FilterFactory( testClassLoader ); + Filter failingMethodsFilter = filterFactory.createFailingMethodFilter( failingTests ); + JUnitCoreWrapper.execute( testsToRun, jUnitCoreParameters, customRunListeners, + filterFactory.and( filter, failingMethodsFilter ) ); + } + } return reporterFactory.close(); } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/ConcurrentRunListenerTest.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/ConcurrentRunListenerTest.java b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/ConcurrentRunListenerTest.java index f41554d..24fb2da 100644 --- a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/ConcurrentRunListenerTest.java +++ b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/ConcurrentRunListenerTest.java @@ -338,7 +338,7 @@ public class ConcurrentRunListenerTest TestSuite suite = new TestSuite(); suite.addTest( new Junit3OddTest1( "testMe" ) ); - suite.addTest( new Junit3OddTest1( "testMe" ) ); + suite.addTest( new Junit3OddTest1( "testMe2" ) ); return suite; } @@ -361,7 +361,7 @@ public class ConcurrentRunListenerTest { TestSuite suite = new TestSuite(); - suite.addTest( new Junit3WithNestedSuite( "testMe2" ) ); + suite.addTest( new Junit3WithNestedSuite( "testMe" ) ); suite.addTest( new Junit3WithNestedSuite( "testMe2" ) ); suite.addTestSuite( Junit3Tc2.class ); return suite; @@ -388,7 +388,6 @@ public class ConcurrentRunListenerTest private void assertReporter( RunStatistics result, int success, int ignored, int failure, String message ) { assertEquals( message, success, result.getCompletedCount() ); - assertEquals( message, failure, result.getFailureSources().size() ); assertEquals( message, ignored, result.getSkipped() ); } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestSuite.java ---------------------------------------------------------------------- diff --git a/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestSuite.java b/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestSuite.java index 2bd6513..989eda4 100644 --- a/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestSuite.java +++ b/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestSuite.java @@ -35,6 +35,8 @@ public class ReportTestSuite private int numberOfSkipped; + private int numberOfFlakes; + private Integer numberOfTests; private String name; @@ -80,6 +82,16 @@ public class ReportTestSuite this.numberOfSkipped = numberOfSkipped; } + public int getNumberOfFlakes() + { + return numberOfFlakes; + } + + public void setNumberOfFlakes( int numberOfFlakes ) + { + this.numberOfFlakes = numberOfFlakes; + } + public int getNumberOfTests() { if ( numberOfTests != null ) http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java ---------------------------------------------------------------------- diff --git a/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java b/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java index e54443d..72afd44 100644 --- a/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java +++ b/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java @@ -218,6 +218,10 @@ public class TestSuiteXmlParser testCase.addFailure( message != null ? message : "skipped", "skipped" ); currentSuite.setNumberOfSkipped( 1 + currentSuite.getNumberOfSkipped() ); } + else if ( "flakyFailure".equals( qName ) || "flakyError".equals( qName ) ) + { + currentSuite.setNumberOfFlakes( 1 + currentSuite.getNumberOfFlakes() ); + } else if ( "failsafe-summary".equals( qName ) ) { valid = false;
