http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java ---------------------------------------------------------------------- diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java deleted file mode 100644 index 5a74c45..0000000 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java +++ /dev/null @@ -1,172 +0,0 @@ -package org.apache.maven.plugin.surefire.report; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; - -import junit.framework.AssertionFailedError; -import junit.framework.TestCase; - -import org.apache.maven.plugin.surefire.booterclient.output.DeserializedStacktraceWriter; -import org.apache.maven.shared.utils.StringUtils; -import org.apache.maven.shared.utils.xml.Xpp3Dom; -import org.apache.maven.shared.utils.xml.Xpp3DomBuilder; -import org.apache.maven.surefire.report.LegacyPojoStackTraceWriter; -import org.apache.maven.surefire.report.ReportEntry; -import org.apache.maven.surefire.report.SimpleReportEntry; -import org.apache.maven.surefire.report.StackTraceWriter; - -@SuppressWarnings( "ResultOfMethodCallIgnored" ) -public class StatelessXMLReporterTest - extends TestCase -{ - - private StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false ); - - private ReportEntry reportEntry; - - private TestSetStats stats; - - private File expectedReportFile; - - protected void setUp() - throws Exception - { - super.setUp(); - reportEntry = new SimpleReportEntry( this.getClass().getName(), "StatelessXMLReporterTest", - new LegacyPojoStackTraceWriter( "", "", new AssertionFailedError() ), 17 ); - stats = new TestSetStats( false, true ); - } - - @Override protected void tearDown() - throws Exception - { - super.tearDown(); - - if ( expectedReportFile != null ) - { - expectedReportFile.delete(); - } - } - - public void testFileNameWithoutSuffix() - { - File reportDir = new File( "." ); - String testName = "org.apache.maven.plugin.surefire.report.StatelessXMLReporterTest"; - reportEntry = new SimpleReportEntry( this.getClass().getName(), testName, 12 ); - WrappedReportEntry testSetReportEntry = - new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null ); - stats.testSucceeded( testSetReportEntry ); - reporter.testSetCompleted( testSetReportEntry, stats ); - - expectedReportFile = new File( reportDir, "TEST-" + testName + ".xml" ); - assertTrue( "Report file (" + expectedReportFile.getAbsolutePath() + ") doesn't exist", - expectedReportFile.exists() ); - } - - - public void testAllFieldsSerialized() - throws IOException - { - File reportDir = new File( "." ); - String testName = "aTestMethod"; - String testName2 = "bTestMethod"; - reportEntry = new SimpleReportEntry( this.getClass().getName(), testName, 12 ); - WrappedReportEntry testSetReportEntry = - new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null ); - expectedReportFile = new File( reportDir, "TEST-" + testName + ".xml" ); - - stats.testSucceeded( testSetReportEntry ); - StackTraceWriter stackTraceWriter = new DeserializedStacktraceWriter( "A fud msg", "trimmed", "fail at foo" ); - Utf8RecodingDeferredFileOutputStream stdOut = new Utf8RecodingDeferredFileOutputStream( "fds" ); - String stdOutPrefix; - String stdErrPrefix; - if ( defaultCharsetSupportsSpecialChar() ) - { - stdErrPrefix = "std-\u0115rr"; - stdOutPrefix = "st]]>d-o\u00DCt"; - } - else - { - stdErrPrefix = "std-err"; - stdOutPrefix = "st]]>d-out"; - } - - byte[] stdOutBytes = (stdOutPrefix + "<null>!\u0020\u0000\u001F").getBytes(); - stdOut.write( stdOutBytes, 0, stdOutBytes.length ); - - Utf8RecodingDeferredFileOutputStream stdErr = new Utf8RecodingDeferredFileOutputStream( "fds" ); - - - byte[] stdErrBytes = (stdErrPrefix + "?&-&£\u0020\u0000\u001F").getBytes(); - stdErr.write( stdErrBytes, 0, stdErrBytes.length ); - WrappedReportEntry t2 = - new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), testName2, stackTraceWriter, 13 ), - ReportEntryType.error, 13, stdOut, stdErr ); - - stats.testSucceeded( t2 ); - StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false ); - reporter.testSetCompleted( testSetReportEntry, stats ); - - FileInputStream fileInputStream = new FileInputStream( expectedReportFile ); - - Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, "UTF-8") ); - assertEquals( "testsuite", testSuite.getName() ); - Xpp3Dom properties = testSuite.getChild( "properties" ); - assertEquals( System.getProperties().size(), properties.getChildCount() ); - Xpp3Dom child = properties.getChild( 1 ); - assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) ); - assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) ); - - Xpp3Dom[] testcase = testSuite.getChildren( "testcase" ); - Xpp3Dom tca = testcase[0]; - assertEquals( testName, tca.getAttribute( "name" ) ); // Hopefully same order on jdk5 - assertEquals( "0.012", tca.getAttribute( "time" ) ); - assertEquals( this.getClass().getName(), tca.getAttribute( "classname" ) ); - - Xpp3Dom tcb = testcase[1]; - assertEquals( testName2, tcb.getAttribute( "name" ) ); - assertEquals( "0.013", tcb.getAttribute( "time" ) ); - assertEquals( Inner.class.getName(), tcb.getAttribute( "classname" ) ); - Xpp3Dom errorNode = tcb.getChild( "error" ); - assertNotNull( errorNode ); - assertEquals( "A fud msg", errorNode.getAttribute( "message" ) ); - assertEquals( "fail at foo", errorNode.getAttribute( "type" ) ); - assertEquals( stdOutPrefix + "<null>! &#0;&#31;", tcb.getChild( "system-out" ).getValue() ); - - - assertEquals( stdErrPrefix + "?&-&£ &#0;&#31;", tcb.getChild( "system-err" ).getValue() ); - } - - private boolean defaultCharsetSupportsSpecialChar() - { - // some charsets are not able to deal with \u0115 on both ways of the conversion - return "\u0115\u00DC".equals( new String( "\u0115\u00DC".getBytes() ) ); - } - - class Inner - { - - } - -}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java ---------------------------------------------------------------------- diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java new file mode 100644 index 0000000..c6c1d15 --- /dev/null +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java @@ -0,0 +1,290 @@ +package org.apache.maven.plugin.surefire.report; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import junit.framework.AssertionFailedError; +import junit.framework.TestCase; +import org.apache.maven.plugin.surefire.booterclient.output.DeserializedStacktraceWriter; +import org.apache.maven.shared.utils.StringUtils; +import org.apache.maven.shared.utils.xml.Xpp3Dom; +import org.apache.maven.shared.utils.xml.Xpp3DomBuilder; +import org.apache.maven.surefire.report.LegacyPojoStackTraceWriter; +import org.apache.maven.surefire.report.ReportEntry; +import org.apache.maven.surefire.report.SimpleReportEntry; +import org.apache.maven.surefire.report.StackTraceWriter; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; + +@SuppressWarnings( "ResultOfMethodCallIgnored" ) +public class StatelessXmlReporterTest + extends TestCase +{ + + private StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false, 0 ); + + private ReportEntry reportEntry; + + private TestSetStats stats; + + private TestSetStats rerunStats; + + private File expectedReportFile; + + private final static String TEST_ONE = "aTestMethod"; + private final static String TEST_TWO = "bTestMethod"; + private final static String TEST_THREE = "cTestMethod"; + + protected void setUp() + throws Exception + { + super.setUp(); + reportEntry = new SimpleReportEntry( this.getClass().getName(), "StatelessXMLReporterTest", + new LegacyPojoStackTraceWriter( "", "", new AssertionFailedError() ), 17 ); + stats = new TestSetStats( false, true ); + rerunStats = new TestSetStats( false, true ); + reporter.cleanTestHistoryMap(); + } + + @Override protected void tearDown() + throws Exception + { + super.tearDown(); + + if ( expectedReportFile != null ) + { + expectedReportFile.delete(); + } + } + + public void testFileNameWithoutSuffix() + { + File reportDir = new File( "." ); + String testName = "org.apache.maven.plugin.surefire.report.StatelessXMLReporterTest"; + reportEntry = new SimpleReportEntry( this.getClass().getName(), testName, 12 ); + WrappedReportEntry testSetReportEntry = + new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null ); + stats.testSucceeded( testSetReportEntry ); + reporter.testSetCompleted( testSetReportEntry, stats ); + + expectedReportFile = new File( reportDir, "TEST-" + testName + ".xml" ); + assertTrue( "Report file (" + expectedReportFile.getAbsolutePath() + ") doesn't exist", + expectedReportFile.exists() ); + } + + + public void testAllFieldsSerialized() + throws IOException + { + File reportDir = new File( "." ); + + reportEntry = new SimpleReportEntry( this.getClass().getName(), TEST_ONE, 12 ); + WrappedReportEntry testSetReportEntry = + new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null ); + expectedReportFile = new File( reportDir, "TEST-" + TEST_ONE + ".xml" ); + + stats.testSucceeded( testSetReportEntry ); + StackTraceWriter stackTraceWriter = new DeserializedStacktraceWriter( "A fud msg", "trimmed", "fail at foo" ); + Utf8RecodingDeferredFileOutputStream stdOut = new Utf8RecodingDeferredFileOutputStream( "fds" ); + String stdOutPrefix; + String stdErrPrefix; + if ( defaultCharsetSupportsSpecialChar() ) + { + stdErrPrefix = "std-\u0115rr"; + stdOutPrefix = "st]]>d-o\u00DCt"; + } + else + { + stdErrPrefix = "std-err"; + stdOutPrefix = "st]]>d-out"; + } + + byte[] stdOutBytes = (stdOutPrefix + "<null>!\u0020\u0000\u001F").getBytes(); + stdOut.write( stdOutBytes, 0, stdOutBytes.length ); + + Utf8RecodingDeferredFileOutputStream stdErr = new Utf8RecodingDeferredFileOutputStream( "fds" ); + + + byte[] stdErrBytes = (stdErrPrefix + "?&-&£\u0020\u0000\u001F").getBytes(); + stdErr.write( stdErrBytes, 0, stdErrBytes.length ); + WrappedReportEntry t2 = + new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_TWO, stackTraceWriter, 13 ), + ReportEntryType.error, 13, stdOut, stdErr ); + + stats.testSucceeded( t2 ); + StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false, 0 ); + reporter.testSetCompleted( testSetReportEntry, stats ); + + FileInputStream fileInputStream = new FileInputStream( expectedReportFile ); + + Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, "UTF-8") ); + assertEquals( "testsuite", testSuite.getName() ); + Xpp3Dom properties = testSuite.getChild( "properties" ); + assertEquals( System.getProperties().size(), properties.getChildCount() ); + Xpp3Dom child = properties.getChild( 1 ); + assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) ); + assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) ); + + Xpp3Dom[] testcase = testSuite.getChildren( "testcase" ); + Xpp3Dom tca = testcase[0]; + assertEquals( TEST_ONE, tca.getAttribute( "name" ) ); // Hopefully same order on jdk5 + assertEquals( "0.012", tca.getAttribute( "time" ) ); + assertEquals( this.getClass().getName(), tca.getAttribute( "classname" ) ); + + Xpp3Dom tcb = testcase[1]; + assertEquals( TEST_TWO, tcb.getAttribute( "name" ) ); + assertEquals( "0.013", tcb.getAttribute( "time" ) ); + assertEquals( Inner.class.getName(), tcb.getAttribute( "classname" ) ); + Xpp3Dom errorNode = tcb.getChild( "error" ); + assertNotNull( errorNode ); + assertEquals( "A fud msg", errorNode.getAttribute( "message" ) ); + assertEquals( "fail at foo", errorNode.getAttribute( "type" ) ); + assertEquals( stdOutPrefix + "<null>! &#0;&#31;", tcb.getChild( "system-out" ).getValue() ); + + + assertEquals( stdErrPrefix + "?&-&£ &#0;&#31;", tcb.getChild( "system-err" ).getValue() ); + } + + public void testOutputRerunFlakyFailure() + throws IOException + { + File reportDir = new File( "." ); + reportEntry = new SimpleReportEntry( this.getClass().getName(), TEST_ONE, 12 ); + + WrappedReportEntry testSetReportEntry = + new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null ); + expectedReportFile = new File( reportDir, "TEST-" + TEST_ONE + ".xml" ); + + stats.testSucceeded( testSetReportEntry ); + StackTraceWriter stackTraceWriterOne = new DeserializedStacktraceWriter( "A fud msg", "trimmed", + "fail at foo" ); + StackTraceWriter stackTraceWriterTwo = + new DeserializedStacktraceWriter( "A fud msg two", "trimmed two", "fail at foo two" ); + + String firstRunOut = "first run out"; + String firstRunErr = "first run err"; + String secondRunOut = "second run out"; + String secondRunErr = "second run err"; + + WrappedReportEntry testTwoFirstError = + new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_TWO, stackTraceWriterOne, 5 ), + ReportEntryType.error, 5, createStdOutput( firstRunOut ), + createStdOutput( firstRunErr ) ); + + WrappedReportEntry testTwoSecondError = + new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_TWO, stackTraceWriterTwo, 13 ), + ReportEntryType.error, 13, createStdOutput( secondRunOut ), + createStdOutput( secondRunErr ) ); + + WrappedReportEntry testThreeFirstRun = + new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_THREE, stackTraceWriterOne, 13 ), + ReportEntryType.failure, 13, createStdOutput( firstRunOut ), + createStdOutput( firstRunErr ) ); + + WrappedReportEntry testThreeSecondRun = + new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_THREE, stackTraceWriterTwo, 2 ), + ReportEntryType.success, 2, createStdOutput( secondRunOut ), + createStdOutput( secondRunErr ) ); + + stats.testSucceeded( testTwoFirstError ); + stats.testSucceeded( testThreeFirstRun ); + rerunStats.testSucceeded( testTwoSecondError ); + rerunStats.testSucceeded( testThreeSecondRun ); + + StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false, 1 ); + reporter.testSetCompleted( testSetReportEntry, stats ); + reporter.testSetCompleted( testSetReportEntry, rerunStats ); + + FileInputStream fileInputStream = new FileInputStream( expectedReportFile ); + + Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, "UTF-8" ) ); + assertEquals( "testsuite", testSuite.getName() ); + // 0.019 = 0.012 + 0.005 +0.002 + assertEquals( "0.019", testSuite.getAttribute( "time" ) ); + Xpp3Dom properties = testSuite.getChild( "properties" ); + assertEquals( System.getProperties().size(), properties.getChildCount() ); + Xpp3Dom child = properties.getChild( 1 ); + assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) ); + assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) ); + + Xpp3Dom[] testcase = testSuite.getChildren( "testcase" ); + Xpp3Dom testCaseOne = testcase[0]; + assertEquals( TEST_ONE, testCaseOne.getAttribute( "name" ) ); + assertEquals( "0.012", testCaseOne.getAttribute( "time" ) ); + assertEquals( this.getClass().getName(), testCaseOne.getAttribute( "classname" ) ); + + Xpp3Dom testCaseTwo = testcase[1]; + assertEquals( TEST_TWO, testCaseTwo.getAttribute( "name" ) ); + // Run time for a rerun failing test is the run time of the first run + assertEquals( "0.005", testCaseTwo.getAttribute( "time" ) ); + assertEquals( Inner.class.getName(), testCaseTwo.getAttribute( "classname" ) ); + Xpp3Dom errorNode = testCaseTwo.getChild( "error" ); + Xpp3Dom rerunErrorNode = testCaseTwo.getChild( "rerunError" ); + assertNotNull( errorNode ); + assertNotNull( rerunErrorNode ); + + assertEquals( "A fud msg", errorNode.getAttribute( "message" ) ); + assertEquals( "fail at foo", errorNode.getAttribute( "type" ) ); + + // Check rerun error node contains all the information + assertEquals( firstRunOut, testCaseTwo.getChild( "system-out" ).getValue() ); + assertEquals( firstRunErr, testCaseTwo.getChild( "system-err" ).getValue() ); + assertEquals( secondRunOut, rerunErrorNode.getChild( "system-out" ).getValue() ); + assertEquals( secondRunErr, rerunErrorNode.getChild( "system-err" ).getValue() ); + assertEquals( "A fud msg two", rerunErrorNode.getAttribute( "message" ) ); + assertEquals( "fail at foo two", rerunErrorNode.getAttribute( "type" ) ); + + // Check flaky failure node + Xpp3Dom testCaseThree = testcase[2]; + assertEquals( TEST_THREE, testCaseThree.getAttribute( "name" ) ); + // Run time for a flaky test is the run time of the first successful run + assertEquals( "0.002", testCaseThree.getAttribute( "time" ) ); + assertEquals( Inner.class.getName(), testCaseThree.getAttribute( "classname" ) ); + Xpp3Dom flakyFailureNode = testCaseThree.getChild( "flakyFailure" ); + assertNotNull( flakyFailureNode ); + assertEquals( firstRunOut, flakyFailureNode.getChild( "system-out" ).getValue() ); + assertEquals( firstRunErr, flakyFailureNode.getChild( "system-err" ).getValue() ); + // system-out and system-err should not be present for flaky failures + assertNull( testCaseThree.getChild( "system-out" ) ); + assertNull( testCaseThree.getChild( "system-err" ) ); + } + + private boolean defaultCharsetSupportsSpecialChar() + { + // some charsets are not able to deal with \u0115 on both ways of the conversion + return "\u0115\u00DC".equals( new String( "\u0115\u00DC".getBytes() ) ); + } + + private Utf8RecodingDeferredFileOutputStream createStdOutput( String content ) + throws IOException + { + Utf8RecodingDeferredFileOutputStream stdOut = new Utf8RecodingDeferredFileOutputStream( "fds2" ); + stdOut.write( content.getBytes(), 0, content.length() ); + return stdOut; + } + + class Inner + { + + } + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java ---------------------------------------------------------------------- diff --git a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java index ca6c469..acb17cc 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java @@ -1,7 +1,5 @@ package org.apache.maven.surefire.report; -import java.util.Collection; - import junit.framework.TestCase; /* @@ -24,85 +22,14 @@ import junit.framework.TestCase; public class RunStatisticsTest extends TestCase { - private static final String Method = "AClass#AMethod"; - - private static final String DUMMY_ERROR_SOURCE = Method + " RuntimeException"; - - private static final String DUMMY_FAILURE_SOURCE = "dummy failure source"; - - private static final String DUMMY_MESSAGE = "foo"; - - public void testAddErrorSourceWithThrowableMessage() - { - RuntimeException throwable = new RuntimeException( DUMMY_MESSAGE ); - RunStatistics statistics = createRunStatisticsAndAddErrorSourceWithThrowable( throwable ); - assertRunStatisticsHasErrorSource( statistics, DUMMY_ERROR_SOURCE + " " + DUMMY_MESSAGE ); - } - - public void testAddErrorSourceWithoutThrowable() - { - RunStatistics statistics = createRunStatisticsAndAddErrorSourceWithThrowable( null ); - assertRunStatisticsHasErrorSource( statistics, Method ); - } - - public void testAddErrorSourceWithThrowableWithoutMessage() - { - RuntimeException throwable = new RuntimeException(); - RunStatistics statistics = createRunStatisticsAndAddErrorSourceWithThrowable( throwable ); - assertRunStatisticsHasErrorSource( statistics, DUMMY_ERROR_SOURCE ); - } - - public void testAddFailureSourceWithThrowableMessage() + public void testSetRunStatistics() { - RuntimeException throwable = new RuntimeException( DUMMY_MESSAGE ); - RunStatistics statistics = createRunStatisticsAndAddFailureSourceWithThrowable( throwable ); - assertRunStatisticsHasFailureSource( statistics, DUMMY_ERROR_SOURCE + " " + DUMMY_MESSAGE ); - } - - public void testAddFailureSourceWithoutThrowable() - { - RunStatistics statistics = createRunStatisticsAndAddFailureSourceWithThrowable( null ); - assertRunStatisticsHasFailureSource( statistics, Method ); - } - - public void testAddFailureSourceWithThrowableWithoutMessage() - { - RuntimeException throwable = new RuntimeException(); - RunStatistics statistics = createRunStatisticsAndAddFailureSourceWithThrowable( throwable ); - assertRunStatisticsHasFailureSource( statistics, DUMMY_ERROR_SOURCE ); - } - - private RunStatistics createRunStatisticsAndAddErrorSourceWithThrowable( Throwable throwable ) - { - StackTraceWriter stackTraceWriter = new LegacyPojoStackTraceWriter( "AClass", "AMethod", throwable ); RunStatistics statistics = new RunStatistics(); - statistics.addErrorSource( stackTraceWriter ); - - return statistics; - } - - private RunStatistics createRunStatisticsAndAddFailureSourceWithThrowable( Throwable throwable ) - { - StackTraceWriter stackTraceWriter = new LegacyPojoStackTraceWriter( "AClass", "AMethod", throwable ); - RunStatistics statistics = new RunStatistics(); - statistics.addFailureSource( stackTraceWriter ); - - return statistics; - } - - private void assertRunStatisticsHasErrorSource( RunStatistics statistics, String expectedErrorSource ) - { - Collection errorSources = statistics.getErrorSources(); - assertNotNull( "No error sources.", errorSources ); - assertEquals( "Wrong number of error sources.", 1, errorSources.size() ); - assertEquals( "Wrong error sources.", expectedErrorSource, errorSources.iterator().next() ); - } - - private void assertRunStatisticsHasFailureSource( RunStatistics statistics, String expectedFailureSource ) - { - Collection failureSources = statistics.getFailureSources(); - assertNotNull( "No failure sources.", failureSources ); - assertEquals( "Wrong number of failure sources.", 1, failureSources.size() ); - assertEquals( "Wrong failure sources.", expectedFailureSource, failureSources.iterator().next() ); + statistics.set( 10, 5, 2, 1, 2 ); + assertEquals( 10, statistics.getCompletedCount() ); + assertEquals( 5, statistics.getErrors() ); + assertEquals( 2, statistics.getFailures() ); + assertEquals( 1, statistics.getSkipped() ); + assertEquals( 2, statistics.getFlakes() ); } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java ---------------------------------------------------------------------- diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java index 131ad74..0b8d030 100644 --- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java +++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java @@ -187,6 +187,18 @@ public class SurefirePlugin @Parameter( property = "surefire.useManifestOnlyJar", defaultValue = "true" ) private boolean useManifestOnlyJar; + /** + * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after + * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake". + * However, all the failing attempts will be recorded. + */ + @Parameter( property = "surefire.rerunFailingTestsCount", defaultValue = "0" ) + protected int rerunFailingTestsCount; + + protected int getRerunFailingTestsCount() { + return rerunFailingTestsCount; + } + protected void handleSummary( RunResult summary, Exception firstForkException ) throws MojoExecutionException, MojoFailureException { http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm ---------------------------------------------------------------------- diff --git a/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm new file mode 100644 index 0000000..42cc98c --- /dev/null +++ b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm @@ -0,0 +1,139 @@ + ------ + Rerun failing tests + ------ + Qingzhou Luo + ------ + 2014-06-27 + ------ + + ~~ Licensed to the Apache Software Foundation (ASF) under one + ~~ or more contributor license agreements. See the NOTICE file + ~~ distributed with this work for additional information + ~~ regarding copyright ownership. The ASF licenses this file + ~~ to you under the Apache License, Version 2.0 (the + ~~ "License"); you may not use this file except in compliance + ~~ with the License. You may obtain a copy of the License at + ~~ + ~~ http://www.apache.org/licenses/LICENSE-2.0 + ~~ + ~~ Unless required by applicable law or agreed to in writing, + ~~ software distributed under the License is distributed on an + ~~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~~ KIND, either express or implied. See the License for the + ~~ specific language governing permissions and limitations + ~~ under the License. + + ~~ NOTE: For help with the syntax of this file, see: + ~~ http://maven.apache.org/doxia/references/apt-format.html + +Rerun Failing Tests + +#{if}(${project.artifactId}=="maven-surefire-plugin") + During development, you may re-run failing tests because they are flaky. + To use this feature through Maven surefire, set the <<<rerunFailingTestsCount>>> property to be a value larger than 0. + Tests will be run until they pass or the number of reruns has been exhausted. + + << NOTE : This feature is supported only for JUnit 4.x. >> + + ++---+ +mvn -DrerunFailingTestsCount=2 test ++---+ + + If <<<rerunFailingTestsCount>>> is set to a value smaller than or euqal to 0, then it will be ignored. + +* Output flaky re-run information on the screen + + When <<<rerunFailingTestsCount>>> is set to a value larger than 0 and the test fails, + then it will be re-run and each run information will also be output. Each run with its number and trimmed stack trace + will be output. + + If the test passes in its first run, then the output on the screen will be identical to the case where + <<<rerunFailingTestsCount>>> is not used. + + It the test fails in the first run, then there are two possible cases: + + 1) The test passes in one of its re-runs: the last run will be marked as PASS + + For example, a test passed in its second run will output on the screen: + ++---+ + Run 1: ... + Run 2: PASS ++---+ + + Then this test will be counted as a flaky test. The build will be successful, but in the end of the summary of all + tests run, the number of flaky tests will be output on the screen, for example: + ++---+ + Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Flakes: 1 ++---+ + + 2) The test fails in all of the re-runs: + + For example, a test failied in all the three runs, then all the runs will be output on the screen: + ++---+ + Run 1: ... + Run 2: ... + Run 3: ... ++---+ + + Then this build will be marked as failure. The type of the failure (failed test or test in error) of this test + depends on its first failure. + +* Output flaky re-run information in test report xml + + When <<<rerunFailingTestsCount>>> is set to a value larger than 0, the output xml of test report will also be extended + to include information of each re-run. + + If the test passes in its first run, then the output xml report will be identical to the case where + <<<rerunFailingTestsCount>>> is not used. + + It the test fails in the first run, then there are also two possible cases. + + 1) The test passes in one of its re-runs: + + <<<flakyFailure>>> and <<<flakyError>>> elements will be used in the generated xml report to include information + of each failing re-runs. <<<system-out>>> and <<<system-err>>> will also be used inside each <<<flakyFailure>>> + or <<<flakyError>>> to include information of System.out and System.err output. The original <<<system-out>>> + and <<<system-err>>> elements will be retained on the top level under <<<testcase>>> for the last successful run. + + For example: + ++---+ +<testcase name=".." classname=".." time="0.1"> + <flakyFailure message="" type=""> flaky failure stack trace + <system-out> flaky failure </system-out> + </flakyFailure> + <system-out> success </system-out> +</testcase> ++---+ + + In the xml report, the running time of a flaky test will be the running time of the <<last successful run>>. + + 2) The test fails in all of the re-runs: + + <<<failure>>> and <<<error>>> elements will still be used in the generated xml report to include information + for the first failing run, the same as without using <<<rerunFailingTests>>>. <<<rerunFailure>>> and <<<rerunError>>> + elements will be used in the generated xml report to include information of each <<subsequent>> failing re-runs. + <<<system-out>>> and <<<system-err>>> will also be used inside each <<<flakyFailure>>> or <<<flakyError>>> to include + information of System.out and System.err output. The original <<<system-out>>> and <<<system-err>>> elements will be + retained on the top level under <<<testcase>>> for the first original failing run. + + For example: + ++---+ +<testcase name=".." classname=".." time="0.1"> + <failure message="" type=""> first failure stack trace </failure> + <system-out> first failure </system-out> + <rerunFailure message="" type=""> rerun failure stack trace + <system-out> rerun failure </system-out> + </rerunFailure> +</testcase> ++---+ + + In the xml report, the running time of a failing test with re-runs will be the running time of the + <<first failing run>>. + +#{end} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-plugin/src/site/site.xml ---------------------------------------------------------------------- diff --git a/maven-surefire-plugin/src/site/site.xml b/maven-surefire-plugin/src/site/site.xml index 3724c87..d98d763 100644 --- a/maven-surefire-plugin/src/site/site.xml +++ b/maven-surefire-plugin/src/site/site.xml @@ -43,6 +43,7 @@ <item name="Skipping Tests" href="examples/skipping-test.html"/> <item name="Inclusions and Exclusions of Tests" href="examples/inclusion-exclusion.html"/> <item name="Running a Single Test" href="examples/single-test.html"/> + <item name="Re-run Failing Tests" href="examples/rerun-failing-tests.html"/> <item name="Class Loading and Forking" href="examples/class-loading.html"/> <item name="Debugging Tests" href="examples/debugging.html"/> <item name="Using System Properties" href="examples/system-properties.html"/> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java ---------------------------------------------------------------------- diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java b/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java index 54a6b5b..2e4bee4 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java @@ -152,11 +152,11 @@ public class SurefireReflector { return null; } - Class[] arguments = { List.class, File.class, String.class, String.class }; + Class[] arguments = { List.class, File.class, String.class, String.class, int.class }; Constructor constructor = ReflectionUtils.getConstructor( this.testRequest, arguments ); - return ReflectionUtils.newInstance( constructor, new Object[]{ suiteDefinition.getSuiteXmlFiles(), + return ReflectionUtils.newInstance(constructor, new Object[]{ suiteDefinition.getSuiteXmlFiles(), suiteDefinition.getTestSourceDirectory(), suiteDefinition.getRequestedTest(), - suiteDefinition.getRequestedTestMethod() } ); + suiteDefinition.getRequestedTestMethod(), suiteDefinition.getRerunFailingTestsCount() }); } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java ---------------------------------------------------------------------- diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java b/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java index 595717b..83d74f6 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java @@ -53,6 +53,8 @@ public class RunResult private final int skipped; + private final int flakes; + private final String failure; private final boolean timeout; @@ -85,14 +87,26 @@ public class RunResult this( completedCount, errors, failures, skipped, null, false ); } + public RunResult( int completedCount, int errors, int failures, int skipped, int flakes ) + { + this( completedCount, errors, failures, skipped, flakes, null, false ); + } + public RunResult( int completedCount, int errors, int failures, int skipped, String failure, boolean timeout ) { + this( completedCount, errors, failures, skipped, 0, failure, timeout ); + } + + public RunResult( int completedCount, int errors, int failures, int skipped, int flakes, String failure, + boolean timeout ) + { this.completedCount = completedCount; this.errors = errors; this.failures = failures; this.skipped = skipped; this.failure = failure; this.timeout = timeout; + this.flakes = flakes; } private static String getStackTrace( Exception e ) @@ -117,6 +131,11 @@ public class RunResult return errors; } + public int getFlakes() + { + return flakes; + } + public int getFailures() { return failures; @@ -176,7 +195,8 @@ public class RunResult int fail = getFailures() + other.getFailures(); int ign = getSkipped() + other.getSkipped(); int err = getErrors() + other.getErrors(); - return new RunResult( completed, err, fail, ign, failureMessage, timeout ); + int flakes = getFlakes() + other.getFlakes(); + return new RunResult( completed, err, fail, ign, flakes, failureMessage, timeout ); } public static RunResult noTestsRun() http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java ---------------------------------------------------------------------- diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java b/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java index 190b71b..a237bbe 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java @@ -36,6 +36,8 @@ public class TestRequest private final String requestedTest; + private final int rerunFailingTestsCount; + /** * @since 2.7.3 */ @@ -51,10 +53,17 @@ public class TestRequest */ public TestRequest( List suiteXmlFiles, File testSourceDirectory, String requestedTest, String requestedTestMethod ) { + this( createFiles( suiteXmlFiles ), testSourceDirectory, requestedTest, requestedTestMethod, 0 ); + } + + public TestRequest( List suiteXmlFiles, File testSourceDirectory, String requestedTest, String requestedTestMethod, + int rerunFailingTestsCount ) + { this.suiteXmlFiles = createFiles( suiteXmlFiles ); this.testSourceDirectory = testSourceDirectory; this.requestedTest = requestedTest; this.requestedTestMethod = requestedTestMethod; + this.rerunFailingTestsCount = rerunFailingTestsCount; } /** @@ -98,6 +107,16 @@ public class TestRequest return requestedTestMethod; } + /** + * How many times to rerun failing tests, issued with -Dsurefire.rerunFailingTestsCount from the command line. + * + * @return The int parameter to indicate how many times to rerun failing tests + */ + public int getRerunFailingTestsCount() + { + return this.rerunFailingTestsCount; + } + private static List<File> createFiles( List suiteXmlFiles ) { if ( suiteXmlFiles != null ) http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java ---------------------------------------------------------------------- diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java index 607c332..2c42c19 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java @@ -138,4 +138,22 @@ public class TestsToRun implements Iterable<Class> } return result.toArray( new Class[result.size()] ); } + + /** + * Get test class which matches className + * + * @param className string used to find the test class + * @return Class object with the matching name, null if could not find a class with the matching name + */ + public Class getClassByName( String className ) + { + for ( Class clazz : this ) + { + if ( clazz.getName().equals( className ) ) + { + return clazz; + } + } + return null; + } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java ---------------------------------------------------------------------- diff --git a/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java b/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java index ec99c64..007a26c 100644 --- a/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java +++ b/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java @@ -58,6 +58,7 @@ public class RunResultTest assertEquals( 3, simple.getErrors() ); assertEquals( 7, simple.getFailures() ); assertEquals( 4, simple.getSkipped() ); + assertEquals( 2, simple.getFlakes() ); } @@ -116,8 +117,8 @@ public class RunResultTest private RunResult getSimpleAggregate() { - RunResult resultOne = new RunResult( 10, 1, 3, 2 ); - RunResult resultTwo = new RunResult( 10, 2, 4, 2 ); + RunResult resultOne = new RunResult( 10, 1, 3, 2, 1 ); + RunResult resultTwo = new RunResult( 10, 2, 4, 2, 1 ); return resultOne.aggregate( resultTwo ); } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java ---------------------------------------------------------------------- diff --git a/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java b/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java index c7029c0..d569c00 100644 --- a/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java +++ b/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java @@ -65,6 +65,14 @@ public class TestsToRunTest assertEquals( 2, locatedClasses.length ); } + public void testGetClassByName() + { + TestsToRun testsToRun = new TestsToRun( Arrays.asList( new Class[]{ T1.class, T2.class } ) ); + assertEquals( T1.class, testsToRun.getClassByName( "org.apache.maven.surefire.util.TestsToRunTest$T1" ) ); + assertEquals( T2.class, testsToRun.getClassByName( "org.apache.maven.surefire.util.TestsToRunTest$T2" ) ); + assertEquals( null, testsToRun.getClassByName( "org.apache.maven.surefire.util.TestsToRunTest$T3" ) ); + } + class T1 { http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java ---------------------------------------------------------------------- diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java index d53bfda..2f637f0 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java @@ -47,4 +47,5 @@ public interface BooterConstants String PROVIDER_CONFIGURATION = "providerConfiguration"; String FORKTESTSET = "forkTestSet"; String FORKTESTSET_PREFER_TESTS_FROM_IN_STREAM = "preferTestsFromInStream"; + String RERUN_FAILING_TESTS_COUNT = "rerunFailingTestsCount"; } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java ---------------------------------------------------------------------- diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java index a9028e0..fce7c42 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java @@ -78,6 +78,8 @@ public class BooterDeserializer final String runOrder = properties.getProperty( RUN_ORDER ); final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE ); + final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT ); + DirectoryScannerParameters dirScannerParams = new DirectoryScannerParameters( testClassesDirectory, includesList, excludesList, specificTestsList, properties.getBooleanObjectProperty( FAILIFNOTESTS ), runOrder ); @@ -86,7 +88,8 @@ public class BooterDeserializer TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion, testArtifactClassifier ); TestRequest testSuiteDefinition = - new TestRequest( testSuiteXmlFiles, sourceDirectory, requestedTest, requestedTestMethod ); + new TestRequest( testSuiteXmlFiles, sourceDirectory, requestedTest, requestedTestMethod, + rerunFailingTestsCount ); ReporterConfiguration reporterConfiguration = new ReporterConfiguration( reportsDirectory, properties.getBooleanObjectProperty( ISTRIMSTACKTRACE ) ); http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java ---------------------------------------------------------------------- diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java index 8a40c63..7a09352 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java @@ -76,6 +76,11 @@ public class PropertiesWrapper return Boolean.valueOf( properties.getProperty( propertyName ) ); } + public int getIntProperty( String propertyName ) + { + return Integer.parseInt( properties.getProperty( propertyName ) ); + } + public File getFileProperty( String key ) { final String property = getProperty( key ); http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java new file mode 100644 index 0000000..47fb71a --- /dev/null +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java @@ -0,0 +1,278 @@ +package org.apache.maven.surefire.its; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +/** + * JUnit4 RunListener Integration Test. + * + * @author <a href="mailto:qingzhou...@google.com";>Qingzhou Luo</a> + */ +public class JUnit4RerunFailingTestsIT + extends SurefireJUnit4IntegrationTestCase +{ + private SurefireLauncher unpack() + { + return unpack( "/junit4-rerun-failing-tests" ); + } + + @Test + public void testRerunFailingErrorTestsWithOneRetry() + throws Exception + { + OutputValidator outputValidator = + unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, + 0 ); + verifyFailuresOneRetryAllClasses( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( + "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 ); + verifyFailuresOneRetryAllClasses( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal( + "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 ); + verifyFailuresOneRetryAllClasses( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal( + "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 ); + verifyFailuresOneRetryAllClasses( outputValidator ); + } + + @Test + public void testRerunFailingErrorTestsTwoRetry() + throws Exception + { + // Four flakes, both tests have been re-run twice + OutputValidator outputValidator = + unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 ); + + verifyFailuresTwoRetryAllClasses( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest() + .assertTestSuiteResults( 5, 0, 0, 0, 4 ); + + verifyFailuresTwoRetryAllClasses( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal( + "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 ); + + verifyFailuresTwoRetryAllClasses( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal( + "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 ); + + verifyFailuresTwoRetryAllClasses( outputValidator ); + } + + @Test + public void testRerunFailingErrorTestsFalse() + throws Exception + { + OutputValidator outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( + "4.7" ).maven().withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 ); + + verifyFailuresNoRetryAllClasses( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-DforkCount=3" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 ); + + verifyFailuresNoRetryAllClasses( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dparallel=methods" ).addGoal( + "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 ); + + verifyFailuresNoRetryAllClasses( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dparallel=classes" ).addGoal( + "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 ); + + verifyFailuresNoRetryAllClasses( outputValidator ); + } + + @Test + public void testRerunOneTestClass() + throws Exception + { + OutputValidator outputValidator = + unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( + "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 ); + + verifyFailuresOneRetryOneClass( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal( + "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 ); + + verifyFailuresOneRetryOneClass( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal( + "-DuseUnlimitedThreads=true" ).addGoal( + "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 ); + + verifyFailuresOneRetryOneClass( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal( + "-DuseUnlimitedThreads=true" ).addGoal( + "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 ); + + verifyFailuresOneRetryOneClass( outputValidator ); + } + + @Test + public void testRerunOneTestMethod() + throws Exception + { + OutputValidator outputValidator = + unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( + "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, + 0, 0 ); + + verifyFailuresOneRetryOneMethod( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal( + "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0, + 0 ); + + verifyFailuresOneRetryOneMethod( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal( + "-DuseUnlimitedThreads=true" ).addGoal( + "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0, + 0 ); + + verifyFailuresOneRetryOneMethod( outputValidator ); + + outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal( + "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal( + "-DuseUnlimitedThreads=true" ).addGoal( + "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0, + 0 ); + + verifyFailuresOneRetryOneMethod( outputValidator ); + } + + private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator ) + { + verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 ); + } + + private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator ) + { + verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 ); + } + + private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator ) + { + verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 ); + } + + private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator ) + { + verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 ); + } + + private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator ) + { + verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 ); + } + + private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors, + int flakes ) + { + outputValidator.verifyTextInLog( "Failed tests" ); + outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" ); + outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" ); + + outputValidator.verifyTextInLog( "Tests in error" ); + outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" ); + outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" ); + + verifyStatistics( outputValidator, run, failures, errors, flakes ); + } + + private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors, + int flakes ) + { + outputValidator.verifyTextInLog( "Failed tests" ); + outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" ); + outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" ); + + verifyStatistics( outputValidator, run, failures, errors, flakes ); + } + + private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures, int errors, + int flakes ) + { + outputValidator.verifyTextInLog( "Flaked tests" ); + outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" ); + outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" ); + outputValidator.verifyTextInLog( "Run 3: PASS" ); + + outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" ); + outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" ); + + verifyStatistics( outputValidator, run, failures, errors, flakes ); + } + + private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes ) + { + outputValidator.verifyTextInLog( "Failed tests" ); + outputValidator.verifyTextInLog( "testFailingTestOne(junit4.FlakyFirstTimeTest)" ); + outputValidator.verifyTextInLog( "ERROR" ); + outputValidator.verifyTextInLog( "testErrorTestOne(junit4.FlakyFirstTimeTest)" ); + + verifyStatistics( outputValidator, run, failures, errors, flakes ); + } + + private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int flakes ) + { + if ( flakes > 0 ) + { + outputValidator.verifyTextInLog( + "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0, Flakes: " + + flakes ); + } + else + { + outputValidator.verifyTextInLog( + "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0" ); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java index 7c8740a..c7afb92 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java @@ -40,6 +40,12 @@ public class HelperAssertions assertTestSuiteResults( total, errors, failures, skipped, suite ); } + public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes, File testDir ) + { + IntegrationTestSuiteResults suite = parseTestResults( new File[]{ testDir } ); + assertTestSuiteResults( total, errors, failures, skipped, flakes, suite ); + } + /** * assert that the reports in the specified testDir have the right summary statistics */ @@ -59,6 +65,13 @@ public class HelperAssertions Assert.assertEquals( "wrong number of skipped", skipped, actualSuite.getSkipped() ); } + public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes, + IntegrationTestSuiteResults actualSuite ) + { + assertTestSuiteResults(total, errors, failures, skipped, actualSuite); + Assert.assertEquals( "wrong number of flaky tests", flakes, actualSuite.getFlakes() ); + } + public static IntegrationTestSuiteResults parseTestResults( File[] testDirs ) { List<ReportTestSuite> reports = extractReports( testDirs ); @@ -77,15 +90,16 @@ public class HelperAssertions public static IntegrationTestSuiteResults parseReportList( List<ReportTestSuite> reports ) { Assert.assertTrue( "No reports!", reports.size() > 0 ); - int total = 0, errors = 0, failures = 0, skipped = 0; + int total = 0, errors = 0, failures = 0, skipped = 0, flakes = 0; for ( ReportTestSuite report : reports ) { total += report.getNumberOfTests(); errors += report.getNumberOfErrors(); failures += report.getNumberOfFailures(); skipped += report.getNumberOfSkipped(); + flakes += report.getNumberOfFlakes(); } - return new IntegrationTestSuiteResults( total, errors, failures, skipped ); + return new IntegrationTestSuiteResults( total, errors, failures, skipped, flakes ); } public static List<ReportTestSuite> extractReports( File[] testDirs ) http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java index 8645027..f147281 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java @@ -22,7 +22,7 @@ package org.apache.maven.surefire.its.fixture; public class IntegrationTestSuiteResults { - private int total, errors, failures, skipped; + private int total, errors, failures, skipped, flakes; public IntegrationTestSuiteResults( int total, int errors, int failures, int skipped ) { @@ -32,6 +32,12 @@ public class IntegrationTestSuiteResults this.skipped = skipped; } + public IntegrationTestSuiteResults( int total, int errors, int failures, int skipped, int flakes ) + { + this(total, errors, failures, skipped); + this.flakes = flakes; + } + public int getTotal() { return total; @@ -72,4 +78,14 @@ public class IntegrationTestSuiteResults this.skipped = skipped; } + public int getFlakes() + { + return flakes; + } + + public void setFlakes( int flakes ) + { + this.flakes = flakes; + } + } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java index 2671879..14dfc47 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java @@ -125,6 +125,12 @@ public class OutputValidator return this; } + public OutputValidator assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes ) + { + HelperAssertions.assertTestSuiteResults( total, errors, failures, skipped, flakes, baseDir ); + return this; + } + public OutputValidator assertIntegrationTestSuiteResults( int total, int errors, int failures, int skipped ) { HelperAssertions.assertIntegrationTestSuiteResults( total, errors, failures, skipped, baseDir ); http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml new file mode 100644 index 0000000..7482811 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.maven.plugins.surefire</groupId> + <artifactId>junit4-rerun-failing-tests</artifactId> + <version>1.0-SNAPSHOT</version> + <name>Test for rerun failing tests in JUnit 4</name> + + + <properties> + <junitVersion>4.4</junitVersion> + </properties> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${junitVersion}</version> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <source>1.5</source> + <target>1.5</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire.version}</version> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java new file mode 100644 index 0000000..264462c --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java @@ -0,0 +1,62 @@ +package junit4; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Assert; +import org.junit.Test; + + +public class FlakyFirstTimeTest +{ + private static int failingCount = 0; + + private static int errorCount = 0; + + + @Test + public void testFailingTestOne() + { + System.out.println( "Failing test" ); + // This test will fail with only one retry, but will pass with two + if ( failingCount < 2 ) + { + failingCount++; + Assert.fail( "Failing test" ); + } + } + + @Test + public void testErrorTestOne() throws Exception + { + System.out.println( "Error test" ); + // This test will error out with only one retry, but will pass with two + if ( errorCount < 2 ) + { + errorCount++; + throw new IllegalArgumentException("..."); + } + } + + @Test + public void testPassingTest() throws Exception + { + System.out.println( "Passing test" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java new file mode 100644 index 0000000..7cb0b57 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java @@ -0,0 +1,39 @@ +package junit4; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Assert; +import org.junit.Test; + + +public class PassingTest +{ + @Test + public void testPassingTestOne() + { + System.out.println( "Passing test one" ); + } + + @Test + public void testPassingTestTwo() throws Exception + { + System.out.println( "Passing test two" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java new file mode 100644 index 0000000..52ce708 --- /dev/null +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java @@ -0,0 +1,109 @@ +package org.apache.maven.surefire.common.junit4; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.util.TestsToRun; +import org.apache.maven.surefire.util.internal.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.runner.notification.Failure; + +import static org.apache.maven.surefire.common.junit4.JUnit4RunListener.isFailureInsideJUnitItself; + +/** + * + * Utility method used among all JUnit4 providers + * + * @author Qingzhou Luo + * + */ +public class JUnit4ProviderUtil +{ + /** + * Organize all the failures in previous run into a map between test classes and corresponding failing test methods + * + * @param allFailures all the failures in previous run + * @param testsToRun all the test classes + * @return a map between failing test classes and their corresponding failing test methods + */ + public static Map<Class<?>, Set<String>> generateFailingTests( List<Failure> allFailures, TestsToRun testsToRun ) + { + Map<Class<?>, Set<String>> testClassMethods = new HashMap<Class<?>, Set<String>>(); + + for ( Failure failure : allFailures ) + { + if ( !isFailureInsideJUnitItself( failure ) ) + { + // failure.getTestHeader() is in the format: method(class) + String[] testMethodClass = StringUtils.split( failure.getTestHeader(), "(" ); + String testMethod = testMethodClass[0]; + String testClass = StringUtils.split( testMethodClass[1], ")" )[0]; + Class testClassObj = testsToRun.getClassByName( testClass ); + + if ( testClassObj == null ) + { + continue; + } + + Set<String> failingMethods = testClassMethods.get( testClassObj ); + if ( failingMethods == null ) + { + failingMethods = new HashSet<String>(); + failingMethods.add( testMethod ); + testClassMethods.put( testClassObj, failingMethods ); + } + else + { + failingMethods.add( testMethod ); + } + } + } + return testClassMethods; + } + + /** + * Get the name of all test methods from a list of Failures + * + * @param allFailures the list of failures for a given test class + * @return the list of test method names + */ + public static Set<String> generateFailingTests( List<Failure> allFailures ) + { + Set<String> failingMethods = new HashSet<String>(); + + for ( Failure failure : allFailures ) + { + if ( !isFailureInsideJUnitItself( failure ) ) + { + // failure.getTestHeader() is in the format: method(class) + String testMethod = StringUtils.split( failure.getTestHeader(), "(" )[0]; + failingMethods.add( testMethod ); + } + } + return failingMethods; + } + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java index c4b5b57..f21ab04 100644 --- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java @@ -209,7 +209,7 @@ public class JUnit4RunListener } } - private static boolean isFailureInsideJUnitItself( Failure failure ) + public static boolean isFailureInsideJUnitItself( Failure failure ) { return failure.getDescription().getDisplayName().equals( "Test mechanism" ); } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java new file mode 100644 index 0000000..5cca0e8 --- /dev/null +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java @@ -0,0 +1,35 @@ +package org.apache.maven.surefire.common.junit4; + +import org.junit.runner.notification.Failure; + +import java.util.ArrayList; +import java.util.List; + +/** + * Test listener to record all the failures during one run + * + * @author Qingzhou Luo + */ +public class JUnitTestFailureListener + extends org.junit.runner.notification.RunListener +{ + + List<Failure> allFailures = new ArrayList<Failure>(); + + @Override + public void testFailure( Failure failure ) + throws java.lang.Exception + { + allFailures.add( failure ); + } + + public List<Failure> getAllFailures() + { + return allFailures; + } + + public void reset() + { + allFailures.clear(); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java new file mode 100644 index 0000000..d21393a --- /dev/null +++ b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java @@ -0,0 +1,85 @@ +package org.apache.maven.surefire.common.junit4; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import junit.framework.TestCase; +import org.apache.maven.surefire.util.TestsToRun; +import org.junit.runner.Description; +import org.junit.runner.notification.Failure; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTests; + +/** + * @author Qingzhou Luo + */ +public class JUnit4ProviderUtilTest + extends TestCase +{ + public void testGenerateFailingTests() + { + TestsToRun testsToRun = new TestsToRun( Arrays.asList( new Class[]{ T1.class, T2.class } ) ); + List<Failure> failures = new ArrayList<Failure>( ); + + Description test1Description = Description.createTestDescription( T1.class, "testOne" ); + Description test2Description = Description.createTestDescription( T1.class, "testTwo" ); + Description test3Description = Description.createTestDescription( T2.class, "testThree" ); + Description test4Description = Description.createTestDescription( T2.class, "testFour" ); + Description test5Description = Description.createSuiteDescription( "Test mechanism" ); + + failures.add( new Failure( test1Description, new AssertionError() ) ); + failures.add( new Failure( test2Description, new AssertionError() ) ); + failures.add( new Failure( test3Description, new RuntimeException() ) ); + failures.add( new Failure( test4Description, new AssertionError() ) ); + failures.add( new Failure( test5Description, new RuntimeException() ) ); + + Map<Class<?>, Set<String>> result = generateFailingTests( failures, testsToRun ); + + assertEquals( 2, result.size() ); + Set<String> resultForT1 = result.get( T1.class ); + Set<String> resultForT2 = result.get( T2.class ); + + Set<String> expectedResultForT1 = new HashSet<String>(); + expectedResultForT1.add( "testOne" ); + expectedResultForT1.add( "testTwo" ); + Set<String> expectedResultForT2 = new HashSet<String>(); + expectedResultForT2.add( "testThree" ); + expectedResultForT2.add( "testFour" ); + + assertEquals( expectedResultForT1, resultForT1 ); + assertEquals( expectedResultForT2, resultForT2 ); + } + + class T1 + { + + } + + class T2 + { + + } +} \ No newline at end of file
