[SUREFIRE-1144] Time for testsuite on commandline does not suit with the time value given in the report file
Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/feda0885 Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/feda0885 Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/feda0885 Branch: refs/heads/master Commit: feda08851f48a6467bd681449eaea1635051bc3b Parents: ce880f1 Author: Tibor17 <tibo...@lycos.com> Authored: Mon Sep 28 02:03:53 2015 +0200 Committer: Tibor17 <tibo...@lycos.com> Committed: Mon Sep 28 02:03:53 2015 +0200 ---------------------------------------------------------------------- .../surefire/report/StatelessXmlReporter.java | 57 +------------- .../report/StatelessXmlReporterTest.java | 3 +- .../surefire/its/XmlReporterRunTimeIT.java | 32 ++++---- .../its/jiras/Surefire1144XmlRunTimeIT.java | 58 ++++++++++++++ .../test/resources/runorder-parallel/pom.xml | 10 +-- .../src/test/java/runorder/parallel/Test1.java | 2 +- .../resources/surefire-1144-xml-runtime/pom.xml | 63 +++++++++++++++ .../src/test/java/surefire1144/Test1.java | 82 ++++++++++++++++++++ 8 files changed, 229 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/feda0885/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java ---------------------------------------------------------------------- diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java index 3a8dc2c..1489f20 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java @@ -129,7 +129,7 @@ public class StatelessXmlReporter ppw.setEncoding( ENCODING ); createTestSuiteElement( ppw, testSetReportEntry, testSetStats, reportNameSuffix, - testSetStats.elapsedTimeAsString( getRunTimeForAllTests( methodRunHistoryMap ) ) ); + testSetReportEntry.elapsedTimeAsString() ); showProperties( ppw ); @@ -268,61 +268,6 @@ public class StatelessXmlReporter return DefaultReporterFactory.getTestResultType( testResultTypeList, rerunFailingTestsCount ); } - /** - * Get run time for the entire test suite (test class) - * For a successful/failed/error test, the run time is the first run - * For a flaky test, the run time is the first successful run's time - * The run time for the entire test class is the sum of all its test methods - * - * - * @param methodRunHistoryMap the input map between test method name and the list of all its runs - * in a given test class - * @return the run time for the entire test class - */ - private int getRunTimeForAllTests( Map<String, List<WrappedReportEntry>> methodRunHistoryMap ) - { - int totalTimeForSuite = 0; - for ( Map.Entry<String, List<WrappedReportEntry>> entry : methodRunHistoryMap.entrySet() ) - { - List<WrappedReportEntry> methodEntryList = entry.getValue(); - if ( methodEntryList == null ) - { - throw new IllegalStateException( "Get null test method run history" ); - } - - if ( !methodEntryList.isEmpty() ) - { - TestResultType resultType = getTestResultType( methodEntryList ); - - switch ( resultType ) - { - case success: - case error: - case failure: - // Get the first run's time for failure/error/success runs - totalTimeForSuite = totalTimeForSuite + methodEntryList.get( 0 ).getElapsed(); - break; - case flake: - // Get the first successful run's time for flaky runs - for ( WrappedReportEntry singleRunEntry : methodEntryList ) - { - if ( singleRunEntry.getReportEntryType() == ReportEntryType.SUCCESS ) - { - totalTimeForSuite = totalTimeForSuite + singleRunEntry.getElapsed(); - break; - } - } - break; - case skipped: - break; - default: - throw new IllegalStateException( "Get unknown test result type" ); - } - } - } - return totalTimeForSuite; - } - private Map<String, List<WrappedReportEntry>> getAddMethodRunHistoryMap( String testClassName ) { Map<String, List<WrappedReportEntry>> methodRunHistoryMap = testClassMethodRunHistoryMap.get( testClassName ); http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/feda0885/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java ---------------------------------------------------------------------- diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java index e6e3d1a..09fa1d1 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java @@ -230,8 +230,7 @@ public class StatelessXmlReporterTest Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, "UTF-8" ) ); assertEquals( "testsuite", testSuite.getName() ); - // 0.019 = 0.012 + 0.005 +0.002 - assertEquals( "0.019", testSuite.getAttribute( "time" ) ); + assertEquals( "0.012", testSuite.getAttribute( "time" ) ); Xpp3Dom properties = testSuite.getChild( "properties" ); assertEquals( System.getProperties().size(), properties.getChildCount() ); Xpp3Dom child = properties.getChild( 1 ); http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/feda0885/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/XmlReporterRunTimeIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/XmlReporterRunTimeIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/XmlReporterRunTimeIT.java index 56cfc43..d96cf61 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/XmlReporterRunTimeIT.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/XmlReporterRunTimeIT.java @@ -20,15 +20,15 @@ package org.apache.maven.surefire.its; */ import org.apache.maven.plugins.surefire.report.ReportTestSuite; -import org.apache.maven.surefire.its.fixture.HelperAssertions; import org.apache.maven.surefire.its.fixture.OutputValidator; import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; import org.junit.Test; -import java.io.File; -import java.util.List; - -import static org.junit.Assert.assertTrue; +import static org.apache.maven.surefire.its.fixture.HelperAssertions.extractReports; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; /** * Test reported runtime @@ -42,26 +42,32 @@ public class XmlReporterRunTimeIT public void testForkModeAlways() throws Exception { - OutputValidator outputValidator = unpack( "/runorder-parallel" ).parallelMethods().executeTest(); + // just generate .surefire-<hash> in order to apply runOrder + unpack( "/runorder-parallel" ) + .executeTest() + .verifyErrorFree( 9 ); + + // now assert test results match expected values + OutputValidator outputValidator = unpack( "/runorder-parallel" ) + .executeTest() + .verifyErrorFree( 9 ); - List<ReportTestSuite> reports = HelperAssertions.extractReports( new File[]{ outputValidator.getBaseDir() } ); - for ( ReportTestSuite report : reports ) + for ( ReportTestSuite report : extractReports( outputValidator.getBaseDir() ) ) { if ( "runorder.parallel.Test1".equals( report.getFullClassName() ) ) { - assertTrue( "runorder.parallel.Test1 report.getTimeElapsed found:" + report.getTimeElapsed(), - report.getTimeElapsed() >= 1.2f ); + assertThat( "runorder.parallel.Test1 report.getTimeElapsed found:" + report.getTimeElapsed(), + report.getTimeElapsed(), allOf( greaterThanOrEqualTo( 0.6f ), lessThan( 0.7f ) ) ); } else if ( "runorder.parallel.Test2".equals( report.getFullClassName() ) ) { - assertTrue( "runorder.parallel.Test2 report.getTimeElapsed found:" + report.getTimeElapsed(), - report.getTimeElapsed() >= 0.9f ); + assertThat( "runorder.parallel.Test2 report.getTimeElapsed found:" + report.getTimeElapsed(), + report.getTimeElapsed(), allOf( greaterThanOrEqualTo( 0.5f ), lessThan( 0.6f ) ) ); } else { System.out.println( "report = " + report ); } } - } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/feda0885/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1144XmlRunTimeIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1144XmlRunTimeIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1144XmlRunTimeIT.java new file mode 100644 index 0000000..d81d7c2 --- /dev/null +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1144XmlRunTimeIT.java @@ -0,0 +1,58 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.plugins.surefire.report.ReportTestSuite; +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +import java.util.List; + +import static org.apache.maven.surefire.its.fixture.HelperAssertions.extractReports; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; + +/** + * Test that runtime reported on console matches runtime in XML + * + * @author <a href="mailto:elous...@illinois.edu";>Lamyaa Eloussi</a> + */ +public class Surefire1144XmlRunTimeIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void testXmlRunTime() + throws Exception + { + OutputValidator outputValidator = unpack( "/surefire-1144-xml-runtime" ).forkOnce().executeTest(); + + List<ReportTestSuite> reports = extractReports( outputValidator.getBaseDir() ); + assertThat( reports, hasSize( 1 ) ); + + ReportTestSuite report = reports.get( 0 ); + float xmlTime = report.getTimeElapsed(); + + assertThat( xmlTime, is(greaterThanOrEqualTo( 1.6f ) ) ); //include beforeClass and afterClass + outputValidator.verifyTextInLog( Float.toString( xmlTime ) ); //same time in console + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/feda0885/surefire-integration-tests/src/test/resources/runorder-parallel/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/runorder-parallel/pom.xml b/surefire-integration-tests/src/test/resources/runorder-parallel/pom.xml index f876e2b..f5c967d 100644 --- a/surefire-integration-tests/src/test/resources/runorder-parallel/pom.xml +++ b/surefire-integration-tests/src/test/resources/runorder-parallel/pom.xml @@ -15,10 +15,6 @@ </dependency> </dependencies> - <properties> - <runOrder>balanced</runOrder> - </properties> - <build> <plugins> <plugin> @@ -34,8 +30,10 @@ <artifactId>maven-surefire-plugin</artifactId> <version>${surefire.version}</version> <configuration> - <runOrder>${runOrder}</runOrder> - <threadCount>2</threadCount> + <runOrder>balanced</runOrder> + <parallel>classesAndMethods</parallel> + <threadCountClasses>2</threadCountClasses> + <threadCountMethods>6</threadCountMethods> <perCoreThreadCount>false</perCoreThreadCount> <includes> <include>**/Test*.java</include> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/feda0885/surefire-integration-tests/src/test/resources/runorder-parallel/src/test/java/runorder/parallel/Test1.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/runorder-parallel/src/test/java/runorder/parallel/Test1.java b/surefire-integration-tests/src/test/resources/runorder-parallel/src/test/java/runorder/parallel/Test1.java index 8fbddd6..5429fda 100755 --- a/surefire-integration-tests/src/test/resources/runorder-parallel/src/test/java/runorder/parallel/Test1.java +++ b/surefire-integration-tests/src/test/resources/runorder-parallel/src/test/java/runorder/parallel/Test1.java @@ -38,7 +38,7 @@ public class Test1 { do { - Thread.sleep( ms ); + Thread.sleep( 1L ); } while ( System.currentTimeMillis() < target ); } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/feda0885/surefire-integration-tests/src/test/resources/surefire-1144-xml-runtime/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1144-xml-runtime/pom.xml b/surefire-integration-tests/src/test/resources/surefire-1144-xml-runtime/pom.xml new file mode 100644 index 0000000..799f87e --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1144-xml-runtime/pom.xml @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>it-parent</artifactId> + <version>1.0</version> + </parent> + + <groupId>org.apache.maven.plugins.surefire</groupId> + <artifactId>surefire1144-xml-runtime</artifactId> + <version>1.0</version> + + <url>http://maven.apache.org</url> + + <contributors> + <contributor> + <name>lamyaa (Lamyaa Eloussi)</name> + <email>elous...@illinois.edu</email> + </contributor> + </contributors> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.0</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <source>1.5</source> + <target>1.5</target> + </configuration> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/feda0885/surefire-integration-tests/src/test/resources/surefire-1144-xml-runtime/src/test/java/surefire1144/Test1.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1144-xml-runtime/src/test/java/surefire1144/Test1.java b/surefire-integration-tests/src/test/resources/surefire-1144-xml-runtime/src/test/java/surefire1144/Test1.java new file mode 100644 index 0000000..047a741 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1144-xml-runtime/src/test/java/surefire1144/Test1.java @@ -0,0 +1,82 @@ +package surefire1144; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class Test1 +{ + static void sleep( int ms ) + { + long target = System.currentTimeMillis() + ms; + try + { + do + { + Thread.sleep( 1L ); + } + while ( System.currentTimeMillis() < target ); + } + catch ( InterruptedException e ) + { + throw new RuntimeException( e ); + } + } + + static void printTimeAndSleep( String msg, int ms ) + { + System.out.println( msg + " started @ " + System.currentTimeMillis() ); + sleep( ms ); + } + + @Test + public void testSleep100() + { + printTimeAndSleep( "Test1.sleep100", 100 ); + } + + @Test + public void testSleep200() + { + printTimeAndSleep( "Test1.sleep200", 200 ); + } + + @Test + public void testSleep300() + { + printTimeAndSleep( "Test1.sleep300", 300 ); + } + + @BeforeClass + public static void setUpBeforeClass() + throws Exception + { + printTimeAndSleep( "beforeClass sleep 500", 500 ); + } + + @AfterClass + public static void tearDownAfterClass() + throws Exception + { + printTimeAndSleep( "afterClass sleep 500", 500 ); + } +}
