[SUREFIRE-654] fix TestNG successPercentage support by ensuring that "flaky" test detection is disabled when rerunFailingTests=0
Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/7bd22e0a Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/7bd22e0a Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/7bd22e0a Branch: refs/heads/master Commit: 7bd22e0ad1539412c729582b590e86e910a6c340 Parents: 2340c01 Author: Andreas Gudian <agud...@apache.org> Authored: Sun Oct 26 19:45:01 2014 +0100 Committer: Andreas Gudian <agud...@apache.org> Committed: Sun Oct 26 22:27:44 2014 +0100 ---------------------------------------------------------------------- .../surefire/report/DefaultReporterFactory.java | 53 ++++++------ .../surefire/report/StatelessXmlReporter.java | 6 +- .../report/DefaultReporterFactoryTest.java | 53 +++++++----- .../surefire/its/TestNgSuccessPercentageIT.java | 30 +++++-- .../testng-succes-percentage-fail/pom.xml | 90 -------------------- .../java/testng/TestNGSuccessPercentTest.java | 44 ---------- .../testng-succes-percentage-pass/pom.xml | 90 -------------------- .../java/testng/TestNGSuccessPercentTest.java | 44 ---------- .../resources/testng-succes-percentage/pom.xml | 70 +++++++++++++++ .../testng/TestNGSuccessPercentFailingTest.java | 44 ++++++++++ .../testng/TestNGSuccessPercentPassingTest.java | 43 ++++++++++ surefire-providers/surefire-testng/pom.xml | 5 ++ 12 files changed, 248 insertions(+), 324 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java ---------------------------------------------------------------------- diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java index 62619a3..dfbec97 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java @@ -162,56 +162,60 @@ public class DefaultReporterFactory * if it only has errors or failures, then count its result based on its first run * * @param reportEntryList the list of test run report type for a given test + * @param rerunFailingTestsCount configured rerun count for failing tests * @return the type of test result */ // Use default visibility for testing - static TestResultType getTestResultType( List<ReportEntryType> reportEntryList ) + static TestResultType getTestResultType( List<ReportEntryType> reportEntryList, int rerunFailingTestsCount ) { if ( reportEntryList == null || reportEntryList.size() == 0 ) { return TestResultType.unknown; } - boolean seenSuccess = false, seenFailure = false; + boolean seenSuccess = false, seenFailure = false, seenError = false; for ( ReportEntryType resultType : reportEntryList ) { if ( resultType == ReportEntryType.SUCCESS ) { seenSuccess = true; } - else if ( resultType == ReportEntryType.FAILURE - || resultType == ReportEntryType.ERROR ) + else if ( resultType == ReportEntryType.FAILURE ) { seenFailure = true; } + else if ( resultType == ReportEntryType.ERROR ) + { + seenError = true; + } } - if ( seenSuccess && !seenFailure ) - { - return TestResultType.success; - } - - if ( seenSuccess && seenFailure ) - { - return TestResultType.flake; - } - - if ( !seenSuccess && seenFailure ) + if ( seenFailure || seenError ) { - if ( reportEntryList.get( 0 ) == ReportEntryType.FAILURE ) - { - return TestResultType.failure; - } - else if ( reportEntryList.get( 0 ) == ReportEntryType.ERROR ) + if ( seenSuccess && rerunFailingTestsCount > 0 ) { - return TestResultType.error; + return TestResultType.flake; } else { - // Reach here if the first one is skipped but later ones have failure, should be impossible - return TestResultType.skipped; + if ( seenError ) + { + return TestResultType.error; + } + else if ( seenFailure ) + { + return TestResultType.failure; + } + else + { + return TestResultType.skipped; + } } } + else if ( seenSuccess ) + { + return TestResultType.success; + } else { return TestResultType.skipped; @@ -266,7 +270,8 @@ public class DefaultReporterFactory resultTypeList.add( methodStats.getResultType() ); } - TestResultType resultType = getTestResultType( resultTypeList ); + TestResultType resultType = getTestResultType( resultTypeList, + reportConfiguration.getRerunFailingTestsCount() ); switch ( resultType ) { http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java ---------------------------------------------------------------------- diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java index dedeb27..e37940a 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java @@ -258,7 +258,7 @@ public class StatelessXmlReporter * @param methodEntryList the list of runs for a given test * @return the TestResultType for the given test */ - private static TestResultType getTestResultType( List<WrappedReportEntry> methodEntryList ) + private TestResultType getTestResultType( List<WrappedReportEntry> methodEntryList ) { List<ReportEntryType> testResultTypeList = new ArrayList<ReportEntryType>(); for ( WrappedReportEntry singleRunEntry : methodEntryList ) @@ -266,7 +266,7 @@ public class StatelessXmlReporter testResultTypeList.add( singleRunEntry.getReportEntryType() ); } - return DefaultReporterFactory.getTestResultType( testResultTypeList ); + return DefaultReporterFactory.getTestResultType( testResultTypeList, rerunFailingTestsCount ); } /** @@ -280,7 +280,7 @@ public class StatelessXmlReporter * in a given test class * @return the run time for the entire test class */ - private static int getRunTimeForAllTests( Map<String, List<WrappedReportEntry>> methodRunHistoryMap ) + private int getRunTimeForAllTests( Map<String, List<WrappedReportEntry>> methodRunHistoryMap ) { int totalTimeForSuite = 0; for ( Map.Entry<String, List<WrappedReportEntry>> entry : methodRunHistoryMap.entrySet() ) http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java ---------------------------------------------------------------------- diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java index 1c18283..fe8b7c1 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java @@ -19,19 +19,25 @@ package org.apache.maven.plugin.surefire.report; * under the License. */ +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import junit.framework.TestCase; + import org.apache.maven.plugin.surefire.StartupReportConfiguration; import org.apache.maven.surefire.report.DefaultDirectConsoleReporter; -import org.apache.maven.surefire.report.ReportEntry; import org.apache.maven.surefire.report.RunStatistics; import org.apache.maven.surefire.report.SafeThrowable; import org.apache.maven.surefire.report.StackTraceWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.*; +import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.error; +import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.failure; +import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.flake; +import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.skipped; +import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.success; +import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.unknown; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -55,7 +61,10 @@ public class DefaultReporterFactoryTest public void testMergeTestHistoryResult() { - DefaultReporterFactory factory = new DefaultReporterFactory( StartupReportConfiguration.defaultValue() ); + StartupReportConfiguration reportConfig = new StartupReportConfiguration( true, true, "PLAIN", false, false, new File("target"), false, null, "TESTHASH", + false, 1 ); + + DefaultReporterFactory factory = new DefaultReporterFactory( reportConfig ); // First run, four tests failed and one passed List<TestMethodStats> firstRunStats = new ArrayList<TestMethodStats>(); @@ -99,8 +108,8 @@ public class DefaultReporterFactoryTest // Only TEST_THREE is a failing test, other three are flaky tests assertEquals( 5, mergedStatistics.getCompletedCount() ); - assertEquals( 0, mergedStatistics.getErrors() ); - assertEquals( 1, mergedStatistics.getFailures() ); + assertEquals( 1, mergedStatistics.getErrors() ); + assertEquals( 0, mergedStatistics.getFailures() ); assertEquals( 3, mergedStatistics.getFlakes() ); assertEquals( 0, mergedStatistics.getSkipped() ); @@ -114,14 +123,14 @@ public class DefaultReporterFactoryTest assertEquals( Arrays.asList( expectedFlakeOutput ), reporter.getMessages() ); reporter = new DummyTestReporter(); - factory.printTestFailures( reporter, DefaultReporterFactory.TestResultType.failure ); + factory.printTestFailures( reporter, DefaultReporterFactory.TestResultType.error ); String[] expectedFailureOutput = - { "Failed tests: ", TEST_THREE, " Run 1: " + ASSERTION_FAIL, " Run 2: " + ERROR, " Run 3: " + ERROR, "", + { "Tests in error: ", TEST_THREE, " Run 1: " + ASSERTION_FAIL, " Run 2: " + ERROR, " Run 3: " + ERROR, "", "" }; assertEquals( Arrays.asList( expectedFailureOutput ), reporter.getMessages() ); reporter = new DummyTestReporter(); - factory.printTestFailures( reporter, DefaultReporterFactory.TestResultType.error ); + factory.printTestFailures( reporter, DefaultReporterFactory.TestResultType.failure ); String[] expectedErrorOutput = { "" }; assertEquals( Arrays.asList( expectedErrorOutput ), reporter.getMessages() ); } @@ -151,40 +160,42 @@ public class DefaultReporterFactoryTest public void testGetTestResultType() { - DefaultReporterFactory factory = new DefaultReporterFactory( StartupReportConfiguration.defaultValue() ); - List<ReportEntryType> emptyList = new ArrayList<ReportEntryType>(); - assertEquals( unknown, factory.getTestResultType( emptyList ) ); + assertEquals( unknown, DefaultReporterFactory.getTestResultType( emptyList, 1 ) ); List<ReportEntryType> successList = new ArrayList<ReportEntryType>(); successList.add( ReportEntryType.SUCCESS ); successList.add( ReportEntryType.SUCCESS ); - assertEquals( success, factory.getTestResultType( successList ) ); + assertEquals( success, DefaultReporterFactory.getTestResultType( successList, 1 ) ); List<ReportEntryType> failureErrorList = new ArrayList<ReportEntryType>(); failureErrorList.add( ReportEntryType.FAILURE ); failureErrorList.add( ReportEntryType.ERROR ); - assertEquals( failure, factory.getTestResultType( failureErrorList ) ); + assertEquals( error, DefaultReporterFactory.getTestResultType( failureErrorList, 1 ) ); List<ReportEntryType> errorFailureList = new ArrayList<ReportEntryType>(); errorFailureList.add( ReportEntryType.ERROR ); errorFailureList.add( ReportEntryType.FAILURE ); - assertEquals( error, factory.getTestResultType( errorFailureList ) ); + assertEquals( error, DefaultReporterFactory.getTestResultType( errorFailureList, 1 ) ); List<ReportEntryType> flakeList = new ArrayList<ReportEntryType>(); flakeList.add( ReportEntryType.SUCCESS ); flakeList.add( ReportEntryType.FAILURE ); - assertEquals( flake, factory.getTestResultType( flakeList ) ); + assertEquals( flake, DefaultReporterFactory.getTestResultType( flakeList, 1 ) ); + + assertEquals( failure, DefaultReporterFactory.getTestResultType( flakeList, 0 ) ); flakeList = new ArrayList<ReportEntryType>(); flakeList.add( ReportEntryType.ERROR ); flakeList.add( ReportEntryType.SUCCESS ); flakeList.add( ReportEntryType.FAILURE ); - assertEquals( flake, factory.getTestResultType( flakeList ) ); + assertEquals( flake, DefaultReporterFactory.getTestResultType( flakeList, 1 ) ); + + assertEquals( error, DefaultReporterFactory.getTestResultType( flakeList, 0 ) ); List<ReportEntryType> skippedList = new ArrayList<ReportEntryType>(); skippedList.add( ReportEntryType.SKIPPED ); - assertEquals( skipped, factory.getTestResultType( skippedList ) ); + assertEquals( skipped, DefaultReporterFactory.getTestResultType( skippedList, 1 ) ); } static class DummyStackTraceWriter http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/TestNgSuccessPercentageIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/TestNgSuccessPercentageIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/TestNgSuccessPercentageIT.java index 0df5cb6..a9f26e7 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/TestNgSuccessPercentageIT.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/TestNgSuccessPercentageIT.java @@ -1,5 +1,24 @@ package org.apache.maven.surefire.its; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + import org.apache.maven.surefire.its.fixture.OutputValidator; import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; import org.junit.Test; @@ -9,19 +28,14 @@ import org.junit.Test; * have passed. * * @author Jon Todd + * @author Andreas Gudian */ public class TestNgSuccessPercentageIT extends SurefireJUnit4IntegrationTestCase { @Test public void testPassesWhenFailuresLessThanSuccessPercentage() { - OutputValidator validator = unpack("/testng-succes-percentage-pass").executeTest(); - validator.assertTestSuiteResults(4, 0, 0, 0); + OutputValidator validator = unpack("/testng-succes-percentage").mavenTestFailureIgnore( true ).executeTest(); + validator.assertTestSuiteResults(8, 0, 1, 0); } - @Test - public void testFailsWhenFailuresMoreThanSuccessPercentage() - { - OutputValidator validator = unpack("/testng-succes-percentage-fail").executeTest(); - validator.assertTestSuiteResults(4, 0, 1, 0); - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/pom.xml b/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/pom.xml deleted file mode 100644 index 42933f2..0000000 --- a/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/pom.xml +++ /dev/null @@ -1,90 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, - ~ software distributed under the License is distributed on an - ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - ~ KIND, either express or implied. See the License for the - ~ specific language governing permissions and limitations - ~ under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <groupId>org.apache.maven.plugins.surefire</groupId> - <artifactId>junit4</artifactId> - <version>1.0-SNAPSHOT</version> - <name>Test for Testng</name> - - - <properties> - <testNgVersion>5.7</testNgVersion> - <testNgClassifier>jdk15</testNgClassifier> - </properties> - - <profiles> - <profile> - <id>testng-old</id> - <activation> - <property><name>testNgClassifier</name></property> - </activation> - <dependencies> - <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> - <version>${testNgVersion}</version> - <classifier>${testNgClassifier}</classifier> - </dependency> - </dependencies> - </profile> - <profile> - <id>testng-new</id> - <activation> - <property><name>!testNgClassifier</name></property> - </activation> - <dependencies> - <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> - <version>${testNgVersion}</version> - </dependency> - </dependencies> - </profile> - </profiles> - - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>2.3.2</version> - <configuration> - <source>1.5</source> - <target>1.5</target> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire.version}</version> - <configuration> - <test>TestNGSuccessPercentTest</test> - </configuration> - </plugin> - </plugins> - </build> - -</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/src/test/java/testng/TestNGSuccessPercentTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/src/test/java/testng/TestNGSuccessPercentTest.java b/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/src/test/java/testng/TestNGSuccessPercentTest.java deleted file mode 100644 index a5b7a25..0000000 --- a/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/src/test/java/testng/TestNGSuccessPercentTest.java +++ /dev/null @@ -1,44 +0,0 @@ -package testng; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.testng.annotations.*; -import static org.testng.Assert.*; -import java.util.concurrent.atomic.AtomicInteger; - -public class TestNGSuccessPercentTest -{ - - private static AtomicInteger counter = new AtomicInteger(0); - - // Pass 2 of 4 tests, expect this test to fail when 60% success is required - @Test(invocationCount = 4, successPercentage = 60) - public void testFailure() { - if (isOdd(counter.get())) { - assertTrue(false); // Fail - } - counter.addAndGet(1); - } - - private boolean isOdd(int number) { - return number % 2 == 0; - } - -} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/pom.xml b/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/pom.xml deleted file mode 100644 index 42933f2..0000000 --- a/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/pom.xml +++ /dev/null @@ -1,90 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, - ~ software distributed under the License is distributed on an - ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - ~ KIND, either express or implied. See the License for the - ~ specific language governing permissions and limitations - ~ under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <groupId>org.apache.maven.plugins.surefire</groupId> - <artifactId>junit4</artifactId> - <version>1.0-SNAPSHOT</version> - <name>Test for Testng</name> - - - <properties> - <testNgVersion>5.7</testNgVersion> - <testNgClassifier>jdk15</testNgClassifier> - </properties> - - <profiles> - <profile> - <id>testng-old</id> - <activation> - <property><name>testNgClassifier</name></property> - </activation> - <dependencies> - <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> - <version>${testNgVersion}</version> - <classifier>${testNgClassifier}</classifier> - </dependency> - </dependencies> - </profile> - <profile> - <id>testng-new</id> - <activation> - <property><name>!testNgClassifier</name></property> - </activation> - <dependencies> - <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> - <version>${testNgVersion}</version> - </dependency> - </dependencies> - </profile> - </profiles> - - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>2.3.2</version> - <configuration> - <source>1.5</source> - <target>1.5</target> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire.version}</version> - <configuration> - <test>TestNGSuccessPercentTest</test> - </configuration> - </plugin> - </plugins> - </build> - -</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/src/test/java/testng/TestNGSuccessPercentTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/src/test/java/testng/TestNGSuccessPercentTest.java b/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/src/test/java/testng/TestNGSuccessPercentTest.java deleted file mode 100644 index 775587d..0000000 --- a/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/src/test/java/testng/TestNGSuccessPercentTest.java +++ /dev/null @@ -1,44 +0,0 @@ -package testng; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.testng.annotations.*; -import static org.testng.Assert.*; -import java.util.concurrent.atomic.AtomicInteger; - -public class TestNGSuccessPercentTest -{ - - private static AtomicInteger counter = new AtomicInteger(0); - - // Pass 2 of 4 tests, expect this test to pass when 50% success is required - @Test(invocationCount = 4, successPercentage = 50) - public void testSuccess() { - if (isOdd(counter.get())) { - assertTrue(false); // Fail - } - counter.addAndGet(1); - } - - private boolean isOdd(int number) { - return number % 2 == 0; - } - -} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/surefire-integration-tests/src/test/resources/testng-succes-percentage/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage/pom.xml b/surefire-integration-tests/src/test/resources/testng-succes-percentage/pom.xml new file mode 100644 index 0000000..13bc831 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/testng-succes-percentage/pom.xml @@ -0,0 +1,70 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>it-parent</artifactId> + <version>1.0</version> + </parent> + + <artifactId>testng-success-percentage</artifactId> + <version>1.0-SNAPSHOT</version> + <name>Test for Testng</name> + + <properties> + <testNgVersion>5.7</testNgVersion> + <testNgClassifier>jdk15</testNgClassifier> + </properties> + + <profiles> + <profile> + <id>testng-old</id> + <activation> + <property><name>testNgClassifier</name></property> + </activation> + <dependencies> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <version>${testNgVersion}</version> + <classifier>${testNgClassifier}</classifier> + </dependency> + </dependencies> + </profile> + <profile> + <id>testng-new</id> + <activation> + <property><name>!testNgClassifier</name></property> + </activation> + <dependencies> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <version>${testNgVersion}</version> + </dependency> + </dependencies> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/surefire-integration-tests/src/test/resources/testng-succes-percentage/src/test/java/testng/TestNGSuccessPercentFailingTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage/src/test/java/testng/TestNGSuccessPercentFailingTest.java b/surefire-integration-tests/src/test/resources/testng-succes-percentage/src/test/java/testng/TestNGSuccessPercentFailingTest.java new file mode 100644 index 0000000..791869c --- /dev/null +++ b/surefire-integration-tests/src/test/resources/testng-succes-percentage/src/test/java/testng/TestNGSuccessPercentFailingTest.java @@ -0,0 +1,44 @@ +package testng; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.testng.annotations.*; +import static org.testng.Assert.*; +import java.util.concurrent.atomic.AtomicInteger; + +public class TestNGSuccessPercentFailingTest +{ + + private static final AtomicInteger counter = new AtomicInteger( 0 ); + + // Pass 2 of 4 tests, expect this test to fail when 60% success is required + @Test( invocationCount = 4, successPercentage = 60 ) + public void testFailure() + { + int value = counter.addAndGet( 1 ); + assertTrue( isOdd( value ), "is odd: " + value ); + } + + private boolean isOdd( int number ) + { + return number % 2 == 0; + } + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/surefire-integration-tests/src/test/resources/testng-succes-percentage/src/test/java/testng/TestNGSuccessPercentPassingTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage/src/test/java/testng/TestNGSuccessPercentPassingTest.java b/surefire-integration-tests/src/test/resources/testng-succes-percentage/src/test/java/testng/TestNGSuccessPercentPassingTest.java new file mode 100644 index 0000000..35a1be4 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/testng-succes-percentage/src/test/java/testng/TestNGSuccessPercentPassingTest.java @@ -0,0 +1,43 @@ +package testng; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.testng.annotations.*; +import static org.testng.Assert.*; +import java.util.concurrent.atomic.AtomicInteger; + +public class TestNGSuccessPercentPassingTest +{ + + private static final AtomicInteger counter = new AtomicInteger( 0 ); + + // Pass 2 of 4 tests, expect this test to pass when 50% success is required + @Test( invocationCount = 4, successPercentage = 50 ) + public void testSuccess() + { + int value = counter.addAndGet( 1 ); + assertTrue( isOdd( value ), "is odd: " + value ); + } + + private boolean isOdd( int number ) + { + return number % 2 == 0; + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/7bd22e0a/surefire-providers/surefire-testng/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-testng/pom.xml b/surefire-providers/surefire-testng/pom.xml index 4bc36e2..cca78aa 100644 --- a/surefire-providers/surefire-testng/pom.xml +++ b/surefire-providers/surefire-testng/pom.xml @@ -43,6 +43,11 @@ <artifactId>common-java5</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>surefire-testng-utils</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> <build>
