Repository: maven-surefire Updated Branches: refs/heads/master 710a3d58d -> 5c3a3138e
[SUREFIRE-1152] Support rerunFailingTestsCount for test suites Submitted by: Sean Flanigan Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/cb977633 Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/cb977633 Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/cb977633 Branch: refs/heads/master Commit: cb977633f0f1d2be4b2556a4743a5f52d8a7008a Parents: 1f19156 Author: Sean Flanigan <sflan...@redhat.com> Authored: Sat Apr 11 11:32:37 2015 +1000 Committer: Sean Flanigan <sflan...@redhat.com> Committed: Sat Apr 11 11:47:33 2015 +1000 ---------------------------------------------------------------------- .../Surefire1152RerunFailingTestsInSuiteIT.java | 42 +++++++++ .../pom.xml | 91 ++++++++++++++++++++ .../test/java/jiras/surefire1152/FlakyIT.java | 35 ++++++++ .../java/jiras/surefire1152/FlakyITSuite.java | 28 ++++++ .../test/java/jiras/surefire1152/FlakyTest.java | 35 ++++++++ .../java/jiras/surefire1152/FlakyTestSuite.java | 28 ++++++ .../common/junit4/JUnit4ProviderUtil.java | 10 +-- .../maven/surefire/junit4/JUnit4Provider.java | 20 +++-- 8 files changed, 279 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/cb977633/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java new file mode 100644 index 0000000..7385b22 --- /dev/null +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java @@ -0,0 +1,42 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * SUREFIRE-1152 Assert rerunFailingTestsCount works with test suites + * + * @author Sean Flanigan + */ +public class Surefire1152RerunFailingTestsInSuiteIT + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void rerunsFailingTestInSuite() + { + OutputValidator outputValidator = unpack( "surefire-1152-rerunFailingTestsCount-suite" ).executeVerify(); + outputValidator.assertTestSuiteResults( 1, 0, 0, 0 ); + outputValidator.assertIntegrationTestSuiteResults( 1, 0, 0, 0 ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/cb977633/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml new file mode 100644 index 0000000..b0206b7 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml @@ -0,0 +1,91 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>it-parent</artifactId> + <version>1.0</version> + <relativePath>../pom.xml</relativePath> + </parent> + <groupId>org.apache.maven.plugins.surefire</groupId> + <artifactId>jiras-surefire-1152</artifactId> + + <version>1.0</version> + + <properties> + <failIfNoTests>true</failIfNoTests> + <failsafe.rerunFailingTestsCount>2</failsafe.rerunFailingTestsCount> + <surefire.rerunFailingTestsCount>2</surefire.rerunFailingTestsCount> + </properties> + + <build> + <plugins> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <source>1.5</source> + <target>1.5</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <includes> + <include>**/FlakyTestSuite.java</include> + </includes> + <!--<skip>true</skip>--> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + </execution> + </executions> + <configuration> + <includes> + <include>**/FlakyITSuite.java</include> + </includes> + </configuration> + </plugin> + </plugins> + </build> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.11</version> + <scope>test</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/cb977633/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java new file mode 100644 index 0000000..54888db --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java @@ -0,0 +1,35 @@ +package jiras.surefire1152; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Test; + +public class FlakyIT { + + static int n = 0; + + @Test + public void testFlaky() { + if (n++ == 0) { + throw new AssertionError("deliberately flaky test (should pass the next time)"); + } + } + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/cb977633/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java new file mode 100644 index 0000000..77d1c9f --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java @@ -0,0 +1,28 @@ +package jiras.surefire1152; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({FlakyIT.class}) +public class FlakyITSuite { +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/cb977633/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java new file mode 100644 index 0000000..613b7df --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java @@ -0,0 +1,35 @@ +package jiras.surefire1152; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Test; + +public class FlakyTest { + + static int n = 0; + + @Test + public void testFlaky() { + if (n++ == 0) { + throw new AssertionError("deliberately flaky test (should pass the next time)"); + } + } + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/cb977633/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java new file mode 100644 index 0000000..09e100b --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java @@ -0,0 +1,28 @@ +package jiras.surefire1152; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({FlakyTest.class}) +public class FlakyTestSuite { +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/cb977633/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java index 610ac2c..1e35552 100644 --- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java @@ -90,14 +90,14 @@ public final class JUnit4ProviderUtil } /** - * Get the name of all test methods from a list of Failures + * Get all test methods from a list of Failures * * @param allFailures the list of failures for a given test class - * @return the list of test method names + * @return the list of test methods */ - public static Set<String> generateFailingTests( List<Failure> allFailures ) + public static Set<ClassMethod> generateFailingTests( List<Failure> allFailures ) { - Set<String> failingMethods = new HashSet<String>(); + Set<ClassMethod> failingMethods = new HashSet<ClassMethod>(); for ( Failure failure : allFailures ) { @@ -107,7 +107,7 @@ public final class JUnit4ProviderUtil ClassMethod classMethod = cutTestClassAndMethod( description ); if ( classMethod.isValid() ) { - failingMethods.add( classMethod.getMethod() ); + failingMethods.add( classMethod ); } } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/cb977633/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java index 12c3983..b622aea 100644 --- a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java +++ b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java @@ -171,11 +171,12 @@ public class JUnit4Provider { for ( int i = 0; i < rerunFailingTestsCount && !failureListener.getAllFailures().isEmpty(); i++ ) { - Set<String> failedTests = JUnit4ProviderUtil.generateFailingTests( failureListener.getAllFailures() ); + Set<ClassMethod> failedTests = JUnit4ProviderUtil.generateFailingTests( + failureListener.getAllFailures( ) ); failureListener.reset(); if ( !failedTests.isEmpty() ) { - executeFailedMethod( clazz, listeners, failedTests ); + executeFailedMethod( listeners, failedTests ); } } } @@ -273,11 +274,20 @@ public class JUnit4Provider } } - private static void executeFailedMethod( Class<?> testClass, RunNotifier notifier, Set<String> failedMethods ) + private void executeFailedMethod( RunNotifier notifier, Set<ClassMethod> failedMethods ) { - for ( String failedMethod : failedMethods ) + for ( ClassMethod failedMethod : failedMethods ) { - Request.method( testClass, failedMethod ).getRunner().run( notifier ); + try + { + Class<?> methodClass = Class.forName( failedMethod.getClazz(), true, testClassLoader ); + String methodName = failedMethod.getMethod(); + Request.method( methodClass, methodName ).getRunner().run( notifier ); + } + catch ( ClassNotFoundException e ) + { + throw new RuntimeException( "Unable to create test class '" + failedMethod.getClazz() + "'", e ); + } } }
