This is an automated email from the ASF dual-hosted git repository.

psteitz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-pool.git


The following commit(s) were added to refs/heads/master by this push:
     new 7c63f29b JIRA:POOL-407. Add ResilientPooledObjectFactory.
     new b50d1626 Merge branch 'master' of 
https://github.com/apache/commons-pool
7c63f29b is described below

commit 7c63f29b9b90b3fbbc829d3ce0b0e4f98c832351
Author: Phil Steitz <phil.ste...@gmail.com>
AuthorDate: Fri May 31 10:53:12 2024 -0700

    JIRA:POOL-407. Add ResilientPooledObjectFactory.
---
 src/changes/changes.xml                            |   4 +
 .../pool3/impl/ResilientPooledObjectFactory.java   | 379 ++++++++++++++++++
 .../commons/pool3/impl/TestGenericObjectPool.java  | 429 ++++++++++++---------
 .../impl/TestResilientPooledObjectFactory.java     | 339 ++++++++++++++++
 4 files changed, 971 insertions(+), 180 deletions(-)

diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index f6d3603b..d9756758 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -60,6 +60,10 @@ The <action> type attribute can be add,update,fix,remove.
     <action dev="ggregory" type="update">Bump org.ow2.asm:asm-util from 9.5 to 
9.7 #252, #294.</action>
     <action type="update" dev="ggregory" due-to="Gary Gregory">Bump 
commons-lang3 from 3.13.0 to 3.14.0.</action>
     <action type="update" dev="ggregory" due-to="Dependabot, Gary 
Gregory">Bump org.apache.bcel:bcel from 6.7.0 to 6.9.0 #263, #306.</action>
+    <!-- FIX -->
+    <action dev="psteitz" type="fix" issue="POOL-407">
+      Add ReslientPooledObjectFactory to provide resilence against factory 
outages.
+    </action>
   </release> 
   <release version="2.12.0" date="2023-MM-DD" description="This is a feature 
and maintenance release (Java 8 or above).">
     <!-- FIX -->
diff --git 
a/src/main/java/org/apache/commons/pool3/impl/ResilientPooledObjectFactory.java 
b/src/main/java/org/apache/commons/pool3/impl/ResilientPooledObjectFactory.java
new file mode 100644
index 00000000..6cc8877e
--- /dev/null
+++ 
b/src/main/java/org/apache/commons/pool3/impl/ResilientPooledObjectFactory.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.pool3.impl;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.apache.commons.pool3.PooledObject;
+import org.apache.commons.pool3.PooledObjectFactory;
+
+/**
+ * Wraps a PooledObjectFactory, extending to provide resilient features.
+ * <p>
+ * Maintains a cicrular log of makeObject calls and makes strategy-based
+ * decisions on whether to keep trying proactively to create objects.
+ * Decisions use data in the makeObject log and information reported by
+ * the pool that the factory is attached to.
+ *
+ * @param <T> Type of object managed by the factory
+ * @param <E> Type of exception that the factory may throw
+ */
+public class ResilientPooledObjectFactory<T, E extends Exception> implements 
PooledObjectFactory<T, E> {
+    private static final int DEFAULT_LOG_SIZE = 10;
+    private static final Duration DEFAULT_DELAY = Duration.ofSeconds(1);
+    private static final Duration DEFAULT_LOOK_BACK = Duration.ofMinutes(5);
+    private static final Duration DEFAULT_TIME_BETWEEN_CHECKS = 
Duration.ofSeconds(10);
+    /** Wrapped factory */
+    private final PooledObjectFactory<T, E> factory;
+    /** GOP that factory is attached to. */
+    private GenericObjectPool<T, E> pool;
+    /** Size of the circular log of makeObject events */
+    private int logSize;
+    /** Duration of time window for statistics */
+    private final Duration lookBack;
+    /** Circular log of makeObject events */
+    private final ConcurrentLinkedQueue<MakeEvent> makeObjectLog = new 
ConcurrentLinkedQueue<>();
+    /** Time of last factory failure */
+    private Instant downStart;
+    /** Time factory last returned to "up" state. */
+    private Instant upStart;
+    /** Exception counts */
+    @SuppressWarnings("rawtypes")
+    private ConcurrentHashMap<Class, Integer> exceptionCounts = new 
ConcurrentHashMap<>();
+    /** Whether or not the factory is "up" */
+    private boolean up = true;
+    /** Whether or not the monitor thread is running */
+    private boolean monitoring = false;
+    /** Time to wait between object creations by the adder thread */
+    private final Duration delay;
+    /** Time between monitor checks */
+    private Duration timeBetweenChecks = Duration.ofSeconds(10);
+
+    /** Adder thread */
+    private Adder adder = null;
+
+    /**
+     * Construct a ResilientPooledObjectFactory from a factory with specified
+     * parameters.
+     *
+     * @param factory           PooledObjectFactory to wrap
+     * @param logSize           length of the makeObject log
+     * @param delay             time to wait between object creations by the 
adder
+     *                          thread
+     * @param lookBack          length of time over which metrics are kept
+     * @param timeBetweenChecks time between checks by the monitor thread
+     */
+    public ResilientPooledObjectFactory(PooledObjectFactory<T, E> factory,
+            int logSize, Duration delay, Duration lookBack, Duration 
timeBetweenChecks) {
+        this.logSize = logSize;
+        this.factory = factory;
+        this.delay = delay;
+        this.lookBack = lookBack;
+        this.timeBetweenChecks = timeBetweenChecks;
+    }
+
+    /**
+     * Construct a ResilientPooledObjectFactory from a factory and pool, using
+     * defaults for logSize, delay, and lookBack.
+     *
+     * @param factory PooledObjectFactory to wrap
+     */
+    public ResilientPooledObjectFactory(PooledObjectFactory<T, E> factory) {
+        this(factory, DEFAULT_LOG_SIZE, DEFAULT_DELAY, DEFAULT_LOOK_BACK, 
DEFAULT_TIME_BETWEEN_CHECKS);
+    }
+
+    public void setPool(GenericObjectPool<T, E> pool) {
+        this.pool = pool;
+    }
+
+    public void setTimeBetweenChecks(Duration timeBetweenChecks) {
+        this.timeBetweenChecks = timeBetweenChecks;
+    }
+
+    public void setLogSize(int logSize) {
+        this.logSize = logSize;
+    }
+
+    /**
+     * Delegate to the wrapped factory, but log the makeObject call.
+     */
+    @Override
+    public PooledObject<T> makeObject() throws E {
+        final MakeEvent makeEvent = new MakeEvent();
+        try {
+            PooledObject<T> obj = factory.makeObject();
+            makeEvent.setSuccess(!PooledObject.isNull(obj));
+            return obj;
+        } catch (Throwable t) {
+            makeEvent.setSuccess(false);
+            exceptionCounts.put(t.getClass(), exceptionCounts.getOrDefault(t, 
0) + 1);
+            throw t;
+        } finally {
+            makeEvent.end();
+            makeObjectLog.add(makeEvent);
+        }
+    }
+
+    // Delegate all other methods to the wrapped factory.
+
+    @Override
+    public void destroyObject(PooledObject<T> p) throws E {
+        factory.destroyObject(p);
+    }
+
+    @Override
+    public boolean validateObject(PooledObject<T> p) {
+        return factory.validateObject(p);
+    }
+
+    @Override
+    public void activateObject(PooledObject<T> p) throws E {
+        factory.activateObject(p);
+    }
+
+    @Override
+    public void passivateObject(PooledObject<T> p) throws E {
+        factory.passivateObject(p);
+    }
+
+    /**
+     * Default implementation considers the factory down as soon as a single
+     * makeObject call fails and considers it back up if the last logSize 
makes have
+     * succeeded.
+     * <p>
+     * Sets downStart to time of the first failure found in makeObjectLog and
+     * upStart to the time when logSize consecutive makes have succeeded.
+     * <p>
+     * When a failure is observed, the adder thread is started if the pool
+     * is not closed and has take waiters.
+     * <p>
+     * Removes the oldest event from the log if it is full.
+     *
+     */
+    protected void runChecks() {
+        boolean upOverLog = true;
+        // 1. If the log is full, remove the oldest (first) event.
+        //
+        // 2. Walk the event log. If we find a failure, set downStart, set up 
to false
+        // and start the adder thread.
+        //
+        // 3. If the log contains only successes, if up is false, set upStart 
and up to
+        // true
+        // and kill the adder thread.
+        while (makeObjectLog.size() > logSize) {
+            makeObjectLog.poll();
+        }
+        for (MakeEvent makeEvent : makeObjectLog) {
+            if (!makeEvent.isSuccess()) {
+                upOverLog = false;
+                downStart = Instant.now();
+                up = false;
+                if (pool.getNumWaiters() > 0 && !pool.isClosed() && adder == 
null) {
+                    adder = new Adder();
+                    adder.start();
+                }
+            }
+
+        }
+        if (upOverLog && !up) {
+            // Kill adder thread and set up to true
+            upStart = Instant.now();
+            up = true;
+            adder.kill();
+            adder = null;
+        }
+    }
+
+    /**
+     * @return true if the factory is considered "up".
+     */
+    public boolean isUp() {
+        return up;
+    }
+
+    /**
+     * @return true if the adder is running
+     */
+    public boolean isAdderRunning() {
+        return adder != null && adder.isRunning();
+    }
+
+    /**
+     * @return true if the monitor is running
+     */
+    public boolean isMonitorRunning() {
+        return monitoring;
+    }
+
+    public void startMonitor(Duration timeBetweenChecks) {
+        this.timeBetweenChecks = timeBetweenChecks;
+        startMonitor();
+    }
+
+    public void startMonitor() {
+        monitoring = true;
+        new Monitor().start();
+    }
+
+    public void stopMonitor() {
+        monitoring = false;
+    }
+
+    /**
+     * Adder thread that adds objects to the pool, waiting for a fixed delay 
between
+     * adds.
+     * <p>
+     * The adder thread will stop under any of the following conditions:
+     * <ul>
+     * <li>The pool is closed.</li>
+     * <li>The factory is down.</li>
+     * <li>The pool is full.</li>
+     * <li>The pool has no waiters.</li>
+     * </ul>
+     */
+    class Adder extends Thread {
+        private boolean killed = false;
+        private boolean running = false;
+        private int MAX_FAILURES = 5;
+        private int failures = 0;
+
+        @Override
+        public void run() {
+            running = true;
+            while (!up && !killed && !pool.isClosed()) {
+                try {
+                    pool.addObject();
+                    if (pool.getNumWaiters() == 0 || pool.getNumActive() + 
pool.getNumIdle() == pool.getMaxTotal()) {
+                        kill();
+                    }
+                } catch (Throwable e) {
+                    failures++;
+                    if (failures > MAX_FAILURES) {
+                        kill();
+                    }
+                } finally {
+                    // Wait for delay
+                    try {
+                        Thread.sleep(delay.toMillis());
+                    } catch (InterruptedException e) {
+                        // Ignore
+                    } catch (Throwable e) {
+                        killed = true;
+                        running = false;
+                        throw (e);
+                    }
+                }
+            }
+            killed = true;
+            running = false;
+        }
+
+        public void start() {
+            if (killed) {
+                killed = false;
+            }
+            run();
+        }
+
+        public boolean isRunning() {
+            return running;
+        }
+
+        public void kill() {
+            killed = true;
+        }
+    }
+
+    /**
+     * Record of a makeObject event.
+     */
+    static class MakeEvent {
+        private final Instant startTime;
+        private Instant endTime;
+        private boolean success;
+        private Throwable exception;
+        private String message;
+
+        /**
+         * Constructor set statTime to now.
+         */
+        public MakeEvent() {
+            startTime = Instant.now();
+        }
+
+        /**
+         * @return the time the makeObject call ended
+         */
+        public Instant getEndTime() {
+            return endTime;
+        }
+
+        public void end() {
+            this.endTime = Instant.now();
+        }
+
+        public boolean isSuccess() {
+            return success;
+        }
+
+        public void setSuccess(boolean success) {
+            this.success = success;
+        }
+
+        public Throwable getException() {
+            return exception;
+        }
+
+        public void setException(Throwable exception) {
+            this.exception = exception;
+        }
+
+        public Instant getStartTime() {
+            return startTime;
+        }
+
+        public String getMessage() {
+            return message;
+        }
+
+        public void setMessage(String message) {
+            this.message = message;
+        }
+
+    }
+
+    class Monitor extends Thread {
+        @Override
+        public void run() {
+            while (monitoring && !pool.isClosed()) {
+                runChecks();
+                try {
+                    Thread.sleep(timeBetweenChecks.toMillis());
+                } catch (InterruptedException e) {
+                    monitoring = false;
+                } catch (Throwable e) {
+                    monitoring = false;
+                    throw (e);
+                }
+            }
+            monitoring = false;
+        }
+    }
+}
diff --git 
a/src/test/java/org/apache/commons/pool3/impl/TestGenericObjectPool.java 
b/src/test/java/org/apache/commons/pool3/impl/TestGenericObjectPool.java
index 3a3066eb..f09656bc 100644
--- a/src/test/java/org/apache/commons/pool3/impl/TestGenericObjectPool.java
+++ b/src/test/java/org/apache/commons/pool3/impl/TestGenericObjectPool.java
@@ -148,6 +148,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         public Object create() {
             return null;
         }
+
         @Override
         public PooledObject<Object> wrap(final Object value) {
             return new DefaultPooledObject<>(value);
@@ -173,9 +174,10 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     }
 
     /**
-     * Factory that creates HashSets.  Note that this means
-     *  0) All instances are initially equal (not discernible by equals)
-     *  1) Instances are mutable and mutation can cause change in identity / 
hash code.
+     * Factory that creates HashSets. Note that this means
+     * 0) All instances are initially equal (not discernible by equals)
+     * 1) Instances are mutable and mutation can cause change in identity / 
hash
+     * code.
      */
     private static final class HashSetFactory
             extends BasePooledObjectFactory<HashSet<String>, RuntimeException> 
{
@@ -183,6 +185,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         public HashSet<String> create() {
             return new HashSet<>();
         }
+
         @Override
         public PooledObject<HashSet<String>> wrap(final HashSet<String> value) 
{
             return new DefaultPooledObject<>(value);
@@ -196,13 +199,16 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         private final String obj;
         private final ObjectPool<String, ? extends Exception> pool;
         private boolean done;
+
         public InvalidateThread(final ObjectPool<String, ? extends Exception> 
pool, final String obj) {
             this.obj = obj;
             this.pool = pool;
         }
+
         public boolean complete() {
             return done;
         }
+
         @Override
         public void run() {
             try {
@@ -224,6 +230,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         public Object create() {
             return new Object();
         }
+
         @Override
         public boolean validateObject(final PooledObject<Object> obj) {
             Waiter.sleepQuietly(1000);
@@ -272,7 +279,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         }
 
         public SimpleFactory(final boolean valid) {
-            this(valid,valid);
+            this(valid, valid);
         }
 
         public SimpleFactory(final boolean evalid, final boolean ovalid) {
@@ -292,7 +299,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
                 oddTest = oddValid;
                 counter = activationCounter++;
             }
-            if (hurl && !(counter%2 == 0 ? evenTest : oddTest)) {
+            if (hurl && !(counter % 2 == 0 ? evenTest : oddTest)) {
                 throw new TestException();
             }
         }
@@ -339,7 +346,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
                 activeCount++;
                 if (activeCount > maxTotal) {
                     throw new IllegalStateException(
-                        "Too many active instances: " + activeCount);
+                            "Too many active instances: " + activeCount);
                 }
                 waitLatency = makeLatency;
             }
@@ -436,7 +443,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
                 throw new RuntimeException("validation failed");
             }
             if (validate) {
-                return counter%2 == 0 ? evenTest : oddTest;
+                return counter % 2 == 0 ? evenTest : oddTest;
             }
             return true;
         }
@@ -503,15 +510,15 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         }
 
         public TestThread(final ObjectPool<T, E> pool, final int iter, final 
int startDelay,
-            final int holdTime, final boolean randomDelay, final Object obj) {
-       this.pool = pool;
-       this.iter = iter;
-       this.startDelay = startDelay;
-       this.holdTime = holdTime;
-       this.randomDelay = randomDelay;
-       this.random = this.randomDelay ? new Random() : null;
-       this.expectedObject = obj;
-    }
+                final int holdTime, final boolean randomDelay, final Object 
obj) {
+            this.pool = pool;
+            this.iter = iter;
+            this.startDelay = startDelay;
+            this.holdTime = holdTime;
+            this.randomDelay = randomDelay;
+            this.random = this.randomDelay ? new Random() : null;
+            this.expectedObject = obj;
+        }
 
         public boolean complete() {
             return complete;
@@ -568,7 +575,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         private Throwable thrown;
 
         private long preBorrowMillis; // just before borrow
-        private long postBorrowMillis; //  borrow returned
+        private long postBorrowMillis; // borrow returned
         private long postReturnMillis; // after object was returned
         private long endedMillis;
         private String objectId;
@@ -591,14 +598,14 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
                 postReturnMillis = System.currentTimeMillis();
             } catch (final Throwable e) {
                 thrown = e;
-            } finally{
+            } finally {
                 endedMillis = System.currentTimeMillis();
             }
         }
     }
 
-    private static final boolean DISPLAY_THREAD_DETAILS=
-    Boolean.getBoolean("TestGenericObjectPool.display.thread.details");
+    private static final boolean DISPLAY_THREAD_DETAILS = Boolean
+            .getBoolean("TestGenericObjectPool.display.thread.details");
     // To pass this to a Maven test, use:
     // mvn test -DargLine="-DTestGenericObjectPool.display.thread.details=true"
     // @see https://issues.apache.org/jira/browse/SUREFIRE-121
@@ -674,7 +681,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         assertNotEquals("0", obj, "oldest not evicted");
         assertNotEquals("1", obj, "second oldest not evicted");
         // 2 should be next out for FIFO, 4 for LIFO
-        assertEquals(lifo ? "4" : "2" , obj,"Wrong instance returned");
+        assertEquals(lifo ? "4" : "2", obj, "Wrong instance returned");
     }
 
     private void checkEvictionOrderPart2(final boolean lifo) throws Exception {
@@ -689,13 +696,14 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         genericObjectPool.evict(); // Should evict "0" and "1"
         genericObjectPool.evict(); // Should evict "2" and "3"
         final Object obj = genericObjectPool.borrowObject();
-        assertEquals("4", obj,"Wrong instance remaining in pool");
+        assertEquals("4", obj, "Wrong instance remaining in pool");
     }
 
     private void checkEvictorVisiting(final boolean lifo) throws Exception {
         VisitTracker<Object> obj;
         VisitTrackerFactory<Object> trackerFactory = new 
VisitTrackerFactory<>();
-        try (GenericObjectPool<VisitTracker<Object>,RuntimeException> 
trackerPool = new GenericObjectPool<>(trackerFactory)) {
+        try (GenericObjectPool<VisitTracker<Object>, RuntimeException> 
trackerPool = new GenericObjectPool<>(
+                trackerFactory)) {
             trackerPool.setNumTestsPerEvictionRun(2);
             trackerPool.setMinEvictableIdleDuration(Duration.ofMillis(-1));
             trackerPool.setTestWhileIdle(true);
@@ -717,16 +725,17 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
             for (int i = 0; i < 8; i++) {
                 final VisitTracker<Object> tracker = 
trackerPool.borrowObject();
                 if (tracker.getId() >= 4) {
-                    assertEquals( 0, tracker.getValidateCount(),"Unexpected 
instance visited " + tracker.getId());
+                    assertEquals(0, tracker.getValidateCount(), "Unexpected 
instance visited " + tracker.getId());
                 } else {
-                    assertEquals( 1, tracker.getValidateCount(),
+                    assertEquals(1, tracker.getValidateCount(),
                             "Instance " + tracker.getId() + " visited wrong 
number of times.");
                 }
             }
         }
 
         trackerFactory = new VisitTrackerFactory<>();
-        try (GenericObjectPool<VisitTracker<Object>, RuntimeException> 
trackerPool = new GenericObjectPool<>(trackerFactory)) {
+        try (GenericObjectPool<VisitTracker<Object>, RuntimeException> 
trackerPool = new GenericObjectPool<>(
+                trackerFactory)) {
             trackerPool.setNumTestsPerEvictionRun(3);
             trackerPool.setMinEvictableIdleDuration(Duration.ofMillis(-1));
             trackerPool.setTestWhileIdle(true);
@@ -753,10 +762,10 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
             for (int i = 0; i < 8; i++) {
                 final VisitTracker<Object> tracker = 
trackerPool.borrowObject();
                 if (tracker.getId() != 0) {
-                    assertEquals( 1, tracker.getValidateCount(),
+                    assertEquals(1, tracker.getValidateCount(),
                             "Instance " + tracker.getId() + " visited wrong 
number of times.");
                 } else {
-                    assertEquals( 2, tracker.getValidateCount(),
+                    assertEquals(2, tracker.getValidateCount(),
                             "Instance " + tracker.getId() + " visited wrong 
number of times.");
                 }
             }
@@ -769,7 +778,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         random.setSeed(System.currentTimeMillis());
         for (int i = 0; i < 4; i++) {
             for (int j = 0; j < 5; j++) {
-                try (GenericObjectPool<VisitTracker<Object>, RuntimeException> 
trackerPool = new GenericObjectPool<>(trackerFactory)) {
+                try (GenericObjectPool<VisitTracker<Object>, RuntimeException> 
trackerPool = new GenericObjectPool<>(
+                        trackerFactory)) {
                     trackerPool.setNumTestsPerEvictionRun(smallPrimes[i]);
                     
trackerPool.setMinEvictableIdleDuration(Duration.ofMillis(-1));
                     trackerPool.setTestWhileIdle(true);
@@ -839,7 +849,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         };
     }
 
-    private BasePooledObjectFactory<String, InterruptedException> 
createSlowObjectFactory(final long elapsedTimeMillis) {
+    private BasePooledObjectFactory<String, InterruptedException> 
createSlowObjectFactory(
+            final long elapsedTimeMillis) {
         return new BasePooledObjectFactory<String, InterruptedException>() {
             @Override
             public String create() throws InterruptedException {
@@ -874,8 +885,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     protected ObjectPool<String, TestException> makeEmptyPool(final int 
minCap) {
         final GenericObjectPool<String, TestException> mtPool = new 
GenericObjectPool<>(new SimpleFactory());
         mtPool.setMaxTotal(minCap);
-       mtPool.setMaxIdle(minCap);
-       return mtPool;
+        mtPool.setMaxIdle(minCap);
+        return mtPool;
     }
 
     @Override
@@ -888,7 +899,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
      * <iterations> borrow-return cycles with random delay times <= delay
      * in between.
      */
-    private <T, E extends Exception> void runTestThreads(final int numThreads, 
final int iterations, final int delay, final GenericObjectPool<T, E> testPool) {
+    private <T, E extends Exception> void runTestThreads(final int numThreads, 
final int iterations, final int delay,
+            final GenericObjectPool<T, E> testPool) {
         final TestThread<T, E>[] threads = new TestThread[numThreads];
         for (int i = 0; i < numThreads; i++) {
             threads[i] = new TestThread<>(testPool, iterations, delay);
@@ -922,7 +934,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         simpleFactory = null;
 
         final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
-        final Set<ObjectName> result = mbs.queryNames(new 
ObjectName("org.apache.commoms.pool3:type=GenericObjectPool,*"), null);
+        final Set<ObjectName> result = mbs
+                .queryNames(new 
ObjectName("org.apache.commoms.pool3:type=GenericObjectPool,*"), null);
         // There should be no registered pools at this point
         final int registeredPoolCount = result.size();
         final StringBuilder msg = new StringBuilder("Current pool is: ");
@@ -947,12 +960,15 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     }
 
     /**
-     * Check that a pool that starts an evictor, but is never closed does not 
leave EvictionTimer executor running. Confirmation check is in
+     * Check that a pool that starts an evictor, but is never closed does not 
leave
+     * EvictionTimer executor running. Confirmation check is in
      * {@link #tearDown()}.
      *
-     * @throws TestException Custom exception
-     * @throws InterruptedException if any thread has interrupted the current 
thread. The <em>interrupted status</em> of the current thread is cleared when 
this
-     *         exception is thrown.
+     * @throws TestException        Custom exception
+     * @throws InterruptedException if any thread has interrupted the current
+     *                              thread. The <em>interrupted status</em> of 
the
+     *                              current thread is cleared when this
+     *                              exception is thrown.
      */
     @SuppressWarnings("deprecation")
     @Test
@@ -975,16 +991,16 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     @Test
     @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
     public void testAddObject() throws Exception {
-        assertEquals( 0, genericObjectPool.getNumIdle(),"should be zero idle");
+        assertEquals(0, genericObjectPool.getNumIdle(), "should be zero idle");
         genericObjectPool.addObject();
-        assertEquals( 1, genericObjectPool.getNumIdle(),"should be one idle");
-        assertEquals( 0, genericObjectPool.getNumActive(),"should be zero 
active");
+        assertEquals(1, genericObjectPool.getNumIdle(), "should be one idle");
+        assertEquals(0, genericObjectPool.getNumActive(), "should be zero 
active");
         final String obj = genericObjectPool.borrowObject();
-        assertEquals( 0, genericObjectPool.getNumIdle(),"should be zero idle");
-        assertEquals( 1, genericObjectPool.getNumActive(),"should be one 
active");
+        assertEquals(0, genericObjectPool.getNumIdle(), "should be zero idle");
+        assertEquals(1, genericObjectPool.getNumActive(), "should be one 
active");
         genericObjectPool.returnObject(obj);
-        assertEquals( 1, genericObjectPool.getNumIdle(),"should be one idle");
-        assertEquals( 0, genericObjectPool.getNumActive(),"should be zero 
active");
+        assertEquals(1, genericObjectPool.getNumIdle(), "should be one idle");
+        assertEquals(0, genericObjectPool.getNumActive(), "should be zero 
active");
     }
 
     @Test
@@ -1005,7 +1021,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
      * aware of this if you see a failure of this test.
      */
     @SuppressWarnings({
-        "rawtypes", "unchecked"
+            "rawtypes", "unchecked"
     })
     @Test
     @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
@@ -1030,7 +1046,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
 
         // Start and park threads waiting to borrow objects
         final TestThread[] threads = new TestThread[numThreads];
-        for(int i=0;i<numThreads;i++) {
+        for (int i = 0; i < numThreads; i++) {
             threads[i] = new TestThread(genericObjectPool, 1, 0, 2000, false, 
String.valueOf(i % maxTotal));
             final Thread t = new Thread(threads[i]);
             t.start();
@@ -1059,9 +1075,12 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         // Borrow
         final String object = genericObjectPool.borrowObject();
         final PooledObject<String> po = 
genericObjectPool.getPooledObject(object);
-        // In the initial state, all instants are the creation instant: last 
borrow, last use, last return.
-        // In the initial state, the active duration is the time between "now" 
and the creation time.
-        // In the initial state, the idle duration is the time between "now" 
and the last return, which is the creation time.
+        // In the initial state, all instants are the creation instant: last 
borrow,
+        // last use, last return.
+        // In the initial state, the active duration is the time between "now" 
and the
+        // creation time.
+        // In the initial state, the idle duration is the time between "now" 
and the
+        // last return, which is the creation time.
         // But... this PO might have already been used in other tests in this 
class.
 
         final Instant lastBorrowInstant1 = po.getLastBorrowInstant();
@@ -1072,13 +1091,15 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         assertThat(po.getCreateInstant(), 
lessThanOrEqualTo(lastReturnInstant1));
         assertThat(po.getCreateInstant(), lessThanOrEqualTo(lastUsedInstant1));
 
-        // Sleep MUST be "long enough" to detect that more than 0 milliseconds 
have elapsed.
+        // Sleep MUST be "long enough" to detect that more than 0 milliseconds 
have
+        // elapsed.
         // Need an API in Java 8 to get the clock granularity.
         Thread.sleep(200);
 
         assertFalse(po.getActiveDuration().isNegative());
         assertFalse(po.getActiveDuration().isZero());
-        // We use greaterThanOrEqualTo instead of equal because "now" many be 
different when each argument is evaluated.
+        // We use greaterThanOrEqualTo instead of equal because "now" many be 
different
+        // when each argument is evaluated.
         assertThat(1L, lessThanOrEqualTo(2L)); // sanity check
         assertThat(Duration.ZERO, 
lessThanOrEqualTo(Duration.ZERO.plusNanos(1))); // sanity check
         assertThat(po.getActiveDuration(), 
lessThanOrEqualTo(po.getIdleDuration()));
@@ -1110,6 +1131,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     /**
      * On first borrow, first object fails validation, second object is OK.
      * Subsequent borrows are OK. This was POOL-152.
+     * 
      * @throws Exception
      */
     @Test
@@ -1152,7 +1174,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         final SimpleFactory factory = new SimpleFactory();
         // Give makeObject a little latency
         factory.setMakeLatency(200);
-        try (final GenericObjectPool<String, TestException> pool = new 
GenericObjectPool<>(factory, new GenericObjectPoolConfig<>())) {
+        try (final GenericObjectPool<String, TestException> pool = new 
GenericObjectPool<>(factory,
+                new GenericObjectPoolConfig<>())) {
             final String s = pool.borrowObject();
             // First borrow waits on create, so wait time should be at least 
200 ms
             // Allow 100ms error in clock times
@@ -1170,7 +1193,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     @Test
     @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
     public void testCloseMultiplePools1() {
-        try (final GenericObjectPool<String, TestException> genericObjectPool2 
= new GenericObjectPool<>(simpleFactory)) {
+        try (final GenericObjectPool<String, TestException> genericObjectPool2 
= new GenericObjectPool<>(
+                simpleFactory)) {
             
genericObjectPool.setDurationBetweenEvictionRuns(TestConstants.ONE_MILLISECOND_DURATION);
             
genericObjectPool2.setDurationBetweenEvictionRuns(TestConstants.ONE_MILLISECOND_DURATION);
         }
@@ -1180,8 +1204,10 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     @Test
     @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
     public void testCloseMultiplePools2() throws Exception {
-        try (final GenericObjectPool<String, TestException> genericObjectPool2 
= new GenericObjectPool<>(simpleFactory)) {
-            // Ensure eviction takes a long time, during which time 
EvictionTimer.executor's queue is empty
+        try (final GenericObjectPool<String, TestException> genericObjectPool2 
= new GenericObjectPool<>(
+                simpleFactory)) {
+            // Ensure eviction takes a long time, during which time 
EvictionTimer.executor's
+            // queue is empty
             simpleFactory.setDestroyLatency(1000L);
             // Ensure there is an object to evict, so that above latency takes 
effect
             
genericObjectPool.setDurationBetweenEvictionRuns(TestConstants.ONE_MILLISECOND_DURATION);
@@ -1203,10 +1229,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         genericObjectPool.addObject();
 
         for (int i = 0; i < 5000; i++) {
-            final ConcurrentBorrowAndEvictThread one =
-                    new ConcurrentBorrowAndEvictThread(true);
-            final ConcurrentBorrowAndEvictThread two =
-                    new ConcurrentBorrowAndEvictThread(false);
+            final ConcurrentBorrowAndEvictThread one = new 
ConcurrentBorrowAndEvictThread(true);
+            final ConcurrentBorrowAndEvictThread two = new 
ConcurrentBorrowAndEvictThread(false);
 
             one.start();
             two.start();
@@ -1215,11 +1239,12 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
 
             genericObjectPool.returnObject(one.obj);
 
-            /* Uncomment this for a progress indication
-            if (i % 10 == 0) {
-                System.out.println(i/10);
-            }
-            */
+            /*
+             * Uncomment this for a progress indication
+             * if (i % 10 == 0) {
+             * System.out.println(i/10);
+             * }
+             */
         }
     }
 
@@ -1308,13 +1333,20 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
             assertEquals(BaseObjectPoolConfig.DEFAULT_MAX_WAIT, 
dummyPool.getMaxWaitDuration());
             assertEquals(GenericObjectPoolConfig.DEFAULT_MIN_IDLE, 
dummyPool.getMinIdle());
             assertEquals(GenericObjectPoolConfig.DEFAULT_MAX_TOTAL, 
dummyPool.getMaxTotal());
-            
assertEquals(BaseObjectPoolConfig.DEFAULT_MIN_EVICTABLE_IDLE_DURATION, 
dummyPool.getMinEvictableIdleDuration());
-            
assertEquals(BaseObjectPoolConfig.DEFAULT_NUM_TESTS_PER_EVICTION_RUN, 
dummyPool.getNumTestsPerEvictionRun());
-            
assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_TEST_ON_BORROW), 
Boolean.valueOf(dummyPool.getTestOnBorrow()));
-            
assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN), 
Boolean.valueOf(dummyPool.getTestOnReturn()));
-            
assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_TEST_WHILE_IDLE), 
Boolean.valueOf(dummyPool.getTestWhileIdle()));
-            
assertEquals(BaseObjectPoolConfig.DEFAULT_DURATION_BETWEEN_EVICTION_RUNS, 
dummyPool.getDurationBetweenEvictionRuns());
-            
assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_BLOCK_WHEN_EXHAUSTED),
 Boolean.valueOf(dummyPool.getBlockWhenExhausted()));
+            
assertEquals(BaseObjectPoolConfig.DEFAULT_MIN_EVICTABLE_IDLE_DURATION,
+                    dummyPool.getMinEvictableIdleDuration());
+            
assertEquals(BaseObjectPoolConfig.DEFAULT_NUM_TESTS_PER_EVICTION_RUN,
+                    dummyPool.getNumTestsPerEvictionRun());
+            
assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_TEST_ON_BORROW),
+                    Boolean.valueOf(dummyPool.getTestOnBorrow()));
+            
assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN),
+                    Boolean.valueOf(dummyPool.getTestOnReturn()));
+            
assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_TEST_WHILE_IDLE),
+                    Boolean.valueOf(dummyPool.getTestWhileIdle()));
+            
assertEquals(BaseObjectPoolConfig.DEFAULT_DURATION_BETWEEN_EVICTION_RUNS,
+                    dummyPool.getDurationBetweenEvictionRuns());
+            
assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_BLOCK_WHEN_EXHAUSTED),
+                    Boolean.valueOf(dummyPool.getBlockWhenExhausted()));
             assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_LIFO), 
Boolean.valueOf(dummyPool.getLifo()));
         }
 
@@ -1352,7 +1384,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     @Test
     @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
     public void testDefaultConfiguration() {
-        assertConfiguration(new GenericObjectPoolConfig<>(),genericObjectPool);
+        assertConfiguration(new GenericObjectPoolConfig<>(), 
genericObjectPool);
     }
 
     /**
@@ -1377,7 +1409,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     public void testErrorFactoryDoesNotBlockThreads() throws Exception {
 
         final CreateErrorFactory factory = new CreateErrorFactory();
-        try (final GenericObjectPool<String, InterruptedException> 
createFailFactoryPool = new GenericObjectPool<>(factory)) {
+        try (final GenericObjectPool<String, InterruptedException> 
createFailFactoryPool = new GenericObjectPool<>(
+                factory)) {
 
             createFailFactoryPool.setMaxTotal(1);
 
@@ -1442,7 +1475,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         final Thread borrowerThread = new Thread(borrower);
         // Set evictor to run in 100 ms - will create idle instance
         
genericObjectPool.setDurationBetweenEvictionRuns(Duration.ofMillis(100));
-        borrowerThread.start();  // Off to the races
+        borrowerThread.start(); // Off to the races
         borrowerThread.join();
         assertFalse(borrower.failed());
     }
@@ -1472,17 +1505,22 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         }
 
         Waiter.sleepQuietly(1000L);
-        assertTrue(genericObjectPool.getNumIdle() < 500,"Should be less than 
500 idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() < 500,
+                "Should be less than 500 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(600L);
-        assertTrue(genericObjectPool.getNumIdle() < 400,"Should be less than 
400 idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() < 400,
+                "Should be less than 400 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(600L);
-        assertTrue(genericObjectPool.getNumIdle() < 300,"Should be less than 
300 idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() < 300,
+                "Should be less than 300 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(600L);
-        assertTrue(genericObjectPool.getNumIdle() < 200,"Should be less than 
200 idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() < 200,
+                "Should be less than 200 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(600L);
-        assertTrue(genericObjectPool.getNumIdle() < 100,"Should be less than 
100 idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() < 100,
+                "Should be less than 100 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(600L);
-        assertEquals(0,genericObjectPool.getNumIdle(),"Should be zero idle, 
found " + genericObjectPool.getNumIdle());
+        assertEquals(0, genericObjectPool.getNumIdle(), "Should be zero idle, 
found " + genericObjectPool.getNumIdle());
 
         for (int i = 0; i < 500; i++) {
             active[i] = genericObjectPool.borrowObject();
@@ -1492,24 +1530,30 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         }
 
         Waiter.sleepQuietly(1000L);
-        assertTrue(genericObjectPool.getNumIdle() < 500,"Should be less than 
500 idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() < 500,
+                "Should be less than 500 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(600L);
-        assertTrue(genericObjectPool.getNumIdle() < 400,"Should be less than 
400 idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() < 400,
+                "Should be less than 400 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(600L);
-        assertTrue(genericObjectPool.getNumIdle() < 300,"Should be less than 
300 idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() < 300,
+                "Should be less than 300 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(600L);
-        assertTrue(genericObjectPool.getNumIdle() < 200,"Should be less than 
200 idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() < 200,
+                "Should be less than 200 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(600L);
-        assertTrue(genericObjectPool.getNumIdle() < 100,"Should be less than 
100 idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() < 100,
+                "Should be less than 100 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(600L);
-        assertEquals(0,genericObjectPool.getNumIdle(),"Should be zero idle, 
found " + genericObjectPool.getNumIdle());
+        assertEquals(0, genericObjectPool.getNumIdle(), "Should be zero idle, 
found " + genericObjectPool.getNumIdle());
     }
 
     @Test
     @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
     public void testEvictionInvalid() throws Exception {
 
-        try (final GenericObjectPool<Object, RuntimeException> 
invalidFactoryPool = new GenericObjectPool<>(new InvalidFactory())) {
+        try (final GenericObjectPool<Object, RuntimeException> 
invalidFactoryPool = new GenericObjectPool<>(
+                new InvalidFactory())) {
 
             invalidFactoryPool.setMaxIdle(1);
             invalidFactoryPool.setMaxTotal(1);
@@ -1539,8 +1583,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
             Thread.sleep(1000);
 
             // Should have an empty pool
-            assertEquals( 0, invalidFactoryPool.getNumIdle(),"Idle count 
different than expected.");
-            assertEquals( 0, invalidFactoryPool.getNumActive(),"Total count 
different than expected.");
+            assertEquals(0, invalidFactoryPool.getNumIdle(), "Idle count 
different than expected.");
+            assertEquals(0, invalidFactoryPool.getNumActive(), "Total count 
different than expected.");
         }
     }
 
@@ -1572,18 +1616,22 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         genericObjectPool.setTestWhileIdle(true);
 
         // ClassNotFoundException
-        assertThrows(IllegalArgumentException.class, () -> 
genericObjectPool.setEvictionPolicyClassName(Long.toString(System.currentTimeMillis())),
+        assertThrows(IllegalArgumentException.class,
+                () -> 
genericObjectPool.setEvictionPolicyClassName(Long.toString(System.currentTimeMillis())),
                 "setEvictionPolicyClassName must throw an error if the class 
name is invalid.");
 
         // InstantiationException
-        assertThrows(IllegalArgumentException.class, () -> 
genericObjectPool.setEvictionPolicyClassName(java.io.Serializable.class.getName()),
+        assertThrows(IllegalArgumentException.class,
+                () -> 
genericObjectPool.setEvictionPolicyClassName(java.io.Serializable.class.getName()),
                 "setEvictionPolicyClassName must throw an error if the class 
name is invalid.");
 
         // IllegalAccessException
-        assertThrows(IllegalArgumentException.class, () -> 
genericObjectPool.setEvictionPolicyClassName(java.util.Collections.class.getName()),
+        assertThrows(IllegalArgumentException.class,
+                () -> 
genericObjectPool.setEvictionPolicyClassName(java.util.Collections.class.getName()),
                 "setEvictionPolicyClassName must throw an error if the class 
name is invalid.");
 
-        assertThrows(IllegalArgumentException.class, () -> 
genericObjectPool.setEvictionPolicyClassName(java.lang.String.class.getName()),
+        assertThrows(IllegalArgumentException.class,
+                () -> 
genericObjectPool.setEvictionPolicyClassName(java.lang.String.class.getName()),
                 () -> "setEvictionPolicyClassName must throw an error if a 
class that does not implement EvictionPolicy is specified.");
 
         genericObjectPool.setEvictionPolicy(new TestEvictionPolicy<>());
@@ -1671,7 +1719,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     @Test
     @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
     public void testEvictionWithNegativeNumTests() throws Exception {
-        // when numTestsPerEvictionRun is negative, it represents a fraction 
of the idle objects to test
+        // when numTestsPerEvictionRun is negative, it represents a fraction 
of the idle
+        // objects to test
         genericObjectPool.setMaxIdle(6);
         genericObjectPool.setMaxTotal(6);
         genericObjectPool.setNumTestsPerEvictionRun(-2);
@@ -1687,13 +1736,16 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         }
 
         Waiter.sleepQuietly(100L);
-        assertTrue(genericObjectPool.getNumIdle() <= 6,"Should at most 6 idle, 
found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() <= 6,
+                "Should at most 6 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(100L);
-        assertTrue(genericObjectPool.getNumIdle() <= 3,"Should at most 3 idle, 
found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() <= 3,
+                "Should at most 3 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(100L);
-        assertTrue(genericObjectPool.getNumIdle() <= 2,"Should be at most 2 
idle, found " + genericObjectPool.getNumIdle());
+        assertTrue(genericObjectPool.getNumIdle() <= 2,
+                "Should be at most 2 idle, found " + 
genericObjectPool.getNumIdle());
         Waiter.sleepQuietly(100L);
-        assertEquals(0,genericObjectPool.getNumIdle(),"Should be zero idle, 
found " + genericObjectPool.getNumIdle());
+        assertEquals(0, genericObjectPool.getNumIdle(), "Should be zero idle, 
found " + genericObjectPool.getNumIdle());
     }
 
     @Test
@@ -1794,14 +1846,15 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         final String obj = genericObjectPool.borrowObject();
         simpleFactory.setThrowExceptionOnPassivate(true);
         genericObjectPool.returnObject(obj);
-        assertEquals(0,genericObjectPool.getNumIdle());
+        assertEquals(0, genericObjectPool.getNumIdle());
     }
 
     @Test
     public void testFailingFactoryDoesNotBlockThreads() throws Exception {
 
         final CreateFailFactory factory = new CreateFailFactory();
-        try (final GenericObjectPool<String, InterruptedException> 
createFailFactoryPool = new GenericObjectPool<>(factory)) {
+        try (final GenericObjectPool<String, InterruptedException> 
createFailFactoryPool = new GenericObjectPool<>(
+                factory)) {
 
             createFailFactoryPool.setMaxTotal(1);
 
@@ -1853,26 +1906,28 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         genericObjectPool.addObject(); // "0"
         genericObjectPool.addObject(); // "1"
         genericObjectPool.addObject(); // "2"
-        assertEquals( "0", genericObjectPool.borrowObject(),"Oldest");
-        assertEquals( "1", genericObjectPool.borrowObject(),"Middle");
-        assertEquals( "2", genericObjectPool.borrowObject(),"Youngest");
+        assertEquals("0", genericObjectPool.borrowObject(), "Oldest");
+        assertEquals("1", genericObjectPool.borrowObject(), "Middle");
+        assertEquals("2", genericObjectPool.borrowObject(), "Youngest");
         final String o = genericObjectPool.borrowObject();
-        assertEquals( "3", o,"new-3");
+        assertEquals("3", o, "new-3");
         genericObjectPool.returnObject(o);
-        assertEquals( o, genericObjectPool.borrowObject(),"returned-3");
-        assertEquals( "4", genericObjectPool.borrowObject(),"new-4");
+        assertEquals(o, genericObjectPool.borrowObject(), "returned-3");
+        assertEquals("4", genericObjectPool.borrowObject(), "new-4");
     }
 
     @Test
     public void testGetFactoryType_DefaultPooledObjectFactory() {
-        try (final GenericObjectPool<String, RuntimeException> pool = new 
GenericObjectPool<>(createDefaultPooledObjectFactory())) {
+        try (final GenericObjectPool<String, RuntimeException> pool = new 
GenericObjectPool<>(
+                createDefaultPooledObjectFactory())) {
             assertNotNull(pool.getFactoryType());
         }
     }
 
     @Test
     public void testGetFactoryType_NullPooledObjectFactory() {
-        try (final GenericObjectPool<String, RuntimeException> pool = new 
GenericObjectPool<>(createNullPooledObjectFactory())) {
+        try (final GenericObjectPool<String, RuntimeException> pool = new 
GenericObjectPool<>(
+                createNullPooledObjectFactory())) {
             assertNotNull(pool.getFactoryType());
         }
     }
@@ -1918,7 +1973,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     }
 
     /**
-     * Verify that threads waiting on a depleted pool get served when a 
checked out object is
+     * Verify that threads waiting on a depleted pool get served when a 
checked out
+     * object is
      * invalidated.
      *
      * JIRA: POOL-240
@@ -1939,7 +1995,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
             // Launch another thread - will block, but fail in 500 ms
             final WaitingTestThread<TestException> thread2 = new 
WaitingTestThread<>(pool, 100);
             thread2.start();
-            // Invalidate the object borrowed by this thread - should allow 
thread2 to create
+            // Invalidate the object borrowed by this thread - should allow 
thread2 to
+            // create
             Thread.sleep(20);
             pool.invalidateObject(obj);
             Thread.sleep(600); // Wait for thread2 to timeout
@@ -1963,14 +2020,16 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
 
         final GenericObjectPoolConfig<String> config = new 
GenericObjectPoolConfig<>();
         config.setJmxEnabled(false);
-        try (final GenericObjectPool<String, TestException> poolWithoutJmx = 
new GenericObjectPool<>(simpleFactory, config)) {
+        try (final GenericObjectPool<String, TestException> poolWithoutJmx = 
new GenericObjectPool<>(simpleFactory,
+                config)) {
             assertNull(poolWithoutJmx.getJmxName());
             config.setJmxEnabled(true);
             poolWithoutJmx.jmxUnregister();
         }
 
         config.setJmxNameBase(null);
-        try (final GenericObjectPool<String, TestException> 
poolWithDefaultJmxNameBase = new GenericObjectPool<>(simpleFactory, config)) {
+        try (final GenericObjectPool<String, TestException> 
poolWithDefaultJmxNameBase = new GenericObjectPool<>(
+                simpleFactory, config)) {
             assertNotNull(poolWithDefaultJmxNameBase.getJmxName());
         }
     }
@@ -1983,33 +2042,37 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         genericObjectPool.addObject(); // "0"
         genericObjectPool.addObject(); // "1"
         genericObjectPool.addObject(); // "2"
-        assertEquals( "2", genericObjectPool.borrowObject(),"Youngest");
-        assertEquals( "1", genericObjectPool.borrowObject(),"Middle");
-        assertEquals( "0", genericObjectPool.borrowObject(),"Oldest");
+        assertEquals("2", genericObjectPool.borrowObject(), "Youngest");
+        assertEquals("1", genericObjectPool.borrowObject(), "Middle");
+        assertEquals("0", genericObjectPool.borrowObject(), "Oldest");
         o = genericObjectPool.borrowObject();
-        assertEquals( "3", o,"new-3");
+        assertEquals("3", o, "new-3");
         genericObjectPool.returnObject(o);
-        assertEquals( o, genericObjectPool.borrowObject(),"returned-3");
-        assertEquals( "4", genericObjectPool.borrowObject(),"new-4");
+        assertEquals(o, genericObjectPool.borrowObject(), "returned-3");
+        assertEquals("4", genericObjectPool.borrowObject(), "new-4");
     }
 
     /**
      * Simplest example of recovery from factory outage.
-     * A thread gets into parked wait on the deque when there is capacity to 
create, but
-     * creates are failing due to factory outage.  Verify that the borrower is 
served
+     * A thread gets into parked wait on the deque when there is capacity to 
create,
+     * but
+     * creates are failing due to factory outage. Verify that the borrower is 
served
      * once the factory is back online.
      */
     @Test
-    @Disabled
     @Timeout(value = 1000, unit = TimeUnit.MILLISECONDS)
     public void testLivenessOnTransientFactoryFailure() throws 
InterruptedException {
         final DisconnectingWaiterFactory<String> factory = new 
DisconnectingWaiterFactory<>(
-            DisconnectingWaiterFactory.DEFAULT_DISCONNECTED_CREATE_ACTION,
-            DisconnectingWaiterFactory.DEFAULT_DISCONNECTED_LIFECYCLE_ACTION,
-            obj -> false // all instances fail validation
+                DisconnectingWaiterFactory.DEFAULT_DISCONNECTED_CREATE_ACTION,
+                
DisconnectingWaiterFactory.DEFAULT_DISCONNECTED_LIFECYCLE_ACTION,
+                obj -> false // all instances fail validation
         );
         final AtomicBoolean failed = new AtomicBoolean();
-        try (GenericObjectPool<Waiter, IllegalStateException> pool = new 
GenericObjectPool<>(factory)) {
+        final ResilientPooledObjectFactory<Waiter, IllegalStateException> 
resilientFactory = new ResilientPooledObjectFactory<>(
+                factory, 10, Duration.ofMillis(20), Duration.ofMinutes(10), 
Duration.ofMillis(20));
+        try (GenericObjectPool<Waiter, IllegalStateException> pool = new 
GenericObjectPool<>(resilientFactory)) {
+            resilientFactory.setPool(pool);
+            resilientFactory.startMonitor();
             pool.setMaxWait(Duration.ofMillis(100));
             pool.setTestOnReturn(true);
             pool.setMaxTotal(1);
@@ -2039,9 +2102,11 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
 
     /**
      * Test the following scenario:
-     *   Thread 1 borrows an instance
-     *   Thread 2 starts to borrow another instance before thread 1 returns 
its instance
-     *   Thread 1 returns its instance while thread 2 is validating its newly 
created instance
+     * Thread 1 borrows an instance
+     * Thread 2 starts to borrow another instance before thread 1 returns its
+     * instance
+     * Thread 1 returns its instance while thread 2 is validating its newly 
created
+     * instance
      * The test verifies that the instance created by Thread 2 is not leaked.
      *
      * @throws Exception May occur in some failure modes
@@ -2070,15 +2135,15 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         genericObjectPool.setMaxTotal(100);
         genericObjectPool.setMaxIdle(8);
         final String[] active = new String[100];
-        for(int i=0;i<100;i++) {
+        for (int i = 0; i < 100; i++) {
             active[i] = genericObjectPool.borrowObject();
         }
-        assertEquals(100,genericObjectPool.getNumActive());
-        assertEquals(0,genericObjectPool.getNumIdle());
-        for(int i=0;i<100;i++) {
+        assertEquals(100, genericObjectPool.getNumActive());
+        assertEquals(0, genericObjectPool.getNumIdle());
+        for (int i = 0; i < 100; i++) {
             genericObjectPool.returnObject(active[i]);
-            assertEquals(99 - i,genericObjectPool.getNumActive());
-            assertEquals(i < 8 ? i+1 : 8,genericObjectPool.getNumIdle());
+            assertEquals(99 - i, genericObjectPool.getNumActive());
+            assertEquals(i < 8 ? i + 1 : 8, genericObjectPool.getNumIdle());
         }
     }
 
@@ -2088,14 +2153,14 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         genericObjectPool.setMaxTotal(100);
         genericObjectPool.setMaxIdle(0);
         final String[] active = new String[100];
-        for(int i=0;i<100;i++) {
+        for (int i = 0; i < 100; i++) {
             active[i] = genericObjectPool.borrowObject();
         }
-        assertEquals(100,genericObjectPool.getNumActive());
-        assertEquals(0,genericObjectPool.getNumIdle());
-        for(int i=0;i<100;i++) {
+        assertEquals(100, genericObjectPool.getNumActive());
+        assertEquals(0, genericObjectPool.getNumIdle());
+        for (int i = 0; i < 100; i++) {
             genericObjectPool.returnObject(active[i]);
-            assertEquals(99 - i,genericObjectPool.getNumActive());
+            assertEquals(99 - i, genericObjectPool.getNumActive());
             assertEquals(0, genericObjectPool.getNumIdle());
         }
     }
@@ -2123,7 +2188,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
 
         // Start threads to borrow objects
         final TestThread[] threads = new TestThread[numThreads];
-        for(int i=0;i<numThreads;i++) {
+        for (int i = 0; i < numThreads; i++) {
             // Factor of 2 on iterations so main thread does work whilst other
             // threads are running. Factor of 2 on delay so average delay for
             // other threads == actual delay for main thread
@@ -2191,8 +2256,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
     public void testMaxTotalInvariant() {
         final int maxTotal = 15;
-        simpleFactory.setEvenValid(false);     // Every other validation fails
-        simpleFactory.setDestroyLatency(100);  // Destroy takes 100 ms
+        simpleFactory.setEvenValid(false); // Every other validation fails
+        simpleFactory.setDestroyLatency(100); // Destroy takes 100 ms
         simpleFactory.setMaxTotal(maxTotal); // (makes - destroys) bound
         simpleFactory.setValidationEnabled(true);
         genericObjectPool.setMaxTotal(maxTotal);
@@ -2219,7 +2284,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
 
         // Start threads to borrow objects
         final TestThread[] threads = new TestThread[numThreads];
-        for(int i=0;i<numThreads;i++) {
+        for (int i = 0; i < numThreads; i++) {
             // Factor of 2 on iterations so main thread does work whilst other
             // threads are running. Factor of 2 on delay so average delay for
             // other threads == actual delay for main thread
@@ -2256,10 +2321,10 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         }
 
         for (int i = 0; i < numThreads; i++) {
-            while(!threads[i].complete()) {
+            while (!threads[i].complete()) {
                 Waiter.sleepQuietly(500L);
             }
-            if(threads[i].failed()) {
+            if (threads[i].failed()) {
                 fail("Thread " + i + " failed: " + 
threads[i].error.toString());
             }
         }
@@ -2277,7 +2342,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
      * Test multi-threaded pool access.
      * Multiple threads, but maxTotal only allows half the threads to succeed.
      *
-     * This test was prompted by Continuum build failures in the Commons DBCP 
test case:
+     * This test was prompted by Continuum build failures in the Commons DBCP 
test
+     * case:
      * TestPerUserPoolDataSource.testMultipleThreads2()
      * Let's see if the this fails on Continuum too!
      */
@@ -2302,30 +2368,29 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         int failed = 0;
         for (final WaitingTestThread<TestException> element : wtt) {
             element.join();
-            if (element.thrown != null){
+            if (element.thrown != null) {
                 failed++;
             }
         }
-        if (DISPLAY_THREAD_DETAILS || wtt.length/2 != failed){
+        if (DISPLAY_THREAD_DETAILS || wtt.length / 2 != failed) {
             System.out.println(
                     "MaxWait: " + maxWait +
-                    " HoldTime: " + holdTime +
-                     " MaxTotal: " + threads +
-                    " Threads: " + wtt.length +
-                    " Failed: " + failed
-                    );
+                            " HoldTime: " + holdTime +
+                            " MaxTotal: " + threads +
+                            " Threads: " + wtt.length +
+                            " Failed: " + failed);
             for (final WaitingTestThread<TestException> wt : wtt) {
                 System.out.println(
                         "PreBorrow: " + (wt.preBorrowMillis - originMillis) +
-                        " PostBorrow: " + (wt.postBorrowMillis != 0 ? 
wt.postBorrowMillis - originMillis : -1) +
-                        " BorrowTime: " + (wt.postBorrowMillis != 0 ? 
wt.postBorrowMillis - wt.preBorrowMillis : -1) +
-                        " PostReturn: " + (wt.postReturnMillis != 0 ? 
wt.postReturnMillis - originMillis : -1) +
-                        " Ended: " + (wt.endedMillis - originMillis) +
-                        " ObjId: " + wt.objectId
-                        );
+                                " PostBorrow: " + (wt.postBorrowMillis != 0 ? 
wt.postBorrowMillis - originMillis : -1) +
+                                " BorrowTime: "
+                                + (wt.postBorrowMillis != 0 ? 
wt.postBorrowMillis - wt.preBorrowMillis : -1) +
+                                " PostReturn: " + (wt.postReturnMillis != 0 ? 
wt.postReturnMillis - originMillis : -1) +
+                                " Ended: " + (wt.endedMillis - originMillis) +
+                                " ObjId: " + wt.objectId);
             }
         }
-        assertEquals(wtt.length / 2, failed,"Expected half the threads to 
fail");
+        assertEquals(wtt.length / 2, failed, "Expected half the threads to 
fail");
     }
 
     @Test
@@ -2389,7 +2454,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         Waiter.sleepQuietly(150L);
         assertEquals(5, genericObjectPool.getNumIdle(), "Should be 5 idle, 
found " + genericObjectPool.getNumIdle());
 
-        for(int i = 0 ; i < 5 ; i++) {
+        for (int i = 0; i < 5; i++) {
             genericObjectPool.returnObject(active[i]);
         }
 
@@ -2412,7 +2477,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     }
 
     /**
-     * Verifies that returning an object twice (without borrow in between) 
causes ISE
+     * Verifies that returning an object twice (without borrow in between) 
causes
+     * ISE
      * but does not re-validate or re-passivate the instance.
      *
      * JIRA: POOL-285
@@ -2440,7 +2506,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     // POOL-248
     @Test
     public void testMultipleReturnOfSameObject() throws Exception {
-        try (final GenericObjectPool<String, TestException> pool = new 
GenericObjectPool<>(simpleFactory, new GenericObjectPoolConfig<>())) {
+        try (final GenericObjectPool<String, TestException> pool = new 
GenericObjectPool<>(simpleFactory,
+                new GenericObjectPoolConfig<>())) {
 
             assertEquals(0, pool.getNumActive());
             assertEquals(0, pool.getNumIdle());
@@ -2489,7 +2556,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         genericObjectPool.setMaxTotal(-1);
         genericObjectPool.setBlockWhenExhausted(false);
         final String obj = genericObjectPool.borrowObject();
-        assertEquals(getNthObject(0),obj);
+        assertEquals(getNthObject(0), obj);
         genericObjectPool.returnObject(obj);
     }
 
@@ -2538,16 +2605,16 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
 
     /**
      * Verify that when a factory returns a null object, pool methods throw 
NPE.
+     * 
      * @throws InterruptedException
      */
     @Test
     @Timeout(value = 1000, unit = TimeUnit.MILLISECONDS)
     public void testNPEOnFactoryNull() throws InterruptedException {
         final DisconnectingWaiterFactory<String> factory = new 
DisconnectingWaiterFactory<>(
-            () -> null,  // Override default to always return null from 
makeObject
-            DisconnectingWaiterFactory.DEFAULT_DISCONNECTED_LIFECYCLE_ACTION,
-            DisconnectingWaiterFactory.DEFAULT_DISCONNECTED_VALIDATION_ACTION
-        );
+                () -> null, // Override default to always return null from 
makeObject
+                
DisconnectingWaiterFactory.DEFAULT_DISCONNECTED_LIFECYCLE_ACTION,
+                
DisconnectingWaiterFactory.DEFAULT_DISCONNECTED_VALIDATION_ACTION);
         try (GenericObjectPool<Waiter, IllegalStateException> pool = new 
GenericObjectPool<>(factory)) {
             pool.setTestOnBorrow(true);
             pool.setMaxTotal(-1);
@@ -2575,7 +2642,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         assertEquals(1, genericObjectPool.getNumIdle());
     }
 
-    @Test/* maxWaitMillis x2 + padding */
+    @Test /* maxWaitMillis x2 + padding */
     @Timeout(value = 1200, unit = TimeUnit.MILLISECONDS)
     public void testReturnBorrowObjectWithingMaxWaitMillis() throws Exception {
         final long maxWaitMillis = 500;
@@ -2586,7 +2653,8 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
             
createSlowObjectFactoryPool.setMaxWait(Duration.ofMillis(maxWaitMillis));
 
             // thread1 tries creating a slow object to make pool full.
-            final WaitingTestThread<InterruptedException> thread1 = new 
WaitingTestThread<>(createSlowObjectFactoryPool, 0);
+            final WaitingTestThread<InterruptedException> thread1 = new 
WaitingTestThread<>(createSlowObjectFactoryPool,
+                    0);
             thread1.start();
 
             // Wait for thread1's reaching to create().
@@ -2608,7 +2676,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
      * inserted just before the final call to isLifo() in the returnObject()
      * method.
      */
-    //@Test
+    // @Test
     @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
     public void testReturnObject() throws Exception {
 
@@ -2636,7 +2704,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
     @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
     public void testSetConfig() throws Exception {
         final GenericObjectPoolConfig<String> expected = new 
GenericObjectPoolConfig<>();
-        assertConfiguration(expected,genericObjectPool);
+        assertConfiguration(expected, genericObjectPool);
         expected.setMaxTotal(2);
         expected.setMaxIdle(3);
         expected.setMaxWait(Duration.ofMillis(5));
@@ -2649,7 +2717,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         expected.setDurationBetweenEvictionRuns(Duration.ofMillis(11L));
         expected.setBlockWhenExhausted(false);
         genericObjectPool.setConfig(expected);
-        assertConfiguration(expected,genericObjectPool);
+        assertConfiguration(expected, genericObjectPool);
     }
 
     @Test
@@ -2733,11 +2801,11 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         }
         {
             
genericObjectPool.setDurationBetweenEvictionRuns(Duration.ofMillis(11235));
-            
assertEquals(11235L,genericObjectPool.getDurationBetweenEvictionRuns().toMillis());
+            assertEquals(11235L, 
genericObjectPool.getDurationBetweenEvictionRuns().toMillis());
         }
         {
             
genericObjectPool.setSoftMinEvictableIdleDuration(Duration.ofMillis(12135));
-            
assertEquals(12135L,genericObjectPool.getSoftMinEvictableIdleDuration().toMillis());
+            assertEquals(12135L, 
genericObjectPool.getSoftMinEvictableIdleDuration().toMillis());
         }
         {
             genericObjectPool.setBlockWhenExhausted(true);
@@ -2769,7 +2837,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
             }
 
             // note that it stays populated
-            assertEquals(6,genericObjectPool.getNumIdle(),"Should have 6 
idle");
+            assertEquals(6, genericObjectPool.getNumIdle(), "Should have 6 
idle");
 
             // start the evictor
             
genericObjectPool.setDurationBetweenEvictionRuns(Duration.ofMillis(50));
@@ -2778,7 +2846,7 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
             Waiter.sleepQuietly(200L);
 
             // assert that the evictor has cleared out the pool
-            assertEquals(0,genericObjectPool.getNumIdle(),"Should have 0 
idle");
+            assertEquals(0, genericObjectPool.getNumIdle(), "Should have 0 
idle");
 
             // stop the evictor
             genericObjectPool.startEvictor(Duration.ZERO);
@@ -2867,10 +2935,11 @@ public class TestGenericObjectPool extends 
TestBaseObjectPool {
         // Should have one idle, one out now
         assertEquals(1, genericObjectPool.getNumIdle());
         assertEquals(1, genericObjectPool.getNumActive());
-       }
+    }
 
     /**
-     * Verify that threads waiting on a depleted pool get served when a 
returning object fails
+     * Verify that threads waiting on a depleted pool get served when a 
returning
+     * object fails
      * validation.
      *
      * JIRA: POOL-240
diff --git 
a/src/test/java/org/apache/commons/pool3/impl/TestResilientPooledObjectFactory.java
 
b/src/test/java/org/apache/commons/pool3/impl/TestResilientPooledObjectFactory.java
new file mode 100644
index 00000000..29b03665
--- /dev/null
+++ 
b/src/test/java/org/apache/commons/pool3/impl/TestResilientPooledObjectFactory.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.pool3.impl;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.util.UUID;
+
+import org.apache.commons.pool3.PooledObject;
+import org.apache.commons.pool3.PooledObjectFactory;
+import org.junit.jupiter.api.Test;
+
+public class TestResilientPooledObjectFactory {
+    @Test
+    public void testTransientFailure() throws Exception {
+        FailingFactory ff = new FailingFactory();
+        // Make the factory fail with exception immmmediately on make
+        ff.setHang(false);
+        ff.setSilentFail(false);
+        ResilientPooledObjectFactory<String, Exception> rf = new 
ResilientPooledObjectFactory<String, Exception>(ff,
+                5, Duration.ofMillis(100), Duration.ofMinutes(10), 
Duration.ofMillis(100));
+        // Create a pool with a max size of 2, using the resilient factory
+        GenericObjectPool<String, Exception> pool = new 
GenericObjectPool<>(rf);
+        pool.setMaxTotal(2);
+        pool.setBlockWhenExhausted(true);
+        pool.setTestOnReturn(true);
+        // Tell the factory what pool it is attached to (a little awkward)
+        rf.setPool(pool);
+        rf.startMonitor(Duration.ofMillis(20)); // 20ms monitor interval
+        // Base factory is up
+        assertTrue(rf.isUp());
+        // Check out a couple of objects
+        String s1 = pool.borrowObject();
+        String s2 = pool.borrowObject();
+        // Start a borrower that will wait
+        new Thread() {
+            @Override
+            public void run() {
+                try {
+                    pool.borrowObject();
+                } catch (Exception e) {
+                }
+            }
+        }.start();
+        // Wait for the borrower to join wait queue
+        try {
+            Thread.sleep(200);
+        } catch (InterruptedException e) {
+        }
+
+        // Crash the base factory
+        ff.crash();
+        // Resilient factory does not know the base factory is down until a 
make is
+        // attempted
+        assertTrue(rf.isUp());
+        assertEquals(1, pool.getNumWaiters());
+        pool.returnObject(s1);
+        pool.returnObject(s2);
+        assertEquals(0, pool.getNumIdle());
+        assertEquals(1, pool.getNumWaiters());
+        // Wait for the monitor to pick up the failed create which should 
happen on
+        // validation destroy
+        try {
+            Thread.sleep(100);
+        } catch (InterruptedException e) {
+        }
+        assertFalse(rf.isUp());
+        // Adder should be running, but failing
+        assertTrue(rf.isAdderRunning());
+
+        // Pool should have one take waiter
+        assertEquals(1, pool.getNumWaiters());
+
+        // Restart the factory
+        ff.recover();
+        // Wait for the adder to succeed
+        try {
+            Thread.sleep(100);
+        } catch (InterruptedException e) {
+        }
+        // Pool should have no waiters
+        assertTrue(pool.getNumWaiters() == 0);
+        // Add 5 objects to clear the rf log
+        // First have to expand the pool
+        pool.setMaxTotal(10);
+        for (int i = 0; i < 5; i++) {
+            pool.addObject();
+        }
+        // Wait for monitor to run
+        try {
+            Thread.sleep(200);
+        } catch (InterruptedException e) {
+        }
+        // rf should be up now
+        assertTrue(rf.isUp());
+
+        // Adder should be stopped
+        assertFalse(rf.isAdderRunning());
+
+        pool.close();
+        rf.stopMonitor();
+    }
+
+    @Test
+    public void testNulls() throws Exception {
+        FailingFactory ff = new FailingFactory();
+        // Make the factory fail with exception immmmediately on make
+        ff.setSilentFail(true);
+        ResilientPooledObjectFactory<String, Exception> rf = new 
ResilientPooledObjectFactory<String, Exception>(ff,
+                5, Duration.ofMillis(50), Duration.ofMinutes(10), 
Duration.ofMillis(50));
+        // Create a pool with a max size of 2, using the resilient factory
+        GenericObjectPool<String, Exception> pool = new 
GenericObjectPool<>(rf);
+        pool.setMaxTotal(2);
+        pool.setBlockWhenExhausted(true);
+        pool.setTestOnReturn(true);
+        // Tell the factory what pool it is attached to (a little awkward)
+        rf.setPool(pool);
+        rf.startMonitor(Duration.ofMillis(20)); // 20ms monitor interval
+
+        // Exhaust the pool
+        final String s1 = pool.borrowObject();
+        final String s2 = pool.borrowObject();
+        ff.crash();
+        // Start two borrowers that will block waiting
+        new Thread() {
+            @Override
+            public void run() {
+                try {
+                    final String s = pool.borrowObject();
+                    pool.returnObject(s);
+                } catch (Exception e) {
+                }
+            }
+        }.start();
+        new Thread() {
+            @Override
+            public void run() {
+                try {
+                    final String s = pool.borrowObject();
+                    pool.returnObject(s);
+                } catch (Exception e) {
+                }
+            }
+        }.start();
+        // Return borrowed objects - validation will fail
+        // Wait for the borrowers to get in the queue
+        try {
+            Thread.sleep(50);
+        } catch (InterruptedException e) {
+        }
+        pool.returnObject(s1);
+        pool.returnObject(s2);
+        assertEquals(0, pool.getNumIdle());
+        assertTrue(pool.getNumWaiters() > 0);
+        // Wait for the monitor to pick up the failed create which should 
happen on
+        // validation destroy
+        try {
+            Thread.sleep(200);
+        } catch (InterruptedException e) {
+        }
+        assertFalse(rf.isUp());
+        // Restart the factory
+        ff.recover();
+        // Wait for the adder to succeed
+        try {
+            Thread.sleep(200);
+        } catch (InterruptedException e) {
+        }
+        // Pool should have no waiters
+        assertEquals(0, pool.getNumWaiters());
+        pool.close();
+        // Wait for monitor to run
+        try {
+            Thread.sleep(200);
+        } catch (InterruptedException e) {
+        }
+        // Monitor and adder should be stopped by pool close
+        assertFalse(rf.isAdderRunning());
+        assertFalse(rf.isMonitorRunning());
+    }
+
+    @Test
+    public void testAdderStartStop() throws Exception {
+        FailingFactory ff = new FailingFactory();
+        // Make the factory fail with exception immmmediately on make
+        ff.setSilentFail(true);
+        ResilientPooledObjectFactory<String, Exception> rf = new 
ResilientPooledObjectFactory<String, Exception>(ff,
+                5, Duration.ofMillis(200), Duration.ofMinutes(10), 
Duration.ofMillis(20));
+        // Create a pool with a max size of 2, using the resilient factory
+        GenericObjectPool<String, Exception> pool = new 
GenericObjectPool<>(rf);
+        pool.setMaxTotal(2);
+        pool.setBlockWhenExhausted(true);
+        pool.setTestOnReturn(true);
+        rf.setPool(pool);
+        rf.startMonitor();
+        // Exhasut the pool
+        final String s1 = pool.borrowObject();
+        final String s2 = pool.borrowObject();
+        // Start a borrower that will block waiting
+        new Thread() {
+            @Override
+            public void run() {
+                try {
+                    final String s = pool.borrowObject();
+                } catch (Exception e) {
+                } finally {
+                    System.out.println("Borrower done");
+                }
+            }
+        }.start();
+        // Wait for the borrower to get in the queue
+        try {
+            Thread.sleep(50);
+        } catch (InterruptedException e) {
+        }
+        // Crash the base factory
+        ff.crash();
+        // Return object will create capacity in the pool
+        try {
+            pool.returnObject(s1);
+        } catch (Exception e) {
+        }
+        // Wait for the adder to run
+        try {
+            Thread.sleep(100);
+        } catch (InterruptedException e) {
+        }
+        // Adder should be running
+        assertTrue(rf.isAdderRunning());
+        // Restart the factory
+        ff.recover();
+        // Wait for the adder to succeed
+        try {
+            Thread.sleep(200);
+        } catch (InterruptedException e) {
+        }
+        // Pool should have no waiters
+        assertEquals(0, pool.getNumWaiters());
+        // Adder should still be running because there is a failure in the log
+        assertTrue(rf.isAdderRunning());
+        // Expand the pool - don't try this at home
+        pool.setMaxTotal(10);
+        // Run enough adds to homogenize the log
+        for (int i = 0; i < 6; i++) {
+            pool.addObject();
+        }
+        // Wait for the monitor to run
+        try {
+            Thread.sleep(200);
+        } catch (InterruptedException e) {
+        }
+        assertTrue(rf.isUp());
+        // Adder should be stopped
+        assertFalse(rf.isAdderRunning());
+    }
+
+    /**
+     * Factory that suffers outages and fails in configurable ways when it is 
down.
+     */
+    class FailingFactory implements PooledObjectFactory<String, Exception> {
+        /** Wheter or not the factory is up */
+        private boolean up = true;
+
+        /** Whether or not to fail silently */
+        private boolean silentFail = true;
+
+        /** Whether or not to hang */
+        private boolean hang = false;
+
+        public void crash() {
+            this.up = false;
+        }
+
+        public void recover() {
+            this.up = true;
+        }
+
+        public void setSilentFail(boolean silentFail) {
+            this.silentFail = silentFail;
+        }
+
+        public void setHang(boolean hang) {
+            this.hang = hang;
+        }
+
+        @Override
+        public PooledObject<String> makeObject() throws Exception {
+            if (up) {
+                return new 
DefaultPooledObject<String>(UUID.randomUUID().toString());
+            }
+            if (!silentFail) {
+                throw new Exception("makeObject failed");
+            }
+            if (hang) {
+                while (!up) {
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException e) {
+                    }
+                }
+            }
+            return null;
+        }
+
+        @Override
+        public void destroyObject(PooledObject<String> p) throws Exception {
+        }
+
+        @Override
+        public boolean validateObject(PooledObject<String> p) {
+            return up;
+        }
+
+        @Override
+        public void activateObject(PooledObject<String> p) throws Exception {
+        }
+
+        @Override
+        public void passivateObject(PooledObject<String> p) throws Exception {
+        }
+    }
+}
\ No newline at end of file

Reply via email to