This is an automated email from the ASF dual-hosted git repository. jlli pushed a commit to branch randomly-pick-endpoint in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 39f40f8e14553c7229376c1a09f70a286ca655cf Author: Jack Li(Analytics Engineering) <[email protected]> AuthorDate: Mon May 9 15:55:28 2022 -0700 Randomly pick the first URI for RoundRobinURIProvider --- .../pinot/common/utils/RoundRobinURIProvider.java | 4 +- .../common/utils/RoundRobinURIProviderTest.java | 149 +++++++++++---------- .../minion/tasks/SegmentConversionUtils.java | 2 +- 3 files changed, 83 insertions(+), 72 deletions(-) diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/RoundRobinURIProvider.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/RoundRobinURIProvider.java index 36f837e5ff..39fe142ea2 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/utils/RoundRobinURIProvider.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/RoundRobinURIProvider.java @@ -23,6 +23,7 @@ import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; +import java.util.Random; import org.apache.http.client.utils.URIBuilder; @@ -33,7 +34,7 @@ import org.apache.http.client.utils.URIBuilder; public class RoundRobinURIProvider { private final URI[] _uris; - private int _index = 0; + private int _index; public RoundRobinURIProvider(URI originalUri) throws UnknownHostException, URISyntaxException { @@ -50,6 +51,7 @@ public class RoundRobinURIProvider { _uris[i] = uriBuilder.setHost(ip).build(); } } + _index = new Random().nextInt(_uris.length); } public int numAddresses() { diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/RoundRobinURIProviderTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/RoundRobinURIProviderTest.java index edd1c90af5..b73fe1a6c6 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/utils/RoundRobinURIProviderTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/RoundRobinURIProviderTest.java @@ -23,6 +23,8 @@ import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.List; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.testng.Assert; @@ -50,128 +52,135 @@ public class RoundRobinURIProviderTest { mock.when(() -> InetAddress.getAllByName("testweb.com")).thenReturn(testWebAddresses); TestCase[] testCases = new TestCase[]{ - new TestCase("http://127.0.0.1";, new String[]{"http://127.0.0.1"}), - new TestCase("http://127.0.0.1/";, new String[]{"http://127.0.0.1/"}), - new TestCase("http://127.0.0.1/?";, new String[]{"http://127.0.0.1/?"}), - new TestCase("http://127.0.0.1/?it=5";, new String[]{"http://127.0.0.1/?it=5"}), - new TestCase("http://127.0.0.1/me/out?it=5";, new String[]{"http://127.0.0.1/me/out?it=5"}), - new TestCase("http://127.0.0.1:20000";, new String[]{"http://127.0.0.1:20000"}), - new TestCase("http://127.0.0.1:20000/";, new String[]{"http://127.0.0.1:20000/"}), - new TestCase("http://127.0.0.1:20000/?";, new String[]{"http://127.0.0.1:20000/?"}), - new TestCase("http://127.0.0.1:20000/?it=5";, new String[]{"http://127.0.0.1:20000/?it=5"}), - new TestCase("http://127.0.0.1:20000/me/out?it=5";, new String[]{"http://127.0.0.1:20000/me/out?it=5"}), + new TestCase("http://127.0.0.1";, Arrays.asList("http://127.0.0.1";)), + new TestCase("http://127.0.0.1/";, Arrays.asList("http://127.0.0.1/";)), + new TestCase("http://127.0.0.1/?";, Arrays.asList("http://127.0.0.1/?";)), + new TestCase("http://127.0.0.1/?it=5";, Arrays.asList("http://127.0.0.1/?it=5";)), + new TestCase("http://127.0.0.1/me/out?it=5";, Arrays.asList("http://127.0.0.1/me/out?it=5";)), + new TestCase("http://127.0.0.1:20000";, Arrays.asList("http://127.0.0.1:20000";)), + new TestCase("http://127.0.0.1:20000/";, Arrays.asList("http://127.0.0.1:20000/";)), + new TestCase("http://127.0.0.1:20000/?";, Arrays.asList("http://127.0.0.1:20000/?";)), + new TestCase("http://127.0.0.1:20000/?it=5";, Arrays.asList("http://127.0.0.1:20000/?it=5";)), + new TestCase("http://127.0.0.1:20000/me/out?it=5";, Arrays.asList("http://127.0.0.1:20000/me/out?it=5";)), - new TestCase("http://localhost";, new String[]{"http://127.0.0.1";, "http://[0:0:0:0:0:0:0:1]"}), - new TestCase("http://localhost/";, new String[]{"http://127.0.0.1/";, "http://[0:0:0:0:0:0:0:1]/"}), - new TestCase("http://localhost/?";, new String[]{"http://127.0.0.1/?";, "http://[0:0:0:0:0:0:0:1]/?"}), + new TestCase("http://localhost";, Arrays.asList("http://127.0.0.1";, "http://[0:0:0:0:0:0:0:1]";)), + new TestCase("http://localhost/";, Arrays.asList("http://127.0.0.1/";, "http://[0:0:0:0:0:0:0:1]/";)), + new TestCase("http://localhost/?";, Arrays.asList("http://127.0.0.1/?";, "http://[0:0:0:0:0:0:0:1]/?";)), new TestCase("http://localhost/?it=5";, - new String[]{"http://127.0.0.1/?it=5";, "http://[0:0:0:0:0:0:0:1]/?it=5"}), + Arrays.asList("http://127.0.0.1/?it=5";, "http://[0:0:0:0:0:0:0:1]/?it=5";)), new TestCase("http://localhost/me/out?it=5";, - new String[]{"http://127.0.0.1/me/out?it=5";, "http://[0:0:0:0:0:0:0:1]/me/out?it=5"}), + Arrays.asList("http://127.0.0.1/me/out?it=5";, "http://[0:0:0:0:0:0:0:1]/me/out?it=5";)), new TestCase("http://localhost:20000";, - new String[]{"http://127.0.0.1:20000";, "http://[0:0:0:0:0:0:0:1]:20000"}), + Arrays.asList("http://127.0.0.1:20000";, "http://[0:0:0:0:0:0:0:1]:20000";)), new TestCase("http://localhost:20000/";, - new String[]{"http://127.0.0.1:20000/";, "http://[0:0:0:0:0:0:0:1]:20000/"}), + Arrays.asList("http://127.0.0.1:20000/";, "http://[0:0:0:0:0:0:0:1]:20000/";)), new TestCase("http://localhost:20000/?";, - new String[]{"http://127.0.0.1:20000/?";, "http://[0:0:0:0:0:0:0:1]:20000/?"}), + Arrays.asList("http://127.0.0.1:20000/?";, "http://[0:0:0:0:0:0:0:1]:20000/?";)), new TestCase("http://localhost:20000/?it=5";, - new String[]{"http://127.0.0.1:20000/?it=5";, "http://[0:0:0:0:0:0:0:1]:20000/?it=5"}), + Arrays.asList("http://127.0.0.1:20000/?it=5";, "http://[0:0:0:0:0:0:0:1]:20000/?it=5";)), new TestCase("http://localhost:20000/me/out?it=5";, - new String[]{"http://127.0.0.1:20000/me/out?it=5";, "http://[0:0:0:0:0:0:0:1]:20000/me/out?it=5"}), + Arrays.asList("http://127.0.0.1:20000/me/out?it=5";, "http://[0:0:0:0:0:0:0:1]:20000/me/out?it=5";)), new TestCase("http://testweb.com";, - new String[]{"http://192.168.3.1";, "http://192.168.3.2";, "http://192.168.3.3"}), + Arrays.asList("http://192.168.3.1";, "http://192.168.3.2";, "http://192.168.3.3";)), new TestCase("http://testweb.com/";, - new String[]{"http://192.168.3.1/";, "http://192.168.3.2/";, "http://192.168.3.3/"}), + Arrays.asList("http://192.168.3.1/";, "http://192.168.3.2/";, "http://192.168.3.3/";)), new TestCase("http://testweb.com/?";, - new String[]{"http://192.168.3.1/?";, "http://192.168.3.2/?";, "http://192.168.3.3/?"}), + Arrays.asList("http://192.168.3.1/?";, "http://192.168.3.2/?";, "http://192.168.3.3/?";)), new TestCase("http://testweb.com/?it=5";, - new String[]{"http://192.168.3.1/?it=5";, "http://192.168.3.2/?it=5";, "http://192.168.3.3/?it=5"}), + Arrays.asList("http://192.168.3.1/?it=5";, "http://192.168.3.2/?it=5";, "http://192.168.3.3/?it=5";)), new TestCase("http://testweb.com/me/out?it=5";, - new String[]{"http://192.168.3.1/me/out?it=5";, "http://192.168.3.2/me/out?it=5";, - "http://192.168.3.3/me/out?it=5"}), + Arrays.asList("http://192.168.3.1/me/out?it=5";, "http://192.168.3.2/me/out?it=5";, + "http://192.168.3.3/me/out?it=5";)), new TestCase("http://testweb.com:20000";, - new String[]{"http://192.168.3.1:20000";, "http://192.168.3.2:20000";, "http://192.168.3.3:20000"}), + Arrays.asList("http://192.168.3.1:20000";, "http://192.168.3.2:20000";, "http://192.168.3.3:20000";)), new TestCase("http://testweb.com:20000/";, - new String[]{"http://192.168.3.1:20000/";, "http://192.168.3.2:20000/";, "http://192.168.3.3:20000/"}), + Arrays.asList("http://192.168.3.1:20000/";, "http://192.168.3.2:20000/";, "http://192.168.3.3:20000/";)), new TestCase("http://testweb.com:20000/?";, - new String[]{"http://192.168.3.1:20000/?";, "http://192.168.3.2:20000/?";, "http://192.168.3.3:20000/?"}), + Arrays.asList("http://192.168.3.1:20000/?";, "http://192.168.3.2:20000/?";, "http://192.168.3.3:20000/?";)), new TestCase("http://testweb.com:20000/?it=5";, - new String[]{"http://192.168.3.1:20000/?it=5";, "http://192.168.3.2:20000/?it=5";, - "http://192.168.3.3:20000/?it=5"}), + Arrays.asList("http://192.168.3.1:20000/?it=5";, "http://192.168.3.2:20000/?it=5";, + "http://192.168.3.3:20000/?it=5";)), new TestCase("http://testweb.com:20000/me/out?it=5";, - new String[]{"http://192.168.3.1:20000/me/out?it=5";, "http://192.168.3.2:20000/me/out?it=5";, - "http://192.168.3.3:20000/me/out?it=5"}), + Arrays.asList("http://192.168.3.1:20000/me/out?it=5";, "http://192.168.3.2:20000/me/out?it=5";, + "http://192.168.3.3:20000/me/out?it=5";)), - new TestCase("https://127.0.0.1";, new String[]{"https://127.0.0.1"}), - new TestCase("https://127.0.0.1/";, new String[]{"https://127.0.0.1/"}), - new TestCase("https://127.0.0.1/?";, new String[]{"https://127.0.0.1/?"}), - new TestCase("https://127.0.0.1/?it=5";, new String[]{"https://127.0.0.1/?it=5"}), - new TestCase("https://127.0.0.1/me/out?it=5";, new String[]{"https://127.0.0.1/me/out?it=5"}), - new TestCase("https://127.0.0.1:20000";, new String[]{"https://127.0.0.1:20000"}), - new TestCase("https://127.0.0.1:20000/";, new String[]{"https://127.0.0.1:20000/"}), - new TestCase("https://127.0.0.1:20000/?";, new String[]{"https://127.0.0.1:20000/?"}), - new TestCase("https://127.0.0.1:20000/?it=5";, new String[]{"https://127.0.0.1:20000/?it=5"}), + new TestCase("https://127.0.0.1";, Arrays.asList("https://127.0.0.1";)), + new TestCase("https://127.0.0.1/";, Arrays.asList("https://127.0.0.1/";)), + new TestCase("https://127.0.0.1/?";, Arrays.asList("https://127.0.0.1/?";)), + new TestCase("https://127.0.0.1/?it=5";, Arrays.asList("https://127.0.0.1/?it=5";)), + new TestCase("https://127.0.0.1/me/out?it=5";, Arrays.asList("https://127.0.0.1/me/out?it=5";)), + new TestCase("https://127.0.0.1:20000";, Arrays.asList("https://127.0.0.1:20000";)), + new TestCase("https://127.0.0.1:20000/";, Arrays.asList("https://127.0.0.1:20000/";)), + new TestCase("https://127.0.0.1:20000/?";, Arrays.asList("https://127.0.0.1:20000/?";)), + new TestCase("https://127.0.0.1:20000/?it=5";, Arrays.asList("https://127.0.0.1:20000/?it=5";)), new TestCase("https://127.0.0.1:20000/me/out?it=5";, - new String[]{"https://127.0.0.1:20000/me/out?it=5"}), + Arrays.asList("https://127.0.0.1:20000/me/out?it=5";)), - new TestCase("https://localhost";, new String[]{"https://127.0.0.1";, "https://[0:0:0:0:0:0:0:1]"}), - new TestCase("https://localhost/";, new String[]{"https://127.0.0.1/";, "https://[0:0:0:0:0:0:0:1]/"}), - new TestCase("https://localhost/?";, new String[]{"https://127.0.0.1/?";, "https://[0:0:0:0:0:0:0:1]/?"}), + new TestCase("https://localhost";, Arrays.asList("https://127.0.0.1";, "https://[0:0:0:0:0:0:0:1]";)), + new TestCase("https://localhost/";, Arrays.asList("https://127.0.0.1/";, "https://[0:0:0:0:0:0:0:1]/";)), + new TestCase("https://localhost/?";, Arrays.asList("https://127.0.0.1/?";, "https://[0:0:0:0:0:0:0:1]/?";)), new TestCase("https://localhost/?it=5";, - new String[]{"https://127.0.0.1/?it=5";, "https://[0:0:0:0:0:0:0:1]/?it=5"}), + Arrays.asList("https://127.0.0.1/?it=5";, "https://[0:0:0:0:0:0:0:1]/?it=5";)), new TestCase("https://localhost/me/out?it=5";, - new String[]{"https://127.0.0.1/me/out?it=5";, "https://[0:0:0:0:0:0:0:1]/me/out?it=5"}), + Arrays.asList("https://127.0.0.1/me/out?it=5";, "https://[0:0:0:0:0:0:0:1]/me/out?it=5";)), new TestCase("https://localhost:20000";, - new String[]{"https://127.0.0.1:20000";, "https://[0:0:0:0:0:0:0:1]:20000"}), + Arrays.asList("https://127.0.0.1:20000";, "https://[0:0:0:0:0:0:0:1]:20000";)), new TestCase("https://localhost:20000/";, - new String[]{"https://127.0.0.1:20000/";, "https://[0:0:0:0:0:0:0:1]:20000/"}), + Arrays.asList("https://127.0.0.1:20000/";, "https://[0:0:0:0:0:0:0:1]:20000/";)), new TestCase("https://localhost:20000/?";, - new String[]{"https://127.0.0.1:20000/?";, "https://[0:0:0:0:0:0:0:1]:20000/?"}), + Arrays.asList("https://127.0.0.1:20000/?";, "https://[0:0:0:0:0:0:0:1]:20000/?";)), new TestCase("https://localhost:20000/?it=5";, - new String[]{"https://127.0.0.1:20000/?it=5";, "https://[0:0:0:0:0:0:0:1]:20000/?it=5"}), + Arrays.asList("https://127.0.0.1:20000/?it=5";, "https://[0:0:0:0:0:0:0:1]:20000/?it=5";)), new TestCase("https://testweb.com";, - new String[]{"https://192.168.3.1";, "https://192.168.3.2";, "https://192.168.3.3"}), + Arrays.asList("https://192.168.3.1";, "https://192.168.3.2";, "https://192.168.3.3";)), new TestCase("https://testweb.com/";, - new String[]{"https://192.168.3.1/";, "https://192.168.3.2/";, "https://192.168.3.3/"}), + Arrays.asList("https://192.168.3.1/";, "https://192.168.3.2/";, "https://192.168.3.3/";)), new TestCase("https://testweb.com/?";, - new String[]{"https://192.168.3.1/?";, "https://192.168.3.2/?";, "https://192.168.3.3/?"}), + Arrays.asList("https://192.168.3.1/?";, "https://192.168.3.2/?";, "https://192.168.3.3/?";)), new TestCase("https://testweb.com/?it=5";, - new String[]{"https://192.168.3.1/?it=5";, "https://192.168.3.2/?it=5";, "https://192.168.3.3/?it=5"}), + Arrays.asList("https://192.168.3.1/?it=5";, "https://192.168.3.2/?it=5";, "https://192.168.3.3/?it=5";)), new TestCase("https://testweb.com/me/out?it=5";, - new String[]{"https://192.168.3.1/me/out?it=5";, "https://192.168.3.2/me/out?it=5";, - "https://192.168.3.3/me/out?it=5"}), + Arrays.asList("https://192.168.3.1/me/out?it=5";, "https://192.168.3.2/me/out?it=5";, + "https://192.168.3.3/me/out?it=5";)), new TestCase("https://testweb.com:20000";, - new String[]{"https://192.168.3.1:20000";, "https://192.168.3.2:20000";, "https://192.168.3.3:20000"}), + Arrays.asList("https://192.168.3.1:20000";, "https://192.168.3.2:20000";, "https://192.168.3.3:20000";)), new TestCase("https://testweb.com:20000/";, - new String[]{"https://192.168.3.1:20000/";, "https://192.168.3.2:20000/";, "https://192.168.3.3:20000/"}), + Arrays.asList("https://192.168.3.1:20000/";, "https://192.168.3.2:20000/";, "https://192.168.3.3:20000/";)), new TestCase("https://testweb.com:20000/?";, - new String[]{"https://192.168.3.1:20000/?";, "https://192.168.3.2:20000/?";, "https://192.168.3.3:20000/?"}), + Arrays.asList("https://192.168.3.1:20000/?";, "https://192.168.3.2:20000/?";, "https://192.168.3.3:20000/?";)), new TestCase("https://testweb.com:20000/?it=5";, - new String[]{"https://192.168.3.1:20000/?it=5";, "https://192.168.3.2:20000/?it=5";, - "https://192.168.3.3:20000/?it=5"}), + Arrays.asList("https://192.168.3.1:20000/?it=5";, "https://192.168.3.2:20000/?it=5";, + "https://192.168.3.3:20000/?it=5";)), new TestCase("https://testweb.com:20000/me/out?it=5";, - new String[]{"https://192.168.3.1:20000/me/out?it=5";, "https://192.168.3.2:20000/me/out?it=5";, - "https://192.168.3.3:20000/me/out?it=5"}), + Arrays.asList("https://192.168.3.1:20000/me/out?it=5";, "https://192.168.3.2:20000/me/out?it=5";, + "https://192.168.3.3:20000/me/out?it=5";)), }; for (TestCase testCase : testCases) { String uri = testCase._originalUri; RoundRobinURIProvider uriProvider = new RoundRobinURIProvider(new URI(uri)); - int n = testCase._expectedUris.length; + int n = testCase._expectedUris.size(); + int previousIndex = -1; + int currentIndex; for (int i = 0; i < 2 * n; i++) { - String expectedUri = testCase._expectedUris[i % n]; - Assert.assertEquals(uriProvider.next().toString(), expectedUri); + String actualUri = uriProvider.next().toString(); + currentIndex = testCase._expectedUris.indexOf(actualUri); + Assert.assertTrue(currentIndex != -1); + if (previousIndex != -1) { + Assert.assertEquals((previousIndex + 1) % n, currentIndex); + } + previousIndex = currentIndex; } } } static class TestCase { String _originalUri; - String[] _expectedUris; + List<String> _expectedUris; - TestCase(String originalUri, String[] expectedUris) { + TestCase(String originalUri, List<String> expectedUris) { _originalUri = originalUri; _expectedUris = expectedUris; } diff --git a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/SegmentConversionUtils.java b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/SegmentConversionUtils.java index cf07398325..87ede0f3e4 100644 --- a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/SegmentConversionUtils.java +++ b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/SegmentConversionUtils.java @@ -63,7 +63,7 @@ public class SegmentConversionUtils { public static void uploadSegment(Map<String, String> configs, List<Header> httpHeaders, List<NameValuePair> parameters, String tableNameWithType, String segmentName, String uploadURL, File fileToUpload) throws Exception { - // Create a RoundRobinURIProvider to round robin IP addresses when retry uploading. Otherwise may always try to + // Create a RoundRobinURIProvider to round-robin IP addresses when retry uploading. Otherwise, it may always try to // upload to a same broken host as: 1) DNS may not RR the IP addresses 2) OS cache the DNS resolution result. RoundRobinURIProvider uriProvider = new RoundRobinURIProvider(new URI(uploadURL)); // Generate retry policy based on the config --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
