This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 59bf3827db By default start 1 minion for all quickstart (#8682)
59bf3827db is described below
commit 59bf3827dbdf54cdd93ea92cbd0f69e5e1eebba8
Author: Xiang Fu <[email protected]>
AuthorDate: Wed May 11 10:36:29 2022 -0700
By default start 1 minion for all quickstart (#8682)
---
.github/workflows/scripts/.pinot_quickstart.sh | 33 +----
pinot-tools/pom.xml | 10 --
.../pinot/tools/BatchQuickstartWithMinion.java | 49 -------
.../org/apache/pinot/tools/EmptyQuickstart.java | 2 +-
.../org/apache/pinot/tools/GenericQuickstart.java | 2 +-
.../apache/pinot/tools/GitHubEventsQuickstart.java | 2 +-
.../org/apache/pinot/tools/HybridQuickstart.java | 2 +-
.../org/apache/pinot/tools/JoinQuickStart.java | 2 +-
.../apache/pinot/tools/JsonIndexQuickStart.java | 2 +-
.../OfflineComplexTypeHandlingQuickStart.java | 2 +-
.../pinot/tools/PartialUpsertQuickStart.java | 2 +-
.../java/org/apache/pinot/tools/Quickstart.java | 9 +-
.../RealtimeComplexTypeHandlingQuickStart.java | 2 +-
.../pinot/tools/RealtimeJsonIndexQuickStart.java | 2 +-
.../org/apache/pinot/tools/RealtimeQuickStart.java | 2 +-
.../pinot/tools/TimestampIndexQuickstart.java | 2 +-
.../apache/pinot/tools/UpsertJsonQuickStart.java | 2 +-
.../org/apache/pinot/tools/UpsertQuickStart.java | 2 +-
.../tools/admin/command/QuickstartRunner.java | 44 ------
.../tools/admin/command/TestQuickStartCommand.java | 159 +++++++++++----------
20 files changed, 98 insertions(+), 234 deletions(-)
diff --git a/.github/workflows/scripts/.pinot_quickstart.sh
b/.github/workflows/scripts/.pinot_quickstart.sh
index 72f93c34f0..f27ecc8dcf 100755
--- a/.github/workflows/scripts/.pinot_quickstart.sh
+++ b/.github/workflows/scripts/.pinot_quickstart.sh
@@ -120,7 +120,8 @@ if [ $? -ne 0 ]; then
fi
# Ingest Data
-bin/pinot-admin.sh LaunchDataIngestionJob -jobSpecFile
examples/batch/baseballStats/ingestionJobSpec.yaml
+d=`pwd`
+INSERT_INTO_RES=`curl -X POST --header 'Content-Type: application/json' -d
"{\"sql\":\"INSERT INTO baseballStats FROM FILE
'${d}/examples/batch/baseballStats/rawdata'\",\"trace\":false}"
http://localhost:8099/query/sql`
if [ $? -ne 0 ]; then
echo 'Failed to ingest data for table baseballStats.'
exit 1
@@ -179,36 +180,6 @@ if [ "${PASS}" -eq 0 ]; then
exit 1
fi
-# Test quick-start-batch-with-minion
-bin/quick-start-batch-with-minion.sh &
-PID=$!
-
-# Print the JVM settings
-jps -lvm
-
-PASS=0
-
-# Wait for 1 minute for table to be set up, then at most 5 minutes to reach
the desired state
-sleep 60
-for i in $(seq 1 150)
-do
- QUERY_RES=`curl -X POST --header 'Accept: application/json' -d
'{"sql":"select count(*) from baseballStats limit 1","trace":false}'
http://localhost:8000/query/sql`
- if [ $? -eq 0 ]; then
- COUNT_STAR_RES=`echo "${QUERY_RES}" | jq '.resultTable.rows[0][0]'`
- if [[ "${COUNT_STAR_RES}" =~ ^[0-9]+$ ]] && [ "${COUNT_STAR_RES}" -eq
97889 ]; then
- PASS=1
- break
- fi
- fi
- sleep 2
-done
-
-cleanup "${PID}"
-if [ "${PASS}" -eq 0 ]; then
- echo 'Batch Quickstart with Minion failed: Cannot get correct result for
count star query.'
- exit 1
-fi
-
# Test quick-start-streaming
# TODO: Streaming test is disabled because Meetup RSVP stream is retired. Find
a replacement and re-enable this test.
#bin/quick-start-streaming.sh &
diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml
index 956d201729..e1e8ca0b94 100644
--- a/pinot-tools/pom.xml
+++ b/pinot-tools/pom.xml
@@ -377,16 +377,6 @@
</extraArguments>
</jvmSettings>
</program>
- <program>
-
<mainClass>org.apache.pinot.tools.BatchQuickstartWithMinion</mainClass>
- <name>quick-start-batch-with-minion</name>
- <jvmSettings>
- <initialMemorySize>4G</initialMemorySize>
- <extraArguments>
-
<extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
- </extraArguments>
- </jvmSettings>
- </program>
<program>
<mainClass>org.apache.pinot.tools.UpsertQuickStart</mainClass>
<name>quick-start-upsert-streaming</name>
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/BatchQuickstartWithMinion.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/BatchQuickstartWithMinion.java
deleted file mode 100644
index 78dc089716..0000000000
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/BatchQuickstartWithMinion.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.tools;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import org.apache.pinot.tools.admin.PinotAdministrator;
-
-
-public class BatchQuickstartWithMinion extends Quickstart {
- @Override
- public List<String> types() {
- return Arrays.asList("OFFLINE_MINION", "BATCH_MINION", "OFFLINE-MINION",
"BATCH-MINION");
- }
-
- public String getBootstrapDataDir() {
- return "examples/minions/batch/baseballStats";
- }
-
- @Override
- public int getNumMinions() {
- return 1;
- }
-
- public static void main(String[] args)
- throws Exception {
- List<String> arguments = new ArrayList<>();
- arguments.addAll(Arrays.asList("QuickStart", "-type", "BATCH-MINION"));
- arguments.addAll(Arrays.asList(args));
- PinotAdministrator.main(arguments.toArray(new String[arguments.size()]));
- }
-}
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/EmptyQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/EmptyQuickstart.java
index 08586ed5d5..d9830a657a 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/EmptyQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/EmptyQuickstart.java
@@ -47,7 +47,7 @@ public class EmptyQuickstart extends QuickStartBase {
}
QuickstartRunner runner =
- new QuickstartRunner(new ArrayList<>(), 1, 1, 1, 0,
+ new QuickstartRunner(new ArrayList<>(), 1, 1, 1, 1,
dataDir, true, getAuthToken(), getConfigOverrides(),
_zkExternalAddress, false);
if (_zkExternalAddress != null) {
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/GenericQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/GenericQuickstart.java
index 400e58a2a2..b0e8ac3069 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/GenericQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/GenericQuickstart.java
@@ -98,7 +98,7 @@ public class GenericQuickstart extends QuickStartBase {
Preconditions.checkState(tempDir.mkdirs());
QuickstartTableRequest request = new
QuickstartTableRequest(_tableDirectory.getAbsolutePath());
final QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, tempDir,
getConfigOverrides());
+ new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, tempDir,
getConfigOverrides());
printStatus(Color.CYAN, "***** Starting Kafka *****");
startKafka();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/GitHubEventsQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/GitHubEventsQuickstart.java
index 50c162d0fd..47a1a9417f 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/GitHubEventsQuickstart.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/GitHubEventsQuickstart.java
@@ -124,7 +124,7 @@ public class GitHubEventsQuickstart extends QuickStartBase {
Preconditions.checkState(tempDir.mkdirs());
QuickstartTableRequest request = new
QuickstartTableRequest(quickStartDataDir.getAbsolutePath());
final QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, tempDir,
getConfigOverrides());
+ new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, tempDir,
getConfigOverrides());
printStatus(Color.CYAN, String.format("***** Starting %s *****",
streamSourceType));
startStreamServer();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
index 61ec1b3a9a..a339292b58 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
@@ -124,7 +124,7 @@ public class HybridQuickstart extends QuickStartBase {
Preconditions.checkState(dataDir.mkdirs());
QuickstartTableRequest bootstrapTableRequest =
prepareTableRequest(baseDir);
final QuickstartRunner runner = new
QuickstartRunner(Lists.newArrayList(bootstrapTableRequest),
- 1, 1, 1, 0, dataDir, getConfigOverrides());
+ 1, 1, 1, 1, dataDir, getConfigOverrides());
printStatus(Color.YELLOW, "***** Starting Kafka *****");
startKafka();
printStatus(Color.YELLOW, "***** Starting airline data stream and
publishing to Kafka *****");
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
index 3270b6699f..c6d9c3bdd0 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
@@ -81,7 +81,7 @@ public class JoinQuickStart extends QuickStartBase {
File tempDir = new File(quickstartTmpDir, "tmp");
FileUtils.forceMkdir(tempDir);
QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request, dimTableRequest), 1,
1, 3, 0, tempDir, getConfigOverrides());
+ new QuickstartRunner(Lists.newArrayList(request, dimTableRequest), 1,
1, 3, 1, tempDir, getConfigOverrides());
printStatus(Quickstart.Color.CYAN, "***** Starting Zookeeper, controller,
broker and server *****");
runner.startAll();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java
index 80ec6d4b80..035bee90e4 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java
@@ -64,7 +64,7 @@ public class JsonIndexQuickStart extends QuickStartBase {
QuickstartTableRequest request = new
QuickstartTableRequest(baseDir.getAbsolutePath());
final QuickstartRunner runner =
- new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, 0,
dataDir, getConfigOverrides());
+ new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, 1,
dataDir, getConfigOverrides());
printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and
server *****");
runner.startAll();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/OfflineComplexTypeHandlingQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/OfflineComplexTypeHandlingQuickStart.java
index 69f07b6805..c5cd73c819 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/OfflineComplexTypeHandlingQuickStart.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/OfflineComplexTypeHandlingQuickStart.java
@@ -66,7 +66,7 @@ public class OfflineComplexTypeHandlingQuickStart extends
QuickStartBase {
QuickstartTableRequest request = new
QuickstartTableRequest(baseDir.getAbsolutePath());
final QuickstartRunner runner =
- new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, 0,
dataDir, getConfigOverrides());
+ new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, 1,
dataDir, getConfigOverrides());
printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and
server *****");
runner.startAll();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/PartialUpsertQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/PartialUpsertQuickStart.java
index e9ec4fcc77..19fd63e72b 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/PartialUpsertQuickStart.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/PartialUpsertQuickStart.java
@@ -73,7 +73,7 @@ public class PartialUpsertQuickStart extends QuickStartBase {
QuickstartTableRequest request = new
QuickstartTableRequest(bootstrapTableDir.getAbsolutePath());
final QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, dataDir,
getConfigOverrides());
+ new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, dataDir,
getConfigOverrides());
printStatus(Color.CYAN, "***** Starting Kafka *****");
final ZkStarter.ZookeeperInstance zookeeperInstance =
ZkStarter.startLocalZkServer();
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
index 01fb8fb716..c3cd2b40ac 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
@@ -40,7 +40,7 @@ public class Quickstart extends QuickStartBase {
private static final String TAB = "\t\t";
private static final String NEW_LINE = "\n";
- private static final String DEFAULT_BOOTSTRAP_DIRECTORY =
"examples/batch/baseballStats";
+ private static final String DEFAULT_BOOTSTRAP_DIRECTORY =
"examples/minions/batch/baseballStats";
public enum Color {
RESET("\u001B[0m"), GREEN("\u001B[32m"), YELLOW("\u001B[33m"),
CYAN("\u001B[36m");
@@ -56,10 +56,6 @@ public class Quickstart extends QuickStartBase {
}
}
- public int getNumMinions() {
- return 0;
- }
-
public String getAuthToken() {
return null;
}
@@ -103,8 +99,7 @@ public class Quickstart extends QuickStartBase {
QuickstartTableRequest request = new
QuickstartTableRequest(baseDir.getAbsolutePath());
QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1,
- getNumMinions(), dataDir, true, getAuthToken(),
+ new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, dataDir,
true, getAuthToken(),
getConfigOverrides(), null, true);
printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and
server *****");
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeComplexTypeHandlingQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeComplexTypeHandlingQuickStart.java
index 88106d3612..2d9e6f7d09 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeComplexTypeHandlingQuickStart.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeComplexTypeHandlingQuickStart.java
@@ -75,7 +75,7 @@ public class RealtimeComplexTypeHandlingQuickStart extends
QuickStartBase {
QuickstartTableRequest request = new
QuickstartTableRequest(baseDir.getAbsolutePath());
QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, dataDir,
getConfigOverrides());
+ new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, dataDir,
getConfigOverrides());
printStatus(Color.CYAN, "***** Starting Kafka *****");
ZkStarter.ZookeeperInstance zookeeperInstance =
ZkStarter.startLocalZkServer();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeJsonIndexQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeJsonIndexQuickStart.java
index 94aadea9e8..e77d611fb0 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeJsonIndexQuickStart.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeJsonIndexQuickStart.java
@@ -74,7 +74,7 @@ public class RealtimeJsonIndexQuickStart extends
QuickStartBase {
QuickstartTableRequest request = new
QuickstartTableRequest(baseDir.getAbsolutePath());
QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, dataDir,
getConfigOverrides());
+ new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, dataDir,
getConfigOverrides());
printStatus(Color.CYAN, "***** Starting Kafka *****");
ZkStarter.ZookeeperInstance zookeeperInstance =
ZkStarter.startLocalZkServer();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
index 975eeb310b..626961c1ee 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
@@ -74,7 +74,7 @@ public class RealtimeQuickStart extends QuickStartBase {
QuickstartTableRequest request = new
QuickstartTableRequest(baseDir.getAbsolutePath());
final QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, dataDir,
getConfigOverrides());
+ new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, dataDir,
getConfigOverrides());
printStatus(Color.CYAN, "***** Starting Kafka *****");
final ZkStarter.ZookeeperInstance zookeeperInstance =
ZkStarter.startLocalZkServer();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/TimestampIndexQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/TimestampIndexQuickstart.java
index 95c59f1bd8..a8be24defc 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/TimestampIndexQuickstart.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/TimestampIndexQuickstart.java
@@ -79,7 +79,7 @@ public class TimestampIndexQuickstart extends QuickStartBase {
Preconditions.checkState(dataDir.mkdirs());
QuickstartTableRequest bootstrapTableRequest =
prepareTableRequest(baseDir);
final QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(bootstrapTableRequest), 1, 1,
1, 0, dataDir, getConfigOverrides());
+ new QuickstartRunner(Lists.newArrayList(bootstrapTableRequest), 1, 1,
1, 1, dataDir, getConfigOverrides());
printStatus(Color.YELLOW, "***** Starting Zookeeper, 1 servers, 1 brokers
and 1 controller *****");
runner.startAll();
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/UpsertJsonQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/UpsertJsonQuickStart.java
index b257d491a4..383d2593ec 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/UpsertJsonQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/UpsertJsonQuickStart.java
@@ -75,7 +75,7 @@ public class UpsertJsonQuickStart extends QuickStartBase {
QuickstartTableRequest request = new
QuickstartTableRequest(baseDir.getAbsolutePath());
QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, dataDir,
getConfigOverrides());
+ new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, dataDir,
getConfigOverrides());
printStatus(Color.CYAN, "***** Starting Kafka *****");
ZkStarter.ZookeeperInstance zookeeperInstance =
ZkStarter.startLocalZkServer();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/UpsertQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/UpsertQuickStart.java
index e17de70c09..ccf219325d 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/UpsertQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/UpsertQuickStart.java
@@ -75,7 +75,7 @@ public class UpsertQuickStart extends QuickStartBase {
QuickstartTableRequest request = new
QuickstartTableRequest(bootstrapTableDir.getAbsolutePath());
final QuickstartRunner runner
- = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0,
dataDir, getConfigOverrides());
+ = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1,
dataDir, getConfigOverrides());
printStatus(Color.CYAN, "***** Starting Kafka *****");
final ZkStarter.ZookeeperInstance zookeeperInstance =
ZkStarter.startLocalZkServer();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index faa005d936..1f479ed46a 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -20,30 +20,21 @@ package org.apache.pinot.tools.admin.command;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableMap;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileReader;
import java.io.IOException;
-import java.io.Reader;
-import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.tenant.TenantRole;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.filesystem.PinotFSFactory;
-import org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher;
-import org.apache.pinot.spi.ingestion.batch.spec.SegmentGenerationJobSpec;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.tools.BootstrapTableTool;
import org.apache.pinot.tools.QuickstartTableRequest;
-import org.apache.pinot.tools.utils.JarUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.Yaml;
public class QuickstartRunner {
@@ -234,41 +225,6 @@ public class QuickstartRunner {
}
}
- @Deprecated
- public void addTable()
- throws Exception {
- for (QuickstartTableRequest request : _tableRequests) {
- new
AddTableCommand().setSchemaFile(request.getSchemaFile().getAbsolutePath())
- .setTableConfigFile(request.getTableRequestFile().getAbsolutePath())
-
.setControllerPort(String.valueOf(_controllerPorts.get(0))).setExecute(true).execute();
- }
- }
-
- @Deprecated
- public void launchDataIngestionJob()
- throws Exception {
- for (QuickstartTableRequest request : _tableRequests) {
- if (request.getTableType() == TableType.OFFLINE) {
- try (Reader reader = new BufferedReader(new
FileReader(request.getIngestionJobFile().getAbsolutePath()))) {
- SegmentGenerationJobSpec spec = new Yaml().loadAs(reader,
SegmentGenerationJobSpec.class);
- String inputDirURI = spec.getInputDirURI();
- if (!new File(inputDirURI).exists()) {
- URL resolvedInputDirURI =
QuickstartRunner.class.getClassLoader().getResource(inputDirURI);
- if (resolvedInputDirURI.getProtocol().equals("jar")) {
- String[] splits = resolvedInputDirURI.getFile().split("!");
- String inputDir = new File(_tempDir, "inputData").toString();
- JarUtils.copyResourcesToDirectory(splits[0],
splits[1].substring(1), inputDir);
- spec.setInputDirURI(inputDir);
- } else {
- spec.setInputDirURI(resolvedInputDirURI.toString());
- }
- }
- IngestionJobLauncher.runIngestionJob(spec);
- }
- }
- }
- }
-
public JsonNode runQuery(String query)
throws Exception {
int brokerPort = _brokerPorts.get(RANDOM.nextInt(_brokerPorts.size()));
diff --git
a/pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/TestQuickStartCommand.java
b/pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/TestQuickStartCommand.java
index 62f0c6accb..e15f07344f 100644
---
a/pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/TestQuickStartCommand.java
+++
b/pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/TestQuickStartCommand.java
@@ -19,7 +19,6 @@
package org.apache.pinot.tools.admin.command;
import java.lang.reflect.InvocationTargetException;
-import org.apache.pinot.tools.BatchQuickstartWithMinion;
import org.apache.pinot.tools.EmptyQuickstart;
import org.apache.pinot.tools.HybridQuickstart;
import org.apache.pinot.tools.JoinQuickStart;
@@ -31,89 +30,91 @@ import
org.apache.pinot.tools.RealtimeComplexTypeHandlingQuickStart;
import org.apache.pinot.tools.RealtimeJsonIndexQuickStart;
import org.apache.pinot.tools.RealtimeQuickStart;
import org.apache.pinot.tools.RealtimeQuickStartWithMinion;
+import org.apache.pinot.tools.TimestampIndexQuickstart;
import org.apache.pinot.tools.UpsertJsonQuickStart;
import org.apache.pinot.tools.UpsertQuickStart;
import org.testng.Assert;
import org.testng.annotations.Test;
+
public class TestQuickStartCommand {
- @Test(expectedExceptions = UnsupportedOperationException.class,
- expectedExceptionsMessageRegExp = "^No QuickStart type provided.
Valid types are: \\[.*\\]$")
- public void testNoArg() throws Exception {
- QuickStartCommand quickStartCommand = new QuickStartCommand();
- quickStartCommand.execute();
- }
-
- @Test(expectedExceptions = UnsupportedOperationException.class,
- expectedExceptionsMessageRegExp = "^Unsupported QuickStart type:
foo. Valid types are: \\[.*\\]$")
- public void testInvalidQuickStart() throws Exception {
- QuickStartCommand quickStartCommand = new QuickStartCommand();
- quickStartCommand.setType("foo");
- quickStartCommand.execute();
- }
-
- @Test
- public void testMatchStringToCommand()
- throws InvocationTargetException, NoSuchMethodException,
InstantiationException, IllegalAccessException {
- Assert.assertEquals(quickStartClassFor("OFFLINE"), Quickstart.class);
- Assert.assertEquals(quickStartClassFor("offline"), Quickstart.class);
- Assert.assertEquals(quickStartClassFor("BATCH"), Quickstart.class);
-
- Assert.assertEquals(quickStartClassFor("EMPTY"),
EmptyQuickstart.class);
- Assert.assertEquals(quickStartClassFor("DEFAULT"),
EmptyQuickstart.class);
-
- Assert.assertEquals(quickStartClassFor("OFFLINE_MINION"),
BatchQuickstartWithMinion.class);
- Assert.assertEquals(quickStartClassFor("BATCH_MINION"),
BatchQuickstartWithMinion.class);
- Assert.assertEquals(quickStartClassFor("OFFLINE-MINION"),
BatchQuickstartWithMinion.class);
- Assert.assertEquals(quickStartClassFor("BATCH-MINION"),
BatchQuickstartWithMinion.class);
-
- Assert.assertEquals(quickStartClassFor("REALTIME_MINION"),
RealtimeQuickStartWithMinion.class);
- Assert.assertEquals(quickStartClassFor("REALTIME-MINION"),
RealtimeQuickStartWithMinion.class);
-
- Assert.assertEquals(quickStartClassFor("REALTIME"),
RealtimeQuickStart.class);
- Assert.assertEquals(quickStartClassFor("REALTIME"),
RealtimeQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("HYBRID"),
HybridQuickstart.class);
-
- Assert.assertEquals(quickStartClassFor("JOIN"), JoinQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("UPSERT"),
UpsertQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("OFFLINE_JSON_INDEX"),
JsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("OFFLINE-JSON-INDEX"),
JsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH_JSON_INDEX"),
JsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH-JSON-INDEX"),
JsonIndexQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("REALTIME_JSON_INDEX"),
RealtimeJsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("REALTIME-JSON-INDEX"),
RealtimeJsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM_JSON_INDEX"),
RealtimeJsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM-JSON-INDEX"),
RealtimeJsonIndexQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("UPSERT_JSON_INDEX"),
UpsertJsonQuickStart.class);
- Assert.assertEquals(quickStartClassFor("UPSERT-JSON-INDEX"),
UpsertJsonQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("OFFLINE_COMPLEX_TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("OFFLINE-COMPLEX-TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH_COMPLEX_TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH-COMPLEX-TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("REALTIME_COMPLEX_TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("REALTIME-COMPLEX-TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM_COMPLEX_TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM-COMPLEX-TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
- }
-
- private Class<? extends QuickStartBase> quickStartClassFor(String offline)
- throws InvocationTargetException, NoSuchMethodException,
InstantiationException, IllegalAccessException {
- return QuickStartCommand.selectQuickStart(offline).getClass();
- }
+ @Test(expectedExceptions = UnsupportedOperationException.class,
+ expectedExceptionsMessageRegExp = "^No QuickStart type provided. Valid
types are: \\[.*\\]$")
+ public void testNoArg()
+ throws Exception {
+ QuickStartCommand quickStartCommand = new QuickStartCommand();
+ quickStartCommand.execute();
+ }
+
+ @Test(expectedExceptions = UnsupportedOperationException.class,
+ expectedExceptionsMessageRegExp = "^Unsupported QuickStart type: foo.
Valid types are: \\[.*\\]$")
+ public void testInvalidQuickStart()
+ throws Exception {
+ QuickStartCommand quickStartCommand = new QuickStartCommand();
+ quickStartCommand.setType("foo");
+ quickStartCommand.execute();
+ }
+
+ @Test
+ public void testMatchStringToCommand()
+ throws InvocationTargetException, NoSuchMethodException,
InstantiationException, IllegalAccessException {
+ Assert.assertEquals(quickStartClassFor("OFFLINE"), Quickstart.class);
+ Assert.assertEquals(quickStartClassFor("offline"), Quickstart.class);
+ Assert.assertEquals(quickStartClassFor("BATCH"), Quickstart.class);
+
+ Assert.assertEquals(quickStartClassFor("EMPTY"), EmptyQuickstart.class);
+ Assert.assertEquals(quickStartClassFor("DEFAULT"), EmptyQuickstart.class);
+
+ Assert.assertEquals(quickStartClassFor("REALTIME_MINION"),
RealtimeQuickStartWithMinion.class);
+ Assert.assertEquals(quickStartClassFor("REALTIME-MINION"),
RealtimeQuickStartWithMinion.class);
+
+ Assert.assertEquals(quickStartClassFor("REALTIME"),
RealtimeQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("REALTIME"),
RealtimeQuickStart.class);
+
+ Assert.assertEquals(quickStartClassFor("HYBRID"), HybridQuickstart.class);
+
+ Assert.assertEquals(quickStartClassFor("JOIN"), JoinQuickStart.class);
+
+ Assert.assertEquals(quickStartClassFor("UPSERT"), UpsertQuickStart.class);
+
+ Assert.assertEquals(quickStartClassFor("OFFLINE_JSON_INDEX"),
JsonIndexQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("OFFLINE-JSON-INDEX"),
JsonIndexQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("BATCH_JSON_INDEX"),
JsonIndexQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("BATCH-JSON-INDEX"),
JsonIndexQuickStart.class);
+
+ Assert.assertEquals(quickStartClassFor("REALTIME_JSON_INDEX"),
RealtimeJsonIndexQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("REALTIME-JSON-INDEX"),
RealtimeJsonIndexQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("STREAM_JSON_INDEX"),
RealtimeJsonIndexQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("STREAM-JSON-INDEX"),
RealtimeJsonIndexQuickStart.class);
+
+ Assert.assertEquals(quickStartClassFor("UPSERT_JSON_INDEX"),
UpsertJsonQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("UPSERT-JSON-INDEX"),
UpsertJsonQuickStart.class);
+
+ Assert.assertEquals(quickStartClassFor("OFFLINE_COMPLEX_TYPE"),
+ OfflineComplexTypeHandlingQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("OFFLINE-COMPLEX-TYPE"),
+ OfflineComplexTypeHandlingQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("BATCH_COMPLEX_TYPE"),
+ OfflineComplexTypeHandlingQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("BATCH-COMPLEX-TYPE"),
+ OfflineComplexTypeHandlingQuickStart.class);
+
+ Assert.assertEquals(quickStartClassFor("REALTIME_COMPLEX_TYPE"),
+ RealtimeComplexTypeHandlingQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("REALTIME-COMPLEX-TYPE"),
+ RealtimeComplexTypeHandlingQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("STREAM_COMPLEX_TYPE"),
+ RealtimeComplexTypeHandlingQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("STREAM-COMPLEX-TYPE"),
+ RealtimeComplexTypeHandlingQuickStart.class);
+
+ Assert.assertEquals(quickStartClassFor("TIMESTAMP"),
+ TimestampIndexQuickstart.class);
+ }
+
+ private Class<? extends QuickStartBase> quickStartClassFor(String offline)
+ throws InvocationTargetException, NoSuchMethodException,
InstantiationException, IllegalAccessException {
+ return QuickStartCommand.selectQuickStart(offline).getClass();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
