This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 7cd53e4 Allow quickstart to get table files from filesystem. (#8093)
7cd53e4 is described below
commit 7cd53e465894c72ef797964b29a68ea86a446ba2
Author: Amrish Lal <amrish.k....@gmail.com>
AuthorDate: Sat Jan 29 22:41:46 2022 -0800
Allow quickstart to get table files from filesystem. (#8093)
* Allow quickstart to get table files from filesystem.
* Move bootstrapTableDir accessor methods to QuickStartBase.java.
* Rebuild.
* Cleanup.
---
.../org/apache/pinot/tools/QuickStartBase.java | 33 ++++++++
.../java/org/apache/pinot/tools/Quickstart.java | 96 ++++++++++++++++------
.../tools/admin/command/QuickStartCommand.java | 16 ++++
3 files changed, 119 insertions(+), 26 deletions(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
index 0fdb528..d58e38b 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
@@ -20,6 +20,7 @@ package org.apache.pinot.tools;
import com.google.common.collect.ImmutableMap;
import java.io.File;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import org.apache.commons.configuration.ConfigurationException;
@@ -31,6 +32,7 @@ import org.apache.pinot.tools.utils.PinotConfigUtils;
public abstract class QuickStartBase {
protected File _dataDir = FileUtils.getTempDirectory();
+ protected String _bootstrapDataDir;
protected String _zkExternalAddress;
protected String _configFilePath;
@@ -39,6 +41,37 @@ public abstract class QuickStartBase {
return this;
}
+ public QuickStartBase setBootstrapDataDir(String bootstrapDataDir) {
+ _bootstrapDataDir = bootstrapDataDir;
+ return this;
+ }
+
+ /**
+ * Assuming that database name is DBNAME, bootstrap path must have the file
structure specified below to properly
+ * load the table:
+ * DBNAME
+ * ├── ingestionJobSpec.yaml
+ * ├── rawdata
+ * │ └── DBNAME_data.csv
+ * ├── DBNAME_offline_table_config.json
+ * └── DBNAME_schema.json
+ *
+ * @return bootstrap path if specified by command line argument
-bootstrapTableDir; otherwise, default.
+ */
+ public String getBootstrapDataDir(String bootstrapDataDir) {
+ return _bootstrapDataDir != null ? _bootstrapDataDir : bootstrapDataDir;
+ }
+
+ /** @return Table name if specified by command line argument
-bootstrapTableDir; otherwise, default. */
+ public String getTableName(String bootstrapDataDir) {
+ return
Paths.get(getBootstrapDataDir(bootstrapDataDir)).getFileName().toString();
+ }
+
+ /** @return true if bootstrapTableDir is not specified by command line
argument -bootstrapTableDir, else false.*/
+ public boolean useDefaultBootstrapTableDir() {
+ return _bootstrapDataDir == null;
+ }
+
public QuickStartBase setZkExternalAddress(String zkExternalAddress) {
_zkExternalAddress = zkExternalAddress;
return this;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
index a7c019e..01fb8fb 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.io.File;
+import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
@@ -39,6 +40,7 @@ public class Quickstart extends QuickStartBase {
private static final String TAB = "\t\t";
private static final String NEW_LINE = "\n";
+ private static final String DEFAULT_BOOTSTRAP_DIRECTORY =
"examples/batch/baseballStats";
public enum Color {
RESET("\u001B[0m"), GREEN("\u001B[32m"), YELLOW("\u001B[33m"),
CYAN("\u001B[36m");
@@ -54,10 +56,6 @@ public class Quickstart extends QuickStartBase {
}
}
- public String getBootstrapDataDir() {
- return "examples/batch/baseballStats";
- }
-
public int getNumMinions() {
return 0;
}
@@ -91,30 +89,17 @@ public class Quickstart extends QuickStartBase {
public void execute()
throws Exception {
+ String tableName = getTableName(DEFAULT_BOOTSTRAP_DIRECTORY);
File quickstartTmpDir = new File(_dataDir,
String.valueOf(System.currentTimeMillis()));
- File baseDir = new File(quickstartTmpDir, "baseballStats");
+ File baseDir = new File(quickstartTmpDir, tableName);
File dataDir = new File(baseDir, "rawdata");
Preconditions.checkState(dataDir.mkdirs());
- File schemaFile = new File(baseDir, "baseballStats_schema.json");
- File tableConfigFile = new File(baseDir,
"baseballStats_offline_table_config.json");
- File ingestionJobSpecFile = new File(baseDir, "ingestionJobSpec.yaml");
- File dataFile = new File(dataDir, "baseballStats_data.csv");
-
- ClassLoader classLoader = Quickstart.class.getClassLoader();
- URL resource = classLoader.getResource(getBootstrapDataDir() +
"/baseballStats_schema.json");
- com.google.common.base.Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, schemaFile);
- resource = classLoader.getResource(getBootstrapDataDir() +
"/rawdata/baseballStats_data.csv");
- com.google.common.base.Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, dataFile);
- resource = classLoader.getResource(getBootstrapDataDir() +
"/ingestionJobSpec.yaml");
- if (resource != null) {
- FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
+ if (useDefaultBootstrapTableDir()) {
+
copyResourceTableToTmpDirectory(getBootstrapDataDir(DEFAULT_BOOTSTRAP_DIRECTORY),
tableName, baseDir, dataDir);
+ } else {
+
copyFilesystemTableToTmpDirectory(getBootstrapDataDir(DEFAULT_BOOTSTRAP_DIRECTORY),
tableName, baseDir);
}
- resource = classLoader.getResource(getBootstrapDataDir() +
"/baseballStats_offline_table_config.json");
- com.google.common.base.Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, tableConfigFile);
QuickstartTableRequest request = new
QuickstartTableRequest(baseDir.getAbsolutePath());
QuickstartRunner runner =
@@ -133,13 +118,74 @@ public class Quickstart extends QuickStartBase {
e.printStackTrace();
}
}));
- printStatus(Color.CYAN, "***** Bootstrap baseballStats table *****");
+ printStatus(Color.CYAN, "***** Bootstrap " + tableName + " table *****");
runner.bootstrapTable();
waitForBootstrapToComplete(runner);
printStatus(Color.YELLOW, "***** Offline quickstart setup complete *****");
+ if (useDefaultBootstrapTableDir()) {
+ // Quickstart is using the default baseballStats sample table, so run
sample queries.
+ runSampleQueries(runner);
+ }
+
+ printStatus(Color.GREEN, "You can always go to http://localhost:9000 to
play around in the query console");
+ }
+
+ private static void copyResourceTableToTmpDirectory(String sourcePath,
String tableName, File baseDir, File dataDir)
+ throws IOException {
+
+ File schemaFile = new File(baseDir, tableName + "_schema.json");
+ File tableConfigFile = new File(baseDir, tableName +
"_offline_table_config.json");
+ File ingestionJobSpecFile = new File(baseDir, "ingestionJobSpec.yaml");
+ File dataFile = new File(dataDir, tableName + "_data.csv");
+
+ ClassLoader classLoader = Quickstart.class.getClassLoader();
+ URL resource = classLoader.getResource(sourcePath + File.separator +
tableName + "_schema.json");
+ com.google.common.base.Preconditions.checkNotNull(resource);
+ FileUtils.copyURLToFile(resource, schemaFile);
+ resource =
+ classLoader.getResource(sourcePath + File.separator + "rawdata" +
File.separator + tableName + "_data.csv");
+ com.google.common.base.Preconditions.checkNotNull(resource);
+ FileUtils.copyURLToFile(resource, dataFile);
+ resource = classLoader.getResource(sourcePath + File.separator +
"ingestionJobSpec.yaml");
+ if (resource != null) {
+ FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
+ }
+ resource = classLoader.getResource(sourcePath + File.separator + tableName
+ "_offline_table_config.json");
+ com.google.common.base.Preconditions.checkNotNull(resource);
+ FileUtils.copyURLToFile(resource, tableConfigFile);
+ }
+
+ private static void copyFilesystemTableToTmpDirectory(String sourcePath,
String tableName, File baseDir)
+ throws IOException {
+ File fileDb = new File(sourcePath);
+
+ if (!fileDb.exists() || !fileDb.isDirectory()) {
+ throw new RuntimeException("Directory " + fileDb.getAbsolutePath() + "
not found.");
+ }
+
+ File schemaFile = new File(fileDb, tableName + "_schema.json");
+ if (!schemaFile.exists()) {
+ throw new RuntimeException("Schema file " + schemaFile.getAbsolutePath()
+ " not found.");
+ }
+
+ File tableFile = new File(fileDb, tableName +
"_offline_table_config.json");
+ if (!tableFile.exists()) {
+ throw new RuntimeException("Table table " + tableFile.getAbsolutePath()
+ " not found.");
+ }
+
+ File data = new File(fileDb, "rawdata" + File.separator + tableName +
"_data.csv");
+ if (!data.exists()) {
+ throw new RuntimeException(("Data file " + data.getAbsolutePath() + "
not found. "));
+ }
+
+ FileUtils.copyDirectory(fileDb, baseDir);
+ }
+
+ private static void runSampleQueries(QuickstartRunner runner)
+ throws Exception {
String q1 = "select count(*) from baseballStats limit 1";
printStatus(Color.YELLOW, "Total number of documents in the table");
printStatus(Color.CYAN, "Query : " + q1);
@@ -173,8 +219,6 @@ public class Quickstart extends QuickStartBase {
printStatus(Color.CYAN, "Query : " + q5);
printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q5)));
printStatus(Color.GREEN,
"***************************************************");
-
- printStatus(Color.GREEN, "You can always go to http://localhost:9000 to
play around in the query console");
}
public static void main(String[] args)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickStartCommand.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickStartCommand.java
index 6a79d4d..440d831 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickStartCommand.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickStartCommand.java
@@ -39,6 +39,10 @@ public class QuickStartCommand extends
AbstractBaseAdminCommand implements Comma
description = "Type of quickstart, supported: STREAM/BATCH/HYBRID")
private String _type;
+ @CommandLine.Option(names = {"-bootstrapTableDir"}, required = false,
+ description = "Directory containing table schema, config, and data.")
+ private String _bootstrapTableDir;
+
@CommandLine.Option(names = {"-tmpDir", "-quickstartDir", "-dataDir"},
required = false,
description = "Temp Directory to host quickstart data")
private String _tmpDir;
@@ -78,6 +82,14 @@ public class QuickStartCommand extends
AbstractBaseAdminCommand implements Comma
_tmpDir = tmpDir;
}
+ public String getBootstrapDataDir() {
+ return _bootstrapTableDir;
+ }
+
+ public void setBootstrapTableDir(String bootstrapTableDir) {
+ _bootstrapTableDir = bootstrapTableDir;
+ }
+
public String getZkExternalAddress() {
return _zkExternalAddress;
}
@@ -130,6 +142,10 @@ public class QuickStartCommand extends
AbstractBaseAdminCommand implements Comma
quickstart.setDataDir(_tmpDir);
}
+ if (_bootstrapTableDir != null) {
+ quickstart.setBootstrapDataDir(_bootstrapTableDir);
+ }
+
if (_zkExternalAddress != null) {
quickstart.setZkExternalAddress(_zkExternalAddress);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
