This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch query-runner-logger-controls
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit f2e397d50d8985e37e63572f84a680143160d4e3
Author: Alexander Pucher <a...@alexpucher.com>
AuthorDate: Tue Oct 27 12:18:19 2020 -0700
add controls for verbosity and query dialect
---
.../pinot/tools/perf/PerfBenchmarkDriver.java | 54 ++++++++++++++--------
.../pinot/tools/perf/PerfBenchmarkDriverConf.java | 20 ++++++++
.../org/apache/pinot/tools/perf/QueryRunner.java | 6 +++
3 files changed, 61 insertions(+), 19 deletions(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
index f56bc38..54d7841 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
@@ -104,7 +104,7 @@ public class PerfBenchmarkDriver {
private PinotHelixResourceManager _helixResourceManager;
public PerfBenchmarkDriver(PerfBenchmarkDriverConf conf) {
- this(conf, "/tmp/", "HEAP", null, false);
+ this(conf, "/tmp/", "HEAP", null, conf.isVerbose());
}
public PerfBenchmarkDriver(PerfBenchmarkDriverConf conf, String tempDir,
String loadMode, String segmentFormatVersion,
@@ -386,13 +386,18 @@ public class PerfBenchmarkDriver {
public JsonNode postQuery(String query)
throws Exception {
- return postQuery(query, null);
+ return postQuery(_conf.getDialect(), query, null);
}
public JsonNode postQuery(String query, String optimizationFlags)
+ throws Exception {
+ return postQuery(_conf.getDialect(), query, optimizationFlags);
+ }
+
+ public JsonNode postQuery(String dialect, String query, String
optimizationFlags)
throws Exception {
ObjectNode requestJson = JsonUtils.newObjectNode();
- requestJson.put("pql", query);
+ requestJson.put(dialect, query);
if (optimizationFlags != null && !optimizationFlags.isEmpty()) {
requestJson.put("debugOptions", "optimizationFlags=" +
optimizationFlags);
@@ -403,30 +408,41 @@ public class PerfBenchmarkDriver {
conn.setDoOutput(true);
try (BufferedWriter writer = new BufferedWriter(new
OutputStreamWriter(conn.getOutputStream(),
- StandardCharsets.UTF_8))) {
+ StandardCharsets.UTF_8))) {
String requestString = requestJson.toString();
writer.write(requestString);
writer.flush();
- StringBuilder stringBuilder = new StringBuilder();
- try (BufferedReader reader = new BufferedReader(new
InputStreamReader(conn.getInputStream(),
- StandardCharsets.UTF_8))) {
- String line;
- while ((line = reader.readLine()) != null) {
- stringBuilder.append(line);
+ try {
+ StringBuilder stringBuilder = new StringBuilder();
+ try (BufferedReader reader = new BufferedReader(new
InputStreamReader(conn.getInputStream(),
+ StandardCharsets.UTF_8))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ stringBuilder.append(line);
+ }
}
- }
- long totalTime = System.currentTimeMillis() - start;
- String responseString = stringBuilder.toString();
- ObjectNode responseJson = (ObjectNode)
JsonUtils.stringToJsonNode(responseString);
- responseJson.put("totalTime", totalTime);
+ long totalTime = System.currentTimeMillis() - start;
+ String responseString = stringBuilder.toString();
+ ObjectNode responseJson = (ObjectNode)
JsonUtils.stringToJsonNode(responseString);
+ responseJson.put("totalTime", totalTime);
+
+ if (_verbose) {
+ if (!responseJson.has("exceptions") ||
responseJson.get("exceptions").size() <= 0) {
+ LOGGER.info("requestString: {}", requestString);
+ LOGGER.info("responseString: {}", responseString);
+ } else {
+ LOGGER.error("requestString: {}", requestString);
+ LOGGER.error("responseString: {}", responseString);
+ }
+ }
- if (_verbose && (responseJson.get("numDocsScanned").asLong() > 0)) {
- LOGGER.info("requestString: {}", requestString);
- LOGGER.info("responseString: {}", responseString);
+ return responseJson;
+ } catch (Exception e) {
+ LOGGER.error("requestString: {}", requestString);
+ throw e;
}
- return responseJson;
}
}
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriverConf.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriverConf.java
index c7436eb..532345d 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriverConf.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriverConf.java
@@ -72,6 +72,10 @@ public class PerfBenchmarkDriverConf {
String resultsOutputDirectory;
+ boolean verbose = false;
+
+ String dialect = "pql";
+
public String getClusterName() {
return clusterName;
}
@@ -263,4 +267,20 @@ public class PerfBenchmarkDriverConf {
public void setSchemaFileNamePath(String schemaFileNamePath) {
this.schemaFileNamePath = schemaFileNamePath;
}
+
+ public boolean isVerbose() {
+ return verbose;
+ }
+
+ public void setVerbose(boolean verbose) {
+ this.verbose = verbose;
+ }
+
+ public String getDialect() {
+ return dialect;
+ }
+
+ public void setDialect(String dialect) {
+ this.dialect = dialect;
+ }
}
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/QueryRunner.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/QueryRunner.java
index d9d6d6a..1030fa0 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/QueryRunner.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/QueryRunner.java
@@ -79,6 +79,10 @@ public class QueryRunner extends AbstractBaseCommand
implements Command {
private int _queueDepth = 64;
@Option(name = "-timeout", required = false, metaVar = "<long>", usage =
"Timeout in milliseconds for completing all queries (default: unlimited).")
private long _timeout = 0;
+ @Option(name = "-verbose", required = false, usage = "Enable verbose query
logging (default: false).")
+ private boolean _verbose = false;
+ @Option(name = "-dialect", required = false, metaVar = "<String>", usage =
"Query dialect to use (pql|sql).")
+ private String _dialect = "pql";
@Option(name = "-help", required = false, help = true, aliases = {"-h",
"--h", "--help"}, usage = "Print this message.")
private boolean _help;
@@ -144,6 +148,8 @@ public class QueryRunner extends AbstractBaseCommand
implements Command {
conf.setStartController(false);
conf.setStartBroker(false);
conf.setStartServer(false);
+ conf.setVerbose(_verbose);
+ conf.setDialect(_dialect);
Stream<String> queries = makeQueries(
IOUtils.readLines(new FileInputStream(_queryFile)),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
