snleee commented on a change in pull request #6224:
URL: https://github.com/apache/incubator-pinot/pull/6224#discussion_r522519514
##########
File path:
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ClusterIntegrationTestUtils.java
##########
@@ -975,6 +814,203 @@ private static boolean isSelectionQuery(BrokerRequest
brokerRequest) {
return false;
}
+ private static void convertToUpperCase(List<String> columns) {
+ for (int i = 0; i < columns.size(); i++) {
+ columns.set(i, columns.get(i).toUpperCase());
+ }
+ }
+
+ private static int getH2ExpectedValues(Set<String> expectedValues,
List<String> expectedOrderByValues,
+ ResultSet h2ResultSet, ResultSetMetaData h2MetaData, Collection<String>
orderByColumns) throws SQLException {
+ Map<String, String> reusableExpectedValueMap = new HashMap<>();
+ Map<String, List<String>> reusableMultiValuesMap = new HashMap<>();
+ List<String> reusableColumnOrder = new ArrayList<>();
+ int h2NumRows;
+ int numColumns = h2MetaData.getColumnCount();
+
+ for (h2NumRows = 0; h2ResultSet.next() && h2NumRows <
MAX_NUM_ROWS_TO_COMPARE; h2NumRows++) {
+ reusableExpectedValueMap.clear();
+ reusableMultiValuesMap.clear();
+ reusableColumnOrder.clear();
+
+ for (int columnIndex = 1; columnIndex <= numColumns; columnIndex++) { //
h2 result set is 1-based
+ String columnName = h2MetaData.getColumnName(columnIndex);
+
+ // Handle null result and convert boolean value to lower case
+ String columnValue = h2ResultSet.getString(columnIndex);
+ if (columnValue == null) {
+ columnValue = "null";
+ } else {
+ columnValue = convertBooleanToLowerCase(columnValue);
+ }
+
+ // Handle multi-value columns
+ int length = columnName.length();
+ if (length > H2_MULTI_VALUE_SUFFIX_LENGTH &&
columnName.substring(length - H2_MULTI_VALUE_SUFFIX_LENGTH, length -
1).equals("__MV")) {
+ // Multi-value column
+ String multiValueColumnName = columnName.substring(0, length -
H2_MULTI_VALUE_SUFFIX_LENGTH);
+ List<String> multiValue =
reusableMultiValuesMap.get(multiValueColumnName);
+ if (multiValue == null) {
+ multiValue = new ArrayList<>();
+ reusableMultiValuesMap.put(multiValueColumnName, multiValue);
+ reusableColumnOrder.add(multiValueColumnName);
+ }
+ multiValue.add(columnValue);
+ } else {
+ // Single-value column
+ String columnDataType = h2MetaData.getColumnTypeName(columnIndex);
+ columnValue = removeTrailingZeroForNumber(columnValue,
columnDataType);
+ reusableExpectedValueMap.put(columnName, columnValue);
+ reusableColumnOrder.add(columnName);
+ }
+ }
+
+ // Add multi-value column results to the expected values
+ // The reason for this step is that Pinot does not maintain order of
elements in multi-value columns
+ for (Map.Entry<String, List<String>> entry :
reusableMultiValuesMap.entrySet()) {
+ List<String> multiValue = entry.getValue();
+ Collections.sort(multiValue);
+ reusableExpectedValueMap.put(entry.getKey(), multiValue.toString());
+ }
+
+ // Build expected value String
+ StringBuilder expectedValue = new StringBuilder();
+ StringBuilder expectedOrderByValue = new StringBuilder();
+ for (String column : reusableColumnOrder) {
+ expectedValue.append(reusableExpectedValueMap.get(column)).append(' ');
+ if (orderByColumns.contains(column)) {
+
expectedOrderByValue.append(reusableExpectedValueMap.get(column)).append(' ');
+ }
+ }
+ expectedValues.add(expectedValue.toString());
+ expectedOrderByValues.add(expectedOrderByValue.toString());
+ }
+
+ return h2NumRows;
+ }
+
+ private static void comparePinotResultsWithExpectedValues(Set<String>
expectedValues, List<String> expectedOrderByValues,
+ org.apache.pinot.client.ResultSet connectionResultSet,
Collection<String> orderByColumns, String pinotQuery, List<String> sqlQueries,
+ int h2NumRows, long pinotNumRecordsSelected) throws IOException,
SQLException {
+
+ int pinotNumRows = connectionResultSet.getRowCount();
+ // No record selected in H2
+ if (h2NumRows== 0) {
+ if (pinotNumRows != 0) {
+ String failureMessage = "No record selected in H2 but number of
records selected in Pinot: " + pinotNumRows;
+ failure(pinotQuery, sqlQueries, failureMessage);
+ return;
+ }
+
+ if (pinotNumRecordsSelected != 0) {
+ String failureMessage =
+ "No selection result returned in Pinot but number of records
selected: " + pinotNumRecordsSelected;
+ failure(pinotQuery, sqlQueries, failureMessage);
+ return;
+ }
+
+ // Skip further comparison
+ return;
+ }
+
+ PinotQuery compiledQuery =
CalciteSqlParser.compileToPinotQuery(pinotQuery);
+ boolean isLimitSet = compiledQuery.isSetLimit();
+ int limit = compiledQuery.getLimit();
+
+ // Only compare exhausted results
+ if (h2NumRows < MAX_NUM_ROWS_TO_COMPARE) {
+
+ for (int rowIndex = 0; rowIndex < pinotNumRows; rowIndex++) {
+ // Build actual value String.
+ StringBuilder actualValueBuilder = new StringBuilder();
+ StringBuilder actualOrderByValueBuilder = new StringBuilder();
+ for (int columnIndex = 0; columnIndex <
connectionResultSet.getColumnCount(); columnIndex++) {
+ // Convert column name to all uppercase to make it compatible with H2
+ String columnName =
connectionResultSet.getColumnName(columnIndex).toUpperCase();
+ String columnResult = connectionResultSet.getString(rowIndex,
columnIndex);
+
+ String columnDataType =
connectionResultSet.getColumnDataType(columnIndex);
+ columnResult = removeTrailingZeroForNumber(columnResult,
columnDataType);
+
+ boolean isMultiValueColumn = false;
+ try {
+ isMultiValueColumn =
JsonUtils.stringToJsonNode(columnResult).isArray();
+ } catch (IOException e) {
+ }
+
+ if (isMultiValueColumn) {
+ // Multi-value column
+ JsonNode columnValues = JsonUtils.stringToJsonNode(columnResult);
Review comment:
We can parse JsonNode once when computing `isMultiValueColumn` and reuse
it. Your code parses JSON 2 times.
##########
File path:
pinot-integration-tests/src/test/resources/On_Time_On_Time_Performance_2014_100k_subset.test_queries_500.sql
##########
@@ -555,3 +555,9 @@
{"sql":"SELECT WheelsOff, TotalAddGTime, \"Month\", OriginState FROM mytable
ORDER BY OriginState LIMIT 24","hsqls":["SELECT WheelsOff, TotalAddGTime,
Month, OriginState FROM mytable ORDER BY OriginState LIMIT 24"]}
{"sql":"SELECT WheelsOn, OriginState FROM mytable WHERE DepartureDelayGroups >
-2 ORDER BY OriginState LIMIT 20"}
{"sql":"SELECT \"Year\", DepTime FROM mytable WHERE CRSArrTime BETWEEN 1237
AND 1534 AND Cancelled BETWEEN 0 AND 1 ORDER BY DepTime LIMIT
3","hsqls":["SELECT Year, DepTime FROM mytable WHERE CRSArrTime BETWEEN 1237
AND 1534 AND Cancelled BETWEEN 0 AND 1 ORDER BY DepTime LIMIT 3"]}
+{"sql":"SELECT DivDistance, ActualElapsedTime FROM mytable LIMIT 29"}
+{"sql":"SELECT DayOfWeek FROM mytable WHERE TotalAddGTime IN (128, 148, 4, 34)
LIMIT 8"}
+{"sql":"SELECT WheelsOff FROM mytable WHERE DivDistance < 436 LIMIT 12"}
+{"sql":"SELECT DivDistance, DepTime FROM mytable WHERE NASDelay IN (45, 55,
31, 9) LIMIT 26"}
+{"sql":"SELECT DepDelay FROM mytable WHERE DepDelayMinutes BETWEEN 292.0 AND
237.0 AND DestState IN ('DE', 'AZ') LIMIT 18"}
+{"sql":"SELECT DestStateFips FROM mytable LIMIT 27"}
Review comment:
add extra line in the end
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
