This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b0ad7ca Support inbuilt transform functions (#5312)
b0ad7ca is described below
commit b0ad7ca706f59cb9c0065040841f41030258dc56
Author: Neha Pawar <neha.pawa...@gmail.com>
AuthorDate: Tue Apr 28 09:24:04 2020 -0700
Support inbuilt transform functions (#5312)
Wiring the FunctionExpressionEvaluator with the ExpressionEvaluatorFactory
to be able to execute inbuilt transform functions.
---
...essionEvaluator.java => DateTimeFunctions.java} | 24 ++++-----
...valuator.java => DefaultFunctionEvaluator.java} | 16 ++++--
...essionEvaluator.java => FunctionEvaluator.java} | 6 +--
...rFactory.java => FunctionEvaluatorFactory.java} | 51 ++++++++++---------
.../pinot/core/data/function/FunctionRegistry.java | 16 ++++++
...Evaluator.java => GroovyFunctionEvaluator.java} | 6 +--
...aluator.java => TimeSpecFunctionEvaluator.java} | 6 +--
.../recordtransformer/ExpressionTransformer.java | 18 +++----
.../org/apache/pinot/core/util/SchemaUtils.java | 16 +++---
.../function/DateTimeFunctionEvaluatorTest.java | 59 ++++++++++++++++++++++
...Test.java => DefaultFunctionEvaluatorTest.java} | 12 +++--
...rTest.java => GroovyFunctionEvaluatorTest.java} | 4 +-
.../apache/pinot/core/util/SchemaUtilsTest.java | 25 ++++++---
.../data/readers/AbstractRecordExtractorTest.java | 2 +-
.../groovy_transform_functions_schema.json | 2 +-
15 files changed, 182 insertions(+), 81 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/ExpressionEvaluator.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimeFunctions.java
similarity index 59%
copy from
pinot-core/src/main/java/org/apache/pinot/core/data/function/ExpressionEvaluator.java
copy to
pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimeFunctions.java
index dc51f0b..533a497 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/ExpressionEvaluator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimeFunctions.java
@@ -18,26 +18,26 @@
*/
package org.apache.pinot.core.data.function;
-import java.util.List;
-import org.apache.pinot.spi.data.readers.GenericRow;
+import java.util.concurrent.TimeUnit;
/**
- * Interface for evaluators of transform function expressions of schema field
specs
- * They transformFunction follows the convention:
- * "transformFunction": "FunctionType({function}, argument1,
argument2,...argumentN)"
- * For example,
- * "transformFunction" : "Groovy({firstName + ' ' + lastName}, firstName,
lastName)"
+ * Inbuilt date time related transform functions
+ * TODO: Exhaustively add all time conversion functions
*/
-public interface ExpressionEvaluator {
+public class DateTimeFunctions {
/**
- * Get the arguments of the function
+ * Convert epoch millis to epoch hours
*/
- List<String> getArguments();
+ static Long toEpochHours(Long millis) {
+ return TimeUnit.MILLISECONDS.toHours(millis);
+ }
/**
- * Evaluate the function on the generic row and return the result
+ * Convert epoch millis to epoch minutes, bucketed by given bucket
granularity
*/
- Object evaluate(GenericRow genericRow);
+ static Long toEpochMinutes(Long millis, String bucket) {
+ return TimeUnit.MILLISECONDS.toMinutes(millis) / Integer.parseInt(bucket);
+ }
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionExpressionEvaluator.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluator.java
similarity index 91%
rename from
pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionExpressionEvaluator.java
rename to
pinot-core/src/main/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluator.java
index 662639f..568fb64 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionExpressionEvaluator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluator.java
@@ -19,6 +19,7 @@
package org.apache.pinot.core.data.function;
import com.google.common.base.Preconditions;
+import java.util.ArrayList;
import java.util.List;
import org.apache.pinot.common.request.transform.TransformExpressionTree;
import org.apache.pinot.spi.data.readers.GenericRow;
@@ -42,12 +43,14 @@ import org.apache.pinot.spi.data.readers.GenericRow;
* </li>
* </ul>
*/
-public class FunctionExpressionEvaluator {
+public class DefaultFunctionEvaluator implements FunctionEvaluator {
// Root of the execution tree
private final ExecutableNode _rootNode;
+ private final List<String> _arguments;
- public FunctionExpressionEvaluator(String expression)
+ public DefaultFunctionEvaluator(String expression)
throws Exception {
+ _arguments = new ArrayList<>();
_rootNode =
planExecution(TransformExpressionTree.compileToExpressionTree(expression));
}
@@ -65,7 +68,9 @@ public class FunctionExpressionEvaluator {
childNode = planExecution(childExpression);
break;
case IDENTIFIER:
- childNode = new ColumnExecutionNode(childExpression.getValue());
+ String columnName = childExpression.getValue();
+ childNode = new ColumnExecutionNode(columnName);
+ _arguments.add(columnName);
break;
case LITERAL:
childNode = new ConstantExecutionNode(childExpression.getValue());
@@ -81,6 +86,11 @@ public class FunctionExpressionEvaluator {
return new FunctionExecutionNode(functionInfo, childNodes);
}
+ @Override
+ public List<String> getArguments() {
+ return _arguments;
+ }
+
public Object evaluate(GenericRow row) {
return _rootNode.execute(row);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/ExpressionEvaluator.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionEvaluator.java
similarity index 81%
rename from
pinot-core/src/main/java/org/apache/pinot/core/data/function/ExpressionEvaluator.java
rename to
pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionEvaluator.java
index dc51f0b..e503f19 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/ExpressionEvaluator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionEvaluator.java
@@ -24,12 +24,8 @@ import org.apache.pinot.spi.data.readers.GenericRow;
/**
* Interface for evaluators of transform function expressions of schema field
specs
- * They transformFunction follows the convention:
- * "transformFunction": "FunctionType({function}, argument1,
argument2,...argumentN)"
- * For example,
- * "transformFunction" : "Groovy({firstName + ' ' + lastName}, firstName,
lastName)"
*/
-public interface ExpressionEvaluator {
+public interface FunctionEvaluator {
/**
* Get the arguments of the function
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/ExpressionEvaluatorFactory.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionEvaluatorFactory.java
similarity index 68%
rename from
pinot-core/src/main/java/org/apache/pinot/core/data/function/ExpressionEvaluatorFactory.java
rename to
pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionEvaluatorFactory.java
index 0959581..ddd4c5c 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/ExpressionEvaluatorFactory.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionEvaluatorFactory.java
@@ -19,36 +19,32 @@
package org.apache.pinot.core.data.function;
import javax.annotation.Nullable;
+import org.apache.pinot.core.util.SchemaUtils;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.TimeFieldSpec;
import org.apache.pinot.spi.data.TimeGranularitySpec;
-import org.apache.pinot.core.util.SchemaUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
- * Factory class to create an {@link ExpressionEvaluator} for the field spec
based on the {@link FieldSpec#getTransformFunction()}
+ * Factory class to create an {@link FunctionEvaluator} for the field spec
based on the {@link FieldSpec#getTransformFunction()}
*/
-public class ExpressionEvaluatorFactory {
+public class FunctionEvaluatorFactory {
- private static final Logger LOGGER =
LoggerFactory.getLogger(ExpressionEvaluatorFactory.class);
-
- private ExpressionEvaluatorFactory() {
+ private FunctionEvaluatorFactory() {
}
/**
- * Creates the {@link ExpressionEvaluator} for the given field spec
+ * Creates the {@link FunctionEvaluator} for the given field spec
*
- * 1. If transform expression is defined, use it to create {@link
ExpressionEvaluator}
- * 2. For TIME column, if conversion is needed, {@link
DefaultTimeSpecEvaluator} for backward compatible handling of time spec. This
is needed until we migrate to {@link
org.apache.pinot.spi.data.DateTimeFieldSpec}
- * 3. For columns ending with __KEYS or __VALUES (used for interpreting Map
column in Avro), create default functions for handing the Map
+ * 1. If transform expression is defined, use it to create the appropriate
{@link FunctionEvaluator}
+ * 2. For TIME column, if conversion is needed, {@link
TimeSpecFunctionEvaluator} for backward compatible handling of time spec. This
is needed until we migrate to {@link
org.apache.pinot.spi.data.DateTimeFieldSpec}
+ * 3. For columns ending with __KEYS or __VALUES (used for interpreting Map
column in Avro), create default groovy functions for handing the Map
* 4. Return null, if none of the above
*/
@Nullable
- public static ExpressionEvaluator getExpressionEvaluator(FieldSpec
fieldSpec) {
- ExpressionEvaluator expressionEvaluator = null;
+ public static FunctionEvaluator getExpressionEvaluator(FieldSpec fieldSpec) {
+ FunctionEvaluator functionEvaluator = null;
String columnName = fieldSpec.getName();
String transformExpression = fieldSpec.getTransformFunction();
@@ -56,7 +52,7 @@ public class ExpressionEvaluatorFactory {
// if transform function expression present, use it to generate function
evaluator
try {
- expressionEvaluator = getExpressionEvaluator(transformExpression);
+ functionEvaluator = getExpressionEvaluator(transformExpression);
} catch (Exception e) {
throw new IllegalStateException(
"Caught exception while constructing expression evaluator for
transform expression:" + transformExpression
@@ -70,7 +66,7 @@ public class ExpressionEvaluatorFactory {
TimeGranularitySpec outgoingGranularitySpec =
timeFieldSpec.getOutgoingGranularitySpec();
if (!incomingGranularitySpec.equals(outgoingGranularitySpec)) {
if
(!incomingGranularitySpec.getName().equals(outgoingGranularitySpec.getName())) {
- expressionEvaluator = new
DefaultTimeSpecEvaluator(incomingGranularitySpec, outgoingGranularitySpec);
+ functionEvaluator = new
TimeSpecFunctionEvaluator(incomingGranularitySpec, outgoingGranularitySpec);
} else {
throw new IllegalStateException(
"Invalid timeSpec - Incoming and outgoing field specs are
different, but name " + incomingGranularitySpec
@@ -84,23 +80,30 @@ public class ExpressionEvaluatorFactory {
String sourceMapName =
columnName.substring(0, columnName.length() -
SchemaUtils.MAP_KEY_COLUMN_SUFFIX.length());
String defaultMapKeysTransformExpression =
getDefaultMapKeysTransformExpression(sourceMapName);
- expressionEvaluator =
getExpressionEvaluator(defaultMapKeysTransformExpression);
+ functionEvaluator =
getExpressionEvaluator(defaultMapKeysTransformExpression);
} else if (columnName.endsWith(SchemaUtils.MAP_VALUE_COLUMN_SUFFIX)) {
// for backward compatible handling of Map type in avro (currently only
in Avro)
String sourceMapName =
columnName.substring(0, columnName.length() -
SchemaUtils.MAP_VALUE_COLUMN_SUFFIX.length());
String defaultMapValuesTransformExpression =
getDefaultMapValuesTransformExpression(sourceMapName);
- expressionEvaluator =
getExpressionEvaluator(defaultMapValuesTransformExpression);
+ functionEvaluator =
getExpressionEvaluator(defaultMapValuesTransformExpression);
}
- return expressionEvaluator;
+ return functionEvaluator;
}
- private static ExpressionEvaluator getExpressionEvaluator(String
transformExpression) {
- ExpressionEvaluator expressionEvaluator = null;
- if
(transformExpression.startsWith(GroovyExpressionEvaluator.getGroovyExpressionPrefix()))
{
- expressionEvaluator = new GroovyExpressionEvaluator(transformExpression);
+ private static FunctionEvaluator getExpressionEvaluator(String
transformExpression) {
+ FunctionEvaluator functionEvaluator;
+ try {
+ if
(transformExpression.startsWith(GroovyFunctionEvaluator.getGroovyExpressionPrefix()))
{
+ functionEvaluator = new GroovyFunctionEvaluator(transformExpression);
+ } else {
+ functionEvaluator = new DefaultFunctionEvaluator(transformExpression);
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ "Could not construct FunctionEvaluator for transformFunction: " +
transformExpression, e);
}
- return expressionEvaluator;
+ return functionEvaluator;
}
private static String getDefaultMapKeysTransformExpression(String
mapColumnName) {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistry.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistry.java
index 3b8d53f..8a8dfac 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistry.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistry.java
@@ -25,12 +25,28 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+/**
+ * Registry for inbuilt Pinot functions
+ */
public class FunctionRegistry {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(FunctionRegistry.class);
+
static Map<String, List<FunctionInfo>> _functionInfoMap = new HashMap<>();
+ static {
+ try {
+
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochHours",
Long.class));
+
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochMinutes",
Long.class, String.class));
+ } catch (NoSuchMethodException e) {
+ LOGGER.error("Caught exception when registering function", e);
+ }
+ }
+
public static FunctionInfo resolve(String functionName, Class<?>[]
argumentTypes) {
List<FunctionInfo> list = _functionInfoMap.get(functionName.toLowerCase());
FunctionInfo bestMatch = null;
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/GroovyExpressionEvaluator.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluator.java
similarity index 92%
rename from
pinot-core/src/main/java/org/apache/pinot/core/data/function/GroovyExpressionEvaluator.java
rename to
pinot-core/src/main/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluator.java
index b0362c9..64e7d9a 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/GroovyExpressionEvaluator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluator.java
@@ -31,7 +31,7 @@ import org.apache.pinot.spi.data.readers.GenericRow;
/**
- * An {@link ExpressionEvaluator} for evaluating schema transform expressions
written in Groovy.
+ * An {@link FunctionEvaluator} for evaluating transform function expressions
of a Schema field spec written in Groovy.
* GroovyShell is used to execute expressions.
*
* The transform expression must follow the convention Groovy({expression},
arguments1, argument2...)
@@ -44,7 +44,7 @@ import org.apache.pinot.spi.data.readers.GenericRow;
* }
* ]
*/
-public class GroovyExpressionEvaluator implements ExpressionEvaluator {
+public class GroovyFunctionEvaluator implements FunctionEvaluator {
private static final String GROOVY_EXPRESSION_PREFIX = "Groovy";
private static final String GROOVY_FUNCTION_REGEX =
"Groovy\\(\\{(?<script>.+)}(,(?<arguments>.+))?\\)";
@@ -58,7 +58,7 @@ public class GroovyExpressionEvaluator implements
ExpressionEvaluator {
private final Binding _binding;
private final Script _script;
- public GroovyExpressionEvaluator(String transformExpression) {
+ public GroovyFunctionEvaluator(String transformExpression) {
Matcher matcher = GROOVY_FUNCTION_PATTERN.matcher(transformExpression);
Preconditions.checkState(matcher.matches(), "Invalid transform expression:
%s", transformExpression);
String arguments = matcher.group(ARGUMENTS_GROUP_NAME);
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DefaultTimeSpecEvaluator.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/TimeSpecFunctionEvaluator.java
similarity index 91%
rename from
pinot-core/src/main/java/org/apache/pinot/core/data/function/DefaultTimeSpecEvaluator.java
rename to
pinot-core/src/main/java/org/apache/pinot/core/data/function/TimeSpecFunctionEvaluator.java
index a0466a0..f2cec35 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DefaultTimeSpecEvaluator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/TimeSpecFunctionEvaluator.java
@@ -29,16 +29,16 @@ import org.apache.pinot.spi.utils.TimeUtils;
/**
- * The {@code DefaultTimeSpecEvaluator} class will convert the time value
based on the {@link TimeFieldSpec}.
+ * An implementation of {@link FunctionEvaluator} for converting the time
value based on the {@link TimeFieldSpec}.
*/
-public class DefaultTimeSpecEvaluator implements ExpressionEvaluator {
+public class TimeSpecFunctionEvaluator implements FunctionEvaluator {
private final String _incomingTimeColumn;
private final String _outgoingTimeColumn;
private final TimeConverter _incomingTimeConverter;
private final TimeConverter _outgoingTimeConverter;
private boolean _isValidated = false;
- public DefaultTimeSpecEvaluator(TimeGranularitySpec incomingGranularitySpec,
+ public TimeSpecFunctionEvaluator(TimeGranularitySpec incomingGranularitySpec,
TimeGranularitySpec outgoingGranularitySpec) {
Preconditions.checkState(!incomingGranularitySpec.equals(outgoingGranularitySpec));
_incomingTimeColumn = incomingGranularitySpec.getName();
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/ExpressionTransformer.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/ExpressionTransformer.java
index facdcf7..00babe5 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/ExpressionTransformer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/ExpressionTransformer.java
@@ -23,8 +23,8 @@ import java.util.Map;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
-import org.apache.pinot.core.data.function.ExpressionEvaluator;
-import org.apache.pinot.core.data.function.ExpressionEvaluatorFactory;
+import org.apache.pinot.core.data.function.FunctionEvaluator;
+import org.apache.pinot.core.data.function.FunctionEvaluatorFactory;
/**
@@ -34,14 +34,14 @@ import
org.apache.pinot.core.data.function.ExpressionEvaluatorFactory;
*/
public class ExpressionTransformer implements RecordTransformer {
- private final Map<String, ExpressionEvaluator> _expressionEvaluators = new
HashMap<>();
+ private final Map<String, FunctionEvaluator> _expressionEvaluators = new
HashMap<>();
public ExpressionTransformer(Schema schema) {
for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) {
if (!fieldSpec.isVirtualColumn()) {
- ExpressionEvaluator expressionEvaluator =
ExpressionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
- if (expressionEvaluator != null) {
- _expressionEvaluators.put(fieldSpec.getName(), expressionEvaluator);
+ FunctionEvaluator functionEvaluator =
FunctionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
+ if (functionEvaluator != null) {
+ _expressionEvaluators.put(fieldSpec.getName(), functionEvaluator);
}
}
}
@@ -49,13 +49,13 @@ public class ExpressionTransformer implements
RecordTransformer {
@Override
public GenericRow transform(GenericRow record) {
- for (Map.Entry<String, ExpressionEvaluator> entry :
_expressionEvaluators.entrySet()) {
+ for (Map.Entry<String, FunctionEvaluator> entry :
_expressionEvaluators.entrySet()) {
String column = entry.getKey();
- ExpressionEvaluator transformExpressionEvaluator = entry.getValue();
+ FunctionEvaluator transformFunctionEvaluator = entry.getValue();
// Skip transformation if column value already exist.
// NOTE: column value might already exist for OFFLINE data
if (record.getValue(column) == null) {
- Object result = transformExpressionEvaluator.evaluate(record);
+ Object result = transformFunctionEvaluator.evaluate(record);
record.putValue(column, result);
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/util/SchemaUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/util/SchemaUtils.java
index d152647..550c754 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/util/SchemaUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/util/SchemaUtils.java
@@ -21,8 +21,8 @@ package org.apache.pinot.core.util;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.pinot.core.data.function.ExpressionEvaluator;
-import org.apache.pinot.core.data.function.ExpressionEvaluatorFactory;
+import org.apache.pinot.core.data.function.FunctionEvaluator;
+import org.apache.pinot.core.data.function.FunctionEvaluatorFactory;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.TimeFieldSpec;
@@ -52,9 +52,9 @@ public class SchemaUtils {
Set<String> sourceFieldNames = new HashSet<>();
for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) {
if (!fieldSpec.isVirtualColumn()) {
- ExpressionEvaluator expressionEvaluator =
ExpressionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
- if (expressionEvaluator != null) {
- sourceFieldNames.addAll(expressionEvaluator.getArguments());
+ FunctionEvaluator functionEvaluator =
FunctionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
+ if (functionEvaluator != null) {
+ sourceFieldNames.addAll(functionEvaluator.getArguments());
}
sourceFieldNames.add(fieldSpec.getName());
}
@@ -83,9 +83,9 @@ public class SchemaUtils {
String column = fieldSpec.getName();
String transformFunction = fieldSpec.getTransformFunction();
if (transformFunction != null) {
- ExpressionEvaluator expressionEvaluator =
ExpressionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
- if (expressionEvaluator != null) {
- List<String> arguments = expressionEvaluator.getArguments();
+ FunctionEvaluator functionEvaluator =
FunctionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
+ if (functionEvaluator != null) {
+ List<String> arguments = functionEvaluator.getArguments();
// output column used as input
if (arguments.contains(column)) {
logger.error("The arguments of transform function: {}, should
not contain the destination column: {}",
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java
new file mode 100644
index 0000000..d6b1a04
--- /dev/null
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.function;
+
+import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+
+/**
+ * Tests the Pinot inbuilt transform functions in {@link DateTimeFunctions}
which perform date time conversion
+ */
+public class DateTimeFunctionEvaluatorTest {
+
+ @Test(dataProvider = "dateTimeFunctionsTestDataProvider")
+ public void testExpressionWithColumn(String transformFunction, List<String>
arguments, GenericRow row, Object result)
+ throws Exception {
+ DefaultFunctionEvaluator evaluator = new
DefaultFunctionEvaluator(transformFunction);
+ Assert.assertEquals(evaluator.getArguments(), arguments);
+ Assert.assertEquals(evaluator.evaluate(row), result);
+ }
+
+ @DataProvider(name = "dateTimeFunctionsTestDataProvider")
+ public Object[][] dateTimeFunctionsDataProvider() {
+ List<Object[]> inputs = new ArrayList<>();
+
+ // toEpochHours
+ GenericRow row1 = new GenericRow();
+ row1.putValue("timestamp", 1585724400000L);
+ inputs.add(new Object[]{"toEpochHours(timestamp)",
Lists.newArrayList("timestamp"), row1, 440479L});
+
+ // toEpochMinutes w/ bucketing fixed
+ GenericRow row2 = new GenericRow();
+ row2.putValue("millis", 1585724400000L);
+ inputs.add(new Object[]{"toEpochMinutes(millis, 5)",
Lists.newArrayList("millis"), row2, 5285748L});
+
+ return inputs.toArray(new Object[0][]);
+ }
+}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/FunctionExpressionEvaluatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluatorTest.java
similarity index 86%
rename from
pinot-core/src/test/java/org/apache/pinot/core/data/function/FunctionExpressionEvaluatorTest.java
rename to
pinot-core/src/test/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluatorTest.java
index 907e822..7c6d4cf 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/FunctionExpressionEvaluatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluatorTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.data.function;
+import com.google.common.collect.Lists;
import java.lang.reflect.Method;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.joda.time.DateTime;
@@ -28,7 +29,7 @@ import org.testng.Assert;
import org.testng.annotations.Test;
-public class FunctionExpressionEvaluatorTest {
+public class DefaultFunctionEvaluatorTest {
@Test
public void testExpressionWithColumn()
@@ -39,7 +40,8 @@ public class FunctionExpressionEvaluatorTest {
System.out.println(functionInfo);
String expression = "reverseString(testColumn)";
- FunctionExpressionEvaluator evaluator = new
FunctionExpressionEvaluator(expression);
+ DefaultFunctionEvaluator evaluator = new
DefaultFunctionEvaluator(expression);
+ Assert.assertEquals(evaluator.getArguments(),
Lists.newArrayList("testColumn"));
GenericRow row = new GenericRow();
for (int i = 0; i < 5; i++) {
String value = "testValue" + i;
@@ -57,7 +59,8 @@ public class FunctionExpressionEvaluatorTest {
String input = "1980-01-01";
String format = "yyyy-MM-dd";
String expression = String.format("daysSinceEpoch('%s', '%s')", input,
format);
- FunctionExpressionEvaluator evaluator = new
FunctionExpressionEvaluator(expression);
+ DefaultFunctionEvaluator evaluator = new
DefaultFunctionEvaluator(expression);
+ Assert.assertTrue(evaluator.getArguments().isEmpty());
GenericRow row = new GenericRow();
Object result = evaluator.evaluate(row);
Assert.assertEquals(result, MyFunc.daysSinceEpoch(input, format));
@@ -73,7 +76,8 @@ public class FunctionExpressionEvaluatorTest {
String reversedInput = MyFunc.reverseString(input);
String format = "yyyy-MM-dd";
String expression = String.format("daysSinceEpoch(reverseString('%s'),
'%s')", reversedInput, format);
- FunctionExpressionEvaluator evaluator = new
FunctionExpressionEvaluator(expression);
+ DefaultFunctionEvaluator evaluator = new
DefaultFunctionEvaluator(expression);
+ Assert.assertTrue(evaluator.getArguments().isEmpty());
GenericRow row = new GenericRow();
Object result = evaluator.evaluate(row);
Assert.assertEquals(result, MyFunc.daysSinceEpoch(input, format));
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyExpressionEvaluatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
similarity index 96%
rename from
pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyExpressionEvaluatorTest.java
rename to
pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
index 0ff8fad..677c05a 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyExpressionEvaluatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
@@ -32,12 +32,12 @@ import org.testng.collections.Lists;
/**
* Tests Groovy functions for transforming schema columns
*/
-public class GroovyExpressionEvaluatorTest {
+public class GroovyFunctionEvaluatorTest {
@Test(dataProvider = "groovyFunctionEvaluationDataProvider")
public void testGroovyFunctionEvaluation(String transformFunction,
List<String> arguments, GenericRow genericRow, Object expectedResult) {
- GroovyExpressionEvaluator groovyExpressionEvaluator = new
GroovyExpressionEvaluator(transformFunction);
+ GroovyFunctionEvaluator groovyExpressionEvaluator = new
GroovyFunctionEvaluator(transformFunction);
Assert.assertEquals(groovyExpressionEvaluator.getArguments(), arguments);
Object result = groovyExpressionEvaluator.evaluate(genericRow);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/util/SchemaUtilsTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/util/SchemaUtilsTest.java
index 43f1812..cb7e6b9 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/util/SchemaUtilsTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/util/SchemaUtilsTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.util;
+import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -47,7 +48,6 @@ public class SchemaUtilsTest {
DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec("d1",
FieldSpec.DataType.STRING, true);
dimensionFieldSpec.setTransformFunction("Groovy({function}, argument1,
argument2)");
schema.addField(dimensionFieldSpec);
-
List<String> extract = new
ArrayList<>(SchemaUtils.extractSourceFields(schema));
Assert.assertEquals(extract.size(), 3);
Assert.assertTrue(extract.containsAll(Arrays.asList("d1", "argument1",
"argument2")));
@@ -57,7 +57,6 @@ public class SchemaUtilsTest {
dimensionFieldSpec = new DimensionFieldSpec("d1",
FieldSpec.DataType.STRING, true);
dimensionFieldSpec.setTransformFunction("Groovy({function})");
schema.addField(dimensionFieldSpec);
-
extract = new ArrayList<>(SchemaUtils.extractSourceFields(schema));
Assert.assertEquals(extract.size(), 1);
Assert.assertTrue(extract.contains("d1"));
@@ -66,7 +65,6 @@ public class SchemaUtilsTest {
schema = new Schema();
dimensionFieldSpec = new DimensionFieldSpec("map__KEYS",
FieldSpec.DataType.INT, false);
schema.addField(dimensionFieldSpec);
-
extract = new ArrayList<>(SchemaUtils.extractSourceFields(schema));
Assert.assertEquals(extract.size(), 2);
Assert.assertTrue(extract.containsAll(Arrays.asList("map", "map__KEYS")));
@@ -75,7 +73,6 @@ public class SchemaUtilsTest {
schema = new Schema();
dimensionFieldSpec = new DimensionFieldSpec("map__VALUES",
FieldSpec.DataType.LONG, false);
schema.addField(dimensionFieldSpec);
-
extract = new ArrayList<>(SchemaUtils.extractSourceFields(schema));
Assert.assertEquals(extract.size(), 2);
Assert.assertTrue(extract.containsAll(Arrays.asList("map",
"map__VALUES")));
@@ -85,7 +82,6 @@ public class SchemaUtilsTest {
schema = new Schema();
TimeFieldSpec timeFieldSpec = new TimeFieldSpec("time",
FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS);
schema.addField(timeFieldSpec);
-
extract = new ArrayList<>(SchemaUtils.extractSourceFields(schema));
Assert.assertEquals(extract.size(), 1);
Assert.assertTrue(extract.contains("time"));
@@ -96,10 +92,27 @@ public class SchemaUtilsTest {
new TimeFieldSpec("in", FieldSpec.DataType.LONG,
TimeUnit.MILLISECONDS, "out", FieldSpec.DataType.LONG,
TimeUnit.MILLISECONDS);
schema.addField(timeFieldSpec);
-
extract = new ArrayList<>(SchemaUtils.extractSourceFields(schema));
Assert.assertEquals(extract.size(), 2);
Assert.assertTrue(extract.containsAll(Arrays.asList("in", "out")));
+
+ // inbuilt functions
+ schema = new Schema();
+ dimensionFieldSpec = new DimensionFieldSpec("hoursSinceEpoch",
FieldSpec.DataType.LONG, true);
+ dimensionFieldSpec.setTransformFunction("toEpochHours(timestamp)");
+ schema.addField(dimensionFieldSpec);
+ extract = new ArrayList<>(SchemaUtils.extractSourceFields(schema));
+ Assert.assertEquals(extract.size(), 2);
+ Assert.assertTrue(extract.containsAll(Arrays.asList("timestamp",
"hoursSinceEpoch")));
+
+ // inbuilt functions with literal
+ schema = new Schema();
+ dimensionFieldSpec = new DimensionFieldSpec("tenMinutesSinceEpoch",
FieldSpec.DataType.LONG, true);
+ dimensionFieldSpec.setTransformFunction("toEpochMinutes(timestamp, 10)");
+ schema.addField(dimensionFieldSpec);
+ extract = new ArrayList<>(SchemaUtils.extractSourceFields(schema));
+ Assert.assertEquals(extract.size(), 2);
+
Assert.assertTrue(extract.containsAll(Lists.newArrayList("tenMinutesSinceEpoch",
"timestamp")));
}
@Test
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/readers/AbstractRecordExtractorTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/readers/AbstractRecordExtractorTest.java
index 636262a..9a9f3af 100644
---
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/readers/AbstractRecordExtractorTest.java
+++
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/readers/AbstractRecordExtractorTest.java
@@ -39,7 +39,7 @@ import org.testng.annotations.Test;
/**
- * Tests the RecordReader for schema with groovy transform functions
+ * Tests the RecordReader for schema with transform functions
*/
public abstract class AbstractRecordExtractorTest {
diff --git
a/pinot-spi/src/test/resources/groovy_transform_functions_schema.json
b/pinot-spi/src/test/resources/groovy_transform_functions_schema.json
index f6bf8b5..8f1ab8f 100644
--- a/pinot-spi/src/test/resources/groovy_transform_functions_schema.json
+++ b/pinot-spi/src/test/resources/groovy_transform_functions_schema.json
@@ -39,6 +39,6 @@
"timeFormat" : "EPOCH",
"timeType": "HOURS"
},
- "transformFunction": "Groovy({timestamp/(1000*60*60)}, timestamp)"
+ "transformFunction": "toEpochHours(timestamp)"
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
