This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 96c4471b4a [feature](udf) udf array/map support decimal and update
doc (#23560)
96c4471b4a is described below
commit 96c4471b4a6a7531a882b206347534efc9596117
Author: Mryange <59914473+mrya...@users.noreply.github.com>
AuthorDate: Thu Aug 31 07:44:18 2023 +0800
[feature](udf) udf array/map support decimal and update doc (#23560)
* update
* decimal
* update table name
* remove log
* add log
---
.../ecosystem/udf/java-user-defined-function.md | 3 +-
.../ecosystem/udf/java-user-defined-function.md | 3 +-
.../apache/doris/common/jni/utils/UdfUtils.java | 96 ++++++++-------------
.../java/org/apache/doris/udf/BaseExecutor.java | 29 ++++---
.../java/org/apache/doris/udf/UdafExecutor.java | 15 ++--
.../java/org/apache/doris/udf/UdfExecutor.java | 28 ++-----
.../data/javaudf_p0/test_javaudf_with_decimal.out | 13 +++
.../java/org/apache/doris/udf/MyArrayDecimal.java | 35 ++++++++
.../java/org/apache/doris/udf/MyMapDecimal.java | 35 ++++++++
.../java/org/apache/doris/udf/MyMapRetDecimal.java | 42 ++++++++++
.../org/apache/doris/udf/MyReturnMapString.java | 1 -
.../org/apache/doris/udf/MySumReturnMapIntDou.java | 1 -
.../suites/javaudf_p0/test_javaudf_agg_map.groovy | 14 ++--
.../javaudf_p0/test_javaudf_with_decimal.groovy | 97 ++++++++++++++++++++++
14 files changed, 294 insertions(+), 118 deletions(-)
diff --git a/docs/en/docs/ecosystem/udf/java-user-defined-function.md
b/docs/en/docs/ecosystem/udf/java-user-defined-function.md
index c6b57fed3c..d1e3ee2998 100644
--- a/docs/en/docs/ecosystem/udf/java-user-defined-function.md
+++ b/docs/en/docs/ecosystem/udf/java-user-defined-function.md
@@ -59,8 +59,9 @@ Java UDF provides users with a Java interface written in UDF
to facilitate the e
|String|String|
|Decimal|BigDecimal|
|```array<Type>```|```ArrayList<Type>```|
+|```map<Type1,Type2>```|```HashMap<Type1,Type2>```|
-* Array types can nested basic types, Eg: In Doris: ```array<int>```
corresponds to JAVA UDF Argument Type: ```ArrayList<Integer>```, Others is also.
+* Array/Map types can nested basic types, Eg: In Doris: ```array<int>```
corresponds to JAVA UDF Argument Type: ```ArrayList<Integer>```, Others is also.
## Write UDF functions
This section mainly introduces how to develop a Java UDF. Samples for the Java
version are provided under `samples/doris-demo/java-udf-demo/` for your
reference, Check it out
[here](https://github.com/apache/incubator-doris/tree/master/samples/doris-demo/java-udf-demo)
diff --git a/docs/zh-CN/docs/ecosystem/udf/java-user-defined-function.md
b/docs/zh-CN/docs/ecosystem/udf/java-user-defined-function.md
index a30fcca614..e7e37b56e5 100644
--- a/docs/zh-CN/docs/ecosystem/udf/java-user-defined-function.md
+++ b/docs/zh-CN/docs/ecosystem/udf/java-user-defined-function.md
@@ -57,8 +57,9 @@ Java UDF 为用户提供UDF编写的Java接口，以方便用户使用Java语言
|String|String|
|Decimal|BigDecimal|
|```array<Type>```|```ArrayList<Type>```|
+|```map<Type1,Type2>```|```HashMap<Type1,Type2>```|
-* array类型可以嵌套基本类型，例如Doris: ```array<int>```对应JAVA UDF Argument Type:
```ArrayList<Integer>```, 其他依此类推
+* array/map类型可以嵌套基本类型，例如Doris: ```array<int>```对应JAVA UDF Argument Type:
```ArrayList<Integer>```, 其他依此类推
## 编写 UDF 函数
本小节主要介绍如何开发一个 Java UDF。在 `samples/doris-demo/java-udf-demo/`
下提供了示例，可供参考，查看点击[这里](https://github.com/apache/doris/tree/master/samples/doris-demo/java-udf-demo)
diff --git
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
index 5f7c050136..1eb0c8d07d 100644
---
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
+++
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
@@ -19,17 +19,12 @@ package org.apache.doris.common.jni.utils;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.MapType;
-import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Pair;
import org.apache.doris.common.exception.InternalException;
import org.apache.doris.thrift.TPrimitiveType;
-import org.apache.doris.thrift.TScalarType;
-import org.apache.doris.thrift.TTypeDesc;
-import org.apache.doris.thrift.TTypeNode;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
import com.vesoft.nebula.client.graph.data.DateTimeWrapper;
import com.vesoft.nebula.client.graph.data.DateWrapper;
@@ -98,6 +93,7 @@ public class UdfUtils {
DECIMAL128("DECIMAL128", TPrimitiveType.DECIMAL128I, 16),
ARRAY_TYPE("ARRAY_TYPE", TPrimitiveType.ARRAY, 0),
MAP_TYPE("MAP_TYPE", TPrimitiveType.MAP, 0);
+
private final String description;
private final TPrimitiveType thriftType;
private final int len;
@@ -106,6 +102,9 @@ public class UdfUtils {
private Type itemType;
private Type keyType;
private Type valueType;
+ private int keyScale;
+ private int valueScale;
+
JavaUdfDataType(String description, TPrimitiveType thriftType, int
len) {
this.description = description;
this.thriftType = thriftType;
@@ -212,63 +211,22 @@ public class UdfUtils {
public void setValueType(Type type) {
this.valueType = type;
}
- }
- public static Pair<Type, Integer> fromThrift(TTypeDesc typeDesc, int
nodeIdx) throws InternalException {
- TTypeNode node = typeDesc.getTypes().get(nodeIdx);
- Type type = null;
- switch (node.getType()) {
- case SCALAR: {
- Preconditions.checkState(node.isSetScalarType());
- TScalarType scalarType = node.getScalarType();
- if (scalarType.getType() == TPrimitiveType.CHAR) {
- Preconditions.checkState(scalarType.isSetLen());
- type = ScalarType.createCharType(scalarType.getLen());
- } else if (scalarType.getType() == TPrimitiveType.VARCHAR) {
- Preconditions.checkState(scalarType.isSetLen());
- type = ScalarType.createVarcharType(scalarType.getLen());
- } else if (scalarType.getType() == TPrimitiveType.DECIMALV2) {
- Preconditions.checkState(scalarType.isSetPrecision()
- && scalarType.isSetScale());
- type =
ScalarType.createDecimalType(scalarType.getPrecision(),
- scalarType.getScale());
- } else if (scalarType.getType() == TPrimitiveType.DECIMAL32
- || scalarType.getType() == TPrimitiveType.DECIMAL64
- || scalarType.getType() == TPrimitiveType.DECIMAL128I)
{
- Preconditions.checkState(scalarType.isSetPrecision()
- && scalarType.isSetScale());
- type =
ScalarType.createDecimalV3Type(scalarType.getPrecision(),
- scalarType.getScale());
- } else {
- type = ScalarType.createType(
- PrimitiveType.fromThrift(scalarType.getType()));
- }
- break;
- }
- case ARRAY: {
- Preconditions.checkState(nodeIdx + 1 <
typeDesc.getTypesSize());
- Pair<Type, Integer> childType = fromThrift(typeDesc, nodeIdx +
1);
- type = new ArrayType(childType.first);
- nodeIdx = childType.second;
- break;
- }
- case MAP: {
- Preconditions.checkState(nodeIdx + 2 <
typeDesc.getTypesSize());
- Pair<Type, Integer> keyType = fromThrift(typeDesc, nodeIdx +
1);
- Pair<Type, Integer> valueType = fromThrift(typeDesc,
keyType.second);
- type = new MapType(keyType.first, valueType.first);
- nodeIdx = valueType.second;
- break;
- }
+ public void setKeyScale(int scale) {
+ this.keyScale = scale;
+ }
- default:
- throw new InternalException("Return type " + node.getType() +
" is not supported now!");
+ public void setValueScale(int scale) {
+ this.valueScale = scale;
+ }
+
+ public int getKeyScale() {
+ return keyScale;
}
- return Pair.of(type, nodeIdx);
- }
- public static long getAddressAtOffset(long base, int offset) {
- return base + 8L * offset;
+ public int getValueScale() {
+ return valueScale;
+ }
}
public static void copyMemory(
@@ -339,9 +297,13 @@ public class UdfUtils {
MapType mapType = (MapType) retType;
result.setKeyType(mapType.getKeyType());
result.setValueType(mapType.getValueType());
- if (mapType.getKeyType().isDatetimeV2() ||
mapType.getKeyType().isDecimalV3()) {
- result.setPrecision(mapType.getKeyType().getPrecision());
- result.setScale(((ScalarType)
mapType.getKeyType()).getScalarScale());
+ Type keyType = mapType.getKeyType();
+ Type valuType = mapType.getValueType();
+ if (keyType.isDatetimeV2() || keyType.isDecimalV3()) {
+ result.setKeyScale(((ScalarType) keyType).getScalarScale());
+ }
+ if (valuType.isDatetimeV2() || valuType.isDecimalV3()) {
+ result.setValueScale(((ScalarType) valuType).getScalarScale());
}
}
return Pair.of(res.length != 0, result);
@@ -368,10 +330,22 @@ public class UdfUtils {
} else if (parameterTypes[finalI].isArrayType()) {
ArrayType arrType = (ArrayType) parameterTypes[finalI];
inputArgTypes[i].setItemType(arrType.getItemType());
+ if (arrType.getItemType().isDatetimeV2() ||
arrType.getItemType().isDecimalV3()) {
+
inputArgTypes[i].setPrecision(arrType.getItemType().getPrecision());
+ inputArgTypes[i].setScale(((ScalarType)
arrType.getItemType()).getScalarScale());
+ }
} else if (parameterTypes[finalI].isMapType()) {
MapType mapType = (MapType) parameterTypes[finalI];
+ Type keyType = mapType.getKeyType();
+ Type valuType = mapType.getValueType();
inputArgTypes[i].setKeyType(mapType.getKeyType());
inputArgTypes[i].setValueType(mapType.getValueType());
+ if (keyType.isDatetimeV2() || keyType.isDecimalV3()) {
+ inputArgTypes[i].setKeyScale(((ScalarType)
keyType).getScalarScale());
+ }
+ if (valuType.isDatetimeV2() || valuType.isDecimalV3()) {
+ inputArgTypes[i].setValueScale(((ScalarType)
valuType).getScalarScale());
+ }
}
if (res.length == 0) {
return Pair.of(false, inputArgTypes);
diff --git
a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
index 41dfe04c2a..9c1cb17ba5 100644
---
a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
+++
b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.exception.UdfRuntimeException;
import org.apache.doris.common.jni.utils.JNINativeMethod;
import org.apache.doris.common.jni.utils.UdfUtils;
import org.apache.doris.common.jni.utils.UdfUtils.JavaUdfDataType;
+import org.apache.doris.thrift.TFunction;
import org.apache.doris.thrift.TJavaUdfExecutorCtorParams;
import com.esotericsoftware.reflectasm.MethodAccess;
@@ -76,6 +77,7 @@ public abstract class BaseExecutor {
protected JavaUdfDataType retType;
protected Class[] argClass;
protected MethodAccess methodAccess;
+ protected TFunction fn;
/**
* Create a UdfExecutor, using parameters from a serialized thrift object.
Used
@@ -95,6 +97,7 @@ public abstract class BaseExecutor {
for (int i = 0; i < request.fn.arg_types.size(); ++i) {
parameterTypes[i] = Type.fromThrift(request.fn.arg_types.get(i));
}
+ fn = request.fn;
String jarFile = request.location;
Type funcRetType = Type.fromThrift(request.fn.ret_type);
init(request, jarFile, funcRetType, parameterTypes);
@@ -125,6 +128,7 @@ public abstract class BaseExecutor {
res = res + " key: " + retType.getValueType().toString() + " sql:
" + retType.getValueType().toSql();
}
res = res + " methodAccess: " + methodAccess.toString();
+ res = res + " fn.toString(): " + fn.toString();
return res;
}
@@ -276,7 +280,6 @@ public abstract class BaseExecutor {
UdfUtils.UNSAFE.putInt(null, offsetsAddr + 4L * row,
Integer.parseUnsignedInt(String.valueOf(offset)));
UdfUtils.copyMemory(bytes, UdfUtils.BYTE_ARRAY_OFFSET, null,
outputBufferBase + offset - bytes.length,
bytes.length);
- updateOutputOffset(offset);
break;
}
case ARRAY_TYPE:
@@ -285,10 +288,6 @@ public abstract class BaseExecutor {
}
}
-
- protected void updateOutputOffset(long offset) {
- }
-
public Object[] convertBasicArg(boolean isUdf, int argIdx, boolean
isNullable, int rowStart, int rowEnd,
long nullMapAddr, long columnAddr, long strOffsetAddr) {
switch (argTypes[argIdx]) {
@@ -470,7 +469,7 @@ public abstract class BaseExecutor {
public Object[] convertMapArg(PrimitiveType type, int argIdx, boolean
isNullable, int rowStart, int rowEnd,
long nullMapAddr,
- long offsetsAddr, long nestedNullMapAddr, long dataAddr, long
strOffsetAddr) {
+ long offsetsAddr, long nestedNullMapAddr, long dataAddr, long
strOffsetAddr, int scale) {
Object[] argument = (Object[]) Array.newInstance(ArrayList.class,
rowEnd - rowStart);
for (int row = rowStart; row < rowEnd; ++row) {
long offsetStart = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L
* (row - 1));
@@ -561,19 +560,19 @@ public abstract class BaseExecutor {
case DECIMALV2:
case DECIMAL128: {
argument[row - rowStart] = UdfConvert
-
.convertArrayDecimalArg(argTypes[argIdx].getScale(), 16L, row, currentRowNum,
+ .convertArrayDecimalArg(scale, 16L, row,
currentRowNum,
offsetStart, isNullable, nullMapAddr,
nestedNullMapAddr, dataAddr);
break;
}
case DECIMAL32: {
argument[row - rowStart] = UdfConvert
-
.convertArrayDecimalArg(argTypes[argIdx].getScale(), 4L, row, currentRowNum,
+ .convertArrayDecimalArg(scale, 4L, row,
currentRowNum,
offsetStart, isNullable, nullMapAddr,
nestedNullMapAddr, dataAddr);
break;
}
case DECIMAL64: {
argument[row - rowStart] = UdfConvert
-
.convertArrayDecimalArg(argTypes[argIdx].getScale(), 8L, row, currentRowNum,
+ .convertArrayDecimalArg(scale, 8L, row,
currentRowNum,
offsetStart, isNullable, nullMapAddr,
nestedNullMapAddr, dataAddr);
break;
}
@@ -813,18 +812,18 @@ public abstract class BaseExecutor {
public void copyBatchArrayResultImpl(boolean isNullable, int numRows,
Object[] result, long nullMapAddr,
long offsetsAddr, long nestedNullMapAddr, long dataAddr, long
strOffsetAddr,
- PrimitiveType type) {
+ PrimitiveType type, int scale) {
long hasPutElementNum = 0;
for (int row = 0; row < numRows; ++row) {
hasPutElementNum = copyTupleArrayResultImpl(hasPutElementNum,
isNullable, row, result[row], nullMapAddr,
- offsetsAddr, nestedNullMapAddr, dataAddr, strOffsetAddr,
type);
+ offsetsAddr, nestedNullMapAddr, dataAddr, strOffsetAddr,
type, scale);
}
}
public long copyTupleArrayResultImpl(long hasPutElementNum, boolean
isNullable, int row, Object result,
long nullMapAddr,
long offsetsAddr, long nestedNullMapAddr, long dataAddr, long
strOffsetAddr,
- PrimitiveType type) {
+ PrimitiveType type, int scale) {
switch (type) {
case BOOLEAN: {
hasPutElementNum = UdfConvert
@@ -914,21 +913,21 @@ public abstract class BaseExecutor {
}
case DECIMAL32: {
hasPutElementNum = UdfConvert
- .copyBatchArrayDecimalV3Result(retType.getScale(), 4L,
hasPutElementNum, isNullable, row,
+ .copyBatchArrayDecimalV3Result(scale, 4L,
hasPutElementNum, isNullable, row,
result, nullMapAddr,
offsetsAddr, nestedNullMapAddr, dataAddr);
break;
}
case DECIMAL64: {
hasPutElementNum = UdfConvert
- .copyBatchArrayDecimalV3Result(retType.getScale(), 8L,
hasPutElementNum, isNullable, row,
+ .copyBatchArrayDecimalV3Result(scale, 8L,
hasPutElementNum, isNullable, row,
result, nullMapAddr,
offsetsAddr, nestedNullMapAddr, dataAddr);
break;
}
case DECIMAL128: {
hasPutElementNum = UdfConvert
- .copyBatchArrayDecimalV3Result(retType.getScale(),
16L, hasPutElementNum, isNullable, row,
+ .copyBatchArrayDecimalV3Result(scale, 16L,
hasPutElementNum, isNullable, row,
result, nullMapAddr,
offsetsAddr, nestedNullMapAddr, dataAddr);
break;
diff --git
a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java
b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java
index bb397f689a..29ac4b272b 100644
---
a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java
+++
b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java
@@ -48,7 +48,6 @@ public class UdafExecutor extends BaseExecutor {
private static final Logger LOG = Logger.getLogger(UdafExecutor.class);
- private long inputPlacesPtr;
private HashMap<String, Method> allMethods;
private HashMap<Long, Object> stateObjMap;
private Class retClass;
@@ -88,11 +87,11 @@ public class UdafExecutor extends BaseExecutor {
PrimitiveType valueType =
argTypes[argIdx].getValueType().getPrimitiveType();
Object[] keyCol = convertMapArg(keyType, argIdx, isNullable, rowStart,
rowEnd, nullMapAddr, offsetsAddr,
keyNestedNullMapAddr, keyDataAddr,
- keyStrOffsetAddr);
+ keyStrOffsetAddr, argTypes[argIdx].getKeyScale());
Object[] valueCol = convertMapArg(valueType, argIdx, isNullable,
rowStart, rowEnd, nullMapAddr, offsetsAddr,
valueNestedNullMapAddr,
valueDataAddr,
- valueStrOffsetAddr);
+ valueStrOffsetAddr, argTypes[argIdx].getValueScale());
return buildHashMap(keyType, valueType, keyCol, valueCol);
}
@@ -304,12 +303,13 @@ public class UdafExecutor extends BaseExecutor {
UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 0);
}
copyTupleArrayResultImpl(hasPutElementNum, isNullable, row, result,
nullMapAddr, offsetsAddr, nestedNullMapAddr,
- dataAddr, strOffsetAddr,
retType.getItemType().getPrimitiveType());
+ dataAddr, strOffsetAddr,
retType.getItemType().getPrimitiveType(), retType.getScale());
}
public void copyTupleMapResult(long hasPutElementNum, boolean isNullable,
int row, Object result, long nullMapAddr,
long offsetsAddr,
- long keyNsestedNullMapAddr, long keyDataAddr, long
keyStrOffsetAddr,
+ long keyNsestedNullMapAddr, long keyDataAddr,
+ long keyStrOffsetAddr,
long valueNsestedNullMapAddr, long valueDataAddr, long
valueStrOffsetAddr) throws UdfRuntimeException {
if (nullMapAddr > 0) {
UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 0);
@@ -323,16 +323,15 @@ public class UdafExecutor extends BaseExecutor {
buildArrayListFromHashMap(resultArr, keyType, valueType, keyCol,
valueCol);
copyTupleArrayResultImpl(hasPutElementNum, isNullable, row,
valueCol[0], nullMapAddr, offsetsAddr,
- valueNsestedNullMapAddr, valueDataAddr, valueStrOffsetAddr,
valueType);
+ valueNsestedNullMapAddr, valueDataAddr, valueStrOffsetAddr,
valueType, retType.getKeyScale());
copyTupleArrayResultImpl(hasPutElementNum, isNullable, row, keyCol[0],
nullMapAddr, offsetsAddr,
- keyNsestedNullMapAddr, keyDataAddr, keyStrOffsetAddr, keyType);
+ keyNsestedNullMapAddr, keyDataAddr, keyStrOffsetAddr, keyType,
retType.getValueScale());
}
@Override
protected void init(TJavaUdfExecutorCtorParams request, String jarPath,
Type funcRetType,
Type... parameterTypes) throws UdfRuntimeException {
String className = request.fn.aggregate_fn.symbol;
- inputPlacesPtr = request.input_places_ptr;
allMethods = new HashMap<>();
stateObjMap = new HashMap<>();
diff --git
a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
index 55a7ef89ea..f0319a5304 100644
---
a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
+++
b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
@@ -44,15 +44,6 @@ public class UdfExecutor extends BaseExecutor {
// setup by init() and cleared by close()
private Method method;
- // Pre-constructed input objects for the UDF. This minimizes object
creation
- // overhead
- // as these objects are reused across calls to evaluate().
- private Object[] inputObjects;
-
- private long outputOffset;
- private long rowIdx;
-
- private long batchSizePtr;
private int evaluateIndex;
/**
@@ -92,10 +83,10 @@ public class UdfExecutor extends BaseExecutor {
PrimitiveType valueType =
argTypes[argIdx].getValueType().getPrimitiveType();
Object[] keyCol = convertMapArg(keyType, argIdx, isNullable, 0,
numRows, nullMapAddr, offsetsAddr,
keyNestedNullMapAddr, keyDataAddr,
- keyStrOffsetAddr);
+ keyStrOffsetAddr, argTypes[argIdx].getKeyScale());
Object[] valueCol = convertMapArg(valueType, argIdx, isNullable, 0,
numRows, nullMapAddr, offsetsAddr,
valueNestedNullMapAddr, valueDataAddr,
- valueStrOffsetAddr);
+ valueStrOffsetAddr, argTypes[argIdx].getValueScale());
return buildHashMap(keyType, valueType, keyCol, valueCol);
}
@@ -131,7 +122,7 @@ public class UdfExecutor extends BaseExecutor {
Preconditions.checkState(result.length == numRows,
"copyBatchArrayResult result size should equal;");
copyBatchArrayResultImpl(isNullable, numRows, result, nullMapAddr,
offsetsAddr, nestedNullMapAddr, dataAddr,
- strOffsetAddr, retType.getItemType().getPrimitiveType());
+ strOffsetAddr, retType.getItemType().getPrimitiveType(),
retType.getScale());
}
public void copyBatchMapResult(boolean isNullable, int numRows, Object[]
result, long nullMapAddr,
@@ -147,10 +138,10 @@ public class UdfExecutor extends BaseExecutor {
copyBatchArrayResultImpl(isNullable, numRows, valueCol, nullMapAddr,
offsetsAddr, valueNsestedNullMapAddr,
valueDataAddr,
- valueStrOffsetAddr, valueType);
+ valueStrOffsetAddr, valueType, retType.getKeyScale());
copyBatchArrayResultImpl(isNullable, numRows, keyCol, nullMapAddr,
offsetsAddr, keyNsestedNullMapAddr,
keyDataAddr,
- keyStrOffsetAddr, keyType);
+ keyStrOffsetAddr, keyType, retType.getValueScale());
}
/**
@@ -168,21 +159,12 @@ public class UdfExecutor extends BaseExecutor {
return method;
}
-
- @Override
- protected void updateOutputOffset(long offset) {
- outputOffset = offset;
- }
-
// Preallocate the input objects that will be passed to the underlying UDF.
// These objects are allocated once and reused across calls to evaluate()
@Override
protected void init(TJavaUdfExecutorCtorParams request, String jarPath,
Type funcRetType,
Type... parameterTypes) throws UdfRuntimeException {
String className = request.fn.scalar_fn.symbol;
- batchSizePtr = request.batch_size_ptr;
- outputOffset = 0L;
- rowIdx = 0L;
ArrayList<String> signatures = Lists.newArrayList();
try {
LOG.debug("Loading UDF '" + className + "' from " + jarPath);
diff --git a/regression-test/data/javaudf_p0/test_javaudf_with_decimal.out
b/regression-test/data/javaudf_p0/test_javaudf_with_decimal.out
new file mode 100644
index 0000000000..4a05a63689
--- /dev/null
+++ b/regression-test/data/javaudf_p0/test_javaudf_with_decimal.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_1 --
+[1.123, 1.123] 3
+[2.123, 2.123] 3
+
+-- !select_2 --
+{1.123:1.123457} 9
+{2.123:2.123457} 9
+
+-- !select_3 --
+1 {1.0000000000:1.1110000000}
+2 {2.0000000000:2.2220000000}
+
diff --git
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyArrayDecimal.java
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyArrayDecimal.java
new file mode 100644
index 0000000000..c689a44af9
--- /dev/null
+++
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyArrayDecimal.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.udf;
+import java.math.BigDecimal;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.log4j.Logger;
+import java.util.*;
+
+public class MyArrayDecimal extends UDF {
+ private static final Logger LOG = Logger.getLogger(MyArrayDecimal.class);
+
+ public Integer evaluate(ArrayList<BigDecimal> arr) {
+ Integer scale = 0;
+ for (BigDecimal value : arr) {
+ scale = value.scale();
+ }
+ return scale;
+ }
+}
diff --git
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyMapDecimal.java
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyMapDecimal.java
new file mode 100644
index 0000000000..f33c230b37
--- /dev/null
+++
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyMapDecimal.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.udf;
+import java.math.BigDecimal;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.log4j.Logger;
+import java.util.*;
+
+public class MyMapDecimal extends UDF {
+ private static final Logger LOG = Logger.getLogger(MyMapDecimal.class);
+
+ public Integer evaluate(HashMap<BigDecimal, BigDecimal> mp) {
+ Integer scale = 0;
+ for (Map.Entry<BigDecimal, BigDecimal> value : mp.entrySet()) {
+ scale = value.getKey().scale() + value.getValue().scale();
+ }
+ return scale;
+ }
+}
diff --git
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyMapRetDecimal.java
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyMapRetDecimal.java
new file mode 100644
index 0000000000..b5c860de94
--- /dev/null
+++
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyMapRetDecimal.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.udf;
+import java.math.BigDecimal;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.log4j.Logger;
+import java.util.*;
+
+public class MyMapRetDecimal extends UDF {
+ private static final Logger LOG = Logger.getLogger(MyMapRetDecimal.class);
+
+ public HashMap<BigDecimal, BigDecimal> evaluate(int id) {
+ BigDecimal idBigDecimal = new BigDecimal(id);
+
+ BigDecimal result = BigDecimal.ZERO;
+ result = result.add(idBigDecimal.divide(new BigDecimal("1")));
+ result = result.add(idBigDecimal.divide(new BigDecimal("10")));
+ result = result.add(idBigDecimal.divide(new BigDecimal("100")));
+ result = result.add(idBigDecimal.divide(new BigDecimal("1000")));
+ HashMap<BigDecimal, BigDecimal> mp = new HashMap<>();
+ for (int i = 0; i < 10; i++) {
+ mp.put(idBigDecimal, result);
+ }
+ return mp;
+ }
+}
diff --git
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyReturnMapString.java
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyReturnMapString.java
index a416a8371e..17daa4e412 100644
---
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyReturnMapString.java
+++
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MyReturnMapString.java
@@ -36,7 +36,6 @@ public class MyReturnMapString {
}
public void add(State state, Integer k, Double v) {
- LOG.info("udaf nest k v " + k + " " + v);
state.counter.put(k, v);
}
diff --git
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MySumReturnMapIntDou.java
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MySumReturnMapIntDou.java
index 7a86666ef3..5e1c8bb265 100644
---
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MySumReturnMapIntDou.java
+++
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MySumReturnMapIntDou.java
@@ -39,7 +39,6 @@ public class MySumReturnMapIntDou {
}
public void add(State state, Integer k, Double v) {
- LOG.info("udaf nest k v " + k + " " + v);
state.counter.put(k, v);
}
diff --git a/regression-test/suites/javaudf_p0/test_javaudf_agg_map.groovy
b/regression-test/suites/javaudf_p0/test_javaudf_agg_map.groovy
index facd8fe1f9..03f84e5b34 100644
--- a/regression-test/suites/javaudf_p0/test_javaudf_agg_map.groovy
+++ b/regression-test/suites/javaudf_p0/test_javaudf_agg_map.groovy
@@ -27,9 +27,9 @@ suite("test_javaudf_agg_map") {
try {
try_sql("DROP FUNCTION IF EXISTS mapii(Map<Int,Int>);")
try_sql("DROP FUNCTION IF EXISTS mapid(Map<Int,Double>);")
- try_sql("DROP TABLE IF EXISTS db")
+ try_sql("DROP TABLE IF EXISTS db_agg_map")
sql """
- CREATE TABLE IF NOT EXISTS db(
+ CREATE TABLE IF NOT EXISTS db_agg_map(
`id` INT NULL COMMENT "",
`i` INT NULL COMMENT "",
`d` Double NULL COMMENT "",
@@ -42,8 +42,8 @@ suite("test_javaudf_agg_map") {
"replication_allocation" = "tag.location.default: 1",
"storage_format" = "V2");
"""
- sql """ INSERT INTO db VALUES(1,
10,1.1,{1:1,10:1,100:1},{1:1.1,11:11.1}); """
- sql """ INSERT INTO db VALUES(2,
20,2.2,{2:2,20:2,200:2},{2:2.2,22:22.2}); """
+ sql """ INSERT INTO db_agg_map VALUES(1,
10,1.1,{1:1,10:1,100:1},{1:1.1,11:11.1}); """
+ sql """ INSERT INTO db_agg_map VALUES(2,
20,2.2,{2:2,20:2,200:2},{2:2.2,22:22.2}); """
sql """
@@ -66,13 +66,13 @@ suite("test_javaudf_agg_map") {
"""
- qt_select_1 """ select mapid(mid) from db; """
+ qt_select_1 """ select mapid(mid) from db_agg_map; """
- qt_select_2 """ select mapii(mii) from db; """
+ qt_select_2 """ select mapii(mii) from db_agg_map; """
} finally {
try_sql("DROP FUNCTION IF EXISTS mapii(Map<Int,Int>);")
try_sql("DROP FUNCTION IF EXISTS mapid(Map<Int,Double>);")
- try_sql("DROP TABLE IF EXISTS db")
+ try_sql("DROP TABLE IF EXISTS db_agg_map")
}
}
diff --git a/regression-test/suites/javaudf_p0/test_javaudf_with_decimal.groovy
b/regression-test/suites/javaudf_p0/test_javaudf_with_decimal.groovy
new file mode 100644
index 0000000000..bf28872831
--- /dev/null
+++ b/regression-test/suites/javaudf_p0/test_javaudf_with_decimal.groovy
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("test_javaudf_with_decimal") {
+ def jarPath =
"""${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar"""
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ try_sql("drop function IF EXISTS getarrscale(Array<Decimal(15,3)>);")
+ try_sql("drop function IF EXISTS
getmapscale(Map<Decimal(15,3),Decimal(15,6)>);")
+ try_sql("drop function IF EXISTS retscale(int);")
+ try_sql("drop table IF EXISTS dbwithDecimal;")
+ sql """
+ CREATE TABLE IF NOT EXISTS dbwithDecimal (
+ `id` INT(11) NULL COMMENT "" ,
+ `arr` Array<Decimal(15,3)> NULL COMMENT "" ,
+ `mp` Map<Decimal(15,3),Decimal(15,6)> NULL COMMENT ""
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2"
+ );
+ """
+ sql """ INSERT INTO dbwithDecimal
VALUES(1,[1.123,1.123456],{1.123:1.123456789}); """
+ sql """ INSERT INTO dbwithDecimal
VALUES(2,[2.123,2.123456],{2.123:2.123456789}); """
+
+
+ sql """
+
+ CREATE FUNCTION getarrscale(Array<Decimal(15,3)>) RETURNS int
PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MyArrayDecimal",
+ "always_nullable"="true",
+ "type"="JAVA_UDF"
+ );
+
+ """
+
+ sql """
+
+ CREATE FUNCTION getmapscale(Map<Decimal(15,3),Decimal(15,6)>) RETURNS
int PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MyMapDecimal",
+ "always_nullable"="true",
+ "type"="JAVA_UDF"
+ );
+
+ """
+
+
+ sql """
+
+ CREATE FUNCTION retscale(int) RETURNS
Map<Decimal(15,10),Decimal(15,10)> PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MyMapRetDecimal",
+ "always_nullable"="true",
+ "type"="JAVA_UDF"
+ );
+ """
+
+ sql """
+ set enable_nereids_planner=false;
+ """
+
+ qt_select_1 """ select arr,getarrscale(arr) from dbwithDecimal order
by id; """
+
+ qt_select_2 """ select mp,getmapscale(mp) from dbwithDecimal order by
id ; """
+
+ qt_select_3 """ select id,retscale(id) from dbwithDecimal order by id;
"""
+ } finally {
+ try_sql("drop function IF EXISTS getarrscale(Array<Decimal(15,3)>);")
+ try_sql("drop function IF EXISTS
getmapscale(Map<Decimal(15,3),Decimal(15,6)>);")
+ try_sql("drop function IF EXISTS retscale(int);")
+ try_sql("drop table IF EXISTS dbwithDecimal;")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
