This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ab359e2b645 [Bug](udf) forbid varchar type and convert to use string
type in java-udf (#38409)
ab359e2b645 is described below
commit ab359e2b6452e1b6161b9766d02ca5ca939c0fde
Author: zhangstar333 <[email protected]>
AuthorDate: Fri Aug 23 17:47:46 2024 +0800
[Bug](udf) forbid varchar type and convert to use string type in java-udf
(#38409)
## Proposed changes
Two point have change:
1. forbid create udf function use varchar type, could use string type.2.
2. in order to the compatibly of old version, convert varchar to string
type to execute udf function.
<!--Describe your changes.-->
---
.../apache/doris/common/jni/utils/JavaUdfDataType.java | 12 +++++++-----
.../java/org/apache/doris/common/jni/utils/UdfUtils.java | 15 +++++++++++----
.../src/main/java/org/apache/doris/catalog/Type.java | 2 --
.../suites/javaudf_p0/test_javaudf_string.groovy | 16 ++++++++++++++++
4 files changed, 34 insertions(+), 11 deletions(-)
diff --git
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JavaUdfDataType.java
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JavaUdfDataType.java
index 18bb90ddb99..9f973543b29 100644
---
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JavaUdfDataType.java
+++
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JavaUdfDataType.java
@@ -43,8 +43,6 @@ public class JavaUdfDataType {
public static final JavaUdfDataType BIGINT = new JavaUdfDataType("BIGINT",
TPrimitiveType.BIGINT, 8);
public static final JavaUdfDataType FLOAT = new JavaUdfDataType("FLOAT",
TPrimitiveType.FLOAT, 4);
public static final JavaUdfDataType DOUBLE = new JavaUdfDataType("DOUBLE",
TPrimitiveType.DOUBLE, 8);
- public static final JavaUdfDataType CHAR = new JavaUdfDataType("CHAR",
TPrimitiveType.CHAR, 0);
- public static final JavaUdfDataType VARCHAR = new
JavaUdfDataType("VARCHAR", TPrimitiveType.VARCHAR, 0);
public static final JavaUdfDataType STRING = new JavaUdfDataType("STRING",
TPrimitiveType.STRING, 0);
public static final JavaUdfDataType DATE = new JavaUdfDataType("DATE",
TPrimitiveType.DATE, 8);
public static final JavaUdfDataType DATETIME = new
JavaUdfDataType("DATETIME", TPrimitiveType.DATETIME, 8);
@@ -72,8 +70,6 @@ public class JavaUdfDataType {
JavaUdfDataTypeSet.add(BIGINT);
JavaUdfDataTypeSet.add(FLOAT);
JavaUdfDataTypeSet.add(DOUBLE);
- JavaUdfDataTypeSet.add(CHAR);
- JavaUdfDataTypeSet.add(VARCHAR);
JavaUdfDataTypeSet.add(STRING);
JavaUdfDataTypeSet.add(DATE);
JavaUdfDataTypeSet.add(DATETIME);
@@ -142,7 +138,9 @@ public class JavaUdfDataType {
} else if (c == double.class || c == Double.class) {
return Sets.newHashSet(JavaUdfDataType.DOUBLE);
} else if (c == char.class || c == Character.class) {
- return Sets.newHashSet(JavaUdfDataType.CHAR);
+ // some users case have create UDF use varchar as parameter not
+ // string type, but evaluate is String Class, so set
TPrimitiveType is STRING
+ return Sets.newHashSet(JavaUdfDataType.STRING);
} else if (c == String.class) {
return Sets.newHashSet(JavaUdfDataType.STRING);
} else if (Type.DATE_SUPPORTED_JAVA_TYPE.contains(c)) {
@@ -171,6 +169,10 @@ public class JavaUdfDataType {
return true;
}
}
+ if (t.getPrimitiveType().toThrift() == TPrimitiveType.VARCHAR
+ || t.getPrimitiveType().toThrift() == TPrimitiveType.CHAR) {
+ return true;
+ }
return false;
}
diff --git
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
index 8cc9ceb014f..2ef1956118b 100644
---
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
+++
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
@@ -25,6 +25,7 @@ import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Pair;
import org.apache.doris.common.exception.InternalException;
+import org.apache.doris.thrift.TPrimitiveType;
import org.apache.log4j.Logger;
import sun.misc.Unsafe;
@@ -112,8 +113,11 @@ public class UdfUtils {
// Check if the evaluate method return type is compatible with the
return type from
// the function definition. This happens when both of them map to the
same primitive
// type.
- Object[] res = javaTypes.stream().filter(
- t -> t.getPrimitiveType() ==
retType.getPrimitiveType().toThrift()).toArray();
+ Object[] res = javaTypes.stream().filter(t -> {
+ TPrimitiveType t1 = t.getPrimitiveType();
+ TPrimitiveType ret = retType.getPrimitiveType().toThrift();
+ return (t1 == ret) || (t1 == TPrimitiveType.STRING && ret ==
TPrimitiveType.VARCHAR);
+ }).toArray();
JavaUdfDataType result = new JavaUdfDataType(
res.length == 0 ? javaTypes.iterator().next() :
(JavaUdfDataType) res[0]);
@@ -160,8 +164,11 @@ public class UdfUtils {
for (int i = 0; i < parameterTypes.length; ++i) {
Set<JavaUdfDataType> javaTypes =
JavaUdfDataType.getCandidateTypes(udfArgTypes[i + firstPos]);
int finalI = i;
- Object[] res = javaTypes.stream().filter(
- t -> t.getPrimitiveType() ==
parameterTypes[finalI].getPrimitiveType().toThrift()).toArray();
+ Object[] res = javaTypes.stream().filter(t -> {
+ TPrimitiveType t1 = t.getPrimitiveType();
+ TPrimitiveType param =
parameterTypes[finalI].getPrimitiveType().toThrift();
+ return (t1 == param) || (t1 == TPrimitiveType.STRING && param
== TPrimitiveType.VARCHAR);
+ }).toArray();
inputArgTypes[i] = new JavaUdfDataType(
res.length == 0 ? javaTypes.iterator().next() :
(JavaUdfDataType) res[0]);
if (parameterTypes[finalI].isDecimalV3() ||
parameterTypes[finalI].isDatetimeV2()) {
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index 9d2abc75d44..1dcd062261b 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -322,8 +322,6 @@ public abstract class Type {
.put(PrimitiveType.DOUBLE, Sets.newHashSet(Double.class,
double.class))
.put(PrimitiveType.BIGINT, Sets.newHashSet(Long.class,
long.class))
.put(PrimitiveType.IPV4, Sets.newHashSet(Integer.class,
int.class))
- .put(PrimitiveType.CHAR, Sets.newHashSet(String.class))
- .put(PrimitiveType.VARCHAR, Sets.newHashSet(String.class))
.put(PrimitiveType.STRING, Sets.newHashSet(String.class))
.put(PrimitiveType.DATE, DATE_SUPPORTED_JAVA_TYPE)
.put(PrimitiveType.DATEV2, DATE_SUPPORTED_JAVA_TYPE)
diff --git a/regression-test/suites/javaudf_p0/test_javaudf_string.groovy
b/regression-test/suites/javaudf_p0/test_javaudf_string.groovy
index 6517c4b08c2..07070cec8b0 100644
--- a/regression-test/suites/javaudf_p0/test_javaudf_string.groovy
+++ b/regression-test/suites/javaudf_p0/test_javaudf_string.groovy
@@ -86,6 +86,22 @@ suite("test_javaudf_string") {
test_javaudf_string
JOIN test_javaudf_string_2 ON test_javaudf_string.user_id =
test_javaudf_string_2.user_id order by 1,2;
"""
+ test {
+ sql """ CREATE FUNCTION java_udf_string_test(varchar, int, int)
RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.StringTest",
+ "type"="JAVA_UDF"
+ ); """
+ exception "does not support type"
+ }
+ test {
+ sql """ CREATE FUNCTION java_udf_string_test(string, int, int)
RETURNS varchar PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.StringTest",
+ "type"="JAVA_UDF"
+ ); """
+ exception "does not support type"
+ }
} finally {
try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int,
int);")
try_sql("DROP TABLE IF EXISTS ${tableName}")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
