This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new 3901055 [feature] support not equals and like and not like filter
push down (#219)
3901055 is described below
commit 390105505a09c9bdc2073094239b629c5d7ce61c
Author: gnehil <adamlee...@gmail.com>
AuthorDate: Mon Jul 29 16:24:17 2024 +0800
[feature] support not equals and like and not like filter push down (#219)
---
.../scala/org/apache/doris/spark/sql/Utils.scala | 7 +++++++
.../org/apache/doris/spark/sql/TestUtils.scala | 21 ++++++++++++++++++---
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git
a/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/Utils.scala
b/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/Utils.scala
index 7cffbe5..0400b04 100644
---
a/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/Utils.scala
+++
b/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/Utils.scala
@@ -51,6 +51,7 @@ private[spark] object Utils {
def compileFilter(filter: Filter, dialect: JdbcDialect, inValueLengthLimit:
Int): Option[String] = {
Option(filter match {
case EqualTo(attribute, value) => s"${quote(attribute)} =
${compileValue(value)}"
+ case Not(EqualTo(attribute, value)) => s"${quote(attribute)} !=
${compileValue(value)}"
case GreaterThan(attribute, value) => s"${quote(attribute)} >
${compileValue(value)}"
case GreaterThanOrEqual(attribute, value) => s"${quote(attribute)} >=
${compileValue(value)}"
case LessThan(attribute, value) => s"${quote(attribute)} <
${compileValue(value)}"
@@ -83,6 +84,12 @@ private[spark] object Utils {
} else {
null
}
+ case StringContains(attribute, value) => s"${quote(attribute)} like
'%$value%'"
+ case Not(StringContains(attribute, value)) => s"${quote(attribute)} not
like '%$value%'"
+ case StringEndsWith(attribute, value) => s"${quote(attribute)} like
'%$value'"
+ case Not(StringEndsWith(attribute, value)) => s"${quote(attribute)} not
like '%$value'"
+ case StringStartsWith(attribute, value) => s"${quote(attribute)} like
'$value%'"
+ case Not(StringStartsWith(attribute, value)) => s"${quote(attribute)}
not like '$value%'"
case _ => null
})
}
diff --git
a/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestUtils.scala
b/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestUtils.scala
index b1affbf..7e7919a 100644
---
a/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestUtils.scala
+++
b/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestUtils.scala
@@ -34,6 +34,7 @@ class TestUtils extends ExpectedExceptionTest {
val inValueLengthLimit = 5
val equalFilter = EqualTo("left", 5)
+ val notEqualFilter = Not(EqualTo("left", 5))
val greaterThanFilter = GreaterThan("left", 5)
val greaterThanOrEqualFilter = GreaterThanOrEqual("left", 5)
val lessThanFilter = LessThan("left", 5)
@@ -41,15 +42,22 @@ class TestUtils extends ExpectedExceptionTest {
val validInFilter = In("left", Array(1, 2, 3, 4))
val emptyInFilter = In("left", Array.empty)
val invalidInFilter = In("left", Array(1, 2, 3, 4, 5))
+ val notInFilter = Not(In("left", Array(1, 2, 3)))
val isNullFilter = IsNull("left")
val isNotNullFilter = IsNotNull("left")
- val notSupportFilter = StringContains("left", "right")
val validAndFilter = And(equalFilter, greaterThanFilter)
- val invalidAndFilter = And(equalFilter, notSupportFilter)
+ val invalidAndFilter = And(equalFilter, invalidInFilter)
val validOrFilter = Or(equalFilter, greaterThanFilter)
- val invalidOrFilter = Or(equalFilter, notSupportFilter)
+ val invalidOrFilter = Or(equalFilter, invalidInFilter)
+ val stringContainsFilter = StringContains("left", "right")
+ val notStringContainsFilter = Not(StringContains("left", "right"))
+ val stringEndsWithFilter = StringEndsWith("left", "right")
+ val notStringEndsWithFilter = Not(StringEndsWith("left", "right"))
+ val stringStartsWithFilter = StringStartsWith("left", "right")
+ val notStringStartsWithFilter = Not(StringStartsWith("left", "right"))
Assert.assertEquals("`left` = 5", Utils.compileFilter(equalFilter,
dialect, inValueLengthLimit).get)
+ Assert.assertEquals("`left` != 5", Utils.compileFilter(notEqualFilter,
dialect, inValueLengthLimit).get)
Assert.assertEquals("`left` > 5", Utils.compileFilter(greaterThanFilter,
dialect, inValueLengthLimit).get)
Assert.assertEquals("`left` >= 5",
Utils.compileFilter(greaterThanOrEqualFilter, dialect, inValueLengthLimit).get)
Assert.assertEquals("`left` < 5", Utils.compileFilter(lessThanFilter,
dialect, inValueLengthLimit).get)
@@ -57,6 +65,7 @@ class TestUtils extends ExpectedExceptionTest {
Assert.assertEquals("`left` in (1, 2, 3, 4)",
Utils.compileFilter(validInFilter, dialect, inValueLengthLimit).get)
Assert.assertTrue(Utils.compileFilter(emptyInFilter, dialect,
inValueLengthLimit).isEmpty)
Assert.assertTrue(Utils.compileFilter(invalidInFilter, dialect,
inValueLengthLimit).isEmpty)
+ Assert.assertEquals("`left` not in (1, 2, 3)",
Utils.compileFilter(notInFilter, dialect, inValueLengthLimit).get)
Assert.assertEquals("`left` is null", Utils.compileFilter(isNullFilter,
dialect, inValueLengthLimit).get)
Assert.assertEquals("`left` is not null",
Utils.compileFilter(isNotNullFilter, dialect, inValueLengthLimit).get)
Assert.assertEquals("(`left` = 5) and (`left` > 5)",
@@ -65,6 +74,12 @@ class TestUtils extends ExpectedExceptionTest {
Assert.assertEquals("(`left` = 5) or (`left` > 5)",
Utils.compileFilter(validOrFilter, dialect, inValueLengthLimit).get)
Assert.assertTrue(Utils.compileFilter(invalidOrFilter, dialect,
inValueLengthLimit).isEmpty)
+ Assert.assertEquals("`left` like '%right%'",
Utils.compileFilter(stringContainsFilter, dialect, inValueLengthLimit).get)
+ Assert.assertEquals("`left` not like '%right%'",
Utils.compileFilter(notStringContainsFilter, dialect, inValueLengthLimit).get)
+ Assert.assertEquals("`left` like '%right'",
Utils.compileFilter(stringEndsWithFilter, dialect, inValueLengthLimit).get)
+ Assert.assertEquals("`left` not like '%right'",
Utils.compileFilter(notStringEndsWithFilter, dialect, inValueLengthLimit).get)
+ Assert.assertEquals("`left` like 'right%'",
Utils.compileFilter(stringStartsWithFilter, dialect, inValueLengthLimit).get)
+ Assert.assertEquals("`left` not like 'right%'",
Utils.compileFilter(notStringStartsWithFilter, dialect, inValueLengthLimit).get)
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
