This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new c5a876d6638 [fix](be) Correct null-aware mark join results for empty
correlated subqueries on 4.1 (#70233)
c5a876d6638 is described below
commit c5a876d66388ac3902457aaef5cf29cc621a1027
Author: Jerry Hu <[email protected]>
AuthorDate: Tue Nov 10 11:40:56 2026 +0800
[fix](be) Correct null-aware mark join results for empty correlated
subqueries on 4.1 (#70233)
## Summary
Backport the null-aware mark join correction from #69762 to branch-4.1.
- Clear the NULL flag when emitting a no-match sentinel from a reusable
probe buffer.
- Preserve a NULL probe key for the existing dedicated handler, with its
lookup state reset.
- Add focused BE unit coverage and a correlated IN/NOT IN SQL
regression.
## Problem and fix
With an additional join predicate, a no-match sentinel could inherit a
NULL flag from an earlier probe row. This could produce NULL instead of
the expected FALSE or TRUE for an empty correlated subquery. The
sentinel now explicitly clears that flag. The branch-4.1 backport also
preserves nullable probe rows so they reach the existing null-key
handler.
## Testing
- `BUILD_TYPE=ASAN DISABLE_BE_JAVA_EXTENSIONS=OFF ./build.sh --be --fe
-j 48` (includes FE checkstyle): passed.
- `BUILD_TYPE=ASAN ./run-be-ut.sh --run --filter='JoinHashTableTest.*'
-j 48`: 2 tests passed.
- `./run-regression-test.sh --conf
regression-test/conf/regression-conf.groovy --run -d nereids_syntax_p0
-s mark_join_correlated_empty`: 1 suite passed on an isolated ASAN FE/BE
cluster.
- `build-support/clang-format.sh`, `build-support/check-format.sh`, and
`git diff --check`: passed.
---
be/src/exec/common/hash_table/join_hash_table.h | 7 +-
.../common/hash_table/join_hash_table_test.cpp | 84 ++++++++++++++++++++++
.../mark_join_correlated_empty.out | 11 +++
.../mark_join_correlated_empty.groovy | 70 ++++++++++++++++++
4 files changed, 170 insertions(+), 2 deletions(-)
diff --git a/be/src/exec/common/hash_table/join_hash_table.h
b/be/src/exec/common/hash_table/join_hash_table.h
index 59eab4cdd63..8c90aaf452b 100644
--- a/be/src/exec/common/hash_table/join_hash_table.h
+++ b/be/src/exec/common/hash_table/join_hash_table.h
@@ -464,6 +464,8 @@ private:
if (!build_idx) {
probe_idxs[matched_cnt] = probe_idx;
build_idxs[matched_cnt] = 0;
+ // A previous NULL-key probe may leave 1 here; the no-match
sentinel is not NULL.
+ null_flags[matched_cnt] = 0;
picking_null_keys = false;
matched_cnt++;
}
@@ -481,7 +483,8 @@ private:
/// If the probe key is null
if constexpr (has_null_map) {
if (null_map[probe_idx]) {
- probe_idx++;
+ build_idx = 0;
+ picking_null_keys = false;
break;
}
}
@@ -514,4 +517,4 @@ private:
template <typename Key, typename Hash, bool DirectMapping>
using JoinHashMap = JoinHashTable<Key, Hash, DirectMapping>;
#include "common/compile_check_end.h"
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/test/exec/common/hash_table/join_hash_table_test.cpp
b/be/test/exec/common/hash_table/join_hash_table_test.cpp
new file mode 100644
index 00000000000..4bf3e3461f0
--- /dev/null
+++ b/be/test/exec/common/hash_table/join_hash_table_test.cpp
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/common/hash_table/join_hash_table.h"
+
+#include <gtest/gtest.h>
+
+#include <array>
+#include <cstdint>
+#include <tuple>
+
+#include "exec/common/hash_table/hash_crc32_return32.h"
+
+namespace doris {
+
+TEST(JoinHashTableTest, NullAwareOtherConjunctSentinelResetsNullFlag) {
+ JoinHashTable<uint32_t, HashCRC32Return32<UInt32>, false> table;
+ table.prepare_build<TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN>(2, 8, false, 0);
+
+ const std::array<uint32_t, 2> build_keys = {0, 7};
+ const std::array<uint32_t, 2> build_buckets = {
+ 0, table.hash(build_keys[1]) & (table.get_bucket_size() - 1)};
+ table.build(build_keys.data(), build_buckets.data(), build_keys.size(),
false);
+
+ const std::array<uint32_t, 1> probe_keys = {42};
+ const uint32_t probe_bucket = table.hash(probe_keys[0]) &
(table.get_bucket_size() - 1);
+ const std::array<uint32_t, 1> build_idx_map =
{table.get_first()[probe_bucket]};
+ std::array<uint32_t, 9> probe_idxs = {};
+ std::array<uint32_t, 9> build_idxs = {};
+ std::array<uint8_t, 9> null_flags = {};
+ null_flags[0] = 1; // A previous NULL-key probe left a NULL mark at this
output position.
+
+ const auto [probe_idx, build_idx, matched_cnt, picking_null_keys] =
+ table.find_null_aware_with_other_conjuncts(
+ probe_keys.data(), build_idx_map.data(), 0, 0,
probe_keys.size(),
+ probe_idxs.data(), build_idxs.data(), null_flags.data(),
false, nullptr);
+
+ EXPECT_EQ(1, probe_idx);
+ EXPECT_EQ(0, build_idx);
+ ASSERT_EQ(1, matched_cnt);
+ EXPECT_FALSE(picking_null_keys);
+ EXPECT_EQ(0, probe_idxs[0]);
+ EXPECT_EQ(0, build_idxs[0]);
+ EXPECT_EQ(0, null_flags[0]);
+}
+
+TEST(JoinHashTableTest, NullAwareOtherConjunctNullProbeIsNotSkipped) {
+ JoinHashTable<uint32_t, HashCRC32Return32<UInt32>, false> table;
+ table.prepare_build<TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN>(2, 8, false, 0);
+
+ const std::array<uint32_t, 2> probe_keys = {0, 42};
+ const std::array<uint32_t, 2> build_idx_map = {0, 0};
+ const std::array<uint8_t, 2> null_map = {1, 0};
+ std::array<uint32_t, 9> probe_idxs = {};
+ std::array<uint32_t, 9> build_idxs = {};
+ std::array<uint8_t, 9> null_flags = {};
+
+ const auto [probe_idx, build_idx, matched_cnt, picking_null_keys] =
+ table.find_null_aware_with_other_conjuncts(probe_keys.data(),
build_idx_map.data(), 0,
+ 0, probe_keys.size(),
probe_idxs.data(),
+ build_idxs.data(),
null_flags.data(), false,
+ null_map.data());
+
+ EXPECT_EQ(0, probe_idx);
+ EXPECT_EQ(0, build_idx);
+ EXPECT_EQ(0, matched_cnt);
+ EXPECT_FALSE(picking_null_keys);
+}
+
+} // namespace doris
diff --git
a/regression-test/data/nereids_syntax_p0/mark_join_correlated_empty.out
b/regression-test/data/nereids_syntax_p0/mark_join_correlated_empty.out
new file mode 100644
index 00000000000..8b2156d2441
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/mark_join_correlated_empty.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !in_empty --
+1 false
+2 false
+
+-- !not_in_empty --
+1 true
+2 true
+
+-- !in_empty_single_row --
+2 false
diff --git
a/regression-test/suites/nereids_syntax_p0/mark_join_correlated_empty.groovy
b/regression-test/suites/nereids_syntax_p0/mark_join_correlated_empty.groovy
new file mode 100644
index 00000000000..bbdadcf46fd
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/mark_join_correlated_empty.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("mark_join_correlated_empty") {
+ sql "DROP TABLE IF EXISTS mark_join_correlated_empty_l"
+ sql "DROP TABLE IF EXISTS mark_join_correlated_empty_r"
+
+ sql """
+ CREATE TABLE mark_join_correlated_empty_l (
+ id INT NOT NULL, k INT NULL, v INT NOT NULL
+ ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ('replication_num' = '1')
+ """
+ sql """
+ CREATE TABLE mark_join_correlated_empty_r (
+ id INT NOT NULL, k INT NOT NULL, v INT NOT NULL
+ ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ('replication_num' = '1')
+ """
+
+ sql "INSERT INTO mark_join_correlated_empty_l VALUES (1, NULL, 0), (2, 42,
-1)"
+ sql "INSERT INTO mark_join_correlated_empty_r VALUES (1, 7, -2)"
+
+ explain {
+ sql """
+ SELECT l.id, l.k IN (SELECT r.k FROM mark_join_correlated_empty_r r
+ WHERE r.v > l.v) AS m
+ FROM (SELECT * FROM mark_join_correlated_empty_l ORDER BY id LIMIT
2) l
+ ORDER BY l.id
+ """
+ contains "NULL AWARE LEFT SEMI JOIN"
+ contains "mark join predicates"
+ contains "other join predicates"
+ }
+
+ order_qt_in_empty """
+ SELECT l.id, l.k IN (SELECT r.k FROM mark_join_correlated_empty_r r
+ WHERE r.v > l.v) AS m
+ FROM (SELECT * FROM mark_join_correlated_empty_l ORDER BY id LIMIT 2) l
+ ORDER BY l.id
+ """
+
+ order_qt_not_in_empty """
+ SELECT l.id, l.k NOT IN (SELECT r.k FROM mark_join_correlated_empty_r r
+ WHERE r.v > l.v) AS m
+ FROM (SELECT * FROM mark_join_correlated_empty_l ORDER BY id LIMIT 2) l
+ ORDER BY l.id
+ """
+
+ order_qt_in_empty_single_row """
+ SELECT l.id, l.k IN (SELECT r.k FROM mark_join_correlated_empty_r r
+ WHERE r.v > l.v) AS m
+ FROM (SELECT * FROM mark_join_correlated_empty_l WHERE id = 2 ORDER BY
id LIMIT 1) l
+ ORDER BY l.id
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
