This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 67dfa392e42 [fix](inverted index) normal process query for null
condition when index is missing #33663 (#33826)
67dfa392e42 is described below
commit 67dfa392e42e2a985dcd30da774c3364a42553a6
Author: zzzxl <33418555+zzzxl1...@users.noreply.github.com>
AuthorDate: Thu Apr 18 14:32:30 2024 +0800
[fix](inverted index) normal process query for null condition when index is
missing #33663 (#33826)
---
be/src/olap/rowset/segment_v2/column_writer.cpp | 40 +++++++++++--
.../rowset/segment_v2/inverted_index_reader.cpp | 8 +++
.../test_no_index_null_fault_injection.out | 7 +++
.../test_no_index_null_fault_injection.groovy | 65 ++++++++++++++++++++++
4 files changed, 116 insertions(+), 4 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp
b/be/src/olap/rowset/segment_v2/column_writer.cpp
index f04fc8b8f4b..19a0c4f3a84 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -43,6 +43,7 @@
#include "olap/types.h"
#include "runtime/collection_value.h"
#include "util/block_compression.h"
+#include "util/debug_points.h"
#include "util/faststring.h"
#include "util/rle_encoding.h"
#include "vec/core/types.h"
@@ -482,10 +483,41 @@ Status ScalarColumnWriter::init() {
}
if (_opts.inverted_index) {
- RETURN_IF_ERROR(InvertedIndexColumnWriter::create(
- get_field(), &_inverted_index_builder,
_file_writer->path().filename().native(),
- _file_writer->path().parent_path().native(),
_opts.inverted_index,
- _file_writer->fs()));
+ do {
+ DBUG_EXECUTE_IF("column_writer.init", {
+ class InvertedIndexColumnWriterEmptyImpl final : public
InvertedIndexColumnWriter {
+ public:
+ Status init() override { return Status::OK(); }
+ Status add_values(const std::string name, const void*
values,
+ size_t count) override {
+ return Status::OK();
+ }
+ Status add_array_values(size_t field_size, const
CollectionValue* values,
+ size_t count) override {
+ return Status::OK();
+ }
+ Status add_array_values(size_t field_size, const void*
value_ptr,
+ const uint8_t* null_map, const
uint8_t* offsets_ptr,
+ size_t count) override {
+ return Status::OK();
+ }
+ Status add_nulls(uint32_t count) override { return
Status::OK(); }
+ Status finish() override { return Status::OK(); }
+ int64_t size() const override { return 0; }
+ int64_t file_size() const override { return 0; }
+ void close_on_error() override {}
+ };
+
+ _inverted_index_builder =
std::make_unique<InvertedIndexColumnWriterEmptyImpl>();
+
+ break;
+ });
+
+ RETURN_IF_ERROR(InvertedIndexColumnWriter::create(
+ get_field(), &_inverted_index_builder,
_file_writer->path().filename().native(),
+ _file_writer->path().parent_path().native(),
_opts.inverted_index,
+ _file_writer->fs()));
+ } while (false);
}
if (_opts.need_bloom_filter) {
if (_opts.is_ngram_bf_index) {
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index ab79c159f44..3da6f7b9a76 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -207,6 +207,14 @@ Status
InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach
return Status::OK();
}
+ bool exists = false;
+ RETURN_IF_ERROR(_fs->exists(index_file_path, &exists));
+ if (!exists) {
+ LOG(WARNING) << "inverted index: " << index_file_path.native() <<
" not exist.";
+ return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
+ "inverted index path: {} not exist.",
index_file_path.native());
+ }
+
if (!dir) {
dir = new DorisCompoundReader(
DorisCompoundDirectoryFactory::getDirectory(_fs,
index_dir.c_str()),
diff --git
a/regression-test/data/fault_injection_p0/test_no_index_null_fault_injection.out
b/regression-test/data/fault_injection_p0/test_no_index_null_fault_injection.out
new file mode 100644
index 00000000000..4da9bf65689
--- /dev/null
+++
b/regression-test/data/fault_injection_p0/test_no_index_null_fault_injection.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+2
+
+-- !sql --
+3
+
diff --git
a/regression-test/suites/fault_injection_p0/test_no_index_null_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_no_index_null_fault_injection.groovy
new file mode 100644
index 00000000000..eda11f9acb9
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_no_index_null_fault_injection.groovy
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_no_index_null", "nonConcurrent") {
+ // define a sql table
+ def testTable = "test_no_index_null"
+
+ def create_httplogs_unique_table = {testTablex ->
+ // multi-line sql
+ def result = sql """
+ CREATE TABLE ${testTablex} (
+ `@timestamp` INT NULL,
+ `clientip` VARCHAR(20) NULL,
+ `request` TEXT NULL,
+ `status` INT NULL,
+ `size` INT NULL,
+ INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" =
"unicode", "support_phrase" = "true", "lower_case" = "true") COMMENT ''''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`@timestamp`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "disable_auto_compaction" = "true"
+ );
+ """
+ }
+
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+ create_httplogs_unique_table.call(testTable)
+
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs("column_writer.init")
+
+ sql """ INSERT INTO ${testTable} VALUES (1, '40.135.0.0', 'GET
/images/hm_bg.jpg HTTP/1.0', 200, 24736); """
+ sql """ INSERT INTO ${testTable} VALUES (1, '40.135.0.0', 'GET
/images/hm_bg.jpg HTTP/1.0', 200, 24736); """
+ sql """ INSERT INTO ${testTable} VALUES (1, '40.135.0.0', NULL, 200,
24736); """
+ sql """ INSERT INTO ${testTable} VALUES (1, '40.135.0.0', 'GET
/images/hm_bg.jpg HTTP/1.0', 200, 24736); """
+ sql """ INSERT INTO ${testTable} VALUES (1, '40.135.0.0', NULL, 200,
24736); """
+ sql 'sync'
+
+ qt_sql """ select count() from ${testTable} where request IS NULL;
"""
+ qt_sql """ select count() from ${testTable} where request IS NOT
NULL; """
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs("column_writer.init")
+ }
+ } finally {
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
