[doris] branch master updated: [Feature-WIP](inverted index) support phrase for inverted index writer (#20193)

[jianliangqi]

This is an automated email from the ASF dual-hosted git repository.

jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 3d8440a1b7 [Feature-WIP](inverted index) support phrase for inverted 
index writer (#20193)
3d8440a1b7 is described below

commit 3d8440a1b7cbcdfe740093a42f8b6d6177c091a5
Author: airborne12 <airborn...@gmail.com>
AuthorDate: Tue May 30 17:07:45 2023 +0800

    [Feature-WIP](inverted index) support phrase for inverted index writer 
(#20193)
---
 be/src/olap/inverted_index_parser.cpp                   | 9 +++++++++
 be/src/olap/inverted_index_parser.h                     | 6 ++++++
 be/src/olap/rowset/segment_v2/inverted_index_writer.cpp | 6 ++++++
 3 files changed, 21 insertions(+)

diff --git a/be/src/olap/inverted_index_parser.cpp 
b/be/src/olap/inverted_index_parser.cpp
index e920a4a930..d1e04e9df1 100644
--- a/be/src/olap/inverted_index_parser.cpp
+++ b/be/src/olap/inverted_index_parser.cpp
@@ -70,4 +70,13 @@ std::string get_parser_mode_string_from_properties(
         return INVERTED_INDEX_PARSER_FINE_GRANULARITY;
     }
 }
+
+std::string get_parser_phrase_support_string_from_properties(
+        const std::map<std::string, std::string>& properties) {
+    if (properties.find(INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY) != 
properties.end()) {
+        return properties.at(INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY);
+    } else {
+        return INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO;
+    }
+}
 } // namespace doris
diff --git a/be/src/olap/inverted_index_parser.h 
b/be/src/olap/inverted_index_parser.h
index aeb3232c39..87e2ef991a 100644
--- a/be/src/olap/inverted_index_parser.h
+++ b/be/src/olap/inverted_index_parser.h
@@ -49,6 +49,10 @@ const std::string INVERTED_INDEX_PARSER_STANDARD = 
"standard";
 const std::string INVERTED_INDEX_PARSER_ENGLISH = "english";
 const std::string INVERTED_INDEX_PARSER_CHINESE = "chinese";
 
+const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY = "support_phrase";
+const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES = "true";
+const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO = "false";
+
 std::string inverted_index_parser_type_to_string(InvertedIndexParserType 
parser_type);
 
 InvertedIndexParserType get_inverted_index_parser_type_from_string(const 
std::string& parser_str);
@@ -56,5 +60,7 @@ InvertedIndexParserType 
get_inverted_index_parser_type_from_string(const std::st
 std::string get_parser_string_from_properties(const std::map<std::string, 
std::string>& properties);
 std::string get_parser_mode_string_from_properties(
         const std::map<std::string, std::string>& properties);
+std::string get_parser_phrase_support_string_from_properties(
+        const std::map<std::string, std::string>& properties);
 
 } // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp 
b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index 253a187fa4..019499c54f 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -182,6 +182,12 @@ public:
             field_config |= int(lucene::document::Field::INDEX_TOKENIZED);
         }
         _field = new lucene::document::Field(_field_name.c_str(), 
field_config);
+        if 
(get_parser_phrase_support_string_from_properties(_index_meta->properties()) ==
+            INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
+            _field->setOmitTermFreqAndPositions(false);
+        } else {
+            _field->setOmitTermFreqAndPositions(true);
+        }
         _doc->add(*_field);
         return Status::OK();
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org