This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a3a951c71d [Fix](multi-catalog) Fix load string dict issue for
transactional hive tables. (#23306)
a3a951c71d is described below
commit a3a951c71d6897f74f4b7282efb6b2997317fb47
Author: Qi Chen <kaka11.c...@gmail.com>
AuthorDate: Sat Aug 26 00:09:12 2023 +0800
[Fix](multi-catalog) Fix load string dict issue for transactional hive
tables. (#23306)
Fix load string dict issue for transactional hive tables. The column name
need to pass 'row.column_name'.
apache/doris-thirdparty#112
---
be/src/apache-orc | 2 +-
be/src/vec/exec/format/orc/vorc_reader.cpp | 12 +++++++-----
be/src/vec/exec/format/orc/vorc_reader.h | 4 ++--
regression-test/pipeline/p0/conf/regression-conf.groovy | 4 +---
4 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/be/src/apache-orc b/be/src/apache-orc
index a4e67d732e..78bbe2e41f 160000
--- a/be/src/apache-orc
+++ b/be/src/apache-orc
@@ -1 +1 @@
-Subproject commit a4e67d732e9acf3acb45e85c4cfe84d630e71ec1
+Subproject commit 78bbe2e41f2140b803855d683fae5e1a4b734a37
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp
b/be/src/vec/exec/format/orc/vorc_reader.cpp
index e1274c1959..362157d41b 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -316,22 +316,24 @@ Status OrcReader::_init_read_columns() {
_missing_cols.emplace_back(col_name);
} else {
int pos = std::distance(orc_cols_lower_case.begin(), iter);
+ std::string read_col;
if (_is_acid && i < _column_names->size() -
TransactionalHive::READ_PARAMS.size()) {
- auto read_col = fmt::format(
+ read_col = fmt::format(
"{}.{}",
TransactionalHive::ACID_COLUMN_NAMES[TransactionalHive::ROW_OFFSET],
orc_cols[pos]);
_read_cols.emplace_back(read_col);
} else {
- _read_cols.emplace_back(orc_cols[pos]);
+ read_col = orc_cols[pos];
+ _read_cols.emplace_back(read_col);
}
_read_cols_lower_case.emplace_back(col_name);
// For hive engine, store the orc column name to schema column
name map.
// This is for Hive 1.x orc file with internal column name _col0,
_col1...
if (_is_hive) {
- _file_col_to_schema_col[orc_cols[pos]] = col_name;
+ _removed_acid_file_col_name_to_schema_col[orc_cols[pos]] =
col_name;
}
- _col_name_to_file_col_name[col_name] = orc_cols[pos];
+ _col_name_to_file_col_name[col_name] = read_col;
}
}
return Status::OK();
@@ -804,7 +806,7 @@ Status OrcReader::_init_select_types(const orc::Type& type,
int idx) {
// For hive engine, translate the column name in orc file to schema
column name.
// This is for Hive 1.x which use internal column name _col0, _col1...
if (_is_hive) {
- name = _file_col_to_schema_col[type.getFieldName(i)];
+ name =
_removed_acid_file_col_name_to_schema_col[type.getFieldName(i)];
} else {
name = _get_field_name_lower_case(&type, i);
}
diff --git a/be/src/vec/exec/format/orc/vorc_reader.h
b/be/src/vec/exec/format/orc/vorc_reader.h
index 9b5c1fe576..4f11eb4de1 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.h
+++ b/be/src/vec/exec/format/orc/vorc_reader.h
@@ -510,10 +510,10 @@ private:
std::list<std::string> _read_cols_lower_case;
std::list<std::string> _missing_cols;
std::unordered_map<std::string, int> _colname_to_idx;
- // Column name in Orc file to column name to schema.
+ // Column name in Orc file after removed acid(remove row.) to column name
to schema.
// This is used for Hive 1.x which use internal column name in Orc file.
// _col0, _col1...
- std::unordered_map<std::string, std::string> _file_col_to_schema_col;
+ std::unordered_map<std::string, std::string>
_removed_acid_file_col_name_to_schema_col;
// Flag for hive engine. True if the external table engine is Hive.
bool _is_hive = false;
std::unordered_map<std::string, std::string> _col_name_to_file_col_name;
diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy
b/regression-test/pipeline/p0/conf/regression-conf.groovy
index d26a127c65..7464b28d03 100644
--- a/regression-test/pipeline/p0/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p0/conf/regression-conf.groovy
@@ -54,9 +54,7 @@ testDirectories = ""
// this groups will not be executed
excludeGroups = ""
// this suites will not be executed
-
-excludeSuites =
"test_full_compaction,test_default_limit,test_profile,test_broker_load,test_spark_load,test_refresh_mtmv,test_bitmap_filter,test_export_parquet,test_doris_jdbc_catalog,test_transactional_hive,nereids_delete_mow_partial_update,test_hdfs_tvf"
-
+excludeSuites =
"test_full_compaction,test_default_limit,test_profile,test_broker_load,test_spark_load,test_refresh_mtmv,test_bitmap_filter,test_export_parquet,test_doris_jdbc_catalog,nereids_delete_mow_partial_update,test_hdfs_tvf"
// this directories will not be executed
excludeDirectories = "workload_manager_p1"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
