[doris] branch master updated: [fix](hive) check hive transactional table's file format (#20888)

[yiguolei]

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 170cc46b12 [fix](hive) check hive transactional table's file format 
(#20888)
170cc46b12 is described below

commit 170cc46b12f1a1fc65e1d6b815d6aef9fd66eb60
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Mon Jun 19 12:33:00 2023 +0800

    [fix](hive) check hive transactional table's file format (#20888)
    
    Sometimes we meet a hive table with parameter: "transactional" = "true", 
but format is parquet, which is not supported.
    So we need to check the input format for transactional table.
---
 .../apache/doris/catalog/external/HMSExternalTable.java    | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
 
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
index 64e06783c3..fe6603c60c 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
@@ -61,6 +61,7 @@ public class HMSExternalTable extends ExternalTable {
     private static final Logger LOG = 
LogManager.getLogger(HMSExternalTable.class);
 
     private static final Set<String> SUPPORTED_HIVE_FILE_FORMATS;
+    private static final Set<String> SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS;
 
     private static final String TBL_PROP_TXN_PROPERTIES = 
"transactional_properties";
     private static final String TBL_PROP_INSERT_ONLY = "insert_only";
@@ -70,6 +71,9 @@ public class HMSExternalTable extends ExternalTable {
         
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat");
         
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
         
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.mapred.TextInputFormat");
+
+        SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS = Sets.newHashSet();
+        
SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
     }
 
     private static final Set<String> SUPPORTED_HUDI_FILE_FORMATS;
@@ -191,7 +195,15 @@ public class HMSExternalTable extends ExternalTable {
     }
 
     public boolean isHiveTransactionalTable() {
-        return dlaType == DLAType.HIVE && 
AcidUtils.isTransactionalTable(remoteTable);
+        return dlaType == DLAType.HIVE && 
AcidUtils.isTransactionalTable(remoteTable)
+                && isSupportedTransactionalFileFormat();
+    }
+
+    private boolean isSupportedTransactionalFileFormat() {
+        // Sometimes we meet "transactional" = "true" but format is parquet, 
which is not supported.
+        // So we need to check the input format for transactional table.
+        String inputFormatName = remoteTable.getSd().getInputFormat();
+        return inputFormatName != null && 
SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS.contains(inputFormatName);
     }
 
     public boolean isFullAcidTable() {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org