[doris] branch dev-1.1.2 updated: [Feature] (ODBC) support read/write emoji of utf16 via odbc table (#11863)

[yiguolei]

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch dev-1.1.2
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/dev-1.1.2 by this push:
     new 54f4a8be57 [Feature] (ODBC) support read/write emoji of utf16 via odbc 
table (#11863)
54f4a8be57 is described below

commit 54f4a8be578c024d4161ea42fa9640d3b3124087
Author: HappenLee <happen...@hotmail.com>
AuthorDate: Thu Aug 18 09:09:02 2022 +0800

    [Feature] (ODBC) support read/write emoji of utf16 via odbc table (#11863)
    
    Co-authored-by: lihaopeng <lihaop...@baidu.com>
---
 be/src/exec/odbc_connector.cpp                     | 26 +++++++---------------
 .../java/org/apache/doris/catalog/Catalog.java     |  1 +
 2 files changed, 9 insertions(+), 18 deletions(-)

diff --git a/be/src/exec/odbc_connector.cpp b/be/src/exec/odbc_connector.cpp
index 1de8337e5d..1a12884cb0 100644
--- a/be/src/exec/odbc_connector.cpp
+++ b/be/src/exec/odbc_connector.cpp
@@ -48,14 +48,9 @@ static constexpr uint32_t BIG_COLUMN_SIZE_BUFFER = 65535;
 // Default max buffer size use in insert to: 50MB, normally a batch is smaller 
than the size
 static constexpr uint32_t INSERT_BUFFER_SIZE = 1024l * 1024 * 50;
 
-static doris::Status utf8_to_wstring(const std::string& str, std::u16string& 
out) {
-    std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> utf8_ucs2_cvt;
-    try {
-        out = utf8_ucs2_cvt.from_bytes(str);
-    } catch (std::range_error& e) {
-        return doris::Status::InternalError("UNICODE out of supported range");
-    }
-    return doris::Status::OK();
+static std::u16string utf8_to_u16string(const char* first, const char* last) {
+    std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> 
utf8_utf16_cvt;
+    return utf8_utf16_cvt.from_bytes(first, last);
 }
 
 namespace doris {
@@ -133,8 +128,7 @@ Status ODBCConnector::query() {
                  "alloc statement");
 
     // Translate utf8 string to utf16 to use unicode encoding
-    std::u16string wquery;
-    RETURN_IF_ERROR(utf8_to_wstring(_sql_str, wquery));
+    auto wquery = utf8_to_u16string(_sql_str.c_str(), _sql_str.c_str() + 
_sql_str.length());
     ODBC_DISPOSE(_stmt, SQL_HANDLE_STMT,
                  SQLExecDirectW(_stmt, (SQLWCHAR*)(wquery.c_str()), SQL_NTS), 
"exec direct");
 
@@ -315,10 +309,8 @@ Status ODBCConnector::append(const std::string& 
table_name, RowBatch* batch,
             }
         }
         // Translate utf8 string to utf16 to use unicode encodeing
-        RETURN_IF_ERROR(utf8_to_wstring(
-                std::string(_insert_stmt_buffer.data(),
-                            _insert_stmt_buffer.data() + 
_insert_stmt_buffer.size()),
-                insert_stmt));
+        insert_stmt = utf8_to_u16string(_insert_stmt_buffer.data(),
+                                        _insert_stmt_buffer.data() + 
_insert_stmt_buffer.size());
     }
 
     {
@@ -501,10 +493,8 @@ Status ODBCConnector::append(const std::string& 
table_name, vectorized::Block* b
             }
         }
         // Translate utf8 string to utf16 to use unicode encodeing
-        RETURN_IF_ERROR(utf8_to_wstring(
-                std::string(_insert_stmt_buffer.data(),
-                            _insert_stmt_buffer.data() + 
_insert_stmt_buffer.size()),
-                insert_stmt));
+        insert_stmt = utf8_to_u16string(_insert_stmt_buffer.data(),
+                                        _insert_stmt_buffer.data() + 
_insert_stmt_buffer.size());
     }
 
     {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java 
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index ffcc3c8172..e5eb9c9232 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -4330,6 +4330,7 @@ public class Catalog {
                 sb.append("\"password\" = \"").append(hidePassword ? "" : 
odbcTable.getPasswd()).append("\",\n");
                 sb.append("\"driver\" = 
\"").append(odbcTable.getOdbcDriver()).append("\",\n");
                 sb.append("\"odbc_type\" = 
\"").append(odbcTable.getOdbcTableTypeName()).append("\",\n");
+                sb.append("\"charest\" = 
\"").append(odbcTable.getCharset()).append("\",\n");
             } else {
                 sb.append("\"odbc_catalog_resource\" = 
\"").append(odbcTable.getOdbcCatalogResourceName()).append("\",\n");
             }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org