minor, make calcite work with unicode
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f52eb0c5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f52eb0c5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f52eb0c5 Branch: refs/heads/KYLIN-2606 Commit: f52eb0c5c667b655d9a20f6226af2a3f5cbe0f4e Parents: 2546e00 Author: Li Yang <liy...@apache.org> Authored: Fri Jun 30 13:42:21 2017 +0800 Committer: Hongbin Ma <m...@kyligence.io> Committed: Fri Jun 30 13:46:14 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfig.java | 21 ++++++++++++++++++++ .../apache/kylin/query/ITCombinationTest.java | 3 +-- .../org/apache/kylin/query/KylinTestBase.java | 4 ++-- .../kylin/query/schema/OLAPSchemaFactory.java | 19 ------------------ 4 files changed, 24 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index 1298807..cc08056 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -28,6 +28,7 @@ import java.io.InputStreamReader; import java.io.PrintWriter; import java.io.StringReader; import java.net.URL; +import java.nio.ByteOrder; import java.nio.charset.Charset; import java.util.Enumeration; import java.util.Map; @@ -58,6 +59,26 @@ public class KylinConfig extends KylinConfigBase { // thread-local instances, will override SYS_ENV_INSTANCE private static transient ThreadLocal<KylinConfig> THREAD_ENV_INSTANCE = new ThreadLocal<>(); + + static { + /* + * Make Calcite to work with Unicode. + * + * Sets default char set for string literals in SQL and row types of + * RelNode. This is more a label used to compare row type equality. For + * both SQL string and row record, they are passed to Calcite in String + * object and does not require additional codec. + * + * Ref SaffronProperties.defaultCharset + * Ref SqlUtil.translateCharacterSetName() + * Ref NlsString constructor() + */ + // copied from org.apache.calcite.util.ConversionUtil.NATIVE_UTF16_CHARSET_NAME + String NATIVE_UTF16_CHARSET_NAME = (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) ? "UTF-16BE" : "UTF-16LE"; + System.setProperty("saffron.default.charset", NATIVE_UTF16_CHARSET_NAME); + System.setProperty("saffron.default.nationalcharset", NATIVE_UTF16_CHARSET_NAME); + System.setProperty("saffron.default.collation.name", NATIVE_UTF16_CHARSET_NAME + "$en_US"); + } public static KylinConfig getInstanceFromEnv() { synchronized (KylinConfig.class) { http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java index 0bed5ed..5051db1 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java @@ -18,7 +18,6 @@ package org.apache.kylin.query; -import java.sql.SQLException; import java.util.Arrays; import java.util.Collection; import java.util.Map; @@ -43,7 +42,7 @@ public class ITCombinationTest extends ITKylinQueryTest { private static final Logger logger = LoggerFactory.getLogger(ITCombinationTest.class); @BeforeClass - public static void setUp() throws SQLException { + public static void setUp() { Map<RealizationType, Integer> priorities = Maps.newHashMap(); priorities.put(RealizationType.HYBRID, 0); priorities.put(RealizationType.CUBE, 0); http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index a05d0c3..af05237 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -21,9 +21,9 @@ package org.apache.kylin.query; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; -import java.io.FileReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.Charset; import java.sql.Connection; import java.sql.DriverManager; @@ -166,7 +166,7 @@ public class KylinTestBase { } public static String getTextFromFile(File file) throws IOException { - BufferedReader reader = new BufferedReader(new FileReader(file)); + BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8")); String line = null; StringBuilder stringBuilder = new StringBuilder(); String ls = System.getProperty("line.separator"); http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java index bfe49cf..c098b23 100644 --- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java +++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java @@ -31,7 +31,6 @@ import java.util.Map.Entry; import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.SchemaFactory; import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.util.ConversionUtil; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; @@ -50,24 +49,6 @@ import com.google.common.collect.Maps; public class OLAPSchemaFactory implements SchemaFactory { public static final Logger logger = LoggerFactory.getLogger(OLAPSchemaFactory.class); - static { - /* - * Tricks Optiq to work with Unicode. - * - * Sets default char set for string literals in SQL and row types of - * RelNode. This is more a label used to compare row type equality. For - * both SQL string and row record, they are passed to Optiq in String - * object and does not require additional codec. - * - * Ref SaffronProperties.defaultCharset - * Ref SqlUtil.translateCharacterSetName() - * Ref NlsString constructor() - */ - System.setProperty("saffron.default.charset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME); - System.setProperty("saffron.default.nationalcharset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME); - System.setProperty("saffron.default.collation.name", ConversionUtil.NATIVE_UTF16_CHARSET_NAME + "$en_US"); - } - private final static String SCHEMA_PROJECT = "project"; @Override
