KYLIN-2801 Make default precision and scale in DataType (for hive) configurable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f7999bf8 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f7999bf8 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f7999bf8 Branch: refs/heads/2.1.x Commit: f7999bf89e65bf457b08d25d5d774b0ee48a8ff9 Parents: 7737aee Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Aug 22 14:12:27 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Aug 22 14:56:57 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 41 ++++++++++++++++++++ .../kylin/metadata/datatype/DataType.java | 24 ++++++------ 2 files changed, 54 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f7999bf8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 0f5f3a9..b622825 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -598,6 +598,47 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.source.hive.flat-table-cluster-by-dict-column"); } + public int getDefaultVarcharPrecision() { + int v = Integer.parseInt(getOptional("kylin.source.hive.default-varchar-precision", "256")); + if (v < 1) { + return 256; + } else if (v > 65355) { + return 65535; + } else { + return v; + } + } + + public int getDefaultCharPrecision() { + //at most 255 according to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-CharcharChar + int v = Integer.parseInt(getOptional("kylin.source.hive.default-char-precision", "255")); + if (v < 1) { + return 255; + } else if (v > 255) { + return 255; + } else { + return v; + } + } + + public int getDefaultDecimalPrecision() { + int v = Integer.parseInt(getOptional("kylin.source.hive.default-decimal-precision", "19")); + if (v < 1) { + return 19; + } else { + return v; + } + } + + public int getDefaultDecimalScale() { + int v = Integer.parseInt(getOptional("kylin.source.hive.default-decimal-scale", "4")); + if (v < 1) { + return 4; + } else { + return v; + } + } + // ============================================================================ // SOURCE.KAFKA // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/f7999bf8/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java index 66ff581..13cc2be 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java @@ -31,6 +31,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.BytesSerializer; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.measure.MeasureTypeFactory; @@ -177,21 +178,22 @@ public class DataType implements Serializable { } } - // FIXME 256 for unknown string precision - if ((name.equals("char") || name.equals("varchar")) && precision == -1) { - precision = 256; // to save memory at frontend, e.g. tableau will - // allocate memory according to this + if (precision == -1) { + // FIXME 256 for unknown string precision + + // why 256(255) as default? + // to save memory at frontend, e.g. tableau will + // allocate memory according to this if (name.equals("char")) { - precision -= 1; //at most 255 according to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-CharcharChar + precision = KylinConfig.getInstanceFromEnv().getDefaultCharPrecision(); + } else if (name.equals("varchar")) { + precision = KylinConfig.getInstanceFromEnv().getDefaultVarcharPrecision(); + } else if ((name.equals("decimal") || name.equals("numeric"))) { + precision = KylinConfig.getInstanceFromEnv().getDefaultDecimalPrecision(); + scale = KylinConfig.getInstanceFromEnv().getDefaultDecimalScale(); } } - // FIXME (19,4) for unknown decimal precision - if ((name.equals("decimal") || name.equals("numeric")) && precision == -1) { - precision = 19; - scale = 4; - } - } private String replaceLegacy(String str) {
