KYLIN-1984: Disable compress in default configuration Signed-off-by: Hongbin Ma <mahong...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/99d0d753 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/99d0d753 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/99d0d753 Branch: refs/heads/v1.5.4-release2 Commit: 99d0d7535ae307a3b34a8ffd23b5d2d50559eea5 Parents: 663820f Author: Yiming Liu <liuyiming....@gmail.com> Authored: Sun Sep 4 17:22:20 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Sun Sep 4 21:15:48 2016 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 4 ++-- build/conf/kylin_hive_conf.xml | 13 ++++++++++++- build/conf/kylin_job_conf.xml | 15 ++++++++++++--- build/conf/kylin_job_conf_inmem.xml | 15 ++++++++++++--- .../org/apache/kylin/common/KylinConfigBase.java | 2 +- .../kylin/storage/hbase/steps/CubeHTableUtil.java | 5 +++-- 6 files changed, 42 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 5607336..c20488a 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -54,8 +54,8 @@ kylin.storage.cleanup.time.threshold=172800000 # Working folder in HDFS, make sure user has the right access to the hdfs directory kylin.hdfs.working.dir=/kylin -# Compression codec for htable, valid value [snappy, lzo, gzip, lz4] -kylin.hbase.default.compression.codec=snappy +# Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4] +kylin.hbase.default.compression.codec=none # HBase Cluster FileSystem, which serving hbase, format as hdfs://hbase-cluster:8020 # Leave empty if hbase running on same cluster with hive and mapreduce http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin_hive_conf.xml ---------------------------------------------------------------------- diff --git a/build/conf/kylin_hive_conf.xml b/build/conf/kylin_hive_conf.xml index 3d6109b..30c4feb 100644 --- a/build/conf/kylin_hive_conf.xml +++ b/build/conf/kylin_hive_conf.xml @@ -39,17 +39,28 @@ <description>enable map-side join</description> </property> + <!-- + The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec, + if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used. + --> + <!-- <property> <name>mapreduce.map.output.compress.codec</name> <value>org.apache.hadoop.io.compress.SnappyCodec</value> <description></description> </property> + --> + <!-- + The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec, + if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used. + --> + <!-- <property> <name>mapreduce.output.fileoutputformat.compress.codec</name> <value>org.apache.hadoop.io.compress.SnappyCodec</value> <description></description> </property> - + --> <property> <name>mapred.output.compression.type</name> <value>BLOCK</value> http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin_job_conf.xml ---------------------------------------------------------------------- diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml index 877e82f..96b806c 100644 --- a/build/conf/kylin_job_conf.xml +++ b/build/conf/kylin_job_conf.xml @@ -31,26 +31,35 @@ <description>Compress map outputs</description> </property> + <!-- + The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec, + if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used. + --> + <!-- <property> <name>mapred.map.output.compression.codec</name> <value>org.apache.hadoop.io.compress.SnappyCodec</value> <description>The compression codec to use for map outputs </description> </property> - + --> <property> <name>mapred.output.compress</name> <value>true</value> <description>Compress the output of a MapReduce job</description> </property> - + <!-- + The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec, + if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used. + --> + <!-- <property> <name>mapred.output.compression.codec</name> <value>org.apache.hadoop.io.compress.SnappyCodec</value> <description>The compression codec to use for job outputs </description> </property> - + --> <property> <name>mapred.output.compression.type</name> <value>BLOCK</value> http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin_job_conf_inmem.xml ---------------------------------------------------------------------- diff --git a/build/conf/kylin_job_conf_inmem.xml b/build/conf/kylin_job_conf_inmem.xml index 1cd809d..fea2f68 100644 --- a/build/conf/kylin_job_conf_inmem.xml +++ b/build/conf/kylin_job_conf_inmem.xml @@ -31,26 +31,35 @@ <description>Compress map outputs</description> </property> + <!-- + The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec, + if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used. + --> + <!-- <property> <name>mapred.map.output.compression.codec</name> <value>org.apache.hadoop.io.compress.SnappyCodec</value> <description>The compression codec to use for map outputs </description> </property> - + --> <property> <name>mapred.output.compress</name> <value>true</value> <description>Compress the output of a MapReduce job</description> </property> - + <!-- + The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec, + if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used. + --> + <!-- <property> <name>mapred.output.compression.codec</name> <value>org.apache.hadoop.io.compress.SnappyCodec</value> <description>The compression codec to use for job outputs </description> </property> - + --> <property> <name>mapred.output.compression.type</name> <value>BLOCK</value> http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 1390e24..f0c91da 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -615,7 +615,7 @@ abstract public class KylinConfigBase implements Serializable { } public String getHbaseDefaultCompressionCodec() { - return getOptional("kylin.hbase.default.compression.codec", ""); + return getOptional("kylin.hbase.default.compression.codec", "none"); } public String getHbaseDefaultEncoding() { http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java index fe65598..9b487a7 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java @@ -183,8 +183,9 @@ public class CubeHTableUtil { cf.setCompressionType(Algorithm.LZ4); break; } + case "none": default: { - logger.info("hbase will not user any compression algorithm to compress data"); + logger.info("hbase will not use any compression algorithm to compress data"); cf.setCompressionType(Algorithm.NONE); } } @@ -194,7 +195,7 @@ public class CubeHTableUtil { DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr); cf.setDataBlockEncoding(encoding); } catch (Exception e) { - logger.info("hbase will not use any encoding"); + logger.info("hbase will not use any encoding", e); cf.setDataBlockEncoding(DataBlockEncoding.NONE); }
