This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 331e296 KYLIN-4618 Upgrade kylin docker image for kylin 3.1.0
331e296 is described below
commit 331e29699aaf386940d233e5a91d522d5a6516f6
Author: Zhichao Zhang <[email protected]>
AuthorDate: Fri Jul 3 10:44:57 2020 +0800
KYLIN-4618 Upgrade kylin docker image for kylin 3.1.0
Upgrade kylin docker image for kylin 3.1.0
---
docker/Dockerfile | 9 ++++++---
docker/Dockerfile_hadoop | 3 +--
docker/README.md | 10 ++++------
docker/build_image.sh | 3 +--
docker/entrypoint.sh | 7 ++-----
docker/run_container.sh | 2 +-
6 files changed, 15 insertions(+), 19 deletions(-)
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 0d14a8e..936f0ac 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -18,7 +18,7 @@
# Docker image for apache kylin, based on the Hadoop image
FROM hadoop2.7-all-in-one
-ENV KYLIN_VERSION 3.0.1
+ENV KYLIN_VERSION 3.1.0
ENV KYLIN_HOME /home/admin/apache-kylin-$KYLIN_VERSION-bin-hbase1x
# Download released Kylin
@@ -28,7 +28,8 @@ RUN wget
https://archive.apache.org/dist/kylin/apache-kylin-$KYLIN_VERSION/apach
RUN cp $HIVE_HOME/hcatalog/share/hcatalog/hive-hcatalog-core-1.2.1.jar
$SPARK_HOME/jars/
-RUN echo "kylin.engine.livy-conf.livy-url=http://127.0.0.1:8998"; >>
$KYLIN_HOME/conf/kylin.properties \
+RUN echo "kylin.engine.livy-conf.livy-enabled=true" >>
$KYLIN_HOME/conf/kylin.properties \
+ && echo "kylin.engine.livy-conf.livy-url=http://127.0.0.1:8998"; >>
$KYLIN_HOME/conf/kylin.properties \
&& echo
kylin.engine.livy-conf.livy-key.file=hdfs://localhost:9000/kylin/livy/kylin-job-$KYLIN_VERSION.jar
>> $KYLIN_HOME/conf/kylin.properties \
&& echo
kylin.engine.livy-conf.livy-arr.jars=hdfs://localhost:9000/kylin/livy/hbase-client-$HBASE_VERSION.jar,hdfs://localhost:9000/kylin/livy/hbase-common-$HBASE_VERSION.jar,hdfs://localhost:9000/kylin/livy/hbase-hadoop-compat-$HBASE_VERSION.jar,hdfs://localhost:9000/kylin/livy/hbase-hadoop2-compat-$HBASE_VERSION.jar,hdfs://localhost:9000/kylin/livy/hbase-server-$HBASE_VERSION.jar,hdfs://localhost:9000/kylin/livy/htrace-core-*-incubating.jar,hdfs://localhost:9000/kylin/livy/metrics-
[...]
&& echo kylin.source.hive.quote-enabled=false >>
$KYLIN_HOME/conf/kylin.properties \
@@ -39,7 +40,9 @@ RUN echo
"kylin.engine.livy-conf.livy-url=http://127.0.0.1:8998"; >> $KYLIN_HOME
&& echo kylin.engine.spark-conf.spark.executor.memory=1G >>
$KYLIN_HOME/conf/kylin.properties \
&& echo kylin.engine.spark-conf.spark.executor.instances=1 >>
$KYLIN_HOME/conf/kylin.properties \
&& echo kylin.engine.spark-conf.spark.executor.memoryOverhead=512M >>
$KYLIN_HOME/conf/kylin.properties \
- && echo kylin.source.hive.redistribute-flat-table=false >>
$KYLIN_HOME/conf/kylin.properties
+ && echo kylin.source.hive.redistribute-flat-table=false >>
$KYLIN_HOME/conf/kylin.properties \
+ && echo kylin.engine.spark-fact-distinct=true >>
$KYLIN_HOME/conf/kylin.properties \
+ && echo kylin.engine.spark-udc-dictionary=true >>
$KYLIN_HOME/conf/kylin.properties
COPY ./entrypoint.sh /home/admin/entrypoint.sh
RUN chmod u+x /home/admin/entrypoint.sh
diff --git a/docker/Dockerfile_hadoop b/docker/Dockerfile_hadoop
index a0312f4..8afd219 100644
--- a/docker/Dockerfile_hadoop
+++ b/docker/Dockerfile_hadoop
@@ -22,7 +22,6 @@ ENV HIVE_VERSION 1.2.1
ENV HADOOP_VERSION 2.7.0
ENV HBASE_VERSION 1.1.2
ENV SPARK_VERSION 2.3.1
-ENV ZK_VERSION 3.4.6
ENV KAFKA_VERSION 1.1.1
ENV LIVY_VERSION 0.6.0
@@ -35,7 +34,6 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop
ENV HBASE_HOME /home/admin/hbase-$HBASE_VERSION
ENV SPARK_HOME /home/admin/spark-$SPARK_VERSION-bin-hadoop2.6
ENV SPARK_CONF_DIR /home/admin/spark-$SPARK_VERSION-bin-hadoop2.6/conf
-ENV ZK_HOME /home/admin/zookeeper-$ZK_VERSION
ENV KAFKA_HOME /home/admin/kafka_2.11-$KAFKA_VERSION
ENV LIVY_HOME /home/admin/apache-livy-$LIVY_VERSION-incubating-bin
ENV PATH
$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HIVE_HOME/bin:$HBASE_HOME/bin:$MVN_HOME/bin:spark-$SPARK_VERSION-bin-hadoop2.6/bin:$KAFKA_HOME/bin
@@ -91,6 +89,7 @@ RUN wget
https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK
&& cp $HIVE_HOME/conf/hive-site.xml $SPARK_HOME/conf \
&& cp $SPARK_HOME/yarn/*.jar $HADOOP_HOME/share/hadoop/yarn/lib
RUN cp $HIVE_HOME/lib/mysql-connector-java-5.1.24.jar $SPARK_HOME/jars
+RUN cp $HBASE_HOME/lib/hbase-protocol-1.1.2.jar $SPARK_HOME/jars
RUN echo spark.sql.catalogImplementation=hive >
$SPARK_HOME/conf/spark-defaults.conf
# setup kafka
diff --git a/docker/README.md b/docker/README.md
index 7959939..7fee598 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -4,19 +4,17 @@ In order to allow users to easily try Kylin, and to
facilitate developers to ver
- Jdk 1.8
- Hadoop 2.7.0
- Hive 1.2.1
-- Hbase 1.1.2
+- Hbase 1.1.2 (With Zookeeper)
- Spark 2.3.1
-- Zookeeper 3.4.6
- Kafka 1.1.1
- MySQL 5.1.73
-- Maven 3.6.1
## Quickly try Kylin with pre-built images
We have pushed the Kylin images to the [docker
hub](https://hub.docker.com/r/apachekylin/apache-kylin-standalone). You do not
need to build the image locally, just pull the image from remote (you can
browse docker hub to check the available versions):
```
-docker pull apachekylin/apache-kylin-standalone:3.0.1
+docker pull apachekylin/apache-kylin-standalone:3.1.0
```
After the pull is successful, execute "sh run_container.sh" or the following
command to start the container:
@@ -30,7 +28,7 @@ docker run -d \
-p 8032:8032 \
-p 8042:8042 \
-p 16010:16010 \
-apache-kylin-standalone
+apachekylin/apache-kylin-standalone:3.1.0
```
The following services are automatically started when the container starts:
@@ -70,7 +68,7 @@ If you want to login into the Docker container, run "docker
ps" to get the conta
```
> docker ps
CONTAINER ID IMAGE COMMAND
CREATED STATUS PORTS
NAMES
-c15d10ff6bf1 apachekylin/apache-kylin-standalone:3.0.1
"/home/admin/entrypo…" 55 minutes ago Up 55 minutes
0.0.0.0:7070->7070/tcp, 0.0.0.0:8032->8032/tcp, 0.0.0.0:8042->8042/tcp,
0.0.0.0:8088->8088/tcp, 0.0.0.0:50070->50070/tcp, 0.0.0.0:16010->16010/tcp
romantic_moser
+c15d10ff6bf1 apachekylin/apache-kylin-standalone:3.1.0
"/home/admin/entrypo…" 55 minutes ago Up 55 minutes
0.0.0.0:7070->7070/tcp, 0.0.0.0:8032->8032/tcp, 0.0.0.0:8042->8042/tcp,
0.0.0.0:8088->8088/tcp, 0.0.0.0:50070->50070/tcp, 0.0.0.0:16010->16010/tcp
romantic_moser
```
Then run "docker -it <container id> bash" to login it with bash:
diff --git a/docker/build_image.sh b/docker/build_image.sh
index 19fbec5..f78c909 100644
--- a/docker/build_image.sh
+++ b/docker/build_image.sh
@@ -21,7 +21,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1
&& pwd )"
cd ${DIR}
echo "build image in dir "${DIR}
-
echo "start build Hadoop docker image"
docker build -f Dockerfile_hadoop -t hadoop2.7-all-in-one .
-docker build -f Dockerfile -t apache-kylin-standalone .
+docker build -f Dockerfile -t apachekylin/apache-kylin-standalone:3.1.0 .
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index e207f8d..262ea4c 100644
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -45,10 +45,7 @@ $HADOOP_HOME/sbin/yarn-daemon.sh start nodemanager
$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start historyserver
# start hbase
-if [ ! -f "/home/admin/first_run" ]
-then
- rm -rf /data/zookeeper/*
-fi
+rm -rf /data/zookeeper/*
$HBASE_HOME/bin/start-hbase.sh
# start kafka
@@ -82,7 +79,7 @@ then
$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper localhost:2181
--replication-factor 1 --partitions 3 --topic kylin_streaming_topic
fi
-nohup $KYLIN_HOME/bin/kylin.sh
org.apache.kylin.source.kafka.util.KafkaSampleProducer --topic
kylin_streaming_topic --broker localhost:9092 > /dev/null 2>&1 >
/tmp/kafka-sample.log &
+# nohup $KYLIN_HOME/bin/kylin.sh
org.apache.kylin.source.kafka.util.KafkaSampleProducer --topic
kylin_streaming_topic --broker localhost:9092 > /dev/null 2>&1 >
/tmp/kafka-sample.log &
# create sample cube at the first time
if [ ! -f "/home/admin/first_run" ]
then
diff --git a/docker/run_container.sh b/docker/run_container.sh
index 8f03d75..3a3d562 100755
--- a/docker/run_container.sh
+++ b/docker/run_container.sh
@@ -24,4 +24,4 @@ docker run -d \
-p 8042:8042 \
-p 16010:16010 \
-p 8998:8998 \
-apache-kylin-standalone
\ No newline at end of file
+apachekylin/apache-kylin-standalone:3.1.0
