Modified: kylin/site/development/index.html URL: http://svn.apache.org/viewvc/kylin/site/development/index.html?rev=1851595&r1=1851594&r2=1851595&view=diff ============================================================================== --- kylin/site/development/index.html (original) +++ kylin/site/development/index.html Fri Jan 18 02:41:45 2019 @@ -6146,6 +6146,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/index.html" class="list-group-item-lay pjaxlink">Development Quick Guide</a></li> @@ -6172,6 +6176,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/dev_env.html" class="list-group-item-lay pjaxlink">Setup Development Env</a></li> @@ -6250,6 +6258,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/howto_test.html" class="list-group-item-lay pjaxlink">How to Test</a></li> @@ -6286,6 +6298,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/howto_contribute.html" class="list-group-item-lay pjaxlink">How to Contribute</a></li> @@ -6318,6 +6334,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/howto_become_apache_committer.html" class="list-group-item-lay pjaxlink">How to become an Apache Committer</a></li> @@ -6358,6 +6378,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/howto_docs.html" class="list-group-item-lay pjaxlink">How to Write Document</a></li> @@ -6406,6 +6430,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/howto_package.html" class="list-group-item-lay pjaxlink">How to Build Binary Package</a></li> @@ -6450,6 +6478,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/howto_hbase_branches.html" class="list-group-item-lay pjaxlink">How to Maintain Hadoop/HBase Branches</a></li> @@ -6502,6 +6534,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/howto_release.html" class="list-group-item-lay pjaxlink">How to Make Release</a></li> @@ -6570,6 +6606,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/plugin_arch.html" class="list-group-item-lay pjaxlink">Plugin Architecture</a></li> @@ -6634,6 +6674,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/new_metadata.html" class="list-group-item-lay pjaxlink">New Metadata Model</a></li> @@ -6706,6 +6750,10 @@ Static Code Analysis: <a href="https://b + + + + <li class="navlist"> <a href="/development/web_tech.html" class="list-group-item-lay pjaxlink">Kylin Web Summary</a></li> @@ -6729,6 +6777,32 @@ Static Code Analysis: <a href="https://b + + + + + + + + + + + + + + + + + + + + + + <li class="navlist"> + <a href="/development/datasource_sdk.html" class="list-group-item-lay pjaxlink">Develop JDBC Data Source</a></li> + + +
Modified: kylin/site/development/new_metadata.html URL: http://svn.apache.org/viewvc/kylin/site/development/new_metadata.html?rev=1851595&r1=1851594&r2=1851595&view=diff ============================================================================== --- kylin/site/development/new_metadata.html (original) +++ kylin/site/development/new_metadata.html Fri Jan 18 02:41:45 2019 @@ -6142,6 +6142,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/index.html" class="list-group-item-lay pjaxlink">Development Quick Guide</a></li> @@ -6168,6 +6172,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/dev_env.html" class="list-group-item-lay pjaxlink">Setup Development Env</a></li> @@ -6246,6 +6254,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_test.html" class="list-group-item-lay pjaxlink">How to Test</a></li> @@ -6282,6 +6294,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_contribute.html" class="list-group-item-lay pjaxlink">How to Contribute</a></li> @@ -6314,6 +6330,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_become_apache_committer.html" class="list-group-item-lay pjaxlink">How to become an Apache Committer</a></li> @@ -6354,6 +6374,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_docs.html" class="list-group-item-lay pjaxlink">How to Write Document</a></li> @@ -6402,6 +6426,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_package.html" class="list-group-item-lay pjaxlink">How to Build Binary Package</a></li> @@ -6446,6 +6474,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_hbase_branches.html" class="list-group-item-lay pjaxlink">How to Maintain Hadoop/HBase Branches</a></li> @@ -6498,6 +6530,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_release.html" class="list-group-item-lay pjaxlink">How to Make Release</a></li> @@ -6566,6 +6602,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/plugin_arch.html" class="list-group-item-lay pjaxlink">Plugin Architecture</a></li> @@ -6630,6 +6670,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/new_metadata.html" class="list-group-item-lay pjaxlink">New Metadata Model</a></li> @@ -6702,6 +6746,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/web_tech.html" class="list-group-item-lay pjaxlink">Kylin Web Summary</a></li> @@ -6725,6 +6773,32 @@ var _hmt = _hmt || []; + + + + + + + + + + + + + + + + + + + + + + <li class="navlist"> + <a href="/development/datasource_sdk.html" class="list-group-item-lay pjaxlink">Develop JDBC Data Source</a></li> + + + Modified: kylin/site/development/plugin_arch.html URL: http://svn.apache.org/viewvc/kylin/site/development/plugin_arch.html?rev=1851595&r1=1851594&r2=1851595&view=diff ============================================================================== --- kylin/site/development/plugin_arch.html (original) +++ kylin/site/development/plugin_arch.html Fri Jan 18 02:41:45 2019 @@ -6164,6 +6164,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/index.html" class="list-group-item-lay pjaxlink">Development Quick Guide</a></li> @@ -6190,6 +6194,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/dev_env.html" class="list-group-item-lay pjaxlink">Setup Development Env</a></li> @@ -6268,6 +6276,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_test.html" class="list-group-item-lay pjaxlink">How to Test</a></li> @@ -6304,6 +6316,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_contribute.html" class="list-group-item-lay pjaxlink">How to Contribute</a></li> @@ -6336,6 +6352,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_become_apache_committer.html" class="list-group-item-lay pjaxlink">How to become an Apache Committer</a></li> @@ -6376,6 +6396,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_docs.html" class="list-group-item-lay pjaxlink">How to Write Document</a></li> @@ -6424,6 +6448,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_package.html" class="list-group-item-lay pjaxlink">How to Build Binary Package</a></li> @@ -6468,6 +6496,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_hbase_branches.html" class="list-group-item-lay pjaxlink">How to Maintain Hadoop/HBase Branches</a></li> @@ -6520,6 +6552,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_release.html" class="list-group-item-lay pjaxlink">How to Make Release</a></li> @@ -6588,6 +6624,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/plugin_arch.html" class="list-group-item-lay pjaxlink">Plugin Architecture</a></li> @@ -6652,6 +6692,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/new_metadata.html" class="list-group-item-lay pjaxlink">New Metadata Model</a></li> @@ -6724,6 +6768,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/web_tech.html" class="list-group-item-lay pjaxlink">Kylin Web Summary</a></li> @@ -6747,6 +6795,32 @@ var _hmt = _hmt || []; + + + + + + + + + + + + + + + + + + + + + + <li class="navlist"> + <a href="/development/datasource_sdk.html" class="list-group-item-lay pjaxlink">Develop JDBC Data Source</a></li> + + + Modified: kylin/site/development/web_tech.html URL: http://svn.apache.org/viewvc/kylin/site/development/web_tech.html?rev=1851595&r1=1851594&r2=1851595&view=diff ============================================================================== --- kylin/site/development/web_tech.html (original) +++ kylin/site/development/web_tech.html Fri Jan 18 02:41:45 2019 @@ -6172,6 +6172,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/index.html" class="list-group-item-lay pjaxlink">Development Quick Guide</a></li> @@ -6198,6 +6202,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/dev_env.html" class="list-group-item-lay pjaxlink">Setup Development Env</a></li> @@ -6276,6 +6284,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_test.html" class="list-group-item-lay pjaxlink">How to Test</a></li> @@ -6312,6 +6324,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_contribute.html" class="list-group-item-lay pjaxlink">How to Contribute</a></li> @@ -6344,6 +6360,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_become_apache_committer.html" class="list-group-item-lay pjaxlink">How to become an Apache Committer</a></li> @@ -6384,6 +6404,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_docs.html" class="list-group-item-lay pjaxlink">How to Write Document</a></li> @@ -6432,6 +6456,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_package.html" class="list-group-item-lay pjaxlink">How to Build Binary Package</a></li> @@ -6476,6 +6504,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_hbase_branches.html" class="list-group-item-lay pjaxlink">How to Maintain Hadoop/HBase Branches</a></li> @@ -6528,6 +6560,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/howto_release.html" class="list-group-item-lay pjaxlink">How to Make Release</a></li> @@ -6596,6 +6632,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/plugin_arch.html" class="list-group-item-lay pjaxlink">Plugin Architecture</a></li> @@ -6660,6 +6700,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/new_metadata.html" class="list-group-item-lay pjaxlink">New Metadata Model</a></li> @@ -6732,6 +6776,10 @@ var _hmt = _hmt || []; + + + + <li class="navlist"> <a href="/development/web_tech.html" class="list-group-item-lay pjaxlink">Kylin Web Summary</a></li> @@ -6755,6 +6803,32 @@ var _hmt = _hmt || []; + + + + + + + + + + + + + + + + + + + + + + <li class="navlist"> + <a href="/development/datasource_sdk.html" class="list-group-item-lay pjaxlink">Develop JDBC Data Source</a></li> + + + Modified: kylin/site/docs/release_notes.html URL: http://svn.apache.org/viewvc/kylin/site/docs/release_notes.html?rev=1851595&r1=1851594&r2=1851595&view=diff ============================================================================== --- kylin/site/docs/release_notes.html (original) +++ kylin/site/docs/release_notes.html Fri Jan 18 02:41:45 2019 @@ -6035,8 +6035,8 @@ there are source code package, binary pa <p>or send to Apache Kylin mailing list:</p> <ul> - <li>User relative: <a href="mailto:u...@kylin.apache.org";>u...@kylin.apache.org</a></li> - <li>Development relative: <a href="mailto:d...@kylin.apache.org";>d...@kylin.apache.org</a></li> + <li>User relative: <a href="mailto:user@kylin.apache.org">user@kylin.apache.org</a></li> + <li>Development relative: <a href="mailto:dev@kylin.apache.org">dev@kylin.apache.org</a></li> </ul> <h2 id="v260---2019-01-12">v2.6.0 - 2019-01-12</h2> @@ -8183,7 +8183,7 @@ This version includes many bug fixs/enha <li>[KYLIN-1396] - minor bug in BigDecimalSerializer - avoidVerbose should be incremented each time when input scale is larger than given scale</li> <li>[KYLIN-1419] - NullPointerException occurs when query from subqueries with order by</li> <li>[KYLIN-1445] - Kylin should throw error if HIVE_CONF dir cannot be found</li> - <li>[KYLIN-1466] - Some environment variables are not used in bin/kylin.sh <RUNNABLE_CLASS_NAME></RUNNABLE_CLASS_NAME></li> + <li>[KYLIN-1466] - Some environment variables are not used in bin/kylin.sh <runnable_class_name></runnable_class_name></li> <li>[KYLIN-1469] - Hive dependency jars are hard coded in test</li> <li>[KYLIN-1471] - LIMIT after having clause should not be pushed down to storage context</li> <li>[KYLIN-1473] - Cannot have comments in the end of New Query textbox</li> @@ -8282,7 +8282,7 @@ This version includes many bug fixs/enha <li>[KYLIN-1443] - For setting Auto Merge Time Ranges, before sending them to backend, the related time ranges should be sorted increasingly</li> <li>[KYLIN-1445] - Kylin should throw error if HIVE_CONF dir cannot be found</li> <li>[KYLIN-1456] - Shouldnât use â1970-01-01â as the default end date</li> - <li>[KYLIN-1466] - Some environment variables are not used in bin/kylin.sh <RUNNABLE_CLASS_NAME></RUNNABLE_CLASS_NAME></li> + <li>[KYLIN-1466] - Some environment variables are not used in bin/kylin.sh <runnable_class_name></runnable_class_name></li> <li>[KYLIN-1469] - Hive dependency jars are hard coded in test</li> </ul> Modified: kylin/site/feed.xml URL: http://svn.apache.org/viewvc/kylin/site/feed.xml?rev=1851595&r1=1851594&r2=1851595&view=diff ============================================================================== --- kylin/site/feed.xml (original) +++ kylin/site/feed.xml Fri Jan 18 02:41:45 2019 @@ -19,11 +19,49 @@ <description>Apache Kylin Home</description> <link>http://kylin.apache.org/</link> <atom:link href="http://kylin.apache.org/feed.xml"; rel="self" type="application/rss+xml"/> - <pubDate>Sun, 13 Jan 2019 18:04:49 -0800</pubDate> - <lastBuildDate>Sun, 13 Jan 2019 18:04:49 -0800</lastBuildDate> + <pubDate>Thu, 17 Jan 2019 18:33:24 -0800</pubDate> + <lastBuildDate>Thu, 17 Jan 2019 18:33:24 -0800</lastBuildDate> <generator>Jekyll v2.5.3</generator> <item> + <title>Introduce data source SDK</title> + <description><h2 id="data-source-sdk">Data source SDK</h2> + +<p>Apache Kylin has already supported several data sources like Amazon Redshift, SQL Server through JDBC. But we found that it takes much efforts to develop an implementation to a new source engine, like supporting metadata sync, cube build and query pushdown. Itâs mainly because the SQL dialects and jdbc implementations between source engines are quite different.</p> + +<p>So since 2.6.0, Kylin provides a new data source SDK, which provides APIs to help developers handle these dialect differences and easily implement a new data source through JDBC.</p> + +<p>With this SDK, users can achieve followings from a JDBC source:</p> + +<ul> + <li>Synchronize metadata and data from JDBC source</li> + <li>Build cube from JDBC source</li> + <li>Query pushdown to JDBC source engine when cube is unmatched</li> +</ul> + +<h2 id="structure">Structure</h2> + +<p class="center"><img src="/images/blog/data-source-sdk.png" alt="" /></p> + +<p>When users want to synchronize metadata or get data from data source, the request pass through the framework, and the framework find the adaptor what has an API for metadata and data.</p> + +<p>To avoid having complex adaptors, when having a push-down query, framework provides sql conversions from ansi sql to target data source dialect(includes sql functions and sql types), and adaptor just provide a function <em>fixSql</em> to fix the sql after conversion.</p> + +<h2 id="how-to-develop">How to develop</h2> + +<p>Please follow this <a href="/development/datasource_sdk.html">doc</a></p> + +</description> + <pubDate>Wed, 16 Jan 2019 12:00:00 -0800</pubDate> + <link>http://kylin.apache.org/blog/2019/01/16/introduce-data-source-sdk-v2.6.0/</link> + <guid isPermaLink="true">http://kylin.apache.org/blog/2019/01/16/introduce-data-source-sdk-v2.6.0/</guid> + + + <category>blog</category> + + </item> + + <item> <title>Why did Meituan develop Kylin On Druid (part 1 of 2)?</title> <description><h2 id="preface">Preface</h2> @@ -31,12 +69,12 @@ <p>During the Apache Kylin Meetup in August 2018, the Meituan team shared their Kylin on Druid (KoD) solution. Why did they develop this hybrid system? Whatâs the rationale behind it? This article will answer these questions and help you to understand the differences and the pros and cons of each OLAP engine.</p> -<h2 id="01-introduction-to-apache-kylin">01 Introduction to Apache Kylin</h2> +<h2 id="introduction-to-apache-kylin">01 Introduction to Apache Kylin</h2> <p>Apache Kylin is an open source distributed big data analytics engine. It constructs data models on top of huge datasets, builds pre-calculated Cubes to support multi-dimensional analysis, and provides a SQL query interface and multi-dimensional analysis on top of Hadoop, with general ODBC, JDBC, and RESTful API interfaces. Apache Kylinâs unique pre-calculation ability enables it to handle extremely large datasets with sub-second query response times.<br /> <img src="/images/blog/Kylin-On-Durid/1 kylin_architecture.png" alt="" /><br /> Graphic 1 Kylin Architecture</p> -<h2 id="02-apache-kylins-advantage">02 Apache Kylinâs Advantage</h2> +<h2 id="apache-kylins-advantage">02 Apache Kylinâs Advantage</h2> <ol> <li>The mature, Hadoop-based computing engines (MapReduce and Spark) that provide strong capability of pre-calculation on super large datasets, which can be deployed out-of-the-box on any mainstream Hadoop platform.</li> <li>Support of ANSI SQL that allows users to do data analysis with SQL directly.</li> @@ -48,34 +86,34 @@ Graphic 1 Kylin Architecture</p> <li>Support of both batch loading of super large historical datasets and micro-batches of data streams.</li> </ol> -<h2 id="03-introduction-to-apache-druid-incubating">03 Introduction to Apache Druid (incubating)</h2> +<h2 id="introduction-to-apache-druid-incubating">03 Introduction to Apache Druid (incubating)</h2> <p>Druid was created in 2012. Itâs an open source distributed data store. Its core design combines the concept of analytical databases, time-series databases, and search systems, and it can support data collection and analytics on fairly large datasets. Druid uses an Apache V2 license and is an Apache incubator project.</p> <p>Druid Architecture<br /> From the perspective of deployment architectures, Druidâs processes mostly fall into 3 categories based on their roles.</p> -<h3 id="-data-node-slave-node-for-data-ingestion-and-calculation">⢠Data Node (Slave node for data ingestion and calculation)</h3> +<h3 id="data-node-slave-node-for-data-ingestion-and-calculation">⢠Data Node (Slave node for data ingestion and calculation)</h3> <p>The Historical node is in charge of loading segments (committed immutable data) and receiving queries on historical data.<br /> Middle Manager is in charge of data ingestion and commit segments. Each task is done by a separate JVM. <br /> Peon is in charge of completing a single task, which is managed and monitored by the Middle Manager.</p> -<h3 id="-query-node">⢠Query Node</h3> +<h3 id="query-node">⢠Query Node</h3> <p>Broker receives query requests, determines on which segment the data resides, and distributes sub-queries and merges query results.</p> -<h3 id="-master-node-task-coordinator-and-cluster-manager">⢠Master Node (Task Coordinator and Cluster Managerï¼</h3> +<h3 id="master-node-task-coordinator-and-cluster-manager">⢠Master Node (Task Coordinator and Cluster Managerï¼</h3> <p>Coordinator monitors Historical nodes, dispatches segments and monitor workload.<br /> Overlord monitors Middle Manager, dispatches tasks to Middle Manager, and assists releasing of segments.</p> <h3 id="external-dependency">External Dependency</h3> <p>At the same time, Druid has 3 replaceable external dependencies.</p> -<h3 id="-deep-storage-distributed-storage">⢠Deep Storage (distributed storage)</h3> +<h3 id="deep-storage-distributed-storage">⢠Deep Storage (distributed storage)</h3> <p>Druid uses Deep storage to transfer data files between nodes.</p> -<h3 id="-metadata-storage">⢠Metadata Storage</h3> +<h3 id="metadata-storage">⢠Metadata Storage</h3> <p>Metadata Storage stores the metadata about segment positions and task output.</p> -<h3 id="-zookeeper-cluster-management-and-task-coordination">⢠Zookeeper (cluster management and task coordination)</h3> +<h3 id="zookeeper-cluster-management-and-task-coordination">⢠Zookeeper (cluster management and task coordination)</h3> <p>Druid uses Zookeeper (ZK) to ensure consistency of the cluster status.<br /> <img src="/images/blog/Kylin-On-Durid/2 druid_architecture.png" alt="" /><br /> Graphic 2 Druid Architecture</p> @@ -98,7 +136,7 @@ Graphic 4 Druid Schema</p> <li>Separation of cold/hot data.</li> </ol> -<h2 id="04-why-did-meituan-develop-kylin-on-druid">04 Why did Meituan develop Kylin on Druid?</h2> +<h2 id="why-did-meituan-develop-kylin-on-druid">04 Why did Meituan develop Kylin on Druid?</h2> <p>Meituan deployed into production an offline OLAP platform with Apache Kylin as its core component in 2015. Since then the platform has served almost all business lines with fast growing data volume and query executions, and the stress on the cluster has increased accordingly. Throughout the time, the tech team in Meituan keeps exploring better solutions for some of Kylinâs challenges. The major one is Apache HBase, the storage that Kylin relies on.</p> <p>Kylin stores its data in HBase by converting the Dimensions and Measures into HBase Keys and Values, respectively. As HBase doesnât support secondary index and only has one RowKey index, Kylinâs Dimension values will be combined into a fixed sequence to store as RowKey. In this way, filtering on a Dimension in the front of the sequence will perform better than those at the back. Hereâs an example:</p> @@ -112,13 +150,14 @@ Graphic 6 Cube2 RowKey Sequence</p> <p><strong>Now letâs query each Cube with the same SQL and compare the response time.</strong></p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>select S_SUPPKEY, C_CUSTKEY, sum(LO_EXTENDEDPRICE) as m1 +<div class="highlighter-rouge"><pre class="highlight"><code>select S_SUPPKEY, C_CUSTKEY, sum(LO_EXTENDEDPRICE) as m1 from P_LINEORDER left join SUPPLIER on P_LINEORDER.LO_SUPPKEY = SUPPLIER.S_SUPPKEY left join CUSTOMER on P_LINEORDER.LO_CUSTKEY = CUSTOMER.C_CUSTKEY WHERE (LO_ORDERKEY &gt; 1799905 and LO_ORDERKEY &lt; 1799915) or (LO_ORDERKEY &gt; 1999905 and LO_ORDERKEY &lt; 1999935) GROUP BY S_SUPPKEY, C_CUSTKEY; -</code></pre></div></div> +</code></pre> +</div> <p><strong>Below shows the time consumed and data scanned:</strong><br /> <img src="/images/blog/Kylin-On-Durid/7 cube1_query_log.png" alt="" /><br /> @@ -135,12 +174,11 @@ Graphic 8 Cube1 Query Log</p> <p>Kylinâs query performance and user experience can be greatly improved with pure columnar storage and multiple indexes on Dimensions. As analyzed above, Druid happens to meet the requirements of columnar + multi-index. So the Meituan Kylin team decided to try replacing HBase with Druid.</p> -<p>Why not just use Druid then? Meituanâs engineers shared their thoughts:</p> +<p>Why not just use Druid then? Meituanâs engineers shared their thoughts:<br /> +1. Druidâs native query language is in its own specific JSON format, which is not as easy to pick up as SQL. Although the Druid community added SQL support later on, the support is not complete and does not meet the data analystsâ requirement of complex SQL queries. On the contrary, Kylin natively supports ANSI SQL, uses Apache Calcite for semantic parsing, and supports SQL features such as join, sub query, window functions, etc. In addition, it provides standard interfaces including ODBC/JDBC, and can directly connect with BI tools such as Tableau, Power BI, Superset, and Redash.</p> + <ol> <li> - <p>Druidâs native query language is in its own specific JSON format, which is not as easy to pick up as SQL. Although the Druid community added SQL support later on, the support is not complete and does not meet the data analystsâ requirement of complex SQL queries. On the contrary, Kylin natively supports ANSI SQL, uses Apache Calcite for semantic parsing, and supports SQL features such as join, sub query, window functions, etc. In addition, it provides standard interfaces including ODBC/JDBC, and can directly connect with BI tools such as Tableau, Power BI, Superset, and Redash.</p> - </li> - <li> <p>Druid can support only single-table query. Multi-table joins are very common in practice, but they cannot be supported by Druid. Kylin, however, supports Star Schema and Snowflake Schema, satisfying multi-table join requirements.</p> </li> <li> @@ -167,9 +205,9 @@ Graphic 8 Cube1 Query Log</p> <p>Therefore, it appears to be a promising OLAP solution to combine Druidâs excellent columnar storage with Kylinâs usability, compatibility, and completeness. Druid has columnar storage, inverted index, better filtering performance than HBase, native OLAP features, and good secondary aggregation capabilities. Meituanâs tech team decided to try replacing HBase with Druid as the storage for Kylin.</p> -<h2 id="05">05</h2> -<h3 id="kylin-on-druid-design">Kylin on Druid Design</h3> -<p>At v1.5, Apache Kylin introduced plugable architecture and de-coupled computing and storage components, which makes the replacement of HBase possible. Here is a brief introduction to the main design concept of Kylin on Druid based on Meituan engineer Kaisen Kangâs design doc. (Graphics 9 and 10 are from reference[1], and text are from reference[1] and [3])</p> +<h2 id="section">05</h2> +<p>### Kylin on Druid Design<br /> +At v1.5, Apache Kylin introduced plugable architecture and de-coupled computing and storage components, which makes the replacement of HBase possible. Here is a brief introduction to the main design concept of Kylin on Druid based on Meituan engineer Kaisen Kangâs design doc. (Graphics 9 and 10 are from reference[1], and text are from reference[1] and [3])</p> <h3 id="process-of-building-cube">Process of Building Cube</h3> <ol> @@ -202,12 +240,12 @@ Graphic 10 Process of Querying Cube</ <li>Kylin measure columns map to Druid measure columns.</li> </ol> -<h2 id="06-summary">06 Summary</h2> +<h2 id="summary">06 Summary</h2> <p>In this article, we first analyzed features and pros/cons of both Kylin and Druid, and the reasons for poor performance of Hbase in Kylin in some cases. Then we searched solutions and found the feasible option of using Druid as the Kylin storage engine. At last, we illustrated the Kylin-on-Druid architecture and the processes developed by Meituan.</p> <p>Stay tuned for our next article about how to use Kylin on Druid, how it performs, and how it can be improved.</p> -<h2 id="07-reference">07 Reference</h2> +<h2 id="reference">07 Reference</h2> <ol> <li> @@ -225,7 +263,7 @@ Graphic 10 Process of Querying Cube</ </ol> </description> - <pubDate>Wed, 12 Dec 2018 17:30:00 +0000</pubDate> + <pubDate>Wed, 12 Dec 2018 09:30:00 -0800</pubDate> <link>http://kylin.apache.org/blog/2018/12/12/why-did-meituan-develop-kylin-on-druid-part1-of-2/</link> <guid isPermaLink="true">http://kylin.apache.org/blog/2018/12/12/why-did-meituan-develop-kylin-on-druid-part1-of-2/</guid> @@ -242,37 +280,37 @@ Graphic 10 Process of Querying Cube</ <p>è¿æ¯ç»§2.4.0 åçä¸ä¸ªæ°åè½çæ¬ã该çæ¬å¼å ¥äºå¾å¤æä»·å¼çæ¹è¿ï¼å®æ´çæ¹å¨å表请åè§<a href="https://kylin.apache.org/docs/release_notes.html">release notes</a>ï¼è¿éæä¸äºä¸»è¦æ¹è¿å说æï¼</p> -<h3 id="all-in-spark-ç-cubing-å¼æ">All-in-Spark ç Cubing å¼æ</h3> +<h3 id="all-in-spark--cubing-">All-in-Spark ç Cubing å¼æ</h3> <p>Kylin ç Spark å¼æå°ä½¿ç¨ Spark è¿è¡ cube 计ç®ä¸çææåå¸å¼ä½ä¸ï¼å æ¬è·åå个维度çä¸åå¼ï¼å° cuboid æ件转æ¢ä¸º HBase HFileï¼å并 segmentï¼å并è¯å ¸çãé»è®¤ç Spark é ç½®ä¹ç»è¿ä¼åï¼ä½¿å¾ç¨æ·å¯ä»¥è·å¾å¼ç®±å³ç¨çä½éªãç¸å ³å¼åä»»å¡æ¯ KYLIN-3427, KYLIN-3441, KYLIN-3442.</p> <p>Spark ä»»å¡ç®¡çä¹æææ¹è¿ï¼ä¸æ¦ Spark ä»»å¡å¼å§è¿è¡ï¼æ¨å°±å¯ä»¥å¨Webæ§å¶å°ä¸è·å¾ä½ä¸é¾æ¥ï¼å¦ææ¨ä¸¢å¼è¯¥ä½ä¸ï¼Kylin å°ç«å»ç»æ¢ Spark ä½ä¸ä»¥åæ¶éæ¾èµæºï¼å¦æéæ°å¯å¨ Kylinï¼å®å¯ä»¥ä»ä¸ä¸ä¸ªä½ä¸æ¢å¤ï¼èä¸æ¯éæ°æ交æ°ä½ä¸.</p> -<h3 id="mysql-å-kylin-å æ°æ®çåå¨">MySQL å Kylin å æ°æ®çåå¨</h3> +<h3 id="mysql--kylin-">MySQL å Kylin å æ°æ®çåå¨</h3> <p>å¨è¿å»ï¼HBase æ¯ Kylin å æ°æ®åå¨çå¯ä¸éæ©ã å¨æäºæ åµä¸ HBaseä¸éç¨ï¼ä¾å¦ä½¿ç¨å¤ä¸ª HBase é群æ¥ä¸º Kylin æä¾è·¨åºåçé«å¯ç¨ï¼è¿éå¤å¶ç HBase é群æ¯åªè¯»çï¼æ以ä¸è½åå æ°æ®åå¨ãç°å¨æ们å¼å ¥äº MySQL Metastore 以满足è¿ç§éæ±ãæ¤åè½ç°å¨å¤äºæµè¯é¶æ®µãæ´å¤å 容åè§ KYLIN-3488ã</p> -<h3 id="hybrid-model-å¾å½¢çé¢">Hybrid model å¾å½¢çé¢</h3> +<h3 id="hybrid-model-">Hybrid model å¾å½¢çé¢</h3> <p>Hybrid æ¯ä¸ç§ç¨äºç»è£ å¤ä¸ª cube çé«çº§æ¨¡åã å®å¯ç¨äºæ»¡è¶³ cube ç schema è¦åçæ¹åçæ åµãè¿ä¸ªåè½è¿å»æ²¡æå¾å½¢çé¢ï¼å æ¤åªæä¸å°é¨åç¨æ·ç¥éå®ãç°å¨æä»¬å¨ Web çé¢ä¸å¼å¯äºå®ï¼ä»¥ä¾¿æ´å¤ç¨æ·å¯ä»¥å°è¯ã</p> -<h3 id="é»è®¤å¼å¯-cube-planner">é»è®¤å¼å¯ Cube planner</h3> +<h3 id="cube-planner">é»è®¤å¼å¯ Cube planner</h3> <p>Cube planner å¯ä»¥æ大å°ä¼å cube ç»æï¼åå°æ建ç cuboid æ°éï¼ä»èèç计ç®/åå¨èµæºå¹¶æé«æ¥è¯¢æ§è½ãå®æ¯å¨v2.3ä¸å¼å ¥çï¼ä½é»è®¤æ åµä¸æ²¡æå¼å¯ã为äºè®©æ´å¤ç¨æ·çå°å¹¶å°è¯å®ï¼æ们é»è®¤å¨v2.5ä¸å¯ç¨å®ã ç®æ³å°å¨ç¬¬ä¸æ¬¡æ建 segment çæ¶åï¼æ ¹æ®æ°æ®ç»è®¡èªå¨ä¼å cuboid éå.</p> -<h3 id="æ¹è¿ç-segment-åªæ">æ¹è¿ç Segment åªæ</h3> +<h3 id="segment-">æ¹è¿ç Segment åªæ</h3> <p>Segmentï¼ååºï¼ä¿®åªå¯ä»¥ææå°åå°ç£çåç½ç»I / Oï¼å æ¤å¤§å¤§æé«äºæ¥è¯¢æ§è½ã è¿å»ï¼Kylin åªæååºå (partition date column) çå¼è¿è¡ segment çä¿®åªã å¦ææ¥è¯¢ä¸æ²¡æå°ååºåä½ä¸ºè¿æ»¤æ¡ä»¶ï¼é£ä¹ä¿®åªå°ä¸èµ·ä½ç¨ï¼ä¼æ«æææsegmentã.<br /> ç°å¨ä»v2.5å¼å§ï¼Kylin å°å¨ segment 级å«è®°å½æ¯ä¸ªç»´åº¦çæå°/æ大å¼ã å¨æ«æ segment ä¹åï¼ä¼å°æ¥è¯¢çæ¡ä»¶ä¸æå°/æ大索å¼è¿è¡æ¯è¾ã å¦æä¸å¹é ï¼å°è·³è¿è¯¥ segmentã æ£æ¥KYLIN-3370äºè§£æ´å¤ä¿¡æ¯ã</p> -<h3 id="å¨-yarn-ä¸å并åå ¸">å¨ YARN ä¸å并åå ¸</h3> +<h3 id="yarn-">å¨ YARN ä¸å并åå ¸</h3> <p>å½ segment å并æ¶ï¼å®ä»¬çè¯å ¸ä¹éè¦å并ãå¨è¿å»ï¼åå ¸å并åçå¨ Kylin ç JVM ä¸ï¼è¿éè¦ä½¿ç¨å¤§éçæ¬å°å åå CPU èµæºã å¨æ端æ åµä¸ï¼å¦ææå 个并åä½ä¸ï¼ï¼å¯è½ä¼å¯¼è´ Kylin è¿ç¨å´©æºã å æ¤ï¼ä¸äºç¨æ·ä¸å¾ä¸ä¸º Kylin ä»»å¡èç¹åé æ´å¤å åï¼æè¿è¡å¤ä¸ªä»»å¡èç¹ä»¥å¹³è¡¡å·¥ä½è´è½½ã<br /> ç°å¨ä»v2.5å¼å§ï¼Kylin å°æè¿é¡¹ä»»å¡æäº¤ç» Hadoop MapReduce å Sparkï¼è¿æ ·å°±å¯ä»¥è§£å³è¿ä¸ªç¶é¢é®é¢ã æ¥çKYLIN-3471äºè§£æ´å¤ä¿¡æ¯.</p> -<h3 id="æ¹è¿ä½¿ç¨å ¨å±åå ¸ç-cube-æ建æ§è½">æ¹è¿ä½¿ç¨å ¨å±åå ¸ç cube æ建æ§è½</h3> +<h3 id="cube-">æ¹è¿ä½¿ç¨å ¨å±åå ¸ç cube æ建æ§è½</h3> <p>å ¨å±åå ¸ (Global Dictionary) æ¯ bitmap 精确å»é计æ°çå¿ è¦æ¡ä»¶ãå¦æå»éåå ·æé常é«çåºæ°ï¼å GD å¯è½é常大ãå¨ cube æ建é¶æ®µï¼Kylin éè¦éè¿ GD å°éæ´æ°å¼è½¬æ¢ä¸ºæ´æ°ã尽管 GD 已被åæå¤ä¸ªåçï¼å¯ä»¥åå¼å è½½å°å åï¼ä½æ¯ç±äºå»éåçå¼æ¯ä¹±åºçãKylin éè¦åå¤è½½å ¥åè½½åº(swap in/out)åçï¼è¿ä¼å¯¼è´æ建任å¡é常ç¼æ ¢ã<br /> 该å¢å¼ºåè½å¼å ¥äºä¸ä¸ªæ°æ¥éª¤ï¼ä¸ºæ¯ä¸ªæ°æ®åä»å ¨å±åå ¸ä¸æ建ä¸ä¸ªç¼©å°çåå ¸ã éåæ¯ä¸ªä»»å¡åªéè¦å 载缩å°çåå ¸ï¼ä»èé¿å é¢ç¹çè½½å ¥åè½½åºãæ§è½å¯ä»¥æ¯ä»¥åå¿«3åãæ¥ç KYLIN-3491 äºè§£æ´å¤ä¿¡æ¯.</p> -<h3 id="æ¹è¿å«-topn-count-distinct-ç-cube-大å°ç估计">æ¹è¿å« TOPN, COUNT DISTINCT ç cube 大å°ç估计</h3> +<h3 id="topn-count-distinct--cube-">æ¹è¿å« TOPN, COUNT DISTINCT ç cube 大å°ç估计</h3> <p>Cube ç大å°å¨æ建æ¶æ¯é¢å 估计çï¼å¹¶è¢«åç»å 个æ¥éª¤ä½¿ç¨ï¼ä¾å¦å³å® MR / Spark ä½ä¸çååºæ°ï¼è®¡ç® HBase region åå²çãå®çåç¡®ä¸å¦ä¼å¯¹æ建æ§è½äº§çå¾å¤§å½±åã å½åå¨ COUNT DISTINCTï¼TOPN ç度éæ¶åï¼å 为å®ä»¬ç大å°æ¯çµæ´»çï¼å æ¤ä¼°è®¡å¼å¯è½è·çå®å¼æå¾å¤§åå·®ã å¨è¿å»ï¼ç¨æ·éè¦è°æ´è¥å¹²ä¸ªåæ°ä»¥ä½¿å°ºå¯¸ä¼°è®¡æ´æ¥è¿å®é 尺寸ï¼è¿å¯¹æ®éç¨æ·æç¹å°é¾ã<br /> ç°å¨ï¼Kylin å°æ ¹æ®æ¶éçç»è®¡ä¿¡æ¯èªå¨è°æ´å¤§å°ä¼°è®¡ãè¿å¯ä»¥ä½¿ä¼°è®¡å¼ä¸å®é 大å°æ´æ¥è¿ãæ¥ç KYLIN-3453 äºè§£æ´å¤ä¿¡æ¯ã</p> -<h3 id="æ¯æhadoop-30hbase-20">æ¯æHadoop 3.0/HBase 2.0</h3> +<h3 id="hadoop-30hbase-20">æ¯æHadoop 3.0/HBase 2.0</h3> <p>Hadoop 3å HBase 2å¼å§è¢«è®¸å¤ç¨æ·éç¨ãç°å¨ Kylin æä¾ä½¿ç¨æ°ç Hadoop å HBase API ç¼è¯çæ°äºè¿å¶å ãæ们已ç»å¨ Hortonworks HDP 3.0 å Cloudera CDH 6.0 ä¸è¿è¡äºæµè¯</p> <p><strong>ä¸è½½</strong></p> @@ -289,7 +327,7 @@ Graphic 10 Process of Querying Cube</ <p><em>é常æè°¢ææè´¡ç®Apache Kylinçæå!</em></p> </description> - <pubDate>Thu, 20 Sep 2018 20:00:00 +0000</pubDate> + <pubDate>Thu, 20 Sep 2018 13:00:00 -0700</pubDate> <link>http://kylin.apache.org/cn/blog/2018/09/20/release-v2.5.0/</link> <guid isPermaLink="true">http://kylin.apache.org/cn/blog/2018/09/20/release-v2.5.0/</guid> @@ -361,7 +399,7 @@ Graphic 10 Process of Querying Cube</ <p><em>Great thanks to everyone who contributed!</em></p> </description> - <pubDate>Thu, 20 Sep 2018 20:00:00 +0000</pubDate> + <pubDate>Thu, 20 Sep 2018 13:00:00 -0700</pubDate> <link>http://kylin.apache.org/blog/2018/09/20/release-v2.5.0/</link> <guid isPermaLink="true">http://kylin.apache.org/blog/2018/09/20/release-v2.5.0/</guid> @@ -407,20 +445,22 @@ GRANT ROLE ssb_write_role TO GROUP ssb_w # Then add kylin_manager_user to kylin_manager_group in OpenLDAP, so kylin_manager_user has access to the ssb database. </pre> <p>2 Assign HDFS directory /user/kylin_manager_user read and write permissions to kylin_manager_user user.<br /> -3 Configure the HADOOP_STREAMING_JAR environment variable under the kylin_manager_user user home directory.</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>Export HADOOP_STREAMING_JAR=/opt/cloudera/parcels/CDH/lib/hadoop-mapreduce/hadoop-streaming.jar -</code></pre></div></div> +3 Configure the HADOOP_STREAMING_JAR environment variable under the kylin_manager_user user home directory.<br /> +<code class="highlighter-rouge"> +Export HADOOP_STREAMING_JAR=/opt/cloudera/parcels/CDH/lib/hadoop-mapreduce/hadoop-streaming.jar +</code></p> <h2 id="download-the-ssb-tool-and-compile">Download the SSB tool and compile</h2> <p>You can quickly download and compile the ssb test tool by entering the following command in the linux terminal.</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>git clone https://github.com/jiangshouzhuang/ssb-kylin.git +<div class="highlighter-rouge"><pre class="highlight"><code>git clone https://github.com/jiangshouzhuang/ssb-kylin.git cd ssb-kylin cd ssb-benchmark make clean make -</code></pre></div></div> +</code></pre> +</div> <h2 id="adjust-the-ssb-parameters">Adjust the SSB parameters</h2> @@ -428,7 +468,7 @@ make <p>Part of the ssb.conf file is:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code> # customer base, default value is 30,000 +<div class="highlighter-rouge"><pre class="highlight"><code> # customer base, default value is 30,000 customer_base = 30000 # part base, default value is 200,000 part_base = 200000 @@ -438,27 +478,30 @@ make date_base = 2556 # lineorder base (purchase record), default value is 6,000,000 lineorder_base = 6000000 -</code></pre></div></div> +</code></pre> +</div> <p>Of course, the above base parameters can be adjusted according to their actual needs, I use the default parameters.<br /> In the ssb.conf file, there are some parameters as follows.</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code># manufacturer max. The value range is (1 .. manu_max) +<div class="highlighter-rouge"><pre class="highlight"><code># manufacturer max. The value range is (1 .. manu_max) manu_max = 5 # category max. The value range is (1 .. cat_max) cat_max = 5 # brand max. The value range is (1 .. brand_max) brand_max = 40 -</code></pre></div></div> +</code></pre> +</div> <p><strong>The explanation is as follows:</strong> <br /> manu_max, cat_max and brand_max are used to define hierarchical scale. For example, manu_max=10, cat_max=10, and brand_max=10 refer to a total of 10 manufactures, and each manufactures has a maximum of 10 category parts, and each category has up to 10 brands. Therefore, the cardinality of manufacture is 10, the cardinality of category is 100, and the cardinality of brand is 1000.</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code># customer: num of cities per country, default value is 100 +<div class="highlighter-rouge"><pre class="highlight"><code># customer: num of cities per country, default value is 100 cust_city_max = 9 # supplier: num of cities per country, default value is 100 supp_city_max = 9 -</code></pre></div></div> +</code></pre> +</div> <p><strong>The explanation is as follows:</strong> <br /> cust_city_max and supp_city_max are used to define the number of city for each country in customer and supplier tables. If the total number of country is 30, and cust_city_max=100, supp_city_max=10, then the customer table will have 3000 different city, and the supplier table will have 300 different city.</p> @@ -489,17 +532,19 @@ ${KYLIN_INSTALL_USER_PASSWD} -d org.apac <p>If your CDH or other big data platform is not using beeline, but hive cli, please modify it yourself.<br /> Once everything is ready, we start running the program and generate test data:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>cd ssb-kylin +<div class="highlighter-rouge"><pre class="highlight"><code>cd ssb-kylin bin/run.sh --scale 20 -</code></pre></div></div> +</code></pre> +</div> <p>We set the scale to 20, the program will run for a while, the maximum lineorder table data has more than 100 million. After the program is executed, we look at the tables in the hive database and the amount of data:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>use ssb; +<div class="highlighter-rouge"><pre class="highlight"><code>use ssb; show tables; select count(1) from lineorder; select count(1) from p_lineorder; -</code></pre></div></div> +</code></pre> +</div> <p><img src="/images/blog/2.1 generated tables.png" alt="" /></p> @@ -512,9 +557,10 @@ select count(1) from p_lineorder; <p>The ssb-kylin project has helped us build the project, model, and cube in advance. Just import the Kylin directly like the learn_kylin example. Cube Metadataâs directory is cubemeta, because our kylin integrates OpenLDAP, there is no ADMIN user, so the owner parameter in cubemeta/cube/ssb.json is set to null.<br /> Execute the following command to import cubemeta:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>cd ssb-kylin +<div class="highlighter-rouge"><pre class="highlight"><code>cd ssb-kylin $KYLIN_HOME/bin/metastore.sh restore cubemeta -</code></pre></div></div> +</code></pre> +</div> <p>Then log in to Kylin and execute Reload Metadata operation. This creates new project, model and cube in Kylin. Before building cube, first Disable, then Purge, delete old temporary files.</p> @@ -524,17 +570,19 @@ $KYLIN_HOME/bin/metastore.sh restore cub <p>Here I test the performance of Spark to build Cube again, disable the previously created Cube, and then Purge. Since the Cube is used by Purge, the useless HBase tables and HDFS files need to be deleted. Here, manually clean up the junk files. First execute the following command:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.StorageCleanupJob --delete false -</code></pre></div></div> +<div class="highlighter-rouge"><pre class="highlight"><code>${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.StorageCleanupJob --delete false +</code></pre> +</div> <p>Then check whether the listed HBase table and the HDFS file are useless. After confirming the error, perform the delete operation:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.StorageCleanupJob --delete true -</code></pre></div></div> +<div class="highlighter-rouge"><pre class="highlight"><code>${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.StorageCleanupJob --delete true +</code></pre> +</div> <p>When using Spark to build a cube, it consumes a lot of memory. After all, using memory resources improves the speed of cube building. Here I will list some of the parameters of Spark in the kylin.properties configuration file:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kylin.engine.spark-conf.spark.master=yarn +<div class="highlighter-rouge"><pre class="highlight"><code>kylin.engine.spark-conf.spark.master=yarn kylin.engine.spark-conf.spark.submit.deployMode=cluster kylin.engine.spark-conf.spark.yarn.queue=root.kylin_manager_group # config Dynamic resource allocation @@ -550,7 +598,8 @@ kylin.engine.spark-conf.spark.driver.mem kylin.engine.spark-conf.spark.executor.memory=4G kylin.engine.spark-conf.spark.executor.cores=1 kylin.engine.spark-conf.spark.network.timeout=600 -</code></pre></div></div> +</code></pre> +</div> <p>The above parameters can meet most of the requirements, so users basically do not need to configure when designing the Cube. Of course, if the situation is special, you can still set Spark-related tuning parameters at the Cube level.</p> @@ -588,7 +637,7 @@ The query result of Scale=10 is as follo </ol> </description> - <pubDate>Mon, 16 Jul 2018 12:28:00 +0000</pubDate> + <pubDate>Mon, 16 Jul 2018 05:28:00 -0700</pubDate> <link>http://kylin.apache.org/blog/2018/07/16/Star-Schema-Benchmark-on-Apache-Kylin/</link> <guid isPermaLink="true">http://kylin.apache.org/blog/2018/07/16/Star-Schema-Benchmark-on-Apache-Kylin/</guid> @@ -638,7 +687,7 @@ The query result of Scale=10 is as follo <p>Wish you have a good time with Redash-Kylin!</p> </description> - <pubDate>Tue, 08 May 2018 20:00:00 +0000</pubDate> + <pubDate>Tue, 08 May 2018 13:00:00 -0700</pubDate> <link>http://kylin.apache.org/blog/2018/05/08/redash-kylin-plugin-strikingly/</link> <guid isPermaLink="true">http://kylin.apache.org/blog/2018/05/08/redash-kylin-plugin-strikingly/</guid> @@ -680,11 +729,11 @@ The query result of Scale=10 is as follo <p>Any issue or question,<br /> open JIRA to Apache Kylin project: <a href="https://issues.apache.org/jira/browse/KYLIN/">https://issues.apache.org/jira/browse/KYLIN/</a><br /> or<br /> -send mail to Apache Kylin dev mailing list: <a href="mailto:d...@kylin.apache.org">d...@kylin.apache.org</a></p> +send mail to Apache Kylin dev mailing list: <a href="&#109;&#097;&#105;&#108;&#116;&#111;:&#100;&#101;&#118;&#064;&#107;&#121;&#108;&#105;&#110;&#046;&#097;&#112;&#097;&#099;&#104;&#101;&#046;&#111;&#114;&#103;">&#100;&#101;&#118;&#064;&#107;&#121;&#108;&#105;&#110;&#046;&#097;&#112;&#097;&#099;&#104;&#101;&#046;&#111;&#114;&#103;</a></p> <p><em>Great thanks to everyone who contributed!</em></p> </description> - <pubDate>Sun, 04 Mar 2018 20:00:00 +0000</pubDate> + <pubDate>Sun, 04 Mar 2018 12:00:00 -0800</pubDate> <link>http://kylin.apache.org/blog/2018/03/04/release-v2.3.0/</link> <guid isPermaLink="true">http://kylin.apache.org/blog/2018/03/04/release-v2.3.0/</guid> @@ -755,14 +804,15 @@ Figure 4: Build Cube in Apache Kylin< <li>Execute SQL in the âInsightâ tab, for example:</li> </ol> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code> select part_dtï¼ +<div class="highlighter-rouge"><pre class="highlight"><code> select part_dtï¼ sum(price) as total_selledï¼ count(distinct seller_id) as sellers from kylin_sales group by part_dt order by part_dt -- #This query will hit on the newly built Cube âKylin_sales_cubeâ. -</code></pre></div></div> +</code></pre> +</div> <ol> <li>Next, we will install Apache Superset and initialize it.<br /> @@ -770,14 +820,15 @@ Figure 4: Build Cube in Apache Kylin< <li>Install kylinpy</li> </ol> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code> $ pip install kylinpy -</code></pre></div></div> +<div class="highlighter-rouge"><pre class="highlight"><code> $ pip install kylinpy +</code></pre> +</div> <ol> <li>Verify your installation, if everything goes well, Apache Superset daemon should be up and running.</li> </ol> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>$ superset runserver -d +<div class="highlighter-rouge"><pre class="highlight"><code>$ superset runserver -d Starting server with command: gunicorn -w 2 --timeout 60 -b 0.0.0.0:8088 --limit-request-line 0 --limit-request-field_size 0 superset:app @@ -786,19 +837,18 @@ gunicorn -w 2 --timeout 60 -b 0.0.0.0:8 [2018-01-03 15:54:03 +0800] [73673] [INFO] Using worker: sync [2018-01-03 15:54:03 +0800] [73676] [INFO] Booting worker with pid: 73676 [2018-01-03 15:54:03 +0800] [73679] [INFO] Booting worker with pid: 73679 -</code></pre></div></div> +</code></pre> +</div> <h2 id="connect-apache-kylin-from-apachesuperset">Connect Apache Kylin from ApacheSuperset</h2> -<p>Now everything you need is installed and ready to go. Letâs try to create an Apache Kylin data source in Apache Superset.</p> -<ol> - <li> - <p>Open up http://localhost:8088 in your web browser with the credential you set during Apache Superset installation.<br /> +<p>Now everything you need is installed and ready to go. Letâs try to create an Apache Kylin data source in Apache Superset.<br /> +1. Open up http://localhost:8088 in your web browser with the credential you set during Apache Superset installation.<br /> <img src="/images/Kylin-and-Superset/png/5. superset_1.png" alt="" /><br /> Figure 5: Apache Superset Login Page</p> - </li> - <li> - <p>Go to Source -&gt; Datasource to configure a new data source.</p> + +<ol> + <li>Go to Source -&gt; Datasource to configure a new data source. <ul> <li>SQLAlchemy URI pattern is : kylin://<username>:<password>@<hostname>:<port>/<project name=""></project></port></hostname></password></username></li> <li>Check âExpose in SQL Labâ if you want to expose this data source in SQL Lab.</li> @@ -844,8 +894,9 @@ Figure 11 Query multiple tables from Apa <img src="/images/Kylin-and-Superset/png/12. SQL_Lab_2.png" alt="" /><br /> Figure 12 Define your query and visualize it immediately</p> -<p>You may copy the entire SQL below to experience how you can query Kylin Cube in SQL Lab.</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>select +<p>You may copy the entire SQL below to experience how you can query Kylin Cube in SQL Lab. <br /> +<code class="highlighter-rouge"> +select YEAR_BEG_DT, MONTH_BEG_DTï¼ WEEK_BEG_DTï¼ @@ -863,8 +914,8 @@ join KYLIN_CATEGORY_GROUPINGS on SITE_ID join KYLIN_ACCOUNT on ACCOUNT_ID=BUYER_ID join KYLIN_COUNTRY on ACCOUNT_COUNTRY=COUNTRY group by YEAR_BEG_DT, MONTH_BEG_DTï¼WEEK_BEG_DTï¼META_CATEG_NAMEï¼CATEG_LVL2_NAME, CATEG_LVL3_NAME, OPS_REGION, NAME -</code></pre></div></div> -<h2 id="experience-all-features-in-apache-superset-with-apache-kylin">Experience All Features in Apache Superset with Apache Kylin</h2> +</code><br /> +## Experience All Features in Apache Superset with Apache Kylin</p> <p>Most of the common reporting features are available in Apache Superset. Now letâs see how we can use those features to analyze data from Apache Kylin.</p> @@ -877,14 +928,13 @@ group by YEAR_BEG_DT, MONTH_BEG_DTï¼ Figure 13 Sort by</p> <h3 id="filtering">Filtering</h3> -<p>There are multiple ways you may filter data from Apache Kylin.</p> -<ol> - <li> - <p>Date Filter<br /> +<p>There are multiple ways you may filter data from Apache Kylin.<br /> +1. Date Filter<br /> You may filter date and time dimension with the calendar filter. <br /> <img src="/images/Kylin-and-Superset/png/14. time_filter.png" alt="" /><br /> Figure 14 Filtering time</p> - </li> + +<ol> <li> <p>Dimension Filter<br /> For other dimensions, you may filter it with SQL conditions like âin, not in, equal to, not equal to, greater than and equal to, smaller than and equal to, greater than, smaller than, likeâ.<br /> @@ -952,7 +1002,7 @@ Figure 13 Sort by</p> </ol> </description> - <pubDate>Mon, 01 Jan 2018 12:28:00 +0000</pubDate> + <pubDate>Mon, 01 Jan 2018 04:28:00 -0800</pubDate> <link>http://kylin.apache.org/blog/2018/01/01/kylin-and-superset/</link> <guid isPermaLink="true">http://kylin.apache.org/blog/2018/01/01/kylin-and-superset/</guid> @@ -981,11 +1031,12 @@ Figure 13 Sort by</p> <h3 id="make-spark-connect-hbase-with-kerberos-enabled">Make Spark connect HBase with Kerberos enabled</h3> <p>If just want to run Spark Cubing in Yarn client mode, we only need to add three line code before new SparkConf() in SparkCubingByLayer:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code> Configuration configuration = HBaseConnection.getCurrentHBaseConfiguration(); +<div class="highlighter-rouge"><pre class="highlight"><code> Configuration configuration = HBaseConnection.getCurrentHBaseConfiguration(); HConnection connection = HConnectionManager.createConnection(configuration); //Obtain an authentication token for the given user and add it to the user's credentials. TokenUtil.obtainAndCacheToken(connection, UserProvider.instantiate(configuration).create(UserGroupInformation.getCurrentUser())); -</code></pre></div></div> +</code></pre> +</div> <p>As for How to make Spark connect HBase using Kerberos in Yarn cluster mode, please refer to SPARK-6918, SPARK-12279, and HBASE-17040. The solution may work, but not elegant. So I tried the sencond solution.</p> @@ -1026,7 +1077,7 @@ This following picture shows the content <p>Following is the Spark configuration I used in our environment. It enables Spark dynamic resource allocation; the goal is to let our user set less Spark configurations.</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>//running in yarn-cluster mode +<div class="highlighter-rouge"><pre class="highlight"><code>//running in yarn-cluster mode kylin.engine.spark-conf.spark.master=yarn kylin.engine.spark-conf.spark.submit.deployMode=cluster @@ -1051,7 +1102,8 @@ kylin.engine.spark-conf.spark.network.ti kylin.engine.spark-conf.spark.yarn.queue=root.hadoop.test kylin.engine.spark.rdd-partition-cut-mb=100 -</code></pre></div></div> +</code></pre> +</div> <h3 id="performance-test-of-spark-cubing">Performance test of Spark Cubing</h3> @@ -1107,7 +1159,7 @@ kylin.engine.spark.rdd-partition-cut-mb= <p>Spark Cubing is a great feature for Kylin 2.0, Thanks Kylin community. We will apply Spark Cubing in real scenarios in our company. I believe Spark Cubing will be more robust and efficient in the future releases.</p> </description> - <pubDate>Fri, 21 Jul 2017 22:22:22 +0000</pubDate> + <pubDate>Fri, 21 Jul 2017 15:22:22 -0700</pubDate> <link>http://kylin.apache.org/blog/2017/07/21/Improving-Spark-Cubing/</link> <guid isPermaLink="true">http://kylin.apache.org/blog/2017/07/21/Improving-Spark-Cubing/</guid> @@ -1127,10 +1179,11 @@ kylin.engine.spark.rdd-partition-cut-mb= <p>In Apache Kylin, we support the similar SQL sytanx like Apache Hive, with a aggregation function called <strong>percentile(&lt;Number Column&gt;, &lt;Double&gt;)</strong>:</p> -<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">SELECT</span> <span class="n">seller_id</span><span class="p">,</span> <span class="n">percentile</span><span class="p">(</span><span class="n">price</span><span class="p">,</span> <span class="mi">0</span><span class="p">.</span><span class="mi">5</span><span class="p">)</span> +<div class="highlighter-rouge"><pre class="highlight"><code><span class="k">SELECT</span> <span class="n">seller_id</span><span class="p">,</span> <span class="n">percentile</span><span class="p">(</span><span class="n">price</span><span class="p">,</span> <span class="mi">0</span><span class="p">.</span><span class="mi">5</span><span class="p">)</span> <span class="k">FROM</span> <span class="n">test_kylin_fact</span> <span class="k">GROUP</span> <span class="k">BY</span> <span class="n">seller_id</span> -</code></pre></div></div> +</code></pre> +</div> <h3 id="how-to-use">How to use</h3> <p>If you know little about <em>Cubes</em>, please go to <a href="http://kylin.apache.org/docs20/tutorial/kylin_sample.html">QuickStart</a> first to learn basic knowledge.</p> @@ -1147,67 +1200,12 @@ kylin.engine.spark.rdd-partition-cut-mb= <p><img src="/images/blog/percentile_3.png" alt="" /></p> </description> - <pubDate>Sat, 01 Apr 2017 22:22:22 +0000</pubDate> + <pubDate>Sat, 01 Apr 2017 15:22:22 -0700</pubDate> <link>http://kylin.apache.org/blog/2017/04/01/percentile-measure/</link> <guid isPermaLink="true">http://kylin.apache.org/blog/2017/04/01/percentile-measure/</guid> <category>blog</category> - - </item> - - <item> - <title>Apache Kylin v2.0.0 beta åå¸</title> - <description><p>Apache Kylin社åºé常é«å ´å°å®£å¸ <a href="http://kylin.apache.org/cn/download/">v2.0.0 beta package</a> å·²ç»å¯ä»¥ä¸è½½å¹¶æµè¯äºã</p> - -<ul> - <li>ä¸è½½é¾æ¥: <a href="http://kylin.apache.org/cn/download/">http://kylin.apache.org/cn/download/</a></li> - <li>æºä»£ç : https://github.com/apache/kylin/tree/kylin-2.0.0-beta</li> -</ul> - -<p>èªä»v1.6.0çæ¬åå¸å·²ç»2个å¤æäºãè¿æ®µæ¶é´éï¼æ´ä¸ªç¤¾åºååå¼åå®æäºä¸ç³»åé大çåè½ï¼å¸æè½å°Apache Kylinæåå°ä¸ä¸ªæ°çé«åº¦ã</p> - -<ul> - <li>æ¯æéªè±æ¨¡å (<a href="https://issues.apache.org/jira/browse/KYLIN-1875">KYLIN-1875</a>)</li> - <li>æ¯æ TPC-H æ¥è¯¢ (<a href="https://issues.apache.org/jira/browse/KYLIN-2467">KYLIN-2467</a>)</li> - <li>Spark æ建å¼æ (<a href="https://issues.apache.org/jira/browse/KYLIN-2331">KYLIN-2331</a>)</li> - <li>Job Engine é«å¯ç¨æ§ (<a href="https://issues.apache.org/jira/browse/KYLIN-2006">KYLIN-2006</a>)</li> - <li>Percentile 度é (<a href="https://issues.apache.org/jira/browse/KYLIN-2396">KYLIN-2396</a>)</li> - <li>å¨ Cloud ä¸éè¿æµè¯ (<a href="https://issues.apache.org/jira/browse/KYLIN-2351">KYLIN-2351</a>)</li> -</ul> - -<p>é常欢è¿å¤§å®¶ä¸è½½å¹¶æµè¯ v2.0.0 betaãæ¨çåé¦å¯¹æ们é常éè¦ï¼è¯·åé®ä»¶å° <a href="mailto:d...@kylin.apache.org">d...@kylin.apache.org</a>ã</p> - -<hr /> - -<h2 id="å®è£ ">å®è£ </h2> - -<p>ææ¶ v2.0.0 beta æ æ³ä» v1.6.0 ç´æ¥å级ï¼å¿ éå ¨æ°å®è£ ãè¿æ¯ç±äºæ°çæ¬çå æ°æ®å¹¶ä¸ååå ¼å®¹ãå¥½å¨ Cube æ°æ®æ¯ååå ¼å®¹çï¼å æ¤åªéè¦å¼åä¸ä¸ªå æ°æ®è½¬æ¢å·¥å ·ï¼å°±è½å¨ä¸ä¹ çå°æ¥å®ç°å¹³æ»å级ãæ们æ£å¨ä¸ºæ¤åªåã</p> - -<hr /> - -<h2 id="è¿è¡-tpc-h-åºåæµè¯">è¿è¡ TPC-H åºåæµè¯</h2> - -<p>å¨ Apache Kylin ä¸è¿è¡ TPC-H çå ·ä½æ¥éª¤: <a href="https://github.com/Kyligence/kylin-tpch">https://github.com/Kyligence/kylin-tpch</a></p> - -<hr /> - -<h2 id="spark-æ建å¼æ">Spark æ建å¼æ</h2> - -<p>Apache Kylin v2.0.0 å¼å ¥äºä¸ä¸ªå ¨æ°çåºäº Apache Spark çæ建å¼æãå®å¯ç¨äºæ¿æ¢åæç MapReduce æ建å¼æãåæ¥æµè¯æ¾ç¤º Cube çæ建æ¶é´ä¸è¬è½ç¼©çå°åå ç 50% å·¦å³ã</p> - -<p>å¯ç¨ Spark æ建å¼æï¼è¯·åè<a href="/docs16/tutorial/cube_spark.html">è¿ç¯ææ¡£</a>.</p> - -<hr /> - -<p><em>æè°¢æ¯ä¸ä½æåçåä¸åè´¡ç®!</em></p> -</description> - <pubDate>Sat, 25 Feb 2017 20:00:00 +0000</pubDate> - <link>http://kylin.apache.org/cn/blog/2017/02/25/v2.0.0-beta-ready/</link> - <guid isPermaLink="true">http://kylin.apache.org/cn/blog/2017/02/25/v2.0.0-beta-ready/</guid> - - - <category>blog</category> </item> Added: kylin/site/images/blog/data-source-sdk.png URL: http://svn.apache.org/viewvc/kylin/site/images/blog/data-source-sdk.png?rev=1851595&view=auto ============================================================================== Binary file - no diff available. Propchange: kylin/site/images/blog/data-source-sdk.png ------------------------------------------------------------------------------ svn:mime-type = application/octet-stream
