Added: kylin/site/docs23/howto/howto_optimize_build.html URL: http://svn.apache.org/viewvc/kylin/site/docs23/howto/howto_optimize_build.html?rev=1822508&view=auto ============================================================================== --- kylin/site/docs23/howto/howto_optimize_build.html (added) +++ kylin/site/docs23/howto/howto_optimize_build.html Mon Jan 29 14:05:18 2018 @@ -0,0 +1,4293 @@ +<!-- +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +--> + +<!doctype html> +<html> + <!-- +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +--> + +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Apache Kylin | Optimize Cube Build</title> + <meta name="description" content="Apache Kylin Home"> + <meta name="author" content="Apache Kylin"> + <link rel="shortcut icon" href="fav.png" type="image/png"> + + + +<link rel="stylesheet" href="/assets/css/animate.css"> +<!-- Bootstrap --> +<link rel="stylesheet" href="/assets/css/bootstrap.min.css"> + +<!-- Fonts --> +<!-- <link rel="stylesheet" href="http://fonts.googleapis.com/css?family=Alice|Open+Sans:400,300,700"> --> + +<!-- Icons --> +<link rel="stylesheet" href="/assets/css/font-awesome.min.css"> + + <!-- Custom styles --> + <link rel="stylesheet" href="/assets/css/styles.css"> + <link rel="stylesheet" href="/assets/css/docs.css"> + <link rel="stylesheet" href="/assets/css/pygments.css"> + + <link rel="canonical" href="http://kylin.apache.org/docs23/howto/howto_optimize_build.html";> + <link rel="alternate" type="application/rss+xml" title="Apache Kylin" href="http://kylin.apache.org/feed.xml"; /> + +<!--[if lt IE 9]> <script src="assets/js/html5shiv.js"></script> <![endif]--> +<script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + //oringal tracker for kylin.io + ga('create', 'UA-55534813-1', 'auto'); + //new tracker for kylin.apache.org + ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'}); + + ga('send', 'pageview'); + ga('toplevel.send', 'pageview'); + + +</script> +<script type="text/javascript" src="/assets/js/jquery-1.9.1.min.js"></script> +<script type="text/javascript" src="/assets/js/nside.js"></script> </script> +<script type="text/javascript" src="/assets/js/nnav.js"></script> </script> +</head> + + <body> + <!-- +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +--> + +<header id="header" > + + <div id="head" class="parallax" parallax-speed="3" > + <div id="logo" class="text-center"> <img class="img-circle" id="circlelogo" src="/assets/images/kylin_logo.jpg"> <span class="title" >Apache Kylinâ¢</span> <span class="tagline">Extreme OLAP Engine for Big Data</span> + </div> + <div class="text-center" style=" + position: relative; + top: 66px; + width: 1080px; + margin: 0 auto; + z-index: 11; + margin-top: -253px; + text-align: right;" + > + <a href="http://apache.org/foundation/contributing.html"; title="Support Apache" style="margin-left: 150px;"> + <img src="https://www.apache.org/images/SupportApache-small.png"; style="height: 150px; width: 150px;"> + </a> + </div> + </div> + + + <!-- Main Menu --> + <nav class="navbar navbar-default" role="navigation" id="nav-wrapper"> + <div class="container-fluid" id="nav"> + <!-- + <img class="img-circle" width="40px" height="40px" id="circlelogo" src="/assets/images/kylin_logo.jpg"> + --> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + + </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav"> + <li><a href="/">Home</a></li> + <li><a href="/docs21" >Docs</a></li> + <li><a href="/download">Download</li> + <li><a href="/community" >Community</a></li> + <li><a href="/development" >Development</a></li> + <li><a href="/blog">Blog</li> + <li><a href="/cn" >ä¸æç</a></li> + <li><a href="https://twitter.com/apachekylin"; target="_blank" class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" ></a></li> + <li><a href="https://github.com/apache/kylin"; target="_blank" class="fa fa-github-alt fa-lg" title="Github: apache/kylin" ></a></li> + <li><a href="https://www.facebook.com/kylinio"; target="_blank" class="fa fa-facebook fa-lg" title="Facebook: kylin.io" ></a></li> + </ul> + </div><!-- /.navbar-collapse --> + </div><!-- /.container-fluid --> +</nav> + </header> + + + <div class="container"> + <div class="row"> + <!-- +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +--> + +<div class="col-md-3 col-lg-3 col-xs-4 aside1 visible-md visible-lg" id="nside1" style=" padding-top: 2em"> + <ul class="nav nav-pills nav-stacked"> + + <li><a href="#titleGetting Started" data-toggle="collapse" id="navtitle">Getting Started</a></li> + <div class="collapse in"> + <div class="list-group" id="list1"> + <ul style="list-style-type:disc"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/index.html" class="list-group-item-lay pjaxlink" id="navlist">Overview</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/release_notes.html" class="list-group-item-lay pjaxlink" id="navlist">Release Notes</a></li> + + + + + + + + + + + + + + + + + + <li><a href="/docs23/gettingstarted/concepts.html" class="list-group-item-lay pjaxlink" id="navlist">Technical Concepts</a></li> + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/gettingstarted/terminology.html" class="list-group-item-lay pjaxlink" id="navlist">Terminology</a></li> + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/gettingstarted/faq.html" class="list-group-item-lay pjaxlink" id="navlist">FAQ</a></li> + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/gettingstarted/events.html" class="list-group-item-lay pjaxlink" id="navlist">Events and Conferences</a></li> + + + + + + + + + + + + + + + + <li><a href="/docs23/gettingstarted/best_practices.html" class="list-group-item-lay pjaxlink" id="navlist">Community Best Practices</a></li> + + + + + + + + + <ul> + </div> +</div> + + <li><a href="#titleInstallation" data-toggle="collapse" id="navtitle">Installation</a></li> + <div class="collapse in"> + <div class="list-group" id="list1"> + <ul style="list-style-type:disc"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/install/index.html" class="list-group-item-lay pjaxlink" id="navlist">Installation Guide</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/install/hadoop_env.html" class="list-group-item-lay pjaxlink" id="navlist">Hadoop Environment</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/install/kylin_cluster.html" class="list-group-item-lay pjaxlink" id="navlist">Deploy in Cluster Mode</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/install/advance_settings.html" class="list-group-item-lay pjaxlink" id="navlist">Advanced Settings</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/install/kylin_docker.html" class="list-group-item-lay pjaxlink" id="navlist">Run Kylin with Docker</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/install/kylin_aws_emr.html" class="list-group-item-lay pjaxlink" id="navlist">Install Kylin on AWS EMR</a></li> + + + + + + + + + <ul> + </div> +</div> + + <li><a href="#titleTutorial" data-toggle="collapse" id="navtitle">Tutorial</a></li> + <div class="collapse in"> + <div class="list-group" id="list1"> + <ul style="list-style-type:disc"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/kylin_sample.html" class="list-group-item-lay pjaxlink" id="navlist">Quick Start with Sample Cube</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/web.html" class="list-group-item-lay pjaxlink" id="navlist">Web Interface</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/create_cube.html" class="list-group-item-lay pjaxlink" id="navlist">Cube Wizard</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/cube_build_job.html" class="list-group-item-lay pjaxlink" id="navlist">Cube Build and Job Monitoring</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/acl.html" class="list-group-item-lay pjaxlink" id="navlist">Cube Permission (v2.1.x)</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/project_level_acl.html" class="list-group-item-lay pjaxlink" id="navlist">Project Level ACL</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/cube_spark.html" class="list-group-item-lay pjaxlink" id="navlist">Build Cube with Spark</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/cube_build_performance.html" class="list-group-item-lay pjaxlink" id="navlist">Cube Build Tuning</a></li> + + + + + + + + + <ul> + </div> +</div> + + <li><a href="#titleIntegration" data-toggle="collapse" id="navtitle">Integration</a></li> + <div class="collapse in"> + <div class="list-group" id="list1"> + <ul style="list-style-type:disc"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/odbc.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin ODBC Driver</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/tableau.html" class="list-group-item-lay pjaxlink" id="navlist">Tableau 8</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/tableau_91.html" class="list-group-item-lay pjaxlink" id="navlist">Tableau 9</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/powerbi.html" class="list-group-item-lay pjaxlink" id="navlist">MS Excel and Power BI</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/microstrategy.html" class="list-group-item-lay pjaxlink" id="navlist">MicroStrategy</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/squirrel.html" class="list-group-item-lay pjaxlink" id="navlist">SQuirreL</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/flink.html" class="list-group-item-lay pjaxlink" id="navlist">Apache Flink</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/hue.html" class="list-group-item-lay pjaxlink" id="navlist">Apache Hue</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/tutorial/Qlik.html" class="list-group-item-lay pjaxlink" id="navlist">Qlik Sense</a></li> + + + + + + + + + <ul> + </div> +</div> + + <li><a href="#titleHow To" data-toggle="collapse" id="navtitle">How To</a></li> + <div class="collapse in"> + <div class="list-group" id="list1"> + <ul style="list-style-type:disc"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_use_restapi.html" class="list-group-item-lay pjaxlink" id="navlist">Use RESTful API</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_build_cube_with_restapi.html" class="list-group-item-lay pjaxlink" id="navlist">Build Cube with API</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_optimize_cubes.html" class="list-group-item-lay pjaxlink" id="navlist">Optimize Cube Design</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_optimize_build.html" class="list-group-item-lay pjaxlink" id="navlist">Optimize Cube Build</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_backup_metadata.html" class="list-group-item-lay pjaxlink" id="navlist">Backup Metadata</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_cleanup_storage.html" class="list-group-item-lay pjaxlink" id="navlist">Cleanup Storage</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_jdbc.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin JDBC Driver</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_upgrade.html" class="list-group-item-lay pjaxlink" id="navlist">Upgrade From Old Versions</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_ldap_and_sso.html" class="list-group-item-lay pjaxlink" id="navlist">Secure with LDAP and SSO</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_use_beeline.html" class="list-group-item-lay pjaxlink" id="navlist">Use Beeline for Hive</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_update_coprocessor.html" class="list-group-item-lay pjaxlink" id="navlist">Update Coprocessor</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_install_ranger_kylin_plugin.html" class="list-group-item-lay pjaxlink" id="navlist">The Ranger Kylin Plugin Installation Guide</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_enable_zookeeper_acl.html" class="list-group-item-lay pjaxlink" id="navlist">Enable zookeeper acl</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_setup_systemcube.html" class="list-group-item-lay pjaxlink" id="navlist">Set Up System Cube</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_use_cube_planner.html" class="list-group-item-lay pjaxlink" id="navlist">Use Cube Planner</a></li> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <li><a href="/docs23/howto/howto_use_dashboard.html" class="list-group-item-lay pjaxlink" id="navlist">Use Dashboard</a></li> + + + + + + + + + <ul> + </div> +</div> + + + </ul> +</div> + + <div class="col-md-9 col-lg-9 col-xs-14 aside2"> + <div id="container"> + <div id="pjax"> + <h1 class="post-title">Optimize Cube Build</h1> + + + <article class="post-content" > + <p>Kylin decomposes a Cube build task into several steps and then executes them in sequence. These steps include Hive operations, MapReduce jobs, and other types job. When you have many Cubes to build daily, then you definitely want to speed up this process. Here are some practices that you probably want to know, and they are organized in the same order as the steps sequence.</p> + +<h2 id="create-intermediate-flat-hive-table">Create Intermediate Flat Hive Table</h2> + +<p>This step extracts data from source Hive tables (with all tables joined) and inserts them into an intermediate flat table. If Cube is partitioned, Kylin will add a time condition so that only the data in the range would be fetched. You can check the related Hive command in the log of this step, e.g:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>hive -e "USE default; +DROP TABLE IF EXISTS kylin_intermediate_airline_cube_v3610f668a3cdb437e8373c034430f6c34; + +CREATE EXTERNAL TABLE IF NOT EXISTS kylin_intermediate_airline_cube_v3610f668a3cdb437e8373c034430f6c34 +(AIRLINE_FLIGHTDATE date,AIRLINE_YEAR int,AIRLINE_QUARTER int,...,AIRLINE_ARRDELAYMINUTES int) +STORED AS SEQUENCEFILE +LOCATION 'hdfs:///kylin/kylin200instance/kylin-0a8d71e8-df77-495f-b501-03c06f785b6c/kylin_intermediate_airline_cube_v3610f668a3cdb437e8373c034430f6c34'; + +SET dfs.replication=2; +SET hive.exec.compress.output=true; +SET hive.auto.convert.join.noconditionaltask=true; +SET hive.auto.convert.join.noconditionaltask.size=100000000; +SET mapreduce.job.split.metainfo.maxsize=-1; + +INSERT OVERWRITE TABLE kylin_intermediate_airline_cube_v3610f668a3cdb437e8373c034430f6c34 SELECT +AIRLINE.FLIGHTDATE +,AIRLINE.YEAR +,AIRLINE.QUARTER +,... +,AIRLINE.ARRDELAYMINUTES +FROM AIRLINE.AIRLINE as AIRLINE +WHERE (AIRLINE.FLIGHTDATE >= '1987-10-01' AND AIRLINE.FLIGHTDATE < '2017-01-01'); +" + +</code></pre> +</div> + +<p>Kylin applies the configuration in conf/kylin_hive_conf.xml while Hive commands are running, for instance, use less replication and enable Hiveâs mapper side join. If it is needed, you can add other configurations which are good for your cluster.</p> + +<p>If Cubeâs partition column (âFLIGHTDATEâ in this case) is the same as Hive tableâs partition column, then filtering on it will let Hive smartly skip those non-matched partitions. So it is highly recommended to use Hive tableâs paritition column (if it is a date column) as the Cubeâs partition column. This is almost required for those very large tables, or Hive has to scan all files each time in this step, costing terribly long time.</p> + +<p>If your Hive enables file merge, you can disable them in âconf/kylin_hive_conf.xmlâ as Kylin has its own way to merge files (in the next step):</p> + +<div class="highlighter-rouge"><pre class="highlight"><code><property> + <name>hive.merge.mapfiles</name> + <value>false</value> + <description>Disable Hive's auto merge</description> +</property> +</code></pre> +</div> + +<h2 id="redistribute-intermediate-table">Redistribute intermediate table</h2> + +<p>After the previous step, Hive generates the data files in HDFS folder: while some files are large, some are small or even empty. The imbalanced file distribution would lead subsequent MR jobs to imbalance as well: some mappers finish quickly yet some others are very slow. To balance them, Kylin adds this step to âredistributeâ the data and here is a sample output:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>total input rows = 159869711 +expected input rows per mapper = 1000000 +num reducers for RedistributeFlatHiveTableStep = 160 + +</code></pre> +</div> + +<p>Redistribute table, cmd:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>hive -e "USE default; +SET dfs.replication=2; +SET hive.exec.compress.output=true; +SET hive.auto.convert.join.noconditionaltask=true; +SET hive.auto.convert.join.noconditionaltask.size=100000000; +SET mapreduce.job.split.metainfo.maxsize=-1; +set mapreduce.job.reduces=160; +set hive.merge.mapredfiles=false; + +INSERT OVERWRITE TABLE kylin_intermediate_airline_cube_v3610f668a3cdb437e8373c034430f6c34 SELECT * FROM kylin_intermediate_airline_cube_v3610f668a3cdb437e8373c034430f6c34 DISTRIBUTE BY RAND(); +" + +</code></pre> +</div> + +<p>Firstly, Kylin gets the row count of this intermediate table; then based on the number of row count, it would get amount of files needed to get data redistributed. By default, Kylin allocates one file per 1 million rows. In this sample, there are 160 million rows and exist 160 reducers, and each reducer would write 1 file. In following MR step over this table, Hadoop will start the same number Mappers as the files to process (usually 1 millionâs data size is small than a HDFS block size). If your daily data scale isnât so large or Hadoop cluster has enough resources, you may want to get more concurrency. Setting âkylin.job.mapreduce.mapper.input.rowsâ in âconf/kylin.propertiesâ to a smaller value will get that, e.g:</p> + +<p><code class="highlighter-rouge">kylin.job.mapreduce.mapper.input.rows=500000</code></p> + +<p>Secondly, Kylin runs a <em>âINSERT OVERWIRTE TABLE â¦. DISTRIBUTE BY â</em> HiveQL to distribute the rows among a specified number of reducers.</p> + +<p>In most cases, Kylin asks Hive to randomly distributes the rows among reducers, then get files very closed in size. The distribute clause is âDISTRIBUTE BY RAND()â.</p> + +<p>If your Cube has specified a âshard byâ dimension (in Cubeâs âAdvanced settingâ page), which is a high cardinality column (like âUSER_IDâ), Kylin will ask Hive to redistribute data by that columnâs value. Then for the rows that have the same value as this column has, they will go to the same file. This is much better than âby randomâ, because the data will be not only redistributed but also pre-categorized without additional cost, thus benefiting the subsequent Cube build process. Under a typical scenario, this optimization can cut off 40% building time. In this case the distribute clause will be âDISTRIBUTE BY USER_IDâ:</p> + +<p><strong>Please note:</strong> 1) The âshard byâ column should be a high cardinality dimension column, and it appears in many cuboids (not just appears in seldom cuboids). Utilize it to distribute properly can get equidistribution in every time range; otherwise it will cause data incline, which will reduce the building speed. Typical good cases are: âUSER_IDâ, âSELLER_IDâ, âPRODUCTâ, âCELL_NUMBERâ, so forth, whose cardinality is higher than one thousand (should be much more than the reducer numbers). 2) Using âshard byâ has other advantage in Cube storage, but it is out of this docâs scope.</p> + +<h2 id="extract-fact-table-distinct-columns">Extract Fact Table Distinct Columns</h2> + +<p>In this step Kylin runs a MR job to fetch distinct values for the dimensions, which are using dictionary encoding.</p> + +<p>Actually this step does more: it collects the Cube statistics by using HyperLogLog counters to estimate the row count of each Cuboid. If you find that mappers work incredible slowly, it usually indicates that the Cube design is too complex, please check <a href="howto_optimize_cubes.html">optimize cube design</a> to make the Cube thinner. If the reducers get OutOfMemory error, it indicates that the Cuboid combination does explode or the default YARN memory allocation cannot meet demands. If this step couldnât finish in a reasonable time by all means, you can give up and revisit the design as the real building will take longer.</p> + +<p>You can reduce the sampling percentage (kylin.job.cubing.inmem.sampling.percen in kylin.properties) to get this step accelerated, but this may not help much and impact on the accuracy of Cube statistics, thus we donât recommend.</p> + +<h2 id="build-dimension-dictionary">Build Dimension Dictionary</h2> + +<p>With the distinct values fetched in previous step, Kylin will build dictionaries in memory (in next version this will be moved to MR). Usually this step is fast, but if the value set is large, Kylin may report error like âToo high cardinality is not suitable for dictionaryâ. For UHC column, please use other encoding method for the UHC column, such as âfixed_lengthâ, âintegerâ and so on.</p> + +<h2 id="save-cuboid-statistics-and-create-htable">Save Cuboid Statistics and Create HTable</h2> + +<p>These two steps are lightweight and fast.</p> + +<h2 id="build-base-cuboid">Build Base Cuboid</h2> + +<p>This step is building the base cuboid from the intermediate table, which is the first round MR of the âby-layerâ cubing algorithm. The mapper number is equals to the reducer number of step 2; The reducer number is estimated with the cube statistics: by default use 1 reducer every 500MB output; If you observed the reducer number is small, you can set âkylin.job.mapreduce.default.reduce.input.mbâ in kylin.properties to a smaller value to get more resources, e.g: <code class="highlighter-rouge">kylin.job.mapreduce.default.reduce.input.mb=200</code></p> + +<h2 id="build-n-dimension-cuboid">Build N-Dimension Cuboid</h2> + +<p>These steps are the âby-layerâ cubing process, each step uses the output of previous step as the input, and then cut off one dimension to aggregate to get one child cuboid. For example, from cuboid ABCD, cut off A get BCD, cut off B get ACD etc.</p> + +<p>Some cuboid can be aggregated from more than 1 parent cubiods, in this case, Kylin will select the minimal parent cuboid. For example, AB can be generated from ABC (id: 1110) and ABD (id: 1101), so ABD will be used as its id is smaller than ABC. Based on this, if Dâs cardinality is small, the aggregation will be cost-efficient. So, when you design the Cube rowkey sequence, please remember to put low cardinality dimensions to the tail position. This not only benefit the Cube build, but also benefit the Cube query as the post-aggregation follows the same rule.</p> + +<p>Usually from the N-D to (N/2)-D the building is slow, because it is the cuboid explosion process: N-D has 1 Cuboid, (N-1)-D has N cuboids, (N-2)-D has N*(N-1) cuboids, etc. After (N/2)-D step, the building gets faster gradually.</p> + +<h2 id="build-cube">Build Cube</h2> + +<p>This step uses a new algorithm to build the Cube: âby-splitâ Cubing (also called as âin-memâ cubing). It will use one round MR to calculate all cuboids, but it requests more memory than normal. The âconf/kylin_job_conf_inmem.xmlâ is made for this step. By default it requests 3GB memory for each mapper. If your cluster has enough memory, you can allocate more in âconf/kylin_job_conf_inmem.xmlâ so it will use as much possible memory to hold the data and gain a better performance, e.g:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code><property> + <name>mapreduce.map.memory.mb</name> + <value>6144</value> + <description></description> +</property> + +<property> + <name>mapreduce.map.java.opts</name> + <value>-Xmx5632m</value> + <description></description> +</property> +</code></pre> +</div> + +<p>Please note, Kylin will automatically select the best algorithm based on the data distribution (get in Cube statistics). The not-selected algorithmâs steps will be skipped. You donât need to select the algorithm explicitly.</p> + +<h2 id="convert-cuboid-data-to-hfile">Convert Cuboid Data to HFile</h2> + +<p>This step starts a MR job to convert the Cuboid files (sequence file format) into HBaseâs HFile format. Kylin calculates the HBase region number with the Cube statistics, by default 1 region per 5GB. The more regions got, the more reducers would be utilized. If you observe the reducerâs number is small and performance is poor, you can set the following parameters in âconf/kylin.propertiesâ to smaller, as follows:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>kylin.hbase.region.cut=2 +kylin.hbase.hfile.size.gb=1 +</code></pre> +</div> + +<p>If youâre not sure what size a region should be, contact your HBase administrator.</p> + +<h2 id="load-hfile-to-hbase-table">Load HFile to HBase Table</h2> + +<p>This step uses HBase API to load the HFile to region servers, it is lightweight and fast.</p> + +<h2 id="update-cube-info">Update Cube Info</h2> + +<p>After loading data into HBase, Kylin marks this Cube segment as ready in metadata. This step is very fast.</p> + +<h2 id="cleanup">Cleanup</h2> + +<p>Drop the intermediate table from Hive. This step doesnât block anything as the segment has been marked ready in the previous step. If this step gets error, no need to worry, the garbage can be collected later when Kylin executes the <a href="howto_cleanup_storage.html">StorageCleanupJob</a>.</p> + +<h2 id="summary">Summary</h2> +<p>There are also many other methods to boost the performance. If you have practices to share, welcome to discuss in <a href="mailto:dev@kylin.apache.org">dev@kylin.apache.org</a>.</p> + + </article> + </div> + </div> + </div> + </div> + </div> + <!-- +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +--> + +<footer id="underfooter"> + <div class="container"> + <div class="row"> + <div class="col-md-12 widget"> + <div class="widget-body" style="text-align:center"> + <a href="http://www.apache.org";> + <img id="asf-logo" alt="Apache Software Foundation" src="/assets/images/feather-small.gif"> + </a> + + <div> + The contents of this website are © 2015 Apache Software Foundation under the terms of the <a + href="http://www.apache.org/licenses/LICENSE-2.0";> Apache License v2 </a>. Apache Kylin and + its logo are trademarks of the Apache Software Foundation. + </div> + + </div> + </div> + </div> + <!-- /row of widgets --> + + </div> + <div></div> + +</footer> + + <script src="/assets/js/jquery-1.9.1.min.js"></script> + <script src="/assets/js/bootstrap.min.js"></script> + <script src="/assets/js/main.js"></script> + </body> +</html>
![https://www.apache.org/images/SupportApache-small.png"](https://www.apache.org/images/SupportApache-small.png"){kind=link}
