KYLIN-2388 Hot load kylin config from web
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/39afa519 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/39afa519 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/39afa519 Branch: refs/heads/master-hbase0.98 Commit: 39afa5197e197d5143843a3bb62c528749b03418 Parents: 7b860ad Author: kangkaisen <kangkai...@live.com> Authored: Thu Jan 12 14:35:59 2017 +0800 Committer: kangkaisen <kangkai...@163.com> Committed: Tue Feb 7 19:57:29 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfig.java | 6 +- .../apache/kylin/common/KylinConfigTest.java | 28 ++++----- .../util/HotLoadKylinPropertiesTestCase.java | 64 ++++++++++++++++++++ .../kylin/cube/CubeSpecificConfigTest.java | 37 ++++++----- .../apache/kylin/job/JobEngineConfigTest.java | 47 ++++++++++++++ .../kylin/rest/controller/CacheController.java | 9 ++- webapp/app/js/controllers/admin.js | 26 ++++++++ webapp/app/js/services/cache.js | 3 +- webapp/app/partials/admin/admin.html | 3 + 9 files changed, 191 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index 0f40654..c7a18c6 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -201,7 +201,7 @@ public class KylinConfig extends KylinConfigBase { return kylinHome + File.separator + "conf"; } - static File getKylinPropertiesFile() { + public static File getKylinPropertiesFile() { String kylinConfHome = System.getProperty(KYLIN_CONF); if (!StringUtils.isEmpty(kylinConfHome)) { logger.info("Use KYLIN_CONF=" + kylinConfHome); @@ -385,4 +385,8 @@ public class KylinConfig extends KylinConfigBase { // } // logger.info(buf.toString()); } + + public synchronized void hotLoadKylinProperties() { + reloadKylinConfig(getKylinProperties()); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java index a426fc6..4d5f130 100644 --- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java @@ -26,24 +26,12 @@ import static org.junit.Assert.assertTrue; import java.util.Map; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.junit.After; -import org.junit.Before; +import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase; import org.junit.Test; import com.google.common.collect.Maps; -public class KylinConfigTest extends LocalFileMetadataTestCase { - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - +public class KylinConfigTest extends HotLoadKylinPropertiesTestCase{ @Test public void testMRConfigOverride() { KylinConfig config = KylinConfig.getInstanceFromEnv(); @@ -78,9 +66,19 @@ public class KylinConfigTest extends LocalFileMetadataTestCase { KylinConfig config = KylinConfig.getInstanceFromEnv(); Map<String, String> override = Maps.newHashMap(); KylinConfig configExt = KylinConfigExt.createInstance(config, override); - assertTrue(config.properties == configExt.properties); config.setProperty("1234", "1234"); assertEquals("1234", configExt.getOptional("1234")); } + + @Test + public void testPropertiesHotLoad() { + KylinConfig config = KylinConfig.getInstanceFromEnv(); + assertEquals("who...@kylin.apache.org", config.getKylinOwner()); + + updateProperty("kylin.storage.hbase.owner-tag", "ky...@kylin.apache.org"); + KylinConfig.getInstanceFromEnv().hotLoadKylinProperties(); + + assertEquals("ky...@kylin.apache.org", config.getKylinOwner()); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java new file mode 100644 index 0000000..9f5b278 --- /dev/null +++ b/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.util; + +import org.apache.kylin.common.KylinConfig; +import org.junit.After; +import org.junit.Before; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.Properties; + +/** + * @author kangkaisen + */ + +public class HotLoadKylinPropertiesTestCase extends LocalFileMetadataTestCase { + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + protected void updateProperty(String key, String value) { + File propFile = KylinConfig.getKylinPropertiesFile(); + Properties conf = new Properties(); + + //load + try (FileInputStream is = new FileInputStream(propFile)) { + conf.load(is); + conf.setProperty(key, value); + } catch (Exception e) { + System.err.println(e.getMessage()); + } + + //store + try (FileOutputStream out = new FileOutputStream(propFile)) { + conf.store(out, null); + } catch (Exception e) { + System.err.println(e.getMessage()); + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java index c61f07f..17c02cc 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java @@ -21,24 +21,13 @@ package org.apache.kylin.cube; import static org.junit.Assert.assertEquals; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase; import org.apache.kylin.cube.model.CubeDesc; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -public class CubeSpecificConfigTest extends LocalFileMetadataTestCase { - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } +import java.io.IOException; +public class CubeSpecificConfigTest extends HotLoadKylinPropertiesTestCase { @Test public void test() { KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); @@ -57,4 +46,24 @@ public class CubeSpecificConfigTest extends LocalFileMetadataTestCase { assertEquals("snappy", base.getHbaseDefaultCompressionCodec()); assertEquals("lz4", override.getHbaseDefaultCompressionCodec()); } + + @Test + public void testPropertiesHotLoad() throws IOException { + KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); + KylinConfig oldCubeDescConfig = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb").getConfig(); + assertEquals(10, oldCubeDescConfig.getMaxConcurrentJobLimit()); + + //hot load Properties + updateProperty("kylin.job.max-concurrent-jobs", "20"); + KylinConfig.getInstanceFromEnv().hotLoadKylinProperties(); + CubeDescManager.getInstance(baseConfig).reloadCubeDescLocal("ssb"); + + //test cubeDescConfig + KylinConfig newCubeDescConfig = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb").getConfig(); + assertEquals(20, newCubeDescConfig.getMaxConcurrentJobLimit()); + + //test cubeConfig + KylinConfig newCubeConfig = CubeManager.getInstance(baseConfig).getCube("ssb").getConfig(); + assertEquals(20, newCubeConfig.getMaxConcurrentJobLimit()); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java ---------------------------------------------------------------------- diff --git a/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java b/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java new file mode 100644 index 0000000..77914ef --- /dev/null +++ b/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.job; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase; +import org.apache.kylin.job.engine.JobEngineConfig; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * @author kangkaisen + */ + +public class JobEngineConfigTest extends HotLoadKylinPropertiesTestCase { + + @Test + public void testPropertiesHotLoad() throws IOException { + KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); + JobEngineConfig jobEngineConfig = new JobEngineConfig(baseConfig); + assertEquals(10, jobEngineConfig.getMaxConcurrentJobLimit()); + + updateProperty("kylin.job.max-concurrent-jobs", "20"); + KylinConfig.getInstanceFromEnv().hotLoadKylinProperties(); + + assertEquals(20, jobEngineConfig.getMaxConcurrentJobLimit()); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java index 254aabf..8d5f00e 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java @@ -20,6 +20,7 @@ package org.apache.kylin.rest.controller; import java.io.IOException; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.rest.service.CacheService; import org.slf4j.Logger; @@ -39,7 +40,7 @@ import org.springframework.web.bind.annotation.ResponseBody; @Controller @RequestMapping(value = "/cache") public class CacheController extends BasicController { - + @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(CacheController.class); @@ -64,6 +65,12 @@ public class CacheController extends BasicController { cacheService.notifyMetadataChange(entity, Broadcaster.Event.getEvent(event), cacheKey); } + @RequestMapping(value = "/announce/config", method = { RequestMethod.POST }) + public void hotLoadKylinConfig() throws IOException { + KylinConfig.getInstanceFromEnv().hotLoadKylinProperties(); + cacheService.notifyMetadataChange(Broadcaster.SYNC_ALL, Broadcaster.Event.UPDATE, Broadcaster.SYNC_ALL); + } + public void setCacheService(CacheService cacheService) { this.cacheService = cacheService; } http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/webapp/app/js/controllers/admin.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/controllers/admin.js b/webapp/app/js/controllers/admin.js index 0d36e0d..783ab17 100644 --- a/webapp/app/js/controllers/admin.js +++ b/webapp/app/js/controllers/admin.js @@ -58,6 +58,32 @@ KylinApp.controller('AdminCtrl', function ($scope, AdminService, CacheService, T }); } + $scope.reloadConfig = function () { + SweetAlert.swal({ + title: '', + text: 'Are you sure to reload config', + type: '', + showCancelButton: true, + confirmButtonColor: '#DD6B55', + confirmButtonText: "Yes", + closeOnConfirm: true + }, function (isConfirm) { + if (isConfirm) { + CacheService.reloadConfig({}, function () { + SweetAlert.swal('Success!', 'config reload successfully', 'success'); + }, function (e) { + if (e.data && e.data.exception) { + var message = e.data.exception; + var msg = !!(message) ? message : 'Failed to take action.'; + SweetAlert.swal('Oops...', msg, 'error'); + } else { + SweetAlert.swal('Oops...', "Failed to take action.", 'error'); + } + }); + } + }); + } + $scope.reloadMeta = function () { SweetAlert.swal({ title: '', http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/webapp/app/js/services/cache.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/services/cache.js b/webapp/app/js/services/cache.js index bcec603..38dc0b8 100644 --- a/webapp/app/js/services/cache.js +++ b/webapp/app/js/services/cache.js @@ -18,6 +18,7 @@ KylinApp.factory('CacheService', ['$resource', function ($resource, config) { return $resource(Config.service.url + 'cache/announce/:type/:name/:action', {}, { - clean: {method: 'PUT', params: {type: 'all', name: 'all', action: 'update'}, isArray: false} + clean: {method: 'PUT', params: {type: 'all', name: 'all', action: 'update'}, isArray: false}, + reloadConfig: {method: 'POST', params: {type: 'config'}, isArray: false} }); }]); http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/webapp/app/partials/admin/admin.html ---------------------------------------------------------------------- diff --git a/webapp/app/partials/admin/admin.html b/webapp/app/partials/admin/admin.html index f2be3d9..b4bca8d 100644 --- a/webapp/app/partials/admin/admin.html +++ b/webapp/app/partials/admin/admin.html @@ -58,6 +58,9 @@ <a class="btn btn-primary btn-lg btn-block" tooltip="Update Server Config" class="btn btn-primary btn-lg" ng-click="toSetConfig()">Set Config</a> </div> <div style="padding-top: 10px;width: 260px;"> + <a class="btn btn-primary btn-lg btn-block" tooltip="Reload Server Config" class="btn btn-primary btn-lg" ng-click="reloadConfig()">Reload Config</a> + </div> + <div style="padding-top: 10px;width: 260px;"> <a ng-click="downloadBadQueryFiles();" tooltip="Download Diagnosis Info For Current Project" class="btn btn-primary btn-lg btn-block"><i class="fa fa-ambulance"></i> Diagnosis</a> </div>
