KYLIN-2033 code review and bug fixes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bf127a91 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bf127a91 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bf127a91 Branch: refs/heads/1.5.x-HBase1.x Commit: bf127a9167c506025616031374a91c57eb1355ed Parents: afa350f Author: Yang Li <liy...@apache.org> Authored: Sat Sep 24 14:29:39 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sat Sep 24 14:29:56 2016 +0800 ---------------------------------------------------------------------- .../kylin/metadata/cachesync/Broadcaster.java | 4 ++-- .../realization/RealizationRegistry.java | 8 ++++++++ .../kylin/rest/controller/CacheController.java | 17 +++++++++------- .../apache/kylin/rest/service/AdminService.java | 1 - .../apache/kylin/rest/service/CacheService.java | 21 ++++++++++++++++---- .../apache/kylin/rest/service/QueryService.java | 4 +--- webapp/app/js/services/cache.js | 2 +- 7 files changed, 39 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java index 380dac8..75b2333 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java @@ -112,7 +112,7 @@ public class Broadcaster { while (true) { try { final BroadcastEvent broadcastEvent = broadcastEvents.takeFirst(); - logger.info("Announcing new broadcast event:" + broadcastEvent); + logger.info("Announcing new broadcast event: " + broadcastEvent); for (final RestClient restClient : restClients) { wipingCachePool.execute(new Runnable() { @Override @@ -326,7 +326,7 @@ public class Broadcaster { @Override public String toString() { - return Objects.toStringHelper(this).add("type", entity).add("name", cacheKey).add("action", event).toString(); + return Objects.toStringHelper(this).add("entity", entity).add("event", event).add("cacheKey", cacheKey).toString(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java index e6b1a86..77e2679 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java @@ -26,6 +26,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ClassUtil; +import org.apache.kylin.metadata.cachesync.Broadcaster; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,6 +76,13 @@ public class RealizationRegistry { logger.info("Initializing RealizationRegistry with metadata url " + config); this.config = config; init(); + + Broadcaster.getInstance(config).registerListener(new Broadcaster.Listener() { + @Override + public void onClearAll(Broadcaster broadcaster) throws IOException { + clearCache(); + } + }, ""); } private void init() { http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java index 667046b..254aabf 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java @@ -47,13 +47,16 @@ public class CacheController extends BasicController { private CacheService cacheService; /** - * Wipe system cache - * - * @param entity {@link Broadcaster.TYPE} - * @param event {@link Broadcaster.Event} - * @param cacheKey - * @return if the action success - * @throws IOException + * Announce wipe cache to all cluster nodes + */ + @RequestMapping(value = "/announce/{entity}/{cacheKey}/{event}", method = { RequestMethod.PUT }) + @ResponseBody + public void announceWipeCache(@PathVariable String entity, @PathVariable String event, @PathVariable String cacheKey) throws IOException { + cacheService.annouceWipeCache(entity, event, cacheKey); + } + + /** + * Wipe cache on this node */ @RequestMapping(value = "/{entity}/{cacheKey}/{event}", method = { RequestMethod.PUT }) @ResponseBody http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java index b92c305..1a94967 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java @@ -48,7 +48,6 @@ public class AdminService extends BasicService { */ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public String getEnv() { - logger.debug("Get Kylin Runtime environment"); PropertiesConfiguration tempConfig = new PropertiesConfiguration(); // Add Java Env http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java index 5d29dcd..c9c2dd7 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -59,15 +59,23 @@ public class CacheService extends BasicService { @Autowired private CacheManager cacheManager; - + private Broadcaster.Listener cacheSyncListener = new Broadcaster.Listener() { @Override + public void onClearAll(Broadcaster broadcaster) throws IOException { + removeAllOLAPDataSources(); + cleanAllDataCache(); + } + + @Override public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException { removeOLAPDataSource(project); + cleanDataCache(project); } @Override public void onProjectDataChange(Broadcaster broadcaster, String project) throws IOException { + removeOLAPDataSource(project); // data availability (cube enabled/disabled) affects exposed schema to SQL cleanDataCache(project); } @@ -84,7 +92,7 @@ public class CacheService extends BasicService { logger.error("Error in updateOnNewSegmentReady()", ex); } } - }.run(); + }.start(); } } }; @@ -94,12 +102,17 @@ public class CacheService extends BasicService { this.cubeService = cubeService; } + public void annouceWipeCache(String entity, String event, String cacheKey) { + Broadcaster broadcaster = Broadcaster.getInstance(getConfig()); + broadcaster.queue(entity, event, cacheKey); + } + public void notifyMetadataChange(String entity, Event event, String cacheKey) throws IOException { Broadcaster broadcaster = Broadcaster.getInstance(getConfig()); - + // broadcaster can be clearCache() too, make sure listener is registered; re-registration will be ignored broadcaster.registerListener(cacheSyncListener, "cube"); - + broadcaster.notifyListener(entity, event, cacheKey); } http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index df296cf..8303cee 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -337,7 +337,6 @@ public class QueryService extends BasicService { conn = dataSource.getConnection(); DatabaseMetaData metaData = conn.getMetaData(); - logger.debug("getting table metas"); JDBCTableMeta = metaData.getTables(null, null, null, null); tableMetas = new LinkedList<TableMeta>(); @@ -355,7 +354,6 @@ public class QueryService extends BasicService { } } - logger.debug("getting column metas"); columnMeta = metaData.getColumns(null, null, null, null); while (columnMeta.next()) { @@ -369,7 +367,7 @@ public class QueryService extends BasicService { tableMap.get(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME()).addColumn(colmnMeta); } } - logger.debug("done column metas"); + } finally { close(columnMeta, null, conn); if (JDBCTableMeta != null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/webapp/app/js/services/cache.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/services/cache.js b/webapp/app/js/services/cache.js index b61a179..bcec603 100644 --- a/webapp/app/js/services/cache.js +++ b/webapp/app/js/services/cache.js @@ -17,7 +17,7 @@ */ KylinApp.factory('CacheService', ['$resource', function ($resource, config) { - return $resource(Config.service.url + 'cache/:type/:name/:action', {}, { + return $resource(Config.service.url + 'cache/announce/:type/:name/:action', {}, { clean: {method: 'PUT', params: {type: 'all', name: 'all', action: 'update'}, isArray: false} }); }]);
