KYLIN-2531 convert old tableExt jackson to new one instead of removing it
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/266a62a5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/266a62a5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/266a62a5 Branch: refs/heads/KYLIN-2506 Commit: 266a62a5ab6b0b39b08b79d3f3d4d1948e111247 Parents: 38fe432 Author: Cheng Wang <cheng.w...@kyligence.io> Authored: Wed Apr 5 14:43:52 2017 +0800 Committer: Dong Li <lid...@apache.org> Committed: Wed Apr 5 14:46:12 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/metadata/MetadataManager.java | 64 +++++++++++++++----- .../kylin/metadata/MetadataManagerTest.java | 41 +++++++++++-- 2 files changed, 85 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/266a62a5/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java index 9427ace..f17983f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java @@ -19,8 +19,10 @@ package org.apache.kylin.metadata; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -30,8 +32,10 @@ import java.util.concurrent.ConcurrentMap; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; +import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; @@ -47,6 +51,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; /** * Serves (and caches) metadata for Kylin instance. @@ -207,11 +212,7 @@ public class MetadataManager { result.setUuid(UUID.randomUUID().toString()); result.setLastModified(0); result.init(); - try { - saveTableExt(result); - } catch (IOException ex) { - logger.warn("Failed to save TableExt", ex); - } + srcTableExdMap.put(result.getName(), result); } return result; } @@ -224,9 +225,14 @@ public class MetadataManager { tableExt.init(); String path = tableExt.getResourcePath(); - getStore().putResource(path, tableExt, TABLE_EXT_SERIALIZER); - srcTableExdMap.put(tableExt.getName(), tableExt); + ResourceStore store = getStore(); + + TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER); + if (t != null && t.getName() == null) + store.deleteResource(path); + + store.putResource(path, tableExt, TABLE_EXT_SERIALIZER); } public void removeTableExt(String tableName) throws IOException { @@ -387,22 +393,52 @@ public class MetadataManager { private TableExtDesc reloadTableExtAt(String path) throws IOException { ResourceStore store = getStore(); TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER); + if (t == null) { return null; } + + // convert old tableExt json to new one + if (t.getName() == null) { + t = convertOldTableExtToNewer(path); + } + t.init(); - String name = t.getName(); + srcTableExdMap.putLocal(t.getName(), t); + return t; + } - // remove old json - if (name == null) { - getStore().deleteResource(path); - return null; + private TableExtDesc convertOldTableExtToNewer(String path) throws IOException { + Map<String, String> attrs = Maps.newHashMap(); + + ResourceStore store = getStore(); + RawResource res = store.getResource(path); + + InputStream is = res.inputStream; + + try { + attrs.putAll(JsonUtil.readValue(is, HashMap.class)); + } finally { + if (is != null) + is.close(); } - srcTableExdMap.putLocal(name, t); + String cardinality = attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY); - return t; + // parse table identity from file name + String file = path; + if (file.indexOf("/") > -1) { + file = file.substring(file.lastIndexOf("/") + 1); + } + String tableIdentity = file.substring(0, file.length() - MetadataConstants.FILE_SURFIX.length()).toUpperCase(); + TableExtDesc result = new TableExtDesc(); + result.setName(tableIdentity); + result.setUuid(UUID.randomUUID().toString()); + result.setLastModified(0); + result.init(); + result.setCardinality(cardinality); + return result; } private void reloadAllExternalFilter() throws IOException { http://git-wip-us.apache.org/repos/asf/kylin/blob/266a62a5/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java index a861ee1..eb04287 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java @@ -23,10 +23,16 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.JoinTableDesc; @@ -51,7 +57,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { public void after() throws Exception { this.cleanupTestMetadata(); } - + @Test public void testCiModel() { MetadataManager mgr = getInstance(getTestConfig()); @@ -59,7 +65,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { DataModelDesc im = mgr.getDataModelDesc("ci_inner_join_model"); assertSnowflakeQuality(lm); assertSnowflakeQuality(im); - + // check inner/left models are identical apart from the left/inner difference assertEquals(lm.getJoinTables().length, im.getJoinTables().length); for (int i = 0, n = im.getJoinTables().length; i < n; i++) { @@ -73,7 +79,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { assertTrue(lt.getJoin().isLeftJoin()); assertTrue(it.getJoin().isInnerJoin()); } - + assertEquals(lm.getDimensions().size(), im.getDimensions().size()); for (int i = 0, n = im.getDimensions().size(); i < n; i++) { ModelDimensionDesc ld = lm.getDimensions().get(i); @@ -81,10 +87,10 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { assertEquals(ld.getTable(), id.getTable()); assertArrayEquals(ld.getColumns(), id.getColumns()); } - + assertArrayEquals(lm.getMetrics(), im.getMetrics()); } - + private void assertSnowflakeQuality(DataModelDesc model) { Assert.assertNotNull(model); try { @@ -93,7 +99,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { } catch (IllegalArgumentException ex) { // excepted } - + Assert.assertNotNull(model.findTable("BUYER_COUNTRY")); Assert.assertNotNull(model.findTable("SELLER_COUNTRY")); Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME")); @@ -148,4 +154,27 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { getInstance(getTestConfig()).removeTableExt("TEST.TEST_TABLE"); } + + @Test + public void testTableExtCompatibility() throws IOException { + String tableName = "TEST.TEST_TABLE"; + Map<String, String> oldTableExt = new HashMap<>(); + oldTableExt.put(MetadataConstants.TABLE_EXD_CARDINALITY, "1,2,3,4"); + mockUpOldTableExtJson(tableName, oldTableExt); + TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt(tableName); + Assert.assertEquals("1,2,3,4,", tableExtDesc.getCardinality()); + getInstance(getTestConfig()).removeTableExt(tableName); + } + + private void mockUpOldTableExtJson(String tableId, Map<String, String> tableExdProperties) throws IOException { + String path = TableDesc.concatExdResourcePath(tableId); + + ByteArrayOutputStream os = new ByteArrayOutputStream(); + JsonUtil.writeValueIndent(os, tableExdProperties); + os.flush(); + InputStream is = new ByteArrayInputStream(os.toByteArray()); + getStore().putResource(path, is, System.currentTimeMillis()); + os.close(); + is.close(); + } }
