This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 620a6ad5bfd6ef06e1bd082deec6bc5e0d74d3cb Author: Hang Jia <754332...@qq.com> AuthorDate: Fri Feb 24 10:59:06 2023 +0800 KYLIN-5529 Support adding "basic agg index" or "basic table index" separately --- .../kylin/metadata/cube/model/IndexPlan.java | 28 ++++++++--- .../rest/controller/NModelControllerTest.java | 57 ++++++++++++++++++++++ .../apache/kylin/rest/request/ModelRequest.java | 6 +++ .../kylin/rest/service/BaseIndexUpdateHelper.java | 34 ++++++++++--- .../apache/kylin/rest/service/ModelService.java | 24 +++++++-- .../apache/kylin/rest/service/BaseIndexTest.java | 47 +++++++++++++----- .../kylin/rest/service/ModelServiceTest.java | 32 +++++++++++- 7 files changed, 197 insertions(+), 31 deletions(-) diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java index 33c708b9e8..b2f70ee752 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java @@ -24,6 +24,7 @@ import static org.apache.kylin.metadata.cube.model.IndexEntity.isAggIndex; import static org.apache.kylin.metadata.cube.model.IndexEntity.isTableIndex; import java.io.Serializable; +import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.Collection; @@ -994,16 +995,29 @@ public class IndexPlan extends RootPersistentEntity implements Serializable, IEn return !curLayout.equalsCols(replace); } - public void createAndAddBaseIndex(NDataModel model) { + public void createAndAddBaseIndex(NDataModel model, List<IndexEntity.Source> sources) { checkIsNotCachedAndShared(); - LayoutEntity agg = createBaseAggIndex(model); - LayoutEntity table = createBaseTableIndex(model); List<LayoutEntity> baseLayouts = Lists.newArrayList(); - baseLayouts.add(agg); - if (table != null) { - baseLayouts.add(table); + if (sources.contains(IndexEntity.Source.BASE_AGG_INDEX)) { + LayoutEntity agg = createBaseAggIndex(model); + baseLayouts.add(agg); + } + + if (sources.contains(IndexEntity.Source.BASE_TABLE_INDEX)) { + LayoutEntity table = createBaseTableIndex(model); + if (table != null) { + baseLayouts.add(table); + } } - createAndAddBaseIndex(baseLayouts); + if (!baseLayouts.isEmpty()) { + createAndAddBaseIndex(baseLayouts); + } + } + + public void createAndAddBaseIndex(NDataModel model) { + ArrayList<IndexEntity.Source> sources = Lists.newArrayList(IndexEntity.Source.BASE_AGG_INDEX, + IndexEntity.Source.BASE_TABLE_INDEX); + createAndAddBaseIndex(model, sources); } public void createAndAddBaseIndex(List<LayoutEntity> needCreateBaseLayouts) { diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java index 47520fa814..1638aa0bad 100644 --- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java +++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java @@ -360,6 +360,63 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase { Mockito.verify(nModelController).createModel(Mockito.any(ModelRequest.class)); } + @Test + public void testCreateModelWithBaseIndexTypeAndWithBaseIndex() throws Exception { + ModelRequest request = new ModelRequest(); + request.setProject("default"); + + request.setBaseIndexType( + Sets.newHashSet(IndexEntity.Source.BASE_AGG_INDEX, IndexEntity.Source.BASE_TABLE_INDEX)); + request.setWithBaseIndex(true); + NDataModel mockModel = new NDataModel(); + mockModel.setUuid("mock"); + mockModel.setProject("default"); + Mockito.doReturn(mockModel).when(modelService).createModel(request.getProject(), request); + Mockito.doReturn(new IndexPlan()).when(modelService).getIndexPlan(mockModel.getId(), mockModel.getProject()); + mockMvc.perform(MockMvcRequestBuilders.post("/api/models").contentType(MediaType.APPLICATION_JSON) + .content(JsonUtil.writeValueAsString(request)) + .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON))) + .andExpect(MockMvcResultMatchers.status().isOk()); + Mockito.verify(nModelController).createModel(Mockito.any(ModelRequest.class)); + } + + @Test + public void testCreateModelWithBaseIndex() throws Exception { + ModelRequest request = new ModelRequest(); + request.setProject("default"); + + request.setWithBaseIndex(true); + NDataModel mockModel = new NDataModel(); + mockModel.setUuid("mock"); + mockModel.setProject("default"); + Mockito.doReturn(mockModel).when(modelService).createModel(request.getProject(), request); + Mockito.doReturn(new IndexPlan()).when(modelService).getIndexPlan(mockModel.getId(), mockModel.getProject()); + mockMvc.perform(MockMvcRequestBuilders.post("/api/models").contentType(MediaType.APPLICATION_JSON) + .content(JsonUtil.writeValueAsString(request)) + .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON))) + .andExpect(MockMvcResultMatchers.status().isOk()); + Mockito.verify(nModelController).createModel(Mockito.any(ModelRequest.class)); + } + + @Test + public void testCreateModelWithBaseIndexType() throws Exception { + ModelRequest request = new ModelRequest(); + request.setProject("default"); + + request.setBaseIndexType( + Sets.newHashSet(IndexEntity.Source.BASE_AGG_INDEX, IndexEntity.Source.BASE_TABLE_INDEX)); + NDataModel mockModel = new NDataModel(); + mockModel.setUuid("mock"); + mockModel.setProject("default"); + Mockito.doReturn(mockModel).when(modelService).createModel(request.getProject(), request); + Mockito.doReturn(new IndexPlan()).when(modelService).getIndexPlan(mockModel.getId(), mockModel.getProject()); + mockMvc.perform(MockMvcRequestBuilders.post("/api/models").contentType(MediaType.APPLICATION_JSON) + .content(JsonUtil.writeValueAsString(request)) + .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON))) + .andExpect(MockMvcResultMatchers.status().isOk()); + Mockito.verify(nModelController).createModel(Mockito.any(ModelRequest.class)); + } + @Test public void checkPartitionDesc() throws Exception { PartitionDesc partitionDesc = new PartitionDesc(); diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java index 054b7e21f0..e585a8cf47 100644 --- a/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java +++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java @@ -18,10 +18,13 @@ package org.apache.kylin.rest.request; + import java.util.Collection; import java.util.List; +import java.util.Set; import java.util.function.BiFunction; +import org.apache.kylin.metadata.cube.model.IndexEntity; import org.apache.kylin.metadata.cube.model.IndexPlan; import org.apache.kylin.metadata.insensitive.ModelInsensitiveRequest; import org.apache.kylin.metadata.model.ColumnDesc; @@ -84,6 +87,9 @@ public class ModelRequest extends NDataModel implements ModelInsensitiveRequest @JsonProperty("with_base_index") private boolean withBaseIndex = false; + @JsonProperty("base_index_type") + private Set<IndexEntity.Source> baseIndexType; + @JsonProperty("with_second_storage") private boolean withSecondStorage = false; diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/BaseIndexUpdateHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/BaseIndexUpdateHelper.java index 94e0915750..881f77a14f 100644 --- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/BaseIndexUpdateHelper.java +++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/BaseIndexUpdateHelper.java @@ -17,14 +17,19 @@ */ package org.apache.kylin.rest.service; +import java.util.List; + import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.rest.util.SpringContext; +import org.apache.kylin.metadata.cube.model.IndexEntity; import org.apache.kylin.metadata.cube.model.IndexPlan; import org.apache.kylin.metadata.cube.model.LayoutEntity; import org.apache.kylin.metadata.cube.model.NIndexPlanManager; import org.apache.kylin.metadata.model.NDataModel; import org.apache.kylin.rest.request.CreateBaseIndexRequest; import org.apache.kylin.rest.response.BuildBaseIndexResponse; +import org.apache.kylin.rest.util.SpringContext; + +import com.google.common.collect.Lists; import io.kyligence.kap.secondstorage.SecondStorageUpdater; import io.kyligence.kap.secondstorage.SecondStorageUtil; @@ -43,14 +48,25 @@ public class BaseIndexUpdateHelper { private String project; private String modelId; - private boolean createIfNotExist; + private List<IndexEntity.Source> updateBaseIndexTypes; private boolean needUpdate; private boolean secondStorageEnabled = false; @Setter private boolean needCleanSecondStorage = true; public BaseIndexUpdateHelper(NDataModel model, boolean createIfNotExist) { + this(model, updateTypesByFlag(createIfNotExist)); + } + + private static List<IndexEntity.Source> updateTypesByFlag(boolean createIfNotExist) { + if (createIfNotExist) { + return Lists.newArrayList(IndexEntity.Source.BASE_AGG_INDEX, IndexEntity.Source.BASE_TABLE_INDEX); + } else { + return Lists.newArrayList(); + } + } + public BaseIndexUpdateHelper(NDataModel model, List<IndexEntity.Source> updateBaseIndexTypes) { NIndexPlanManager indexPlanManager = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(), model.getProject()); IndexPlan indexPlan = indexPlanManager.getIndexPlan(model.getId()); @@ -61,7 +77,7 @@ public class BaseIndexUpdateHelper { if (needUpdate) { project = model.getProject(); modelId = model.getId(); - this.createIfNotExist = createIfNotExist; + this.updateBaseIndexTypes = updateBaseIndexTypes; preBaseAggLayout = getBaseAggLayout(); preBaseTableLayout = getBaseTableLayout(); } @@ -76,18 +92,20 @@ public class BaseIndexUpdateHelper { if (!needUpdate) { return BuildBaseIndexResponse.EMPTY; } - if (notExist(preBaseAggLayout) && notExist(preBaseTableLayout) && !createIfNotExist) { + if (notExist(preBaseAggLayout) && notExist(preBaseTableLayout) + && !updateBaseIndexTypes.contains(IndexEntity.Source.BASE_TABLE_INDEX) + && !updateBaseIndexTypes.contains(IndexEntity.Source.BASE_AGG_INDEX)) { return BuildBaseIndexResponse.EMPTY; } long curBaseTableLayout = getBaseTableLayout(); - boolean needCreateBaseTable = createIfNotExist; + boolean needCreateBaseTable = updateBaseIndexTypes.contains(IndexEntity.Source.BASE_TABLE_INDEX); if (exist(preBaseTableLayout) && notExist(curBaseTableLayout)) { needCreateBaseTable = true; } Long curExistBaseAggLayout = getBaseAggLayout(); - boolean needCreateBaseAgg = createIfNotExist; + boolean needCreateBaseAgg = updateBaseIndexTypes.contains(IndexEntity.Source.BASE_AGG_INDEX); if (exist(preBaseAggLayout) && notExist(curExistBaseAggLayout)) { needCreateBaseAgg = true; } @@ -98,8 +116,8 @@ public class BaseIndexUpdateHelper { CreateBaseIndexRequest indexRequest = new CreateBaseIndexRequest(); indexRequest.setModelId(modelId); indexRequest.setProject(project); - BuildBaseIndexResponse response = service.updateBaseIndex(project, indexRequest, needCreateBaseAgg, - needCreateBaseTable, true); + BuildBaseIndexResponse response = service.updateBaseIndex(project, indexRequest, needCreateBaseTable, + needCreateBaseAgg, true); response.judgeIndexOperateType(exist(preBaseAggLayout), true); response.judgeIndexOperateType(exist(preBaseTableLayout), false); diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java index 9a917fb61c..6c5aced159 100644 --- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java +++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java @@ -2053,14 +2053,30 @@ public class ModelService extends AbstractModelService implements TableModelSupp } public void addBaseIndex(ModelRequest modelRequest, NDataModel model, IndexPlan indexPlan) { - if (!modelRequest.isWithSecondStorage() && NDataModel.ModelType.BATCH == model.getModelType() - && modelRequest.isWithBaseIndex()) { - indexPlan.createAndAddBaseIndex(model); + if (!modelRequest.isWithSecondStorage() && NDataModel.ModelType.BATCH == model.getModelType()) { + List<IndexEntity.Source> sources = needHandleBaseIndexType(modelRequest); + indexPlan.createAndAddBaseIndex(model, sources); } else if (modelRequest.isWithSecondStorage()) { indexPlan.createAndAddBaseIndex(Collections.singletonList(indexPlan.createBaseTableIndex(model))); } } + private List<IndexEntity.Source> needHandleBaseIndexType(ModelRequest modelRequest) { + List<IndexEntity.Source> sources = Lists.newArrayList(); + if (modelRequest.getBaseIndexType() != null) { + if (modelRequest.getBaseIndexType().contains(IndexEntity.Source.BASE_AGG_INDEX)) { + sources.add(IndexEntity.Source.BASE_AGG_INDEX); + } + if (modelRequest.getBaseIndexType().contains(IndexEntity.Source.BASE_TABLE_INDEX)) { + sources.add(IndexEntity.Source.BASE_TABLE_INDEX); + } + } else if (modelRequest.isWithBaseIndex()) { + sources.add(IndexEntity.Source.BASE_AGG_INDEX); + sources.add(IndexEntity.Source.BASE_TABLE_INDEX); + } + return sources; + } + // for streaming & fusion model private void createStreamingJob(String project, NDataModel model, ModelRequest request) { if (NDataModel.ModelType.BATCH != model.getModelType()) { @@ -2995,7 +3011,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp copyModel.init(modelManager.getConfig(), project, modelManager.getCCRelatedModels(copyModel)); BaseIndexUpdateHelper baseIndexUpdater = new BaseIndexUpdateHelper(originModel, - request.isWithBaseIndex()); + needHandleBaseIndexType(request)); preProcessBeforeModelSave(copyModel, project); val updated = modelManager.updateDataModelDesc(copyModel); diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java index 928c903239..8abe394632 100644 --- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java +++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java @@ -142,6 +142,25 @@ public class BaseIndexTest extends SourceTestCase { compareBaseIndex(getModelIdFrom(modelRequest.getAlias()), baseTableLayout, baseAggLayout); } + @Test + public void testCreateBaseTableLayoutWithBaseIndexTypes() { + NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default"); + ModelRequest modelRequest = FormModel(modelManager.getDataModelDesc(COMMON_MODEL_ID)); + modelRequest.setWithBaseIndex(false); + String modelId = modelService.createModel(modelRequest.getProject(), modelRequest).getId(); + BaseIndexUpdateHelper baseIndexUpdater = new BaseIndexUpdateHelper(modelRequest, + Lists.newArrayList(Source.BASE_AGG_INDEX)); + BuildBaseIndexResponse baseIndexResponse = baseIndexUpdater.update(indexPlanService); + LayoutEntity baseAggLayout = LayoutBuilder.builder().colOrder(0, 1, 2, 3, 100000, 100001).id(1L).build(); + LayoutEntity baseTableLayout = null; + compareBaseIndex(getModelIdFrom(modelRequest.getAlias()), baseTableLayout, baseAggLayout); + + baseIndexUpdater = new BaseIndexUpdateHelper(modelRequest, Lists.newArrayList(Source.BASE_TABLE_INDEX)); + baseIndexResponse = baseIndexUpdater.update(indexPlanService); + baseTableLayout = LayoutBuilder.builder().colOrder(0, 1, 2, 3).id(20000000001L).build(); + compareBaseIndex(getModelIdFrom(modelRequest.getAlias()), baseTableLayout, baseAggLayout); + } + @Test public void testCreateBaseLayoutWithProperties() { // create base index is same with index in rulebaseindex or indexes @@ -402,7 +421,7 @@ public class BaseIndexTest extends SourceTestCase { IndexPlan indexPlan = NIndexPlanManager.getInstance(getTestConfig(), getProject()).getIndexPlan(modelId); return indexPlan.getAllLayouts().stream() .filter(layoutEntity -> layoutEntity.isBase() && isAggIndex(layoutEntity.getId())).findFirst() - .orElseGet(null); + .orElse(null); } private String createBaseIndexFromModel(String modelId) { @@ -417,18 +436,24 @@ public class BaseIndexTest extends SourceTestCase { if (expectedBaseTableLayout == null && baseTableLayout == null) { return; } - Assert.assertThat(baseAggLayout.getColOrder(), equalTo(expectedBaseAggLayout.getColOrder())); - Assert.assertThat(baseAggLayout.getShardByColumns(), equalTo(expectedBaseAggLayout.getShardByColumns())); - Assert.assertThat(baseAggLayout.getSortByColumns(), equalTo(expectedBaseAggLayout.getSortByColumns())); - Assert.assertThat(baseTableLayout.getColOrder(), equalTo(expectedBaseTableLayout.getColOrder())); - Assert.assertThat(baseTableLayout.getShardByColumns(), equalTo(expectedBaseTableLayout.getShardByColumns())); - Assert.assertThat(baseTableLayout.getSortByColumns(), equalTo(expectedBaseTableLayout.getSortByColumns())); - if (baseAggLayout.getId() != -1) { - Assert.assertEquals(expectedBaseAggLayout.getId(), baseAggLayout.getId()); + if (expectedBaseAggLayout != null) { + Assert.assertThat(baseAggLayout.getColOrder(), equalTo(expectedBaseAggLayout.getColOrder())); + Assert.assertThat(baseAggLayout.getShardByColumns(), equalTo(expectedBaseAggLayout.getShardByColumns())); + Assert.assertThat(baseAggLayout.getSortByColumns(), equalTo(expectedBaseAggLayout.getSortByColumns())); + if (baseAggLayout.getId() != -1) { + Assert.assertEquals(expectedBaseAggLayout.getId(), baseAggLayout.getId()); + } } - if (baseTableLayout.getId() != -1) { - Assert.assertEquals(expectedBaseTableLayout.getId(), baseTableLayout.getId()); + if (expectedBaseTableLayout != null) { + Assert.assertThat(baseTableLayout.getColOrder(), equalTo(expectedBaseTableLayout.getColOrder())); + Assert.assertThat(baseTableLayout.getShardByColumns(), + equalTo(expectedBaseTableLayout.getShardByColumns())); + Assert.assertThat(baseTableLayout.getSortByColumns(), equalTo(expectedBaseTableLayout.getSortByColumns())); + + if (baseTableLayout.getId() != -1) { + Assert.assertEquals(expectedBaseTableLayout.getId(), baseTableLayout.getId()); + } } } diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java index b69b89585b..58971f625c 100644 --- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java +++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java @@ -5009,8 +5009,38 @@ public class ModelServiceTest extends SourceTestCase { when(modelRequest.isWithSecondStorage()).thenReturn(false); when(model.getModelType()).thenReturn(NDataModel.ModelType.BATCH); when(modelRequest.isWithBaseIndex()).thenReturn(true); + when(modelRequest.getBaseIndexType()).thenReturn(null); modelService.addBaseIndex(modelRequest, model, indexPlan); - Mockito.verify(indexPlan).createAndAddBaseIndex(model); + Mockito.verify(indexPlan).createAndAddBaseIndex(model, + Lists.newArrayList(IndexEntity.Source.BASE_AGG_INDEX, IndexEntity.Source.BASE_TABLE_INDEX)); + when(modelRequest.isWithSecondStorage()).thenReturn(true); + when(indexPlan.createBaseTableIndex(model)).thenReturn(null); + modelService.addBaseIndex(modelRequest, model, indexPlan); + Mockito.verify(indexPlan).createAndAddBaseIndex(anyList()); + } + + @Test + public void testAddBaseAggIndex() { + val modelRequest = mock(ModelRequest.class); + val model = mock(NDataModel.class); + val indexPlan = mock(IndexPlan.class); + + when(modelRequest.isWithSecondStorage()).thenReturn(false); + when(model.getModelType()).thenReturn(NDataModel.ModelType.BATCH); + when(modelRequest.isWithBaseIndex()).thenReturn(true); + when(modelRequest.getBaseIndexType()).thenReturn(null); + modelService.addBaseIndex(modelRequest, model, indexPlan); + Mockito.verify(indexPlan).createAndAddBaseIndex(model, + Lists.newArrayList(IndexEntity.Source.BASE_AGG_INDEX, IndexEntity.Source.BASE_TABLE_INDEX)); + + when(modelRequest.getBaseIndexType()).thenReturn(Collections.singleton(IndexEntity.Source.BASE_AGG_INDEX)); + modelService.addBaseIndex(modelRequest, model, indexPlan); + Mockito.verify(indexPlan).createAndAddBaseIndex(model, Lists.newArrayList(IndexEntity.Source.BASE_AGG_INDEX)); + + when(modelRequest.getBaseIndexType()).thenReturn(Collections.singleton(IndexEntity.Source.BASE_TABLE_INDEX)); + modelService.addBaseIndex(modelRequest, model, indexPlan); + Mockito.verify(indexPlan).createAndAddBaseIndex(model, Lists.newArrayList(IndexEntity.Source.BASE_TABLE_INDEX)); + when(modelRequest.isWithSecondStorage()).thenReturn(true); when(indexPlan.createBaseTableIndex(model)).thenReturn(null); modelService.addBaseIndex(modelRequest, model, indexPlan);