This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 04a956f3983c3873fe38dfa2f597f8cab7681afa
Author: lixiang <447399...@qq.com>
AuthorDate: Sun Apr 9 19:05:53 2023 +0800
KYLIN-5604 add open api for update agg group (#30273)
---
.../common/exception/code/ErrorCodeServer.java | 7 +
.../resources/kylin_error_msg_conf_cn.properties | 7 +
.../resources/kylin_error_msg_conf_en.properties | 7 +
.../kylin_error_suggestion_conf_cn.properties | 1 +
.../kylin_error_suggestion_conf_en.properties | 1 +
.../main/resources/kylin_errorcode_conf.properties | 7 +
.../controller/open/OpenIndexPlanController.java | 74 ++++++++++
.../open/OpenIndexPlanControllerTest.java | 114 ++++++++++++++
.../request/OpenUpdateRuleBasedCuboidRequest.java | 70 +++++++++
.../kylin/rest/service/FusionIndexService.java | 141 +++++++++++++++++-
.../kylin/rest/service/FusionIndexServiceTest.java | 164 ++++++++++++++++++++-
11 files changed, 584 insertions(+), 9 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
index 5e0df273eb..91a14bb8ee 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
@@ -115,6 +115,12 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
INDEX_PARAMETER_INVALID("KE-010012203"),
SHARD_BY_COLUMN_NOT_IN_INDEX("KE-010012204"),
OUT_OF_MAX_DIM_COMBINATION("KE-010012205"),
+ DIMENSION_NOT_IN_MODEL("KE-010012206"),
+ MEASURE_NOT_IN_MODEL("KE-010012207"),
+ MANDATORY_NOT_IN_DIMENSION("KE-010012208"),
+ HIERARCHY_NOT_IN_DIMENSION("KE-010012209"),
+ JOINT_NOT_IN_DIMENSION("KE-010012210"),
+ DIMENSION_ONLY_SET_ONCE("KE-010012211"),
// 10043XX parameter check
REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY("KE-010043201"),
@@ -138,6 +144,7 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
USERNAME_COMPANY_NAME_INVALID_VALUE("KE-010043219"),
USER_GROUP_NOT_EXIST("KE-010043220"),
REPEATED_PARAMETER("KE-010043221"),
+ INTEGER_POSITIVE_CHECK("KE-010043222"),
// 100313xx async query
ASYNC_QUERY_RESULT_NOT_FOUND("KE-010031301"),
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 2e4a068fe0..ee75b68336 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -116,6 +116,12 @@ KE-010012202=因为存在相同的索引，无法新建该索引。请修改。
KE-010012203=参数 “%s” 仅支持 “%s”。
KE-010012204=ShardBy 列不在索引包含的列中，请修改后重试。
KE-010012205=聚合组生成的索引数超出系统允许的最大索引数(%s)。
+KE-010012206=聚合组包含的维度必须已经添加在模型中，请修改后重试。
+KE-010012207=聚合组包含的度量必须已经添加在模型中，请修改后重试。
+KE-010012208=必需维度必须在 'dimension' 参数中，请修改后重试。
+KE-010012209=层级维度必须在 'dimension' 参数中，请修改后重试。
+KE-010012210=联合维度必须在 'dimension' 参数中，请修改后重试。
+KE-010012211=任一维度只能在必需维度、层级维度或联合维度中设置一次，请修改后重试。
## 10043XX parameter check
KE-010043201=请求参数 “%s” 为空或值为空。请检查请求参数是否正确填写。
@@ -139,6 +145,7 @@ KE-010043218=无效请求参数值 “%s”。请输入正确的sort_by字段。
KE-010043219=请使用中文、英文、空格命名用户名和公司。
KE-010043220=找不到用户组 “%s”。请检查后重试。
KE-010043221=参数 “%s” 已存在。请检查后重试。
+KE-010043222=输入的参数值无效，参数值需为正整数。请检查后重试。
## Streaming
KE-010035202=使用解析器 “%s” 解析Topic “%s” 的消息时发生异常，请检查后重试。
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index 187c23a6e4..511f9b6ee0 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -115,6 +115,12 @@ KE-010012202=Can't add this index, as the same index
already exists. Please modi
KE-010012203=The parameter "%s" only supports "%s".
KE-010012204=The ShardBy column is not included in the index. Please fix and
try again.
KE-010012205=The number of indexes generated by the aggregate group exceeds
the maximum number(%s) of indexes allowed by the system.
+KE-010012206=The dimension contained in the aggregate group must have been
added to the model, please modify and try again.
+KE-010012207=The measure contained in the aggregate group must have been added
to the model, please modify and try again.
+KE-010012208=The mandatory dimension must be included in the 'dimension'
parameter. Please modify it and try again.
+KE-010012209=The hierarchy dimension must be included in the 'dimension'
parameter. Please modify it and try again.
+KE-010012210=The joint dimension must be included in the 'dimension'
parameter. Please modify it and try again.
+KE-010012211=Any dimension can only be set once in mandatory dimension,
hierarchy dimension or joint dimension. Please modify and try again.
## 10043XX parameter check
KE-010043201=Request parameter "%s" is empty or value is empty. Please check
the request parameters.
@@ -138,6 +144,7 @@ KE-010043218=Invalid request parameter value "%s". Please
enter the correct sort
KE-010043219=Please use characters or spaces for username and company.
KE-010043220=Can't find user group "%s". Please check and try again.
KE-010043221=The parameter “%s” already exists. Please check and try again.
+KE-010043222=The entered parameter value is invalid. The parameter value must
be a positive integer. Please check and try again.
## Streaming
KE-010035202=An exception occurred while parsing the messages of Topic "%2$s"
with parser "%1$s". Please check and try again.
diff --git
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
index 42dc91cf84..43af745ba9 100644
---
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
+++
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
@@ -132,6 +132,7 @@ KE-010043218=
KE-010043219=
KE-010043220=
KE-010043221=
+KE-010043222=
## Streaming
KE-010035215=
diff --git
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
index 105f521360..6dc32307e2 100644
---
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
+++
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
@@ -132,6 +132,7 @@ KE-010043218=
KE-010043219=
KE-010043220=
KE-010043221=
+KE-010043222=
## Streaming
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index ae53e187e4..30f304c8e0 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -127,6 +127,12 @@ KE-010012202
KE-010012203
KE-010012204
KE-010012205
+KE-010012206
+KE-010012207
+KE-010012208
+KE-010012209
+KE-010012210
+KE-010012211
## 10043XX parameter check
KE-010043201
@@ -150,6 +156,7 @@ KE-010043218
KE-010043219
KE-010043220
KE-010043221
+KE-010043222
## Streaming
KE-010035202
diff --git
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenIndexPlanController.java
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenIndexPlanController.java
new file mode 100644
index 0000000000..432528b511
--- /dev/null
+++
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenIndexPlanController.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.controller.open;
+
+import static
org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NOT_EXIST;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.rest.controller.NBasicController;
+import org.apache.kylin.rest.controller.NIndexPlanController;
+import org.apache.kylin.rest.request.OpenUpdateRuleBasedCuboidRequest;
+import org.apache.kylin.rest.request.UpdateRuleBasedCuboidRequest;
+import org.apache.kylin.rest.response.DiffRuleBasedIndexResponse;
+import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.service.FusionIndexService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import lombok.val;
+
+@RestController
+@RequestMapping(value = "/api/index_plans", produces = {
HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON })
+public class OpenIndexPlanController extends NBasicController {
+
+ private static final String MODEL_ALIAS = "model";
+
+ @Autowired
+ @Qualifier("fusionIndexService")
+ private FusionIndexService fusionIndexService;
+
+ @Autowired
+ NIndexPlanController indexPlanController;
+
+ @PutMapping(value = "/agg_groups")
+ public EnvelopeResponse<DiffRuleBasedIndexResponse> updateRule(
+ @RequestBody OpenUpdateRuleBasedCuboidRequest request) {
+ String projectName = checkProjectName(request.getProject());
+ checkRequiredArg(MODEL_ALIAS, request.getModelAlias());
+ request.setProject(projectName);
+ val modelManager =
NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), projectName);
+ NDataModel model =
modelManager.getDataModelDescByAlias(request.getModelAlias());
+ if (model == null) {
+ throw new KylinException(MODEL_NOT_EXIST);
+ }
+ UpdateRuleBasedCuboidRequest internalRequest =
fusionIndexService.convertOpenToInternal(request, model);
+ EnvelopeResponse<DiffRuleBasedIndexResponse> response =
indexPlanController
+ .calculateDiffRuleBasedIndex(internalRequest);
+ indexPlanController.updateRule(internalRequest);
+ return response;
+ }
+}
diff --git
a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenIndexPlanControllerTest.java
b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenIndexPlanControllerTest.java
new file mode 100644
index 0000000000..529a491c92
--- /dev/null
+++
b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenIndexPlanControllerTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.controller.open;
+
+import static
org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.controller.NIndexPlanController;
+import org.apache.kylin.rest.request.OpenUpdateRuleBasedCuboidRequest;
+import org.apache.kylin.rest.request.UpdateRuleBasedCuboidRequest;
+import org.apache.kylin.rest.service.FusionIndexService;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.springframework.http.MediaType;
+import org.springframework.security.authentication.TestingAuthenticationToken;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
+import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+
+import lombok.val;
+
+public class OpenIndexPlanControllerTest extends NLocalFileMetadataTestCase {
+
+ @InjectMocks
+ private final OpenIndexPlanController openIndexPlanController =
Mockito.spy(new OpenIndexPlanController());
+
+ @Mock
+ private FusionIndexService fusionIndexService;
+
+ @Mock
+ private NIndexPlanController nIndexPlanController;
+
+ private MockMvc mockMvc;
+
+ private final Authentication authentication = new
TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN);
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ mockMvc = MockMvcBuilders.standaloneSetup(openIndexPlanController)
+ .defaultRequest(MockMvcRequestBuilders.get("/"))
+
.defaultResponseCharacterEncoding(StandardCharsets.UTF_8).build();
+ SecurityContextHolder.getContext().setAuthentication(authentication);
+ createTestMetadata();
+ }
+
+ @After
+ public void tearDown() {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void testCreateAggGroups() throws Exception {
+ val request = UpdateRuleBasedCuboidRequest.builder().project("default")
+
.modelId("89af4ee2-2cdb-4b07-b39e-4c29856309aa").aggregationGroups(Lists.newArrayList()).build();
+
+ OpenUpdateRuleBasedCuboidRequest openRequest = new
OpenUpdateRuleBasedCuboidRequest();
+ openRequest.setModelAlias("nmodel_basic");
+ openRequest.setProject("default");
+ val modelManager =
NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(),
request.getProject());
+ NDataModel model =
modelManager.getDataModelDescByAlias(openRequest.getModelAlias());
+
+ Mockito.when(fusionIndexService.convertOpenToInternal(openRequest,
model)).thenReturn(request);
+
mockMvc.perform(MockMvcRequestBuilders.put("/api/index_plans/agg_groups")
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(openRequest))
+
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(openIndexPlanController).updateRule(openRequest);
+ }
+
+ @Test
+ public void testCreateAggGroupsNotExistModel() throws Exception {
+ OpenUpdateRuleBasedCuboidRequest openRequest = new
OpenUpdateRuleBasedCuboidRequest();
+ openRequest.setModelAlias("model_not_exist");
+ openRequest.setProject("default");
+
mockMvc.perform(MockMvcRequestBuilders.put("/api/index_plans/agg_groups")
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(openRequest))
+
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().is5xxServerError());
+ Mockito.verify(openIndexPlanController).updateRule(openRequest);
+ }
+}
diff --git
a/src/modeling-service/src/main/java/org/apache/kylin/rest/request/OpenUpdateRuleBasedCuboidRequest.java
b/src/modeling-service/src/main/java/org/apache/kylin/rest/request/OpenUpdateRuleBasedCuboidRequest.java
new file mode 100644
index 0000000000..2424fab513
--- /dev/null
+++
b/src/modeling-service/src/main/java/org/apache/kylin/rest/request/OpenUpdateRuleBasedCuboidRequest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.request;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@AllArgsConstructor
+@NoArgsConstructor
+@Data
+public class OpenUpdateRuleBasedCuboidRequest {
+
+ @JsonProperty("project")
+ private String project;
+
+ @JsonProperty("model")
+ private String modelAlias;
+
+ @JsonProperty("aggregation_groups")
+ private List<OpenAggGroupRequest> aggregationGroups;
+
+ @JsonProperty("global_dim_cap")
+ private Integer globalDimCap;
+
+ @JsonProperty("restore_deleted_index")
+ private boolean restoreDeletedIndex;
+
+ @Data
+ public static class OpenAggGroupRequest {
+
+ @JsonProperty("dimensions")
+ private String[] dimensions;
+
+ @JsonProperty("measures")
+ private String[] measures;
+
+ @JsonProperty("mandatory_dims")
+ private String[] mandatoryDims;
+
+ @JsonProperty("hierarchy_dims")
+ private String[][] hierarchyDims;
+
+ @JsonProperty("joint_dims")
+ private String[][] jointDims;
+
+ @JsonProperty("dim_cap")
+ private Integer dimCap;
+ }
+}
diff --git
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
index fd3b99a283..71cece5de9 100644
---
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
+++
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
@@ -18,20 +18,35 @@
package org.apache.kylin.rest.service;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.INTEGER_POSITIVE_CHECK;
+
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
+import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
+import org.apache.kylin.common.exception.code.ErrorCodeServer;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.model.SelectRule;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.execution.JobTypeEnum;
import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup;
@@ -50,6 +65,7 @@ import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.rest.aspect.Transaction;
import org.apache.kylin.rest.request.AggShardByColumnsRequest;
import org.apache.kylin.rest.request.CreateTableIndexRequest;
+import org.apache.kylin.rest.request.OpenUpdateRuleBasedCuboidRequest;
import org.apache.kylin.rest.request.UpdateRuleBasedCuboidRequest;
import org.apache.kylin.rest.response.AggIndexResponse;
import org.apache.kylin.rest.response.BuildIndexResponse;
@@ -62,8 +78,6 @@ import org.apache.kylin.streaming.metadata.StreamingJobMeta;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -73,6 +87,8 @@ public class FusionIndexService extends BasicService {
private static final List<JobStatusEnum> runningStatus =
Arrays.asList(JobStatusEnum.STARTING,
JobStatusEnum.RUNNING, JobStatusEnum.STOPPING);
+ private static final String COUNT_ALL_MEASURE = "COUNT_ALL";
+
@Autowired
private IndexPlanService indexPlanService;
@@ -423,6 +439,127 @@ public class FusionIndexService extends BasicService {
batchIndexIds);
}
+ public UpdateRuleBasedCuboidRequest
convertOpenToInternal(OpenUpdateRuleBasedCuboidRequest request,
+ NDataModel model) {
+ checkParamPositive(request.getGlobalDimCap());
+ val dimMap =
model.getEffectiveDimensions().entrySet().stream().collect(Collectors
+ .toMap(e -> e.getValue().getAliasDotName(), Map.Entry::getKey,
(u, v) -> v, LinkedHashMap::new));
+ val meaMap = model.getEffectiveMeasures().entrySet().stream().collect(
+ Collectors.toMap(e -> e.getValue().getName(),
Map.Entry::getKey, (u, v) -> v, LinkedHashMap::new));
+
+ List<NAggregationGroup> newAdded =
request.getAggregationGroups().stream().map(aggGroup -> {
+ NAggregationGroup group = new NAggregationGroup();
+ Preconditions.checkNotNull(aggGroup.getDimensions(), "dimension
should not null");
+ checkParamPositive(aggGroup.getDimCap());
+ val selectedDimMap = extractIds(aggGroup.getDimensions(), dimMap,
AggGroupParams.DIMENSION);
+ group.setIncludes(selectedDimMap.values().toArray(new Integer[0]));
+ String[] measures = extractMeasures(aggGroup.getMeasures());
+ val selectedMeaMap = extractIds(measures, meaMap,
AggGroupParams.MEASURE);
+ group.setMeasures(selectedMeaMap.values().toArray(new Integer[0]));
+ SelectRule selectRule = new SelectRule();
+ val mandatoryDimMap = extractIds(aggGroup.getMandatoryDims(),
selectedDimMap, AggGroupParams.MANDATORY);
+ Set<String> allDims = new HashSet<>(mandatoryDimMap.keySet());
+ selectRule.setMandatoryDims(mandatoryDimMap.values().toArray(new
Integer[0]));
+
selectRule.setHierarchyDims(extractJointOrHierarchyIds(aggGroup.getHierarchyDims(),
selectedDimMap, allDims,
+ AggGroupParams.HIERARCHY));
+ selectRule.setJointDims(
+ extractJointOrHierarchyIds(aggGroup.getJointDims(),
selectedDimMap, allDims, AggGroupParams.JOINT));
+ selectRule.setDimCap(aggGroup.getDimCap() != null ?
aggGroup.getDimCap() : request.getGlobalDimCap());
+ group.setSelectRule(selectRule);
+ return group;
+ }).collect(Collectors.toList());
+
+ RuleBasedIndex ruleBasedIndex = getRule(request.getProject(),
model.getUuid());
+ List<NAggregationGroup> groups = ruleBasedIndex.getAggregationGroups();
+ groups.addAll(newAdded);
+
+ UpdateRuleBasedCuboidRequest result = new
UpdateRuleBasedCuboidRequest();
+ result.setModelId(model.getUuid());
+ result.setProject(request.getProject());
+ result.setLoadData(false);
+ result.setRestoreDeletedIndex(request.isRestoreDeletedIndex());
+ result.setAggregationGroups(groups);
+ return result;
+ }
+
+ private String[] extractMeasures(String[] measures) {
+ if (ArrayUtils.isEmpty(measures)) {
+ return new String[] { COUNT_ALL_MEASURE };
+ } else {
+ List<String> list = Arrays.stream(measures).filter(m ->
!m.equals(COUNT_ALL_MEASURE))
+ .collect(Collectors.toList());
+ list.add(0, COUNT_ALL_MEASURE);
+ return list.toArray(new String[0]);
+ }
+ }
+
+ private void checkParamPositive(Integer dimCap) {
+ if (dimCap != null && dimCap <= 0) {
+ throw new KylinException(INTEGER_POSITIVE_CHECK);
+ }
+ }
+
+ private Map<String, Integer> extractIds(String[] dimOrMeaNames,
Map<String, Integer> nameIdMap,
+ AggGroupParams aggGroupParams) {
+ if (dimOrMeaNames == null || dimOrMeaNames.length == 0) {
+ return new HashMap<>();
+ }
+ Set<String> set = Arrays.stream(dimOrMeaNames)
+ .map(str -> aggGroupParams == AggGroupParams.MEASURE ? str :
StringUtils.upperCase(str, Locale.ROOT))
+ .collect(Collectors.toCollection(TreeSet::new));
+ if (set.size() < dimOrMeaNames.length) {
+ throw new IllegalStateException(
+ "Dimension or measure in agg group must not contain
duplication: " + Arrays.asList(dimOrMeaNames));
+ }
+
+ Map<String, Integer> upperCaseMap = nameIdMap.entrySet().stream()
+ .collect(Collectors.toMap(entry -> aggGroupParams ==
AggGroupParams.MEASURE ? entry.getKey()
+ : StringUtils.upperCase(entry.getKey(), Locale.ROOT),
Map.Entry::getValue));
+ if (!upperCaseMap.keySet().containsAll(set)) {
+ switch (aggGroupParams) {
+ case DIMENSION:
+ throw new
KylinException(ErrorCodeServer.DIMENSION_NOT_IN_MODEL);
+ case MEASURE:
+ throw new KylinException(ErrorCodeServer.MEASURE_NOT_IN_MODEL);
+ case MANDATORY:
+ throw new
KylinException(ErrorCodeServer.MANDATORY_NOT_IN_DIMENSION);
+ case HIERARCHY:
+ throw new
KylinException(ErrorCodeServer.HIERARCHY_NOT_IN_DIMENSION);
+ case JOINT:
+ throw new
KylinException(ErrorCodeServer.JOINT_NOT_IN_DIMENSION);
+ default:
+ throw new IllegalStateException("this should not happen");
+ }
+ }
+ return set.stream()
+ .collect(Collectors.toMap(Function.identity(),
upperCaseMap::get, (v1, v2) -> v1, LinkedHashMap::new));
+ }
+
+ private Integer[][] extractJointOrHierarchyIds(String[][] origins,
Map<String, Integer> selectedDimMap, Set<String> allDims,
+ AggGroupParams aggGroupParams) {
+ if (origins == null || origins.length == 0) {
+ return new Integer[0][];
+ }
+ Integer[][] result = new Integer[origins.length][];
+ for (int i = 0; i < origins.length; i++) {
+ if (ArrayUtils.isEmpty(origins[i])) {
+ continue;
+ }
+ Map<String, Integer> tmp = extractIds(origins[i], selectedDimMap,
aggGroupParams);
+ if (Sets.intersection(tmp.keySet(), allDims).isEmpty()) {
+ allDims.addAll(tmp.keySet());
+ result[i] = tmp.values().toArray(new Integer[0]);
+ } else {
+ throw new
KylinException(ErrorCodeServer.DIMENSION_ONLY_SET_ONCE);
+ }
+ }
+ return result;
+ }
+
+ enum AggGroupParams {
+ DIMENSION, MEASURE, MANDATORY, HIERARCHY, JOINT
+ }
+
private String getBatchModel(String project, String modelId) {
FusionModel fusionModel = getManager(FusionModelManager.class,
project).getFusionModel(modelId);
return fusionModel.getBatchModel().getId();
diff --git
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionIndexServiceTest.java
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionIndexServiceTest.java
index 1f78db3f14..4684c0fcdb 100644
---
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionIndexServiceTest.java
+++
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionIndexServiceTest.java
@@ -25,13 +25,10 @@ import java.util.Collections;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
+import org.apache.kylin.common.exception.code.ErrorCodeServer;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.cube.model.SelectRule;
-import org.apache.kylin.metadata.model.SegmentRange;
-import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.response.AggIndexResponse;
-import org.apache.kylin.rest.util.AclEvaluate;
-import org.apache.kylin.rest.util.AclUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.IndexEntity.Range;
@@ -40,9 +37,17 @@ import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.CreateTableIndexRequest;
+import org.apache.kylin.rest.request.OpenUpdateRuleBasedCuboidRequest;
import org.apache.kylin.rest.request.UpdateRuleBasedCuboidRequest;
+import org.apache.kylin.rest.response.AggIndexResponse;
import org.apache.kylin.rest.response.FusionRuleDataResult;
+import org.apache.kylin.rest.util.AclEvaluate;
+import org.apache.kylin.rest.util.AclUtil;
import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Assert;
@@ -57,8 +62,6 @@ import
org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.test.util.ReflectionTestUtils;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import lombok.val;
import lombok.var;
@@ -860,4 +863,151 @@ public class FusionIndexServiceTest extends
SourceTestCase {
"data_size", true, Lists.newArrayList());
Assert.assertEquals(64, indexResponses2.size());
}
+
+ @Test
+ public void testNormalConvertOpenToInternal() {
+ OpenUpdateRuleBasedCuboidRequest request = new
OpenUpdateRuleBasedCuboidRequest();
+ request.setProject("default");
+ request.setModelAlias("test_bank");
+ val modelManager =
NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(),
request.getProject());
+ NDataModel model =
modelManager.getDataModelDescByAlias(request.getModelAlias());
+ val aggGroup = new
OpenUpdateRuleBasedCuboidRequest.OpenAggGroupRequest();
+ request.setAggregationGroups(Collections.singletonList(aggGroup));
+
+ {
+ aggGroup.setDimensions(new String[]{"TEST_BANK_INCOME.NAME",
"TEST_BANK_INCOME.DT"});
+ aggGroup.setMeasures(new String[]{"SUM_INCOME"});
+ String[][] jointDims = new String[][]{{"TEST_BANK_INCOME.DT"}, {}};
+ aggGroup.setJointDims(jointDims);
+ UpdateRuleBasedCuboidRequest internal =
fusionIndexService.convertOpenToInternal(request, model);
+ Assert.assertEquals(2, internal.getAggregationGroups().size());
+ }
+
+ {
+ aggGroup.setDimensions(new String[]{"TEST_BANK_INCOME.NAME",
"TEST_BANK_INCOME.DT"});
+ aggGroup.setMeasures(null);
+ String[][] hierarchyDims = { { "TEST_BANK_INCOME.NAME" } };
+ aggGroup.setHierarchyDims(hierarchyDims);
+ String[][] jointDims = new String[][]{{"TEST_BANK_INCOME.DT"}, {}};
+ aggGroup.setJointDims(jointDims);
+ UpdateRuleBasedCuboidRequest internal =
fusionIndexService.convertOpenToInternal(request, model);
+ Assert.assertEquals(2, internal.getAggregationGroups().size());
+ }
+
+ {
+ aggGroup.setDimensions(new String[]{"TEST_BANK_INCOME.NAME",
"TEST_BANK_INCOME.DT"});
+ aggGroup.setMeasures(new String[]{"SUM_INCOME"});
+ aggGroup.setHierarchyDims(null);
+ aggGroup.setJointDims(new String[][]{});
+ aggGroup.setMandatoryDims(new String[] {"TEST_BANK_INCOME.NAME"});
+ UpdateRuleBasedCuboidRequest internal =
fusionIndexService.convertOpenToInternal(request, model);
+ Assert.assertEquals(2, internal.getAggregationGroups().size());
+ }
+ }
+
+ @Test
+ public void testIllegalConvertOpenToInternal() {
+ OpenUpdateRuleBasedCuboidRequest request = new
OpenUpdateRuleBasedCuboidRequest();
+ request.setProject("default");
+ request.setModelAlias("test_bank");
+ val modelManager =
NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(),
request.getProject());
+ NDataModel model =
modelManager.getDataModelDescByAlias(request.getModelAlias());
+ val aggGroup = new
OpenUpdateRuleBasedCuboidRequest.OpenAggGroupRequest();
+ request.setAggregationGroups(Collections.singletonList(aggGroup));
+
+ request.setGlobalDimCap(-1);
+ try {
+ fusionIndexService.convertOpenToInternal(request, model);
+ Assert.fail();
+ } catch (KylinException e) {
+
Assert.assertEquals(ErrorCodeServer.INTEGER_POSITIVE_CHECK.getMsg(),
e.getMessage());
+ }
+
+ checkDimAndMeas(aggGroup, request, model);
+
+ checkSelectRuleDims(aggGroup, request, model);
+ }
+
+ private void
checkDimAndMeas(OpenUpdateRuleBasedCuboidRequest.OpenAggGroupRequest aggGroup,
+ OpenUpdateRuleBasedCuboidRequest request, NDataModel model) {
+ request.setGlobalDimCap(null);
+ aggGroup.setDimensions(null);
+ try {
+ fusionIndexService.convertOpenToInternal(request, model);
+ Assert.fail();
+ } catch (NullPointerException e) {
+ Assert.assertEquals("dimension should not null", e.getMessage());
+ }
+
+ aggGroup.setDimCap(1);
+ aggGroup.setDimensions(new String[] { "TEST_BANK_INCOME.NAME",
"TEST_BANK_INCOME.NAME" });
+ try {
+ fusionIndexService.convertOpenToInternal(request, model);
+ Assert.fail();
+ } catch (IllegalStateException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Dimension or measure in agg
group must not contain duplication"));
+ }
+
+ aggGroup.setDimensions(new String[] { "TEST_KYLIN_FACT.LSTG_SITE_ID1",
"TEST_KYLIN_FACT.LSTG_SITE_ID" });
+ try {
+ fusionIndexService.convertOpenToInternal(request, model);
+ Assert.fail();
+ } catch (KylinException e) {
+
Assert.assertEquals(ErrorCodeServer.DIMENSION_NOT_IN_MODEL.getMsg(),
e.getMessage());
+ }
+
+ aggGroup.setDimensions(new String[] { "TEST_BANK_INCOME.NAME",
"TEST_BANK_INCOME.DT" });
+ aggGroup.setMeasures(new String[] { "TRANS_CNT1" });
+ try {
+ fusionIndexService.convertOpenToInternal(request, model);
+ Assert.fail();
+ } catch (KylinException e) {
+ Assert.assertEquals(ErrorCodeServer.MEASURE_NOT_IN_MODEL.getMsg(),
e.getMessage());
+ }
+ }
+
+ private void
checkSelectRuleDims(OpenUpdateRuleBasedCuboidRequest.OpenAggGroupRequest
aggGroup,
+ OpenUpdateRuleBasedCuboidRequest request, NDataModel model) {
+ aggGroup.setMeasures(new String[] { "SUM_INCOME" });
+ aggGroup.setMandatoryDims(new String[] {
"TEST_KYLIN_FACT.LSTG_FORMAT_NAME" });
+ try {
+ fusionIndexService.convertOpenToInternal(request, model);
+ Assert.fail();
+ } catch (KylinException e) {
+
Assert.assertEquals(ErrorCodeServer.MANDATORY_NOT_IN_DIMENSION.getMsg(),
e.getMessage());
+ }
+
+ aggGroup.setMandatoryDims(new String[] {});
+ String[][] hierarchyDims = { { "TEST_KYLIN_FACT.LSTG_FORMAT_NAME" } };
+ aggGroup.setHierarchyDims(hierarchyDims);
+ try {
+ fusionIndexService.convertOpenToInternal(request, model);
+ Assert.fail();
+ } catch (KylinException e) {
+
Assert.assertEquals(ErrorCodeServer.HIERARCHY_NOT_IN_DIMENSION.getMsg(),
e.getMessage());
+ }
+
+ aggGroup.setMeasures(null);
+ aggGroup.setHierarchyDims(null);
+ String[][] jointDims = { { "TEST_KYLIN_FACT.LSTG_FORMAT_NAME" }, {} };
+ aggGroup.setJointDims(jointDims);
+ try {
+ fusionIndexService.convertOpenToInternal(request, model);
+ Assert.fail();
+ } catch (KylinException e) {
+
Assert.assertEquals(ErrorCodeServer.JOINT_NOT_IN_DIMENSION.getMsg(),
e.getMessage());
+ }
+
+ hierarchyDims = new String[][] { { "TEST_BANK_INCOME.NAME" } };
+ jointDims = new String[][] { { "TEST_BANK_INCOME.NAME" } };
+ aggGroup.setHierarchyDims(hierarchyDims);
+ aggGroup.setJointDims(jointDims);
+ try {
+ fusionIndexService.convertOpenToInternal(request, model);
+ Assert.fail();
+ } catch (KylinException e) {
+
Assert.assertEquals(ErrorCodeServer.DIMENSION_ONLY_SET_ONCE.getMsg(),
e.getMessage());
+ }
+ }
}
