This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 40a10818a045b1ec5e909df12f7365f7679eae9a
Author: lixiang <447399...@qq.com>
AuthorDate: Fri Aug 11 11:48:41 2023 +0800
Add some code to compatible with the enterprise version
---------
Co-authored-by: Zhiting Guo <zhiting....@kyligence.io>
---
.../common/exception/code/ErrorCodeServer.java | 3 +
.../resources/kylin_error_msg_conf_cn.properties | 3 +
.../resources/kylin_error_msg_conf_en.properties | 3 +
.../kylin_error_suggestion_conf_cn.properties | 3 +
.../kylin_error_suggestion_conf_en.properties | 3 +
.../main/resources/kylin_errorcode_conf.properties | 3 +
.../apache/kylin/metadata/model/NDataModel.java | 9 ++
.../recommendation/entity/DimensionRecItemV2.java | 17 +++-
.../kylin/rest/controller/SegmentController.java | 6 +-
.../rest/controller/SegmentControllerTest.java | 43 ++++++++-
.../kylin/rest/controller/NModelController.java | 11 ++-
.../rest/controller/NModelControllerTest.java | 107 ++++++++++++++++++++-
.../org/apache/kylin/rest/util/ModelUtils.java | 36 ++++++-
13 files changed, 234 insertions(+), 13 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
index 26f558be38..f2562e3a9f 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
@@ -37,6 +37,8 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
MODEL_SUM_LC_INVALID_DATA_TYPE("KE-010002303"),
MODEL_SUM_LC_INVALID_TIMESTAMP_TYPE("KE-010002304"),
MODEL_NAME_TOO_LONG("KE-010002305"),
+ MODEL_SECOND_STORAGE_PARTITION_INVALID("KE-010002306"),
+ PARTITION_SECOND_STORAGE_PARTITION_INVALID("KE-010002307"),
// 100252XX Cube
CUBE_NOT_EXIST("KE-010025201"),
@@ -64,6 +66,7 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
SEGMENT_INDEX_CONFLICT_PARAMETER("KE-010022220"),
SEGMENT_INDEX_STATUS_INVALID("KE-010022221"),
SEGMENT_SINGLE_JOB_THRESHOLD("KE-010022222"),
+ SEGMENT_SECOND_STORAGE_PARTITION_INVALID("KE-010022223"),
// 100072XX table
TABLE_RELOAD_MODEL_RETRY("KE-010007204"),
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index ffb3f55714..010c2b10c3 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -37,6 +37,8 @@ KE-010002302=模型中的列名 %s 与度量名 %s 重复，无法导出 TDS。
KE-010002303=SUM_LC度量的返回类型 '%s' 不合法。返回类型必须是这其中的一个：%s。
KE-010002304=SUM_LC度量的时间类型 '%s' 不合法。
KE-010002305=模型名称最长 127 字符，请修改后重试。
+KE-010002306=无法保存模型。当增量加载的模型开启分层存储时，必须将时间分区列加入维度。
+KE-010002307=无法保存分区设置。当增量加载的模型开启分层存储时，必须将时间分区列加入维度。
## 100252XX Cube
KE-010025201=无法找到相关 Cube。
@@ -64,6 +66,7 @@ KE-010022219=当前 Segments 所包含的分区不一致，请先构建分区并
KE-010022220=index_ids或index_status不能同时设置，请修改后重试。
KE-010022221=索引状态错误，请输入正确的索引状态（NO_BUILD, ONLINE, BUILDING）后重试。
KE-010022222=单个任务中最多包含100个Segments，请修改后重试。
+KE-010022223=无法构建索引。当增量加载的模型开启分层存储时，必须将时间分区列加入维度。
## 100072XX table
KE-010007204=源表 %1$s 中列 %2$s 的数据类型发生变更。请从模型 %3$s 中删除该列，或修改该列的数据类型。
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index b2bcbac2bd..75f05b384b 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -37,6 +37,8 @@ KE-010002302=There are duplicated names among model column %s
and measure name %
KE-010002303=SUM_LC Measure's return type '%s' is illegal. It must be one of
%s.
KE-010002304=SUM_LC Measure's time column type '%s' is illegal.
KE-010002305=The maximum length of the model name is 127 characters, please
modify and try again.
+KE-010002306=Can't save the model. When the model uses incremental load method
and the tiered storage is ON, the time partition column must be added as a
dimension.
+KE-010002307=Can't save the model partition. When the model uses incremental
load method and the tiered storage is ON, the time partition column must be
added as a dimension.
## 100252XX Cube
KE-010025201=Can't find the cube.
@@ -64,6 +66,7 @@ KE-010022219=The partitions included in the selected segments
are not consistent
KE-010022220=Index_ids or index_status can not be set at the same time, please
modify and try again.
KE-010022221=The index status is wrong, please enter the correct index
status(NO_BUILD, ONLINE, BUILDING) and try again.
KE-010022222=A single job can contain up to 100 segments, please modify and
try again.
+KE-010022223=Can't build index. When the model uses incremental load method
and the tiered storage is ON, the time partition column must be added as a
dimension.
## 100072XX table
KE-010007204=The data type of column %2$s from the source table %1$s has
changed. Please remove the column from model %3$s, or modify the data type.
diff --git
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
index 946726bdbd..46fafcfd18 100644
---
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
+++
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
@@ -35,6 +35,8 @@ KE-010002208=
KE-010002303=
KE-010002304=
KE-010002305=
+KE-010002306=
+KE-010002307=
## 100252XX Cube
KE-010025201=
@@ -62,6 +64,7 @@ KE-010022219=
KE-010022220=
KE-010022221=
KE-010022222=
+KE-010022223=
## 100072XX table
KE-010007204=
diff --git
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
index b34c268e50..1547a0431e 100644
---
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
+++
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
@@ -35,6 +35,8 @@ KE-010002208=
KE-010002303=
KE-010002304=
KE-010002305=
+KE-010002306=
+KE-010002307=
## 100252XX Cube
KE-010025201=
@@ -62,6 +64,7 @@ KE-010022219=
KE-010022220=
KE-010022221=
KE-010022222=
+KE-010022223=
## 100072XX table
KE-010007204=
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index b274ff6399..f8ed420188 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -38,6 +38,8 @@ KE-010002302
KE-010002303
KE-010002304
KE-010002305
+KE-010002306
+KE-010002307
## 100252XX Cube
KE-010025201
@@ -63,6 +65,7 @@ KE-010022217
KE-010022220
KE-010022221
KE-010022222
+KE-010022223
## 100072XX table
KE-010007204
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index 8ecc7fc5a3..9366cbf387 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -1296,6 +1296,15 @@ public class NDataModel extends RootPersistentEntity {
return Collections.unmodifiableSet(ccColumnNames);
}
+ public Map<String, ComputedColumnDesc> getCcMap() {
+ Map<String, ComputedColumnDesc> ccMap = Maps.newHashMap();
+ getComputedColumnDescs().forEach(cc -> {
+ String aliasDotName = cc.getTableAlias() + "." +
cc.getColumnName();
+ ccMap.putIfAbsent(aliasDotName, cc);
+ });
+ return ccMap;
+ }
+
public int getMaxColumnId() {
return this.getAllNamedColumns().stream() //
.mapToInt(NDataModel.NamedColumn::getId) //
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/entity/DimensionRecItemV2.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/entity/DimensionRecItemV2.java
index e7cf391360..9701bd0dbd 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/entity/DimensionRecItemV2.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/entity/DimensionRecItemV2.java
@@ -19,19 +19,23 @@
package org.apache.kylin.metadata.recommendation.entity;
import java.io.Serializable;
+import java.util.Locale;
import java.util.Map;
+import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.recommendation.candidate.RawRecItem;
import org.apache.kylin.metadata.recommendation.util.RawRecUtil;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import lombok.Getter;
+import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@@ -39,12 +43,21 @@ import lombok.extern.slf4j.Slf4j;
@Getter
@Setter
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE,
getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility =
JsonAutoDetect.Visibility.NONE, setterVisibility =
JsonAutoDetect.Visibility.NONE)
+@NoArgsConstructor
public class DimensionRecItemV2 extends RecItemV2 implements Serializable {
@JsonProperty("column")
private NDataModel.NamedColumn column;
@JsonProperty("data_type")
private String dataType;
+ public DimensionRecItemV2(NDataModel.NamedColumn column, TblColRef
tblColRef, String uniqueContent) {
+ setColumn(column);
+ setDataType(tblColRef.getDatatype());
+ setCreateTime(System.currentTimeMillis());
+ setUniqueContent(uniqueContent);
+ setUuid(String.format(Locale.ROOT, "dimension_%s",
RandomUtil.randomUUIDStr()));
+ }
+
public int[] genDependIds(Map<String, RawRecItem> uniqueRecItemMap, String
content, NDataModel dataModel) {
if (uniqueRecItemMap.containsKey(content)) {
return new int[] { -1 * uniqueRecItemMap.get(content).getId() };
diff --git
a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
index 594dd90ef4..0b8ee47301 100644
---
a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
+++
b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
@@ -33,6 +33,7 @@ import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.request.BuildIndexRequest;
@@ -58,6 +59,7 @@ import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.params.IncrementBuildSegmentParams;
import org.apache.kylin.rest.service.params.MergeSegmentParams;
import org.apache.kylin.rest.service.params.RefreshSegmentParams;
+import org.apache.kylin.rest.util.ModelUtils;
import org.apache.kylin.util.DataRangeUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
@@ -72,8 +74,6 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
import io.swagger.annotations.ApiOperation;
import lombok.val;
import lombok.extern.log4j.Log4j;
@@ -300,6 +300,8 @@ public class SegmentController extends BaseController {
String partitionColumnFormat =
buildSegmentsRequest.getPartitionDesc().getPartitionDateFormat();
DataRangeUtils.validateDataRange(buildSegmentsRequest.getStart(),
buildSegmentsRequest.getEnd(), partitionColumnFormat);
modelService.validateCCType(modelId,
buildSegmentsRequest.getProject());
+
ModelUtils.checkSecondStoragePartition(buildSegmentsRequest.getProject(),
modelId,
+ buildSegmentsRequest.getPartitionDesc(),
ModelUtils.MessageType.SEGMENT);
IncrementBuildSegmentParams incrParams = new
IncrementBuildSegmentParams(buildSegmentsRequest.getProject(),
modelId, buildSegmentsRequest.getStart(),
buildSegmentsRequest.getEnd(),
diff --git
a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SegmentControllerTest.java
b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SegmentControllerTest.java
index c36bf4b87b..b312cdc286 100644
---
a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SegmentControllerTest.java
+++
b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SegmentControllerTest.java
@@ -27,9 +27,11 @@ import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import org.apache.kylin.common.exception.code.ErrorCodeServer;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.job.execution.JobTypeEnum;
import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.Segments;
@@ -66,12 +68,12 @@ import
org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
+import io.kyligence.kap.secondstorage.SecondStorageUtil;
import lombok.val;
public class SegmentControllerTest extends NLocalFileMetadataTestCase {
@@ -268,6 +270,43 @@ public class SegmentControllerTest extends
NLocalFileMetadataTestCase {
Mockito.any(IncrementBuildSegmentsRequest.class));
}
+ private PartitionDesc makePartition(String column) {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ partitionDesc.setPartitionDateColumn(column);
+ return partitionDesc;
+ }
+
+ @Test
+ public void testBuildSegments_withSecondStoragePartitionCheck() throws
Exception {
+ IncrementBuildSegmentsRequest request = new
IncrementBuildSegmentsRequest();
+ String project = "default";
+ String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
+ request.setProject(project);
+ request.setStart("100");
+ request.setEnd("200");
+ Mockito.mockStatic(SecondStorageUtil.class);
+ Mockito.when(SecondStorageUtil.isModelEnable(project,
modelId)).thenReturn(true);
+ request.setPartitionDesc(makePartition("TEST_KYLIN_FACT.ORDER_ID"));
+ IncrementBuildSegmentParams incrParams = new
IncrementBuildSegmentParams(project, modelId, request.getStart(),
+ request.getEnd(), request.getPartitionDesc(), null,
request.getSegmentHoles(), true, null);
+ Mockito.doAnswer(x ->
null).when(fusionModelService).incrementBuildSegmentsManually(incrParams);
+
mockMvc.perform(MockMvcRequestBuilders.put("/api/models/{model}/model_segments",
modelId)
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+
Mockito.verify(segmentController).incrementBuildSegmentsManually(eq(modelId),
+ Mockito.any(IncrementBuildSegmentsRequest.class));
+
+ request.setPartitionDesc(makePartition("TEST_KYLIN_FACT.PRICE"));
+ MvcResult mvcResult = mockMvc
+
.perform(MockMvcRequestBuilders.put("/api/models/{model}/model_segments",
modelId)
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(request))
+
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+
.andExpect(MockMvcResultMatchers.status().is5xxServerError()).andReturn();
+
Assert.assertEquals(ErrorCodeServer.SEGMENT_SECOND_STORAGE_PARTITION_INVALID.getMsg(),
+ mvcResult.getResolvedException().getMessage());
+ }
+
@Test
public void testBuildSegments_DataRangeEndLessThanStart() throws Exception
{
BuildSegmentsRequest request = new BuildSegmentsRequest();
diff --git
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
index ac99e38d29..ccfdad7cc0 100644
---
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
+++
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
@@ -39,6 +39,8 @@ import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.constant.Constant;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.exception.LookupTableException;
@@ -78,6 +80,7 @@ import org.apache.kylin.rest.service.IndexPlanService;
import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.ModelTdsService;
import org.apache.kylin.rest.service.params.ModelQueryParams;
+import org.apache.kylin.rest.util.ModelUtils;
import org.apache.kylin.tool.bisync.SyncContext;
import org.apache.kylin.tool.bisync.model.SyncModel;
import org.apache.kylin.util.DataRangeUtils;
@@ -95,8 +98,6 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
import io.swagger.annotations.ApiOperation;
import lombok.val;
@@ -446,6 +447,8 @@ public class NModelController extends NBasicController {
DataRangeUtils.validateDataRange(request.getStart(), request.getEnd(),
partitionColumnFormat);
modelService.validatePartitionDesc(request.getPartitionDesc());
checkRequiredArg(MODEL_ID, request.getUuid());
+ ModelUtils.checkSecondStoragePartition(request.getProject(),
request.getUuid(), request.getPartitionDesc(),
+ ModelUtils.MessageType.MODEL);
try {
BuildBaseIndexResponse response = BuildBaseIndexResponse.EMPTY;
if (request.getBrokenReason() == NDataModel.BrokenReason.SCHEMA) {
@@ -472,6 +475,8 @@ public class NModelController extends NBasicController {
checkProjectName(request.getProject());
modelService.validatePartitionDesc(request.getPartitionDesc());
checkRequiredArg(MODEL_ID, modelId);
+ ModelUtils.checkSecondStoragePartition(request.getProject(), modelId,
request.getPartitionDesc(),
+ ModelUtils.MessageType.PARTITION);
try {
modelService.updatePartitionColumn(request.getProject(), modelId,
request.getPartitionDesc(),
request.getMultiPartitionDesc());
@@ -627,6 +632,8 @@ public class NModelController extends NBasicController {
@ResponseBody
public EnvelopeResponse<ModelSaveCheckResponse>
checkBeforeModelSave(@RequestBody ModelRequest modelRequest) {
checkProjectName(modelRequest.getProject());
+ ModelUtils.checkSecondStoragePartition(modelRequest.getProject(),
modelRequest.getUuid(),
+ modelRequest.getPartitionDesc(), ModelUtils.MessageType.MODEL);
ModelSaveCheckResponse response =
modelService.checkBeforeModelSave(modelRequest);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, response,
"");
}
diff --git
a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
index 79ba76abd7..7ba4232b46 100644
---
a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
+++
b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
@@ -31,9 +31,13 @@ import java.util.List;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.exception.code.ErrorCodeServer;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
@@ -52,6 +56,7 @@ import org.apache.kylin.rest.request.ModelUpdateRequest;
import org.apache.kylin.rest.request.ModelValidationRequest;
import org.apache.kylin.rest.request.MultiPartitionMappingRequest;
import org.apache.kylin.rest.request.OwnerChangeRequest;
+import org.apache.kylin.rest.request.PartitionColumnRequest;
import org.apache.kylin.rest.request.UnlinkModelRequest;
import org.apache.kylin.rest.request.UpdateMultiPartitionValueRequest;
import org.apache.kylin.rest.response.IndicesResponse;
@@ -66,14 +71,20 @@ import org.apache.kylin.rest.service.ModelTdsService;
import org.apache.kylin.tool.bisync.SyncContext;
import org.apache.kylin.tool.bisync.model.SyncModel;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.http.MediaType;
import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.Authentication;
@@ -84,12 +95,12 @@ import
org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
+import io.kyligence.kap.secondstorage.SecondStorageUtil;
import lombok.val;
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({SecondStorageUtil.class})
+@PowerMockIgnore({"javax.net.ssl.*", "javax.management.*",
"org.apache.hadoop.*", "javax.security.*", "javax.crypto.*", "javax.script.*"})
public class NModelControllerTest extends NLocalFileMetadataTestCase {
private MockMvc mockMvc;
@@ -571,6 +582,94 @@ public class NModelControllerTest extends
NLocalFileMetadataTestCase {
Mockito.verify(nModelController).updateSemantic(Mockito.any(ModelRequest.class));
}
+ @Test
+ public void testUpdateModelSemantics_WithSecondStoragePartitionCheck()
throws Exception {
+ ModelRequest request =
makeModelRequest(makePartition("TEST_KYLIN_FACT.ORDER_ID"));
+ PowerMockito.mockStatic(SecondStorageUtil.class);
+
PowerMockito.when(SecondStorageUtil.isModelEnable(request.getProject(),
request.getUuid())).thenReturn(true);
+
+
mockMvc.perform(MockMvcRequestBuilders.put("/api/models/semantic").contentType(MediaType.APPLICATION_JSON)
+ .content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+
+ ModelRequest request1 =
makeModelRequest(makePartition("TEST_KYLIN_FACT.PRICE"));
+ MvcResult mvcResult = mockMvc
+
.perform(MockMvcRequestBuilders.put("/api/models/semantic").contentType(MediaType.APPLICATION_JSON)
+ .content(JsonUtil.writeValueAsString(request1))
+
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+
.andExpect(MockMvcResultMatchers.status().is5xxServerError()).andReturn();
+
Assert.assertEquals(ErrorCodeServer.MODEL_SECOND_STORAGE_PARTITION_INVALID.getMsg(),
+ mvcResult.getResolvedException().getMessage());
+
+ Mockito.verify(nModelController,
Mockito.times(2)).updateSemantic(Mockito.any(ModelRequest.class));
+ }
+
+ private ModelRequest makeModelRequest(PartitionDesc partitionDesc) {
+ ModelRequest request = new ModelRequest();
+ String project = "default";
+ String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
+ request.setProject(project);
+ request.setUuid(modelId);
+ request.setPartitionDesc(partitionDesc);
+ return request;
+ }
+
+ private PartitionDesc makePartition(String column) {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ partitionDesc.setPartitionDateColumn(column);
+ return partitionDesc;
+ }
+
+ @Test
+ public void testUpdatePartition_withSecondStoragePartitionCheck() throws
Exception {
+ String project = "default";
+ String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
+ PowerMockito.mockStatic(SecondStorageUtil.class);
+ PowerMockito.when(SecondStorageUtil.isModelEnable(project,
modelId)).thenReturn(true);
+ PartitionColumnRequest request = new PartitionColumnRequest();
+ request.setProject(project);
+
+ request.setPartitionDesc(makePartition("TEST_KYLIN_FACT.ORDER_ID"));
+
mockMvc.perform(MockMvcRequestBuilders.put("/api/models/{model}/partition",
modelId)
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(nModelController).updatePartitionSemantic(modelId,
request);
+
+ request.setPartitionDesc(makePartition("TEST_KYLIN_FACT.PRICE"));
+ MvcResult mvcResult = mockMvc
+
.perform(MockMvcRequestBuilders.put("/api/models/{model}/partition", modelId)
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(request))
+
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+
.andExpect(MockMvcResultMatchers.status().is5xxServerError()).andReturn();
+
Assert.assertEquals(ErrorCodeServer.PARTITION_SECOND_STORAGE_PARTITION_INVALID.getMsg(),
+ mvcResult.getResolvedException().getMessage());
+ Mockito.verify(nModelController).updatePartitionSemantic(modelId,
request);
+ }
+
+ @Test
+ public void testCheckBeforeModelSave_withSecondStoragePartitionCheck()
throws Exception {
+ ModelRequest request =
makeModelRequest(makePartition("TEST_KYLIN_FACT.ORDER_ID"));
+ PowerMockito.mockStatic(SecondStorageUtil.class);
+
PowerMockito.when(SecondStorageUtil.isModelEnable(request.getProject(),
request.getUuid())).thenReturn(true);
+
+
mockMvc.perform(MockMvcRequestBuilders.post("/api/models/model_save/check")
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+
+ ModelRequest request1 =
makeModelRequest(makePartition("TEST_KYLIN_FACT.PRICE"));
+ MvcResult mvcResult = mockMvc
+
.perform(MockMvcRequestBuilders.post("/api/models/model_save/check")
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(request1))
+
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+
.andExpect(MockMvcResultMatchers.status().is5xxServerError()).andReturn();
+
Assert.assertEquals(ErrorCodeServer.MODEL_SECOND_STORAGE_PARTITION_INVALID.getMsg(),
+ mvcResult.getResolvedException().getMessage());
+ Mockito.verify(nModelController,
Mockito.times(2)).checkBeforeModelSave(Mockito.any(ModelRequest.class));
+ }
+
@Test
public void testUnlinkModel() throws Exception {
UnlinkModelRequest request = new UnlinkModelRequest();
diff --git
a/src/modeling-service/src/main/java/org/apache/kylin/rest/util/ModelUtils.java
b/src/modeling-service/src/main/java/org/apache/kylin/rest/util/ModelUtils.java
index 488c053931..76ff9d4276 100644
---
a/src/modeling-service/src/main/java/org/apache/kylin/rest/util/ModelUtils.java
+++
b/src/modeling-service/src/main/java/org/apache/kylin/rest/util/ModelUtils.java
@@ -18,6 +18,9 @@
package org.apache.kylin.rest.util;
import static
org.apache.kylin.common.exception.ServerErrorCode.INVALID_PARTITION_COLUMN;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_SECOND_STORAGE_PARTITION_INVALID;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.PARTITION_SECOND_STORAGE_PARTITION_INVALID;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_SECOND_STORAGE_PARTITION_INVALID;
import java.math.BigDecimal;
import java.util.Collections;
@@ -30,9 +33,9 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.response.NDataModelResponse;
@@ -128,4 +131,35 @@ public class ModelUtils {
}
}
+ public static void checkSecondStoragePartition(String project, String
modelId, PartitionDesc partitionDesc,
+ MessageType type) {
+ if (partitionDesc == null || partitionDesc.getPartitionDateColumn() ==
null) {
+ return;
+ }
+
+ if (!SecondStorageUtil.isModelEnable(project, modelId)) {
+ return;
+ }
+
+ val model =
NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(),
project).getDataModelDesc(modelId);
+ String partitionColumn = partitionDesc.getPartitionDateColumn();
+ if (model.getAllNamedColumns().stream()
+ .noneMatch(col -> col.isDimension() &&
col.getAliasDotColumn().equalsIgnoreCase(partitionColumn))) {
+ switch (type) {
+ case MODEL:
+ throw new
KylinException(MODEL_SECOND_STORAGE_PARTITION_INVALID);
+ case SEGMENT:
+ throw new
KylinException(SEGMENT_SECOND_STORAGE_PARTITION_INVALID);
+ case PARTITION:
+ throw new
KylinException(PARTITION_SECOND_STORAGE_PARTITION_INVALID);
+ default:
+ throw new IllegalStateException("this should not happen");
+ }
+ }
+ }
+
+ public enum MessageType {
+ SEGMENT, PARTITION, MODEL
+ }
+
}
