This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1612a76dfafaaab68354b6050beb28ae9cff5c6b
Author: lixiang <447399...@qq.com>
AuthorDate: Fri Mar 17 20:29:41 2023 +0800
KYLIN-5572 Add a new REST API to build the specified indexes
---
.../common/exception/code/ErrorCodeServer.java | 3 +
.../resources/kylin_error_msg_conf_cn.properties | 4 +-
.../resources/kylin_error_msg_conf_en.properties | 3 +
.../kylin_error_suggestion_conf_cn.properties | 3 +
.../kylin_error_suggestion_conf_en.properties | 3 +
.../main/resources/kylin_errorcode_conf.properties | 3 +
.../kylin/rest/controller/SegmentController.java | 5 +
.../controller/open/OpenSegmentController.java | 59 +++++++++++-
.../rest/controller/SegmentControllerTest.java | 38 +++++++-
.../controller/open/OpenSegmentControllerTest.java | 102 ++++++++++++++++++++-
10 files changed, 216 insertions(+), 7 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
index c2d6c51c0c..5e0df273eb 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
@@ -60,6 +60,9 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
SEGMENT_STATUS("KE-010022217"),
SEGMENT_MERGE_CHECK_INDEX_ILLEGAL("KE-010022218"),
SEGMENT_MERGE_CHECK_PARTITION_ILLEGAL("KE-010022219"),
+ SEGMENT_INDEX_CONFLICT_PARAMETER("KE-010022220"),
+ SEGMENT_INDEX_STATUS_INVALID("KE-010022221"),
+ SEGMENT_SINGLE_JOB_THRESHOLD("KE-010022222"),
// 100072XX table
TABLE_RELOAD_MODEL_RETRY("KE-010007204"),
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 3b4eb00a04..0edf55b29c 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -60,7 +60,9 @@ KE-010022216=Segment “%s” 被锁定，无法删除、刷新或合并。请
KE-010022217=Segment “%s” 处于 “%s” 状态，无法刷新或合并。请稍后重试。
KE-010022218=当前 Segments 所包含的索引不一致，请先构建索引并确保其一致后再合并。
KE-010022219=当前 Segments 所包含的分区不一致，请先构建分区并确保其一致后再合并。
-
+KE-010022220=index_ids或index_status不能同时设置，请修改后重试。
+KE-010022221=索引状态错误，请输入正确的索引状态（NO_BUILD, ONLINE, BUILDING）后重试。
+KE-010022222=单个任务中最多包含100个Segments，请修改后重试。
## 100072XX table
KE-010007204=源表 %1$s 中列 %2$s 的数据类型发生变更。请从模型 %3$s 中删除该列，或修改该列的数据类型。
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index e5dd9f4fb6..132c78cdb3 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -60,6 +60,9 @@ KE-010022216=Can't remove, refresh or merge segment "%s", as
it's LOCKED. Please
KE-010022217=Can't refresh or merge segment "%s", as it's in "%s". Please try
again later.
KE-010022218=The indexes included in the selected segments are not consistent.
Please build index first and try merging again.
KE-010022219=The partitions included in the selected segments are not
consistent. Please build the partitions first and try merging again.
+KE-010022220=Index_ids or index_status can not be set at the same time, please
modify and try again.
+KE-010022221=The index status is wrong, please enter the correct index
status(NO_BUILD, ONLINE, BUILDING) and try again.
+KE-010022222=A single job can contain up to 100 segments, please modify and
try again.
## 100072XX table
KE-010007204=The data type of column %2$s from the source table %1$s has
changed. Please remove the column from model %3$s, or modify the data type.
diff --git
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
index b9b1865c7b..42dc91cf84 100644
---
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
+++
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
@@ -58,6 +58,9 @@ KE-010022216=
KE-010022217=
KE-010022218=
KE-010022219=
+KE-010022220=
+KE-010022221=
+KE-010022222=
## 100072XX table
KE-010007204=
diff --git
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
index 53dc857806..105f521360 100644
---
a/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
+++
b/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
@@ -58,6 +58,9 @@ KE-010022216=
KE-010022217=
KE-010022218=
KE-010022219=
+KE-010022220=
+KE-010022221=
+KE-010022222=
## 100072XX table
KE-010007204=
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index 3b891fbf05..ae53e187e4 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -59,6 +59,9 @@ KE-010022214
KE-010022215
KE-010022216
KE-010022217
+KE-010022220
+KE-010022221
+KE-010022222
## 100072XX table
KE-010007204
diff --git
a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
index cd2701948a..594dd90ef4 100644
---
a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
+++
b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
@@ -22,6 +22,7 @@ import static
org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_EMPTY_ID;
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_MERGE_LESS_THAN_TWO;
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_REFRESH_SELECT_EMPTY;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_SINGLE_JOB_THRESHOLD;
import java.util.ArrayList;
import java.util.Arrays;
@@ -323,6 +324,10 @@ public class SegmentController extends BaseController {
ProjectInstance prjInstance =
NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
.getProject(buildSegmentsRequest.getProject());
checkParamLength("tag", buildSegmentsRequest.getTag(),
prjInstance.getConfig().getJobTagMaxSize());
+ if (!buildSegmentsRequest.isParallelBuildBySegment() &&
buildSegmentsRequest.getSegmentIds() != null
+ && buildSegmentsRequest.getSegmentIds().size() > 100) {
+ throw new KylinException(SEGMENT_SINGLE_JOB_THRESHOLD);
+ }
val response = fusionModelService.addIndexesToSegments(modelId,
buildSegmentsRequest);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, response,
"");
}
diff --git
a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSegmentController.java
b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSegmentController.java
index 63eaf1a3da..c528f8b39f 100644
---
a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSegmentController.java
+++
b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSegmentController.java
@@ -22,15 +22,24 @@ import static
org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import static
org.apache.kylin.common.exception.ServerErrorCode.INVALID_PARAMETER;
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_NOT_EXIST;
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.PROJECT_MULTI_PARTITION_DISABLE;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_INDEX_CONFLICT_PARAMETER;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_INDEX_STATUS_INVALID;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
+import java.util.stream.Collectors;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.job.execution.JobTypeEnum;
+import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.project.NProjectManager;
@@ -48,12 +57,16 @@ import org.apache.kylin.rest.response.BuildIndexResponse;
import org.apache.kylin.rest.response.CheckSegmentResponse;
import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.response.IndexResponse;
import org.apache.kylin.rest.response.JobInfoResponse;
import org.apache.kylin.rest.response.JobInfoResponseWithFailure;
import org.apache.kylin.rest.response.NDataSegmentResponse;
import org.apache.kylin.rest.response.SegmentPartitionResponse;
+import org.apache.kylin.rest.service.FusionIndexService;
import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.ModelService;
+import org.apache.kylin.rest.service.params.IndexPlanParams;
+import org.apache.kylin.rest.service.params.PaginationParams;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.util.DataRangeUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -90,6 +103,9 @@ public class OpenSegmentController extends BaseController {
@Autowired
private AclEvaluate aclEvaluate;
+ @Autowired
+ private FusionIndexService fusionIndexService;
+
@VisibleForTesting
public NDataModel getModel(String modelAlias, String project) {
NDataModel model = modelService.getManager(NDataModelManager.class,
project).listAllModels().stream() //
@@ -203,9 +219,10 @@ public class OpenSegmentController extends BaseController {
@RequestParam(value = "partial_build", required = false,
defaultValue = "false") boolean partialBuild,
@RequestParam(value = "priority", required = false, defaultValue =
"3") Integer priority,
@RequestParam(value = "yarn_queue", required = false) String
yarnQueue,
- @RequestParam(value = "tag", required = false) Object tag) {
+ @RequestParam(value = "tag", required = false) Object tag,
+ @RequestParam(value = "index_status", required = false)
List<String> indexStatusStr) {
String projectName = checkProjectName(project);
- checkSegmentParams(ids, names);
+ checkParams(ids, names, batchIndexIds, indexStatusStr);
String modelId = getModel(modelAlias, projectName).getUuid();
Pair<String, String[]> pair =
fusionModelService.convertSegmentIdWithName(modelId, projectName, ids, names);
IndexesToSegmentsRequest req = new IndexesToSegmentsRequest();
@@ -213,6 +230,21 @@ public class OpenSegmentController extends BaseController {
req.setParallelBuildBySegment(parallel);
req.setSegmentIds(Lists.newArrayList(pair.getSecond()));
req.setPartialBuild(partialBuild);
+ if (CollectionUtils.isNotEmpty(indexStatusStr)) {
+ List<IndexEntity.Status> indexStatus =
formatStatus(indexStatusStr);
+ IndexPlanParams indexPlanParams = new IndexPlanParams(project,
modelId, null, null, Collections.emptyList(),
+ indexStatus, null);
+ PaginationParams paginationParams = new PaginationParams(null,
null, null, false);
+ val indexes = fusionIndexService.getIndexes(indexPlanParams,
paginationParams, null);
+ if (indexes.isEmpty()) {
+ JobInfoResponseWithFailure result = new
JobInfoResponseWithFailure();
+ List<JobInfoResponse.JobInfo> jobs = new LinkedList<>();
+ jobs.add(new
JobInfoResponse.JobInfo(JobTypeEnum.INDEX_BUILD.toString(), null));
+ result.setJobs(jobs);
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
result, "");
+ }
+ batchIndexIds =
indexes.stream().map(IndexResponse::getId).collect(Collectors.toList());
+ }
req.setIndexIds(batchIndexIds);
req.setPriority(priority);
req.setYarnQueue(yarnQueue);
@@ -220,6 +252,29 @@ public class OpenSegmentController extends BaseController {
return segmentController.addIndexesToSegments(pair.getFirst(), req);
}
+ private void checkParams(String[] ids, String[] names, List<Long>
batchIndexIds, List<String> indexStatus) {
+ checkSegmentParams(ids, names);
+ if (CollectionUtils.isNotEmpty(batchIndexIds) &&
CollectionUtils.isNotEmpty(indexStatus)) {
+ throw new KylinException(SEGMENT_INDEX_CONFLICT_PARAMETER);
+ }
+ }
+
+ private List<IndexEntity.Status> formatStatus(List<String> indexStatus) {
+ List<IndexEntity.Status> result = new ArrayList<>();
+ indexStatus.forEach(str -> {
+ try {
+ IndexEntity.Status status =
IndexEntity.Status.valueOf(str.toUpperCase(Locale.ROOT));
+ if (status == IndexEntity.Status.LOCKED) {
+ throw new KylinException(SEGMENT_INDEX_STATUS_INVALID);
+ }
+ result.add(status);
+ } catch (Exception e) {
+ throw new KylinException(SEGMENT_INDEX_STATUS_INVALID);
+ }
+ });
+ return result;
+ }
+
@ApiOperation(value = "buildIndicesManually", tags = { "AI" })
@PostMapping(value = "/{model_name:.+}/indexes")
@ResponseBody
diff --git
a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SegmentControllerTest.java
b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SegmentControllerTest.java
index fbf01832d8..c36bf4b87b 100644
---
a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SegmentControllerTest.java
+++
b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SegmentControllerTest.java
@@ -24,17 +24,20 @@ import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.job.execution.JobTypeEnum;
import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.rest.request.BuildIndexRequest;
import org.apache.kylin.rest.request.BuildSegmentsRequest;
import org.apache.kylin.rest.request.IncrementBuildSegmentsRequest;
+import org.apache.kylin.rest.request.IndexesToSegmentsRequest;
import org.apache.kylin.rest.request.PartitionsBuildRequest;
import org.apache.kylin.rest.request.PartitionsRefreshRequest;
import org.apache.kylin.rest.request.SegmentFixRequest;
@@ -425,4 +428,37 @@ public class SegmentControllerTest extends
NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
}
+
+ @Test
+ public void testAddIndexToSegment() throws Exception {
+ IndexesToSegmentsRequest request = new IndexesToSegmentsRequest();
+ request.setSegmentIds(IntStream.rangeClosed(1, 101).mapToObj(t -> t +
"").collect(Collectors.toList()));
+ request.setProject("default");
+ mockMvc.perform(MockMvcRequestBuilders
+ .post("/api/models/{model}/model_segments/indexes",
"89af4ee2-2cdb-4b07-b39e-4c29856309aa")
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+
.andExpect(MockMvcResultMatchers.status().isInternalServerError());
+
+ request.setParallelBuildBySegment(true);
+ mockMvc.perform(MockMvcRequestBuilders
+ .post("/api/models/{model}/model_segments/indexes",
"89af4ee2-2cdb-4b07-b39e-4c29856309aa")
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+
+ request.setSegmentIds(IntStream.rangeClosed(1, 10).mapToObj(t -> t +
"").collect(Collectors.toList()));
+ mockMvc.perform(MockMvcRequestBuilders
+ .post("/api/models/{model}/model_segments/indexes",
"89af4ee2-2cdb-4b07-b39e-4c29856309aa")
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+
+ request.setParallelBuildBySegment(false);
+ mockMvc.perform(MockMvcRequestBuilders
+ .post("/api/models/{model}/model_segments/indexes",
"89af4ee2-2cdb-4b07-b39e-4c29856309aa")
+
.contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ }
}
diff --git
a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSegmentControllerTest.java
b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSegmentControllerTest.java
index 32bfdca1d4..8a1de250ff 100644
---
a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSegmentControllerTest.java
+++
b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSegmentControllerTest.java
@@ -22,6 +22,8 @@ import static
org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_NOT_EXIST;
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.PROJECT_MULTI_PARTITION_DISABLE;
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_CONFLICT_PARAMETER;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_INDEX_CONFLICT_PARAMETER;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_INDEX_STATUS_INVALID;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
@@ -33,8 +35,10 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
@@ -44,6 +48,7 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.Segments;
@@ -64,8 +69,11 @@ import org.apache.kylin.rest.response.IndexResponse;
import org.apache.kylin.rest.response.NDataModelResponse;
import org.apache.kylin.rest.response.NDataSegmentResponse;
import org.apache.kylin.rest.response.SegmentPartitionResponse;
+import org.apache.kylin.rest.service.FusionIndexService;
import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.ModelService;
+import org.apache.kylin.rest.service.params.IndexPlanParams;
+import org.apache.kylin.rest.service.params.PaginationParams;
import org.apache.kylin.rest.util.AclEvaluate;
import org.junit.After;
import org.junit.Assert;
@@ -92,6 +100,9 @@ import
org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import shaded.parquet.com.fasterxml.jackson.core.JsonProcessingException;
+import shaded.parquet.com.fasterxml.jackson.databind.ObjectMapper;
+
@RunWith(MockitoJUnitRunner.class)
public class OpenSegmentControllerTest extends NLocalFileMetadataTestCase {
@@ -109,11 +120,16 @@ public class OpenSegmentControllerTest extends
NLocalFileMetadataTestCase {
@Mock
private AclEvaluate aclEvaluate;
+ @Mock
+ private FusionIndexService fusionIndexService;
+
@InjectMocks
private final OpenSegmentController openSegmentController =
Mockito.spy(new OpenSegmentController());
private final Authentication authentication = new
TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN);
+ private ObjectMapper objectMapper = new ObjectMapper();
+
@Before
public void setup() {
MockitoAnnotations.initMocks(this);
@@ -285,7 +301,7 @@ public class OpenSegmentControllerTest extends
NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
Mockito.verify(openSegmentController).completeSegments(modelName,
project, false, ids, null, null, false, 3,
- null, null);
+ null, null, null);
mockMvc.perform(MockMvcRequestBuilders.post("/api/models/{model_name}/segments/completion",
modelName)
.param("project", "default") //
.param("parallel", "false") //
@@ -294,7 +310,7 @@ public class OpenSegmentControllerTest extends
NLocalFileMetadataTestCase {
.param("priority",
"0").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
Mockito.verify(openSegmentController).completeSegments(modelName,
project, false, ids, null, null, false, 0,
- null, null);
+ null, null, null);
}
@Test
@@ -323,7 +339,7 @@ public class OpenSegmentControllerTest extends
NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
Mockito.verify(openSegmentController).completeSegments(modelName,
project, false, ids, null, batchIndexIds,
- true, 3, null, null);
+ true, 3, null, null, null);
}
@Test
@@ -372,6 +388,86 @@ public class OpenSegmentControllerTest extends
NLocalFileMetadataTestCase {
Assert.assertTrue(contentAsString.contains(SEGMENT_CONFLICT_PARAMETER.getMsg()));
}
+ @Test
+ public void testCompleteSegmentsByIndexStatus() throws Exception {
+ String modelName = "default_model_name";
+ String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
+ String project = "default";
+ String[] ids = { "ef5e0663-feba-4ed2-b71c-21958122bbff" };
+ Pair<String, String[]> pair = new Pair<>(modelId, ids);
+ IndexesToSegmentsRequest req = new IndexesToSegmentsRequest();
+ req.setProject(project);
+ req.setParallelBuildBySegment(false);
+ req.setSegmentIds(Lists.newArrayList(ids));
+ mockGetModelName(modelName, project, modelId);
+ lenient().doReturn(new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
"", "")).when(nModelController)
+ .addIndexesToSegments(modelId, req);
+
Mockito.doReturn(pair).when(fusionModelService).convertSegmentIdWithName(modelId,
project, ids, null);
+ MvcResult result = mockMvc
+
.perform(MockMvcRequestBuilders.post("/api/models/{model_name}/segments/completion",
modelName)
+ .param("project", "default") //
+ .param("parallel", "false") //
+ .param("ids", ids) //
+ .param("names", (String) null) //
+ .param("batch_index_ids", "1,2") //
+ .param("index_status", "NO_BUILD")
+
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+
.andExpect(MockMvcResultMatchers.status().isInternalServerError()).andReturn();
+ String contentAsString = result.getResponse().getContentAsString();
+
Assert.assertTrue(contentAsString.contains(SEGMENT_INDEX_CONFLICT_PARAMETER.getMsg()));
+
+ MvcResult result1 = mockMvc.perform(MockMvcRequestBuilders
+ .post("/api/models/{model_name}/segments/completion",
modelName).param("project", "default") //
+ .param("parallel", "false") //
+ .param("ids", ids) //
+ .param("names", (String) null) //
+ .param("index_status",
"LOCKED").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+
.andExpect(MockMvcResultMatchers.status().isInternalServerError()).andReturn();
+ String content1 = result1.getResponse().getContentAsString();
+
Assert.assertTrue(content1.contains(SEGMENT_INDEX_STATUS_INVALID.getMsg()));
+
+ IndexPlanParams indexPlanParams = new IndexPlanParams(project,
modelId, null, null, Collections.emptyList(),
+ Collections.singletonList(IndexEntity.Status.NO_BUILD), null);
+ PaginationParams paginationParams = new PaginationParams(null, null,
null, false);
+ List<IndexResponse> emptyIndex = new ArrayList<>();
+
Mockito.doReturn(emptyIndex).when(fusionIndexService).getIndexes(indexPlanParams,
paginationParams, null);
+ MvcResult result2 = mockMvc
+
.perform(MockMvcRequestBuilders.post("/api/models/{model_name}/segments/completion",
modelName)
+ .param("project", "default") //
+ .param("parallel", "false") //
+ .param("ids", ids) //
+ .param("names", (String) null) //
+ .param("index_status", "NO_BUILD")
+
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
+ String content2 = result2.getResponse().getContentAsString();
+ Assert.assertNull(getJobId(content2));
+
+ List<IndexResponse> indexResponseList = new ArrayList<>();
+ IndexResponse indexResponse = new IndexResponse();
+ indexResponse.setId(1L);
+ indexResponseList.add(indexResponse);
+
Mockito.doReturn(indexResponseList).when(fusionIndexService).getIndexes(indexPlanParams,
paginationParams,
+ null);
+
mockMvc.perform(MockMvcRequestBuilders.post("/api/models/{model_name}/segments/completion",
modelName)
+ .param("project", "default") //
+ .param("parallel", "false") //
+ .param("ids", ids) //
+ .param("names", (String) null) //
+ .param("index_status", "NO_BUILD")
+
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ }
+
+ @SuppressWarnings("unchecked")
+ private String getJobId(String content) throws JsonProcessingException {
+ LinkedHashMap<String, Object> envelopeResponse =
objectMapper.readValue(content, LinkedHashMap.class);
+ Map<String, Object> data = (Map<String, Object>)
envelopeResponse.get("data");
+ List<?> list = (List<?>) data.get("jobs");
+ Map<String, String> map = (Map<String, String>) list.get(0);
+ return map.get("job_id");
+ }
+
@Test
public void testDeleteSegmentsAll() throws Exception {
String modelName = "default_model_name";
