This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3ef254379f6a93f1208d4610fc85f6d30bc30621
Author: Hongrong Cao 曹鸿荣 <148043261+aligad...@users.noreply.github.com>
AuthorDate: Sat Nov 11 09:44:59 2023 +0800
KYLIN-5873 Check segment overlap before returning models
---
.../kylin/rest/service/AccessServiceTest.java | 4 +-
.../kylin/rest/service/AclTCRServiceTest.java | 2 +-
.../org/apache/kylin/job/common/SegmentUtil.java | 4 +-
.../org/apache/kylin/job/common/SegmentsTest.java | 39 +++++
.../apache/kylin/metadata/model/NDataModel.java | 2 +-
.../metadata/model/SegmentStatusEnumToDisplay.java | 2 +-
.../metadata/project/NProjectManagerTest.java | 2 +-
.../kylin/metrics/HdfsCapacityMetricsTest.java | 2 +-
.../_global/project/segment_overlap_test.json | 35 ++++
.../d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json | 179 +++++++++++++++++++++
.../19d129a3-0070-d228-9cc5-acafe78179f5.json | 36 +++++
.../6c272580-f1aa-7eb9-65bf-4d8113c695fe.json | 36 +++++
.../a549d3b2-6c37-a7a1-0ac5-94bc8d67f1b0.json | 36 +++++
.../d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json | 64 ++++++++
.../d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json | 146 +++++++++++++++++
.../segment_overlap_test/table/SSB.LINEORDER.json | 114 +++++++++++++
.../kylin/rest/response/NDataModelResponse.java | 5 +
.../apache/kylin/rest/service/ModelService.java | 16 ++
.../kylin/rest/service/ModelServiceTest.java | 19 +++
.../kylin/rest/service/ProjectServiceTest.java | 6 +-
.../kylin/rest/service/AsyncQueryJobTest.java | 2 +-
.../rest/service/QueryHistoryServiceTest.java | 2 +-
22 files changed, 740 insertions(+), 13 deletions(-)
diff --git
a/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
b/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
index 253f0c15a0..22e6942a84 100644
---
a/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
+++
b/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
@@ -610,14 +610,14 @@ public class AccessServiceTest extends
NLocalFileMetadataTestCase {
@Test
public void testGetGrantedProjectsOfUser() throws IOException {
List<String> result = accessService.getGrantedProjectsOfUser("ADMIN");
- assertEquals(31, result.size());
+ assertEquals(32, result.size());
}
@Test
public void testGetGrantedProjectsOfUserOrGroup() throws IOException {
// admin user
List<String> result =
accessService.getGrantedProjectsOfUserOrGroup("ADMIN", true);
- assertEquals(31, result.size());
+ assertEquals(32, result.size());
// normal user
result = accessService.getGrantedProjectsOfUserOrGroup("ANALYST",
true);
diff --git
a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
index b915bd6879..c40f59092e 100644
---
a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
+++
b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
@@ -1317,7 +1317,7 @@ public class AclTCRServiceTest extends
NLocalFileMetadataTestCase {
Mockito.when(userService.isGlobalAdmin("ADMIN")).thenReturn(true);
List<SidPermissionWithAclResponse> responses =
accessService.getUserOrGroupAclPermissions(projects, "ADMIN",
true);
- Assert.assertEquals(31, responses.size());
+ Assert.assertEquals(32, responses.size());
Assert.assertTrue(responses.stream().allMatch(response ->
"ADMIN".equals(response.getProjectPermission())));
// test normal group
diff --git
a/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
b/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
index 8532c1e0c5..1e40973436 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
@@ -91,7 +91,9 @@ public class SegmentUtil {
}
if (CollectionUtils.isNotEmpty(overlapSegs)) {
-
Preconditions.checkState(CollectionUtils.isNotEmpty(overlapSegs.getSegments(SegmentStatusEnum.NEW)));
+
if(CollectionUtils.isEmpty(overlapSegs.getSegments(SegmentStatusEnum.NEW))){
+ return SegmentStatusEnumToDisplay.OVERLAP;
+ }
return SegmentStatusEnumToDisplay.LOCKED;
}
diff --git
a/src/core-job/src/test/java/org/apache/kylin/job/common/SegmentsTest.java
b/src/core-job/src/test/java/org/apache/kylin/job/common/SegmentsTest.java
index 7c76c58197..524ccfb72a 100644
--- a/src/core-job/src/test/java/org/apache/kylin/job/common/SegmentsTest.java
+++ b/src/core-job/src/test/java/org/apache/kylin/job/common/SegmentsTest.java
@@ -18,6 +18,8 @@
package org.apache.kylin.job.common;
+import java.util.ArrayList;
+
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.junit.TimeZoneTestRunner;
import org.apache.kylin.metadata.cube.model.NDataSegment;
@@ -225,4 +227,41 @@ public class SegmentsTest {
return seg;
}
+ @Test
+ public void testSegmentsWithRangeOverlap() {
+ val segmentList = new ArrayList();
+ Segments segments = new Segments();
+ val seg = NDataSegment.empty();
+ seg.setId(RandomUtil.randomUUIDStr());
+ seg.setSegmentRange(new SegmentRange.TimePartitionedSegmentRange(0L,
10000L));
+ seg.setStatus(SegmentStatusEnum.READY);
+ segments.add(seg);
+ segmentList.add(seg);
+
+ val seg2 = NDataSegment.empty();
+ seg2.setId(RandomUtil.randomUUIDStr());
+ seg2.setSegmentRange(new
SegmentRange.TimePartitionedSegmentRange(9999L, 20000L));
+ seg2.setStatus(SegmentStatusEnum.READY);
+ segments.add(seg2);
+ segmentList.add(seg);
+
+ val newSeg = NDataSegment.empty();
+ newSeg.setId(RandomUtil.randomUUIDStr());
+ newSeg.setSegmentRange(new
SegmentRange.TimePartitionedSegmentRange(20000L, 40000L));
+ newSeg.setStatus(SegmentStatusEnum.NEW);
+ segments.add(newSeg);
+ segmentList.add(seg);
+
+ Mockito.mockStatic(SegmentUtil.class);
+ Mockito.when(SegmentUtil.getSegmentStatusToDisplay(
+ Mockito.any(),
+ Mockito.any(NDataSegment.class),
+ Mockito.any())).thenCallRealMethod();
+
Mockito.when(SegmentUtil.anyIndexJobRunning(Mockito.anyObject())).thenReturn(false);
+ SegmentStatusEnumToDisplay status =
SegmentUtil.getSegmentStatusToDisplay(segments, seg2, null);
+ Assert.assertEquals(status, SegmentStatusEnumToDisplay.OVERLAP);
+
+
+ }
+
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index 3e981ade03..0b7dbf5b73 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -291,7 +291,7 @@ public class NDataModel extends RootPersistentEntity {
}
public enum BrokenReason {
- SCHEMA, NULL, EVENT
+ SCHEMA, NULL, EVENT, SEGMENT_OVERLAP
}
@Data
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnumToDisplay.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnumToDisplay.java
index 77fd690227..d9868ddb21 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnumToDisplay.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnumToDisplay.java
@@ -30,7 +30,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* @author xduo
*/
public enum SegmentStatusEnumToDisplay implements Serializable {
- ONLINE, LOCKED, REFRESHING, MERGING, LOADING, WARNING,
+ ONLINE, LOCKED, REFRESHING, MERGING, LOADING, WARNING, OVERLAP,
@JsonProperty(value = "ONLINE(HDFS)")
ONLINE_HDFS {
@Override
diff --git
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
index d1b7c000f6..bfcf809f14 100644
---
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
+++
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
@@ -69,7 +69,7 @@ public class NProjectManagerTest extends
NLocalFileMetadataTestCase {
}
val projects = projectManager.listAllProjects();
- Assert.assertEquals(31, projects.size());
+ Assert.assertEquals(32, projects.size());
Assert.assertTrue(projects.stream().noneMatch(p ->
p.getName().equals("test")));
}
diff --git
a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
b/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
index 24fdb0e549..1189892633 100644
---
a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
+++
b/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
@@ -90,7 +90,7 @@ public class HdfsCapacityMetricsTest extends
NLocalFileMetadataTestCase {
}
Assert.assertTrue(hdfsCapacityMetrics.getWorkingDirCapacity().isEmpty());
hdfsCapacityMetrics.writeHdfsMetrics();
- Assert.assertEquals(31,
hdfsCapacityMetrics.getWorkingDirCapacity().size());
+ Assert.assertEquals(32,
hdfsCapacityMetrics.getWorkingDirCapacity().size());
}
diff --git
a/src/examples/test_case_data/localmeta/metadata/_global/project/segment_overlap_test.json
b/src/examples/test_case_data/localmeta/metadata/_global/project/segment_overlap_test.json
new file mode 100644
index 0000000000..326b7c8963
--- /dev/null
+++
b/src/examples/test_case_data/localmeta/metadata/_global/project/segment_overlap_test.json
@@ -0,0 +1,35 @@
+{
+ "uuid" : "9d1c17a8-71bd-fbc1-7f8f-23c9351fa095",
+ "last_modified" : 1699429611441,
+ "create_time" : 1699352459287,
+ "version" : "4.0.0.0",
+ "name" : "segment_overlap_test",
+ "owner" : "ADMIN",
+ "status" : "ENABLED",
+ "create_time_utc" : 1699352459287,
+ "default_database" : "DEFAULT",
+ "description" : "",
+ "principal" : null,
+ "keytab" : null,
+ "maintain_model_type" : "MANUAL_MAINTAIN",
+ "override_kylin_properties" : {
+ "kylin.metadata.semi-automatic-mode" : "false",
+ "kylin.query.metadata.expose-computed-column" : "true",
+ "kylin.source.default" : "1"
+ },
+ "segment_config" : {
+ "auto_merge_enabled" : false,
+ "auto_merge_time_ranges" : [ "WEEK", "MONTH", "QUARTER", "YEAR" ],
+ "volatile_range" : {
+ "volatile_range_number" : 0,
+ "volatile_range_enabled" : false,
+ "volatile_range_type" : "DAY"
+ },
+ "retention_range" : {
+ "retention_range_number" : 1,
+ "retention_range_enabled" : false,
+ "retention_range_type" : "MONTH"
+ },
+ "create_empty_segment_enabled" : false
+ }
+}
\ No newline at end of file
diff --git
a/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json
new file mode 100644
index 0000000000..20dfbc691f
--- /dev/null
+++
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json
@@ -0,0 +1,179 @@
+{
+ "uuid" : "d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "last_modified" : 1699501377516,
+ "create_time" : 1699352541918,
+ "version" : "4.0.0.0",
+ "status" : "ONLINE",
+ "last_status" : "OFFLINE",
+ "cost" : 50,
+ "query_hit_count" : 0,
+ "last_query_time" : 0,
+ "last_data_refresh_time" : 0,
+ "layout_query_hit_count" : { },
+ "segments" : [ {
+ "id" : "a549d3b2-6c37-a7a1-0ac5-94bc8d67f1b0",
+ "name" : "19910101000000_19950101000000",
+ "create_time_utc" : 1699500777474,
+ "status" : "READY",
+ "segRange" : {
+ "@class" :
"org.apache.kylin.metadata.model.SegmentRange$TimePartitionedSegmentRange",
+ "date_range_start" : 662659200000,
+ "date_range_end" : 788889600000
+ },
+ "timeRange" : null,
+ "dimension_range_info_map" : {
+ "11" : {
+ "min" : "1-URGENT",
+ "max" : "5-LOW"
+ },
+ "1" : {
+ "min" : "1",
+ "max" : "20000"
+ },
+ "16" : {
+ "min" : "1",
+ "max" : "2999"
+ },
+ "7" : {
+ "min" : "1",
+ "max" : "200"
+ },
+ "10" : {
+ "min" : "19920101",
+ "max" : "19941231"
+ }
+ },
+ "parameters" : null,
+ "dictionaries" : null,
+ "snapshots" : null,
+ "last_build_time" : 1699501438508,
+ "source_count" : 273096,
+ "source_bytes_size" : 57003296,
+ "column_source_bytes" : {
+ "SSB.LINEORDER.LO_ORDERPRIOTITY" : 2103112,
+ "SSB.LINEORDER.LO_ORDERDATE" : 2184768,
+ "SSB.LINEORDER.LO_SUPPKEY" : 675093,
+ "SSB.LINEORDER.LO_PARTKEY" : 1214184,
+ "SSB.LINEORDER.LO_CUSTKEY" : 992977
+ },
+ "ori_snapshot_size" : { },
+ "additionalInfo" : { },
+ "is_realtime_segment" : false,
+ "is_snapshot_ready" : false,
+ "is_dict_ready" : false,
+ "is_flat_table_ready" : true,
+ "is_fact_view_ready" : false,
+ "multi_partitions" : [ ],
+ "max_bucket_id" : -1
+ }, {
+ "id" : "6c272580-f1aa-7eb9-65bf-4d8113c695fe",
+ "name" : "19950101000000_19980101000000",
+ "create_time_utc" : 1699500787014,
+ "status" : "READY",
+ "segRange" : {
+ "@class" :
"org.apache.kylin.metadata.model.SegmentRange$TimePartitionedSegmentRange",
+ "date_range_start" : 788889600000,
+ "date_range_end" : 883584000000
+ },
+ "timeRange" : null,
+ "dimension_range_info_map" : {
+ "11" : {
+ "min" : "1-URGENT",
+ "max" : "5-LOW"
+ },
+ "1" : {
+ "min" : "1",
+ "max" : "20000"
+ },
+ "16" : {
+ "min" : "1",
+ "max" : "2999"
+ },
+ "7" : {
+ "min" : "1",
+ "max" : "200"
+ },
+ "10" : {
+ "min" : "19950101",
+ "max" : "19971231"
+ }
+ },
+ "parameters" : null,
+ "dictionaries" : null,
+ "snapshots" : null,
+ "last_build_time" : 1699501443345,
+ "source_count" : 272955,
+ "source_bytes_size" : 57003296,
+ "column_source_bytes" : {
+ "SSB.LINEORDER.LO_ORDERPRIOTITY" : 2115674,
+ "SSB.LINEORDER.LO_ORDERDATE" : 2183640,
+ "SSB.LINEORDER.LO_SUPPKEY" : 669831,
+ "SSB.LINEORDER.LO_PARTKEY" : 1221200,
+ "SSB.LINEORDER.LO_CUSTKEY" : 980727
+ },
+ "ori_snapshot_size" : { },
+ "additionalInfo" : { },
+ "is_realtime_segment" : false,
+ "is_snapshot_ready" : false,
+ "is_dict_ready" : false,
+ "is_flat_table_ready" : true,
+ "is_fact_view_ready" : false,
+ "multi_partitions" : [ ],
+ "max_bucket_id" : -1
+ }, {
+ "id" : "19d129a3-0070-d228-9cc5-acafe78179f5",
+ "name" : "19980101000000_19990101000000",
+ "create_time_utc" : 1699500797946,
+ "status" : "READY",
+ "segRange" : {
+ "@class" :
"org.apache.kylin.metadata.model.SegmentRange$TimePartitionedSegmentRange",
+ "date_range_start" : 883583000000,
+ "date_range_end" : 915120000000
+ },
+ "timeRange" : null,
+ "dimension_range_info_map" : {
+ "11" : {
+ "min" : "1-URGENT",
+ "max" : "5-LOW"
+ },
+ "1" : {
+ "min" : "1",
+ "max" : "20000"
+ },
+ "16" : {
+ "min" : "1",
+ "max" : "2999"
+ },
+ "7" : {
+ "min" : "1",
+ "max" : "200"
+ },
+ "10" : {
+ "min" : "19980101",
+ "max" : "19980802"
+ }
+ },
+ "parameters" : null,
+ "dictionaries" : null,
+ "snapshots" : null,
+ "last_build_time" : 1699501443515,
+ "source_count" : 54521,
+ "source_bytes_size" : 57003296,
+ "column_source_bytes" : {
+ "SSB.LINEORDER.LO_ORDERPRIOTITY" : 416704,
+ "SSB.LINEORDER.LO_ORDERDATE" : 436168,
+ "SSB.LINEORDER.LO_SUPPKEY" : 133794,
+ "SSB.LINEORDER.LO_PARTKEY" : 243381,
+ "SSB.LINEORDER.LO_CUSTKEY" : 198183
+ },
+ "ori_snapshot_size" : { },
+ "additionalInfo" : { },
+ "is_realtime_segment" : false,
+ "is_snapshot_ready" : false,
+ "is_dict_ready" : false,
+ "is_flat_table_ready" : true,
+ "is_fact_view_ready" : false,
+ "multi_partitions" : [ ],
+ "max_bucket_id" : -1
+ } ]
+}
\ No newline at end of file
diff --git
a/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow_details/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb/19d129a3-0070-d228-9cc5-acafe78179f5.json
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow_details/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb/19d129a3-0070-d228-9cc5-acafe78179f5.json
new file mode 100644
index 0000000000..7cf2e12f02
--- /dev/null
+++
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow_details/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb/19d129a3-0070-d228-9cc5-acafe78179f5.json
@@ -0,0 +1,36 @@
+{
+ "uuid" : "19d129a3-0070-d228-9cc5-acafe78179f5",
+ "last_modified" : 1699500797947,
+ "create_time" : 1699500797947,
+ "version" : "4.0.0.0",
+ "dataflow" : "d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "layout_instances" : [ {
+ "layout_id" : 1,
+ "build_job_id" :
"b9333c90-19ad-9ce9-4e5d-904c5d385b58-d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "rows" : 54521,
+ "byte_size" : 385248,
+ "file_count" : 1,
+ "source_rows" : 54521,
+ "source_byte_size" : 0,
+ "partition_num" : 1,
+ "partition_values" : [ ],
+ "is_ready" : false,
+ "create_time" : 1699501327234,
+ "multi_partition" : [ ],
+ "abnormal_type" : null
+ }, {
+ "layout_id" : 20000000001,
+ "build_job_id" :
"b9333c90-19ad-9ce9-4e5d-904c5d385b58-d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "rows" : 54521,
+ "byte_size" : 384827,
+ "file_count" : 1,
+ "source_rows" : 54521,
+ "source_byte_size" : 0,
+ "partition_num" : 1,
+ "partition_values" : [ ],
+ "is_ready" : false,
+ "create_time" : 1699501327235,
+ "multi_partition" : [ ],
+ "abnormal_type" : null
+ } ]
+}
\ No newline at end of file
diff --git
a/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow_details/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb/6c272580-f1aa-7eb9-65bf-4d8113c695fe.json
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow_details/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb/6c272580-f1aa-7eb9-65bf-4d8113c695fe.json
new file mode 100644
index 0000000000..7f8707bc00
--- /dev/null
+++
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow_details/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb/6c272580-f1aa-7eb9-65bf-4d8113c695fe.json
@@ -0,0 +1,36 @@
+{
+ "uuid" : "6c272580-f1aa-7eb9-65bf-4d8113c695fe",
+ "last_modified" : 1699500787016,
+ "create_time" : 1699500787015,
+ "version" : "4.0.0.0",
+ "dataflow" : "d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "layout_instances" : [ {
+ "layout_id" : 1,
+ "build_job_id" :
"f8e681fa-517e-8093-54a0-f4786fd996ee-d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "rows" : 272955,
+ "byte_size" : 1298468,
+ "file_count" : 1,
+ "source_rows" : 272955,
+ "source_byte_size" : 0,
+ "partition_num" : 1,
+ "partition_values" : [ ],
+ "is_ready" : false,
+ "create_time" : 1699501326166,
+ "multi_partition" : [ ],
+ "abnormal_type" : null
+ }, {
+ "layout_id" : 20000000001,
+ "build_job_id" :
"f8e681fa-517e-8093-54a0-f4786fd996ee-d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "rows" : 272955,
+ "byte_size" : 1297233,
+ "file_count" : 1,
+ "source_rows" : 272955,
+ "source_byte_size" : 0,
+ "partition_num" : 1,
+ "partition_values" : [ ],
+ "is_ready" : false,
+ "create_time" : 1699501326166,
+ "multi_partition" : [ ],
+ "abnormal_type" : null
+ } ]
+}
\ No newline at end of file
diff --git
a/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow_details/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb/a549d3b2-6c37-a7a1-0ac5-94bc8d67f1b0.json
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow_details/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb/a549d3b2-6c37-a7a1-0ac5-94bc8d67f1b0.json
new file mode 100644
index 0000000000..29fdcfce17
--- /dev/null
+++
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/dataflow_details/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb/a549d3b2-6c37-a7a1-0ac5-94bc8d67f1b0.json
@@ -0,0 +1,36 @@
+{
+ "uuid" : "a549d3b2-6c37-a7a1-0ac5-94bc8d67f1b0",
+ "last_modified" : 1699500777475,
+ "create_time" : 1699500777475,
+ "version" : "4.0.0.0",
+ "dataflow" : "d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "layout_instances" : [ {
+ "layout_id" : 1,
+ "build_job_id" :
"502da487-01ea-291e-193e-c9a48751a003-d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "rows" : 273096,
+ "byte_size" : 1298794,
+ "file_count" : 1,
+ "source_rows" : 273096,
+ "source_byte_size" : 0,
+ "partition_num" : 1,
+ "partition_values" : [ ],
+ "is_ready" : false,
+ "create_time" : 1699501319236,
+ "multi_partition" : [ ],
+ "abnormal_type" : null
+ }, {
+ "layout_id" : 20000000001,
+ "build_job_id" :
"502da487-01ea-291e-193e-c9a48751a003-d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "rows" : 273096,
+ "byte_size" : 1297559,
+ "file_count" : 1,
+ "source_rows" : 273096,
+ "source_byte_size" : 0,
+ "partition_num" : 1,
+ "partition_values" : [ ],
+ "is_ready" : false,
+ "create_time" : 1699501319236,
+ "multi_partition" : [ ],
+ "abnormal_type" : null
+ } ]
+}
\ No newline at end of file
diff --git
a/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/index_plan/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/index_plan/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json
new file mode 100644
index 0000000000..2e65c38e50
--- /dev/null
+++
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/index_plan/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json
@@ -0,0 +1,64 @@
+{
+ "uuid" : "d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "last_modified" : 1699501377518,
+ "create_time" : 1699352541842,
+ "version" : "4.0.0.0",
+ "description" : null,
+ "rule_based_index" : null,
+ "indexes" : [ {
+ "id" : 0,
+ "dimensions" : [ 1, 7, 10, 11, 16 ],
+ "measures" : [ 100000 ],
+ "layouts" : [ {
+ "id" : 1,
+ "name" : null,
+ "owner" : null,
+ "col_order" : [ 1, 7, 10, 11, 16, 100000 ],
+ "shard_by_columns" : [ ],
+ "partition_by_columns" : [ ],
+ "sort_by_columns" : [ ],
+ "storage_type" : 20,
+ "update_time" : 1699352541853,
+ "manual" : false,
+ "auto" : false,
+ "base" : true,
+ "draft_version" : null,
+ "index_range" : null
+ } ],
+ "next_layout_offset" : 2
+ }, {
+ "id" : 20000000000,
+ "dimensions" : [ 1, 7, 10, 11, 16 ],
+ "measures" : [ ],
+ "layouts" : [ {
+ "id" : 20000000001,
+ "name" : null,
+ "owner" : null,
+ "col_order" : [ 1, 7, 10, 11, 16 ],
+ "shard_by_columns" : [ ],
+ "partition_by_columns" : [ ],
+ "sort_by_columns" : [ ],
+ "storage_type" : 20,
+ "update_time" : 1699352541854,
+ "manual" : false,
+ "auto" : false,
+ "base" : true,
+ "draft_version" : null,
+ "index_range" : null
+ } ],
+ "next_layout_offset" : 2
+ } ],
+ "override_properties" : { },
+ "to_be_deleted_indexes" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 80,
+ "next_aggregation_index_id" : 10000,
+ "next_table_index_id" : 20000010000,
+ "agg_shard_by_columns" : [ ],
+ "extend_partition_columns" : [ ],
+ "layout_bucket_num" : { },
+ "approved_additional_recs" : 0,
+ "approved_removal_recs" : 0,
+ "base_agg_index_reduce_high_cardinality_dim" : false
+}
\ No newline at end of file
diff --git
a/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/model_desc/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/model_desc/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json
new file mode 100644
index 0000000000..bd35ca05b3
--- /dev/null
+++
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/model_desc/d0bbfa51-9c16-b6e5-1d33-76b47d8853eb.json
@@ -0,0 +1,146 @@
+{
+ "uuid" : "d0bbfa51-9c16-b6e5-1d33-76b47d8853eb",
+ "last_modified" : 1699500778271,
+ "create_time" : 1699352540889,
+ "version" : "4.0.0.0",
+ "alias" : "time_range_overlap",
+ "owner" : "ADMIN",
+ "config_last_modifier" : null,
+ "config_last_modified" : 0,
+ "description" : "",
+ "fact_table" : "SSB.LINEORDER",
+ "fact_table_alias" : null,
+ "management_type" : "MODEL_BASED",
+ "join_tables" : [ ],
+ "filter_condition" : "",
+ "partition_desc" : {
+ "partition_date_column" : "LINEORDER.LO_ORDERDATE",
+ "partition_date_start" : 0,
+ "partition_date_format" : "yyyyMMdd",
+ "partition_type" : "APPEND",
+ "partition_condition_builder" :
"org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
+ },
+ "capacity" : "MEDIUM",
+ "segment_config" : {
+ "auto_merge_enabled" : null,
+ "auto_merge_time_ranges" : null,
+ "volatile_range" : null,
+ "retention_range" : null,
+ "create_empty_segment_enabled" : false
+ },
+ "data_check_desc" : {
+ "check_options" : 0,
+ "fault_threshold" : 0,
+ "fault_actions" : 0
+ },
+ "semantic_version" : 0,
+ "storage_type" : 0,
+ "model_type" : "BATCH",
+ "all_named_columns" : [ {
+ "id" : 0,
+ "name" : "LO_ORDERKEY",
+ "column" : "LINEORDER.LO_ORDERKEY"
+ }, {
+ "id" : 1,
+ "name" : "LO_PARTKEY",
+ "column" : "LINEORDER.LO_PARTKEY",
+ "status" : "DIMENSION"
+ }, {
+ "id" : 2,
+ "name" : "LO_DISCOUNT",
+ "column" : "LINEORDER.LO_DISCOUNT"
+ }, {
+ "id" : 3,
+ "name" : "LO_SUPPLYCOST",
+ "column" : "LINEORDER.LO_SUPPLYCOST"
+ }, {
+ "id" : 4,
+ "name" : "LO_COMMITDATE",
+ "column" : "LINEORDER.LO_COMMITDATE"
+ }, {
+ "id" : 5,
+ "name" : "LO_EXTENDEDPRICE",
+ "column" : "LINEORDER.LO_EXTENDEDPRICE"
+ }, {
+ "id" : 6,
+ "name" : "LO_TAX",
+ "column" : "LINEORDER.LO_TAX"
+ }, {
+ "id" : 7,
+ "name" : "LO_SUPPKEY",
+ "column" : "LINEORDER.LO_SUPPKEY",
+ "status" : "DIMENSION"
+ }, {
+ "id" : 8,
+ "name" : "LO_ORDTOTALPRICE",
+ "column" : "LINEORDER.LO_ORDTOTALPRICE"
+ }, {
+ "id" : 9,
+ "name" : "LO_REVENUE",
+ "column" : "LINEORDER.LO_REVENUE"
+ }, {
+ "id" : 10,
+ "name" : "LO_ORDERDATE",
+ "column" : "LINEORDER.LO_ORDERDATE",
+ "status" : "DIMENSION"
+ }, {
+ "id" : 11,
+ "name" : "LO_ORDERPRIOTITY",
+ "column" : "LINEORDER.LO_ORDERPRIOTITY",
+ "status" : "DIMENSION"
+ }, {
+ "id" : 12,
+ "name" : "LO_SHIPPRIOTITY",
+ "column" : "LINEORDER.LO_SHIPPRIOTITY"
+ }, {
+ "id" : 13,
+ "name" : "LO_QUANTITY",
+ "column" : "LINEORDER.LO_QUANTITY"
+ }, {
+ "id" : 14,
+ "name" : "LO_SHIPMODE",
+ "column" : "LINEORDER.LO_SHIPMODE"
+ }, {
+ "id" : 15,
+ "name" : "LO_LINENUMBER",
+ "column" : "LINEORDER.LO_LINENUMBER"
+ }, {
+ "id" : 16,
+ "name" : "LO_CUSTKEY",
+ "column" : "LINEORDER.LO_CUSTKEY",
+ "status" : "DIMENSION"
+ } ],
+ "all_measures" : [ {
+ "name" : "COUNT_ALL",
+ "function" : {
+ "expression" : "COUNT",
+ "parameters" : [ {
+ "type" : "constant",
+ "value" : "1"
+ } ],
+ "returntype" : "bigint"
+ },
+ "column" : null,
+ "comment" : null,
+ "id" : 100000,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ } ],
+ "recommendations_count" : 0,
+ "computed_columns" : [ ],
+ "canvas" : {
+ "coordinate" : {
+ "LINEORDER" : {
+ "x" : 547.3333062065973,
+ "y" : 231.72220865885419,
+ "width" : 200.0,
+ "height" : 230.0
+ }
+ },
+ "zoom" : 9.0
+ },
+ "broken_reason" : "NULL",
+ "multi_partition_desc" : null,
+ "multi_partition_key_mapping" : null,
+ "fusion_id" : null
+}
\ No newline at end of file
diff --git
a/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/table/SSB.LINEORDER.json
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/table/SSB.LINEORDER.json
new file mode 100644
index 0000000000..32156cf230
--- /dev/null
+++
b/src/examples/test_case_data/localmeta/metadata/segment_overlap_test/table/SSB.LINEORDER.json
@@ -0,0 +1,114 @@
+{
+ "uuid" : "35a35cdf-40a8-00b2-7abc-afdd4ca8f5fc",
+ "last_modified" : 0,
+ "create_time" : 1699352480176,
+ "version" : "4.0.0.0",
+ "name" : "LINEORDER",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "LO_ORDERKEY",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "lo_orderkey"
+ }, {
+ "id" : "2",
+ "name" : "LO_LINENUMBER",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "lo_linenumber"
+ }, {
+ "id" : "3",
+ "name" : "LO_CUSTKEY",
+ "datatype" : "integer",
+ "case_sensitive_name" : "lo_custkey"
+ }, {
+ "id" : "4",
+ "name" : "LO_PARTKEY",
+ "datatype" : "integer",
+ "case_sensitive_name" : "lo_partkey"
+ }, {
+ "id" : "5",
+ "name" : "LO_SUPPKEY",
+ "datatype" : "integer",
+ "case_sensitive_name" : "lo_suppkey"
+ }, {
+ "id" : "6",
+ "name" : "LO_ORDERDATE",
+ "datatype" : "integer",
+ "case_sensitive_name" : "lo_orderdate"
+ }, {
+ "id" : "7",
+ "name" : "LO_ORDERPRIOTITY",
+ "datatype" : "varchar(4096)",
+ "case_sensitive_name" : "lo_orderpriotity"
+ }, {
+ "id" : "8",
+ "name" : "LO_SHIPPRIOTITY",
+ "datatype" : "integer",
+ "case_sensitive_name" : "lo_shippriotity"
+ }, {
+ "id" : "9",
+ "name" : "LO_QUANTITY",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "lo_quantity"
+ }, {
+ "id" : "10",
+ "name" : "LO_EXTENDEDPRICE",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "lo_extendedprice"
+ }, {
+ "id" : "11",
+ "name" : "LO_ORDTOTALPRICE",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "lo_ordtotalprice"
+ }, {
+ "id" : "12",
+ "name" : "LO_DISCOUNT",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "lo_discount"
+ }, {
+ "id" : "13",
+ "name" : "LO_REVENUE",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "lo_revenue"
+ }, {
+ "id" : "14",
+ "name" : "LO_SUPPLYCOST",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "lo_supplycost"
+ }, {
+ "id" : "15",
+ "name" : "LO_TAX",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "lo_tax"
+ }, {
+ "id" : "16",
+ "name" : "LO_COMMITDATE",
+ "datatype" : "integer",
+ "case_sensitive_name" : "lo_commitdate"
+ }, {
+ "id" : "17",
+ "name" : "LO_SHIPMODE",
+ "datatype" : "varchar(4096)",
+ "case_sensitive_name" : "lo_shipmode"
+ } ],
+ "source_type" : 9,
+ "table_type" : "MANAGED",
+ "top" : false,
+ "increment_loading" : false,
+ "last_snapshot_path" : null,
+ "last_snapshot_size" : 0,
+ "snapshot_last_modified" : 0,
+ "query_hit_count" : 0,
+ "partition_column" : null,
+ "snapshot_partitions" : { },
+ "snapshot_partitions_info" : { },
+ "snapshot_total_rows" : 0,
+ "snapshot_partition_col" : null,
+ "selected_snapshot_partition_col" : null,
+ "temp_snapshot_path" : null,
+ "snapshot_has_broken" : false,
+ "database" : "SSB",
+ "transactional" : false,
+ "rangePartition" : false,
+ "partition_desc" : null,
+ "table_comment" : null
+}
\ No newline at end of file
diff --git
a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java
b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java
index 66e8b488c8..8475640a8b 100644
---
a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java
+++
b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java
@@ -302,6 +302,11 @@ public class NDataModelResponse extends NDataModel {
return this.isBroken();
}
+ @JsonProperty("has_segment_overlap")
+ public boolean isHasSegmentOverlap() {
+ return NDataModel.BrokenReason.SEGMENT_OVERLAP ==
this.getBrokenReason();
+ }
+
@JsonProperty("simplified_tables")
public List<SimplifiedTableResponse> getSimpleTables() {
List<SimplifiedTableResponse> simpleTables = new ArrayList<>();
diff --git
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 6e31ef05e1..44e22cef26 100644
---
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -467,6 +467,7 @@ public class ModelService extends AbstractModelService
((NDataModelResponse) model).setSegments(segments);
((NDataModelResponse) model).setHasSegments(
((NDataModelResponse) model).isHasSegments() ||
CollectionUtils.isNotEmpty(segments));
+ checkSegmentOverlap((NDataModelResponse) model, segments);
}
if (model.isFusionModel()) {
@@ -679,6 +680,7 @@ public class ModelService extends AbstractModelService
modelResponse.setSegments(segments);
modelResponse.setHasSegments(modelResponse.isHasSegments() ||
CollectionUtils.isNotEmpty(segments));
+ checkSegmentOverlap(modelResponse, segments);
}
oldParams.setName(modelResponse.getAlias());
@@ -4597,6 +4599,20 @@ public class ModelService extends AbstractModelService
}
}
+ public void checkSegmentOverlap(NDataModelResponse model,
List<NDataSegmentResponse> segments) {
+ if (ModelStatusToDisplayEnum.BROKEN != model.getStatus()) {
+ boolean hasAnyOverlapSegment = segments.stream()
+ .anyMatch(seg -> SegmentStatusEnumToDisplay.OVERLAP ==
seg.getStatusToDisplay());
+ if (hasAnyOverlapSegment) {
+ logger.warn("model {} segments overlap found, mark as
broken!", model.getId());
+ model.setBroken(true);
+ model.setBrokenReason(NDataModel.BrokenReason.SEGMENT_OVERLAP);
+ model.setStatus(ModelStatusToDisplayEnum.BROKEN);
+ }
+ }
+ }
+
+
public Pair<ModelRequest, ComputedColumnConflictResponse>
checkCCConflict(ModelRequest modelRequest) {
String project = modelRequest.getProject();
validatePartitionDateColumn(modelRequest);
diff --git
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index d7f635697c..4de6c8c16a 100644
---
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -5713,4 +5713,23 @@ public class ModelServiceTest extends SourceTestCase {
ModelRequest request = JsonUtil.readValue(modelRequest,
ModelRequest.class);
Assert.assertThrows(KylinException.class, () ->
modelService.checkBeforeModelSave(request));
}
+
+
+ @Test
+ public void testCheckModelWithSegmentOverlap() {
+ val project = "segment_overlap_test";
+ val modelId = "d0bbfa51-9c16-b6e5-1d33-76b47d8853eb";
+ val modelName = "time_range_overlap";
+ val models = modelService.getModels(modelName, project,
+ false, null, Lists.newArrayList(), null, false, null, null,
null, true);
+ NDataModelResponse model = models.get(0);
+ Assert.assertSame(NDataModel.BrokenReason.NULL,
model.getBrokenReason());
+
+ modelService.addOldParams(project, (List)models);
+ Assert.assertSame(NDataModel.BrokenReason.SEGMENT_OVERLAP,
model.getBrokenReason());
+
+ model = modelService.getCubes0(modelName, project).get(0);
+ Assert.assertSame(NDataModel.BrokenReason.SEGMENT_OVERLAP,
model.getBrokenReason());
+
+ }
}
diff --git
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
index 9f8265dc79..89e0d5895e 100644
---
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
+++
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
@@ -239,14 +239,14 @@ public class ProjectServiceTest extends
NLocalFileMetadataTestCase {
public void testGetReadableProjects() {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances =
projectService.getReadableProjects("", false);
- Assert.assertEquals(31, projectInstances.size());
+ Assert.assertEquals(32, projectInstances.size());
}
@Test
public void testGetAdminProjects() throws Exception {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances =
projectService.getAdminProjects();
- Assert.assertEquals(31, projectInstances.size());
+ Assert.assertEquals(32, projectInstances.size());
}
@Test
@@ -260,7 +260,7 @@ public class ProjectServiceTest extends
NLocalFileMetadataTestCase {
public void testGetReadableProjectsHasNoPermissionProject() {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances =
projectService.getReadableProjects("", false);
- Assert.assertEquals(31, projectInstances.size());
+ Assert.assertEquals(32, projectInstances.size());
}
diff --git
a/src/query-service/src/test/java/org/apache/kylin/rest/service/AsyncQueryJobTest.java
b/src/query-service/src/test/java/org/apache/kylin/rest/service/AsyncQueryJobTest.java
index 64b5f8ce32..7fbe890122 100644
---
a/src/query-service/src/test/java/org/apache/kylin/rest/service/AsyncQueryJobTest.java
+++
b/src/query-service/src/test/java/org/apache/kylin/rest/service/AsyncQueryJobTest.java
@@ -247,7 +247,7 @@ public class AsyncQueryJobTest extends
NLocalFileMetadataTestCase {
rawResourceMap.put(zipEntry.getName(), raw);
}
}
- Assert.assertEquals(87, rawResourceMap.size());
+ Assert.assertEquals(88, rawResourceMap.size());
}
private void testKylinConfig(FileSystem workingFileSystem, FileStatus
metaFileStatus) throws IOException {
diff --git
a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
index 426506435b..62b427b3ab 100644
---
a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
+++
b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
@@ -368,7 +368,7 @@ public class QueryHistoryServiceTest extends
NLocalFileMetadataTestCase {
// get all tables
tableMap = queryHistoryService.getQueryHistoryTableMap(null);
- Assert.assertEquals(31, tableMap.size());
+ Assert.assertEquals(32, tableMap.size());
// not existing project
tableMap =
queryHistoryService.getQueryHistoryTableMap(Lists.newArrayList("not_existing_project"));
