This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 098862e0b9d9eb3bd89a5d7d12f927a6ae2c5efe
Author: Hongrong Cao 曹鸿荣 <148043261+aligad...@users.noreply.github.com>
AuthorDate: Wed Nov 1 10:36:17 2023 +0800
KYLIN-5863 Fix query bug caused by concurrent model deletions
---
.../kylin/metadata/project/NProjectManager.java | 3 +-
.../apache/kylin/rest/service/TableService.java | 10 +++-
.../apache/kylin/rest/DeleteDuringQueryTest.java | 65 ++++++++++++++++++++++
3 files changed, 76 insertions(+), 2 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
index 2319e5bccd..b718236278 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
@@ -22,6 +22,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -206,7 +207,7 @@ public class NProjectManager {
}
public List<NDataModel> listHealthyModels(String project) {
- return
listAllRealizations(project).stream().map(IRealization::getModel).collect(Collectors.toList());
+ return
listAllRealizations(project).stream().map(IRealization::getModel).filter(Objects::nonNull).collect(Collectors.toList());
}
public List<MeasureDesc> listEffectiveRewriteMeasures(String project,
String factTable) {
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
index b0b4534f95..14a4826fe0 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -611,6 +611,10 @@ public class TableService extends BasicService {
long size = 0;
for (val model : models) {
val df = dfManger.getDataflow(model.getUuid());
+ if(df==null){
+ logger.warn("cannot get model size {} of {}（might be deleted）
", model.getAlias(), project);
+ continue;
+ }
val readySegs = df.getSegments(SegmentStatusEnum.READY,
SegmentStatusEnum.WARNING);
if (CollectionUtils.isNotEmpty(readySegs)) {
for (NDataSegment segment : readySegs) {
@@ -628,7 +632,11 @@ public class TableService extends BasicService {
Set<String> primaryKey = new HashSet<>();
Set<String> foreignKey = new HashSet<>();
for (val model : modelsUsingTable) {
- val joinTables =
dataModelManager.getDataModelDesc(model.getUuid()).getJoinTables();
+ var dataMode = dataModelManager.getDataModelDesc(model.getUuid());
+ if(dataMode==null){
+ continue;
+ }
+ val joinTables = dataMode.getJoinTables();
for (JoinTableDesc joinTable : joinTables) {
if (joinTable.getTable().equals(table.getIdentity())) {
foreignKey.addAll(Arrays.asList(joinTable.getJoin().getForeignKey()));
diff --git
a/src/server/src/test/java/org/apache/kylin/rest/DeleteDuringQueryTest.java
b/src/server/src/test/java/org/apache/kylin/rest/DeleteDuringQueryTest.java
new file mode 100644
index 0000000000..02a266b284
--- /dev/null
+++ b/src/server/src/test/java/org/apache/kylin/rest/DeleteDuringQueryTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.rest;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.rest.service.TableService;
+import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mockito;
+import org.springframework.test.util.ReflectionTestUtils;
+
+@MetadataInfo
+class DeleteDuringQueryTest {
+ @InjectMocks
+ private final TableService tableService = Mockito.spy(new TableService());
+
+ @Test
+ void testModelSize() {
+ List<NDataModel> models = new ArrayList<>();
+ long res = ReflectionTestUtils.invokeMethod(tableService,
"getStorageSize", "default", models, null);
+ Assert.assertEquals(0L, res);
+ NDataModel model = new NDataModel();
+ model.setUuid("testUUid");
+ models.add(model);
+ long res2 = ReflectionTestUtils.invokeMethod(tableService,
"getStorageSize", "default", models, null);
+ Assert.assertEquals(0L, res2);
+ }
+
+
+ @Test
+ void tesTableColumnType() {
+ List<NDataModel> models = new ArrayList<>();
+ TableDesc table = new TableDesc();
+ Pair<Set<String>, Set<String>> res =
ReflectionTestUtils.invokeMethod(tableService, "getTableColumnType", "default",
table, models);
+ Assert.assertEquals(Boolean.TRUE, res.getFirst().size()==0);
+ NDataModel model = new NDataModel();
+ model.setUuid("testUUid");
+ models.add(model);
+ Pair<Set<String>, Set<String>> res2 =
ReflectionTestUtils.invokeMethod(tableService, "getTableColumnType", "default",
table, models);
+ Assert.assertEquals(Boolean.TRUE, res2.getFirst().size()==0);
+ }
+}
