This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b2e847a0e3ba72b556546552a0c8523c120b8a3a
Author: Guoliang Sun <guoliang....@kyligence.io>
AuthorDate: Thu Oct 26 18:32:39 2023 +0800
KYLIN-5859 Add cache in ProjectBundle for effectiveRewriteMeasures of table
---
.../kylin/metadata/project/NProjectLoader.java | 58 ++++++++++++++--------
.../kylin/metadata/project/NProjectManager.java | 2 +-
.../kylin/metadata/project/NProjectLoaderTest.java | 24 +++++++--
3 files changed, 60 insertions(+), 24 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectLoader.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectLoader.java
index 1b80aaf299..ddca82d024 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectLoader.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectLoader.java
@@ -95,25 +95,10 @@ public class NProjectLoader {
return Collections.unmodifiableSet(realizationsByTable);
}
- public List<MeasureDesc> listEffectiveRewriteMeasures(String project,
String table, boolean onlyRewriteMeasure) {
- Set<IRealization> realizations = getRealizationsByTable(project,
table);
- Set<String> modelIds =
NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project)
- .listAllModelIds();
- List<MeasureDesc> result = Lists.newArrayList();
- List<IRealization> existingRealizations =
realizations.stream().filter(r -> modelIds.contains(r.getUuid()))
- .collect(Collectors.toList());
- for (IRealization r : existingRealizations) {
- if (!r.isOnline())
- continue;
- NDataModel model = r.getModel();
- for (MeasureDesc m : r.getMeasures()) {
- FunctionDesc func = m.getFunction();
- if (belongToFactTable(table, model) && (!onlyRewriteMeasure ||
func.needRewrite())) {
- result.add(m);
- }
- }
- }
- return result;
+ public List<MeasureDesc> listEffectiveRewriteMeasures(String project,
String table) {
+ List<MeasureDesc> effectiveRewriteMeasures =
load(project).tableToMeasuresMap.get(StringUtils.upperCase(table));
+ return effectiveRewriteMeasures == null ? Collections.emptyList()
+ : Collections.unmodifiableList(effectiveRewriteMeasures);
}
private boolean belongToFactTable(String table, NDataModel model) {
@@ -173,7 +158,7 @@ public class NProjectLoader {
mapTableToRealization(projectBundle, realization);
}
});
-
+ mapEffectiveRewriteMeasuresByTable(projectBundle, project,
projectAllTables);
return projectBundle;
}
@@ -217,9 +202,42 @@ public class NProjectLoader {
}
}
+ private void mapEffectiveRewriteMeasuresByTable(ProjectBundle prjCache,
String project,
+ Map<String, TableDesc> projectAllTables) {
+ Set<String> modelIds =
NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project)
+ .listAllModelIds();
+
+ projectAllTables.forEach((tableKey, tableDesc) -> {
+ Set<IRealization> realizations =
prjCache.realizationsByTable.get(tableKey);
+ if (realizations == null) {
+ return;
+ }
+ List<IRealization> existingRealizations = realizations.stream()
+ .filter(realization ->
modelIds.contains(realization.getUuid())).collect(Collectors.toList());
+ List<MeasureDesc> measureDescs = Lists.newArrayList();
+ for (IRealization realization : existingRealizations) {
+ if (!realization.isOnline()) {
+ continue;
+ }
+ NDataModel model = realization.getModel();
+ if (model == null || model.isBroken()) {
+ continue;
+ }
+ for (MeasureDesc measureDesc : realization.getMeasures()) {
+ FunctionDesc func = measureDesc.getFunction();
+ if (belongToFactTable(tableKey, model) &&
func.needRewrite()) {
+ measureDescs.add(measureDesc);
+ }
+ }
+ }
+ prjCache.tableToMeasuresMap.put(tableDesc.getIdentity(),
measureDescs);
+ });
+ }
+
private static class ProjectBundle {
private String project;
private final Map<String, Set<IRealization>> realizationsByTable = new
ConcurrentHashMap<>();
+ private final Map<String, List<MeasureDesc>> tableToMeasuresMap = new
ConcurrentHashMap<>();
ProjectBundle(String project) {
this.project = project;
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
index a10b271b01..2319e5bccd 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
@@ -210,7 +210,7 @@ public class NProjectManager {
}
public List<MeasureDesc> listEffectiveRewriteMeasures(String project,
String factTable) {
- return projectLoader.listEffectiveRewriteMeasures(project,
factTable.toUpperCase(Locale.ROOT), true);
+ return projectLoader.listEffectiveRewriteMeasures(project, factTable);
}
KylinConfig getConfig() {
diff --git
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectLoaderTest.java
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectLoaderTest.java
index 72c6561fbe..a2c3bc7f92 100644
---
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectLoaderTest.java
+++
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectLoaderTest.java
@@ -20,13 +20,16 @@ package org.apache.kylin.metadata.project;
import static
org.apache.kylin.metadata.project.NProjectLoaderTest.PROJECT_NAME;
+import java.util.List;
import java.util.Set;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.realization.IRealization;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.RepeatedTest;
+import org.junit.jupiter.api.Test;
@MetadataInfo(overlay = "src/test/resources/ut_meta/project_loader", project =
PROJECT_NAME)
class NProjectLoaderTest {
@@ -38,9 +41,24 @@ class NProjectLoaderTest {
NProjectLoader.removeCache();
NProjectLoader.updateCache(PROJECT_NAME);
Set<IRealization> realizations = new NProjectLoader(
-
NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())).getRealizationsByTable(PROJECT_NAME,
- "SSB.P_LINEORDER");
+ NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()))
+ .getRealizationsByTable(PROJECT_NAME, "SSB.P_LINEORDER");
Assertions.assertEquals(5, realizations.size());
NProjectLoader.removeCache();
}
+
+ @Test
+ public void testGetEffectiveRewriteMeasures() {
+ NProjectLoader.removeCache();
+ NProjectLoader.updateCache(PROJECT_NAME);
+ List<MeasureDesc> effectiveRewriteMeasures = new NProjectLoader(
+ NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()))
+ .listEffectiveRewriteMeasures(PROJECT_NAME, "SSB.P_LINEORDER");
+
+ List<MeasureDesc> cacheEffectiveRewriteMeasures = new NProjectLoader(
+ NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()))
+ .listEffectiveRewriteMeasures(PROJECT_NAME, "SSB.P_LINEORDER");
+ Assertions.assertEquals(effectiveRewriteMeasures.size(),
cacheEffectiveRewriteMeasures.size());
+ NProjectLoader.removeCache();
+ }
}
