This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 191d08c272b48d04441783290459291f70c2e9e6
Author: Pengfei Zhan <dethr...@gmail.com>
AuthorDate: Wed Sep 20 10:20:56 2023 +0800
KYLIN-5830 Push columns of sortRel to subOlapContexts
---
.../query/routing/RealizationChooserTest.java | 24 +++++++++++++++++++++-
.../org/apache/kylin/util/OlapContextTestUtil.java | 13 ++++++++++++
.../org/apache/kylin/query/relnode/KapSortRel.java | 19 +++++++++++++++--
.../apache/kylin/query/relnode/KapTableScan.java | 10 +++++----
4 files changed, 59 insertions(+), 7 deletions(-)
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
index 2cea9cf51b..267be6b1d8 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
@@ -21,12 +21,14 @@ package org.apache.kylin.query.routing;
import java.util.List;
import java.util.Map;
+import org.apache.calcite.rel.RelNode;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
+import org.apache.kylin.query.relnode.KapSortRel;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.util.OlapContextTestUtil;
import org.junit.After;
@@ -97,6 +99,25 @@ public class RealizationChooserTest extends
NLocalWithSparkSessionTest {
}
}
+ @Test
+ public void testPushSortRelToSubOlapContexts() throws SqlParseException {
+ overwriteSystemProp("kylin.query.print-logical-plan", "true");
+ String project = "joins_graph_left_or_inner";
+ String sql = "select a.NAME from TEST_BANK_INCOME a inner join
TEST_BANK_LOCATION b on a.COUNTRY = b.COUNTRY\n"
+ + "order by a.INCOME nulls last";
+ RelNode relNode =
OlapContextTestUtil.cutOlapContextsAndReturnRelNode(project, sql);
+ KapSortRel sortRel = null;
+ while (relNode != null) {
+ if (relNode instanceof KapSortRel) {
+ sortRel = (KapSortRel) relNode;
+ break;
+ }
+ relNode = relNode.getInput(0);
+ }
+ Assert.assertNotNull(sortRel);
+ Assert.assertTrue(sortRel.isNeedPushToSubCtx());
+ }
+
@Test
public void testCanMatchModelInnerQueryLeft() throws SqlParseException {
// model: TEST_BANK_INCOME inner join TEST_BANK_LOCATION
@@ -127,7 +148,8 @@ public class RealizationChooserTest extends
NLocalWithSparkSessionTest {
String sql = "select a.NAME from TEST_BANK_INCOME a left join
TEST_BANK_LOCATION b \n"
+ " on a.COUNTRY = b.COUNTRY where " + filter;
OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(),
+ olapContext);
Assert.assertTrue(sqlAlias2ModelNameMap.isEmpty());
}
}
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
b/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
index 094fa8c3c5..e7e2a88b5c 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
@@ -56,6 +56,19 @@ public class OlapContextTestUtil {
return getOlapContexts();
}
+ public static RelNode cutOlapContextsAndReturnRelNode(String project,
String sql) throws SqlParseException {
+ QueryExec queryExec = new QueryExec(project,
KylinConfig.getInstanceFromEnv());
+ RelNode rel = null;
+ try {
+ rel = queryExec.parseAndOptimize(sql);
+ QueryContextCutter.selectRealization(project, rel, false);
+ } catch (NoRealizationFoundException |
NoStreamingRealizationFoundException e) {
+ // When NoRealizationFoundException occurs, do nothing
+ // because we only need to obtain OlapContexts.
+ }
+ return rel;
+ }
+
public static List<OLAPContext> getOlapContexts(String project, String
sql, boolean reCutBanned,
Consumer<NoRealizationFoundException> consumer) throws
SqlParseException {
QueryExec queryExec = new QueryExec(project,
KylinConfig.getInstanceFromEnv());
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapSortRel.java
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapSortRel.java
index 2fe32a6fca..9f91b0f355 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapSortRel.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapSortRel.java
@@ -18,7 +18,10 @@
package org.apache.kylin.query.relnode;
+import java.util.Collection;
+import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelTraitSet;
@@ -26,15 +29,21 @@ import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rex.RexNode;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.query.util.ICutContextStrategy;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import lombok.Getter;
+import lombok.Setter;
public class KapSortRel extends OLAPSortRel implements KapRel {
private Set<OLAPContext> subContexts = Sets.newHashSet();
+ @Setter
+ @Getter
+ protected boolean needPushToSubCtx;
public KapSortRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode
child, RelCollation collation,
RexNode offset, RexNode fetch) {
@@ -101,6 +110,12 @@ public class KapSortRel extends OLAPSortRel implements
KapRel {
this.context.addSort(orderCol, order);
this.context.allColumns.addAll(orderCol.getSourceColumns());
}
+ } else if (needPushToSubCtx) {
+ List<Set<TblColRef>> sourceColumns =
this.columnRowType.getSourceColumns();
+ if (CollectionUtils.isNotEmpty(sourceColumns)) {
+ ContextUtil.updateSubContexts(
+
sourceColumns.stream().flatMap(Collection::stream).collect(Collectors.toSet()),
subContexts);
+ }
}
}
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapTableScan.java
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapTableScan.java
index 92bfbc4225..bf2c46fc1e 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapTableScan.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapTableScan.java
@@ -33,16 +33,15 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.query.engine.KECalciteConfig;
import org.apache.kylin.query.schema.OLAPSchema;
import org.apache.kylin.query.schema.OLAPTable;
-import org.apache.kylin.query.engine.KECalciteConfig;
import org.apache.kylin.query.util.ICutContextStrategy;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
/**
*/
public class KapTableScan extends OLAPTableScan implements EnumerableRel,
KapRel {
@@ -153,6 +152,9 @@ public class KapTableScan extends OLAPTableScan implements
EnumerableRel, KapRel
break;
}
KapRel parent = (KapRel) tempParent;
+ if (parent instanceof KapSortRel) {
+ ((KapSortRel) parent).setNeedPushToSubCtx(true);
+ }
if (topProjParent == null && parent instanceof OLAPProjectRel
&& !((OLAPProjectRel) parent).isMerelyPermutation()) {
