This is an automated email from the ASF dual-hosted git repository.
pfzhan pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 4a6e1893f5 KYLIN-6034 fix columns of OlapContext when top node is
OlapWindowRel
4a6e1893f5 is described below
commit 4a6e1893f5fc0dd9b332e63be268e32587ae1c99
Author: Xuecheng Shan <[email protected]>
AuthorDate: Tue Jul 9 11:12:57 2024 +0800
KYLIN-6034 fix columns of OlapContext when top node is OlapWindowRel
Co-authored-by: Xuecheng Shan <[email protected]>
---
.../query/engine/WindowFunctionPlannerTest.java | 162 +++++++++++++++++++++
.../query/sql_window_olap_context/query00.sql | 20 +++
.../query/sql_window_olap_context/query01.sql | 21 +++
.../query/sql_window_olap_context/query02.sql | 20 +++
.../query/sql_window_olap_context/query03.sql | 20 +++
.../query/sql_window_olap_context/query04.sql | 21 +++
.../query/sql_window_olap_context/query05.sql | 28 ++++
.../apache/kylin/query/relnode/ContextUtil.java | 6 +-
.../apache/kylin/query/relnode/OlapWindowRel.java | 26 +++-
9 files changed, 322 insertions(+), 2 deletions(-)
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/WindowFunctionPlannerTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/WindowFunctionPlannerTest.java
new file mode 100644
index 0000000000..1f8244f01c
--- /dev/null
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/WindowFunctionPlannerTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.engine;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.NoRealizationFoundException;
+import org.apache.kylin.query.relnode.ContextUtil;
+import org.apache.kylin.query.relnode.OlapContext;
+import org.apache.kylin.query.relnode.OlapLimitRel;
+import org.apache.kylin.query.relnode.OlapWindowRel;
+import org.apache.kylin.query.rules.CalciteRuleTestBase;
+import org.apache.kylin.query.util.QueryContextCutter;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class WindowFunctionPlannerTest extends CalciteRuleTestBase {
+
+ String project = "default";
+
+ QueryExec queryExec;
+
+ @Before
+ public void setup() {
+ createTestMetadata();
+ queryExec = new QueryExec(project, getTestConfig());
+ }
+
+ private void prepareOlapContext(String sql) throws SqlParseException {
+ RelNode relNode = queryExec.parseAndOptimize(sql);
+ List<RelNode> optimizedRelNodes = queryExec.postOptimize(relNode);
+
+ try {
+ QueryContextCutter.selectRealization(project,
optimizedRelNodes.get(0),
+ QueryContext.current().isForModeling());
+ } catch (NoRealizationFoundException e) {
+ log.info("No realization found for sql: {}", sql);
+ }
+ }
+
+ private String getSql(String fileName) throws IOException {
+ Pair<String, String> sql = readOneSQL(getTestConfig(), project,
"query/sql_window_olap_context", fileName);
+ return sql.getSecond();
+ }
+
+ @Test
+ public void testSelectStarAndWindow() throws SqlParseException,
IOException {
+ prepareOlapContext(getSql("query00.sql"));
+
+ Assert.assertEquals(1, ContextUtil.listContexts().size());
+ OlapContext olapContext = ContextUtil.listContexts().get(0);
+ Assert.assertEquals(17, olapContext.getAllColumns().size());
+ Assert.assertTrue(olapContext.getTopNode() instanceof OlapWindowRel);
+ }
+
+ @Test
+ public void testSelectStarAndWindowWithLimit() throws SqlParseException,
IOException {
+ prepareOlapContext(getSql("query01.sql"));
+
+ Assert.assertEquals(1, ContextUtil.listContexts().size());
+ OlapContext olapContext = ContextUtil.listContexts().get(0);
+ Assert.assertEquals(17, olapContext.getAllColumns().size());
+ Assert.assertTrue(olapContext.getTopNode() instanceof OlapLimitRel);
+ }
+
+ @Test
+ public void testSelectOnlyWindow() throws IOException, SqlParseException {
+ prepareOlapContext(getSql("query02.sql"));
+
+ Assert.assertEquals(1, ContextUtil.listContexts().size());
+ OlapContext olapContext = ContextUtil.listContexts().get(0);
+ Set<TblColRef> allColumns = olapContext.getAllColumns();
+ Assert.assertEquals(2, allColumns.size());
+ Set<String> expectedColumns =
Sets.newHashSet("SSB.LINEORDER.LO_ORDERDATE", "SSB.LINEORDER.LO_QUANTITY");
+ allColumns.forEach(tblColRef -> expectedColumns
+ .remove(tblColRef.getTableRef().getTableIdentity() + '.' +
tblColRef.getColumnDesc().getName()));
+ Assert.assertTrue(expectedColumns.isEmpty());
+ }
+
+ @Test
+ public void testSelectWindowAndSpecifiedColumn() throws IOException,
SqlParseException {
+ prepareOlapContext(getSql("query03.sql"));
+
+ Assert.assertEquals(1, ContextUtil.listContexts().size());
+ OlapContext olapContext = ContextUtil.listContexts().get(0);
+ Set<TblColRef> allColumns = olapContext.getAllColumns();
+ Assert.assertEquals(3, allColumns.size());
+ Set<String> expectedColumns =
Sets.newHashSet("SSB.LINEORDER.LO_ORDERDATE", "SSB.LINEORDER.LO_QUANTITY",
+ "SSB.LINEORDER.LO_CUSTKEY");
+ allColumns.forEach(tblColRef -> expectedColumns
+ .remove(tblColRef.getTableRef().getTableIdentity() + '.' +
tblColRef.getColumnDesc().getName()));
+ Assert.assertTrue(expectedColumns.isEmpty());
+ }
+
+ @Test
+ public void testSelectWindowAndFilter() throws IOException,
SqlParseException {
+ prepareOlapContext(getSql("query04.sql"));
+
+ Assert.assertEquals(1, ContextUtil.listContexts().size());
+ OlapContext olapContext = ContextUtil.listContexts().get(0);
+ Set<TblColRef> allColumns = olapContext.getAllColumns();
+ Assert.assertEquals(4, allColumns.size());
+ Set<String> expectedColumns =
Sets.newHashSet("SSB.LINEORDER.LO_ORDERDATE", "SSB.LINEORDER.LO_QUANTITY",
+ "SSB.LINEORDER.LO_CUSTKEY", "SSB.LINEORDER.LO_PARTKEY");
+ allColumns.forEach(tblColRef -> expectedColumns
+ .remove(tblColRef.getTableRef().getTableIdentity() + '.' +
tblColRef.getColumnDesc().getName()));
+ Assert.assertTrue(expectedColumns.isEmpty());
+ }
+
+ @Test
+ public void testSubQueryWithWidow() throws IOException, SqlParseException {
+ prepareOlapContext(getSql("query05.sql"));
+
+ Assert.assertEquals(2, ContextUtil.listContexts().size());
+
+ OlapContext olapContext1 = ContextUtil.listContexts().get(0);
+ Set<TblColRef> allColumns1 = olapContext1.getAllColumns();
+ Assert.assertEquals(4, allColumns1.size());
+ Set<String> expectedColumns1 =
Sets.newHashSet("SSB.LINEORDER.LO_ORDERDATE", "SSB.LINEORDER.LO_QUANTITY",
+ "SSB.LINEORDER.LO_ORDERKEY", "SSB.LINEORDER.LO_PARTKEY");
+ allColumns1.forEach(tblColRef -> expectedColumns1
+ .remove(tblColRef.getTableRef().getTableIdentity() + '.' +
tblColRef.getColumnDesc().getName()));
+ Assert.assertTrue(expectedColumns1.isEmpty());
+
+ OlapContext olapContext2 = ContextUtil.listContexts().get(1);
+ Set<TblColRef> allColumns2 = olapContext2.getAllColumns();
+ Assert.assertEquals(5, allColumns2.size());
+ Set<String> expectedColumns2 =
Sets.newHashSet("SSB.LINEORDER.LO_ORDERDATE", "SSB.LINEORDER.LO_QUANTITY",
+ "SSB.LINEORDER.LO_ORDERKEY", "SSB.LINEORDER.LO_PARTKEY",
"SSB.LINEORDER.LO_CUSTKEY");
+ allColumns2.forEach(tblColRef -> expectedColumns2
+ .remove(tblColRef.getTableRef().getTableIdentity() + '.' +
tblColRef.getColumnDesc().getName()));
+ Assert.assertTrue(expectedColumns2.isEmpty());
+ }
+}
diff --git
a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query00.sql
b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query00.sql
new file mode 100644
index 0000000000..c00f0ab8eb
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query00.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT *, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY)
AS RN
+FROM SSB.LINEORDER
\ No newline at end of file
diff --git
a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query01.sql
b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query01.sql
new file mode 100644
index 0000000000..f248deb293
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query01.sql
@@ -0,0 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT *, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY)
AS RN
+FROM SSB.LINEORDER
+LIMIT 100
\ No newline at end of file
diff --git
a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query02.sql
b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query02.sql
new file mode 100644
index 0000000000..832667e729
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query02.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY) AS RN
+FROM SSB.LINEORDER
\ No newline at end of file
diff --git
a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query03.sql
b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query03.sql
new file mode 100644
index 0000000000..3cdacbb2b4
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query03.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT LO_ORDERDATE, LO_CUSTKEY, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE
ORDER BY LO_QUANTITY) AS RN
+FROM SSB.LINEORDER
\ No newline at end of file
diff --git
a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query04.sql
b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query04.sql
new file mode 100644
index 0000000000..d00a6c404b
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query04.sql
@@ -0,0 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT LO_ORDERDATE, LO_CUSTKEY, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE
ORDER BY LO_QUANTITY) AS RN
+FROM SSB.LINEORDER
+WHERE LO_PARTKEY < 1000
\ No newline at end of file
diff --git
a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query05.sql
b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query05.sql
new file mode 100644
index 0000000000..7872fce655
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query05.sql
@@ -0,0 +1,28 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select t1.LO_ORDERDATE, t1.LO_ORDERKEY, t2.LO_CUSTKEY, t1.RN_1, t2.RN_2
+from
+(SELECT *, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY)
AS RN_1
+ FROM SSB.LINEORDER
+where LO_PARTKEY<1000) t1
+left join
+(SELECT *, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY)
AS RN_2
+ FROM SSB.LINEORDER
+where LO_PARTKEY>2000) t2
+on t1.LO_ORDERKEY=t2.LO_ORDERKEY
\ No newline at end of file
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
index 661a30cb8f..6d819573e4 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
@@ -105,8 +105,12 @@ public class ContextUtil {
((OlapRel)
rel).getColumnRowType().getAllColumns().stream().filter(context::isOriginAndBelongToCtxTables)
.forEach(context.getAllColumns()::add);
} else if (rel instanceof OlapWindowRel) {
- ((OlapWindowRel)
rel).getGroupingColumns().stream().filter(context::isOriginAndBelongToCtxTables)
+ OlapWindowRel olapWindowRel = (OlapWindowRel) rel;
+
olapWindowRel.getGroupingColumns().stream().filter(context::isOriginAndBelongToCtxTables)
.forEach(context.getAllColumns()::add);
+ if (!olapWindowRel.isExistParentProjectNeedPushInfo()) {
+ amendAllColsIfNoAgg(olapWindowRel.getInput());
+ }
} else if (rel instanceof OlapJoinRel) {
amendAllColsIfNoAgg(rel.getInput(0));
amendAllColsIfNoAgg(rel.getInput(1));
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OlapWindowRel.java
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OlapWindowRel.java
index 6d7afe0e1f..48cd6d7fa2 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OlapWindowRel.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OlapWindowRel.java
@@ -20,6 +20,7 @@ package org.apache.kylin.query.relnode;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Deque;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -55,6 +56,8 @@ public class OlapWindowRel extends Window implements OlapRel {
@Setter
private Set<OlapContext> subContexts = Sets.newHashSet();
+ private boolean existParentProjectNeedPushInfo;
+
public OlapWindowRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode
input, List<RexLiteral> constants,
RelDataType rowType, List<Window.Group> groups) {
super(cluster, traitSet, input, constants, rowType, groups);
@@ -87,12 +90,33 @@ public class OlapWindowRel extends Window implements
OlapRel {
if (context != null) {
this.context.setHasWindow(true);
if (this == context.getTopNode() && !context.isHasAgg())
- ContextUtil.amendAllColsIfNoAgg(this);
+ amendContextColumns(olapImpl);
} else {
ContextUtil.updateSubContexts(getGroupingColumns(), subContexts);
}
}
+ private void amendContextColumns(OlapImpl olapImpl) {
+ this.existParentProjectNeedPushInfo =
checkParentProjectNeedPushInfo(olapImpl.getParentNodeStack());
+ ContextUtil.amendAllColsIfNoAgg(this);
+ }
+
+ public boolean checkParentProjectNeedPushInfo(Deque<RelNode> allParents) {
+ for (RelNode parent : allParents) {
+ if (parent instanceof OlapProjectRel) {
+ OlapProjectRel parentProject = (OlapProjectRel) parent;
+ if (parentProject.isNeedPushInfoToSubCtx()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ public boolean isExistParentProjectNeedPushInfo() {
+ return existParentProjectNeedPushInfo;
+ }
+
protected ColumnRowType buildColumnRowType() {
OlapRel olapChild = (OlapRel) getInput(0);
ColumnRowType inputColumnRowType = olapChild.getColumnRowType();
