This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e722aa6722bf3daae3bee9289441d7bc00144975
Author: Pengfei Zhan <dethr...@gmail.com>
AuthorDate: Sun Apr 23 21:31:23 2023 +0800
KYLIN-5633 fix recommendation on modelView & restore query behavior of
hybrid model
---
.../routing/DataflowCapabilityCheckerTest.java | 16 +--
.../routing/HeterogeneousSegmentPruningTest.java | 32 +++---
.../query/routing/QueryLayoutChooserTest.java | 118 ++++++++++-----------
.../kylin/query/routing/QueryLayoutFilterTest.java | 4 +-
.../query/routing/RealizationChooserTest.java | 22 ++--
.../query/routing/SegmentPruningRuleTest.java | 4 +-
.../routing/TableIndexAnswerSelectStarTest.java | 14 +--
.../query/routing/VacantIndexPruningRuleTest.java | 24 ++---
...apContextUtil.java => OlapContextTestUtil.java} | 23 +++-
.../engine/exec/sparder/SparderQueryPlanExec.java | 37 ++-----
.../kylin/query/util/QueryContextCutter.java | 34 +++---
11 files changed, 166 insertions(+), 162 deletions(-)
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/DataflowCapabilityCheckerTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/DataflowCapabilityCheckerTest.java
index 0b33b0c382..45d07cd629 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/DataflowCapabilityCheckerTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/DataflowCapabilityCheckerTest.java
@@ -32,7 +32,7 @@ import org.apache.kylin.metadata.cube.model.NDataflowUpdate;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.util.OlapContextUtil;
+import org.apache.kylin.util.OlapContextTestUtil;
import org.junit.Assert;
import org.junit.Test;
@@ -43,8 +43,8 @@ public class DataflowCapabilityCheckerTest extends
NLocalWithSparkSessionTest {
NDataflow dataflow =
NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject())
.getDataflow("abe3bf1a-c4bc-458d-8278-7ea8b00f5e96");
String sql = "SELECT seller_ID FROM TEST_KYLIN_FACT LEFT JOIN
TEST_ACCOUNT ON SELLER_ID = ACCOUNT_ID";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
Candidate candidate = new Candidate(dataflow, olapContext,
sqlAlias2ModelNameMap);
CapabilityResult result = DataflowCapabilityChecker.check(dataflow,
candidate, olapContext.getSQLDigest());
@@ -64,8 +64,8 @@ public class DataflowCapabilityCheckerTest extends
NLocalWithSparkSessionTest {
// case 1. raw-query answered by Lookup
{
String sql = "select SITE_ID from EDW.TEST_SITES";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
Candidate candidate = new Candidate(dataflow, olapContext,
sqlAlias2ModelNameMap);
CapabilityResult result =
DataflowCapabilityChecker.check(dataflow, candidate,
olapContext.getSQLDigest());
@@ -78,8 +78,8 @@ public class DataflowCapabilityCheckerTest extends
NLocalWithSparkSessionTest {
// case 2. aggregate-query answered by lookup
{
String sql = "select sum(SITE_ID) from EDW.TEST_SITES";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
Candidate candidate = new Candidate(dataflow, olapContext,
sqlAlias2ModelNameMap);
CapabilityResult result =
DataflowCapabilityChecker.check(dataflow, candidate,
olapContext.getSQLDigest());
@@ -92,7 +92,7 @@ public class DataflowCapabilityCheckerTest extends
NLocalWithSparkSessionTest {
{
// case 3. cannot answer when there are no ready segment
String sql = "select sum(SITE_ID) from EDW.TEST_SITES";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
removeAllSegments(dataflow);
dataflow =
NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject())
.getDataflow("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/HeterogeneousSegmentPruningTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/HeterogeneousSegmentPruningTest.java
index 715591aca2..f18ba30b2a 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/HeterogeneousSegmentPruningTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/HeterogeneousSegmentPruningTest.java
@@ -47,7 +47,7 @@ import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.util.MetadataTestUtils;
-import org.apache.kylin.util.OlapContextUtil;
+import org.apache.kylin.util.OlapContextTestUtil;
import org.junit.Assert;
import org.junit.Test;
@@ -398,7 +398,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
{ // composite filters
val sqlFilter = sql + "select * from T1\n"
+ "where (cal_dt = DATE'2012-01-01' or (cast(cal_dt as
date) = '2012-01-02' or cal_dt = '2012-01-03')) and (cal_dt is not null or
cal_dt in ('2012-01-01', '2012-01-02'))";
- val context = OlapContextUtil.getOlapContexts(project,
sqlFilter).get(0);
+ val context = OlapContextTestUtil.getOlapContexts(project,
sqlFilter).get(0);
assertFiltersAndLayout(context, null,
">=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01),<=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-03),"
+ "OR(=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01), =(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-02),
=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-03)),"
@@ -408,7 +408,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
{ // invalid filters with or
val sqlFilter = sql + "select * from T1\n"
+ "where trans_id = 123 or trans_id + 1 = 123 or
(trans_id + 2 = 234 and trans_id = 345)";
- val context = OlapContextUtil.getOlapContexts(project,
sqlFilter).get(0);
+ val context = OlapContextTestUtil.getOlapContexts(project,
sqlFilter).get(0);
assertFiltersAndLayout(context, null,
">=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01),<=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-03)");
}
@@ -416,7 +416,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
{ // invalid filters with and
val sqlFilter = sql + "select * from T1\n"
+ "where trans_id = 123 and (trans_id + 2 = 234 or
trans_id = 345)";
- val context = OlapContextUtil.getOlapContexts(project,
sqlFilter).get(0);
+ val context = OlapContextTestUtil.getOlapContexts(project,
sqlFilter).get(0);
assertFiltersAndLayout(context, null,
">=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01),<=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-03),"
+ "=(DEFAULT.TEST_KYLIN_FACT.TRANS_ID, 123)");
@@ -425,7 +425,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
{ // invalid filters with not
val sqlFilter = sql + "select * from T1\n"
+ "where trans_id = 123 and (not(trans_id = 234) or
trans_id = 345) and (not(trans_id + 1 = 132))";
- val context = OlapContextUtil.getOlapContexts(project,
sqlFilter).get(0);
+ val context = OlapContextTestUtil.getOlapContexts(project,
sqlFilter).get(0);
assertFiltersAndLayout(context, null,
">=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01),<=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-03),"
+ "=(DEFAULT.TEST_KYLIN_FACT.TRANS_ID,
123),OR(<>(DEFAULT.TEST_KYLIN_FACT.TRANS_ID, 234),
=(DEFAULT.TEST_KYLIN_FACT.TRANS_ID, 345))");
@@ -464,7 +464,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
{ // inner join
val sqlInnerJoin = sql + " select * from T1 inner join T2 on
T1.cal_dt = T2.cal_dt \n"
+ " where T1.cal_dt = '2012-01-01' and T2.cal_dt = DATE
'2012-01-02'";
- val contexts = OlapContextUtil.getOlapContexts(project,
sqlInnerJoin);
+ val contexts = OlapContextTestUtil.getOlapContexts(project,
sqlInnerJoin);
assertFiltersAndLayout(contexts.get(0), seg1Id,
">=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01),<=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-03),=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-01)");
assertFiltersAndLayout(contexts.get(1), seg2Id,
@@ -474,7 +474,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
{ // left join
val sqlLeftJoin = sql + " select * from T1 left join T2 on
T1.cal_dt = T2.cal_dt \n"
+ " where T1.cal_dt = DATE '2012-01-01' and T2.cal_dt =
DATE '2012-01-02'";
- val contexts = OlapContextUtil.getOlapContexts(project,
sqlLeftJoin);
+ val contexts = OlapContextTestUtil.getOlapContexts(project,
sqlLeftJoin);
assertFiltersAndLayout(contexts.get(0), seg1Id,
">=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01),<=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-03),=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-01)");
assertFiltersAndLayout(contexts.get(1), null,
@@ -484,7 +484,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
{ // right join
val sqlRightJoin = sql + " select * from T1 right join T2 on
T1.cal_dt = T2.cal_dt \n"
+ " where T1.cal_dt = DATE '2012-01-01' and T2.cal_dt =
DATE '2012-01-02'";
- val contexts = OlapContextUtil.getOlapContexts(project,
sqlRightJoin);
+ val contexts = OlapContextTestUtil.getOlapContexts(project,
sqlRightJoin);
assertFiltersAndLayout(contexts.get(0), seg2Id,
">=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01),<=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-02),=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-02)");
assertFiltersAndLayout(contexts.get(1), null,
@@ -494,7 +494,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
{ // full join
val sqlFullJoin = sql + " select * from T1 full join T2 on
T1.cal_dt = T2.cal_dt \n"
+ " where T1.cal_dt = DATE '2012-01-01' and T2.cal_dt =
DATE '2012-01-02'";
- val contexts = OlapContextUtil.getOlapContexts(project,
sqlFullJoin);
+ val contexts = OlapContextTestUtil.getOlapContexts(project,
sqlFullJoin);
assertFiltersAndLayout(contexts.get(0), null,
">=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01),<=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-03)");
assertFiltersAndLayout(contexts.get(1), null,
@@ -538,7 +538,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
+ "(cal_dt='2012-01-01' and trans_id = 15) or\n" +
"(cal_dt='2012-01-01' and trans_id = 16) or\n"
+ "(cal_dt='2012-01-01' and trans_id = 17) or\n" +
"(cal_dt='2012-01-01' and trans_id = 18) or\n"
+ "(cal_dt='2012-01-01' and trans_id = 19) or\n" +
"(cal_dt='2012-01-01' and trans_id = 20)";
- val contexts = OlapContextUtil.getOlapContexts(project,
sqlWithTooManyOrs);
+ val contexts = OlapContextTestUtil.getOlapContexts(project,
sqlWithTooManyOrs);
Assert.assertEquals(
">=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01),<=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-03)",
contexts.get(0).getExpandedFilterConditions().stream().map(RexNode::toString)
@@ -552,7 +552,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
+ "(cal_dt='2012-01-01' and trans_id = 6) or\n" +
"(cal_dt='2012-01-01' and trans_id = 7) or\n"
+ "(cal_dt='2012-01-01' and trans_id = 8) or\n" +
"(cal_dt='2012-01-01' and trans_id = 9) or\n"
+ "(cal_dt='2012-01-01' and trans_id = 10)";
- val contexts = OlapContextUtil.getOlapContexts(project,
sqlWithFilter);
+ val contexts = OlapContextTestUtil.getOlapContexts(project,
sqlWithFilter);
Assert.assertNotEquals(
">=(DEFAULT.TEST_KYLIN_FACT.CAL_DT,
2012-01-01),<=(DEFAULT.TEST_KYLIN_FACT.CAL_DT, 2012-01-03)",
contexts.get(0).getExpandedFilterConditions().stream().map(RexNode::toString)
@@ -606,7 +606,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
+ "group by cal_dt\n";
// can not query
{
- OLAPContext olapContext = OlapContextUtil.getOlapContexts(project,
sql).get(0);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(project, sql).get(0);
StorageContext storageContext = olapContext.storageContext;
Assert.assertEquals(-1L, storageContext.getLayoutId().longValue());
}
@@ -615,7 +615,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
MetadataTestUtils.updateProjectConfig(project,
"kylin.query.index-match-rules",
QueryRouter.USE_VACANT_INDEXES);
try (QueryContext queryContext = QueryContext.current()) {
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(project, sql).get(0);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(project, sql).get(0);
StorageContext storageContext = olapContext.storageContext;
Assert.assertEquals(10001L,
storageContext.getLayoutId().longValue());
Assert.assertFalse(queryContext.getQueryTagInfo().isVacant());
@@ -662,7 +662,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
MetadataTestUtils.updateProjectConfig(project,
"kylin.query.index-match-rules", QueryRouter.USE_VACANT_INDEXES);
try (QueryContext queryContext = QueryContext.current()) {
- OLAPContext olapContext = OlapContextUtil.getOlapContexts(project,
sql).get(0);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(project, sql).get(0);
StorageContext storageContext = olapContext.storageContext;
Assert.assertEquals(-1L, storageContext.getLayoutId().longValue());
Assert.assertFalse(queryContext.getQueryTagInfo().isVacant());
@@ -680,7 +680,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
}
private void assertNoRealizationFound(String project, String sql) throws
SqlParseException {
- val context = OlapContextUtil.getOlapContexts(project, sql).get(0);
+ val context = OlapContextTestUtil.getOlapContexts(project, sql).get(0);
Assert.assertNull(context.realization);
}
@@ -693,7 +693,7 @@ public class HeterogeneousSegmentPruningTest extends
NLocalWithSparkSessionTest
private void assertPrunedSegmentsRange(String project, String sql, String
dfId,
List<Pair<String, String>> expectedRanges, long expectedLayoutId,
Map<String, List<Long>> expectedPartitions, String
expectedFilterCond) throws SqlParseException {
- val context = OlapContextUtil.getOlapContexts(project, sql).get(0);
+ val context = OlapContextTestUtil.getOlapContexts(project, sql).get(0);
if (expectedLayoutId == -1L) {
Assert.assertTrue(context.storageContext.isEmptyLayout());
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutChooserTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutChooserTest.java
index f767554461..4a47b4db74 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutChooserTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutChooserTest.java
@@ -45,7 +45,7 @@ import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.util.MetadataTestUtils;
-import org.apache.kylin.util.OlapContextUtil;
+import org.apache.kylin.util.OlapContextTestUtil;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -69,12 +69,12 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
NDataflowManager dataflowManager =
NDataflowManager.getInstance(getTestConfig(), getProject());
{
String sql = "select distinct DEAL_AMOUNT from test_kylin_fact \n";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
// model with computedColumns
String modelWithCCId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
NDataflow dataflow = dataflowManager.getDataflow(modelWithCCId);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
// model without computedColumns
@@ -88,12 +88,12 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
// match aggIndex - null in agg col
{
String sql = "select sum(DEAL_AMOUNT) from test_kylin_fact \n";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
// model with computedColumns
String modelWithCCId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
NDataflow dataflow = dataflowManager.getDataflow(modelWithCCId);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
// model without computedColumns
@@ -107,12 +107,12 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
// match tableIndex
{
String sql = "select DEAL_AMOUNT from test_kylin_fact \n";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
// model with computedColumns
String modelWithCCId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
NDataflow dataflow = dataflowManager.getDataflow(modelWithCCId);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
// model without computedColumns
@@ -146,8 +146,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select CAL_DT, count(price) as GMV from
test_kylin_fact \n"
+ " where CAL_DT='2012-01-10' group by CAL_DT ";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -160,8 +160,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
// 2. tableIndex match
{
String sql = "select CAL_DT from test_kylin_fact where
CAL_DT='2012-01-10'";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -184,8 +184,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select CAL_DT, TRANS_ID, count(*) as GMV from
test_kylin_fact \n"
+ " where CAL_DT='2012-01-10' and TRANS_ID > 10000 group
by CAL_DT, TRANS_ID ";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -196,8 +196,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select CAL_DT, TRANS_ID, count(*) as GMV from
test_kylin_fact \n"
+ " where CAL_DT > '2012-01-10' and TRANS_ID = 10000 group
by CAL_DT, TRANS_ID ";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -210,8 +210,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
String sql = "select CAL_DT, TRANS_ID, count(*) as GMV from
test_kylin_fact \n"
+ " where CAL_DT = '2012-01-10' and TRANS_ID = 10000 group
by CAL_DT, TRANS_ID ";
mockTableStats();
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -287,8 +287,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
+ "AND test_kylin_fact.lstg_site_id =
test_category_groupings.site_id\n"
+ " left JOIN edw.test_sites as test_sites ON
test_kylin_fact.lstg_site_id = test_sites.site_id\n"
+ " group by test_kylin_fact.lstg_format_name,
META_CATEG_NAME";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(), olapContext.getSQLDigest(),
null);
@@ -355,8 +355,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
String sql = "select b.ACCOUNT_BUYER_LEVEL from
\"DEFAULT\".\"TEST_KYLIN_FACT\" a\n"
+ "left join \"DEFAULT\".\"TEST_ACCOUNT\" b on a.SELLER_ID =
b.ACCOUNT_ID";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(), olapContext.getSQLDigest(),
null);
@@ -380,11 +380,11 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
+ "group by 1.1000000000000001\n";
getTestConfig().setProperty("kylin.query.calcite.aggregate-pushdown-enabled",
"true");
- List<OLAPContext> olapContexts =
OlapContextUtil.getHepRulesOptimizedOlapContexts(getProject(), sql, false);
+ List<OLAPContext> olapContexts =
OlapContextTestUtil.getHepRulesOptimizedOlapContexts(getProject(), sql, false);
// validate the first
OLAPContext oneOlapContext = olapContexts.get(0);
- Map<String, String> oneMap =
OlapContextUtil.matchJoins(dataflow.getModel(), oneOlapContext);
+ Map<String, String> oneMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), oneOlapContext);
oneOlapContext.fixModel(dataflow.getModel(), oneMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
oneOlapContext.getSQLDigest(), null);
@@ -393,7 +393,7 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
// validate the second
OLAPContext anotherOlapContext = olapContexts.get(1);
- Map<String, String> anotherMap =
OlapContextUtil.matchJoins(dataflow.getModel(), anotherOlapContext);
+ Map<String, String> anotherMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), anotherOlapContext);
anotherOlapContext.fixModel(dataflow.getModel(), anotherMap);
NLayoutCandidate anotherCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
anotherOlapContext.getSQLDigest(), null);
@@ -418,11 +418,11 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
+ "group by 1.1000000000000001\n";
getTestConfig().setProperty("kylin.query.calcite.aggregate-pushdown-enabled",
"false");
- List<OLAPContext> olapContexts =
OlapContextUtil.getHepRulesOptimizedOlapContexts(getProject(), sql, false);
+ List<OLAPContext> olapContexts =
OlapContextTestUtil.getHepRulesOptimizedOlapContexts(getProject(), sql, false);
// validate the first
OLAPContext oneOlapContext = olapContexts.get(0);
- Map<String, String> oneMap =
OlapContextUtil.matchJoins(dataflow.getModel(), oneOlapContext);
+ Map<String, String> oneMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), oneOlapContext);
oneOlapContext.fixModel(dataflow.getModel(), oneMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
oneOlapContext.getSQLDigest(), null);
@@ -431,7 +431,7 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
// validate the second
OLAPContext anotherOlapContext = olapContexts.get(1);
- Map<String, String> anotherMap =
OlapContextUtil.matchJoins(dataflow.getModel(), anotherOlapContext);
+ Map<String, String> anotherMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), anotherOlapContext);
anotherOlapContext.fixModel(dataflow.getModel(), anotherMap);
NLayoutCandidate anotherCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
anotherOlapContext.getSQLDigest(), null);
@@ -456,8 +456,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select CAL_DT, TRANS_ID, count(*) as GMV from
test_kylin_fact \n"
+ " where CAL_DT = '2012-01-10' and TRANS_ID = 10000 group
by CAL_DT, TRANS_ID ";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
// hit layout 1010002
@@ -474,8 +474,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select CAL_DT, TRANS_ID, count(*) as GMV from
test_kylin_fact \n"
+ " where CAL_DT = '2012-01-10' and TRANS_ID > 10000 group
by CAL_DT, TRANS_ID ";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -533,8 +533,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
String modelId = "abe3bf1a-c4bc-458d-8278-7ea8b00f5e96";
NDataflow dataflow = NDataflowManager.getInstance(getTestConfig(),
getProject()).getDataflow(modelId);
String sql = "select avg(TEST_KYLIN_FACT.ITEM_COUNT) from
TEST_KYLIN_FACT";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(), olapContext.getSQLDigest(),
null);
@@ -559,8 +559,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select sum(ORDER_ID) from TEST_KYLIN_FACT";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(emptyProject, sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(emptyProject, sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -570,8 +570,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select max(ORDER_ID) from TEST_KYLIN_FACT";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(emptyProject, sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(emptyProject, sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -581,8 +581,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select min(ORDER_ID) from TEST_KYLIN_FACT";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(emptyProject, sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(emptyProject, sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -592,8 +592,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select count(ORDER_ID) from TEST_KYLIN_FACT";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(emptyProject, sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(emptyProject, sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -603,8 +603,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select count(distinct ORDER_ID) from
TEST_KYLIN_FACT";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(emptyProject, sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(emptyProject, sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -614,8 +614,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
String sql = "select collect_set(ORDER_ID) from TEST_KYLIN_FACT";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(emptyProject, sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(emptyProject, sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -625,8 +625,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
getTestConfig().setProperty("kylin.engine.segment-online-mode",
"ANY");
String sql = "select max(PRICE)from TEST_KYLIN_FACT";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(emptyProject, sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(emptyProject, sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -661,8 +661,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
String sql = "select LSTG_FORMAT_NAME,count(*) from TEST_KYLIN_FACT
group by LSTG_FORMAT_NAME";
{
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> tableAlias2ModelAliasMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> tableAlias2ModelAliasMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(),
tableAlias2ModelAliasMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -672,8 +672,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
{
overwriteSystemProp("kylin.query.layout.prefer-aggindex", "false");
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> tableAlias2ModelAliasMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> tableAlias2ModelAliasMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(),
tableAlias2ModelAliasMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -687,11 +687,11 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
String project = "table_index";
MetadataTestUtils.updateProjectConfig(project,
"kylin.query.use-tableindex-answer-non-raw-query", "true");
String sql = "select max(PRICE)from TEST_KYLIN_FACT";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
String modelId = "acfde546-2cc9-4eec-bc92-e3bd46d4e2ee";
NDataflow dataflow = NDataflowManager.getInstance(getTestConfig(),
project).getDataflow(modelId);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(), olapContext.getSQLDigest(),
null);
@@ -712,9 +712,9 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
copyForWrite.setOverrideProps(props);
});
String sql = "select max(PRICE) from TEST_KYLIN_FACT";
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(), olapContext.getSQLDigest(),
null);
@@ -739,8 +739,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
for (String filter : filters) {
String sql = "select CAL_DT from test_kylin_fact a inner join
EDW.test_sites b \n"
+ " on a.LSTG_SITE_ID = b.SITE_ID where " + filter;
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelName =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelName =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelName);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(),
olapContext.getSQLDigest(), null);
@@ -776,8 +776,8 @@ public class QueryLayoutChooserTest extends
NLocalWithSparkSessionTest {
+ "GROUP BY \"TEST_KYLIN_FACT\".\"TRANS_ID\"";
NDataflow dataflow =
NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject())
.getDataflow("741ca86a-1f13-46da-a59f-95fb68615e3a");
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelName =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelName =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelName);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(), olapContext.getSQLDigest(),
null);
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutFilterTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutFilterTest.java
index f03e4175c1..64f4a2b3ca 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutFilterTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutFilterTest.java
@@ -38,7 +38,7 @@ import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.engine.TypeSystem;
import org.apache.kylin.query.engine.meta.SimpleDataContext;
import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.util.OlapContextUtil;
+import org.apache.kylin.util.OlapContextTestUtil;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparderEnv;
import org.apache.spark.sql.SparkSession;
@@ -105,7 +105,7 @@ public class QueryLayoutFilterTest extends
NLocalWithSparkSessionTest {
+ "and TEST_BANK_INCOME.DT = '2021-11-02'\n" //
+ "and TEST_BANK_INCOME.COUNTRY in ('INDONESIA')\n" //
+ "and TEST_BANK_INCOME.COUNTRY in ('KENYA')";
- List<OLAPContext> contexts = OlapContextUtil.getOlapContexts(project,
sql);
+ List<OLAPContext> contexts =
OlapContextTestUtil.getOlapContexts(project, sql);
OLAPContext context = contexts.get(0);
CalciteSchema rootSchema = new QueryExec(project,
kylinConfig).getRootSchema();
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
index f4363eaa36..2cea9cf51b 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
@@ -28,7 +28,7 @@ import
org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.util.OlapContextUtil;
+import org.apache.kylin.util.OlapContextTestUtil;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -59,8 +59,8 @@ public class RealizationChooserTest extends
NLocalWithSparkSessionTest {
for (String filter : filters) {
String sql = "select a.NAME from TEST_BANK_INCOME a left join
TEST_BANK_LOCATION b \n"
+ " on a.COUNTRY = b.COUNTRY where " + filter;
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelName =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelName =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
Assert.assertFalse(sqlAlias2ModelName.isEmpty());
}
}
@@ -74,8 +74,8 @@ public class RealizationChooserTest extends
NLocalWithSparkSessionTest {
overwriteSystemProp("kylin.query.join-match-optimization-enabled",
"true");
String sql = "select a.NAME from TEST_BANK_INCOME a inner join
TEST_BANK_LOCATION b on a.COUNTRY = b.COUNTRY";
NDataflow dataflow = NDataflowManager.getInstance(getTestConfig(),
project).getDataflow(modelId);
- OLAPContext olapContext = OlapContextUtil.getOlapContexts(project,
sql, true).get(0);
- Map<String, String> sqlAlias2ModelName =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext = OlapContextTestUtil.getOlapContexts(project,
sql, true).get(0);
+ Map<String, String> sqlAlias2ModelName =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
Assert.assertTrue(sqlAlias2ModelName.isEmpty());
}
@@ -91,8 +91,8 @@ public class RealizationChooserTest extends
NLocalWithSparkSessionTest {
for (String filter : filters) {
String sql = "select CAL_DT from test_kylin_fact a inner join
EDW.test_sites b \n"
+ " on a.LSTG_SITE_ID = b.SITE_ID where " + filter;
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelName =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelName =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
Assert.assertTrue(sqlAlias2ModelName.isEmpty());
}
}
@@ -109,8 +109,8 @@ public class RealizationChooserTest extends
NLocalWithSparkSessionTest {
for (String filter : filters) {
String sql = "select a.NAME from TEST_BANK_INCOME a left join
TEST_BANK_LOCATION b \n"
+ " on a.COUNTRY = b.COUNTRY where " + filter;
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql, true).get(0);
- Map<String, String> sqlAlias2ModelName =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql, true).get(0);
+ Map<String, String> sqlAlias2ModelName =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
Assert.assertFalse(sqlAlias2ModelName.isEmpty());
}
}
@@ -126,8 +126,8 @@ public class RealizationChooserTest extends
NLocalWithSparkSessionTest {
for (String filter : filters) {
String sql = "select a.NAME from TEST_BANK_INCOME a left join
TEST_BANK_LOCATION b \n"
+ " on a.COUNTRY = b.COUNTRY where " + filter;
- OLAPContext olapContext =
OlapContextUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap =
OlapContextUtil.matchJoins(dataflow.getModel(), olapContext);
+ OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
Assert.assertTrue(sqlAlias2ModelNameMap.isEmpty());
}
}
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/SegmentPruningRuleTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/SegmentPruningRuleTest.java
index 062d10b200..5ffafcb0a6 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/SegmentPruningRuleTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/SegmentPruningRuleTest.java
@@ -38,7 +38,7 @@ import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.engine.TypeSystem;
import org.apache.kylin.query.engine.meta.SimpleDataContext;
import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.util.OlapContextUtil;
+import org.apache.kylin.util.OlapContextTestUtil;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparderEnv;
import org.apache.spark.sql.SparkSession;
@@ -94,7 +94,7 @@ public class SegmentPruningRuleTest extends
NLocalWithSparkSessionTest {
private List<NDataSegment> startRealizationPruner(NDataflowManager
dataflowManager, String dataflowId, String sql,
String project, KylinConfig kylinConfig) throws Exception {
NDataflow dataflow = dataflowManager.getDataflow(dataflowId);
- List<OLAPContext> olapContexts =
OlapContextUtil.getOlapContexts(getProject(), sql);
+ List<OLAPContext> olapContexts =
OlapContextTestUtil.getOlapContexts(getProject(), sql);
OLAPContext context = olapContexts.get(0);
CalciteSchema rootSchema = new QueryExec(project,
kylinConfig).getRootSchema();
SimpleDataContext dataContext = new
SimpleDataContext(rootSchema.plus(), TypeSystem.javaTypeFactory(),
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/TableIndexAnswerSelectStarTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/TableIndexAnswerSelectStarTest.java
index c228f87579..07d4dd2e80 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/TableIndexAnswerSelectStarTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/TableIndexAnswerSelectStarTest.java
@@ -47,7 +47,7 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.query.engine.SchemaMetaData;
import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.util.OlapContextUtil;
+import org.apache.kylin.util.OlapContextTestUtil;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparderEnv;
import org.apache.spark.sql.SparkSession;
@@ -99,11 +99,11 @@ public class TableIndexAnswerSelectStarTest extends
NLocalWithSparkSessionTest {
public void testTableIndexAnswerSelectStarPartialMatch() throws Exception {
String sql = "select * from kylin_sales";
overwriteSystemProp("kylin.query.use-tableindex-answer-select-star.enabled",
"true");
- OLAPContext context = OlapContextUtil.getOlapContexts(getProject(),
sql).get(0);
+ OLAPContext context =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
NDataflow dataflow =
NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject())
.getDataflow("ccb82d81-1497-ca6d-f226-3258a0f0ba4f");
Assert.assertEquals(dataflow.getAllColumns().size(),
context.allColumns.size());
- Map<String, String> sqlAlias2ModelName =
OlapContextUtil.matchJoins(dataflow.getModel(), context);
+ Map<String, String> sqlAlias2ModelName =
OlapContextTestUtil.matchJoins(dataflow.getModel(), context);
context.fixModel(dataflow.getModel(), sqlAlias2ModelName);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(), context.getSQLDigest(), null);
@@ -115,11 +115,11 @@ public class TableIndexAnswerSelectStarTest extends
NLocalWithSparkSessionTest {
public void testTableIndexAnswerSelectStarBaseTableIndex() throws
Exception {
String sql = "select * from test_kylin_fact \n";
overwriteSystemProp("kylin.query.use-tableindex-answer-select-star.enabled",
"true");
- OLAPContext context = OlapContextUtil.getOlapContexts(getProject(),
sql).get(0);
+ OLAPContext context =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
NDataflow dataflow =
NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject())
.getDataflow("c7a44f37-8481-e78b-5cac-faa7d76767db");
Assert.assertEquals(dataflow.getAllColumns().size(),
context.allColumns.size());
- Map<String, String> sqlAlias2ModelName =
OlapContextUtil.matchJoins(dataflow.getModel(), context);
+ Map<String, String> sqlAlias2ModelName =
OlapContextTestUtil.matchJoins(dataflow.getModel(), context);
context.fixModel(dataflow.getModel(), sqlAlias2ModelName);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(), context.getSQLDigest(), null);
@@ -180,13 +180,13 @@ public class TableIndexAnswerSelectStarTest extends
NLocalWithSparkSessionTest {
dataflowManager.updateDataflow(updateOps);
String sql = "select cal_dt, new_cc from test_kylin_fact";
- OLAPContext context = OlapContextUtil.getOlapContexts(getProject(),
sql).get(0);
+ OLAPContext context =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
Set<TblColRef> allColumns = context.realization.getAllColumns();
Assert.assertEquals(13, allColumns.size());
SchemaMetaData schemaMetaData = new SchemaMetaData(getProject(),
KylinConfig.getInstanceFromEnv());
Assert.assertEquals(26,
schemaMetaData.getTables().get(1).getFields().size());
- Map<String, String> sqlAlias2ModelName =
OlapContextUtil.matchJoins(dataflow.getModel(), context);
+ Map<String, String> sqlAlias2ModelName =
OlapContextTestUtil.matchJoins(dataflow.getModel(), context);
context.fixModel(dataflow.getModel(), sqlAlias2ModelName);
NLayoutCandidate layoutCandidate =
QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(), context.getSQLDigest(), null);
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/VacantIndexPruningRuleTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/VacantIndexPruningRuleTest.java
index dad29ba967..60f57eccd4 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/VacantIndexPruningRuleTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/VacantIndexPruningRuleTest.java
@@ -39,7 +39,7 @@ import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.util.MetadataTestUtils;
-import org.apache.kylin.util.OlapContextUtil;
+import org.apache.kylin.util.OlapContextTestUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@@ -73,7 +73,7 @@ class VacantIndexPruningRuleTest extends
NLocalWithSparkSessionTest {
QueryRouter.USE_VACANT_INDEXES);
try (QueryContext queryContext = QueryContext.current()) {
String sql = "select max(LO_ORDERDATE) from ssb.lineorder";
- List<OLAPContext> olapContexts =
OlapContextUtil.getOlapContexts(getProject(), sql);
+ List<OLAPContext> olapContexts =
OlapContextTestUtil.getOlapContexts(getProject(), sql);
OLAPContext olapContext = olapContexts.get(0);
StorageContext storageContext = olapContext.storageContext;
Assertions.assertTrue(storageContext.isEmptyLayout());
@@ -85,10 +85,10 @@ class VacantIndexPruningRuleTest extends
NLocalWithSparkSessionTest {
void testUnmatchedWithNullResult() throws SqlParseException {
String modelId = "d67bf0e4-30f4-9248-2528-52daa80be91a";
String sql = "select max(LO_ORDERPRIOTITY) from ssb.lineorder";
- List<OLAPContext> olapContexts =
OlapContextUtil.getOlapContexts(getProject(), sql);
+ List<OLAPContext> olapContexts =
OlapContextTestUtil.getOlapContexts(getProject(), sql);
OLAPContext olapContext = olapContexts.get(0);
NDataflow df = NDataflowManager.getInstance(getTestConfig(),
getProject()).getDataflow(modelId);
- Map<String, String> matchedJoinGraphAliasMap =
OlapContextUtil.matchJoins(df.getModel(), olapContext);
+ Map<String, String> matchedJoinGraphAliasMap =
OlapContextTestUtil.matchJoins(df.getModel(), olapContext);
olapContext.fixModel(df.getModel(), matchedJoinGraphAliasMap);
Candidate candidate = new Candidate(df, olapContext,
matchedJoinGraphAliasMap);
@@ -104,10 +104,10 @@ class VacantIndexPruningRuleTest extends
NLocalWithSparkSessionTest {
NDataModelManager modelMgr =
NDataModelManager.getInstance(getTestConfig(), getProject());
modelMgr.listAllModels().stream().filter(model -> !model.isBroken())
.forEach(model ->
cleanAlreadyExistingLayoutsInSegments(model.getId()));
- List<OLAPContext> olapContexts =
OlapContextUtil.getOlapContexts(getProject(), sql);
+ List<OLAPContext> olapContexts =
OlapContextTestUtil.getOlapContexts(getProject(), sql);
OLAPContext olapContext = olapContexts.get(0);
NDataflow df = NDataflowManager.getInstance(getTestConfig(),
getProject()).getDataflow(modelId);
- Map<String, String> matchedJoinGraphAliasMap =
OlapContextUtil.matchJoins(df.getModel(), olapContext);
+ Map<String, String> matchedJoinGraphAliasMap =
OlapContextTestUtil.matchJoins(df.getModel(), olapContext);
olapContext.fixModel(df.getModel(), matchedJoinGraphAliasMap);
Candidate candidate = new Candidate(df, olapContext,
matchedJoinGraphAliasMap);
@@ -132,10 +132,10 @@ class VacantIndexPruningRuleTest extends
NLocalWithSparkSessionTest {
void testUnmatchedAggIndex() throws SqlParseException {
String modelId = "d67bf0e4-30f4-9248-2528-52daa80be91a";
String sql = "select max(LO_ORDERPRIOTITY) from ssb.lineorder";
- List<OLAPContext> olapContexts =
OlapContextUtil.getOlapContexts(getProject(), sql);
+ List<OLAPContext> olapContexts =
OlapContextTestUtil.getOlapContexts(getProject(), sql);
OLAPContext olapContext = olapContexts.get(0);
NDataflow df = NDataflowManager.getInstance(getTestConfig(),
getProject()).getDataflow(modelId);
- Map<String, String> matchedJoinGraphAliasMap =
OlapContextUtil.matchJoins(df.getModel(), olapContext);
+ Map<String, String> matchedJoinGraphAliasMap =
OlapContextTestUtil.matchJoins(df.getModel(), olapContext);
olapContext.fixModel(df.getModel(), matchedJoinGraphAliasMap);
Candidate candidate = new Candidate(df, olapContext,
matchedJoinGraphAliasMap);
@@ -152,10 +152,10 @@ class VacantIndexPruningRuleTest extends
NLocalWithSparkSessionTest {
NDataModelManager modelMgr =
NDataModelManager.getInstance(getTestConfig(), getProject());
modelMgr.listAllModels().stream().filter(model -> !model.isBroken())
.forEach(model ->
cleanAlreadyExistingLayoutsInSegments(model.getId()));
- List<OLAPContext> olapContexts =
OlapContextUtil.getOlapContexts(getProject(), sql);
+ List<OLAPContext> olapContexts =
OlapContextTestUtil.getOlapContexts(getProject(), sql);
OLAPContext olapContext = olapContexts.get(0);
NDataflow df = NDataflowManager.getInstance(getTestConfig(),
getProject()).getDataflow(modelId);
- Map<String, String> matchedJoinGraphAliasMap =
OlapContextUtil.matchJoins(df.getModel(), olapContext);
+ Map<String, String> matchedJoinGraphAliasMap =
OlapContextTestUtil.matchJoins(df.getModel(), olapContext);
olapContext.fixModel(df.getModel(), matchedJoinGraphAliasMap);
Candidate candidate = new Candidate(df, olapContext,
matchedJoinGraphAliasMap);
@@ -175,10 +175,10 @@ class VacantIndexPruningRuleTest extends
NLocalWithSparkSessionTest {
NDataModelManager modelMgr =
NDataModelManager.getInstance(getTestConfig(), getProject());
modelMgr.listAllModels().stream().filter(model -> !model.isBroken())
.forEach(model ->
cleanAlreadyExistingLayoutsInSegments(model.getId()));
- List<OLAPContext> olapContexts =
OlapContextUtil.getOlapContexts(getProject(), sql);
+ List<OLAPContext> olapContexts =
OlapContextTestUtil.getOlapContexts(getProject(), sql);
OLAPContext olapContext = olapContexts.get(0);
NDataflow df = NDataflowManager.getInstance(getTestConfig(),
getProject()).getDataflow(modelId);
- Map<String, String> matchedJoinGraphAliasMap =
OlapContextUtil.matchJoins(df.getModel(), olapContext);
+ Map<String, String> matchedJoinGraphAliasMap =
OlapContextTestUtil.matchJoins(df.getModel(), olapContext);
olapContext.fixModel(df.getModel(), matchedJoinGraphAliasMap);
Candidate candidate = new Candidate(df, olapContext,
matchedJoinGraphAliasMap);
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextUtil.java
b/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
similarity index 82%
rename from
src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextUtil.java
rename to
src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
index ab9494b64a..c5462d0c12 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextUtil.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
@@ -21,21 +21,22 @@ package org.apache.kylin.util;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.function.Consumer;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.realization.NoRealizationFoundException;
+import
org.apache.kylin.metadata.realization.NoStreamingRealizationFoundException;
import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.RealizationChooser;
import org.apache.kylin.query.util.QueryContextCutter;
-import com.clearspring.analytics.util.Lists;
-
-public class OlapContextUtil {
+public class OlapContextTestUtil {
public static List<OLAPContext> getOlapContexts(String project, String
sql) throws SqlParseException {
return getOlapContexts(project, sql, false);
@@ -44,10 +45,10 @@ public class OlapContextUtil {
public static List<OLAPContext> getOlapContexts(String project, String
sql, boolean reCutBanned)
throws SqlParseException {
QueryExec queryExec = new QueryExec(project,
KylinConfig.getInstanceFromEnv());
- RelNode rel = queryExec.parseAndOptimize(sql);
try {
+ RelNode rel = queryExec.parseAndOptimize(sql);
QueryContextCutter.selectRealization(rel, reCutBanned);
- } catch (NoRealizationFoundException e) {
+ } catch (NoRealizationFoundException |
NoStreamingRealizationFoundException e) {
// When NoRealizationFoundException occurs, do nothing
// because we only need to obtain OlapContexts.
}
@@ -55,6 +56,18 @@ public class OlapContextUtil {
return getOlapContexts();
}
+ public static List<OLAPContext> getOlapContexts(String project, String
sql, boolean reCutBanned,
+ Consumer<NoRealizationFoundException> consumer) throws
SqlParseException {
+ QueryExec queryExec = new QueryExec(project,
KylinConfig.getInstanceFromEnv());
+ try {
+ RelNode rel = queryExec.parseAndOptimize(sql);
+ QueryContextCutter.selectRealization(rel, reCutBanned);
+ } catch (NoRealizationFoundException e) {
+ consumer.accept(e);
+ }
+ return getOlapContexts();
+ }
+
public static List<OLAPContext> getHepRulesOptimizedOlapContexts(String
project, String sql, boolean reCutBanned)
throws SqlParseException {
QueryExec queryExec = new QueryExec(project,
KylinConfig.getInstanceFromEnv());
diff --git
a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
index 0e4409bac4..ea4624cecb 100644
---
a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
+++
b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
@@ -32,6 +32,8 @@ import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.query.engine.exec.ExecuteResult;
@@ -48,9 +50,6 @@ import org.apache.kylin.query.runtime.SparkEngine;
import org.apache.kylin.query.util.QueryContextCutter;
import org.apache.spark.SparkException;
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -69,7 +68,12 @@ public class SparderQueryPlanExec implements QueryPlanExec {
public ExecuteResult executeToIterable(RelNode rel, MutableDataContext
dataContext) {
QueryContext.currentTrace().startSpan(QueryTrace.MODEL_MATCHING);
// select realizations
- selectRealization(rel);
+ ContextUtil.dumpCalcitePlan("EXECUTION PLAN BEFORE
(SparderQueryPlanExec) SELECT REALIZATION", rel, log);
+ QueryContext.current().record("end_plan");
+ QueryContext.current().getQueryTagInfo().setWithoutSyntaxError(true);
+
+ QueryContextCutter.selectRealization(rel,
BackdoorToggles.getIsQueryFromAutoModeling());
+ ContextUtil.dumpCalcitePlan("EXECUTION PLAN AFTER
(SparderQueryPlanExec) SELECT REALIZATION IS SET", rel, log);
val contexts = ContextUtil.listContexts();
for (OLAPContext context : contexts) {
@@ -93,18 +97,9 @@ public class SparderQueryPlanExec implements QueryPlanExec {
// rewrite
rewrite(rel);
- return doExecute(rel, dataContext);
- }
- /**
- * submit rel and dataContext to query engine
- * @param rel
- * @param dataContext
- * @return
- */
- private ExecuteResult doExecute(RelNode rel, DataContext dataContext) {
- QueryEngine queryEngine = new SparkEngine();
- return internalCompute(queryEngine, dataContext, rel.getInput(0));
+ // submit rel and dataContext to query engine
+ return internalCompute(new SparkEngine(), dataContext,
rel.getInput(0));
}
private static boolean forceTableIndexAtException(Exception e) {
@@ -153,18 +148,6 @@ public class SparderQueryPlanExec implements QueryPlanExec
{
return queryEngine.computeToIterable(dataContext, rel);
}
- /**
- * match cubes
- */
- private void selectRealization(RelNode rel) {
- ContextUtil.dumpCalcitePlan("EXECUTION PLAN BEFORE OLAPImplementor",
rel, log);
- QueryContext.current().record("end_plan");
-
- QueryContext.current().getQueryTagInfo().setWithoutSyntaxError(true);
- QueryContextCutter.selectRealization(rel,
BackdoorToggles.getIsQueryFromAutoModeling());
- ContextUtil.dumpCalcitePlan("EXECUTION PLAN AFTER REALIZATION IS SET",
rel, log);
- }
-
/**
* rewrite relNodes
*/
diff --git
a/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
b/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
index bd41205514..55006842b4 100644
---
a/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
+++
b/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
@@ -47,12 +47,14 @@ public class QueryContextCutter {
}
/**
- * For a parser tree of one query, there are 3 steps to get it matched
with pre-calculated realizations
- * 1. first-round cut the tree off several parts correspond to
OLAPContext, which make the part as big as possible
- * 2. collect contexts' info and choose lowest-cost realization
according to the context's info. if there are contexts cannot match
realizations, goto 3
- * 3. seek proper context-cut methods to ensure as many contexts as
possible match a realization, then goto 2.
- *
- * @return The cut OLAPContext with selected realizations, which is a
subset of OLAPContext.getThreadLocalContexts().
+ * For each query parse tree, the following steps are used for generating
OlapContexts
+ * and matching the precomputed indexes.
+ * <p> 1. The larger the OlapContext for the first cut, the better;</p>
+ * <p> 2. Traverse the RelNode operator tree to collect all attributes for
each split OlapContext;</p>
+ * <p> 3. Choose the most appropriate index for each OlapContext;</p>
+ * <p> 4. If there exists an OlapContext that does not match any index,
then use the re-cut strategy
+ * to get multiple smaller OlapContexts and use the previous steps
to continue matching.
+ * @return Each of the returned OlapContexts matches an index, or throws
an exception.
*/
public static List<OLAPContext> selectRealization(RelNode root, boolean
isReCutBanned) {
ContextInitialCutStrategy firstRoundStrategy = new
ContextInitialCutStrategy();
@@ -62,11 +64,16 @@ public class QueryContextCutter {
int retryCutTimes = 0;
while (retryCutTimes++ < MAX_RETRY_TIMES_OF_CONTEXT_CUT) {
try {
- return collectContextInfoAndSelectRealization(root);
+ fillOlapContextPropertiesWithRelTree(root);
+ List<OLAPContext> olapContexts = chooseCandidate();
+ if (isReCutBanned) {
+ throw new NoRealizationFoundException("There is no need to
select realizations for OlapContexts.");
+ }
+ return olapContexts;
} catch (NoRealizationFoundException |
NoStreamingRealizationFoundException e) {
if (isReCutBanned && e instanceof
NoStreamingRealizationFoundException) {
checkStreamingTableWithAutoModeling();
- } else if (isReCutBanned) {
+ } else if (isReCutBanned || e instanceof
NoStreamingRealizationFoundException) {
throw e;
}
reCutStrategy.tryCutToSmallerContexts(root, e);
@@ -88,18 +95,19 @@ public class QueryContextCutter {
throw new NoRealizationFoundException(errorMsg);
}
- private static List<OLAPContext>
collectContextInfoAndSelectRealization(RelNode queryRoot) {
+ private static void fillOlapContextPropertiesWithRelTree(RelNode
queryRoot) {
// post-order travel children
OLAPRel.OLAPImplementor kapImplementor = new OLAPRel.OLAPImplementor();
kapImplementor.visitChild(queryRoot.getInput(0), queryRoot);
QueryContext.current().record("collect_olap_context_info");
- // identify model
- List<OLAPContext> contexts = ContextUtil.listContextsHavingScan();
+ }
- for (OLAPContext olapContext : contexts) {
+ private static List<OLAPContext> chooseCandidate() {
+ List<OLAPContext> contexts = ContextUtil.listContextsHavingScan();
+ contexts.forEach(olapContext -> {
olapContext.setHasSelected(true);
log.info("Context for realization matching: {}", olapContext);
- }
+ });
long selectLayoutStartTime = System.currentTimeMillis();
if (contexts.size() > 1) {
