morrySnow commented on code in PR #24691:
URL: https://github.com/apache/doris/pull/24691#discussion_r1335422428
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java:
##########
@@ -359,4 +359,14 @@ public void initColumnNameMap() {
definedNameToColumn.put(normalizeName(column.getDefineName()),
column);
}
}
+
+ // rollup or base column not have define expr
+ public boolean haveDefineExpr() {
Review Comment:
not use?
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java:
##########
@@ -689,93 +685,52 @@ private SelectResult select(
Map<Boolean, List<MaterializedIndex>> indexesGroupByIsBaseOrNot =
table.getVisibleIndex()
.stream()
.collect(Collectors.groupingBy(index -> index.getId() ==
table.getBaseIndexId()));
- if (table.isDupKeysOrMergeOnWrite()) {
- // Duplicate-keys table could use base index and indexes that
pre-aggregation status is on.
- Set<MaterializedIndex> candidatesWithoutRewriting =
- indexesGroupByIsBaseOrNot.getOrDefault(false,
ImmutableList.of())
- .stream()
- .filter(index -> checkPreAggStatus(scan,
index.getId(), predicates,
- aggregateFunctions, groupingExprs).isOn())
- .collect(Collectors.toSet());
-
- // try to rewrite bitmap, hll by materialized index columns.
- List<AggRewriteResult> candidatesWithRewriting =
indexesGroupByIsBaseOrNot.getOrDefault(false,
- ImmutableList.of())
- .stream()
- .filter(index ->
!candidatesWithoutRewriting.contains(index))
- .map(index -> rewriteAgg(index, scan,
nonVirtualRequiredScanOutput, predicates,
- aggregateFunctions,
- groupingExprs))
- .filter(aggRewriteResult -> checkPreAggStatus(scan,
aggRewriteResult.index.getId(),
- predicates,
- // check pre-agg status of aggregate function that
couldn't rewrite.
- aggFuncsDiff(aggregateFunctions, aggRewriteResult),
- groupingExprs).isOn())
- .filter(result -> result.success)
- .collect(Collectors.toList());
-
- List<MaterializedIndex> haveAllRequiredColumns = Streams.concat(
- candidatesWithoutRewriting.stream()
- .filter(index -> containAllRequiredColumns(index,
scan, nonVirtualRequiredScanOutput,
- requiredExpr, predicates)),
- candidatesWithRewriting.stream()
- .filter(aggRewriteResult ->
containAllRequiredColumns(aggRewriteResult.index, scan,
- aggRewriteResult.requiredScanOutput,
- requiredExpr.stream().map(e ->
aggRewriteResult.exprRewriteMap.replaceAgg(e))
- .collect(Collectors.toSet()),
- predicates))
- .map(aggRewriteResult -> aggRewriteResult.index))
- .collect(Collectors.toList());
-
- long selectIndexId = selectBestIndex(haveAllRequiredColumns, scan,
predicates);
- Optional<AggRewriteResult> rewriteResultOpt =
candidatesWithRewriting.stream()
- .filter(aggRewriteResult -> aggRewriteResult.index.getId()
== selectIndexId)
- .findAny();
- // Pre-aggregation is set to `on` by default for duplicate-keys
table.
- return new SelectResult(PreAggStatus.on(), selectIndexId,
- rewriteResultOpt.map(r -> r.exprRewriteMap).orElse(new
ExprRewriteMap()));
- } else {
- if (scan.getPreAggStatus().isOff()) {
- return new SelectResult(scan.getPreAggStatus(),
- scan.getTable().getBaseIndexId(), new
ExprRewriteMap());
- }
- Set<MaterializedIndex> candidatesWithoutRewriting = new
HashSet<>();
+ if (!table.isDupKeysOrMergeOnWrite() &&
scan.getPreAggStatus().isOff()) {
+ return new SelectResult(scan.getPreAggStatus(),
scan.getTable().getBaseIndexId(), new ExprRewriteMap());
+ }
- for (MaterializedIndex index :
indexesGroupByIsBaseOrNot.getOrDefault(false, ImmutableList.of())) {
- final PreAggStatus preAggStatus;
- if (preAggEnabledByHint(scan)) {
- preAggStatus = PreAggStatus.on();
- } else {
- preAggStatus = checkPreAggStatus(scan, index.getId(),
predicates,
- aggregateFunctions, groupingExprs);
- }
+ Set<MaterializedIndex> candidatesWithoutRewriting =
indexesGroupByIsBaseOrNot
+ .getOrDefault(false, ImmutableList.of()).stream()
+ .filter(index -> preAggEnabledByHint(scan)
+ || checkPreAggStatus(scan, index.getId(), predicates,
aggregateFunctions, groupingExprs).isOn())
+ .collect(Collectors.toSet());
+
+ // try to rewrite bitmap, hll by materialized index columns.
+ List<AggRewriteResult> candidatesWithRewriting =
indexesGroupByIsBaseOrNot
+ .getOrDefault(false, ImmutableList.of()).stream()
+ .filter(index -> !candidatesWithoutRewriting.contains(index))
+ .map(index -> rewriteAgg(index, scan,
nonVirtualRequiredScanOutput, predicates, aggregateFunctions,
+ groupingExprs))
+ .filter(aggRewriteResult -> checkPreAggStatus(scan,
aggRewriteResult.index.getId(), predicates,
+ // check pre-agg status of aggregate function that
couldn't rewrite.
+ aggFuncsDiff(aggregateFunctions, aggRewriteResult),
groupingExprs).isOn())
+ .filter(result -> result.success).collect(Collectors.toList());
+
+ List<MaterializedIndex> haveAllRequiredColumns = Streams.concat(
+ candidatesWithoutRewriting.stream()
+ .filter(index -> containAllRequiredColumns(index,
scan, nonVirtualRequiredScanOutput,
+ requiredExpr, predicates)),
+ candidatesWithRewriting.stream()
+ .filter(aggRewriteResult ->
containAllRequiredColumns(aggRewriteResult.index, scan,
+ aggRewriteResult.requiredScanOutput,
+ requiredExpr.stream().map(e ->
aggRewriteResult.exprRewriteMap.replaceAgg(e))
+ .collect(Collectors.toSet()),
+ predicates))
+ .map(aggRewriteResult -> aggRewriteResult.index))
+ .collect(Collectors.toList());
- if (preAggStatus.isOn()) {
- candidatesWithoutRewriting.add(index);
- }
- }
- SelectResult baseIndexSelectResult = new SelectResult(
- checkPreAggStatus(scan, scan.getTable().getBaseIndexId(),
- predicates, aggregateFunctions, groupingExprs),
- scan.getTable().getBaseIndexId(), new ExprRewriteMap());
- if (candidatesWithoutRewriting.isEmpty()) {
- // return early if pre agg status if off.
- return baseIndexSelectResult;
- } else {
- List<MaterializedIndex> rollupsWithAllRequiredCols =
- Stream.concat(candidatesWithoutRewriting.stream(),
indexesGroupByIsBaseOrNot.get(true).stream())
- .filter(index ->
containAllRequiredColumns(index, scan, nonVirtualRequiredScanOutput,
- requiredExpr, predicates))
- .collect(Collectors.toList());
-
- long selectedIndex =
selectBestIndex(rollupsWithAllRequiredCols, scan, predicates);
- if (selectedIndex == scan.getTable().getBaseIndexId()) {
- return baseIndexSelectResult;
- }
- return new SelectResult(PreAggStatus.on(), selectedIndex, new
ExprRewriteMap());
- }
+ long selectIndexId = selectBestIndex(haveAllRequiredColumns, scan,
predicates);
+ if (!table.isDupKeysOrMergeOnWrite() && (new CheckContext(scan,
selectIndexId)).isBaseIndex()) {
Review Comment:
could u add some comments to explain what only check
`!table.isDupKeysOrMergeOnWrite()`
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java:
##########
@@ -689,93 +685,52 @@ private SelectResult select(
Map<Boolean, List<MaterializedIndex>> indexesGroupByIsBaseOrNot =
table.getVisibleIndex()
.stream()
.collect(Collectors.groupingBy(index -> index.getId() ==
table.getBaseIndexId()));
- if (table.isDupKeysOrMergeOnWrite()) {
- // Duplicate-keys table could use base index and indexes that
pre-aggregation status is on.
- Set<MaterializedIndex> candidatesWithoutRewriting =
- indexesGroupByIsBaseOrNot.getOrDefault(false,
ImmutableList.of())
- .stream()
- .filter(index -> checkPreAggStatus(scan,
index.getId(), predicates,
- aggregateFunctions, groupingExprs).isOn())
- .collect(Collectors.toSet());
-
- // try to rewrite bitmap, hll by materialized index columns.
- List<AggRewriteResult> candidatesWithRewriting =
indexesGroupByIsBaseOrNot.getOrDefault(false,
- ImmutableList.of())
- .stream()
- .filter(index ->
!candidatesWithoutRewriting.contains(index))
- .map(index -> rewriteAgg(index, scan,
nonVirtualRequiredScanOutput, predicates,
- aggregateFunctions,
- groupingExprs))
- .filter(aggRewriteResult -> checkPreAggStatus(scan,
aggRewriteResult.index.getId(),
- predicates,
- // check pre-agg status of aggregate function that
couldn't rewrite.
- aggFuncsDiff(aggregateFunctions, aggRewriteResult),
- groupingExprs).isOn())
- .filter(result -> result.success)
- .collect(Collectors.toList());
-
- List<MaterializedIndex> haveAllRequiredColumns = Streams.concat(
- candidatesWithoutRewriting.stream()
- .filter(index -> containAllRequiredColumns(index,
scan, nonVirtualRequiredScanOutput,
- requiredExpr, predicates)),
- candidatesWithRewriting.stream()
- .filter(aggRewriteResult ->
containAllRequiredColumns(aggRewriteResult.index, scan,
- aggRewriteResult.requiredScanOutput,
- requiredExpr.stream().map(e ->
aggRewriteResult.exprRewriteMap.replaceAgg(e))
- .collect(Collectors.toSet()),
- predicates))
- .map(aggRewriteResult -> aggRewriteResult.index))
- .collect(Collectors.toList());
-
- long selectIndexId = selectBestIndex(haveAllRequiredColumns, scan,
predicates);
- Optional<AggRewriteResult> rewriteResultOpt =
candidatesWithRewriting.stream()
- .filter(aggRewriteResult -> aggRewriteResult.index.getId()
== selectIndexId)
- .findAny();
- // Pre-aggregation is set to `on` by default for duplicate-keys
table.
- return new SelectResult(PreAggStatus.on(), selectIndexId,
- rewriteResultOpt.map(r -> r.exprRewriteMap).orElse(new
ExprRewriteMap()));
- } else {
- if (scan.getPreAggStatus().isOff()) {
- return new SelectResult(scan.getPreAggStatus(),
- scan.getTable().getBaseIndexId(), new
ExprRewriteMap());
- }
- Set<MaterializedIndex> candidatesWithoutRewriting = new
HashSet<>();
+ if (!table.isDupKeysOrMergeOnWrite() &&
scan.getPreAggStatus().isOff()) {
Review Comment:
could u add some comment to explain why add `scan.getPreAggStatus().isOff()`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
