yashmayya opened a new pull request, #17181: URL: https://github.com/apache/pinot/pull/17181
- This rule was added in https://github.com/apache/pinot/pull/16035. - For certain queries with complex predicates, this can trigger a recursive Calcite optimization path that is very expensive in terms of CPU. The symptom is the following stack trace in the planner thread: ``` org.apache.calcite.rex.RexUtil.isConstant(org.apache.calcite.rex.RexNode) at void org.apache.calcite.rex.RexUtil.gatherConstraint(java.lang.Class, org.apache.calcite.rex.RexNode, org.apache.calcite.rex.RexNode, java.util.Map, java.util.Set, org.apache.calcite.rex.RexBuilder) at void org.apache.calcite.rex.RexUtil.gatherConstraints(java.lang.Class, org.apache.calcite.rex.RexNode, java.util.Map, java.util.Set, org.apache.calcite.rex.RexBuilder) at org.apache.pinot.shaded.com.google.common.collect.ImmutableMap org.apache.calcite.rex.RexUtil.predicateConstants(java.lang.Class, org.apache.calcite.rex.RexBuilder, java.util.List) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.plan.RelOptPredicateList.of(org.apache.calcite.rex.RexBuilder, java.lang.Iterable, java.lang.Iterable, java.lang.Iterable) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.plan.RelOptPredicateList.union(org.apache.calcite.rex.RexBuilder, org.apache.calcite.plan.RelOptPredicateList) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMdPredicates.getPredicates(org.apache.calcite.rel.core.Filter, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates_$(org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler, org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates(org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMetadataQuery.getPulledUpPredicates(org.apache.calcite.rel.RelNode) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMdPredicates.getPredicates(org.apache.calcite.rel.core.Filter, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates_$(org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler, org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates(org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMetadataQuery.getPulledUpPredicates(org.apache.calcite.rel.RelNode) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMdPredicates.getPredicates(org.apache.calcite.rel.core.Filter, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates_$(org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler, org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates(org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMetadataQuery.getPulledUpPredicates(org.apache.calcite.rel.RelNode) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMdPredicates.getPredicates(org.apache.calcite.rel.core.Filter, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates_$(org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler, org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates(org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMetadataQuery.getPulledUpPredicates(org.apache.calcite.rel.RelNode) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMdPredicates.getPredicates(org.apache.calcite.rel.core.Filter, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates_$(org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler, org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates(org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMetadataQuery.getPulledUpPredicates(org.apache.calcite.rel.RelNode) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMdPredicates.getPredicates(org.apache.calcite.rel.core.Filter, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates_$(org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler, org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates(org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMetadataQuery.getPulledUpPredicates(org.apache.calcite.rel.RelNode) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMdPredicates.getPredicates(org.apache.calcite.rel.core.Join, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates_$(org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler, org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.janino.GeneratedMetadata_PredicatesHandler.getPredicates(org.apache.calcite.rel.RelNode, org.apache.calcite.rel.metadata.RelMetadataQuery) at org.apache.calcite.plan.RelOptPredicateList org.apache.calcite.rel.metadata.RelMetadataQuery.getPulledUpPredicates(org.apache.calcite.rel.RelNode) at void org.apache.pinot.calcite.rel.rules.PinotJoinPushTransitivePredicatesRule.onMatch(org.apache.calcite.plan.RelOptRuleCall) at void org.apache.calcite.plan.AbstractRelOptPlanner.fireRule(org.apache.calcite.plan.RelOptRuleCall) at org.apache.calcite.plan.hep.HepRelVertex org.apache.calcite.plan.hep.HepPlanner.applyRule(org.apache.calcite.plan.RelOptRule, org.apache.calcite.plan.hep.HepRelVertex, boolean) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) at int org.apache.calcite.plan.hep.HepPlanner.depthFirstApply(org.apache.calcite.plan.hep.HepProgram$State, java.util.Iterator, java.util.Collection, boolean, int) ``` - In certain environments and query workloads, this caused the broker CPU to jump from ~5% to ~70% alongside query timeouts during planning. - This patch disables the rule by default to prevent such performance regressions. It can be enabled on a case by case basis using the query option `usePlannerRules`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
