From f68f7256d045b9e1383bd1b3bad94d32c79183f3 Mon Sep 17 00:00:00 2001 From: Seaven Date: Thu, 16 Jan 2025 11:24:09 +0800 Subject: [PATCH 1/3] [Refactor] Refactor optimizer interface Signed-off-by: Seaven --- .../authorization/ColumnPrivilege.java | 8 +- .../java/com/starrocks/sql/DeletePlanner.java | 7 +- .../java/com/starrocks/sql/InsertPlanner.java | 7 +- .../com/starrocks/sql/StatementPlanner.java | 37 ++- .../java/com/starrocks/sql/UpdatePlanner.java | 11 +- .../sql/analyzer/AnalyzeStmtAnalyzer.java | 7 +- .../analyzer/MaterializedViewAnalyzer.java | 9 +- .../starrocks/sql/optimizer/Optimizer.java | 123 +++----- .../sql/optimizer/OptimizerConfig.java | 13 +- .../sql/optimizer/OptimizerContext.java | 284 ++++++++---------- .../sql/optimizer/OptimizerFactory.java | 59 ++++ .../sql/optimizer/OptimizerIterface.java | 17 ++ .../sql/optimizer/dump/QueryDumpInfo.java | 2 +- .../rewrite/JoinPredicatePushdown.java | 4 +- .../materialization/MvUtils.java | 16 +- .../sql/optimizer/task/ApplyRuleTask.java | 2 - .../sql/optimizer/task/TaskContext.java | 14 - .../sql/optimizer/task/TaskScheduler.java | 27 +- .../validate/MVRewriteValidator.java | 18 +- .../connector/hive/HiveMetadataTest.java | 4 +- .../hive/HiveStatisticsProviderTest.java | 4 +- .../iceberg/IcebergMetadataTest.java | 8 +- .../connector/paimon/PaimonMetadataTest.java | 5 +- .../sql/optimizer/OptimizerTaskTest.java | 184 ++++++------ .../sql/optimizer/rule/BinderTest.java | 7 +- .../OlapScanImplementationRuleTest.java | 7 +- .../DistributionPrunerRuleTest.java | 5 +- .../MergeLimitWithSortRuleTest.java | 5 +- .../PartitionPruneRuleTest.java | 11 +- .../transformation/PushDownAggRuleTest.java | 5 +- .../transformation/PushDownScanRuleTest.java | 5 +- .../materialization/MVTestBase.java | 7 +- .../MvRewritePreprocessorTest.java | 74 +++-- .../MvRewriteStrategyTest.java | 12 +- .../statistics/StatisticsCalculatorTest.java | 4 +- .../sql/plan/PartitionPruneTest.java | 4 +- .../com/starrocks/utframe/UtFrameUtils.java | 12 +- 37 files changed, 506 insertions(+), 522 deletions(-) create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerIterface.java diff --git a/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java b/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java index 843a3e30a3d8e..92575b70bb8e0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java +++ b/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java @@ -38,6 +38,7 @@ import com.starrocks.sql.optimizer.OptExpressionVisitor; import com.starrocks.sql.optimizer.Optimizer; import com.starrocks.sql.optimizer.OptimizerConfig; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; @@ -114,9 +115,10 @@ public static void check(ConnectContext context, QueryStatement stmt, List outputColumns, LogicalPlan logicalPlan, ColumnRefFactory columnRefFactory, QueryRelation queryRelation, Table targetTable) { - Optimizer optimizer = new Optimizer(); PhysicalPropertySet requiredPropertySet = createPhysicalPropertySet(insertStmt, outputColumns, session.getSessionVariable()); OptExpression optimizedPlan; try (Timer ignore2 = Tracers.watchScope("Optimizer")) { + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.initContext(session, columnRefFactory)); optimizedPlan = optimizer.optimize( - session, logicalPlan.getRoot(), requiredPropertySet, - new ColumnRefSet(logicalPlan.getOutputColumn()), - columnRefFactory); + new ColumnRefSet(logicalPlan.getOutputColumn())); } //8. Build fragment exec plan diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java index ababf5b585c2c..3e44deabec4e8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java @@ -60,6 +60,8 @@ import com.starrocks.sql.common.UnsupportedException; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; +import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.OptimizerTraceUtil; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; @@ -234,7 +236,7 @@ private static ExecPlan createQueryPlan(StatementBase stmt, ConnectContext session, TResultSinkType resultSinkType) { QueryStatement queryStmt = (QueryStatement) stmt; - QueryRelation query = (QueryRelation) queryStmt.getQueryRelation(); + QueryRelation query = queryStmt.getQueryRelation(); List colNames = query.getColumnOutputNames(); // 1. Build Logical plan ColumnRefFactory columnRefFactory = new ColumnRefFactory(); @@ -252,15 +254,14 @@ private static ExecPlan createQueryPlan(StatementBase stmt, OptExpression optimizedPlan; try (Timer ignored = Tracers.watchScope("Optimizer")) { // 2. Optimize logical plan and build physical plan - Optimizer optimizer = new Optimizer(); - optimizedPlan = optimizer.optimize( - session, - root, - mvTransformerContext, - stmt, + OptimizerContext optimizerContext = OptimizerFactory.initContext(session, columnRefFactory); + optimizerContext.setMvTransformerContext(mvTransformerContext); + optimizerContext.setStatement(stmt); + + Optimizer optimizer = OptimizerFactory.create(optimizerContext); + optimizedPlan = optimizer.optimize(root, new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), - columnRefFactory); + new ColumnRefSet(logicalPlan.getOutputColumn())); } try (Timer ignored = Tracers.watchScope("ExecPlanBuild")) { @@ -316,21 +317,19 @@ public static ExecPlan createQueryPlanWithReTry(QueryStatement queryStmt, OptExpression optimizedPlan; try (Timer ignored = Tracers.watchScope("Optimizer")) { + OptimizerContext optimizerContext = OptimizerFactory.initContext(session, columnRefFactory); // 2. Optimize logical plan and build physical plan - Optimizer optimizer = new Optimizer(); // FIXME: refactor this into Optimizer.optimize() method. // set query tables into OptimizeContext so can be added for mv rewrite if (Config.skip_whole_phase_lock_mv_limit >= 0) { - optimizer.setQueryTables(olapTables); + optimizerContext.setQueryTables(olapTables); } - optimizedPlan = optimizer.optimize( - session, - root, - mvTransformerContext, - queryStmt, - new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), - columnRefFactory); + optimizerContext.setMvTransformerContext(mvTransformerContext); + optimizerContext.setStatement(queryStmt); + + Optimizer optimizer = OptimizerFactory.create(optimizerContext); + optimizedPlan = optimizer.optimize(root, new PhysicalPropertySet(), + new ColumnRefSet(logicalPlan.getOutputColumn())); } try (Timer ignored = Tracers.watchScope("ExecPlanBuild")) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/UpdatePlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/UpdatePlanner.java index 6cbfdcde33ceb..82c42b0ea08b7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/UpdatePlanner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/UpdatePlanner.java @@ -40,6 +40,8 @@ import com.starrocks.sql.ast.UpdateStmt; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; +import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; @@ -95,15 +97,14 @@ public ExecPlan plan(UpdateStmt updateStmt, ConnectContext session) { session.getSessionVariable().setEnableLocalShuffleAgg(false); long tableId = targetTable.getId(); - Optimizer optimizer = new Optimizer(); - optimizer.setUpdateTableId(tableId); + OptimizerContext optimizerContext = OptimizerFactory.initContext(session, columnRefFactory); + optimizerContext.setUpdateTableId(tableId); + Optimizer optimizer = OptimizerFactory.create(optimizerContext); OptExpression optimizedPlan = optimizer.optimize( - session, optExprBuilder.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(outputColumns), - columnRefFactory); + new ColumnRefSet(outputColumns)); ExecPlan execPlan = PlanFragmentBuilder.createPhysicalPlan(optimizedPlan, session, outputColumns, columnRefFactory, colNames, TResultSinkType.MYSQL_PROTOCAL, false); DescriptorTable descriptorTable = execPlan.getDescTbl(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java index e205e66655325..90ce27d270843 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java @@ -42,9 +42,7 @@ import com.starrocks.sql.ast.DropStatsStmt; import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.common.MetaUtils; -import com.starrocks.sql.optimizer.Memo; -import com.starrocks.sql.optimizer.OptimizerConfig; -import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.statistics.Statistics; import com.starrocks.statistic.StatisticUtils; @@ -381,8 +379,7 @@ private void analyzeAnalyzeTypeDesc(ConnectContext session, StatementBase statem } Statistics tableStats = session.getGlobalStateMgr().getMetadataMgr(). - getTableStatistics(new OptimizerContext(new Memo(), new ColumnRefFactory(), session, - OptimizerConfig.defaultConfig()), + getTableStatistics(OptimizerFactory.initContext(session, new ColumnRefFactory()), tableName.getCatalog(), analyzeTable, Maps.newHashMap(), keys, null); totalRows = tableStats.getOutputRowCount(); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java index b3ccf1800ae66..231d5c7702e37 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java @@ -96,6 +96,8 @@ import com.starrocks.sql.common.PListCell; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; +import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; @@ -457,14 +459,13 @@ private void planMVQuery(CreateMaterializedViewStatement createStmt, QueryStatem // Build logical plan for view query OptExprBuilder optExprBuilder = logicalPlan.getRootBuilder(); logicalPlan = new LogicalPlan(optExprBuilder, outputColumns, logicalPlan.getCorrelation()); - Optimizer optimizer = new Optimizer(); + OptimizerContext optimizerContext = OptimizerFactory.initContext(ctx, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(optimizerContext); PhysicalPropertySet requiredPropertySet = PhysicalPropertySet.EMPTY; OptExpression optimizedPlan = optimizer.optimize( - ctx, logicalPlan.getRoot(), requiredPropertySet, - new ColumnRefSet(logicalPlan.getOutputColumn()), - columnRefFactory); + new ColumnRefSet(logicalPlan.getOutputColumn())); optimizedPlan.deriveMVProperty(); // TODO: refine rules for mv plan diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java index 003e5fd4a4be3..86291f8c17175 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java @@ -19,7 +19,6 @@ import com.google.common.collect.Lists; import com.starrocks.analysis.JoinOperator; import com.starrocks.catalog.MaterializedView; -import com.starrocks.catalog.OlapTable; import com.starrocks.common.profile.Timer; import com.starrocks.common.profile.Tracers; import com.starrocks.qe.ConnectContext; @@ -27,7 +26,6 @@ import com.starrocks.qe.feedback.OperatorTuningGuides; import com.starrocks.qe.feedback.PlanTuningAdvisor; import com.starrocks.sql.Explain; -import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; @@ -120,14 +118,12 @@ import com.starrocks.sql.optimizer.task.PrepareCollectMetaTask; import com.starrocks.sql.optimizer.task.TaskContext; import com.starrocks.sql.optimizer.task.TaskScheduler; -import com.starrocks.sql.optimizer.transformer.MVTransformerContext; import com.starrocks.sql.optimizer.validate.MVRewriteValidator; import com.starrocks.sql.optimizer.validate.OptExpressionValidator; import com.starrocks.sql.optimizer.validate.PlanValidator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; @@ -143,26 +139,17 @@ */ public class Optimizer { private static final Logger LOG = LogManager.getLogger(Optimizer.class); - private OptimizerContext context; - private final OptimizerConfig optimizerConfig; + private final OptimizerContext context; + private OptimizerConfig optimizerConfig; private MvRewriteStrategy mvRewriteStrategy = new MvRewriteStrategy(); - private TaskScheduler scheduler; + private final TaskScheduler scheduler = new TaskScheduler(); + private Memo memo; - private long updateTableId = -1; + // collect all LogicalOlapScanOperators in the query before any optimization + private final List allLogicalOlapScanOperators = Lists.newArrayList(); - private Set queryTables; - - public Optimizer() { - this(OptimizerConfig.defaultConfig()); - } - - public Optimizer(OptimizerConfig config) { - this.optimizerConfig = config; - } - - @VisibleForTesting - public OptimizerConfig getOptimizerConfig() { - return optimizerConfig; + Optimizer(OptimizerContext context) { + this.context = context; } public OptimizerContext getContext() { @@ -174,52 +161,50 @@ public MvRewriteStrategy getMvRewriteStrategy() { return mvRewriteStrategy; } - public OptExpression optimize(ConnectContext connectContext, - OptExpression logicOperatorTree, - PhysicalPropertySet requiredProperty, - ColumnRefSet requiredColumns, - ColumnRefFactory columnRefFactory) { - return optimize(connectContext, logicOperatorTree, null, null, requiredProperty, - requiredColumns, columnRefFactory); + private void prepare(OptExpression logicOperatorTree) { + optimizerConfig = context.getOptimizerConfig(); + + if (!optimizerConfig.isRuleBased()) { + memo = new Memo(); + context.setMemo(memo); + } + context.setTaskScheduler(scheduler); + + // collect all olap scan operator + Utils.extractOperator(logicOperatorTree, allLogicalOlapScanOperators, + op -> op instanceof LogicalOlapScanOperator); + } + + public OptExpression optimize(OptExpression logicOperatorTree, ColumnRefSet requiredColumns) { + return optimize(logicOperatorTree, new PhysicalPropertySet(), requiredColumns); } - public OptExpression optimize(ConnectContext connectContext, - OptExpression logicOperatorTree, - MVTransformerContext mvTransformerContext, - StatementBase stmt, - PhysicalPropertySet requiredProperty, - ColumnRefSet requiredColumns, - ColumnRefFactory columnRefFactory) { + public OptExpression optimize(OptExpression logicOperatorTree, PhysicalPropertySet requiredProperty, + ColumnRefSet requiredColumns) { try { // prepare for optimizer - prepare(connectContext, columnRefFactory, logicOperatorTree); + prepare(logicOperatorTree); // prepare for mv rewrite - prepareMvRewrite(connectContext, logicOperatorTree, columnRefFactory, requiredColumns); + prepareMvRewrite(context.getConnectContext(), logicOperatorTree, context.getColumnRefFactory(), + requiredColumns); try (Timer ignored = Tracers.watchScope("MVTextRewrite")) { - logicOperatorTree = new TextMatchBasedRewriteRule(connectContext, stmt, mvTransformerContext) - .transform(logicOperatorTree, context).get(0); + logicOperatorTree = new TextMatchBasedRewriteRule(context.getConnectContext(), context.getStatement(), + context.getMvTransformerContext()).transform(logicOperatorTree, context).get(0); } OptExpression result = optimizerConfig.isRuleBased() ? optimizeByRule(logicOperatorTree, requiredProperty, requiredColumns) : - optimizeByCost(connectContext, logicOperatorTree, requiredProperty, requiredColumns); + optimizeByCost(context.getConnectContext(), logicOperatorTree, requiredProperty, + requiredColumns); return result; } finally { // make sure clear caches in OptimizerContext - context.clear(); - connectContext.setQueryMVContext(null); + context.getQueryMaterializationContext().clear(); + context.getConnectContext().setQueryMVContext(null); } } - public void setQueryTables(Set queryTables) { - this.queryTables = queryTables; - } - - public void setUpdateTableId(long updateTableId) { - this.updateTableId = updateTableId; - } - // Optimize by rule will return logical plan. // Used by materialized view query rewrite optimization. private OptExpression optimizeByRule(OptExpression logicOperatorTree, @@ -250,7 +235,6 @@ private OptExpression optimizeByCost(ConnectContext connectContext, // Phase 1: none OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", logicOperatorTree); // Phase 2: rewrite based on memo and group - Memo memo = context.getMemo(); TaskContext rootTaskContext = new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE); @@ -262,6 +246,7 @@ private OptExpression optimizeByCost(ConnectContext connectContext, return logicOperatorTree; } + Preconditions.checkNotNull(memo); memo.init(logicOperatorTree); if (context.getQueryMaterializationContext() != null) { // LogicalTreeWithView is logically equivalent to logicOperatorTree @@ -310,20 +295,20 @@ private OptExpression optimizeByCost(ConnectContext connectContext, } // collect all mv scan operator - collectAllPhysicalOlapScanOperators(result, rootTaskContext); - List mvScan = rootTaskContext.getAllPhysicalOlapScanOperators().stream(). + List mvScan = collectAllPhysicalOlapScanOperators(result).stream(). filter(scan -> scan.getTable().isMaterializedView()).collect(Collectors.toList()); // add mv db id to currentSqlDbIds, the resource group could use this to distinguish sql patterns - Set currentSqlDbIds = rootTaskContext.getOptimizerContext().getCurrentSqlDbIds(); + Set currentSqlDbIds = context.getConnectContext().getCurrentSqlDbIds(); mvScan.stream().map(scan -> ((MaterializedView) scan.getTable()).getDbId()).forEach(currentSqlDbIds::add); try (Timer ignored = Tracers.watchScope("PlanValidate")) { // valid the final plan PlanValidator.getInstance().validatePlan(finalPlan, rootTaskContext); // validate mv and log tracer if needed - MVRewriteValidator.getInstance().validateMV(connectContext, finalPlan, rootTaskContext); + MVRewriteValidator mvRewriteValidator = new MVRewriteValidator(allLogicalOlapScanOperators); + mvRewriteValidator.validateMV(connectContext, finalPlan, rootTaskContext); // audit mv - MVRewriteValidator.getInstance().auditMv(connectContext, finalPlan, rootTaskContext); + mvRewriteValidator.auditMv(connectContext, finalPlan, rootTaskContext); return finalPlan; } } @@ -336,23 +321,7 @@ private void addViewBasedPlanIntoMemo(OptExpression logicalTreeWithView) { memo.copyIn(memo.getRootGroup(), logicalTreeWithView); } - private void prepare(ConnectContext connectContext, - ColumnRefFactory columnRefFactory, - OptExpression logicOperatorTree) { - Memo memo = null; - if (!optimizerConfig.isRuleBased()) { - memo = new Memo(); - } - - context = new OptimizerContext(memo, columnRefFactory, connectContext, optimizerConfig); - context.setQueryTables(queryTables); - context.setUpdateTableId(updateTableId); - this.scheduler = context.getTaskScheduler(); - // collect all olap scan operator - collectAllLogicalOlapScanOperators(logicOperatorTree, context); - - } private void prepareMvRewrite(ConnectContext connectContext, OptExpression logicOperatorTree, ColumnRefFactory columnRefFactory, ColumnRefSet requiredColumns) { @@ -1025,16 +994,10 @@ private OptExpression extractBestPlan(PhysicalPropertySet requiredProperty, return expression; } - private void collectAllLogicalOlapScanOperators(OptExpression tree, OptimizerContext optimizerContext) { - List list = Lists.newArrayList(); - Utils.extractOperator(tree, list, op -> op instanceof LogicalOlapScanOperator); - optimizerContext.setAllLogicalOlapScanOperators(Collections.unmodifiableList(list)); - } - - private void collectAllPhysicalOlapScanOperators(OptExpression tree, TaskContext rootTaskContext) { + private List collectAllPhysicalOlapScanOperators(OptExpression tree) { List list = Lists.newArrayList(); Utils.extractOperator(tree, list, op -> op instanceof PhysicalOlapScanOperator); - rootTaskContext.setAllPhysicalOlapScanOperators(Collections.unmodifiableList(list)); + return list; } private void prepareMetaOnlyOnce(OptExpression tree, TaskContext rootTaskContext) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java index 2839d980c6b08..065e017f5b867 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package com.starrocks.sql.optimizer; import com.starrocks.sql.optimizer.rule.RuleType; @@ -29,12 +28,6 @@ public enum OptimizerAlgorithm { private final BitSet ruleSwitches; - private static final OptimizerConfig DEFAULT_CONFIG = new OptimizerConfig(); - - public static OptimizerConfig defaultConfig() { - return DEFAULT_CONFIG; - } - public OptimizerConfig() { this(OptimizerAlgorithm.COST_BASED); } @@ -56,4 +49,10 @@ public void disableRule(RuleType ruleType) { public boolean isRuleDisable(RuleType ruleType) { return !ruleSwitches.get(ruleType.ordinal()); } + + private static final OptimizerConfig DEFAULT_CONFIG = new OptimizerConfig(); + + public static OptimizerConfig defaultConfig() { + return DEFAULT_CONFIG; + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java index 7b7c5d77e7723..7f7c3136ccf91 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java @@ -14,27 +14,25 @@ package com.starrocks.sql.optimizer; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.starrocks.catalog.OlapTable; import com.starrocks.qe.ConnectContext; import com.starrocks.qe.SessionVariable; -import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.common.ErrorType; import com.starrocks.sql.common.StarRocksPlannerException; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.dump.DumpInfo; -import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.operator.scalar.IsNullPredicateOperator; import com.starrocks.sql.optimizer.rewrite.JoinPredicatePushdown; import com.starrocks.sql.optimizer.rule.RuleSet; import com.starrocks.sql.optimizer.rule.RuleType; import com.starrocks.sql.optimizer.task.TaskContext; import com.starrocks.sql.optimizer.task.TaskScheduler; +import com.starrocks.sql.optimizer.transformer.MVTransformerContext; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -42,228 +40,175 @@ import java.util.concurrent.TimeUnit; public class OptimizerContext { - private final UUID queryId; - private final Memo memo; + // ============================ Query ============================ + private StatementBase statement; + private ConnectContext connectContext; + private ColumnRefFactory columnRefFactory; + private Set queryTables; + private long updateTableId = -1; + + private OptimizerConfig optimizerConfig; + + // ============================ Optimizer ============================ + private Memo memo; private final RuleSet ruleSet; - private final GlobalStateMgr globalStateMgr; - private final TaskScheduler taskScheduler; - private final ColumnRefFactory columnRefFactory; - private SessionVariable sessionVariable; - private DumpInfo dumpInfo; - private Set currentSqlDbIds; - private CTEContext cteContext; - private TaskContext currentTaskContext; - private final OptimizerConfig optimizerConfig; + private TaskScheduler taskScheduler; - private Set queryTables; + private final CTEContext cteContext; + private TaskContext currentTaskContext; + private final QueryMaterializationContext queryMaterializationContext = new QueryMaterializationContext(); - private long updateTableId = -1; + private MVTransformerContext mvTransformerContext; - private boolean isObtainedFromInternalStatistics = false; + // uniquePartitionIdGenerator for external catalog + private long uniquePartitionIdGenerator = 0L; private final Stopwatch optimizerTimer = Stopwatch.createStarted(); private final Map ruleWatchMap = Maps.newHashMap(); + // ============================ Task Variables ============================ // The context for join predicate pushdown rule - private JoinPredicatePushdown.JoinPredicatePushDownContext joinPredicatePushDownContext = + private final JoinPredicatePushdown.JoinPredicatePushDownContext joinPredicatePushDownContext = new JoinPredicatePushdown.JoinPredicatePushDownContext(); // QueryMaterializationContext is different from MaterializationContext that it keeps the context during the query // lifecycle instead of per materialized view. - private QueryMaterializationContext queryMaterializationContext = new QueryMaterializationContext(); + private boolean isObtainedFromInternalStatistics = false; private boolean isShortCircuit = false; private boolean inMemoPhase = false; // Is not null predicate can be derived from inner join or semi join, // which should be kept to be used to convert outer join into inner join. - private List pushdownNotNullPredicates = Lists.newArrayList(); - - // uniquePartitionIdGenerator for external catalog - private long uniquePartitionIdGenerator = 0L; + private final List pushdownNotNullPredicates = Lists.newArrayList(); - // collect all LogicalOlapScanOperators in the query before any optimization - private List allLogicalOlapScanOperators; - - @VisibleForTesting - public OptimizerContext(Memo memo, ColumnRefFactory columnRefFactory) { - this.memo = memo; + OptimizerContext(ConnectContext context) { + this.connectContext = context; this.ruleSet = new RuleSet(); - this.globalStateMgr = GlobalStateMgr.getCurrentState(); - this.taskScheduler = TaskScheduler.create(); - this.columnRefFactory = columnRefFactory; - this.sessionVariable = GlobalStateMgr.getCurrentState().getVariableMgr().newSessionVariable(); - this.optimizerConfig = new OptimizerConfig(); - this.queryId = UUID.randomUUID(); - this.allLogicalOlapScanOperators = Collections.emptyList(); - } - - @VisibleForTesting - public OptimizerContext(Memo memo, ColumnRefFactory columnRefFactory, ConnectContext connectContext) { - this(memo, columnRefFactory, connectContext, OptimizerConfig.defaultConfig()); - } - - public OptimizerContext(Memo memo, ColumnRefFactory columnRefFactory, ConnectContext connectContext, - OptimizerConfig optimizerConfig) { - this.memo = memo; - this.ruleSet = new RuleSet(); - this.globalStateMgr = GlobalStateMgr.getCurrentState(); - this.taskScheduler = TaskScheduler.create(); - this.columnRefFactory = columnRefFactory; - this.queryId = connectContext.getQueryId(); - this.sessionVariable = connectContext.getSessionVariable(); - this.dumpInfo = connectContext.getDumpInfo(); - this.currentSqlDbIds = connectContext.getCurrentSqlDbIds(); this.cteContext = new CTEContext(); cteContext.reset(); - this.cteContext.setEnableCTE(sessionVariable.isCboCteReuse()); - this.cteContext.setInlineCTERatio(sessionVariable.getCboCTERuseRatio()); - this.cteContext.setMaxCTELimit(sessionVariable.getCboCTEMaxLimit()); - this.optimizerConfig = optimizerConfig; + this.cteContext.setEnableCTE(getSessionVariable().isCboCteReuse()); + this.cteContext.setInlineCTERatio(getSessionVariable().getCboCTERuseRatio()); + this.cteContext.setMaxCTELimit(getSessionVariable().getCboCTEMaxLimit()); + + this.optimizerConfig = OptimizerConfig.defaultConfig(); } - public Memo getMemo() { - return memo; + // ============================ Query ============================ + public StatementBase getStatement() { + return statement; } - public RuleSet getRuleSet() { - return ruleSet; + public void setStatement(StatementBase statement) { + this.statement = statement; } - public GlobalStateMgr getCatalog() { - return globalStateMgr; + public ConnectContext getConnectContext() { + return connectContext; } - public TaskScheduler getTaskScheduler() { - return taskScheduler; + public void setConnectContext(ConnectContext connectContext) { + this.connectContext = connectContext; } - public ColumnRefFactory getColumnRefFactory() { - return columnRefFactory; + public UUID getQueryId() { + return connectContext.getQueryId(); } public final SessionVariable getSessionVariable() { - return sessionVariable; + return connectContext.getSessionVariable(); } - public void setSessionVariable(SessionVariable sessionVariable) { - this.sessionVariable = sessionVariable; + public DumpInfo getDumpInfo() { + return connectContext.getDumpInfo(); } - public DumpInfo getDumpInfo() { - return dumpInfo; + public void setColumnRefFactory(ColumnRefFactory columnRefFactory) { + this.columnRefFactory = columnRefFactory; } - public Set getCurrentSqlDbIds() { - return currentSqlDbIds; + public ColumnRefFactory getColumnRefFactory() { + return columnRefFactory; } - public CTEContext getCteContext() { - return cteContext; + public Set getQueryTables() { + return queryTables; } - public void setTaskContext(TaskContext context) { - this.currentTaskContext = context; + public void setQueryTables(Set queryTables) { + this.queryTables = queryTables; } - public TaskContext getTaskContext() { - return currentTaskContext; + public void setUpdateTableId(long updateTableId) { + this.updateTableId = updateTableId; } - public UUID getQueryId() { - return queryId; + public long getUpdateTableId() { + return updateTableId; } public OptimizerConfig getOptimizerConfig() { return optimizerConfig; } - /** - * Get all valid candidate materialized views for the query: - * - The materialized view is valid to rewrite by rule(SPJG) - * - The materialized view's refresh-ness is valid to rewrite. - */ - public List getCandidateMvs() { - return queryMaterializationContext.getValidCandidateMVs(); + public void setOptimizerConfig(OptimizerConfig optimizerConfig) { + this.optimizerConfig = optimizerConfig; } - public JoinPredicatePushdown.JoinPredicatePushDownContext getJoinPushDownParams() { - return joinPredicatePushDownContext; + // ============================ Optimizer ============================ + public Memo getMemo() { + return memo; } - public void setUpdateTableId(long updateTableId) { - this.updateTableId = updateTableId; + public void setMemo(Memo memo) { + this.memo = memo; } - public long getUpdateTableId() { - return updateTableId; + public RuleSet getRuleSet() { + return ruleSet; } - public long optimizerElapsedMs() { - return optimizerTimer.elapsed(TimeUnit.MILLISECONDS); + public TaskScheduler getTaskScheduler() { + return taskScheduler; } - public Stopwatch getStopwatch(RuleType ruleType) { - return ruleWatchMap.computeIfAbsent(ruleType, (k) -> Stopwatch.createStarted()); + public void setTaskScheduler(TaskScheduler taskScheduler) { + this.taskScheduler = taskScheduler; } - public boolean ruleExhausted(RuleType ruleType) { - Stopwatch watch = getStopwatch(ruleType); - long elapsed = watch.elapsed(TimeUnit.MILLISECONDS); - long timeLimit = Math.min(sessionVariable.getOptimizerMaterializedViewTimeLimitMillis(), - sessionVariable.getOptimizerExecuteTimeout()); - return elapsed > timeLimit; + public CTEContext getCteContext() { + return cteContext; } - public boolean isObtainedFromInternalStatistics() { - return isObtainedFromInternalStatistics; + public void setTaskContext(TaskContext context) { + this.currentTaskContext = context; } - public void setObtainedFromInternalStatistics(boolean obtainedFromInternalStatistics) { - isObtainedFromInternalStatistics = obtainedFromInternalStatistics; + public TaskContext getTaskContext() { + return currentTaskContext; } - /** - * Whether reach optimizer timeout - */ - public boolean reachTimeout() { - long timeout = getSessionVariable().getOptimizerExecuteTimeout(); - return optimizerElapsedMs() > timeout; + public QueryMaterializationContext getQueryMaterializationContext() { + return queryMaterializationContext; } - public Set getQueryTables() { - return queryTables; + public MVTransformerContext getMvTransformerContext() { + return mvTransformerContext; } - public void setQueryTables(Set queryTables) { - this.queryTables = queryTables; + public void setMvTransformerContext(MVTransformerContext mvTransformerContext) { + this.mvTransformerContext = mvTransformerContext; } - /** - * Throw exception if reach optimizer timeout - */ - public void checkTimeout() { - if (!reachTimeout()) { - return; - } - Memo memo = getMemo(); - Group group = memo == null ? null : memo.getRootGroup(); - throw new StarRocksPlannerException("StarRocks planner use long time " + optimizerElapsedMs() + - " ms in " + (group == null ? "logical" : "memo") + " phase, This probably because " + - "1. FE Full GC, " + - "2. Hive external table fetch metadata took a long time, " + - "3. The SQL is very complex. " + - "You could " + - "1. adjust FE JVM config, " + - "2. try query again, " + - "3. enlarge new_planner_optimize_timeout session variable", - ErrorType.INTERNAL_ERROR); + // ============================ Task Variables ============================ + public JoinPredicatePushdown.JoinPredicatePushDownContext getJoinPushDownParams() { + return joinPredicatePushDownContext; } - public void setQueryMaterializationContext(QueryMaterializationContext queryMaterializationContext) { - this.queryMaterializationContext = queryMaterializationContext; + public boolean isObtainedFromInternalStatistics() { + return isObtainedFromInternalStatistics; } - public QueryMaterializationContext getQueryMaterializationContext() { - return queryMaterializationContext; + public void setObtainedFromInternalStatistics(boolean obtainedFromInternalStatistics) { + isObtainedFromInternalStatistics = obtainedFromInternalStatistics; } public boolean isShortCircuit() { @@ -274,12 +219,6 @@ public void setShortCircuit(boolean shortCircuit) { isShortCircuit = shortCircuit; } - public void clear() { - if (this.queryMaterializationContext != null) { - this.queryMaterializationContext.clear(); - } - } - public void setInMemoPhase(boolean inMemoPhase) { this.inMemoPhase = inMemoPhase; } @@ -288,6 +227,27 @@ public boolean isInMemoPhase() { return this.inMemoPhase; } + /** + * Get all valid candidate materialized views for the query: + * - The materialized view is valid to rewrite by rule(SPJG) + * - The materialized view's refresh-ness is valid to rewrite. + */ + public List getCandidateMvs() { + return queryMaterializationContext.getValidCandidateMVs(); + } + + public Stopwatch getStopwatch(RuleType ruleType) { + return ruleWatchMap.computeIfAbsent(ruleType, (k) -> Stopwatch.createStarted()); + } + + public boolean ruleExhausted(RuleType ruleType) { + Stopwatch watch = getStopwatch(ruleType); + long elapsed = watch.elapsed(TimeUnit.MILLISECONDS); + long timeLimit = Math.min(getSessionVariable().getOptimizerMaterializedViewTimeLimitMillis(), + getSessionVariable().getOptimizerExecuteTimeout()); + return elapsed > timeLimit; + } + public List getPushdownNotNullPredicates() { return pushdownNotNullPredicates; } @@ -305,11 +265,23 @@ public long getNextUniquePartitionId() { return uniquePartitionIdGenerator++; } - public void setAllLogicalOlapScanOperators(List allScanOperators) { - this.allLogicalOlapScanOperators = allScanOperators; - } - - public List getAllLogicalOlapScanOperators() { - return allLogicalOlapScanOperators; + /** + * Throw exception if reach optimizer timeout + */ + public void checkTimeout() { + long timeout = getSessionVariable().getOptimizerExecuteTimeout(); + long now = optimizerTimer.elapsed(TimeUnit.MILLISECONDS); + if (timeout > 0 && now > timeout) { + throw new StarRocksPlannerException("StarRocks planner use long time " + now + + " ms in " + (inMemoPhase ? "memo" : "logical") + " phase, This probably because " + + "1. FE Full GC, " + + "2. Hive external table fetch metadata took a long time, " + + "3. The SQL is very complex. " + + "You could " + + "1. adjust FE JVM config, " + + "2. try query again, " + + "3. enlarge new_planner_optimize_timeout session variable", + ErrorType.INTERNAL_ERROR); + } } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java new file mode 100644 index 0000000000000..8b189086ee712 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java @@ -0,0 +1,59 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer; + +import com.google.common.annotations.VisibleForTesting; +import com.starrocks.qe.ConnectContext; +import com.starrocks.sql.optimizer.base.ColumnRefFactory; + +public class OptimizerFactory { + @VisibleForTesting + public static OptimizerContext mockContext(ConnectContext context, ColumnRefFactory columnRefFactory, + OptimizerConfig config) { + OptimizerContext oc = new OptimizerContext(context); + oc.setColumnRefFactory(columnRefFactory); + oc.setOptimizerConfig(config); + return oc; + } + + @VisibleForTesting + public static OptimizerContext mockContext(ConnectContext context, ColumnRefFactory columnRefFactory) { + return mockContext(context, columnRefFactory, OptimizerConfig.defaultConfig()); + } + + @VisibleForTesting + public static OptimizerContext mockContext(ColumnRefFactory columnRefFactory) { + OptimizerContext oc = new OptimizerContext(new ConnectContext()); + oc.setColumnRefFactory(columnRefFactory); + oc.setOptimizerConfig(OptimizerConfig.defaultConfig()); + return oc; + } + + public static OptimizerContext initContext(ConnectContext context, ColumnRefFactory columnRefFactory, + OptimizerConfig config) { + OptimizerContext oc = new OptimizerContext(context); + oc.setColumnRefFactory(columnRefFactory); + oc.setOptimizerConfig(config); + return oc; + } + + public static OptimizerContext initContext(ConnectContext context, ColumnRefFactory columnRefFactory) { + return initContext(context, columnRefFactory, OptimizerConfig.defaultConfig()); + } + + public static Optimizer create(OptimizerContext context) { + return new Optimizer(context); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerIterface.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerIterface.java new file mode 100644 index 0000000000000..36b4244a6f7ed --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerIterface.java @@ -0,0 +1,17 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.starrocks.sql.optimizer; + +public interface OptimizerIterface { +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java index 55036cf51eb60..f3aea9d56b177 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java @@ -62,7 +62,7 @@ public class QueryDumpInfo implements DumpInfo { private final List exceptionList = new ArrayList<>(); private int beNum; private int cachedAvgNumOfHardwareCores = -1; - private Map numOfHardwareCoresPerBe = Maps.newHashMap(); + private final Map numOfHardwareCoresPerBe = Maps.newHashMap(); private SessionVariable sessionVariable; private final ConnectContext connectContext; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java index d405d85f7f0c4..d37aa41bbcdc7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java @@ -59,8 +59,8 @@ public class JoinPredicatePushdown { private OptExpression joinOptExpression; - private List leftPushDown; - private List rightPushDown; + private final List leftPushDown; + private final List rightPushDown; /** * Whether to do complete equivalence derive in diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java index f9c4b9bc37814..f191c749abcd9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java @@ -73,6 +73,7 @@ import com.starrocks.sql.optimizer.Optimizer; import com.starrocks.sql.optimizer.OptimizerConfig; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.QueryMaterializationContext; import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.base.ColumnRefFactory; @@ -486,13 +487,13 @@ public static Pair getRuleOptimizedLogicalPlan(State TransformerContext transformerContext = new TransformerContext(columnRefFactory, connectContext, inlineView, null); LogicalPlan logicalPlan = new RelationTransformer(transformerContext).transform(query); - Optimizer optimizer = new Optimizer(optimizerConfig); + Optimizer optimizer = + OptimizerFactory.create( + OptimizerFactory.initContext(connectContext, columnRefFactory, optimizerConfig)); OptExpression optimizedPlan = optimizer.optimize( - connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), - columnRefFactory); + new ColumnRefSet(logicalPlan.getOutputColumn())); return Pair.create(optimizedPlan, logicalPlan); } @@ -1572,9 +1573,10 @@ public static OptExpression optimizeViewPlan(OptExpression logicalTree, OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); optimizerConfig.disableRule(RuleType.GP_SINGLE_TABLE_MV_REWRITE); optimizerConfig.disableRule(RuleType.GP_MULTI_TABLE_MV_REWRITE); - Optimizer optimizer = new Optimizer(optimizerConfig); - OptExpression optimizedViewPlan = optimizer.optimize(connectContext, logicalTree, - new PhysicalPropertySet(), requiredColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.initContext(connectContext, columnRefFactory, optimizerConfig)); + OptExpression optimizedViewPlan = optimizer.optimize(logicalTree, + new PhysicalPropertySet(), requiredColumns); return optimizedViewPlan; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/ApplyRuleTask.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/ApplyRuleTask.java index 9865ba2522b1b..7baebceb1fb86 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/ApplyRuleTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/ApplyRuleTask.java @@ -73,7 +73,6 @@ public void execute() { final Stopwatch ruleStopWatch = optimizerContext.getStopwatch(rule.type()); final Binder binder = new Binder(optimizerContext, pattern, groupExpression, ruleStopWatch); final List newExpressions = Lists.newArrayList(); - final List extractExpressions = Lists.newArrayList(); OptExpression extractExpr = binder.next(); while (extractExpr != null) { // Check if the rule has exhausted or not to avoid optimization time exceeding the limit.: @@ -88,7 +87,6 @@ public void execute() { extractExpr = binder.next(); continue; } - extractExpressions.add(extractExpr); List targetExpressions; OptimizerTraceUtil.logApplyRuleBefore(context.getOptimizerContext(), rule, extractExpr); try (Timer ignore = Tracers.watchScope(Tracers.Module.OPTIMIZER, rule.getClass().getSimpleName())) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskContext.java index 9eff255ab4e1c..3f98c8f796da6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskContext.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskContext.java @@ -18,10 +18,6 @@ import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; -import com.starrocks.sql.optimizer.operator.physical.PhysicalOlapScanOperator; - -import java.util.Collections; -import java.util.List; // The context for optimizer task public class TaskContext { @@ -29,7 +25,6 @@ public class TaskContext { private final PhysicalPropertySet requiredProperty; private ColumnRefSet requiredColumns; private double upperBoundCost; - private List allPhysicalOlapScanOperators; public TaskContext(OptimizerContext context, PhysicalPropertySet physicalPropertySet, @@ -39,7 +34,6 @@ public TaskContext(OptimizerContext context, this.requiredProperty = physicalPropertySet; this.requiredColumns = requiredColumns; this.upperBoundCost = cost; - this.allPhysicalOlapScanOperators = Collections.emptyList(); } public OptimizerContext getOptimizerContext() { @@ -65,12 +59,4 @@ public void setRequiredColumns(ColumnRefSet requiredColumns) { public void setUpperBoundCost(double upperBoundCost) { this.upperBoundCost = upperBoundCost; } - - public void setAllPhysicalOlapScanOperators(List allScanOperators) { - this.allPhysicalOlapScanOperators = allScanOperators; - } - - public List getAllPhysicalOlapScanOperators() { - return allPhysicalOlapScanOperators; - } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java index a5eb700b9648a..1bbf01d98f5c8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java @@ -16,10 +16,6 @@ import com.starrocks.common.profile.Timer; import com.starrocks.common.profile.Tracers; -import com.starrocks.sql.common.ErrorType; -import com.starrocks.sql.common.StarRocksPlannerException; -import com.starrocks.sql.optimizer.Group; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.rule.Rule; @@ -39,29 +35,8 @@ public static TaskScheduler create() { } public void executeTasks(TaskContext context) { - long timeout = context.getOptimizerContext().getSessionVariable().getOptimizerExecuteTimeout(); while (!tasks.empty()) { - long watch = context.getOptimizerContext().optimizerElapsedMs(); - if (timeout > 0 && watch > timeout) { - // Should have at least one valid plan - // group will be null when in rewrite phase - // memo may be null for rule-based optimizer - Memo memo = context.getOptimizerContext().getMemo(); - Group group = memo == null ? null : memo.getRootGroup(); - if (group == null || !group.hasBestExpression(context.getRequiredProperty())) { - throw new StarRocksPlannerException("StarRocks planner use long time " + timeout + - " ms in " + (group == null ? "logical" : "memo") + " phase, This probably because " + - "1. FE Full GC, " + - "2. Hive external table fetch metadata took a long time, " + - "3. The SQL is very complex. " + - "You could " + - "1. adjust FE JVM config, " + - "2. try query again, " + - "3. enlarge new_planner_optimize_timeout session variable", - ErrorType.INTERNAL_ERROR); - } - break; - } + context.getOptimizerContext().checkTimeout(); OptimizerTask task = tasks.pop(); context.getOptimizerContext().setTaskContext(context); try (Timer ignore = Tracers.watchScope(Tracers.Module.OPTIMIZER, task.getClass().getSimpleName())) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java index 8d141a99e9bb1..ba06194d0f2fd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java @@ -24,6 +24,7 @@ import com.starrocks.sql.optimizer.MaterializationContext; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.rule.transformation.materialization.MaterializedViewRewriter; import com.starrocks.sql.optimizer.task.TaskContext; import org.apache.commons.collections.CollectionUtils; @@ -36,10 +37,15 @@ import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.collectMaterializedViews; public class MVRewriteValidator { - private static final MVRewriteValidator INSTANCE = new MVRewriteValidator(); - - public static MVRewriteValidator getInstance() { - return INSTANCE; + // private static final MVRewriteValidator INSTANCE = new MVRewriteValidator(); + // + // public static MVRewriteValidator getInstance() { + // return INSTANCE; + // } + private List allLogicalOlapScanOperators; + + public MVRewriteValidator(List allLogicalOlapScanOperators) { + this.allLogicalOlapScanOperators = allLogicalOlapScanOperators; } private static boolean isUpdateMaterializedViewMetrics(ConnectContext connectContext) { @@ -79,7 +85,7 @@ public void auditMv(ConnectContext connectContext, OptExpression physicalPlan, T List mvs = collectMaterializedViews(physicalPlan); // To avoid queries that query the materialized view directly, only consider materialized views // that are not used in rewriting before. - Set beforeTableIds = optimizerContext.getAllLogicalOlapScanOperators().stream() + Set beforeTableIds = allLogicalOlapScanOperators.stream() .map(op -> op.getTable().getId()) .collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(mvs)) { @@ -130,7 +136,7 @@ public void validateMV(ConnectContext connectContext, OptExpression physicalPlan } List mvs = collectMaterializedViews(physicalPlan); - Set beforeTableIds = taskContext.getOptimizerContext().getAllLogicalOlapScanOperators().stream() + Set beforeTableIds = allLogicalOlapScanOperators.stream() .map(op -> op.getTable().getId()) .collect(Collectors.toSet()); diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveMetadataTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveMetadataTest.java index 27f698e464c6d..203d3713965f2 100644 --- a/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveMetadataTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveMetadataTest.java @@ -51,8 +51,8 @@ import com.starrocks.sql.analyzer.AstToStringBuilder; import com.starrocks.sql.ast.CreateTableStmt; import com.starrocks.sql.ast.DropTableStmt; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.sql.optimizer.statistics.ColumnStatistic; @@ -133,7 +133,7 @@ public void setUp() throws Exception { // create connect context connectContext = UtFrameUtils.createDefaultCtx(); columnRefFactory = new ColumnRefFactory(); - optimizerContext = new OptimizerContext(new Memo(), columnRefFactory, connectContext); + optimizerContext = OptimizerFactory.mockContext(connectContext, columnRefFactory); hiveMetadata = new HiveMetadata("hive_catalog", new HdfsEnvironment(), hmsOps, fileOps, statisticsProvider, Optional.empty(), executorForHmsRefresh, executorForHmsRefresh, new ConnectorProperties(ConnectorType.HIVE)); diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveStatisticsProviderTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveStatisticsProviderTest.java index b761a36cef254..cf3dd8d90ea84 100644 --- a/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveStatisticsProviderTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveStatisticsProviderTest.java @@ -26,8 +26,8 @@ import com.starrocks.connector.PartitionUtil; import com.starrocks.connector.RemoteFileOperations; import com.starrocks.qe.ConnectContext; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.sql.optimizer.statistics.ColumnStatistic; @@ -98,7 +98,7 @@ public void setUp() throws Exception { // create connect context connectContext = UtFrameUtils.createDefaultCtx(); columnRefFactory = new ColumnRefFactory(); - optimizerContext = new OptimizerContext(new Memo(), columnRefFactory, connectContext); + optimizerContext = OptimizerFactory.mockContext(connectContext, columnRefFactory); } @After diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java index 49358b31f711d..d2e5c69252349 100644 --- a/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java @@ -79,8 +79,8 @@ import com.starrocks.sql.ast.ModifyColumnClause; import com.starrocks.sql.ast.ModifyTablePropertiesClause; import com.starrocks.sql.ast.TableRenameClause; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator; import com.starrocks.sql.optimizer.operator.scalar.CallOperator; @@ -794,7 +794,7 @@ public void testGetTableStatistics() { ColumnRefOperator columnRefOperator2 = new ColumnRefOperator(4, Type.STRING, "data", true); colRefToColumnMetaMap.put(columnRefOperator1, new Column("id", Type.INT)); colRefToColumnMetaMap.put(columnRefOperator2, new Column("data", Type.STRING)); - OptimizerContext context = new OptimizerContext(new Memo(), new ColumnRefFactory()); + OptimizerContext context = OptimizerFactory.mockContext(new ColumnRefFactory()); Assert.assertFalse(context.getSessionVariable().enableIcebergColumnStatistics()); Assert.assertTrue(context.getSessionVariable().enableReadIcebergPuffinNdv()); TableVersionRange versionRange = TableVersionRange.withEnd(Optional.of( @@ -830,7 +830,7 @@ public void testGetTableStatisticsWithColumnStats() { TableVersionRange versionRange = TableVersionRange.withEnd(Optional.of( mockedNativeTableB.currentSnapshot().snapshotId())); Statistics statistics = metadata.getTableStatistics( - new OptimizerContext(null, null, ConnectContext.get()), + OptimizerFactory.mockContext(ConnectContext.get(), null), icebergTable, colRefToColumnMetaMap, null, null, -1, versionRange); Assert.assertEquals(4.0, statistics.getOutputRowCount(), 0.001); Assert.assertEquals(2, statistics.getColumnStatistics().size()); @@ -940,7 +940,7 @@ public void testGetRepeatedTableStats() { mockedNativeTableA.refresh(); new ConnectContext().setThreadLocalInfo(); - OptimizerContext context = new OptimizerContext(new Memo(), new ColumnRefFactory(), ConnectContext.get()); + OptimizerContext context = OptimizerFactory.mockContext(ConnectContext.get(), new ColumnRefFactory()); context.getSessionVariable().setEnableIcebergColumnStatistics(true); TableVersionRange version = TableVersionRange.withEnd(Optional.of( diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java index 57372c09a3b3c..a8863686b76a9 100644 --- a/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java @@ -29,9 +29,8 @@ import com.starrocks.credential.CloudConfiguration; import com.starrocks.credential.CloudType; import com.starrocks.server.MetadataMgr; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptExpression; -import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.logical.LogicalPaimonScanOperator; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; @@ -443,7 +442,7 @@ public long getTableCreateTime(String dbName, String tblName) { OptExpression scan = new OptExpression(new LogicalPaimonScanOperator(paimonTable, colRefToColumnMetaMap, columnMetaToColRefMap, -1, null)); - rule0.transform(scan, new OptimizerContext(new Memo(), new ColumnRefFactory())); + rule0.transform(scan, OptimizerFactory.mockContext(new ColumnRefFactory())); assertEquals(1, ((LogicalPaimonScanOperator) scan.getOp()).getScanOperatorPredicates() .getSelectedPartitionIds().size()); } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/OptimizerTaskTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/OptimizerTaskTest.java index 3917eadd17aa1..8df67e9e1fcff 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/OptimizerTaskTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/OptimizerTaskTest.java @@ -39,7 +39,6 @@ import com.starrocks.sql.optimizer.base.DistributionSpec; import com.starrocks.sql.optimizer.base.HashDistributionDesc; import com.starrocks.sql.optimizer.base.Ordering; -import com.starrocks.sql.optimizer.base.PhysicalPropertySet; import com.starrocks.sql.optimizer.dump.MockDumpInfo; import com.starrocks.sql.optimizer.operator.AggType; import com.starrocks.sql.optimizer.operator.Operator; @@ -169,9 +168,8 @@ public void testTaskScheduler(@Mocked OlapTable olapTable1, } }; - Optimizer optimizer = new Optimizer(); - optimizer.optimize(ctx, logicOperatorTree, new PhysicalPropertySet(), new ColumnRefSet(), - columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(logicOperatorTree, new ColumnRefSet()); Memo memo = optimizer.getContext().getMemo(); assertEquals(3, memo.getGroups().size()); assertEquals(8, memo.getGroupExpressions().size()); @@ -257,9 +255,8 @@ public void testTwoJoin(@Mocked OlapTable olapTable1, } }; - Optimizer optimizer = new Optimizer(); - optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(), - columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(topJoin, new ColumnRefSet()); Memo memo = optimizer.getContext().getMemo(); @@ -379,9 +376,8 @@ public void testThreeJoin(@Mocked OlapTable olapTable1, scan4ColumnMap, Maps.newHashMap(), null, -1, null))); - Optimizer optimizer = new Optimizer(); - optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(), - columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(topJoin, new ColumnRefSet()); } @Test @@ -475,9 +471,8 @@ public void testFourJoin(@Mocked OlapTable olapTable1, } }; - Optimizer optimizer = new Optimizer(); - optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(), - columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(topJoin, new ColumnRefSet()); } @Test @@ -622,9 +617,8 @@ public void testSevenJoin(@Mocked OlapTable olapTable1, } }; - Optimizer optimizer = new Optimizer(); - optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(), - columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(topJoin, new ColumnRefSet()); } @Test @@ -670,8 +664,9 @@ public void testDeriveOutputColumns(@Mocked OlapTable olapTable1, outputColumns.addAll(outputColumns2); ColumnRefSet outputColumnsSet = new ColumnRefSet(outputColumns); - Optimizer optimizer = new Optimizer(); - optimizer.optimize(ctx, logicOperatorTree, new PhysicalPropertySet(), outputColumnsSet, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(logicOperatorTree, outputColumnsSet); + Memo memo = optimizer.getContext().getMemo(); MemoStatusChecker checker = new MemoStatusChecker(memo, 2, outputColumnsSet); @@ -715,10 +710,10 @@ public void testExtractBestPlanForThreeTable(@Mocked OlapTable olapTable1, new LogicalOlapScanOperator(olapTable3, scanColumnMap, Maps.newHashMap(), null, -1, null))); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), - new ColumnRefSet(Lists.newArrayList(column1)), - columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(topJoin, + new ColumnRefSet(List.of(column1))); + assertEquals(physicalTree.getOp().getOpType(), OperatorType.PHYSICAL_NESTLOOP_JOIN); assertEquals(physicalTree.inputAt(0).getOp().getOpType(), OperatorType.PHYSICAL_NESTLOOP_JOIN); assertEquals(physicalTree.inputAt(1).getOp().getOpType(), OperatorType.PHYSICAL_DISTRIBUTION); @@ -750,9 +745,9 @@ public void testTopDownRewrite(@Mocked OlapTable olapTable1) { } }; - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), new ColumnRefSet(), - columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet()); + Operator root = physicalTree.getOp(); assertEquals(root.getOpType(), OperatorType.PHYSICAL_LIMIT); } @@ -783,9 +778,9 @@ public void testPruneOlapScanColumnsRule(@Mocked OlapTable olapTable1) { new LogicalOlapScanOperator(olapTable1, scanColumnMap, Maps.newHashMap(), null, -1, null))); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns1), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, + new ColumnRefSet(outputColumns1)); assertEquals(OperatorType.PHYSICAL_OLAP_SCAN, physicalTree.getOp().getOpType()); PhysicalOlapScanOperator physicalOlapScan = (PhysicalOlapScanOperator) physicalTree.getOp(); @@ -829,9 +824,10 @@ public void testPruneOlapScanColumnsRuleWithConstant(@Mocked OlapTable olapTable } }; - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns1), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, + new ColumnRefSet(outputColumns1)); assertNotNull(physicalTree.getOp().getProjection()); assertEquals(physicalTree.getOp().getOpType(), OperatorType.PHYSICAL_OLAP_SCAN); @@ -901,9 +897,9 @@ public void testPruneAggregateColumnsRule(@Mocked OlapTable olapTable1) { ColumnRefSet outputColumns = new ColumnRefSet(column2.getId()); - Optimizer optimizer = new Optimizer(); try { - optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(expression, outputColumns); fail("invalid plan. can not optimize success"); } catch (Exception e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 2: t2 in " + @@ -917,8 +913,10 @@ public void testPruneAggregateColumnsRule(@Mocked OlapTable olapTable1) { tmp.getSessionVariable().setJoinImplementationMode("auto"); tmp.getSessionVariable().setEnablePlanValidation(false); tmp.setDumpInfo(new MockDumpInfo()); - OptExpression expression1 = optimizer.optimize(tmp, expression, new PhysicalPropertySet(), outputColumns, - columnRefFactory); + + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression expression1 = optimizer.optimize(expression, outputColumns); Map aggs = ((PhysicalHashAggregateOperator) expression1.getOp()).getAggregations(); assertEquals(Type.INT, column2.getType()); @@ -994,9 +992,10 @@ public void testPruneCountStarRule(@Mocked OlapTable olapTable1) { ColumnRefSet outputColumns = new ColumnRefSet(column5.getId()); - Optimizer optimizer = new Optimizer(); try { - optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(expression, outputColumns); + fail("invalid plan. can not optimize success"); } catch (Exception e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 5: t5 in " + @@ -1047,9 +1046,11 @@ public void testPruneAggregateConstantRule(@Mocked OlapTable olapTable1) { } }; - Optimizer optimizer = new Optimizer(); try { - optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(expression, outputColumns); + fail("invalid plan. can not optimize success"); } catch (Exception e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 3: t3 in " + @@ -1104,9 +1105,11 @@ public void testMergeAggregateWithLimitRule(@Mocked OlapTable olapTable1) { } }; - Optimizer optimizer = new Optimizer(); try { - optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(expression, outputColumns); + fail("invalid plan. can not optimize success"); } catch (Exception e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 3: t3 in " + @@ -1154,9 +1157,9 @@ public void testPruneSortColumnsRule(@Mocked OlapTable olapTable1, ColumnRefSet outputColumns = new ColumnRefSet(column2.getId()); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, outputColumns); assertEquals(physicalTree.getLogicalProperty().getOutputColumns(), new ColumnRefSet(column2.getId())); @@ -1209,9 +1212,9 @@ public void testSplitAggregateRule(@Mocked OlapTable olapTable1) { ColumnRefSet outputColumns = new ColumnRefSet(column1.getId()); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, outputColumns); Operator operator = physicalTree.getOp(); assertEquals(operator.getOpType(), OperatorType.PHYSICAL_HASH_AGG); @@ -1281,9 +1284,11 @@ public void testSplitAggregateRuleNoGroupBy(@Mocked OlapTable olapTable1) { } }; - Optimizer optimizer = new Optimizer(); try { - optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(expression, outputColumns); + fail("invalid plan. can not optimize success"); } catch (Exception e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 3: t3 in " + @@ -1342,9 +1347,10 @@ public void testSplitAggregateRuleWithDistinctAndGroupBy(@Mocked OlapTable olapT ColumnRefSet outputColumns = new ColumnRefSet(Lists.newArrayList(column3, column2)); - Optimizer optimizer = new Optimizer(); try { - optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(expression, outputColumns); fail("invalid plan. can not optimize success"); } catch (Exception e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 3: t3 in " + @@ -1403,9 +1409,10 @@ public void testSplitAggregateRuleWithOnlyOneDistinct(@Mocked OlapTable olapTabl ColumnRefSet outputColumns = new ColumnRefSet(Lists.newArrayList(column3)); - Optimizer optimizer = new Optimizer(); try { - optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(expression, outputColumns); + fail("invalid plan. can not optimize success"); } catch (Exception e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 3: t3 in " + @@ -1458,9 +1465,8 @@ public void testSplitAggregateRuleWithProject(@Mocked OlapTable olapTable1) { ColumnRefSet outputColumns = new ColumnRefSet(column2.getId()); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, topProject, new PhysicalPropertySet(), - outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(topProject, outputColumns); Operator operator = physicalTree.getOp(); assertEquals(operator.getOpType(), OperatorType.PHYSICAL_HASH_AGG); @@ -1524,9 +1530,10 @@ public void testFilterPushDownWithHaving(@Mocked OlapTable olapTable1) { ColumnRefSet outputColumns = new ColumnRefSet(Lists.newArrayList(column6)); - Optimizer optimizer = new Optimizer(); try { - optimizer.optimize(ctx, root, new PhysicalPropertySet(), outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + optimizer.optimize(root, outputColumns); + fail("invalid plan. can not optimize success"); } catch (Exception e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 5: t5 in " + @@ -1583,9 +1590,9 @@ public void testFilterPushDownWithHaving2(@Mocked OlapTable olapTable1) { ColumnRefSet outputColumns = new ColumnRefSet(Lists.newArrayList(column4)); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, root, new PhysicalPropertySet(), - outputColumns, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(root, outputColumns); Operator operator = physicalTree.getOp(); assertEquals(operator.getOpType(), OperatorType.PHYSICAL_HASH_AGG); @@ -1636,9 +1643,8 @@ public void testFilterPushDownRule(@Mocked OlapTable olapTable1) { null, -1, null)))); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns)); assertNotNull(physicalTree.getOp().getProjection()); assertEquals(physicalTree.getOp().getOpType(), OperatorType.PHYSICAL_OLAP_SCAN); @@ -1685,9 +1691,8 @@ public void testFilterPushDownRuleWithMultiProjects(@Mocked OlapTable olapTable1 OptExpression filter = OptExpression.create(new LogicalFilterOperator(predicate), project2); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, filter, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(filter, new ColumnRefSet(outputColumns)); assertNotNull(physicalTree.getOp().getProjection()); Projection pp = physicalTree.getOp().getProjection(); @@ -1748,9 +1753,9 @@ public void testCommonOperatorReuseRule(@Mocked OlapTable olapTable1) { new LogicalOlapScanOperator(olapTable1, scanColumnMap, Maps.newHashMap(), null, -1, null))); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns)); PhysicalOlapScanOperator olapScanOperator = (PhysicalOlapScanOperator) physicalTree.getOp(); Projection projection = olapScanOperator.getProjection(); @@ -1834,9 +1839,8 @@ public void testShuffleTwoJoin(@Mocked OlapTable olapTable1, OptExpression.create(scan1), OptExpression.create(scan2)); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns)); } @Test @@ -1932,9 +1936,8 @@ public void testShuffleThreeJoin(@Mocked OlapTable olapTable1, OptExpression.create(scan2)); OptExpression topJoin = OptExpression.create(join2, join1, OptExpression.create(scan3)); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(topJoin, new ColumnRefSet(outputColumns)); } @Test @@ -2031,9 +2034,10 @@ public void testBroadcastExceedRowLimitWithHugeGapInRowCount(@Mocked OlapTable o OptExpression.create(scan1), OptExpression.create(scan2)); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns)); + assertEquals(physicalTree.getInputs().get(1).getOp().getOpType(), OperatorType.PHYSICAL_DISTRIBUTION); PhysicalDistributionOperator rightOperator = (PhysicalDistributionOperator) physicalTree.getInputs().get(1).getOp(); @@ -2152,9 +2156,10 @@ public void testBroadcastExceedRowLimitWithoutHugeGapInRowCount(@Mocked OlapTabl OptExpression.create(scan1), OptExpression.create(scan2)); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns)); + assertEquals(physicalTree.getInputs().get(1).getOp().getOpType(), OperatorType.PHYSICAL_DISTRIBUTION); PhysicalDistributionOperator rightOperator = (PhysicalDistributionOperator) physicalTree.getInputs().get(1).getOp(); @@ -2277,9 +2282,9 @@ public void testOlapTablePartitionRowCount(@Mocked OlapTable olapTable1, OptExpression.create(scan1), OptExpression.create(scan2)); - Optimizer optimizer = new Optimizer(); - OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns)); assertEquals(physicalTree.getInputs().get(1).getOp().getOpType(), OperatorType.PHYSICAL_DISTRIBUTION); PhysicalDistributionOperator rightOperator = (PhysicalDistributionOperator) physicalTree.getInputs().get(1).getOp(); @@ -2334,7 +2339,6 @@ public void testOlapTablePartitionRowCount(@Mocked OlapTable olapTable1, } }; - optimizer = new Optimizer(); expression = OptExpression.create(new LogicalJoinOperator(JoinOperator.INNER_JOIN, predicate), OptExpression.create(new LogicalOlapScanOperator(olapTable1, scan1ColumnMap, Maps.newHashMap(), DistributionSpec.createHashDistributionSpec( @@ -2344,8 +2348,10 @@ public void testOlapTablePartitionRowCount(@Mocked OlapTable olapTable1, DistributionSpec.createHashDistributionSpec( new HashDistributionDesc(Lists.newArrayList(this.column4.getId()), HashDistributionDesc.SourceType.LOCAL)), -1, null))); - physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), - new ColumnRefSet(outputColumns), columnRefFactory); + optimizer = OptimizerFactory.create( + OptimizerFactory.mockContext(ctx, columnRefFactory)); + physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns)); + assertEquals(physicalTree.getInputs().get(1).getOp().getOpType(), OperatorType.PHYSICAL_DISTRIBUTION); rightOperator = (PhysicalDistributionOperator) physicalTree.getInputs().get(1).getOp(); assertEquals(rightOperator.getDistributionSpec().getType(), DistributionSpec.DistributionType.BROADCAST); @@ -2395,7 +2401,7 @@ public void testSplitAggregateRuleConstantColumns(@Mocked OlapTable olapTable1) SplitTwoPhaseAggRule splitTwoPhaseAggRule = SplitTwoPhaseAggRule.getInstance(); List list = splitTwoPhaseAggRule.transform( - expression, new OptimizerContext(new Memo(), new ColumnRefFactory())); + expression, OptimizerFactory.mockContext(new ColumnRefFactory())); assertEquals(OperatorType.LOGICAL_AGGR, list.get(0).getOp().getOpType()); LogicalAggregationOperator result = (LogicalAggregationOperator) list.get(0).getOp(); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/BinderTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/BinderTest.java index 2eff6fa0ee1c8..f31946f16dee4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/BinderTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/BinderTest.java @@ -22,6 +22,7 @@ import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator; @@ -37,21 +38,21 @@ public class BinderTest { private Binder buildBinder(Pattern pattern, OptExpression expr) { Memo memo = new Memo(); - OptimizerContext optimizerContext = new OptimizerContext(memo, new ColumnRefFactory()); + OptimizerContext optimizerContext = OptimizerFactory.mockContext(new ColumnRefFactory()); Stopwatch stopwatch = Stopwatch.createStarted(); return new Binder(optimizerContext, pattern, memo.init(expr), stopwatch); } private Binder buildBinder(Pattern pattern, GroupExpression qe) { Memo memo = new Memo(); - OptimizerContext optimizerContext = new OptimizerContext(memo, new ColumnRefFactory()); + OptimizerContext optimizerContext = OptimizerFactory.mockContext(new ColumnRefFactory()); Stopwatch stopwatch = Stopwatch.createStarted(); return new Binder(optimizerContext, pattern, qe, stopwatch); } private OptExpression bindNext(Pattern pattern, OptExpression expr) { Memo memo = new Memo(); - OptimizerContext optimizerContext = new OptimizerContext(memo, new ColumnRefFactory()); + OptimizerContext optimizerContext = OptimizerFactory.mockContext(new ColumnRefFactory()); Stopwatch stopwatch = Stopwatch.createStarted(); Binder binder = new Binder(optimizerContext, pattern, memo.init(expr), stopwatch); return binder.next(); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/implementation/OlapScanImplementationRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/implementation/OlapScanImplementationRuleTest.java index ea7e8126092b5..0878c80271f01 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/implementation/OlapScanImplementationRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/implementation/OlapScanImplementationRuleTest.java @@ -17,9 +17,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.starrocks.catalog.OlapTable; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptExpression; -import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.operator.physical.PhysicalOlapScanOperator; @@ -41,8 +40,8 @@ public void transform(@Mocked OlapTable table) { false, Lists.newArrayList(4L), null, null, false); List output = - new OlapScanImplementationRule().transform(new OptExpression(logical), new OptimizerContext( - new Memo(), new ColumnRefFactory())); + new OlapScanImplementationRule().transform(new OptExpression(logical), + OptimizerFactory.mockContext(new ColumnRefFactory())); assertEquals(1, output.size()); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java index c7f286f4c2a18..57944bd89d625 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java @@ -32,9 +32,8 @@ import com.starrocks.catalog.ScalarType; import com.starrocks.catalog.Type; import com.starrocks.planner.PartitionColumnFilter; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptExpression; -import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; @@ -197,7 +196,7 @@ public void transform(@Mocked OlapTable olapTable, @Mocked Partition partition, assertEquals(0, operator.getSelectedTabletId().size()); OptExpression optExpression = - rule.transform(new OptExpression(operator), new OptimizerContext(new Memo(), new ColumnRefFactory())) + rule.transform(new OptExpression(operator), OptimizerFactory.mockContext(new ColumnRefFactory())) .get(0); assertEquals(20, ((LogicalOlapScanOperator) optExpression.getOp()).getSelectedTabletId().size()); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitWithSortRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitWithSortRuleTest.java index 2f19bf394d95a..93070f493c0c7 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitWithSortRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitWithSortRuleTest.java @@ -17,9 +17,8 @@ import com.google.common.collect.Lists; import com.starrocks.catalog.Type; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptExpression; -import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.Ordering; import com.starrocks.sql.optimizer.operator.OperatorType; @@ -43,7 +42,7 @@ public void transform() { limit.getInputs().add(sort); MergeLimitWithSortRule rule = new MergeLimitWithSortRule(); - List list = rule.transform(limit, new OptimizerContext(new Memo(), new ColumnRefFactory())); + List list = rule.transform(limit, OptimizerFactory.mockContext(new ColumnRefFactory())); assertEquals(OperatorType.LOGICAL_TOPN, list.get(0).getOp().getOpType()); assertEquals(2, ((LogicalTopNOperator) list.get(0).getOp()).getOffset()); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRuleTest.java index fe6cf641e18ec..d28b3228c702f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRuleTest.java @@ -37,9 +37,8 @@ import com.starrocks.pseudocluster.PseudoCluster; import com.starrocks.sql.ast.PartitionNames; import com.starrocks.sql.ast.PartitionValue; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptExpression; -import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; @@ -161,7 +160,7 @@ public void transform1(@Mocked OlapTable olapTable, @Mocked RangePartitionInfo p assertNull(operator.getSelectedPartitionId()); OptExpression optExpression = - rule.transform(new OptExpression(operator), new OptimizerContext(new Memo(), columnRefFactory)).get(0); + rule.transform(new OptExpression(operator), OptimizerFactory.mockContext(columnRefFactory)).get(0); assertEquals(3, ((LogicalOlapScanOperator) optExpression.getOp()).getSelectedPartitionId().size()); } @@ -282,7 +281,7 @@ public void transform2(@Mocked OlapTable olapTable, @Mocked RangePartitionInfo p assertNull(operator.getSelectedPartitionId()); OptExpression optExpression = - rule.transform(new OptExpression(operator), new OptimizerContext(new Memo(), columnRefFactory)).get(0); + rule.transform(new OptExpression(operator), OptimizerFactory.mockContext(columnRefFactory)).get(0); assertEquals(3, ((LogicalOlapScanOperator) optExpression.getOp()).getSelectedPartitionId().size()); } @@ -363,7 +362,7 @@ public void transformForSingleItemListPartition(@Mocked OlapTable olapTable, PartitionPruneRule rule = new PartitionPruneRule(); assertNull(operator.getSelectedPartitionId()); OptExpression optExpression = - rule.transform(new OptExpression(operator), new OptimizerContext(new Memo(), columnRefFactory)).get(0); + rule.transform(new OptExpression(operator), OptimizerFactory.mockContext(columnRefFactory)).get(0); List selectPartitionIds = ((LogicalOlapScanOperator) optExpression.getOp()).getSelectedPartitionId(); assertEquals(1, selectPartitionIds.size()); @@ -454,7 +453,7 @@ public void transformForSingleItemListPartitionWithTemp(@Mocked OlapTable olapTa PartitionPruneRule rule = new PartitionPruneRule(); assertNull(operator.getSelectedPartitionId()); OptExpression optExpression = - rule.transform(new OptExpression(operator), new OptimizerContext(new Memo(), columnRefFactory)).get(0); + rule.transform(new OptExpression(operator), OptimizerFactory.mockContext(columnRefFactory)).get(0); List selectPartitionIds = ((LogicalOlapScanOperator) optExpression.getOp()).getSelectedPartitionId(); assertEquals(1, selectPartitionIds.size()); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownAggRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownAggRuleTest.java index 753a41d8cedd4..3371d38e731c3 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownAggRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownAggRuleTest.java @@ -20,9 +20,8 @@ import com.starrocks.analysis.BinaryType; import com.starrocks.catalog.Column; import com.starrocks.catalog.Type; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptExpression; -import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.AggType; import com.starrocks.sql.optimizer.operator.OperatorType; @@ -77,7 +76,7 @@ public void transform(@Mocked LogicalOlapScanOperator scanOp) { PushDownPredicateAggRule rule = new PushDownPredicateAggRule(); - List list = rule.transform(filter, new OptimizerContext(new Memo(), new ColumnRefFactory())); + List list = rule.transform(filter, OptimizerFactory.mockContext(new ColumnRefFactory())); assertEquals(OperatorType.LOGICAL_AGGR, list.get(0).getOp().getOpType()); assertEquals(OperatorType.COMPOUND, diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java index c56e0ca90bddc..a2b34453f9e9a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java @@ -19,9 +19,8 @@ import com.starrocks.analysis.BinaryType; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Type; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptExpression; -import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.OperatorType; @@ -57,7 +56,7 @@ public void transform(@Mocked OlapTable table) { assertNull(((LogicalOlapScanOperator) scan.getOp()).getPredicate()); List result = - rule.transform(optExpression, new OptimizerContext(new Memo(), new ColumnRefFactory())); + rule.transform(optExpression, OptimizerFactory.mockContext(new ColumnRefFactory())); Operator scanOperator = result.get(0).inputAt(0).getOp(); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java index 27b9fcf259cc2..0e0f8f1ade69d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java @@ -51,6 +51,7 @@ import com.starrocks.sql.optimizer.CachingMvPlanContextBuilder; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.QueryMaterializationContext; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; @@ -228,13 +229,11 @@ public static OptExpression getOptimizedPlan(String sql, ConnectContext connectC ColumnRefFactory columnRefFactory = new ColumnRefFactory(); LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext).transformWithSelectLimit(query); - Optimizer optimizer = new Optimizer(); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, columnRefFactory)); return optimizer.optimize( - connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), - columnRefFactory); + new ColumnRefSet(logicalPlan.getOutputColumn())); } public List getScanOperators(OptExpression root, String name) { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java index 9c87dd4f7fbc0..8fcdcae3bba98 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java @@ -26,12 +26,12 @@ import com.starrocks.sql.optimizer.CachingMvPlanContextBuilder; import com.starrocks.sql.optimizer.MaterializationContext; import com.starrocks.sql.optimizer.MaterializedViewOptimizer; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.MvRewritePreprocessor; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; import com.starrocks.sql.optimizer.OptimizerConfig; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; @@ -92,21 +92,6 @@ public List transform(OptExpression input, OptimizerContext conte @Test public void testOptimizer() throws Exception { - Optimizer optimizer = new Optimizer(); - Assert.assertFalse(optimizer.getOptimizerConfig().isRuleBased()); - Assert.assertFalse(optimizer.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); - Assert.assertFalse(optimizer.getOptimizerConfig().isRuleDisable(RuleType.GP_AGGREGATE_REWRITE)); - - OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); - optimizerConfig.disableRule(RuleType.TF_MERGE_TWO_PROJECT); - optimizerConfig.disableRule(RuleType.GP_PUSH_DOWN_PREDICATE); - Optimizer optimizer1 = new Optimizer(optimizerConfig); - Assert.assertTrue(optimizer1.getOptimizerConfig().isRuleBased()); - Assert.assertFalse(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_AGG_RULE)); - Assert.assertTrue(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); - Assert.assertFalse(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.GP_COLLECT_CTE)); - Assert.assertTrue(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.GP_PUSH_DOWN_PREDICATE)); - String sql = "select v1, sum(v3) from t0 where v1 < 10 group by v1"; Pair result = UtFrameUtils.getPlanAndFragment(connectContext, sql); Assert.assertNotNull(result); @@ -118,13 +103,33 @@ public void testOptimizer() throws Exception { ColumnRefFactory columnRefFactory = new ColumnRefFactory(); LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext) .transformWithSelectLimit(query.getQueryRelation()); - OptExpression expr = optimizer.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory); + + OptimizerContext optimizerContext = OptimizerFactory.mockContext(connectContext, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(optimizerContext); + Assert.assertFalse(optimizerContext.getOptimizerConfig().isRuleBased()); + Assert.assertFalse(optimizerContext.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); + Assert.assertFalse(optimizerContext.getOptimizerConfig().isRuleDisable(RuleType.GP_AGGREGATE_REWRITE)); + + OptExpression expr = optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), + new ColumnRefSet(logicalPlan.getOutputColumn())); Assert.assertTrue(expr.getInputs().get(0).getOp() instanceof PhysicalOlapScanOperator); Assert.assertNotNull(expr.getInputs().get(0).getOp().getPredicate()); - OptExpression expr1 = optimizer1.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory); + + OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); + optimizerConfig.disableRule(RuleType.TF_MERGE_TWO_PROJECT); + optimizerConfig.disableRule(RuleType.GP_PUSH_DOWN_PREDICATE); + OptimizerContext optimizerContext1 = OptimizerFactory.mockContext(connectContext, columnRefFactory, + optimizerConfig); + Optimizer optimizer1 = OptimizerFactory.create(optimizerContext1); + Assert.assertTrue(optimizerContext1.getOptimizerConfig().isRuleBased()); + Assert.assertFalse(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_AGG_RULE)); + Assert.assertTrue(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); + Assert.assertFalse(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.GP_COLLECT_CTE)); + Assert.assertTrue(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.GP_PUSH_DOWN_PREDICATE)); + + OptExpression expr1 = optimizer1.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), + new ColumnRefSet(logicalPlan.getOutputColumn())); Assert.assertTrue(expr1.getInputs().get(0).getOp() instanceof LogicalFilterOperator); // test timeout @@ -163,9 +168,9 @@ public void testPreprocessMvNonPartitionMv() throws Exception { ColumnRefFactory columnRefFactory = new ColumnRefFactory(); LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext) .transformWithSelectLimit(query.getQueryRelation()); - Optimizer optimizer = new Optimizer(); - OptExpression expr = optimizer.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, columnRefFactory)); + OptExpression expr = optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), + new ColumnRefSet(logicalPlan.getOutputColumn())); Assert.assertNotNull(expr); Assert.assertEquals(2, optimizer.getContext().getCandidateMvs().size()); @@ -208,9 +213,10 @@ public void testPreprocessMvPartitionMv() throws Exception { ColumnRefFactory columnRefFactory = new ColumnRefFactory(); LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext) .transformWithSelectLimit(query.getQueryRelation()); - Optimizer optimizer = new Optimizer(); - OptExpression expr = optimizer.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, + columnRefFactory)); + OptExpression expr = optimizer.optimize(logicalPlan.getRoot(), + new ColumnRefSet(logicalPlan.getOutputColumn())); Assert.assertNotNull(expr); Assert.assertEquals(1, optimizer.getContext().getCandidateMvs().size()); MaterializationContext materializationContext = optimizer.getContext().getCandidateMvs().iterator().next(); @@ -222,9 +228,10 @@ public void testPreprocessMvPartitionMv() throws Exception { refreshMaterializedView("test", "mv_4"); executeInsertSql(connectContext, "insert into tbl_with_mv partition(p2) values(\"2020-02-20\", 20, 30)"); - Optimizer optimizer2 = new Optimizer(); - OptExpression expr2 = optimizer2.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory); + Optimizer optimizer2 = OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, + columnRefFactory)); + OptExpression expr2 = optimizer2.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), + new ColumnRefSet(logicalPlan.getOutputColumn())); Assert.assertNotNull(expr2); MaterializationContext materializationContext2 = optimizer2.getContext().getCandidateMvs().iterator().next(); @@ -245,12 +252,13 @@ public void testPreprocessMvPartitionMv() throws Exception { StatementBase stmt = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); QueryStatement query = (QueryStatement) stmt; - Optimizer optimizer3 = new Optimizer(); ColumnRefFactory columnRefFactory = new ColumnRefFactory(); LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext) .transformWithSelectLimit(query.getQueryRelation()); - OptExpression expr3 = optimizer3.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory); + Optimizer optimizer3 = + OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, columnRefFactory)); + OptExpression expr3 = optimizer3.optimize(logicalPlan.getRoot(), + new ColumnRefSet(logicalPlan.getOutputColumn())); Assert.assertNotNull(expr3); MaterializationContext materializationContext3 = optimizer3.getContext().getCandidateMvs().iterator().next(); @@ -265,7 +273,7 @@ public void testPreprocessMvPartitionMv() throws Exception { private Pair buildMvProcessor(String query) { ColumnRefFactory columnRefFactory = new ColumnRefFactory(); OptimizerConfig optimizerConfig = new OptimizerConfig(); - OptimizerContext context = new OptimizerContext(new Memo(), columnRefFactory, connectContext, optimizerConfig); + OptimizerContext context = OptimizerFactory.mockContext(connectContext, columnRefFactory, optimizerConfig); try { StatementBase stmt = UtFrameUtils.parseStmtWithNewParser(query, connectContext); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java index f688c8ed147ac..a2f4556b9cd9c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java @@ -18,6 +18,7 @@ import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; @@ -40,11 +41,11 @@ private OptExpression optimize(Optimizer optimizer, String sql) { try { StatementBase stmt = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); QueryStatement queryStatement = (QueryStatement) stmt; - ColumnRefFactory columnRefFactory = new ColumnRefFactory(); - LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext) + LogicalPlan logicalPlan = + new RelationTransformer(optimizer.getContext().getColumnRefFactory(), connectContext) .transformWithSelectLimit(queryStatement.getQueryRelation()); - return optimizer.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory); + return optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), + new ColumnRefSet(logicalPlan.getOutputColumn())); } catch (Exception e) { Assert.fail(e.getMessage()); return null; @@ -59,7 +60,8 @@ public void testSingleTableRewriteStrategy() throws Exception { " as" + " select t1a, id_date, t1b from table_with_partition"); String sql = "select t1a, id_date, t1b from table_with_partition"; - Optimizer optimizer = new Optimizer(); + Optimizer optimizer = + OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, new ColumnRefFactory())); OptExpression optExpression = optimize(optimizer, sql); Assert.assertTrue(optExpression != null); MvRewriteStrategy mvRewriteStrategy = optimizer.getMvRewriteStrategy(); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java index 987cc90772f50..1107c44a379f8 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java @@ -33,8 +33,8 @@ import com.starrocks.sql.optimizer.ExpressionContext; import com.starrocks.sql.optimizer.Group; import com.starrocks.sql.optimizer.GroupExpression; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; @@ -87,7 +87,7 @@ public static void beforeClass() throws Exception { // create connect context connectContext = UtFrameUtils.createDefaultCtx(); columnRefFactory = new ColumnRefFactory(); - optimizerContext = new OptimizerContext(new Memo(), columnRefFactory, connectContext); + optimizerContext = OptimizerFactory.mockContext(connectContext, columnRefFactory); starRocksAssert = new StarRocksAssert(connectContext); ConnectorPlanTestBase.mockAllCatalogs(connectContext, temp.newFolder().toURI().toString()); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java index efb143cb6dbb0..a30e3a277afc7 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java @@ -19,8 +19,8 @@ import com.starrocks.catalog.PartitionInfo; import com.starrocks.common.FeConstants; import com.starrocks.common.Pair; -import com.starrocks.sql.optimizer.Memo; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; @@ -251,7 +251,7 @@ private static Pair buildConjunctAndScan(St private void testRemovePredicate(String sql, String expected) throws Exception { Pair pair = buildConjunctAndScan(sql); StatisticsCalculator calculator = new StatisticsCalculator(); - OptimizerContext context = new OptimizerContext(new Memo(), new ColumnRefFactory()); + OptimizerContext context = OptimizerFactory.mockContext(new ColumnRefFactory()); ScalarOperator newPredicate = calculator.removePartitionPredicate(pair.first, pair.second, context); Assert.assertEquals(expected, newPredicate.toString()); } diff --git a/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java index 3edae4e55ee87..8b3be0c618dbd 100644 --- a/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java +++ b/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java @@ -120,6 +120,8 @@ import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; import com.starrocks.sql.optimizer.OptimizerConfig; +import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerFactory; import com.starrocks.sql.optimizer.QueryMaterializationContext; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; @@ -851,17 +853,15 @@ public static OptExpression getQueryOptExpression(ConnectContext connectContext, OptExpression optimizedPlan; try (Timer t = Tracers.watchScope("Optimizer")) { Optimizer optimizer = null; + OptimizerContext context = OptimizerFactory.mockContext(connectContext, columnRefFactory); if (optimizerConfig != null) { - optimizer = new Optimizer(optimizerConfig); - } else { - optimizer = new Optimizer(); + context.setOptimizerConfig(optimizerConfig); } + optimizer = OptimizerFactory.create(context); optimizedPlan = optimizer.optimize( - connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), - columnRefFactory); + new ColumnRefSet(logicalPlan.getOutputColumn())); } return optimizedPlan; } From bc457de6ea2f66098f144691f1045a676918de42 Mon Sep 17 00:00:00 2001 From: Seaven Date: Thu, 16 Jan 2025 15:52:06 +0800 Subject: [PATCH 2/3] up Signed-off-by: Seaven --- .../starrocks/sql/optimizer/Optimizer.java | 985 +--------------- .../sql/optimizer/OptimizerFactory.java | 2 +- .../sql/optimizer/OptimizerIterface.java | 17 - .../sql/optimizer/QueryOptimizer.java | 1002 +++++++++++++++++ .../MvRewriteStrategyTest.java | 5 +- .../materialization/MvRewriteTest.java | 12 +- 6 files changed, 1015 insertions(+), 1008 deletions(-) delete mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerIterface.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java index 86291f8c17175..032a40454397d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java @@ -11,142 +11,13 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package com.starrocks.sql.optimizer; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.starrocks.analysis.JoinOperator; -import com.starrocks.catalog.MaterializedView; -import com.starrocks.common.profile.Timer; -import com.starrocks.common.profile.Tracers; -import com.starrocks.qe.ConnectContext; -import com.starrocks.qe.SessionVariable; -import com.starrocks.qe.feedback.OperatorTuningGuides; -import com.starrocks.qe.feedback.PlanTuningAdvisor; -import com.starrocks.sql.Explain; -import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; -import com.starrocks.sql.optimizer.cost.CostEstimate; -import com.starrocks.sql.optimizer.operator.Operator; -import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; -import com.starrocks.sql.optimizer.operator.logical.LogicalTreeAnchorOperator; -import com.starrocks.sql.optimizer.operator.logical.LogicalViewScanOperator; -import com.starrocks.sql.optimizer.operator.physical.PhysicalOlapScanOperator; -import com.starrocks.sql.optimizer.rewrite.JoinPredicatePushdown; -import com.starrocks.sql.optimizer.rule.RuleSet; -import com.starrocks.sql.optimizer.rule.implementation.OlapScanImplementationRule; -import com.starrocks.sql.optimizer.rule.join.JoinReorderFactory; -import com.starrocks.sql.optimizer.rule.join.ReorderJoinRule; -import com.starrocks.sql.optimizer.rule.mv.MaterializedViewRule; -import com.starrocks.sql.optimizer.rule.transformation.ApplyExceptionRule; -import com.starrocks.sql.optimizer.rule.transformation.ArrayDistinctAfterAggRule; -import com.starrocks.sql.optimizer.rule.transformation.CTEProduceAddProjectionRule; -import com.starrocks.sql.optimizer.rule.transformation.ConvertToEqualForNullRule; -import com.starrocks.sql.optimizer.rule.transformation.DeriveRangeJoinPredicateRule; -import com.starrocks.sql.optimizer.rule.transformation.EliminateAggRule; -import com.starrocks.sql.optimizer.rule.transformation.EliminateConstantCTERule; -import com.starrocks.sql.optimizer.rule.transformation.EliminateSortColumnWithEqualityPredicateRule; -import com.starrocks.sql.optimizer.rule.transformation.ForceCTEReuseRule; -import com.starrocks.sql.optimizer.rule.transformation.GroupByCountDistinctRewriteRule; -import com.starrocks.sql.optimizer.rule.transformation.IcebergEqualityDeleteRewriteRule; -import com.starrocks.sql.optimizer.rule.transformation.IcebergPartitionsTableRewriteRule; -import com.starrocks.sql.optimizer.rule.transformation.JoinLeftAsscomRule; -import com.starrocks.sql.optimizer.rule.transformation.MaterializedViewTransparentRewriteRule; -import com.starrocks.sql.optimizer.rule.transformation.MergeProjectWithChildRule; -import com.starrocks.sql.optimizer.rule.transformation.MergeTwoAggRule; -import com.starrocks.sql.optimizer.rule.transformation.MergeTwoProjectRule; -import com.starrocks.sql.optimizer.rule.transformation.OnPredicateMoveAroundRule; -import com.starrocks.sql.optimizer.rule.transformation.PartitionColumnMinMaxRewriteRule; -import com.starrocks.sql.optimizer.rule.transformation.PartitionColumnValueOnlyOnScanRule; -import com.starrocks.sql.optimizer.rule.transformation.PruneEmptyWindowRule; -import com.starrocks.sql.optimizer.rule.transformation.PullUpScanPredicateRule; -import com.starrocks.sql.optimizer.rule.transformation.PushDownAggregateGroupingSetsRule; -import com.starrocks.sql.optimizer.rule.transformation.PushDownJoinOnExpressionToChildProject; -import com.starrocks.sql.optimizer.rule.transformation.PushDownLimitRankingWindowRule; -import com.starrocks.sql.optimizer.rule.transformation.PushDownPredicateRankingWindowRule; -import com.starrocks.sql.optimizer.rule.transformation.PushDownProjectLimitRule; -import com.starrocks.sql.optimizer.rule.transformation.PushDownTopNBelowOuterJoinRule; -import com.starrocks.sql.optimizer.rule.transformation.PushDownTopNBelowUnionRule; -import com.starrocks.sql.optimizer.rule.transformation.PushLimitAndFilterToCTEProduceRule; -import com.starrocks.sql.optimizer.rule.transformation.RemoveAggregationFromAggTable; -import com.starrocks.sql.optimizer.rule.transformation.RewriteGroupingSetsByCTERule; -import com.starrocks.sql.optimizer.rule.transformation.RewriteMultiDistinctRule; -import com.starrocks.sql.optimizer.rule.transformation.RewriteSimpleAggToHDFSScanRule; -import com.starrocks.sql.optimizer.rule.transformation.RewriteUnnestBitmapRule; -import com.starrocks.sql.optimizer.rule.transformation.SchemaTableEvaluateRule; -import com.starrocks.sql.optimizer.rule.transformation.SeparateProjectRule; -import com.starrocks.sql.optimizer.rule.transformation.SkewJoinOptimizeRule; -import com.starrocks.sql.optimizer.rule.transformation.SplitScanORToUnionRule; -import com.starrocks.sql.optimizer.rule.transformation.UnionToValuesRule; -import com.starrocks.sql.optimizer.rule.transformation.materialization.MVCompensationPruneUnionRule; -import com.starrocks.sql.optimizer.rule.transformation.materialization.MvRewriteStrategy; -import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils; -import com.starrocks.sql.optimizer.rule.transformation.materialization.rule.TextMatchBasedRewriteRule; -import com.starrocks.sql.optimizer.rule.transformation.pruner.CboTablePruneRule; -import com.starrocks.sql.optimizer.rule.transformation.pruner.PrimaryKeyUpdateTableRule; -import com.starrocks.sql.optimizer.rule.transformation.pruner.RboTablePruneRule; -import com.starrocks.sql.optimizer.rule.transformation.pruner.UniquenessBasedTablePruneRule; -import com.starrocks.sql.optimizer.rule.tree.AddDecodeNodeForDictStringRule; -import com.starrocks.sql.optimizer.rule.tree.AddIndexOnlyPredicateRule; -import com.starrocks.sql.optimizer.rule.tree.ApplyTuningGuideRule; -import com.starrocks.sql.optimizer.rule.tree.CloneDuplicateColRefRule; -import com.starrocks.sql.optimizer.rule.tree.DataCachePopulateRewriteRule; -import com.starrocks.sql.optimizer.rule.tree.EliminateOveruseColumnAccessPathRule; -import com.starrocks.sql.optimizer.rule.tree.ExchangeSortToMergeRule; -import com.starrocks.sql.optimizer.rule.tree.ExtractAggregateColumn; -import com.starrocks.sql.optimizer.rule.tree.InlineCteProjectPruneRule; -import com.starrocks.sql.optimizer.rule.tree.JoinLocalShuffleRule; -import com.starrocks.sql.optimizer.rule.tree.MarkParentRequiredDistributionRule; -import com.starrocks.sql.optimizer.rule.tree.PhysicalDistributionAggOptRule; -import com.starrocks.sql.optimizer.rule.tree.PreAggregateTurnOnRule; -import com.starrocks.sql.optimizer.rule.tree.PredicateReorderRule; -import com.starrocks.sql.optimizer.rule.tree.PruneAggregateNodeRule; -import com.starrocks.sql.optimizer.rule.tree.PruneShuffleColumnRule; -import com.starrocks.sql.optimizer.rule.tree.PruneSubfieldsForComplexType; -import com.starrocks.sql.optimizer.rule.tree.PushDownAggregateRule; -import com.starrocks.sql.optimizer.rule.tree.PushDownDistinctAggregateRule; -import com.starrocks.sql.optimizer.rule.tree.ScalarOperatorsReuseRule; -import com.starrocks.sql.optimizer.rule.tree.SimplifyCaseWhenPredicateRule; -import com.starrocks.sql.optimizer.rule.tree.SubfieldExprNoCopyRule; -import com.starrocks.sql.optimizer.rule.tree.lowcardinality.LowCardinalityRewriteRule; -import com.starrocks.sql.optimizer.rule.tree.prunesubfield.PruneSubfieldRule; -import com.starrocks.sql.optimizer.rule.tree.prunesubfield.PushDownSubfieldRule; -import com.starrocks.sql.optimizer.task.OptimizeGroupTask; -import com.starrocks.sql.optimizer.task.PrepareCollectMetaTask; -import com.starrocks.sql.optimizer.task.TaskContext; -import com.starrocks.sql.optimizer.task.TaskScheduler; -import com.starrocks.sql.optimizer.validate.MVRewriteValidator; -import com.starrocks.sql.optimizer.validate.OptExpressionValidator; -import com.starrocks.sql.optimizer.validate.PlanValidator; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_TRANSPARENT_REWRITE; -import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_UNION_REWRITE; -import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED; -import static com.starrocks.sql.optimizer.rule.RuleType.TF_MATERIALIZED_VIEW; -/** - * Optimizer's entrance class - */ -public class Optimizer { - private static final Logger LOG = LogManager.getLogger(Optimizer.class); - private final OptimizerContext context; - private OptimizerConfig optimizerConfig; - private MvRewriteStrategy mvRewriteStrategy = new MvRewriteStrategy(); - private final TaskScheduler scheduler = new TaskScheduler(); - private Memo memo; - - // collect all LogicalOlapScanOperators in the query before any optimization - private final List allLogicalOlapScanOperators = Lists.newArrayList(); +public abstract class Optimizer { + protected final OptimizerContext context; Optimizer(OptimizerContext context) { this.context = context; @@ -156,854 +27,8 @@ public OptimizerContext getContext() { return context; } - @VisibleForTesting - public MvRewriteStrategy getMvRewriteStrategy() { - return mvRewriteStrategy; - } - - private void prepare(OptExpression logicOperatorTree) { - optimizerConfig = context.getOptimizerConfig(); - - if (!optimizerConfig.isRuleBased()) { - memo = new Memo(); - context.setMemo(memo); - } - context.setTaskScheduler(scheduler); - - // collect all olap scan operator - Utils.extractOperator(logicOperatorTree, allLogicalOlapScanOperators, - op -> op instanceof LogicalOlapScanOperator); - } - - public OptExpression optimize(OptExpression logicOperatorTree, ColumnRefSet requiredColumns) { - return optimize(logicOperatorTree, new PhysicalPropertySet(), requiredColumns); - } - - public OptExpression optimize(OptExpression logicOperatorTree, PhysicalPropertySet requiredProperty, - ColumnRefSet requiredColumns) { - try { - // prepare for optimizer - prepare(logicOperatorTree); - - // prepare for mv rewrite - prepareMvRewrite(context.getConnectContext(), logicOperatorTree, context.getColumnRefFactory(), - requiredColumns); - try (Timer ignored = Tracers.watchScope("MVTextRewrite")) { - logicOperatorTree = new TextMatchBasedRewriteRule(context.getConnectContext(), context.getStatement(), - context.getMvTransformerContext()).transform(logicOperatorTree, context).get(0); - } - - OptExpression result = optimizerConfig.isRuleBased() ? - optimizeByRule(logicOperatorTree, requiredProperty, requiredColumns) : - optimizeByCost(context.getConnectContext(), logicOperatorTree, requiredProperty, - requiredColumns); - return result; - } finally { - // make sure clear caches in OptimizerContext - context.getQueryMaterializationContext().clear(); - context.getConnectContext().setQueryMVContext(null); - } - } - - // Optimize by rule will return logical plan. - // Used by materialized view query rewrite optimization. - private OptExpression optimizeByRule(OptExpression logicOperatorTree, - PhysicalPropertySet requiredProperty, - ColumnRefSet requiredColumns) { - OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", logicOperatorTree); - TaskContext rootTaskContext = - new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE); - logicOperatorTree = rewriteAndValidatePlan(logicOperatorTree, rootTaskContext); - OptimizerTraceUtil.log("after logical rewrite, new logicOperatorTree:\n%s", logicOperatorTree); - return logicOperatorTree; - } - - /** - * Optimizer will transform and implement the logical operator based on - * the {@see Rule}, then cost the physical operator, and finally find the - * lowest cost physical operator tree - * - * @param logicOperatorTree the input for query Optimizer - * @param requiredProperty the required physical property from sql or groupExpression - * @param requiredColumns the required output columns from sql or groupExpression - * @return the lowest cost physical operator for this query - */ - private OptExpression optimizeByCost(ConnectContext connectContext, - OptExpression logicOperatorTree, - PhysicalPropertySet requiredProperty, - ColumnRefSet requiredColumns) { - // Phase 1: none - OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", logicOperatorTree); - // Phase 2: rewrite based on memo and group - TaskContext rootTaskContext = - new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE); - - try (Timer ignored = Tracers.watchScope("RuleBaseOptimize")) { - logicOperatorTree = rewriteAndValidatePlan(logicOperatorTree, rootTaskContext); - } - - if (logicOperatorTree.getShortCircuit()) { - return logicOperatorTree; - } - - Preconditions.checkNotNull(memo); - memo.init(logicOperatorTree); - if (context.getQueryMaterializationContext() != null) { - // LogicalTreeWithView is logically equivalent to logicOperatorTree - addViewBasedPlanIntoMemo(context.getQueryMaterializationContext().getQueryOptPlanWithView()); - } - OptimizerTraceUtil.log("after logical rewrite, root group:\n%s", memo.getRootGroup()); - - // Currently, we cache output columns in logic property. - // We derive logic property Bottom Up firstly when new group added to memo, - // but we do column prune rewrite top down later. - // So after column prune rewrite, the output columns for each operator maybe change, - // but the logic property is cached and never change. - // So we need to explicitly derive all group logic property again - memo.deriveAllGroupLogicalProperty(); - - // Phase 3: optimize based on memo and group - try (Timer ignored = Tracers.watchScope("CostBaseOptimize")) { - memoOptimize(connectContext, memo, rootTaskContext); - } - - OptExpression result; - if (connectContext.getSessionVariable().isSetUseNthExecPlan()) { - // extract the nth execution plan - int nthExecPlan = connectContext.getSessionVariable().getUseNthExecPlan(); - result = EnumeratePlan.extractNthPlan(requiredProperty, memo.getRootGroup(), nthExecPlan); - } else { - result = extractBestPlan(requiredProperty, memo.getRootGroup()); - } - OptimizerTraceUtil.logOptExpression("after extract best plan:\n%s", result); - - // set costs audio log before physicalRuleRewrite - // statistics won't set correctly after physicalRuleRewrite. - // we need set plan costs before physical rewrite stage. - final CostEstimate costs = Explain.buildCost(result); - connectContext.getAuditEventBuilder().setPlanCpuCosts(costs.getCpuCost()) - .setPlanMemCosts(costs.getMemoryCost()); - OptExpression finalPlan; - try (Timer ignored = Tracers.watchScope("PhysicalRewrite")) { - finalPlan = physicalRuleRewrite(connectContext, rootTaskContext, result); - OptimizerTraceUtil.logOptExpression("final plan after physical rewrite:\n%s", finalPlan); - } - - try (Timer ignored = Tracers.watchScope("DynamicRewrite")) { - finalPlan = dynamicRewrite(connectContext, rootTaskContext, finalPlan); - OptimizerTraceUtil.logOptExpression("final plan after dynamic rewrite:\n%s", finalPlan); - } - - // collect all mv scan operator - List mvScan = collectAllPhysicalOlapScanOperators(result).stream(). - filter(scan -> scan.getTable().isMaterializedView()).collect(Collectors.toList()); - // add mv db id to currentSqlDbIds, the resource group could use this to distinguish sql patterns - Set currentSqlDbIds = context.getConnectContext().getCurrentSqlDbIds(); - mvScan.stream().map(scan -> ((MaterializedView) scan.getTable()).getDbId()).forEach(currentSqlDbIds::add); - - try (Timer ignored = Tracers.watchScope("PlanValidate")) { - // valid the final plan - PlanValidator.getInstance().validatePlan(finalPlan, rootTaskContext); - // validate mv and log tracer if needed - MVRewriteValidator mvRewriteValidator = new MVRewriteValidator(allLogicalOlapScanOperators); - mvRewriteValidator.validateMV(connectContext, finalPlan, rootTaskContext); - // audit mv - mvRewriteValidator.auditMv(connectContext, finalPlan, rootTaskContext); - return finalPlan; - } - } - - private void addViewBasedPlanIntoMemo(OptExpression logicalTreeWithView) { - if (logicalTreeWithView == null) { - return; - } - Memo memo = context.getMemo(); - memo.copyIn(memo.getRootGroup(), logicalTreeWithView); - } - - - - private void prepareMvRewrite(ConnectContext connectContext, OptExpression logicOperatorTree, - ColumnRefFactory columnRefFactory, ColumnRefSet requiredColumns) { - SessionVariable sessionVariable = connectContext.getSessionVariable(); - // MV Rewrite will be used when cbo is enabled. - if (context.getOptimizerConfig().isRuleBased() || sessionVariable.isDisableMaterializedViewRewrite() || - !sessionVariable.isEnableMaterializedViewRewrite()) { - return; - } - // prepare related mvs if needed and initialize mv rewrite strategy - new MvRewritePreprocessor(connectContext, columnRefFactory, context, requiredColumns) - .prepare(logicOperatorTree); - - // initialize mv rewrite strategy finally - mvRewriteStrategy = MvRewriteStrategy.prepareRewriteStrategy(context, connectContext, logicOperatorTree); - OptimizerTraceUtil.logMVPrepare("MV rewrite strategy: {}", mvRewriteStrategy); - - // TODO(stephen): enable agg push down when query exists related mvs. - if (context.getQueryMaterializationContext() != null && - !context.getQueryMaterializationContext().getValidCandidateMVs().isEmpty()) { - context.getSessionVariable().setCboPushDownAggregateMode(-1); - } - } - - private void pruneTables(OptExpression tree, TaskContext rootTaskContext, ColumnRefSet requiredColumns) { - if (rootTaskContext.getOptimizerContext().getSessionVariable().isEnableRboTablePrune()) { - if (!Utils.hasPrunableJoin(tree)) { - return; - } - // PARTITION_PRUNE is required to run before ReorderJoinRule because ReorderJoinRule's - // Statistics calculation on Operators depends on row count yielded by the PARTITION_PRUNE. - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES); - // ReorderJoinRule is a in-memo rule, when it is used outside memo, we must apply - // MergeProjectWithChildRule to merge LogicalProjectionOperator into its child's - // projection before ReorderJoinRule's application, after that, we must separate operator's - // projection as LogicalProjectionOperator from the operator by applying SeparateProjectRule. - scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); - scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule()); - CTEUtils.collectForceCteStatisticsOutsideMemo(tree, context); - tree = new UniquenessBasedTablePruneRule().rewrite(tree, rootTaskContext); - deriveLogicalProperty(tree); - tree = new ReorderJoinRule().rewrite(tree, context); - tree = new SeparateProjectRule().rewrite(tree, rootTaskContext); - deriveLogicalProperty(tree); - // TODO(by satanson): bucket shuffle join interpolation in PK table's update query can adjust layout - // of the data ingested by OlapTableSink and eliminate race introduced by multiple concurrent write - // operations on the same tablets, pruning this bucket shuffle join make update statement performance - // regression, so we can turn on this rule after we put an bucket-shuffle exchange in front of - // OlapTableSink in future, at present we turn off this rule. - if (rootTaskContext.getOptimizerContext().getSessionVariable().isEnableTablePruneOnUpdate()) { - tree = new PrimaryKeyUpdateTableRule().rewrite(tree, rootTaskContext); - deriveLogicalProperty(tree); - } - tree = new RboTablePruneRule().rewrite(tree, rootTaskContext); - scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); - rootTaskContext.setRequiredColumns(requiredColumns.clone()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); - } - } - - /** - * Rewrite transparent materialized view. - */ - private OptExpression transparentMVRewrite(OptExpression tree, TaskContext rootTaskContext) { - scheduler.rewriteOnce(tree, rootTaskContext, new MaterializedViewTransparentRewriteRule()); - if (Utils.isOptHasAppliedRule(tree, OP_MV_TRANSPARENT_REWRITE)) { - tree = new SeparateProjectRule().rewrite(tree, rootTaskContext); - } - return tree; - } - - private void ruleBasedMaterializedViewRewrite(OptExpression tree, - TaskContext rootTaskContext, - ColumnRefSet requiredColumns) { - // skip if mv rewrite is disabled - if (!mvRewriteStrategy.enableMaterializedViewRewrite || context.getQueryMaterializationContext() == null) { - return; - } - - // do rule based mv rewrite if needed - if (!context.getQueryMaterializationContext().hasRewrittenSuccess()) { - doRuleBasedMaterializedViewRewrite(tree, rootTaskContext); - } - - // NOTE: Since union rewrite will generate Filter -> Union -> OlapScan -> OlapScan, need to push filter below Union - // and do partition predicate again. - // TODO: move this into doRuleBasedMaterializedViewRewrite - // TODO: Do it in CBO if needed later. - boolean isNeedFurtherPartitionPrune = - Utils.isOptHasAppliedRule(tree, op -> op.isOpRuleBitSet(OP_MV_UNION_REWRITE)); - OptimizerTraceUtil.logMVPrepare("is further partition prune: {}", isNeedFurtherPartitionPrune); - if (isNeedFurtherPartitionPrune && context.getQueryMaterializationContext().hasRewrittenSuccess()) { - // reset partition prune bit to do partition prune again. - MvUtils.getScanOperator(tree).forEach(scan -> { - scan.resetOpRuleBit(OP_PARTITION_PRUNED); - }); - // Do predicate push down if union rewrite successes. - tree = new SeparateProjectRule().rewrite(tree, rootTaskContext); - deriveLogicalProperty(tree); - // Do partition prune again to avoid unnecessary scan. - rootTaskContext.setRequiredColumns(requiredColumns.clone()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); - // It's necessary for external table since its predicate is not used directly after push down. - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_EMPTY_OPERATOR_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, new MVCompensationPruneUnionRule()); - scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); - } - } - - private void doRuleBasedMaterializedViewRewrite(OptExpression tree, - TaskContext rootTaskContext) { - if (mvRewriteStrategy.enableViewBasedRewrite) { - // try view based mv rewrite first, then try normal mv rewrite rules - viewBasedMvRuleRewrite(tree, rootTaskContext); - } - if (mvRewriteStrategy.enableForceRBORewrite) { - // use rule based mv rewrite strategy to do mv rewrite for multi tables query - if (mvRewriteStrategy.enableMultiTableRewrite) { - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.MULTI_TABLE_MV_REWRITE_RULES); - } - if (mvRewriteStrategy.enableSingleTableRewrite) { - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES); - } - } else if (mvRewriteStrategy.enableSingleTableRewrite) { - // now add single table materialized view rewrite rules in rule based rewrite phase to boost optimization - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES); - } - } - - private void doMVRewriteWithMultiStages(OptExpression tree, - TaskContext rootTaskContext) { - if (!mvRewriteStrategy.enableMaterializedViewRewrite || !mvRewriteStrategy.mvStrategy.isMultiStages()) { - return; - } - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); - scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule()); - // do rule based mv rewrite - doRuleBasedMaterializedViewRewrite(tree, rootTaskContext); - new SeparateProjectRule().rewrite(tree, rootTaskContext); - deriveLogicalProperty(tree); - } - - private OptExpression logicalRuleRewrite( - OptExpression tree, - TaskContext rootTaskContext) { - rootTaskContext.getOptimizerContext().setShortCircuit(tree.getShortCircuit()); - tree = OptExpression.createForShortCircuit(new LogicalTreeAnchorOperator(), tree, tree.getShortCircuit()); - // for short circuit - Optional result = ruleRewriteForShortCircuit(tree, rootTaskContext); - if (result.isPresent()) { - return result.get(); - } - - ColumnRefSet requiredColumns = rootTaskContext.getRequiredColumns().clone(); - deriveLogicalProperty(tree); - - SessionVariable sessionVariable = rootTaskContext.getOptimizerContext().getSessionVariable(); - CTEContext cteContext = context.getCteContext(); - CTEUtils.collectCteOperators(tree, context); - - // see JoinPredicatePushdown - JoinPredicatePushdown.JoinPredicatePushDownContext joinPredicatePushDownContext = - context.getJoinPushDownParams(); - joinPredicatePushDownContext.prepare(context, sessionVariable, mvRewriteStrategy); - - // inline CTE if consume use once - while (cteContext.hasInlineCTE()) { - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INLINE_CTE_RULES); - CTEUtils.collectCteOperators(tree, context); - } - - scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule()); - CTEUtils.collectCteOperators(tree, context); - - scheduler.rewriteOnce(tree, rootTaskContext, new IcebergPartitionsTableRewriteRule()); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.AGGREGATE_REWRITE_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_SUBQUERY_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_COMMON_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_TO_WINDOW_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_TO_JOIN_RULES); - scheduler.rewriteOnce(tree, rootTaskContext, new ApplyExceptionRule()); - CTEUtils.collectCteOperators(tree, context); - - if (sessionVariable.isEnableFineGrainedRangePredicate()) { - scheduler.rewriteAtMostOnce(tree, rootTaskContext, RuleSet.FINE_GRAINED_RANGE_PREDICATE_RULES); - } - - // rewrite transparent materialized view - tree = transparentMVRewrite(tree, rootTaskContext); - - // Note: PUSH_DOWN_PREDICATE tasks should be executed before MERGE_LIMIT tasks - // because of the Filter node needs to be merged first to avoid the Limit node - // cannot merge - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); - scheduler.rewriteOnce(tree, rootTaskContext, SchemaTableEvaluateRule.getInstance()); - - scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.ELIMINATE_OP_WITH_CONSTANT_RULES); - scheduler.rewriteOnce(tree, rootTaskContext, new PushDownPredicateRankingWindowRule()); - - scheduler.rewriteOnce(tree, rootTaskContext, new ConvertToEqualForNullRule()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); - // Put EliminateAggRule after PRUNE_COLUMNS to give a chance to prune group bys before eliminate aggregations. - scheduler.rewriteOnce(tree, rootTaskContext, EliminateAggRule.getInstance()); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_UKFK_JOIN_RULES); - deriveLogicalProperty(tree); - - scheduler.rewriteOnce(tree, rootTaskContext, new PushDownJoinOnExpressionToChildProject()); - - scheduler.rewriteIterative(tree, rootTaskContext, new PruneEmptyWindowRule()); - // @todo: resolve recursive optimization question: - // MergeAgg -> PruneColumn -> PruneEmptyWindow -> MergeAgg/Project -> PruneColumn... - scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoAggRule()); - - rootTaskContext.setRequiredColumns(requiredColumns.clone()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); - - pruneTables(tree, rootTaskContext, requiredColumns); - - scheduler.rewriteIterative(tree, rootTaskContext, new PruneEmptyWindowRule()); - scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); - - // rule-based materialized view rewrite: early stage - doMVRewriteWithMultiStages(tree, rootTaskContext); - joinPredicatePushDownContext.reset(); - - // Limit push must be after the column prune, - // otherwise the Node containing limit may be prune - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, new PushDownProjectLimitRule()); - - scheduler.rewriteOnce(tree, rootTaskContext, new PushDownLimitRankingWindowRule()); - rewriteGroupingSets(tree, rootTaskContext, sessionVariable); - - // No heavy metadata operation before external table partition prune - prepareMetaOnlyOnce(tree, rootTaskContext); - - // apply skew join optimize after push down join on expression to child project, - // we need to compute the stats of child project(like subfield). - skewJoinOptimize(tree, rootTaskContext); - scheduler.rewriteOnce(tree, rootTaskContext, new IcebergEqualityDeleteRewriteRule()); - - tree = pruneSubfield(tree, rootTaskContext, requiredColumns); - - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_ASSERT_ROW_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES); - - CTEUtils.collectCteOperators(tree, context); - if (cteContext.needOptimizeCTE()) { - cteContext.reset(); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.COLLECT_CTE_RULES); - rootTaskContext.setRequiredColumns(requiredColumns.clone()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); - if (cteContext.needPushLimit() || cteContext.needPushPredicate()) { - scheduler.rewriteOnce(tree, rootTaskContext, new PushLimitAndFilterToCTEProduceRule()); - } - - if (cteContext.needPushPredicate()) { - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); - } - - if (cteContext.needPushLimit()) { - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES); - } - - scheduler.rewriteOnce(tree, rootTaskContext, new ForceCTEReuseRule()); - } - - // Add a config to decide whether to rewrite sync mv. - if (!optimizerConfig.isRuleDisable(TF_MATERIALIZED_VIEW) - && sessionVariable.isEnableSyncMaterializedViewRewrite()) { - // Split or predicates to union all so can be used by mv rewrite to choose the best sort key indexes. - // TODO: support adaptive for or-predicates to union all. - if (SplitScanORToUnionRule.isForceRewrite()) { - scheduler.rewriteOnce(tree, rootTaskContext, SplitScanORToUnionRule.getInstance()); - } - - OptimizerTraceUtil.logOptExpression("before MaterializedViewRule:\n%s", tree); - tree = new MaterializedViewRule().transform(tree, context).get(0); - OptimizerTraceUtil.logOptExpression("after MaterializedViewRule:\n%s", tree); - - deriveLogicalProperty(tree); - } - - scheduler.rewriteDownTop(tree, rootTaskContext, OnPredicateMoveAroundRule.INSTANCE); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); - - scheduler.rewriteIterative(tree, rootTaskContext, new PartitionColumnMinMaxRewriteRule()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, new RewriteMultiDistinctRule()); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_EMPTY_OPERATOR_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, new CTEProduceAddProjectionRule()); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES); - - // ArrayDistinctAfterAggRule must run before pushDownAggregation, - // because push down agg won't have array_distinct project - if (sessionVariable.getEnableArrayDistinctAfterAggOpt()) { - scheduler.rewriteOnce(tree, rootTaskContext, new ArrayDistinctAfterAggRule()); - } - - tree = pushDownAggregation(tree, rootTaskContext, requiredColumns); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES); - - CTEUtils.collectCteOperators(tree, context); - // inline CTE if consume use once - while (cteContext.hasInlineCTE()) { - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INLINE_CTE_RULES); - CTEUtils.collectCteOperators(tree, context); - } - - scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.META_SCAN_REWRITE_RULES); - scheduler.rewriteOnce(tree, rootTaskContext, new PartitionColumnValueOnlyOnScanRule()); - // before MergeProjectWithChildRule, after INLINE_CTE and MergeApplyWithTableFunction - scheduler.rewriteIterative(tree, rootTaskContext, RewriteUnnestBitmapRule.getInstance()); - - // After this rule, we shouldn't generate logical project operator - scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule()); - - scheduler.rewriteOnce(tree, rootTaskContext, new EliminateSortColumnWithEqualityPredicateRule()); - scheduler.rewriteOnce(tree, rootTaskContext, new PushDownTopNBelowOuterJoinRule()); - // intersect rewrite depend on statistics - Utils.calculateStatistics(tree, rootTaskContext.getOptimizerContext()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INTERSECT_REWRITE_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, new RemoveAggregationFromAggTable()); - - scheduler.rewriteOnce(tree, rootTaskContext, SplitScanORToUnionRule.getInstance()); - scheduler.rewriteOnce(tree, rootTaskContext, new PushDownTopNBelowUnionRule()); - - // rule based materialized view rewrite - ruleBasedMaterializedViewRewrite(tree, rootTaskContext, requiredColumns); - - // this rewrite rule should be after mv. - scheduler.rewriteOnce(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.SCAN_NO_PROJECT); - - // NOTE: This rule should be after MV Rewrite because MV Rewrite cannot handle - // select count(distinct c) from t group by a, b - // if this rule has applied before MV. - scheduler.rewriteOnce(tree, rootTaskContext, new GroupByCountDistinctRewriteRule()); - - scheduler.rewriteOnce(tree, rootTaskContext, new DeriveRangeJoinPredicateRule()); - - scheduler.rewriteOnce(tree, rootTaskContext, UnionToValuesRule.getInstance()); - - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.VECTOR_REWRITE_RULES); - // this rule should be after mv - // @TODO: it can also be applied to other table scan operator - if (context.getSessionVariable().isEnableScanPredicateExprReuse()) { - scheduler.rewriteOnce(tree, rootTaskContext, PullUpScanPredicateRule.OLAP_SCAN); - } - - tree = SimplifyCaseWhenPredicateRule.INSTANCE.rewrite(tree, rootTaskContext); - deriveLogicalProperty(tree); - return tree.getInputs().get(0); - } - - private void rewriteGroupingSets(OptExpression tree, TaskContext rootTaskContext, SessionVariable sessionVariable) { - if (sessionVariable.isEnableRewriteGroupingsetsToUnionAll()) { - scheduler.rewriteIterative(tree, rootTaskContext, new RewriteGroupingSetsByCTERule()); - } - if (sessionVariable.isCboPushDownGroupingSet()) { - scheduler.rewriteOnce(tree, rootTaskContext, new PushDownAggregateGroupingSetsRule()); - } - } - - private Optional ruleRewriteForShortCircuit(OptExpression tree, TaskContext rootTaskContext) { - Boolean isShortCircuit = tree.getShortCircuit(); - - if (isShortCircuit) { - deriveLogicalProperty(tree); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SHORT_CIRCUIT_SET_RULES); - scheduler.rewriteOnce(tree, rootTaskContext, new MergeProjectWithChildRule()); - OptExpression result = tree.getInputs().get(0); - result.setShortCircuit(true); - return Optional.of(result); - } - return Optional.empty(); - } - - // for single scan node, to make sure we can rewrite - private void viewBasedMvRuleRewrite(OptExpression tree, TaskContext rootTaskContext) { - QueryMaterializationContext queryMaterializationContext = context.getQueryMaterializationContext(); - Preconditions.checkArgument(queryMaterializationContext != null); - - try (Timer ignored = Tracers.watchScope("MVViewRewrite")) { - OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "try VIEW_BASED_MV_REWRITE"); - OptExpression treeWithView = queryMaterializationContext.getQueryOptPlanWithView(); - // should add a LogicalTreeAnchorOperator for rewrite - treeWithView = OptExpression.create(new LogicalTreeAnchorOperator(), treeWithView); - if (mvRewriteStrategy.enableMultiTableRewrite) { - scheduler.rewriteIterative(treeWithView, rootTaskContext, RuleSet.MULTI_TABLE_MV_REWRITE_RULES); - } - if (mvRewriteStrategy.enableSingleTableRewrite) { - scheduler.rewriteIterative(treeWithView, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES); - } - - List leftViewScanOperators = Lists.newArrayList(); - MvUtils.collectViewScanOperator(treeWithView, leftViewScanOperators); - List origQueryViewScanOperators = - queryMaterializationContext.getQueryViewScanOps(); - if (leftViewScanOperators.size() < origQueryViewScanOperators.size()) { - // replace original tree plan - tree.setChild(0, treeWithView.inputAt(0)); - deriveLogicalProperty(tree); - - // if there are view scan operator left, we should replace it back to original plans - if (!leftViewScanOperators.isEmpty()) { - MvUtils.replaceLogicalViewScanOperator(tree); - } - } - OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "original view scans size: {}, " + - "left view scans size: {}", origQueryViewScanOperators.size(), leftViewScanOperators.size()); - } catch (Exception e) { - OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", - "single table view based mv rule rewrite failed.", e); - } - } - - private OptExpression rewriteAndValidatePlan( - OptExpression tree, - TaskContext rootTaskContext) { - OptExpression result = logicalRuleRewrite(tree, rootTaskContext); - OptExpressionValidator validator = new OptExpressionValidator(); - validator.validate(result); - // skip memo - if (result.getShortCircuit()) { - result = new OlapScanImplementationRule().transform(result, null).get(0); - result.setShortCircuit(true); - } - return result; - } - - private OptExpression pushDownAggregation(OptExpression tree, TaskContext rootTaskContext, - ColumnRefSet requiredColumns) { - boolean pushDistinctFlag = false; - boolean pushAggFlag = false; - if (context.getSessionVariable().isCboPushDownDistinctBelowWindow()) { - // TODO(by satanson): in future, PushDownDistinctAggregateRule and PushDownAggregateRule should be - // fused one rule to tackle with all scenarios of agg push-down. - PushDownDistinctAggregateRule rule = new PushDownDistinctAggregateRule(rootTaskContext); - tree = rule.rewrite(tree, rootTaskContext); - pushDistinctFlag = rule.getRewriter().hasRewrite(); - } - - if (context.getSessionVariable().getCboPushDownAggregateMode() != -1) { - if (context.getSessionVariable().isCboPushDownAggregateOnBroadcastJoin()) { - // Reorder joins before applying PushDownAggregateRule to better decide where to push down aggregator. - // For example, do not push down a not very efficient aggregator below a very small broadcast join. - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES); - scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); - scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule()); - CTEUtils.collectForceCteStatisticsOutsideMemo(tree, context); - deriveLogicalProperty(tree); - tree = new ReorderJoinRule().rewrite(tree, JoinReorderFactory.createJoinReorderAdaptive(), context); - tree = new SeparateProjectRule().rewrite(tree, rootTaskContext); - deriveLogicalProperty(tree); - Utils.calculateStatistics(tree, context); - } - - PushDownAggregateRule rule = new PushDownAggregateRule(rootTaskContext); - rule.getRewriter().collectRewriteContext(tree); - if (rule.getRewriter().isNeedRewrite()) { - pushAggFlag = true; - tree = rule.rewrite(tree, rootTaskContext); - } - } - - if (pushDistinctFlag || pushAggFlag) { - deriveLogicalProperty(tree); - rootTaskContext.setRequiredColumns(requiredColumns.clone()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); - scheduler.rewriteOnce(tree, rootTaskContext, EliminateAggRule.getInstance()); - } - - return tree; - } - - private void skewJoinOptimize(OptExpression tree, TaskContext rootTaskContext) { - if (context.getSessionVariable().isEnableStatsToOptimizeSkewJoin()) { - // merge projects before calculate statistics - scheduler.rewriteOnce(tree, rootTaskContext, new MergeTwoProjectRule()); - Utils.calculateStatistics(tree, rootTaskContext.getOptimizerContext()); - } - scheduler.rewriteOnce(tree, rootTaskContext, new SkewJoinOptimizeRule()); - } - - private OptExpression pruneSubfield(OptExpression tree, TaskContext rootTaskContext, ColumnRefSet requiredColumns) { - if (!context.getSessionVariable().isCboPruneSubfield()) { - return tree; - } - - PushDownSubfieldRule pushDownRule = new PushDownSubfieldRule(); - tree = pushDownRule.rewrite(tree, rootTaskContext); - - if (pushDownRule.hasRewrite()) { - rootTaskContext.setRequiredColumns(requiredColumns.clone()); - scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); - } - scheduler.rewriteOnce(tree, rootTaskContext, new PruneSubfieldRule()); - - return tree; - } - - private void deriveLogicalProperty(OptExpression root) { - for (OptExpression child : root.getInputs()) { - deriveLogicalProperty(child); - } - - ExpressionContext context = new ExpressionContext(root); - context.deriveLogicalProperty(); - root.setLogicalProperty(context.getRootProperty()); - } - - void memoOptimize(ConnectContext connectContext, Memo memo, TaskContext rootTaskContext) { - context.setInMemoPhase(true); - OptExpression tree = memo.getRootGroup().extractLogicalTree(); - SessionVariable sessionVariable = connectContext.getSessionVariable(); - // add CboTablePruneRule - if (Utils.countJoinNodeSize(tree, CboTablePruneRule.JOIN_TYPES) < 10 && - sessionVariable.isEnableCboTablePrune()) { - context.getRuleSet().addCboTablePruneRule(); - } - // Join reorder - int innerCrossJoinNode = Utils.countJoinNodeSize(tree, JoinOperator.innerCrossJoinSet()); - if (!sessionVariable.isDisableJoinReorder() && innerCrossJoinNode < sessionVariable.getCboMaxReorderNode()) { - if (innerCrossJoinNode > sessionVariable.getCboMaxReorderNodeUseExhaustive()) { - CTEUtils.collectForceCteStatistics(memo, context); - - OptimizerTraceUtil.logOptExpression("before ReorderJoinRule:\n%s", tree); - new ReorderJoinRule().transform(tree, context); - OptimizerTraceUtil.logOptExpression("after ReorderJoinRule:\n%s", tree); - - context.getRuleSet().addJoinCommutativityWithoutInnerRule(); - } else { - if (Utils.countJoinNodeSize(tree, JoinOperator.semiAntiJoinSet()) < - sessionVariable.getCboMaxReorderNodeUseExhaustive()) { - context.getRuleSet().getTransformRules().add(JoinLeftAsscomRule.INNER_JOIN_LEFT_ASSCOM_RULE); - } - context.getRuleSet().addJoinTransformationRules(); - } - } - - if (!sessionVariable.isDisableJoinReorder() && sessionVariable.isEnableOuterJoinReorder() - && Utils.capableOuterReorder(tree, sessionVariable.getCboReorderThresholdUseExhaustive())) { - context.getRuleSet().addOuterJoinTransformationRules(); - } - - if (!sessionVariable.isMVPlanner()) { - // add join implementRule - String joinImplementationMode = connectContext.getSessionVariable().getJoinImplementationMode(); - if ("merge".equalsIgnoreCase(joinImplementationMode)) { - context.getRuleSet().addMergeJoinImplementationRule(); - } else if ("hash".equalsIgnoreCase(joinImplementationMode)) { - context.getRuleSet().addHashJoinImplementationRule(); - } else if ("nestloop".equalsIgnoreCase(joinImplementationMode)) { - context.getRuleSet().addNestLoopJoinImplementationRule(); - } else { - context.getRuleSet().addAutoJoinImplementationRule(); - } - } else { - context.getRuleSet().addRealtimeMVRules(); - } - - if (mvRewriteStrategy.enableMultiTableRewrite) { - context.getRuleSet().addSingleTableMvRewriteRule(); - context.getRuleSet().addMultiTableMvRewriteRule(); - } - - scheduler.pushTask(new OptimizeGroupTask(rootTaskContext, memo.getRootGroup())); - scheduler.executeTasks(rootTaskContext); - } - - private OptExpression physicalRuleRewrite(ConnectContext connectContext, TaskContext rootTaskContext, - OptExpression result) { - Preconditions.checkState(result.getOp().isPhysical()); - - int planCount = result.getPlanCount(); - - // Since there may be many different plans in the logic phase, it's possible - // that this switch can't turned on after logical optimization, so we only determine - // whether the PreAggregate can be turned on in the final - result = new PreAggregateTurnOnRule().rewrite(result, rootTaskContext); - - // Rewrite Exchange on top of Sort to Final Sort - result = new ExchangeSortToMergeRule().rewrite(result, rootTaskContext); - result = new PruneAggregateNodeRule().rewrite(result, rootTaskContext); - result = new PruneShuffleColumnRule().rewrite(result, rootTaskContext); - result = new PhysicalDistributionAggOptRule().rewrite(result, rootTaskContext); - result = new AddDecodeNodeForDictStringRule().rewrite(result, rootTaskContext); - result = new LowCardinalityRewriteRule().rewrite(result, rootTaskContext); - // Put before ScalarOperatorsReuseRule - result = new PruneSubfieldsForComplexType().rewrite(result, rootTaskContext); - result = new InlineCteProjectPruneRule().rewrite(result, rootTaskContext); - // This rule should be last - result = new ScalarOperatorsReuseRule().rewrite(result, rootTaskContext); - // Reorder predicates - result = new PredicateReorderRule(rootTaskContext.getOptimizerContext().getSessionVariable()).rewrite(result, - rootTaskContext); - result = new ExtractAggregateColumn().rewrite(result, rootTaskContext); - result = new JoinLocalShuffleRule().rewrite(result, rootTaskContext); - - // This must be put at last of the optimization. Because wrapping reused ColumnRefOperator with CloneOperator - // too early will prevent it from certain optimizations that depend on the equivalence of the ColumnRefOperator. - result = new CloneDuplicateColRefRule().rewrite(result, rootTaskContext); - - // set subfield expr copy flag - if (rootTaskContext.getOptimizerContext().getSessionVariable().getEnableSubfieldNoCopy()) { - result = new SubfieldExprNoCopyRule().rewrite(result, rootTaskContext); - } - - result = new AddIndexOnlyPredicateRule().rewrite(result, rootTaskContext); - result = new DataCachePopulateRewriteRule(connectContext).rewrite(result, rootTaskContext); - result = new EliminateOveruseColumnAccessPathRule().rewrite(result, rootTaskContext); - result.setPlanCount(planCount); - return result; - } - - private OptExpression dynamicRewrite(ConnectContext connectContext, TaskContext rootTaskContext, - OptExpression result) { - // update the existRequiredDistribution value in optExpression. The next rules need it to determine - // if we can change the distribution to adjust the plan because of skew data, bad statistics or something else. - result = new MarkParentRequiredDistributionRule().rewrite(result, rootTaskContext); - result = new ApplyTuningGuideRule(connectContext).rewrite(result, rootTaskContext); + public abstract OptExpression optimize(OptExpression logicOperatorTree, ColumnRefSet requiredColumns); - OperatorTuningGuides.OptimizedRecord optimizedRecord = PlanTuningAdvisor.getInstance() - .getOptimizedRecord(context.getQueryId()); - if (optimizedRecord != null) { - Tracers.record(Tracers.Module.BASE, "DynamicApplyTuningGuides", optimizedRecord.getExplainString()); - } - return result; - } - - /** - * Extract the lowest cost physical operator tree from memo - * - * @param requiredProperty the required physical property from sql or groupExpression - * @param rootGroup the current group to find the lowest cost physical operator - * @return the lowest cost physical operator for this query - */ - private OptExpression extractBestPlan(PhysicalPropertySet requiredProperty, - Group rootGroup) { - GroupExpression groupExpression = rootGroup.getBestExpression(requiredProperty); - if (groupExpression == null) { - String msg = "no executable plan for this sql. group: %s. required property: %s"; - throw new IllegalArgumentException(String.format(msg, rootGroup, requiredProperty)); - } - List inputProperties = groupExpression.getInputProperties(requiredProperty); - - List childPlans = Lists.newArrayList(); - for (int i = 0; i < groupExpression.arity(); ++i) { - OptExpression childPlan = extractBestPlan(inputProperties.get(i), groupExpression.inputAt(i)); - childPlans.add(childPlan); - } - - OptExpression expression = OptExpression.create(groupExpression.getOp(), - childPlans); - // record inputProperties at optExpression, used for planFragment builder to determine join type - expression.setRequiredProperties(inputProperties); - expression.setStatistics(groupExpression.getGroup().getStatistics()); - expression.setCost(groupExpression.getCost(requiredProperty)); - expression.setOutputProperty(requiredProperty); - - // When build plan fragment, we need the output column of logical property - expression.setLogicalProperty(rootGroup.getLogicalProperty()); - return expression; - } - - private List collectAllPhysicalOlapScanOperators(OptExpression tree) { - List list = Lists.newArrayList(); - Utils.extractOperator(tree, list, op -> op instanceof PhysicalOlapScanOperator); - return list; - } - - private void prepareMetaOnlyOnce(OptExpression tree, TaskContext rootTaskContext) { - if (rootTaskContext.getOptimizerContext().getSessionVariable().enableParallelPrepareMetadata()) { - scheduler.pushTask(new PrepareCollectMetaTask(rootTaskContext, tree)); - scheduler.executeTasks(rootTaskContext); - } - } + public abstract OptExpression optimize(OptExpression logicOperatorTree, PhysicalPropertySet requiredProperty, + ColumnRefSet requiredColumns); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java index 8b189086ee712..8c06e7eae9430 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java @@ -54,6 +54,6 @@ public static OptimizerContext initContext(ConnectContext context, ColumnRefFact } public static Optimizer create(OptimizerContext context) { - return new Optimizer(context); + return new QueryOptimizer(context); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerIterface.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerIterface.java deleted file mode 100644 index 36b4244a6f7ed..0000000000000 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerIterface.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2021-present StarRocks, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.starrocks.sql.optimizer; - -public interface OptimizerIterface { -} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java new file mode 100644 index 0000000000000..d8b549eb4e2b9 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java @@ -0,0 +1,1002 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.starrocks.analysis.JoinOperator; +import com.starrocks.catalog.MaterializedView; +import com.starrocks.common.profile.Timer; +import com.starrocks.common.profile.Tracers; +import com.starrocks.qe.ConnectContext; +import com.starrocks.qe.SessionVariable; +import com.starrocks.qe.feedback.OperatorTuningGuides; +import com.starrocks.qe.feedback.PlanTuningAdvisor; +import com.starrocks.sql.Explain; +import com.starrocks.sql.optimizer.base.ColumnRefFactory; +import com.starrocks.sql.optimizer.base.ColumnRefSet; +import com.starrocks.sql.optimizer.base.PhysicalPropertySet; +import com.starrocks.sql.optimizer.cost.CostEstimate; +import com.starrocks.sql.optimizer.operator.Operator; +import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; +import com.starrocks.sql.optimizer.operator.logical.LogicalTreeAnchorOperator; +import com.starrocks.sql.optimizer.operator.logical.LogicalViewScanOperator; +import com.starrocks.sql.optimizer.operator.physical.PhysicalOlapScanOperator; +import com.starrocks.sql.optimizer.rewrite.JoinPredicatePushdown; +import com.starrocks.sql.optimizer.rule.RuleSet; +import com.starrocks.sql.optimizer.rule.implementation.OlapScanImplementationRule; +import com.starrocks.sql.optimizer.rule.join.JoinReorderFactory; +import com.starrocks.sql.optimizer.rule.join.ReorderJoinRule; +import com.starrocks.sql.optimizer.rule.mv.MaterializedViewRule; +import com.starrocks.sql.optimizer.rule.transformation.ApplyExceptionRule; +import com.starrocks.sql.optimizer.rule.transformation.ArrayDistinctAfterAggRule; +import com.starrocks.sql.optimizer.rule.transformation.CTEProduceAddProjectionRule; +import com.starrocks.sql.optimizer.rule.transformation.ConvertToEqualForNullRule; +import com.starrocks.sql.optimizer.rule.transformation.DeriveRangeJoinPredicateRule; +import com.starrocks.sql.optimizer.rule.transformation.EliminateAggRule; +import com.starrocks.sql.optimizer.rule.transformation.EliminateConstantCTERule; +import com.starrocks.sql.optimizer.rule.transformation.EliminateSortColumnWithEqualityPredicateRule; +import com.starrocks.sql.optimizer.rule.transformation.ForceCTEReuseRule; +import com.starrocks.sql.optimizer.rule.transformation.GroupByCountDistinctRewriteRule; +import com.starrocks.sql.optimizer.rule.transformation.IcebergEqualityDeleteRewriteRule; +import com.starrocks.sql.optimizer.rule.transformation.IcebergPartitionsTableRewriteRule; +import com.starrocks.sql.optimizer.rule.transformation.JoinLeftAsscomRule; +import com.starrocks.sql.optimizer.rule.transformation.MaterializedViewTransparentRewriteRule; +import com.starrocks.sql.optimizer.rule.transformation.MergeProjectWithChildRule; +import com.starrocks.sql.optimizer.rule.transformation.MergeTwoAggRule; +import com.starrocks.sql.optimizer.rule.transformation.MergeTwoProjectRule; +import com.starrocks.sql.optimizer.rule.transformation.OnPredicateMoveAroundRule; +import com.starrocks.sql.optimizer.rule.transformation.PartitionColumnMinMaxRewriteRule; +import com.starrocks.sql.optimizer.rule.transformation.PartitionColumnValueOnlyOnScanRule; +import com.starrocks.sql.optimizer.rule.transformation.PruneEmptyWindowRule; +import com.starrocks.sql.optimizer.rule.transformation.PullUpScanPredicateRule; +import com.starrocks.sql.optimizer.rule.transformation.PushDownAggregateGroupingSetsRule; +import com.starrocks.sql.optimizer.rule.transformation.PushDownJoinOnExpressionToChildProject; +import com.starrocks.sql.optimizer.rule.transformation.PushDownLimitRankingWindowRule; +import com.starrocks.sql.optimizer.rule.transformation.PushDownPredicateRankingWindowRule; +import com.starrocks.sql.optimizer.rule.transformation.PushDownProjectLimitRule; +import com.starrocks.sql.optimizer.rule.transformation.PushDownTopNBelowOuterJoinRule; +import com.starrocks.sql.optimizer.rule.transformation.PushDownTopNBelowUnionRule; +import com.starrocks.sql.optimizer.rule.transformation.PushLimitAndFilterToCTEProduceRule; +import com.starrocks.sql.optimizer.rule.transformation.RemoveAggregationFromAggTable; +import com.starrocks.sql.optimizer.rule.transformation.RewriteGroupingSetsByCTERule; +import com.starrocks.sql.optimizer.rule.transformation.RewriteMultiDistinctRule; +import com.starrocks.sql.optimizer.rule.transformation.RewriteSimpleAggToHDFSScanRule; +import com.starrocks.sql.optimizer.rule.transformation.RewriteUnnestBitmapRule; +import com.starrocks.sql.optimizer.rule.transformation.SchemaTableEvaluateRule; +import com.starrocks.sql.optimizer.rule.transformation.SeparateProjectRule; +import com.starrocks.sql.optimizer.rule.transformation.SkewJoinOptimizeRule; +import com.starrocks.sql.optimizer.rule.transformation.SplitScanORToUnionRule; +import com.starrocks.sql.optimizer.rule.transformation.UnionToValuesRule; +import com.starrocks.sql.optimizer.rule.transformation.materialization.MVCompensationPruneUnionRule; +import com.starrocks.sql.optimizer.rule.transformation.materialization.MvRewriteStrategy; +import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils; +import com.starrocks.sql.optimizer.rule.transformation.materialization.rule.TextMatchBasedRewriteRule; +import com.starrocks.sql.optimizer.rule.transformation.pruner.CboTablePruneRule; +import com.starrocks.sql.optimizer.rule.transformation.pruner.PrimaryKeyUpdateTableRule; +import com.starrocks.sql.optimizer.rule.transformation.pruner.RboTablePruneRule; +import com.starrocks.sql.optimizer.rule.transformation.pruner.UniquenessBasedTablePruneRule; +import com.starrocks.sql.optimizer.rule.tree.AddDecodeNodeForDictStringRule; +import com.starrocks.sql.optimizer.rule.tree.AddIndexOnlyPredicateRule; +import com.starrocks.sql.optimizer.rule.tree.ApplyTuningGuideRule; +import com.starrocks.sql.optimizer.rule.tree.CloneDuplicateColRefRule; +import com.starrocks.sql.optimizer.rule.tree.DataCachePopulateRewriteRule; +import com.starrocks.sql.optimizer.rule.tree.EliminateOveruseColumnAccessPathRule; +import com.starrocks.sql.optimizer.rule.tree.ExchangeSortToMergeRule; +import com.starrocks.sql.optimizer.rule.tree.ExtractAggregateColumn; +import com.starrocks.sql.optimizer.rule.tree.InlineCteProjectPruneRule; +import com.starrocks.sql.optimizer.rule.tree.JoinLocalShuffleRule; +import com.starrocks.sql.optimizer.rule.tree.MarkParentRequiredDistributionRule; +import com.starrocks.sql.optimizer.rule.tree.PhysicalDistributionAggOptRule; +import com.starrocks.sql.optimizer.rule.tree.PreAggregateTurnOnRule; +import com.starrocks.sql.optimizer.rule.tree.PredicateReorderRule; +import com.starrocks.sql.optimizer.rule.tree.PruneAggregateNodeRule; +import com.starrocks.sql.optimizer.rule.tree.PruneShuffleColumnRule; +import com.starrocks.sql.optimizer.rule.tree.PruneSubfieldsForComplexType; +import com.starrocks.sql.optimizer.rule.tree.PushDownAggregateRule; +import com.starrocks.sql.optimizer.rule.tree.PushDownDistinctAggregateRule; +import com.starrocks.sql.optimizer.rule.tree.ScalarOperatorsReuseRule; +import com.starrocks.sql.optimizer.rule.tree.SimplifyCaseWhenPredicateRule; +import com.starrocks.sql.optimizer.rule.tree.SubfieldExprNoCopyRule; +import com.starrocks.sql.optimizer.rule.tree.lowcardinality.LowCardinalityRewriteRule; +import com.starrocks.sql.optimizer.rule.tree.prunesubfield.PruneSubfieldRule; +import com.starrocks.sql.optimizer.rule.tree.prunesubfield.PushDownSubfieldRule; +import com.starrocks.sql.optimizer.task.OptimizeGroupTask; +import com.starrocks.sql.optimizer.task.PrepareCollectMetaTask; +import com.starrocks.sql.optimizer.task.TaskContext; +import com.starrocks.sql.optimizer.task.TaskScheduler; +import com.starrocks.sql.optimizer.validate.MVRewriteValidator; +import com.starrocks.sql.optimizer.validate.OptExpressionValidator; +import com.starrocks.sql.optimizer.validate.PlanValidator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_TRANSPARENT_REWRITE; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_UNION_REWRITE; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED; +import static com.starrocks.sql.optimizer.rule.RuleType.TF_MATERIALIZED_VIEW; + +/** + * QueryOptimizer's entrance class + */ +public class QueryOptimizer extends Optimizer { + private static final Logger LOG = LogManager.getLogger(QueryOptimizer.class); + private OptimizerConfig optimizerConfig; + private MvRewriteStrategy mvRewriteStrategy = new MvRewriteStrategy(); + private final TaskScheduler scheduler = new TaskScheduler(); + private Memo memo; + + // collect all LogicalOlapScanOperators in the query before any optimization + private final List allLogicalOlapScanOperators = Lists.newArrayList(); + + QueryOptimizer(OptimizerContext context) { + super(context); + } + + @VisibleForTesting + public MvRewriteStrategy getMvRewriteStrategy() { + return mvRewriteStrategy; + } + + private void prepare(OptExpression logicOperatorTree) { + optimizerConfig = context.getOptimizerConfig(); + + if (!optimizerConfig.isRuleBased()) { + memo = new Memo(); + context.setMemo(memo); + } + context.setTaskScheduler(scheduler); + + // collect all olap scan operator + Utils.extractOperator(logicOperatorTree, allLogicalOlapScanOperators, + op -> op instanceof LogicalOlapScanOperator); + } + + public OptExpression optimize(OptExpression logicOperatorTree, ColumnRefSet requiredColumns) { + return optimize(logicOperatorTree, new PhysicalPropertySet(), requiredColumns); + } + + public OptExpression optimize(OptExpression logicOperatorTree, PhysicalPropertySet requiredProperty, + ColumnRefSet requiredColumns) { + try { + // prepare for optimizer + prepare(logicOperatorTree); + + // prepare for mv rewrite + prepareMvRewrite(context.getConnectContext(), logicOperatorTree, context.getColumnRefFactory(), + requiredColumns); + try (Timer ignored = Tracers.watchScope("MVTextRewrite")) { + logicOperatorTree = new TextMatchBasedRewriteRule(context.getConnectContext(), context.getStatement(), + context.getMvTransformerContext()).transform(logicOperatorTree, context).get(0); + } + + OptExpression result = optimizerConfig.isRuleBased() ? + optimizeByRule(logicOperatorTree, requiredProperty, requiredColumns) : + optimizeByCost(context.getConnectContext(), logicOperatorTree, requiredProperty, + requiredColumns); + return result; + } finally { + // make sure clear caches in OptimizerContext + context.getQueryMaterializationContext().clear(); + context.getConnectContext().setQueryMVContext(null); + } + } + + // Optimize by rule will return logical plan. + // Used by materialized view query rewrite optimization. + private OptExpression optimizeByRule(OptExpression logicOperatorTree, + PhysicalPropertySet requiredProperty, + ColumnRefSet requiredColumns) { + OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", logicOperatorTree); + TaskContext rootTaskContext = + new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE); + logicOperatorTree = rewriteAndValidatePlan(logicOperatorTree, rootTaskContext); + OptimizerTraceUtil.log("after logical rewrite, new logicOperatorTree:\n%s", logicOperatorTree); + return logicOperatorTree; + } + + /** + * Optimizer will transform and implement the logical operator based on + * the {@see Rule}, then cost the physical operator, and finally find the + * lowest cost physical operator tree + * + * @param logicOperatorTree the input for query Optimizer + * @param requiredProperty the required physical property from sql or groupExpression + * @param requiredColumns the required output columns from sql or groupExpression + * @return the lowest cost physical operator for this query + */ + private OptExpression optimizeByCost(ConnectContext connectContext, + OptExpression logicOperatorTree, + PhysicalPropertySet requiredProperty, + ColumnRefSet requiredColumns) { + // Phase 1: none + OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", logicOperatorTree); + // Phase 2: rewrite based on memo and group + TaskContext rootTaskContext = + new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE); + + try (Timer ignored = Tracers.watchScope("RuleBaseOptimize")) { + logicOperatorTree = rewriteAndValidatePlan(logicOperatorTree, rootTaskContext); + } + + if (logicOperatorTree.getShortCircuit()) { + return logicOperatorTree; + } + + Preconditions.checkNotNull(memo); + memo.init(logicOperatorTree); + if (context.getQueryMaterializationContext() != null) { + // LogicalTreeWithView is logically equivalent to logicOperatorTree + addViewBasedPlanIntoMemo(context.getQueryMaterializationContext().getQueryOptPlanWithView()); + } + OptimizerTraceUtil.log("after logical rewrite, root group:\n%s", memo.getRootGroup()); + + // Currently, we cache output columns in logic property. + // We derive logic property Bottom Up firstly when new group added to memo, + // but we do column prune rewrite top down later. + // So after column prune rewrite, the output columns for each operator maybe change, + // but the logic property is cached and never change. + // So we need to explicitly derive all group logic property again + memo.deriveAllGroupLogicalProperty(); + + // Phase 3: optimize based on memo and group + try (Timer ignored = Tracers.watchScope("CostBaseOptimize")) { + memoOptimize(connectContext, memo, rootTaskContext); + } + + OptExpression result; + if (connectContext.getSessionVariable().isSetUseNthExecPlan()) { + // extract the nth execution plan + int nthExecPlan = connectContext.getSessionVariable().getUseNthExecPlan(); + result = EnumeratePlan.extractNthPlan(requiredProperty, memo.getRootGroup(), nthExecPlan); + } else { + result = extractBestPlan(requiredProperty, memo.getRootGroup()); + } + OptimizerTraceUtil.logOptExpression("after extract best plan:\n%s", result); + + // set costs audio log before physicalRuleRewrite + // statistics won't set correctly after physicalRuleRewrite. + // we need set plan costs before physical rewrite stage. + final CostEstimate costs = Explain.buildCost(result); + connectContext.getAuditEventBuilder().setPlanCpuCosts(costs.getCpuCost()) + .setPlanMemCosts(costs.getMemoryCost()); + OptExpression finalPlan; + try (Timer ignored = Tracers.watchScope("PhysicalRewrite")) { + finalPlan = physicalRuleRewrite(connectContext, rootTaskContext, result); + OptimizerTraceUtil.logOptExpression("final plan after physical rewrite:\n%s", finalPlan); + } + + try (Timer ignored = Tracers.watchScope("DynamicRewrite")) { + finalPlan = dynamicRewrite(connectContext, rootTaskContext, finalPlan); + OptimizerTraceUtil.logOptExpression("final plan after dynamic rewrite:\n%s", finalPlan); + } + + // collect all mv scan operator + List mvScan = collectAllPhysicalOlapScanOperators(result).stream(). + filter(scan -> scan.getTable().isMaterializedView()).collect(Collectors.toList()); + // add mv db id to currentSqlDbIds, the resource group could use this to distinguish sql patterns + Set currentSqlDbIds = context.getConnectContext().getCurrentSqlDbIds(); + mvScan.stream().map(scan -> ((MaterializedView) scan.getTable()).getDbId()).forEach(currentSqlDbIds::add); + + try (Timer ignored = Tracers.watchScope("PlanValidate")) { + // valid the final plan + PlanValidator.getInstance().validatePlan(finalPlan, rootTaskContext); + // validate mv and log tracer if needed + MVRewriteValidator mvRewriteValidator = new MVRewriteValidator(allLogicalOlapScanOperators); + mvRewriteValidator.validateMV(connectContext, finalPlan, rootTaskContext); + // audit mv + mvRewriteValidator.auditMv(connectContext, finalPlan, rootTaskContext); + return finalPlan; + } + } + + private void addViewBasedPlanIntoMemo(OptExpression logicalTreeWithView) { + if (logicalTreeWithView == null) { + return; + } + Memo memo = context.getMemo(); + memo.copyIn(memo.getRootGroup(), logicalTreeWithView); + } + + private void prepareMvRewrite(ConnectContext connectContext, OptExpression logicOperatorTree, + ColumnRefFactory columnRefFactory, ColumnRefSet requiredColumns) { + SessionVariable sessionVariable = connectContext.getSessionVariable(); + // MV Rewrite will be used when cbo is enabled. + if (context.getOptimizerConfig().isRuleBased() || sessionVariable.isDisableMaterializedViewRewrite() || + !sessionVariable.isEnableMaterializedViewRewrite()) { + return; + } + // prepare related mvs if needed and initialize mv rewrite strategy + new MvRewritePreprocessor(connectContext, columnRefFactory, context, requiredColumns) + .prepare(logicOperatorTree); + + // initialize mv rewrite strategy finally + mvRewriteStrategy = MvRewriteStrategy.prepareRewriteStrategy(context, connectContext, logicOperatorTree); + OptimizerTraceUtil.logMVPrepare("MV rewrite strategy: {}", mvRewriteStrategy); + + // TODO(stephen): enable agg push down when query exists related mvs. + if (context.getQueryMaterializationContext() != null && + !context.getQueryMaterializationContext().getValidCandidateMVs().isEmpty()) { + context.getSessionVariable().setCboPushDownAggregateMode(-1); + } + } + + private void pruneTables(OptExpression tree, TaskContext rootTaskContext, ColumnRefSet requiredColumns) { + if (rootTaskContext.getOptimizerContext().getSessionVariable().isEnableRboTablePrune()) { + if (!Utils.hasPrunableJoin(tree)) { + return; + } + // PARTITION_PRUNE is required to run before ReorderJoinRule because ReorderJoinRule's + // Statistics calculation on Operators depends on row count yielded by the PARTITION_PRUNE. + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES); + // ReorderJoinRule is a in-memo rule, when it is used outside memo, we must apply + // MergeProjectWithChildRule to merge LogicalProjectionOperator into its child's + // projection before ReorderJoinRule's application, after that, we must separate operator's + // projection as LogicalProjectionOperator from the operator by applying SeparateProjectRule. + scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); + scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule()); + CTEUtils.collectForceCteStatisticsOutsideMemo(tree, context); + tree = new UniquenessBasedTablePruneRule().rewrite(tree, rootTaskContext); + deriveLogicalProperty(tree); + tree = new ReorderJoinRule().rewrite(tree, context); + tree = new SeparateProjectRule().rewrite(tree, rootTaskContext); + deriveLogicalProperty(tree); + // TODO(by satanson): bucket shuffle join interpolation in PK table's update query can adjust layout + // of the data ingested by OlapTableSink and eliminate race introduced by multiple concurrent write + // operations on the same tablets, pruning this bucket shuffle join make update statement performance + // regression, so we can turn on this rule after we put an bucket-shuffle exchange in front of + // OlapTableSink in future, at present we turn off this rule. + if (rootTaskContext.getOptimizerContext().getSessionVariable().isEnableTablePruneOnUpdate()) { + tree = new PrimaryKeyUpdateTableRule().rewrite(tree, rootTaskContext); + deriveLogicalProperty(tree); + } + tree = new RboTablePruneRule().rewrite(tree, rootTaskContext); + scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); + rootTaskContext.setRequiredColumns(requiredColumns.clone()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); + } + } + + /** + * Rewrite transparent materialized view. + */ + private OptExpression transparentMVRewrite(OptExpression tree, TaskContext rootTaskContext) { + scheduler.rewriteOnce(tree, rootTaskContext, new MaterializedViewTransparentRewriteRule()); + if (Utils.isOptHasAppliedRule(tree, OP_MV_TRANSPARENT_REWRITE)) { + tree = new SeparateProjectRule().rewrite(tree, rootTaskContext); + } + return tree; + } + + private void ruleBasedMaterializedViewRewrite(OptExpression tree, + TaskContext rootTaskContext, + ColumnRefSet requiredColumns) { + // skip if mv rewrite is disabled + if (!mvRewriteStrategy.enableMaterializedViewRewrite || context.getQueryMaterializationContext() == null) { + return; + } + + // do rule based mv rewrite if needed + if (!context.getQueryMaterializationContext().hasRewrittenSuccess()) { + doRuleBasedMaterializedViewRewrite(tree, rootTaskContext); + } + + // NOTE: Since union rewrite will generate Filter -> Union -> OlapScan -> OlapScan, need to push filter below Union + // and do partition predicate again. + // TODO: move this into doRuleBasedMaterializedViewRewrite + // TODO: Do it in CBO if needed later. + boolean isNeedFurtherPartitionPrune = + Utils.isOptHasAppliedRule(tree, op -> op.isOpRuleBitSet(OP_MV_UNION_REWRITE)); + OptimizerTraceUtil.logMVPrepare("is further partition prune: {}", isNeedFurtherPartitionPrune); + if (isNeedFurtherPartitionPrune && context.getQueryMaterializationContext().hasRewrittenSuccess()) { + // reset partition prune bit to do partition prune again. + MvUtils.getScanOperator(tree).forEach(scan -> { + scan.resetOpRuleBit(OP_PARTITION_PRUNED); + }); + // Do predicate push down if union rewrite successes. + tree = new SeparateProjectRule().rewrite(tree, rootTaskContext); + deriveLogicalProperty(tree); + // Do partition prune again to avoid unnecessary scan. + rootTaskContext.setRequiredColumns(requiredColumns.clone()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); + // It's necessary for external table since its predicate is not used directly after push down. + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_EMPTY_OPERATOR_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, new MVCompensationPruneUnionRule()); + scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); + } + } + + private void doRuleBasedMaterializedViewRewrite(OptExpression tree, + TaskContext rootTaskContext) { + if (mvRewriteStrategy.enableViewBasedRewrite) { + // try view based mv rewrite first, then try normal mv rewrite rules + viewBasedMvRuleRewrite(tree, rootTaskContext); + } + if (mvRewriteStrategy.enableForceRBORewrite) { + // use rule based mv rewrite strategy to do mv rewrite for multi tables query + if (mvRewriteStrategy.enableMultiTableRewrite) { + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.MULTI_TABLE_MV_REWRITE_RULES); + } + if (mvRewriteStrategy.enableSingleTableRewrite) { + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES); + } + } else if (mvRewriteStrategy.enableSingleTableRewrite) { + // now add single table materialized view rewrite rules in rule based rewrite phase to boost optimization + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES); + } + } + + private void doMVRewriteWithMultiStages(OptExpression tree, + TaskContext rootTaskContext) { + if (!mvRewriteStrategy.enableMaterializedViewRewrite || !mvRewriteStrategy.mvStrategy.isMultiStages()) { + return; + } + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); + scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule()); + // do rule based mv rewrite + doRuleBasedMaterializedViewRewrite(tree, rootTaskContext); + new SeparateProjectRule().rewrite(tree, rootTaskContext); + deriveLogicalProperty(tree); + } + + private OptExpression logicalRuleRewrite( + OptExpression tree, + TaskContext rootTaskContext) { + rootTaskContext.getOptimizerContext().setShortCircuit(tree.getShortCircuit()); + tree = OptExpression.createForShortCircuit(new LogicalTreeAnchorOperator(), tree, tree.getShortCircuit()); + // for short circuit + Optional result = ruleRewriteForShortCircuit(tree, rootTaskContext); + if (result.isPresent()) { + return result.get(); + } + + ColumnRefSet requiredColumns = rootTaskContext.getRequiredColumns().clone(); + deriveLogicalProperty(tree); + + SessionVariable sessionVariable = rootTaskContext.getOptimizerContext().getSessionVariable(); + CTEContext cteContext = context.getCteContext(); + CTEUtils.collectCteOperators(tree, context); + + // see JoinPredicatePushdown + JoinPredicatePushdown.JoinPredicatePushDownContext joinPredicatePushDownContext = + context.getJoinPushDownParams(); + joinPredicatePushDownContext.prepare(context, sessionVariable, mvRewriteStrategy); + + // inline CTE if consume use once + while (cteContext.hasInlineCTE()) { + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INLINE_CTE_RULES); + CTEUtils.collectCteOperators(tree, context); + } + + scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule()); + CTEUtils.collectCteOperators(tree, context); + + scheduler.rewriteOnce(tree, rootTaskContext, new IcebergPartitionsTableRewriteRule()); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.AGGREGATE_REWRITE_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_SUBQUERY_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_COMMON_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_TO_WINDOW_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_TO_JOIN_RULES); + scheduler.rewriteOnce(tree, rootTaskContext, new ApplyExceptionRule()); + CTEUtils.collectCteOperators(tree, context); + + if (sessionVariable.isEnableFineGrainedRangePredicate()) { + scheduler.rewriteAtMostOnce(tree, rootTaskContext, RuleSet.FINE_GRAINED_RANGE_PREDICATE_RULES); + } + + // rewrite transparent materialized view + tree = transparentMVRewrite(tree, rootTaskContext); + + // Note: PUSH_DOWN_PREDICATE tasks should be executed before MERGE_LIMIT tasks + // because of the Filter node needs to be merged first to avoid the Limit node + // cannot merge + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); + scheduler.rewriteOnce(tree, rootTaskContext, SchemaTableEvaluateRule.getInstance()); + + scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.ELIMINATE_OP_WITH_CONSTANT_RULES); + scheduler.rewriteOnce(tree, rootTaskContext, new PushDownPredicateRankingWindowRule()); + + scheduler.rewriteOnce(tree, rootTaskContext, new ConvertToEqualForNullRule()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); + // Put EliminateAggRule after PRUNE_COLUMNS to give a chance to prune group bys before eliminate aggregations. + scheduler.rewriteOnce(tree, rootTaskContext, EliminateAggRule.getInstance()); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_UKFK_JOIN_RULES); + deriveLogicalProperty(tree); + + scheduler.rewriteOnce(tree, rootTaskContext, new PushDownJoinOnExpressionToChildProject()); + + scheduler.rewriteIterative(tree, rootTaskContext, new PruneEmptyWindowRule()); + // @todo: resolve recursive optimization question: + // MergeAgg -> PruneColumn -> PruneEmptyWindow -> MergeAgg/Project -> PruneColumn... + scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoAggRule()); + + rootTaskContext.setRequiredColumns(requiredColumns.clone()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); + + pruneTables(tree, rootTaskContext, requiredColumns); + + scheduler.rewriteIterative(tree, rootTaskContext, new PruneEmptyWindowRule()); + scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); + + // rule-based materialized view rewrite: early stage + doMVRewriteWithMultiStages(tree, rootTaskContext); + joinPredicatePushDownContext.reset(); + + // Limit push must be after the column prune, + // otherwise the Node containing limit may be prune + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, new PushDownProjectLimitRule()); + + scheduler.rewriteOnce(tree, rootTaskContext, new PushDownLimitRankingWindowRule()); + rewriteGroupingSets(tree, rootTaskContext, sessionVariable); + + // No heavy metadata operation before external table partition prune + prepareMetaOnlyOnce(tree, rootTaskContext); + + // apply skew join optimize after push down join on expression to child project, + // we need to compute the stats of child project(like subfield). + skewJoinOptimize(tree, rootTaskContext); + scheduler.rewriteOnce(tree, rootTaskContext, new IcebergEqualityDeleteRewriteRule()); + + tree = pruneSubfield(tree, rootTaskContext, requiredColumns); + + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_ASSERT_ROW_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES); + + CTEUtils.collectCteOperators(tree, context); + if (cteContext.needOptimizeCTE()) { + cteContext.reset(); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.COLLECT_CTE_RULES); + rootTaskContext.setRequiredColumns(requiredColumns.clone()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); + if (cteContext.needPushLimit() || cteContext.needPushPredicate()) { + scheduler.rewriteOnce(tree, rootTaskContext, new PushLimitAndFilterToCTEProduceRule()); + } + + if (cteContext.needPushPredicate()) { + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); + } + + if (cteContext.needPushLimit()) { + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES); + } + + scheduler.rewriteOnce(tree, rootTaskContext, new ForceCTEReuseRule()); + } + + // Add a config to decide whether to rewrite sync mv. + if (!optimizerConfig.isRuleDisable(TF_MATERIALIZED_VIEW) + && sessionVariable.isEnableSyncMaterializedViewRewrite()) { + // Split or predicates to union all so can be used by mv rewrite to choose the best sort key indexes. + // TODO: support adaptive for or-predicates to union all. + if (SplitScanORToUnionRule.isForceRewrite()) { + scheduler.rewriteOnce(tree, rootTaskContext, SplitScanORToUnionRule.getInstance()); + } + + OptimizerTraceUtil.logOptExpression("before MaterializedViewRule:\n%s", tree); + tree = new MaterializedViewRule().transform(tree, context).get(0); + OptimizerTraceUtil.logOptExpression("after MaterializedViewRule:\n%s", tree); + + deriveLogicalProperty(tree); + } + + scheduler.rewriteDownTop(tree, rootTaskContext, OnPredicateMoveAroundRule.INSTANCE); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); + + scheduler.rewriteIterative(tree, rootTaskContext, new PartitionColumnMinMaxRewriteRule()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, new RewriteMultiDistinctRule()); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_EMPTY_OPERATOR_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, new CTEProduceAddProjectionRule()); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES); + + // ArrayDistinctAfterAggRule must run before pushDownAggregation, + // because push down agg won't have array_distinct project + if (sessionVariable.getEnableArrayDistinctAfterAggOpt()) { + scheduler.rewriteOnce(tree, rootTaskContext, new ArrayDistinctAfterAggRule()); + } + + tree = pushDownAggregation(tree, rootTaskContext, requiredColumns); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES); + + CTEUtils.collectCteOperators(tree, context); + // inline CTE if consume use once + while (cteContext.hasInlineCTE()) { + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INLINE_CTE_RULES); + CTEUtils.collectCteOperators(tree, context); + } + + scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.META_SCAN_REWRITE_RULES); + scheduler.rewriteOnce(tree, rootTaskContext, new PartitionColumnValueOnlyOnScanRule()); + // before MergeProjectWithChildRule, after INLINE_CTE and MergeApplyWithTableFunction + scheduler.rewriteIterative(tree, rootTaskContext, RewriteUnnestBitmapRule.getInstance()); + + // After this rule, we shouldn't generate logical project operator + scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule()); + + scheduler.rewriteOnce(tree, rootTaskContext, new EliminateSortColumnWithEqualityPredicateRule()); + scheduler.rewriteOnce(tree, rootTaskContext, new PushDownTopNBelowOuterJoinRule()); + // intersect rewrite depend on statistics + Utils.calculateStatistics(tree, rootTaskContext.getOptimizerContext()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INTERSECT_REWRITE_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, new RemoveAggregationFromAggTable()); + + scheduler.rewriteOnce(tree, rootTaskContext, SplitScanORToUnionRule.getInstance()); + scheduler.rewriteOnce(tree, rootTaskContext, new PushDownTopNBelowUnionRule()); + + // rule based materialized view rewrite + ruleBasedMaterializedViewRewrite(tree, rootTaskContext, requiredColumns); + + // this rewrite rule should be after mv. + scheduler.rewriteOnce(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.SCAN_NO_PROJECT); + + // NOTE: This rule should be after MV Rewrite because MV Rewrite cannot handle + // select count(distinct c) from t group by a, b + // if this rule has applied before MV. + scheduler.rewriteOnce(tree, rootTaskContext, new GroupByCountDistinctRewriteRule()); + + scheduler.rewriteOnce(tree, rootTaskContext, new DeriveRangeJoinPredicateRule()); + + scheduler.rewriteOnce(tree, rootTaskContext, UnionToValuesRule.getInstance()); + + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.VECTOR_REWRITE_RULES); + // this rule should be after mv + // @TODO: it can also be applied to other table scan operator + if (context.getSessionVariable().isEnableScanPredicateExprReuse()) { + scheduler.rewriteOnce(tree, rootTaskContext, PullUpScanPredicateRule.OLAP_SCAN); + } + + tree = SimplifyCaseWhenPredicateRule.INSTANCE.rewrite(tree, rootTaskContext); + deriveLogicalProperty(tree); + return tree.getInputs().get(0); + } + + private void rewriteGroupingSets(OptExpression tree, TaskContext rootTaskContext, SessionVariable sessionVariable) { + if (sessionVariable.isEnableRewriteGroupingsetsToUnionAll()) { + scheduler.rewriteIterative(tree, rootTaskContext, new RewriteGroupingSetsByCTERule()); + } + if (sessionVariable.isCboPushDownGroupingSet()) { + scheduler.rewriteOnce(tree, rootTaskContext, new PushDownAggregateGroupingSetsRule()); + } + } + + private Optional ruleRewriteForShortCircuit(OptExpression tree, TaskContext rootTaskContext) { + Boolean isShortCircuit = tree.getShortCircuit(); + + if (isShortCircuit) { + deriveLogicalProperty(tree); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SHORT_CIRCUIT_SET_RULES); + scheduler.rewriteOnce(tree, rootTaskContext, new MergeProjectWithChildRule()); + OptExpression result = tree.getInputs().get(0); + result.setShortCircuit(true); + return Optional.of(result); + } + return Optional.empty(); + } + + // for single scan node, to make sure we can rewrite + private void viewBasedMvRuleRewrite(OptExpression tree, TaskContext rootTaskContext) { + QueryMaterializationContext queryMaterializationContext = context.getQueryMaterializationContext(); + Preconditions.checkArgument(queryMaterializationContext != null); + + try (Timer ignored = Tracers.watchScope("MVViewRewrite")) { + OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "try VIEW_BASED_MV_REWRITE"); + OptExpression treeWithView = queryMaterializationContext.getQueryOptPlanWithView(); + // should add a LogicalTreeAnchorOperator for rewrite + treeWithView = OptExpression.create(new LogicalTreeAnchorOperator(), treeWithView); + if (mvRewriteStrategy.enableMultiTableRewrite) { + scheduler.rewriteIterative(treeWithView, rootTaskContext, RuleSet.MULTI_TABLE_MV_REWRITE_RULES); + } + if (mvRewriteStrategy.enableSingleTableRewrite) { + scheduler.rewriteIterative(treeWithView, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES); + } + + List leftViewScanOperators = Lists.newArrayList(); + MvUtils.collectViewScanOperator(treeWithView, leftViewScanOperators); + List origQueryViewScanOperators = + queryMaterializationContext.getQueryViewScanOps(); + if (leftViewScanOperators.size() < origQueryViewScanOperators.size()) { + // replace original tree plan + tree.setChild(0, treeWithView.inputAt(0)); + deriveLogicalProperty(tree); + + // if there are view scan operator left, we should replace it back to original plans + if (!leftViewScanOperators.isEmpty()) { + MvUtils.replaceLogicalViewScanOperator(tree); + } + } + OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "original view scans size: {}, " + + "left view scans size: {}", origQueryViewScanOperators.size(), leftViewScanOperators.size()); + } catch (Exception e) { + OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", + "single table view based mv rule rewrite failed.", e); + } + } + + private OptExpression rewriteAndValidatePlan( + OptExpression tree, + TaskContext rootTaskContext) { + OptExpression result = logicalRuleRewrite(tree, rootTaskContext); + OptExpressionValidator validator = new OptExpressionValidator(); + validator.validate(result); + // skip memo + if (result.getShortCircuit()) { + result = new OlapScanImplementationRule().transform(result, null).get(0); + result.setShortCircuit(true); + } + return result; + } + + private OptExpression pushDownAggregation(OptExpression tree, TaskContext rootTaskContext, + ColumnRefSet requiredColumns) { + boolean pushDistinctFlag = false; + boolean pushAggFlag = false; + if (context.getSessionVariable().isCboPushDownDistinctBelowWindow()) { + // TODO(by satanson): in future, PushDownDistinctAggregateRule and PushDownAggregateRule should be + // fused one rule to tackle with all scenarios of agg push-down. + PushDownDistinctAggregateRule rule = new PushDownDistinctAggregateRule(rootTaskContext); + tree = rule.rewrite(tree, rootTaskContext); + pushDistinctFlag = rule.getRewriter().hasRewrite(); + } + + if (context.getSessionVariable().getCboPushDownAggregateMode() != -1) { + if (context.getSessionVariable().isCboPushDownAggregateOnBroadcastJoin()) { + // Reorder joins before applying PushDownAggregateRule to better decide where to push down aggregator. + // For example, do not push down a not very efficient aggregator below a very small broadcast join. + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES); + scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); + scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule()); + CTEUtils.collectForceCteStatisticsOutsideMemo(tree, context); + deriveLogicalProperty(tree); + tree = new ReorderJoinRule().rewrite(tree, JoinReorderFactory.createJoinReorderAdaptive(), context); + tree = new SeparateProjectRule().rewrite(tree, rootTaskContext); + deriveLogicalProperty(tree); + Utils.calculateStatistics(tree, context); + } + + PushDownAggregateRule rule = new PushDownAggregateRule(rootTaskContext); + rule.getRewriter().collectRewriteContext(tree); + if (rule.getRewriter().isNeedRewrite()) { + pushAggFlag = true; + tree = rule.rewrite(tree, rootTaskContext); + } + } + + if (pushDistinctFlag || pushAggFlag) { + deriveLogicalProperty(tree); + rootTaskContext.setRequiredColumns(requiredColumns.clone()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); + scheduler.rewriteOnce(tree, rootTaskContext, EliminateAggRule.getInstance()); + } + + return tree; + } + + private void skewJoinOptimize(OptExpression tree, TaskContext rootTaskContext) { + if (context.getSessionVariable().isEnableStatsToOptimizeSkewJoin()) { + // merge projects before calculate statistics + scheduler.rewriteOnce(tree, rootTaskContext, new MergeTwoProjectRule()); + Utils.calculateStatistics(tree, rootTaskContext.getOptimizerContext()); + } + scheduler.rewriteOnce(tree, rootTaskContext, new SkewJoinOptimizeRule()); + } + + private OptExpression pruneSubfield(OptExpression tree, TaskContext rootTaskContext, ColumnRefSet requiredColumns) { + if (!context.getSessionVariable().isCboPruneSubfield()) { + return tree; + } + + PushDownSubfieldRule pushDownRule = new PushDownSubfieldRule(); + tree = pushDownRule.rewrite(tree, rootTaskContext); + + if (pushDownRule.hasRewrite()) { + rootTaskContext.setRequiredColumns(requiredColumns.clone()); + scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES); + } + scheduler.rewriteOnce(tree, rootTaskContext, new PruneSubfieldRule()); + + return tree; + } + + private void deriveLogicalProperty(OptExpression root) { + for (OptExpression child : root.getInputs()) { + deriveLogicalProperty(child); + } + + ExpressionContext context = new ExpressionContext(root); + context.deriveLogicalProperty(); + root.setLogicalProperty(context.getRootProperty()); + } + + void memoOptimize(ConnectContext connectContext, Memo memo, TaskContext rootTaskContext) { + context.setInMemoPhase(true); + OptExpression tree = memo.getRootGroup().extractLogicalTree(); + SessionVariable sessionVariable = connectContext.getSessionVariable(); + // add CboTablePruneRule + if (Utils.countJoinNodeSize(tree, CboTablePruneRule.JOIN_TYPES) < 10 && + sessionVariable.isEnableCboTablePrune()) { + context.getRuleSet().addCboTablePruneRule(); + } + // Join reorder + int innerCrossJoinNode = Utils.countJoinNodeSize(tree, JoinOperator.innerCrossJoinSet()); + if (!sessionVariable.isDisableJoinReorder() && innerCrossJoinNode < sessionVariable.getCboMaxReorderNode()) { + if (innerCrossJoinNode > sessionVariable.getCboMaxReorderNodeUseExhaustive()) { + CTEUtils.collectForceCteStatistics(memo, context); + + OptimizerTraceUtil.logOptExpression("before ReorderJoinRule:\n%s", tree); + new ReorderJoinRule().transform(tree, context); + OptimizerTraceUtil.logOptExpression("after ReorderJoinRule:\n%s", tree); + + context.getRuleSet().addJoinCommutativityWithoutInnerRule(); + } else { + if (Utils.countJoinNodeSize(tree, JoinOperator.semiAntiJoinSet()) < + sessionVariable.getCboMaxReorderNodeUseExhaustive()) { + context.getRuleSet().getTransformRules().add(JoinLeftAsscomRule.INNER_JOIN_LEFT_ASSCOM_RULE); + } + context.getRuleSet().addJoinTransformationRules(); + } + } + + if (!sessionVariable.isDisableJoinReorder() && sessionVariable.isEnableOuterJoinReorder() + && Utils.capableOuterReorder(tree, sessionVariable.getCboReorderThresholdUseExhaustive())) { + context.getRuleSet().addOuterJoinTransformationRules(); + } + + if (!sessionVariable.isMVPlanner()) { + // add join implementRule + String joinImplementationMode = connectContext.getSessionVariable().getJoinImplementationMode(); + if ("merge".equalsIgnoreCase(joinImplementationMode)) { + context.getRuleSet().addMergeJoinImplementationRule(); + } else if ("hash".equalsIgnoreCase(joinImplementationMode)) { + context.getRuleSet().addHashJoinImplementationRule(); + } else if ("nestloop".equalsIgnoreCase(joinImplementationMode)) { + context.getRuleSet().addNestLoopJoinImplementationRule(); + } else { + context.getRuleSet().addAutoJoinImplementationRule(); + } + } else { + context.getRuleSet().addRealtimeMVRules(); + } + + if (mvRewriteStrategy.enableMultiTableRewrite) { + context.getRuleSet().addSingleTableMvRewriteRule(); + context.getRuleSet().addMultiTableMvRewriteRule(); + } + + scheduler.pushTask(new OptimizeGroupTask(rootTaskContext, memo.getRootGroup())); + scheduler.executeTasks(rootTaskContext); + } + + private OptExpression physicalRuleRewrite(ConnectContext connectContext, TaskContext rootTaskContext, + OptExpression result) { + Preconditions.checkState(result.getOp().isPhysical()); + + int planCount = result.getPlanCount(); + + // Since there may be many different plans in the logic phase, it's possible + // that this switch can't turned on after logical optimization, so we only determine + // whether the PreAggregate can be turned on in the final + result = new PreAggregateTurnOnRule().rewrite(result, rootTaskContext); + + // Rewrite Exchange on top of Sort to Final Sort + result = new ExchangeSortToMergeRule().rewrite(result, rootTaskContext); + result = new PruneAggregateNodeRule().rewrite(result, rootTaskContext); + result = new PruneShuffleColumnRule().rewrite(result, rootTaskContext); + result = new PhysicalDistributionAggOptRule().rewrite(result, rootTaskContext); + result = new AddDecodeNodeForDictStringRule().rewrite(result, rootTaskContext); + result = new LowCardinalityRewriteRule().rewrite(result, rootTaskContext); + // Put before ScalarOperatorsReuseRule + result = new PruneSubfieldsForComplexType().rewrite(result, rootTaskContext); + result = new InlineCteProjectPruneRule().rewrite(result, rootTaskContext); + // This rule should be last + result = new ScalarOperatorsReuseRule().rewrite(result, rootTaskContext); + // Reorder predicates + result = new PredicateReorderRule(rootTaskContext.getOptimizerContext().getSessionVariable()).rewrite(result, + rootTaskContext); + result = new ExtractAggregateColumn().rewrite(result, rootTaskContext); + result = new JoinLocalShuffleRule().rewrite(result, rootTaskContext); + + // This must be put at last of the optimization. Because wrapping reused ColumnRefOperator with CloneOperator + // too early will prevent it from certain optimizations that depend on the equivalence of the ColumnRefOperator. + result = new CloneDuplicateColRefRule().rewrite(result, rootTaskContext); + + // set subfield expr copy flag + if (rootTaskContext.getOptimizerContext().getSessionVariable().getEnableSubfieldNoCopy()) { + result = new SubfieldExprNoCopyRule().rewrite(result, rootTaskContext); + } + + result = new AddIndexOnlyPredicateRule().rewrite(result, rootTaskContext); + result = new DataCachePopulateRewriteRule(connectContext).rewrite(result, rootTaskContext); + result = new EliminateOveruseColumnAccessPathRule().rewrite(result, rootTaskContext); + result.setPlanCount(planCount); + return result; + } + + private OptExpression dynamicRewrite(ConnectContext connectContext, TaskContext rootTaskContext, + OptExpression result) { + // update the existRequiredDistribution value in optExpression. The next rules need it to determine + // if we can change the distribution to adjust the plan because of skew data, bad statistics or something else. + result = new MarkParentRequiredDistributionRule().rewrite(result, rootTaskContext); + result = new ApplyTuningGuideRule(connectContext).rewrite(result, rootTaskContext); + + OperatorTuningGuides.OptimizedRecord optimizedRecord = PlanTuningAdvisor.getInstance() + .getOptimizedRecord(context.getQueryId()); + if (optimizedRecord != null) { + Tracers.record(Tracers.Module.BASE, "DynamicApplyTuningGuides", optimizedRecord.getExplainString()); + } + return result; + } + + /** + * Extract the lowest cost physical operator tree from memo + * + * @param requiredProperty the required physical property from sql or groupExpression + * @param rootGroup the current group to find the lowest cost physical operator + * @return the lowest cost physical operator for this query + */ + private OptExpression extractBestPlan(PhysicalPropertySet requiredProperty, + Group rootGroup) { + GroupExpression groupExpression = rootGroup.getBestExpression(requiredProperty); + if (groupExpression == null) { + String msg = "no executable plan for this sql. group: %s. required property: %s"; + throw new IllegalArgumentException(String.format(msg, rootGroup, requiredProperty)); + } + List inputProperties = groupExpression.getInputProperties(requiredProperty); + + List childPlans = Lists.newArrayList(); + for (int i = 0; i < groupExpression.arity(); ++i) { + OptExpression childPlan = extractBestPlan(inputProperties.get(i), groupExpression.inputAt(i)); + childPlans.add(childPlan); + } + + OptExpression expression = OptExpression.create(groupExpression.getOp(), + childPlans); + // record inputProperties at optExpression, used for planFragment builder to determine join type + expression.setRequiredProperties(inputProperties); + expression.setStatistics(groupExpression.getGroup().getStatistics()); + expression.setCost(groupExpression.getCost(requiredProperty)); + expression.setOutputProperty(requiredProperty); + + // When build plan fragment, we need the output column of logical property + expression.setLogicalProperty(rootGroup.getLogicalProperty()); + return expression; + } + + private List collectAllPhysicalOlapScanOperators(OptExpression tree) { + List list = Lists.newArrayList(); + Utils.extractOperator(tree, list, op -> op instanceof PhysicalOlapScanOperator); + return list; + } + + private void prepareMetaOnlyOnce(OptExpression tree, TaskContext rootTaskContext) { + if (rootTaskContext.getOptimizerContext().getSessionVariable().enableParallelPrepareMetadata()) { + scheduler.pushTask(new PrepareCollectMetaTask(rootTaskContext, tree)); + scheduler.executeTasks(rootTaskContext); + } + } +} diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java index a2f4556b9cd9c..4980a17247fa3 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java @@ -19,6 +19,7 @@ import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; import com.starrocks.sql.optimizer.OptimizerFactory; +import com.starrocks.sql.optimizer.QueryOptimizer; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; @@ -60,8 +61,8 @@ public void testSingleTableRewriteStrategy() throws Exception { " as" + " select t1a, id_date, t1b from table_with_partition"); String sql = "select t1a, id_date, t1b from table_with_partition"; - Optimizer optimizer = - OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, new ColumnRefFactory())); + QueryOptimizer optimizer = (QueryOptimizer) OptimizerFactory.create( + OptimizerFactory.mockContext(connectContext, new ColumnRefFactory())); OptExpression optExpression = optimize(optimizer, sql); Assert.assertTrue(optExpression != null); MvRewriteStrategy mvRewriteStrategy = optimizer.getMvRewriteStrategy(); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java index d865e757fb6ab..30b90cd2f398c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java @@ -29,15 +29,13 @@ import com.starrocks.common.AnalysisException; import com.starrocks.common.Config; import com.starrocks.common.FeConstants; -import com.starrocks.qe.ConnectContext; import com.starrocks.qe.SessionVariable; import com.starrocks.qe.ShowResultSet; import com.starrocks.schema.MSchema; import com.starrocks.schema.MTable; import com.starrocks.sql.optimizer.CachingMvPlanContextBuilder; import com.starrocks.sql.optimizer.OptExpression; -import com.starrocks.sql.optimizer.Optimizer; -import com.starrocks.sql.optimizer.base.ColumnRefFactory; +import com.starrocks.sql.optimizer.QueryOptimizer; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; @@ -1853,12 +1851,10 @@ public void testPlanCache() throws Exception { MaterializedView mv = getMv("test", "mv_with_window"); - new MockUp() { - + new MockUp() { @Mock - public OptExpression optimize(ConnectContext connectContext, OptExpression logicOperatorTree, - PhysicalPropertySet requiredProperty, ColumnRefSet requiredColumns, - ColumnRefFactory columnRefFactory) { + public OptExpression optimize(OptExpression logicOperatorTree, PhysicalPropertySet requiredProperty, + ColumnRefSet requiredColumns) { throw new RuntimeException("optimize failed"); } }; From 2577e80fb6de71b2972cab01ec68a1a20e2f8550 Mon Sep 17 00:00:00 2001 From: Seaven Date: Fri, 17 Jan 2025 10:55:22 +0800 Subject: [PATCH 3/3] move ShortCircuit Signed-off-by: Seaven --- .../authorization/ColumnPrivilege.java | 12 ++-- .../starrocks/sql/ShortCircuitPlanner.java | 14 +--- .../com/starrocks/sql/StatementPlanner.java | 22 +++--- .../optimizer/MaterializedViewOptimizer.java | 25 +++---- .../sql/optimizer/MvRewritePreprocessor.java | 3 +- .../sql/optimizer/OptExpression.java | 17 ----- .../starrocks/sql/optimizer/Optimizer.java | 16 ++++- .../sql/optimizer/OptimizerContext.java | 21 ++---- .../sql/optimizer/OptimizerFactory.java | 17 +++-- ...mizerConfig.java => OptimizerOptions.java} | 39 +++++++---- .../sql/optimizer/QueryOptimizer.java | 59 ++-------------- .../sql/optimizer/ShortCircuitOptimizer.java | 68 +++++++++++++++++++ .../PruneProjectColumnsRule.java | 2 +- .../materialization/MvRewriteStrategy.java | 12 ++-- .../materialization/MvUtils.java | 14 ++-- .../rule/TextMatchBasedRewriteRule.java | 2 +- .../sql/optimizer/task/OptimizerTask.java | 6 +- .../sql/optimizer/task/RewriteTreeTask.java | 2 +- .../java/com/starrocks/sql/plan/ExecPlan.java | 11 ++- .../sql/plan/PlanFragmentBuilder.java | 25 ++++--- .../MvRewritePreprocessorTest.java | 31 ++++----- .../com/starrocks/utframe/UtFrameUtils.java | 12 ++-- 22 files changed, 236 insertions(+), 194 deletions(-) rename fe/fe-core/src/main/java/com/starrocks/sql/optimizer/{OptimizerConfig.java => OptimizerOptions.java} (54%) create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/ShortCircuitOptimizer.java diff --git a/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java b/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java index 92575b70bb8e0..be4884095484e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java +++ b/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java @@ -37,8 +37,8 @@ import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptExpressionVisitor; import com.starrocks.sql.optimizer.Optimizer; -import com.starrocks.sql.optimizer.OptimizerConfig; import com.starrocks.sql.optimizer.OptimizerFactory; +import com.starrocks.sql.optimizer.OptimizerOptions; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; @@ -111,12 +111,12 @@ public static void check(ConnectContext context, QueryStatement stmt, List
projections) { for (Map.Entry entry : projections.entrySet()) { if (!entry.getKey().equals(entry.getValue())) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java index 3e44deabec4e8..08edaa1cccff2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java @@ -62,6 +62,7 @@ import com.starrocks.sql.optimizer.Optimizer; import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.OptimizerFactory; +import com.starrocks.sql.optimizer.OptimizerOptions; import com.starrocks.sql.optimizer.OptimizerTraceUtil; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; @@ -249,17 +250,19 @@ private static ExecPlan createQueryPlan(StatementBase stmt, logicalPlan = new RelationTransformer(transformerContext).transformWithSelectLimit(query); } - OptExpression root = ShortCircuitPlanner.checkSupportShortCircuitRead(logicalPlan.getRoot(), session); - + boolean isShortCircuit = ShortCircuitPlanner.checkSupportShortCircuitRead(logicalPlan.getRoot(), session); OptExpression optimizedPlan; try (Timer ignored = Tracers.watchScope("Optimizer")) { // 2. Optimize logical plan and build physical plan OptimizerContext optimizerContext = OptimizerFactory.initContext(session, columnRefFactory); optimizerContext.setMvTransformerContext(mvTransformerContext); optimizerContext.setStatement(stmt); + if (isShortCircuit) { + optimizerContext.setOptimizerOptions(OptimizerOptions.newShortCircuitOpt()); + } Optimizer optimizer = OptimizerFactory.create(optimizerContext); - optimizedPlan = optimizer.optimize(root, + optimizedPlan = optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), new ColumnRefSet(logicalPlan.getOutputColumn())); } @@ -274,7 +277,7 @@ private static ExecPlan createQueryPlan(StatementBase stmt, ExecPlan execPlan = PlanFragmentBuilder.createPhysicalPlan( optimizedPlan, session, logicalPlan.getOutputColumn(), columnRefFactory, colNames, resultSinkType, - !session.getSessionVariable().isSingleNodeExecPlan()); + !session.getSessionVariable().isSingleNodeExecPlan(), isShortCircuit); execPlan.setLogicalPlan(logicalPlan); execPlan.setColumnRefFactory(columnRefFactory); return execPlan; @@ -313,8 +316,7 @@ public static ExecPlan createQueryPlanWithReTry(QueryStatement queryStmt, logicalPlan = new RelationTransformer(transformerContext).transformWithSelectLimit(query); } - OptExpression root = ShortCircuitPlanner.checkSupportShortCircuitRead(logicalPlan.getRoot(), session); - + boolean isShortCircuit = ShortCircuitPlanner.checkSupportShortCircuitRead(logicalPlan.getRoot(), session); OptExpression optimizedPlan; try (Timer ignored = Tracers.watchScope("Optimizer")) { OptimizerContext optimizerContext = OptimizerFactory.initContext(session, columnRefFactory); @@ -324,11 +326,15 @@ public static ExecPlan createQueryPlanWithReTry(QueryStatement queryStmt, if (Config.skip_whole_phase_lock_mv_limit >= 0) { optimizerContext.setQueryTables(olapTables); } + + if (isShortCircuit) { + optimizerContext.setOptimizerOptions(OptimizerOptions.newShortCircuitOpt()); + } optimizerContext.setMvTransformerContext(mvTransformerContext); optimizerContext.setStatement(queryStmt); Optimizer optimizer = OptimizerFactory.create(optimizerContext); - optimizedPlan = optimizer.optimize(root, new PhysicalPropertySet(), + optimizedPlan = optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), new ColumnRefSet(logicalPlan.getOutputColumn())); } @@ -340,7 +346,7 @@ public static ExecPlan createQueryPlanWithReTry(QueryStatement queryStmt, ExecPlan plan = PlanFragmentBuilder.createPhysicalPlan( optimizedPlan, session, logicalPlan.getOutputColumn(), columnRefFactory, colNames, resultSinkType, - !session.getSessionVariable().isSingleNodeExecPlan()); + !session.getSessionVariable().isSingleNodeExecPlan(), isShortCircuit); final long finalPlanStartTime = planStartTime; isSchemaValid = olapTables.stream().allMatch(t -> OptimisticVersion.validateTableUpdate(t, finalPlanStartTime)); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java index 4c855eaa92d7a..98b2bc4b86365 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java @@ -36,24 +36,24 @@ public MvPlanContext optimize(MaterializedView mv, ConnectContext connectContext, boolean inlineView) { // optimize the sql by rule and disable rule based materialized view rewrite - OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); + OptimizerOptions optimizerOptions = OptimizerOptions.newRuleBaseOpt(); // Disable partition prune for mv's plan so no needs to compensate pruned predicates anymore. // Only needs to compensate mv's ref-base-table's partition predicates when mv's freshness cannot be satisfied. - optimizerConfig.disableRule(RuleType.GP_PARTITION_PRUNE); - optimizerConfig.disableRule(RuleType.GP_ALL_MV_REWRITE); + optimizerOptions.disableRule(RuleType.GP_PARTITION_PRUNE); + optimizerOptions.disableRule(RuleType.GP_ALL_MV_REWRITE); // INTERSECT_REWRITE is used for INTERSECT related plan optimize, which can not be SPJG; // And INTERSECT_REWRITE should be based on PARTITION_PRUNE rule set. // So exclude it - optimizerConfig.disableRule(RuleType.GP_INTERSECT_REWRITE); - optimizerConfig.disableRule(RuleType.TF_REWRITE_GROUP_BY_COUNT_DISTINCT); - optimizerConfig.disableRule(RuleType.TF_PRUNE_EMPTY_SCAN); - optimizerConfig.disableRule(RuleType.TF_MV_TEXT_MATCH_REWRITE_RULE); - optimizerConfig.disableRule(RuleType.TF_MV_TRANSPARENT_REWRITE_RULE); - optimizerConfig.disableRule(RuleType.TF_ELIMINATE_AGG); - optimizerConfig.disableRule(RuleType.TF_PULL_UP_PREDICATE_SCAN); + optimizerOptions.disableRule(RuleType.GP_INTERSECT_REWRITE); + optimizerOptions.disableRule(RuleType.TF_REWRITE_GROUP_BY_COUNT_DISTINCT); + optimizerOptions.disableRule(RuleType.TF_PRUNE_EMPTY_SCAN); + optimizerOptions.disableRule(RuleType.TF_MV_TEXT_MATCH_REWRITE_RULE); + optimizerOptions.disableRule(RuleType.TF_MV_TRANSPARENT_REWRITE_RULE); + optimizerOptions.disableRule(RuleType.TF_ELIMINATE_AGG); + optimizerOptions.disableRule(RuleType.TF_PULL_UP_PREDICATE_SCAN); // For sync mv, no rewrite query by original sync mv rule to avoid useless rewrite. if (mv.getRefreshScheme().isSync()) { - optimizerConfig.disableRule(RuleType.TF_MATERIALIZED_VIEW); + optimizerOptions.disableRule(RuleType.TF_MATERIALIZED_VIEW); } ColumnRefFactory columnRefFactory = new ColumnRefFactory(); String mvSql = mv.getViewDefineSql(); @@ -77,7 +77,8 @@ public MvPlanContext optimize(MaterializedView mv, try { // get optimized plan of mv's defined query Pair plans = - MvUtils.getRuleOptimizedLogicalPlan(stmt, columnRefFactory, connectContext, optimizerConfig, inlineView); + MvUtils.getRuleOptimizedLogicalPlan(stmt, columnRefFactory, connectContext, optimizerOptions, + inlineView); if (plans == null) { return new MvPlanContext(false, "No query plan for it"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java index 56197134799f4..029c003616076 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java @@ -229,7 +229,8 @@ public void prepare(OptExpression queryOptExpression) { // use a new context rather than reuse the existed context to avoid cache conflict. try { // 1. get related mvs for all input tables - Set relatedMVs = getRelatedMVs(queryTables, context.getOptimizerConfig().isRuleBased()); + Set relatedMVs = + getRelatedMVs(queryTables, context.getOptimizerOptions().isRuleBased()); if (relatedMVs.isEmpty()) { return; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptExpression.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptExpression.java index c876390dc2fb4..ee3d99bf6de4e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptExpression.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptExpression.java @@ -63,8 +63,6 @@ public class OptExpression { private UKFKConstraints constraints; - private Boolean isShortCircuit = false; - // the flag if its parent has required data distribution property for this expression private boolean existRequiredDistribution = true; @@ -82,13 +80,6 @@ public static OptExpression create(Operator op, OptExpression... inputs) { return expr; } - public static OptExpression createForShortCircuit(Operator op, OptExpression input, boolean isShortCircuit) { - OptExpression expr = new OptExpression(op); - expr.inputs = Lists.newArrayList(input); - expr.setShortCircuit(isShortCircuit); - return expr; - } - public static OptExpression create(Operator op, List inputs) { OptExpression expr = new OptExpression(op); expr.inputs = inputs; @@ -231,14 +222,6 @@ public void setCost(double cost) { this.cost = cost; } - public Boolean getShortCircuit() { - return isShortCircuit; - } - - public void setShortCircuit(Boolean shortCircuit) { - isShortCircuit = shortCircuit; - } - @Override public String toString() { return op + " child size " + inputs.size(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java index 032a40454397d..6403470fe490c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java @@ -27,8 +27,20 @@ public OptimizerContext getContext() { return context; } - public abstract OptExpression optimize(OptExpression logicOperatorTree, ColumnRefSet requiredColumns); + public OptExpression optimize(OptExpression tree, ColumnRefSet requiredColumns) { + return optimize(tree, new PhysicalPropertySet(), requiredColumns); + } - public abstract OptExpression optimize(OptExpression logicOperatorTree, PhysicalPropertySet requiredProperty, + public abstract OptExpression optimize(OptExpression tree, PhysicalPropertySet requiredProperty, ColumnRefSet requiredColumns); + + protected void deriveLogicalProperty(OptExpression root) { + for (OptExpression child : root.getInputs()) { + deriveLogicalProperty(child); + } + + ExpressionContext context = new ExpressionContext(root); + context.deriveLogicalProperty(); + root.setLogicalProperty(context.getRootProperty()); + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java index 7f7c3136ccf91..22699912710b6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java @@ -47,7 +47,7 @@ public class OptimizerContext { private Set queryTables; private long updateTableId = -1; - private OptimizerConfig optimizerConfig; + private OptimizerOptions optimizerOptions; // ============================ Optimizer ============================ private Memo memo; @@ -73,7 +73,6 @@ public class OptimizerContext { // lifecycle instead of per materialized view. private boolean isObtainedFromInternalStatistics = false; - private boolean isShortCircuit = false; private boolean inMemoPhase = false; // Is not null predicate can be derived from inner join or semi join, @@ -89,7 +88,7 @@ public class OptimizerContext { this.cteContext.setInlineCTERatio(getSessionVariable().getCboCTERuseRatio()); this.cteContext.setMaxCTELimit(getSessionVariable().getCboCTEMaxLimit()); - this.optimizerConfig = OptimizerConfig.defaultConfig(); + this.optimizerOptions = OptimizerOptions.defaultOpt(); } // ============================ Query ============================ @@ -145,12 +144,12 @@ public long getUpdateTableId() { return updateTableId; } - public OptimizerConfig getOptimizerConfig() { - return optimizerConfig; + public OptimizerOptions getOptimizerOptions() { + return optimizerOptions; } - public void setOptimizerConfig(OptimizerConfig optimizerConfig) { - this.optimizerConfig = optimizerConfig; + public void setOptimizerOptions(OptimizerOptions optimizerOptions) { + this.optimizerOptions = optimizerOptions; } // ============================ Optimizer ============================ @@ -211,14 +210,6 @@ public void setObtainedFromInternalStatistics(boolean obtainedFromInternalStatis isObtainedFromInternalStatistics = obtainedFromInternalStatistics; } - public boolean isShortCircuit() { - return isShortCircuit; - } - - public void setShortCircuit(boolean shortCircuit) { - isShortCircuit = shortCircuit; - } - public void setInMemoPhase(boolean inMemoPhase) { this.inMemoPhase = inMemoPhase; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java index 8c06e7eae9430..e1e55ae111691 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java @@ -21,39 +21,42 @@ public class OptimizerFactory { @VisibleForTesting public static OptimizerContext mockContext(ConnectContext context, ColumnRefFactory columnRefFactory, - OptimizerConfig config) { + OptimizerOptions config) { OptimizerContext oc = new OptimizerContext(context); oc.setColumnRefFactory(columnRefFactory); - oc.setOptimizerConfig(config); + oc.setOptimizerOptions(config); return oc; } @VisibleForTesting public static OptimizerContext mockContext(ConnectContext context, ColumnRefFactory columnRefFactory) { - return mockContext(context, columnRefFactory, OptimizerConfig.defaultConfig()); + return mockContext(context, columnRefFactory, OptimizerOptions.defaultOpt()); } @VisibleForTesting public static OptimizerContext mockContext(ColumnRefFactory columnRefFactory) { OptimizerContext oc = new OptimizerContext(new ConnectContext()); oc.setColumnRefFactory(columnRefFactory); - oc.setOptimizerConfig(OptimizerConfig.defaultConfig()); + oc.setOptimizerOptions(OptimizerOptions.defaultOpt()); return oc; } public static OptimizerContext initContext(ConnectContext context, ColumnRefFactory columnRefFactory, - OptimizerConfig config) { + OptimizerOptions config) { OptimizerContext oc = new OptimizerContext(context); oc.setColumnRefFactory(columnRefFactory); - oc.setOptimizerConfig(config); + oc.setOptimizerOptions(config); return oc; } public static OptimizerContext initContext(ConnectContext context, ColumnRefFactory columnRefFactory) { - return initContext(context, columnRefFactory, OptimizerConfig.defaultConfig()); + return initContext(context, columnRefFactory, OptimizerOptions.defaultOpt()); } public static Optimizer create(OptimizerContext context) { + if (context.getOptimizerOptions().isShortCircuit()) { + return new ShortCircuitOptimizer(context); + } return new QueryOptimizer(context); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerOptions.java similarity index 54% rename from fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java rename to fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerOptions.java index 065e017f5b867..055de4af80066 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerOptions.java @@ -18,28 +18,33 @@ import java.util.BitSet; -public class OptimizerConfig { - public enum OptimizerAlgorithm { +public class OptimizerOptions { + public enum OptimizerStrategy { RULE_BASED, - COST_BASED + COST_BASED, + SHORT_CIRCUIT, } - private final OptimizerAlgorithm optimizerAlgorithm; + private final OptimizerStrategy optimizerStrategy; private final BitSet ruleSwitches; - public OptimizerConfig() { - this(OptimizerAlgorithm.COST_BASED); + public OptimizerOptions() { + this(OptimizerStrategy.COST_BASED); } - - public OptimizerConfig(OptimizerAlgorithm optimizerAlgorithm) { - this.optimizerAlgorithm = optimizerAlgorithm; + + public OptimizerOptions(OptimizerStrategy optimizerStrategy) { + this.optimizerStrategy = optimizerStrategy; this.ruleSwitches = new BitSet(RuleType.NUM_RULES.ordinal()); this.ruleSwitches.flip(0, ruleSwitches.size()); } public boolean isRuleBased() { - return optimizerAlgorithm.equals(OptimizerAlgorithm.RULE_BASED); + return optimizerStrategy.equals(OptimizerStrategy.RULE_BASED); + } + + public boolean isShortCircuit() { + return optimizerStrategy.equals(OptimizerStrategy.SHORT_CIRCUIT); } public void disableRule(RuleType ruleType) { @@ -50,9 +55,17 @@ public boolean isRuleDisable(RuleType ruleType) { return !ruleSwitches.get(ruleType.ordinal()); } - private static final OptimizerConfig DEFAULT_CONFIG = new OptimizerConfig(); + private static final OptimizerOptions DEFAULT_OPTIONS = new OptimizerOptions(OptimizerStrategy.COST_BASED); + + public static OptimizerOptions defaultOpt() { + return DEFAULT_OPTIONS; + } + + public static OptimizerOptions newRuleBaseOpt() { + return new OptimizerOptions(OptimizerStrategy.RULE_BASED); + } - public static OptimizerConfig defaultConfig() { - return DEFAULT_CONFIG; + public static OptimizerOptions newShortCircuitOpt() { + return new OptimizerOptions(OptimizerStrategy.SHORT_CIRCUIT); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java index d8b549eb4e2b9..da92a1a160ee2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java @@ -37,7 +37,6 @@ import com.starrocks.sql.optimizer.operator.physical.PhysicalOlapScanOperator; import com.starrocks.sql.optimizer.rewrite.JoinPredicatePushdown; import com.starrocks.sql.optimizer.rule.RuleSet; -import com.starrocks.sql.optimizer.rule.implementation.OlapScanImplementationRule; import com.starrocks.sql.optimizer.rule.join.JoinReorderFactory; import com.starrocks.sql.optimizer.rule.join.ReorderJoinRule; import com.starrocks.sql.optimizer.rule.mv.MaterializedViewRule; @@ -125,7 +124,6 @@ import org.apache.logging.log4j.Logger; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -139,7 +137,7 @@ */ public class QueryOptimizer extends Optimizer { private static final Logger LOG = LogManager.getLogger(QueryOptimizer.class); - private OptimizerConfig optimizerConfig; + private OptimizerOptions optimizerOptions; private MvRewriteStrategy mvRewriteStrategy = new MvRewriteStrategy(); private final TaskScheduler scheduler = new TaskScheduler(); private Memo memo; @@ -157,9 +155,9 @@ public MvRewriteStrategy getMvRewriteStrategy() { } private void prepare(OptExpression logicOperatorTree) { - optimizerConfig = context.getOptimizerConfig(); + optimizerOptions = context.getOptimizerOptions(); - if (!optimizerConfig.isRuleBased()) { + if (!optimizerOptions.isRuleBased()) { memo = new Memo(); context.setMemo(memo); } @@ -170,10 +168,6 @@ private void prepare(OptExpression logicOperatorTree) { op -> op instanceof LogicalOlapScanOperator); } - public OptExpression optimize(OptExpression logicOperatorTree, ColumnRefSet requiredColumns) { - return optimize(logicOperatorTree, new PhysicalPropertySet(), requiredColumns); - } - public OptExpression optimize(OptExpression logicOperatorTree, PhysicalPropertySet requiredProperty, ColumnRefSet requiredColumns) { try { @@ -188,7 +182,7 @@ public OptExpression optimize(OptExpression logicOperatorTree, PhysicalPropertyS context.getMvTransformerContext()).transform(logicOperatorTree, context).get(0); } - OptExpression result = optimizerConfig.isRuleBased() ? + OptExpression result = optimizerOptions.isRuleBased() ? optimizeByRule(logicOperatorTree, requiredProperty, requiredColumns) : optimizeByCost(context.getConnectContext(), logicOperatorTree, requiredProperty, requiredColumns); @@ -237,10 +231,6 @@ private OptExpression optimizeByCost(ConnectContext connectContext, logicOperatorTree = rewriteAndValidatePlan(logicOperatorTree, rootTaskContext); } - if (logicOperatorTree.getShortCircuit()) { - return logicOperatorTree; - } - Preconditions.checkNotNull(memo); memo.init(logicOperatorTree); if (context.getQueryMaterializationContext() != null) { @@ -320,7 +310,7 @@ private void prepareMvRewrite(ConnectContext connectContext, OptExpression logic ColumnRefFactory columnRefFactory, ColumnRefSet requiredColumns) { SessionVariable sessionVariable = connectContext.getSessionVariable(); // MV Rewrite will be used when cbo is enabled. - if (context.getOptimizerConfig().isRuleBased() || sessionVariable.isDisableMaterializedViewRewrite() || + if (context.getOptimizerOptions().isRuleBased() || sessionVariable.isDisableMaterializedViewRewrite() || !sessionVariable.isEnableMaterializedViewRewrite()) { return; } @@ -464,13 +454,7 @@ private void doMVRewriteWithMultiStages(OptExpression tree, private OptExpression logicalRuleRewrite( OptExpression tree, TaskContext rootTaskContext) { - rootTaskContext.getOptimizerContext().setShortCircuit(tree.getShortCircuit()); - tree = OptExpression.createForShortCircuit(new LogicalTreeAnchorOperator(), tree, tree.getShortCircuit()); - // for short circuit - Optional result = ruleRewriteForShortCircuit(tree, rootTaskContext); - if (result.isPresent()) { - return result.get(); - } + tree = OptExpression.create(new LogicalTreeAnchorOperator(), tree); ColumnRefSet requiredColumns = rootTaskContext.getRequiredColumns().clone(); deriveLogicalProperty(tree); @@ -588,7 +572,7 @@ private OptExpression logicalRuleRewrite( } // Add a config to decide whether to rewrite sync mv. - if (!optimizerConfig.isRuleDisable(TF_MATERIALIZED_VIEW) + if (!optimizerOptions.isRuleDisable(TF_MATERIALIZED_VIEW) && sessionVariable.isEnableSyncMaterializedViewRewrite()) { // Split or predicates to union all so can be used by mv rewrite to choose the best sort key indexes. // TODO: support adaptive for or-predicates to union all. @@ -685,20 +669,6 @@ private void rewriteGroupingSets(OptExpression tree, TaskContext rootTaskContext } } - private Optional ruleRewriteForShortCircuit(OptExpression tree, TaskContext rootTaskContext) { - Boolean isShortCircuit = tree.getShortCircuit(); - - if (isShortCircuit) { - deriveLogicalProperty(tree); - scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SHORT_CIRCUIT_SET_RULES); - scheduler.rewriteOnce(tree, rootTaskContext, new MergeProjectWithChildRule()); - OptExpression result = tree.getInputs().get(0); - result.setShortCircuit(true); - return Optional.of(result); - } - return Optional.empty(); - } - // for single scan node, to make sure we can rewrite private void viewBasedMvRuleRewrite(OptExpression tree, TaskContext rootTaskContext) { QueryMaterializationContext queryMaterializationContext = context.getQueryMaterializationContext(); @@ -744,11 +714,6 @@ private OptExpression rewriteAndValidatePlan( OptExpression result = logicalRuleRewrite(tree, rootTaskContext); OptExpressionValidator validator = new OptExpressionValidator(); validator.validate(result); - // skip memo - if (result.getShortCircuit()) { - result = new OlapScanImplementationRule().transform(result, null).get(0); - result.setShortCircuit(true); - } return result; } @@ -823,16 +788,6 @@ private OptExpression pruneSubfield(OptExpression tree, TaskContext rootTaskCont return tree; } - private void deriveLogicalProperty(OptExpression root) { - for (OptExpression child : root.getInputs()) { - deriveLogicalProperty(child); - } - - ExpressionContext context = new ExpressionContext(root); - context.deriveLogicalProperty(); - root.setLogicalProperty(context.getRootProperty()); - } - void memoOptimize(ConnectContext connectContext, Memo memo, TaskContext rootTaskContext) { context.setInMemoPhase(true); OptExpression tree = memo.getRootGroup().extractLogicalTree(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/ShortCircuitOptimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/ShortCircuitOptimizer.java new file mode 100644 index 0000000000000..a880ce9f9793a --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/ShortCircuitOptimizer.java @@ -0,0 +1,68 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer; + +import com.starrocks.common.profile.Timer; +import com.starrocks.common.profile.Tracers; +import com.starrocks.sql.optimizer.base.ColumnRefSet; +import com.starrocks.sql.optimizer.base.PhysicalPropertySet; +import com.starrocks.sql.optimizer.operator.OperatorType; +import com.starrocks.sql.optimizer.operator.logical.LogicalTreeAnchorOperator; +import com.starrocks.sql.optimizer.rule.RuleSet; +import com.starrocks.sql.optimizer.rule.implementation.OlapScanImplementationRule; +import com.starrocks.sql.optimizer.rule.transformation.MergeProjectWithChildRule; +import com.starrocks.sql.optimizer.task.TaskContext; +import com.starrocks.sql.optimizer.task.TaskScheduler; +import com.starrocks.sql.optimizer.validate.OptExpressionValidator; + +public class ShortCircuitOptimizer extends Optimizer { + + ShortCircuitOptimizer(OptimizerContext context) { + super(context); + } + + @Override + public OptExpression optimize(OptExpression tree, PhysicalPropertySet requiredProperty, + ColumnRefSet requiredColumns) { + TaskScheduler scheduler = new TaskScheduler(); + context.setTaskScheduler(scheduler); + + // Phase 1: none + OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", tree); + // Phase 2: rewrite based on memo and group + TaskContext rootTaskContext = + new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE); + + try (Timer ignored = Tracers.watchScope("ShortCircuitOptimize")) { + tree = OptExpression.create(new LogicalTreeAnchorOperator(), tree); + // for short circuit + deriveLogicalProperty(tree); + scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SHORT_CIRCUIT_SET_RULES); + scheduler.rewriteOnce(tree, rootTaskContext, new MergeProjectWithChildRule()); + tree = tree.getInputs().get(0); + + if (OperatorType.LOGICAL_LIMIT.equals(tree.getOp().getOpType())) { + tree = tree.getInputs().get(0); + } + + OptExpressionValidator validator = new OptExpressionValidator(); + validator.validate(tree); + + // skip memo + tree = new OlapScanImplementationRule().transform(tree, null).get(0); + return tree; + } + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneProjectColumnsRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneProjectColumnsRule.java index 4bff43fc5ea2d..737dff24a94bc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneProjectColumnsRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneProjectColumnsRule.java @@ -67,7 +67,7 @@ public List transform(OptExpression input, OptimizerContext conte ColumnRefOperator constCol = context.getColumnRefFactory() .create("auto_fill_col", Type.TINYINT, false); newMap.put(constCol, ConstantOperator.createTinyInt((byte) 1)); - } else if (newMap.equals(projectOperator.getColumnRefMap()) && context.isShortCircuit()) { + } else if (newMap.equals(projectOperator.getColumnRefMap()) && context.getOptimizerOptions().isShortCircuit()) { // Change the requiredOutputColumns in context requiredOutputColumns.union(requiredInputColumns); // make sure this rule only executed once diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategy.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategy.java index d047b36e25f55..bea986321e321 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategy.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategy.java @@ -19,8 +19,8 @@ import com.starrocks.qe.SessionVariable; import com.starrocks.sql.optimizer.MaterializationContext; import com.starrocks.sql.optimizer.OptExpression; -import com.starrocks.sql.optimizer.OptimizerConfig; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.OptimizerOptions; import com.starrocks.sql.optimizer.rule.RuleType; public class MvRewriteStrategy { @@ -59,14 +59,14 @@ public boolean isMultiStages() { public boolean enableMultiTableRewrite = false; static class MvStrategyArbitrator { - private final OptimizerConfig optimizerConfig; + private final OptimizerOptions optimizerOptions; private final OptimizerContext optimizerContext; private final SessionVariable sessionVariable; public MvStrategyArbitrator(OptimizerContext optimizerContext, ConnectContext connectContext) { this.optimizerContext = optimizerContext; - this.optimizerConfig = optimizerContext.getOptimizerConfig(); + this.optimizerOptions = optimizerContext.getOptimizerOptions(); // from connectContext rather than optimizerContext this.sessionVariable = connectContext.getSessionVariable(); } @@ -84,8 +84,8 @@ private boolean isEnableMaterializedViewRewrite() { optimizerContext.getCandidateMvs().isEmpty()) { return false; } - if (optimizerConfig.isRuleDisable(RuleType.GP_SINGLE_TABLE_MV_REWRITE) && - optimizerConfig.isRuleDisable(RuleType.GP_MULTI_TABLE_MV_REWRITE)) { + if (optimizerOptions.isRuleDisable(RuleType.GP_SINGLE_TABLE_MV_REWRITE) && + optimizerOptions.isRuleDisable(RuleType.GP_MULTI_TABLE_MV_REWRITE)) { return false; } return true; @@ -100,7 +100,7 @@ private boolean isEnableRBOViewBasedRewrite() { private boolean isEnableRBOSingleTableRewrite(OptExpression queryPlan) { // if disable single mv rewrite, return false. - if (optimizerConfig.isRuleDisable(RuleType.GP_SINGLE_TABLE_MV_REWRITE)) { + if (optimizerOptions.isRuleDisable(RuleType.GP_SINGLE_TABLE_MV_REWRITE)) { return false; } // If query only has one table use single table rewrite, view delta only rewrites multi-tables query. diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java index f191c749abcd9..671baf41fcfdf 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java @@ -71,9 +71,9 @@ import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptExpressionVisitor; import com.starrocks.sql.optimizer.Optimizer; -import com.starrocks.sql.optimizer.OptimizerConfig; import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.OptimizerFactory; +import com.starrocks.sql.optimizer.OptimizerOptions; import com.starrocks.sql.optimizer.QueryMaterializationContext; import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.base.ColumnRefFactory; @@ -479,7 +479,7 @@ public static StatementBase parse(MaterializedView mv, public static Pair getRuleOptimizedLogicalPlan(StatementBase mvStmt, ColumnRefFactory columnRefFactory, ConnectContext connectContext, - OptimizerConfig optimizerConfig, + OptimizerOptions optimizerOptions, boolean inlineView) { Preconditions.checkState(mvStmt instanceof QueryStatement); Analyzer.analyze(mvStmt, connectContext); @@ -489,7 +489,7 @@ public static Pair getRuleOptimizedLogicalPlan(State LogicalPlan logicalPlan = new RelationTransformer(transformerContext).transform(query); Optimizer optimizer = OptimizerFactory.create( - OptimizerFactory.initContext(connectContext, columnRefFactory, optimizerConfig)); + OptimizerFactory.initContext(connectContext, columnRefFactory, optimizerOptions)); OptExpression optimizedPlan = optimizer.optimize( logicalPlan.getRoot(), new PhysicalPropertySet(), @@ -1570,11 +1570,11 @@ public static OptExpression optimizeViewPlan(OptExpression logicalTree, ConnectContext connectContext, ColumnRefSet requiredColumns, ColumnRefFactory columnRefFactory) { - OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); - optimizerConfig.disableRule(RuleType.GP_SINGLE_TABLE_MV_REWRITE); - optimizerConfig.disableRule(RuleType.GP_MULTI_TABLE_MV_REWRITE); + OptimizerOptions optimizerOptions = OptimizerOptions.newRuleBaseOpt(); + optimizerOptions.disableRule(RuleType.GP_SINGLE_TABLE_MV_REWRITE); + optimizerOptions.disableRule(RuleType.GP_MULTI_TABLE_MV_REWRITE); Optimizer optimizer = OptimizerFactory.create( - OptimizerFactory.initContext(connectContext, columnRefFactory, optimizerConfig)); + OptimizerFactory.initContext(connectContext, columnRefFactory, optimizerOptions)); OptExpression optimizedViewPlan = optimizer.optimize(logicalTree, new PhysicalPropertySet(), requiredColumns); return optimizedViewPlan; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/TextMatchBasedRewriteRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/TextMatchBasedRewriteRule.java index 7d5bef5af7dd2..7437f2e3feab9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/TextMatchBasedRewriteRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/TextMatchBasedRewriteRule.java @@ -115,7 +115,7 @@ public List transform(OptExpression input, OptimizerContext conte private OptExpression doTransform(OptimizerContext context, OptExpression input, ParseNode parseNode) { - if (context.getOptimizerConfig().isRuleDisable(RuleType.TF_MV_TEXT_MATCH_REWRITE_RULE)) { + if (context.getOptimizerOptions().isRuleDisable(RuleType.TF_MV_TEXT_MATCH_REWRITE_RULE)) { return null; } SessionVariable sessionVariable = connectContext.getSessionVariable(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/OptimizerTask.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/OptimizerTask.java index c3f65210918b4..1d554b6ce780f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/OptimizerTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/OptimizerTask.java @@ -16,7 +16,7 @@ package com.starrocks.sql.optimizer.task; import com.starrocks.sql.optimizer.GroupExpression; -import com.starrocks.sql.optimizer.OptimizerConfig; +import com.starrocks.sql.optimizer.OptimizerOptions; import com.starrocks.sql.optimizer.rule.Rule; import java.util.List; @@ -47,7 +47,7 @@ public void pushTask(OptimizerTask task) { void filterInValidRules(GroupExpression groupExpression, List candidateRules, List validRules) { - OptimizerConfig optimizerConfig = context.getOptimizerContext().getOptimizerConfig(); + OptimizerOptions optimizerOptions = context.getOptimizerContext().getOptimizerOptions(); for (Rule rule : candidateRules) { if (groupExpression.hasRuleExplored(rule)) { continue; @@ -55,7 +55,7 @@ void filterInValidRules(GroupExpression groupExpression, if (!rule.getPattern().matchWithoutChild(groupExpression)) { continue; } - if (optimizerConfig.isRuleDisable(rule.type())) { + if (optimizerOptions.isRuleDisable(rule.type())) { continue; } if (rule.exhausted(context.getOptimizerContext())) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java index 17380d201a7ff..4cdf198750f15 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java @@ -79,7 +79,7 @@ protected void rewrite(OptExpression parent, int childIndex, OptExpression root) protected OptExpression applyRules(OptExpression parent, int childIndex, OptExpression root, List rules) { for (Rule rule : rules) { - if (context.getOptimizerContext().getOptimizerConfig().isRuleDisable(rule.type())) { + if (context.getOptimizerContext().getOptimizerOptions().isRuleDisable(rule.type())) { continue; } if (rule.exhausted(context.getOptimizerContext())) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/plan/ExecPlan.java b/fe/fe-core/src/main/java/com/starrocks/sql/plan/ExecPlan.java index cbdc81af3710d..e32f1b3a12703 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/plan/ExecPlan.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/plan/ExecPlan.java @@ -67,6 +67,8 @@ public class ExecPlan { private List collectExecStatsIds; + private final boolean isShortCircuit; + @VisibleForTesting public ExecPlan() { connectContext = new ConnectContext(); @@ -74,14 +76,16 @@ public ExecPlan() { colNames = new ArrayList<>(); physicalPlan = null; outputColumns = new ArrayList<>(); + isShortCircuit = false; } public ExecPlan(ConnectContext connectContext, List colNames, - OptExpression physicalPlan, List outputColumns) { + OptExpression physicalPlan, List outputColumns, boolean isShortCircuit) { this.connectContext = connectContext; this.colNames = colNames; this.physicalPlan = physicalPlan; this.outputColumns = outputColumns; + this.isShortCircuit = isShortCircuit; } // for broker load plan @@ -91,6 +95,7 @@ public ExecPlan(ConnectContext connectContext, List fragments) { this.physicalPlan = null; this.outputColumns = new ArrayList<>(); this.fragments.addAll(fragments); + this.isShortCircuit = false; } public ConnectContext getConnectContext() { @@ -271,4 +276,8 @@ public List getCollectExecStatsIds() { public void setCollectExecStatsIds(List collectExecStatsIds) { this.collectExecStatsIds = collectExecStatsIds; } + + public boolean isShortCircuit() { + return isShortCircuit; + } } \ No newline at end of file diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java index 01396c8b4b077..1ac67ab3dc730 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java @@ -240,15 +240,24 @@ public static ExecPlan createPhysicalPlan(OptExpression plan, ConnectContext con List outputColumns, ColumnRefFactory columnRefFactory, List colNames, TResultSinkType resultSinkType, - boolean hasOutputFragment) { + boolean hasOutputFragment, boolean isShortCircuit) { UKFKConstraintsCollector.collectColumnConstraints(plan); - ExecPlan execPlan = new ExecPlan(connectContext, colNames, plan, outputColumns); + ExecPlan execPlan = new ExecPlan(connectContext, colNames, plan, outputColumns, isShortCircuit); createOutputFragment(new PhysicalPlanTranslator(columnRefFactory).translate(plan, execPlan), execPlan, outputColumns, hasOutputFragment); execPlan.setPlanCount(plan.getPlanCount()); return finalizeFragments(execPlan, resultSinkType); } + public static ExecPlan createPhysicalPlan(OptExpression plan, ConnectContext connectContext, + List outputColumns, ColumnRefFactory columnRefFactory, + List colNames, + TResultSinkType resultSinkType, + boolean hasOutputFragment) { + return createPhysicalPlan(plan, connectContext, outputColumns, columnRefFactory, colNames, resultSinkType, + hasOutputFragment, false); + } + public static ExecPlan createPhysicalPlanForMV(ConnectContext connectContext, CreateMaterializedViewStatement createStmt, OptExpression optExpr, @@ -257,7 +266,7 @@ public static ExecPlan createPhysicalPlanForMV(ConnectContext connectContext, ColumnRefFactory columnRefFactory) throws DdlException { List colNames = queryRelation.getColumnOutputNames(); List outputColumns = logicalPlan.getOutputColumn(); - ExecPlan execPlan = new ExecPlan(connectContext, colNames, optExpr, outputColumns); + ExecPlan execPlan = new ExecPlan(connectContext, colNames, optExpr, outputColumns, false); PlanFragment planFragment = new PhysicalPlanTranslator(columnRefFactory).translate(optExpr, execPlan); // createOutputFragment(planFragment, execPlan, outputColumns, false); execPlan.setPlanCount(optExpr.getPlanCount()); @@ -288,7 +297,7 @@ public static ExecPlan createPhysicalPlanForMV(ConnectContext connectContext, return execPlan; } - public static TupleDescriptor buildTupleDesc(ExecPlan execPlan, Table table) { + private static TupleDescriptor buildTupleDesc(ExecPlan execPlan, Table table) { DescriptorTable descriptorTable = execPlan.getDescTbl(); TupleDescriptor olapTuple = descriptorTable.createTupleDescriptor(); for (Column column : table.getFullSchema()) { @@ -332,7 +341,7 @@ private static void createOutputFragment(PlanFragment inputFragment, ExecPlan ex && !inputFragment.hashLocalBucketShuffleRightOrFullJoin(inputFragment.getPlanRoot()) && execPlan.getScanNodes().stream().allMatch(d -> d instanceof OlapScanNode) && (execPlan.getScanNodes().stream().map(d -> ((OlapScanNode) d).getScanTabletIds().size()) - .reduce(Integer::sum).orElse(2) <= 1)) || execPlan.getPhysicalPlan().getShortCircuit()) { + .reduce(Integer::sum).orElse(2) <= 1)) || execPlan.isShortCircuit()) { inputFragment.setOutputExprs(outputExprs); return; } @@ -656,7 +665,7 @@ public PlanFragment visitPhysicalProject(OptExpression optExpr, ExecPlan context projectNode.setLimit(inputFragment.getPlanRoot().getLimit()); inputFragment.setPlanRoot(projectNode); - inputFragment.setShortCircuit(optExpr.getShortCircuit()); + inputFragment.setShortCircuit(context.isShortCircuit()); return inputFragment; } @@ -959,11 +968,11 @@ public PlanFragment visitPhysicalOlapScan(OptExpression optExpr, ExecPlan contex new PlanFragment(context.getNextFragmentId(), scanNode, DataPartition.RANDOM); fragment.setQueryGlobalDicts(node.getGlobalDicts()); fragment.setQueryGlobalDictExprs(getGlobalDictsExprs(node.getGlobalDictsExpr(), context)); - fragment.setShortCircuit(optExpr.getShortCircuit()); + fragment.setShortCircuit(context.isShortCircuit()); context.getFragments().add(fragment); // set row store literal - if (optExpr.getShortCircuit()) { + if (context.isShortCircuit()) { scanNode.computePointScanRangeLocations(); } return fragment; diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java index 8fcdcae3bba98..d03edbc4e45b4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java @@ -29,9 +29,9 @@ import com.starrocks.sql.optimizer.MvRewritePreprocessor; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; -import com.starrocks.sql.optimizer.OptimizerConfig; import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.OptimizerFactory; +import com.starrocks.sql.optimizer.OptimizerOptions; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.PhysicalPropertySet; @@ -106,27 +106,26 @@ public void testOptimizer() throws Exception { OptimizerContext optimizerContext = OptimizerFactory.mockContext(connectContext, columnRefFactory); Optimizer optimizer = OptimizerFactory.create(optimizerContext); - Assert.assertFalse(optimizerContext.getOptimizerConfig().isRuleBased()); - Assert.assertFalse(optimizerContext.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); - Assert.assertFalse(optimizerContext.getOptimizerConfig().isRuleDisable(RuleType.GP_AGGREGATE_REWRITE)); + Assert.assertFalse(optimizerContext.getOptimizerOptions().isRuleBased()); + Assert.assertFalse(optimizerContext.getOptimizerOptions().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); + Assert.assertFalse(optimizerContext.getOptimizerOptions().isRuleDisable(RuleType.GP_AGGREGATE_REWRITE)); OptExpression expr = optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), new ColumnRefSet(logicalPlan.getOutputColumn())); Assert.assertTrue(expr.getInputs().get(0).getOp() instanceof PhysicalOlapScanOperator); Assert.assertNotNull(expr.getInputs().get(0).getOp().getPredicate()); - - OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); - optimizerConfig.disableRule(RuleType.TF_MERGE_TWO_PROJECT); - optimizerConfig.disableRule(RuleType.GP_PUSH_DOWN_PREDICATE); + OptimizerOptions optimizerOptions = new OptimizerOptions(OptimizerOptions.OptimizerStrategy.RULE_BASED); + optimizerOptions.disableRule(RuleType.TF_MERGE_TWO_PROJECT); + optimizerOptions.disableRule(RuleType.GP_PUSH_DOWN_PREDICATE); OptimizerContext optimizerContext1 = OptimizerFactory.mockContext(connectContext, columnRefFactory, - optimizerConfig); + optimizerOptions); Optimizer optimizer1 = OptimizerFactory.create(optimizerContext1); - Assert.assertTrue(optimizerContext1.getOptimizerConfig().isRuleBased()); - Assert.assertFalse(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_AGG_RULE)); - Assert.assertTrue(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); - Assert.assertFalse(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.GP_COLLECT_CTE)); - Assert.assertTrue(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.GP_PUSH_DOWN_PREDICATE)); + Assert.assertTrue(optimizerContext1.getOptimizerOptions().isRuleBased()); + Assert.assertFalse(optimizerContext1.getOptimizerOptions().isRuleDisable(RuleType.TF_MERGE_TWO_AGG_RULE)); + Assert.assertTrue(optimizerContext1.getOptimizerOptions().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); + Assert.assertFalse(optimizerContext1.getOptimizerOptions().isRuleDisable(RuleType.GP_COLLECT_CTE)); + Assert.assertTrue(optimizerContext1.getOptimizerOptions().isRuleDisable(RuleType.GP_PUSH_DOWN_PREDICATE)); OptExpression expr1 = optimizer1.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), new ColumnRefSet(logicalPlan.getOutputColumn())); @@ -272,8 +271,8 @@ public void testPreprocessMvPartitionMv() throws Exception { private Pair buildMvProcessor(String query) { ColumnRefFactory columnRefFactory = new ColumnRefFactory(); - OptimizerConfig optimizerConfig = new OptimizerConfig(); - OptimizerContext context = OptimizerFactory.mockContext(connectContext, columnRefFactory, optimizerConfig); + OptimizerOptions optimizerOptions = new OptimizerOptions(); + OptimizerContext context = OptimizerFactory.mockContext(connectContext, columnRefFactory, optimizerOptions); try { StatementBase stmt = UtFrameUtils.parseStmtWithNewParser(query, connectContext); diff --git a/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java index 8b3be0c618dbd..e52726250fe6b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java +++ b/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java @@ -119,9 +119,9 @@ import com.starrocks.sql.optimizer.LogicalPlanPrinter; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; -import com.starrocks.sql.optimizer.OptimizerConfig; import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.OptimizerFactory; +import com.starrocks.sql.optimizer.OptimizerOptions; import com.starrocks.sql.optimizer.QueryMaterializationContext; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; @@ -849,13 +849,13 @@ public static LogicalPlan getQueryLogicalPlan(ConnectContext connectContext, public static OptExpression getQueryOptExpression(ConnectContext connectContext, ColumnRefFactory columnRefFactory, LogicalPlan logicalPlan, - OptimizerConfig optimizerConfig) { + OptimizerOptions optimizerOptions) { OptExpression optimizedPlan; try (Timer t = Tracers.watchScope("Optimizer")) { Optimizer optimizer = null; OptimizerContext context = OptimizerFactory.mockContext(connectContext, columnRefFactory); - if (optimizerConfig != null) { - context.setOptimizerConfig(optimizerConfig); + if (optimizerOptions != null) { + context.setOptimizerOptions(optimizerOptions); } optimizer = OptimizerFactory.create(context); optimizedPlan = optimizer.optimize( @@ -1442,9 +1442,9 @@ public static OptExpression getQueryOptExpression(ConnectContext connectContext, Assert.fail("Parse query failed:" + DebugUtil.getStackTrace(e)); } LogicalPlan logicalPlan = UtFrameUtils.getQueryLogicalPlan(connectContext, columnRefFactory, statement); - OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); + OptimizerOptions optimizerOptions = new OptimizerOptions(OptimizerOptions.OptimizerStrategy.RULE_BASED); OptExpression optExpression = UtFrameUtils.getQueryOptExpression(connectContext, columnRefFactory, - logicalPlan, optimizerConfig); + logicalPlan, optimizerOptions); return optExpression; }