diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index a0bb8a8070726..e381da64fcdb6 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -59,7 +59,8 @@ type testPlanSuite struct { } func (s *testPlanSuite) SetUpSuite(c *C) { - s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockSignedTable(), MockUnsignedTable(), MockView(), MockNoPKTable()}) + s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockSignedTable(), MockUnsignedTable(), MockView(), MockNoPKTable(), + MockRangePartitionTable(), MockHashPartitionTable(), MockListPartitionTable()}) s.ctx = MockContext() domain.GetDomain(s.ctx).MockInfoCacheAndLoadInfoSchema(s.is) s.ctx.GetSessionVars().EnableWindowFunction = true diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index f0c6d5718eaae..58348bd7712de 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -86,6 +86,83 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName string assertRuleSteps []assertTraceStep }{ + { + sql: "select * from pt3 where ptn > 3;", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] has multiple needed partitions[p1,p2] after pruning", + assertAction: "Datasource[1] becomes PartitionUnion[6] with children[TableScan[1],TableScan[1]]", + }, + }, + }, + { + sql: "select * from pt3 where ptn = 1;", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] has one needed partition[p1] after pruning", + assertAction: "Datasource[1] becomes TableScan[1]", + }, + }, + }, + { + sql: "select * from pt2 where ptn in (1,2,3);", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] has multiple needed partitions[p1,p2] after pruning", + assertAction: "Datasource[1] becomes PartitionUnion[7] with children[TableScan[1],TableScan[1]]", + }, + }, + }, + { + sql: "select * from pt2 where ptn = 1;", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] has one needed partition[p2] after pruning", + assertAction: "Datasource[1] becomes TableScan[1]", + }, + }, + }, + { + sql: "select * from pt1 where ptn > 100;", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] doesn't have needed partition table after pruning", + assertAction: "Datasource[1] becomes TableDual[5]", + }, + }, + }, + { + sql: "select * from pt1 where ptn in (10,20);", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] has multiple needed partitions[p1,p2] after pruning", + assertAction: "Datasource[1] becomes PartitionUnion[7] with children[TableScan[1],TableScan[1]]", + }, + }, + }, + { + sql: "select * from pt1 where ptn < 4;", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] has one needed partition[p1] after pruning", + assertAction: "Datasource[1] becomes TableScan[1]", + }, + }, + }, { sql: "select * from (t t1, t t2, t t3,t t4) union all select * from (t t5, t t6, t t7,t t8)", flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagDecorrelate, flagPredicatePushDown, flagEliminateOuterJoin, flagJoinReOrder}, diff --git a/planner/core/mock.go b/planner/core/mock.go index 57375118dfd13..4161e235244f4 100644 --- a/planner/core/mock.go +++ b/planner/core/mock.go @@ -432,3 +432,122 @@ func MockPartitionInfoSchema(definitions []model.PartitionDefinition) infoschema is := infoschema.MockInfoSchema([]*model.TableInfo{tableInfo}) return is } + +// MockRangePartitionTable mocks a range partition table for test +func MockRangePartitionTable() *model.TableInfo { + definitions := []model.PartitionDefinition{ + { + ID: 41, + Name: model.NewCIStr("p1"), + LessThan: []string{"16"}, + }, + { + ID: 42, + Name: model.NewCIStr("p2"), + LessThan: []string{"32"}, + }, + } + tableInfo := MockSignedTable() + tableInfo.Name = model.NewCIStr("pt1") + cols := make([]*model.ColumnInfo, 0, len(tableInfo.Columns)) + cols = append(cols, tableInfo.Columns...) + last := tableInfo.Columns[len(tableInfo.Columns)-1] + cols = append(cols, &model.ColumnInfo{ + State: model.StatePublic, + Offset: last.Offset + 1, + Name: model.NewCIStr("ptn"), + FieldType: newLongType(), + ID: last.ID + 1, + }) + partition := &model.PartitionInfo{ + Type: model.PartitionTypeRange, + Expr: "ptn", + Enable: true, + Definitions: definitions, + } + tableInfo.Columns = cols + tableInfo.Partition = partition + return tableInfo +} + +// MockHashPartitionTable mocks a hash partition table for test +func MockHashPartitionTable() *model.TableInfo { + definitions := []model.PartitionDefinition{ + { + ID: 51, + Name: model.NewCIStr("p1"), + }, + { + ID: 52, + Name: model.NewCIStr("p2"), + }, + } + tableInfo := MockSignedTable() + tableInfo.Name = model.NewCIStr("pt2") + cols := make([]*model.ColumnInfo, 0, len(tableInfo.Columns)) + cols = append(cols, tableInfo.Columns...) + last := tableInfo.Columns[len(tableInfo.Columns)-1] + cols = append(cols, &model.ColumnInfo{ + State: model.StatePublic, + Offset: last.Offset + 1, + Name: model.NewCIStr("ptn"), + FieldType: newLongType(), + ID: last.ID + 1, + }) + partition := &model.PartitionInfo{ + Type: model.PartitionTypeHash, + Expr: "ptn", + Enable: true, + Definitions: definitions, + Num: 2, + } + tableInfo.Columns = cols + tableInfo.Partition = partition + return tableInfo +} + +// MockListPartitionTable mocks a list partition table for test +func MockListPartitionTable() *model.TableInfo { + definitions := []model.PartitionDefinition{ + { + ID: 61, + Name: model.NewCIStr("p1"), + InValues: [][]string{ + { + "1", + }, + }, + }, + { + ID: 62, + Name: model.NewCIStr("p2"), + InValues: [][]string{ + { + "2", + }, + }, + }, + } + tableInfo := MockSignedTable() + tableInfo.Name = model.NewCIStr("pt3") + cols := make([]*model.ColumnInfo, 0, len(tableInfo.Columns)) + cols = append(cols, tableInfo.Columns...) + last := tableInfo.Columns[len(tableInfo.Columns)-1] + cols = append(cols, &model.ColumnInfo{ + State: model.StatePublic, + Offset: last.Offset + 1, + Name: model.NewCIStr("ptn"), + FieldType: newLongType(), + ID: last.ID + 1, + }) + partition := &model.PartitionInfo{ + Type: model.PartitionTypeList, + Expr: "ptn", + Enable: true, + Definitions: definitions, + Num: 2, + } + tableInfo.Columns = cols + tableInfo.Partition = partition + return tableInfo +} diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 1264a47ac97bc..04f572200232b 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -15,6 +15,7 @@ package core import ( + "bytes" "context" "fmt" gomath "math" @@ -58,18 +59,18 @@ const FullRange = -1 type partitionProcessor struct{} func (s *partitionProcessor) optimize(ctx context.Context, lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { - p, err := s.rewriteDataSource(lp) + p, err := s.rewriteDataSource(lp, opt) return p, err } -func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan) (LogicalPlan, error) { +func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { // Assert there will not be sel -> sel in the ast. switch p := lp.(type) { case *DataSource: - return s.prune(p) + return s.prune(p, opt) case *LogicalUnionScan: ds := p.Children()[0] - ds, err := s.prune(ds.(*DataSource)) + ds, err := s.prune(ds.(*DataSource), opt) if err != nil { return nil, err } @@ -94,7 +95,7 @@ func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan) (LogicalPlan, err default: children := lp.Children() for i, child := range children { - newChild, err := s.rewriteDataSource(child) + newChild, err := s.rewriteDataSource(child, opt) if err != nil { return nil, err } @@ -324,7 +325,7 @@ func (s *partitionProcessor) reconstructTableColNames(ds *DataSource) ([]*types. return names, nil } -func (s *partitionProcessor) processHashPartition(ds *DataSource, pi *model.PartitionInfo) (LogicalPlan, error) { +func (s *partitionProcessor) processHashPartition(ds *DataSource, pi *model.PartitionInfo, opt *logicalOptimizeOp) (LogicalPlan, error) { names, err := s.reconstructTableColNames(ds) if err != nil { return nil, err @@ -334,10 +335,11 @@ func (s *partitionProcessor) processHashPartition(ds *DataSource, pi *model.Part return nil, err } if used != nil { - return s.makeUnionAllChildren(ds, pi, convertToRangeOr(used, pi)) + return s.makeUnionAllChildren(ds, pi, convertToRangeOr(used, pi), opt) } tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() + appendNoPartitionChildTraceStep(ds, tableDual, opt) return tableDual, nil } @@ -617,7 +619,7 @@ func (s *partitionProcessor) pruneListPartition(ctx sessionctx.Context, tbl tabl return used, nil } -func (s *partitionProcessor) prune(ds *DataSource) (LogicalPlan, error) { +func (s *partitionProcessor) prune(ds *DataSource, opt *logicalOptimizeOp) (LogicalPlan, error) { pi := ds.tableInfo.GetPartitionInfo() if pi == nil { return ds, nil @@ -631,15 +633,15 @@ func (s *partitionProcessor) prune(ds *DataSource) (LogicalPlan, error) { // Try to locate partition directly for hash partition. switch pi.Type { case model.PartitionTypeRange: - return s.processRangePartition(ds, pi) + return s.processRangePartition(ds, pi, opt) case model.PartitionTypeHash: - return s.processHashPartition(ds, pi) + return s.processHashPartition(ds, pi, opt) case model.PartitionTypeList: - return s.processListPartition(ds, pi) + return s.processListPartition(ds, pi, opt) } // We haven't implement partition by list and so on. - return s.makeUnionAllChildren(ds, pi, fullRange(len(pi.Definitions))) + return s.makeUnionAllChildren(ds, pi, fullRange(len(pi.Definitions)), opt) } // findByName checks whether object name exists in list. @@ -848,7 +850,7 @@ func (s *partitionProcessor) pruneRangePartition(ctx sessionctx.Context, pi *mod if condsToBePruned == nil { return result, nil, nil } - // remove useless predicates after partition pruning + // remove useless predicates after pruning newConds := make([]expression.Expression, 0, len(*condsToBePruned)) for _, cond := range *condsToBePruned { if dataForPrune, ok := pruner.extractDataForPrune(ctx, cond); ok { @@ -872,7 +874,7 @@ func (s *partitionProcessor) pruneRangePartition(ctx sessionctx.Context, pi *mod return result, newConds, nil } -func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.PartitionInfo) (LogicalPlan, error) { +func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.PartitionInfo, opt *logicalOptimizeOp) (LogicalPlan, error) { used, prunedConds, err := s.pruneRangePartition(ds.ctx, pi, ds.table.(table.PartitionedTable), ds.allConds, ds.TblCols, ds.names, &ds.pushedDownConds) if err != nil { return nil, err @@ -880,19 +882,20 @@ func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.Par if prunedConds != nil { ds.pushedDownConds = prunedConds } - return s.makeUnionAllChildren(ds, pi, used) + return s.makeUnionAllChildren(ds, pi, used, opt) } -func (s *partitionProcessor) processListPartition(ds *DataSource, pi *model.PartitionInfo) (LogicalPlan, error) { +func (s *partitionProcessor) processListPartition(ds *DataSource, pi *model.PartitionInfo, opt *logicalOptimizeOp) (LogicalPlan, error) { used, err := s.pruneListPartition(ds.SCtx(), ds.table, ds.partitionNames, ds.allConds) if err != nil { return nil, err } if used != nil { - return s.makeUnionAllChildren(ds, pi, convertToRangeOr(used, pi)) + return s.makeUnionAllChildren(ds, pi, convertToRangeOr(used, pi), opt) } tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() + appendNoPartitionChildTraceStep(ds, tableDual, opt) return tableDual, nil } @@ -1391,9 +1394,11 @@ func (s *partitionProcessor) checkHintsApplicable(ds *DataSource, partitionSet s appendWarnForUnknownPartitions(ds.ctx, HintReadFromStorage, unknownPartitions) } -func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.PartitionInfo, or partitionRangeOR) (LogicalPlan, error) { +func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.PartitionInfo, or partitionRangeOR, opt *logicalOptimizeOp) (LogicalPlan, error) { + children := make([]LogicalPlan, 0, len(pi.Definitions)) partitionNameSet := make(set.StringSet) + usedDefinition := make(map[int64]model.PartitionDefinition) for _, r := range or { for i := r.start; i < r.end; i++ { // This is for `table partition (p0,p1)` syntax, only union the specified partition if has specified partitions. @@ -1421,6 +1426,7 @@ func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.Part return nil, err } children = append(children, &newDataSource) + usedDefinition[pi.Definitions[i].ID] = pi.Definitions[i] } } s.checkHintsApplicable(ds, partitionNameSet) @@ -1429,15 +1435,18 @@ func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.Part // No result after table pruning. tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() + appendMakeUnionAllChildrenTranceStep(ds, usedDefinition, tableDual, children, opt) return tableDual, nil } if len(children) == 1 { // No need for the union all. + appendMakeUnionAllChildrenTranceStep(ds, usedDefinition, children[0], children, opt) return children[0], nil } unionAll := LogicalPartitionUnionAll{}.Init(ds.SCtx(), ds.blockOffset) unionAll.SetChildren(children...) unionAll.SetSchema(ds.schema.Clone()) + appendMakeUnionAllChildrenTranceStep(ds, usedDefinition, unionAll, children, opt) return unionAll, nil } @@ -1565,3 +1574,53 @@ func (p *rangeColumnsPruner) pruneUseBinarySearch(sctx sessionctx.Context, op st } return start, end } + +func appendMakeUnionAllChildrenTranceStep(ds *DataSource, usedMap map[int64]model.PartitionDefinition, plan LogicalPlan, children []LogicalPlan, opt *logicalOptimizeOp) { + if len(children) == 0 { + appendNoPartitionChildTraceStep(ds, plan, opt) + return + } + action := "" + reason := "" + var used []model.PartitionDefinition + for _, def := range usedMap { + used = append(used, def) + } + sort.Slice(used, func(i, j int) bool { + return used[i].ID < used[j].ID + }) + if len(children) == 1 { + action = fmt.Sprintf("Datasource[%v] becomes %s[%v]", ds.ID(), plan.TP(), plan.ID()) + reason = fmt.Sprintf("Datasource[%v] has one needed partition[%s] after pruning", ds.ID(), used[0].Name) + } else { + action = func() string { + buffer := bytes.NewBufferString(fmt.Sprintf("Datasource[%v] becomes %s[%v] with children[", ds.ID(), plan.TP(), plan.ID())) + for i, child := range children { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(fmt.Sprintf("%s[%v]", child.TP(), child.ID())) + } + buffer.WriteString("]") + return buffer.String() + }() + reason = func() string { + buffer := bytes.NewBufferString(fmt.Sprintf("Datasource[%v] has multiple needed partitions[", ds.ID())) + for i, u := range used { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(u.Name.String()) + } + buffer.WriteString("] after pruning") + return buffer.String() + }() + } + opt.appendStepToCurrent(ds.ID(), ds.TP(), reason, action) +} + +func appendNoPartitionChildTraceStep(ds *DataSource, dual LogicalPlan, opt *logicalOptimizeOp) { + action := fmt.Sprintf("Datasource[%v] becomes %v[%v]", ds.ID(), dual.TP(), dual.ID()) + reason := fmt.Sprintf("Datasource[%v] doesn't have needed partition table after pruning", ds.ID()) + opt.appendStepToCurrent(dual.ID(), dual.TP(), reason, action) +}