Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
Signed-off-by: Weizhen Wang <wangweizhen@pingcap.com>
  • Loading branch information
hawkingrei committed Sep 4, 2024
1 parent c6fa020 commit 68b81aa
Show file tree
Hide file tree
Showing 38 changed files with 298 additions and 327 deletions.
12 changes: 6 additions & 6 deletions br/pkg/restore/internal/prealloc_table_id/alloc.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,23 +43,23 @@ func New(tables []*metautil.Table) *PreallocIDs {
}
}

max := int64(0)
maxv := int64(0)

for _, t := range tables {
if t.Info.ID > max && t.Info.ID < insaneTableIDThreshold {
max = t.Info.ID
if t.Info.ID > maxv && t.Info.ID < insaneTableIDThreshold {
maxv = t.Info.ID
}

if t.Info.Partition != nil && t.Info.Partition.Definitions != nil {
for _, part := range t.Info.Partition.Definitions {
if part.ID > max && part.ID < insaneTableIDThreshold {
max = part.ID
if part.ID > maxv && part.ID < insaneTableIDThreshold {
maxv = part.ID
}
}
}
}
return &PreallocIDs{
end: max + 1,
end: maxv + 1,

allocedFrom: math.MaxInt64,
}
Expand Down
14 changes: 7 additions & 7 deletions cmd/benchdb/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -201,8 +201,8 @@ func (ut *benchDB) truncateTable() {
func (ut *benchDB) runCountTimes(name string, count int, f func()) {
var (
sum, first, last time.Duration
min = time.Minute
max = time.Nanosecond
minv = time.Minute
maxv = time.Nanosecond
)
cLogf("%s started", name)
for i := 0; i < count; i++ {
Expand All @@ -213,16 +213,16 @@ func (ut *benchDB) runCountTimes(name string, count int, f func()) {
first = dur
}
last = dur
if dur < min {
min = dur
if dur < minv {
minv = dur
}
if dur > max {
max = dur
if dur > maxv {
maxv = dur
}
sum += dur
}
cLogf("%s done, avg %s, count %d, sum %s, first %s, last %s, max %s, min %s\n\n",
name, sum/time.Duration(count), count, sum, first, last, max, min)
name, sum/time.Duration(count), count, sum, first, last, maxv, minv)
}

// #nosec G404
Expand Down
8 changes: 4 additions & 4 deletions cmd/importer/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,13 @@ func randInt64Value(column *column, minv, maxv int64) int64 {
return data
}

minv, maxv = intRangeValue(column, min, max)
minv, maxv = intRangeValue(column, minv, maxv)
return randInt64(minv, maxv)
}

func nextInt64Value(column *column, min int64, max int64) int64 {
min, max = intRangeValue(column, min, max)
column.data.setInitInt64Value(min, max)
func nextInt64Value(column *column, minv int64, maxv int64) int64 {
minv, maxv = intRangeValue(column, minv, maxv)
column.data.setInitInt64Value(minv, maxv)
return column.data.nextInt64()
}

Expand Down
56 changes: 28 additions & 28 deletions cmd/importer/rand.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,24 +70,24 @@ func randDate(col *column) string {
return col.hist.randDate("DAY", "%Y-%m-%d", dateFormat)
}

min, max := col.min, col.max
if min == "" {
minv, maxv := col.min, col.max
if minv == "" {
year := time.Now().Year()
month := randInt(1, 12)
day := randInt(1, 28)
return fmt.Sprintf("%04d-%02d-%02d", year, month, day)
}

minTime, err := time.Parse(dateFormat, min)
minTime, err := time.Parse(dateFormat, minv)
if err != nil {
log.Warn("parse min date failed", zap.Error(err))
}
if max == "" {
if maxv == "" {
t := minTime.Add(time.Duration(randInt(0, 365)) * 24 * time.Hour) // nolint: durationcheck
return fmt.Sprintf("%04d-%02d-%02d", t.Year(), t.Month(), t.Day())
}

maxTime, err := time.Parse(dateFormat, max)
maxTime, err := time.Parse(dateFormat, maxv)
if err != nil {
log.Warn("parse max date failed", zap.Error(err))
}
Expand All @@ -100,21 +100,21 @@ func randTime(col *column) string {
if col.hist != nil {
return col.hist.randDate("SECOND", "%H:%i:%s", timeFormat)
}
min, max := col.min, col.max
if min == "" || max == "" {
minv, maxv := col.min, col.max
if minv == "" || maxv == "" {
hour := randInt(0, 23)
min := randInt(0, 59)
minute := randInt(0, 59)
sec := randInt(0, 59)
return fmt.Sprintf("%02d:%02d:%02d", hour, min, sec)
return fmt.Sprintf("%02d:%02d:%02d", hour, minute, sec)
}

minTime, err := time.Parse(timeFormat, min)
minTime, err := time.Parse(timeFormat, minv)
if err != nil {
log.Warn("parse min time failed", zap.Error(err))
log.Warn("parse minv time failed", zap.Error(err))
}
maxTime, err := time.Parse(timeFormat, max)
maxTime, err := time.Parse(timeFormat, maxv)
if err != nil {
log.Warn("parse max time failed", zap.Error(err))
log.Warn("parse maxv time failed", zap.Error(err))
}
seconds := int(maxTime.Sub(minTime).Seconds())
t := minTime.Add(time.Duration(randInt(0, seconds)) * time.Second)
Expand All @@ -125,29 +125,29 @@ func randTimestamp(col *column) string {
if col.hist != nil {
return col.hist.randDate("SECOND", "%Y-%m-%d %H:%i:%s", dateTimeFormat)
}
min, max := col.min, col.max
if min == "" {
minv, maxv := col.min, col.max
if minv == "" {
year := time.Now().Year()
month := randInt(1, 12)
day := randInt(1, 28)
hour := randInt(0, 23)
min := randInt(0, 59)
minute := randInt(0, 59)
sec := randInt(0, 59)
return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, min, sec)
return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, sec)
}

minTime, err := time.Parse(dateTimeFormat, min)
minTime, err := time.Parse(dateTimeFormat, minv)
if err != nil {
log.Warn("parse min timestamp failed", zap.Error(err))
log.Warn("parse minv timestamp failed", zap.Error(err))
}
if max == "" {
if maxv == "" {
t := minTime.Add(time.Duration(randInt(0, 365)) * 24 * time.Hour) // nolint: durationcheck
return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
}

maxTime, err := time.Parse(dateTimeFormat, max)
maxTime, err := time.Parse(dateTimeFormat, maxv)
if err != nil {
log.Warn("parse max timestamp failed", zap.Error(err))
log.Warn("parse maxv timestamp failed", zap.Error(err))
}
seconds := int(maxTime.Sub(minTime).Seconds())
t := minTime.Add(time.Duration(randInt(0, seconds)) * time.Second)
Expand All @@ -158,18 +158,18 @@ func randYear(col *column) string {
if col.hist != nil {
return col.hist.randDate("YEAR", "%Y", yearFormat)
}
min, max := col.min, col.max
if min == "" || max == "" {
minv, maxv := col.min, col.max
if minv == "" || maxv == "" {
return fmt.Sprintf("%04d", time.Now().Year()-randInt(0, 10))
}

minTime, err := time.Parse(yearFormat, min)
minTime, err := time.Parse(yearFormat, minv)
if err != nil {
log.Warn("parse min year failed", zap.Error(err))
log.Warn("parse minv year failed", zap.Error(err))
}
maxTime, err := time.Parse(yearFormat, max)
maxTime, err := time.Parse(yearFormat, maxv)
if err != nil {
log.Warn("parse max year failed", zap.Error(err))
log.Warn("parse maxv year failed", zap.Error(err))
}
seconds := int(maxTime.Sub(minTime).Seconds())
t := minTime.Add(time.Duration(randInt(0, seconds)) * time.Second)
Expand Down
26 changes: 13 additions & 13 deletions dumpling/export/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -773,24 +773,24 @@ func (d *Dumper) concurrentDumpTable(tctx *tcontext.Context, conn *BaseConn, met
return d.dumpWholeTableDirectly(tctx, meta, taskChan, "", orderByClause, 0, 1)
}

min, max, err := d.selectMinAndMaxIntValue(tctx, conn, db, tbl, field)
minv, maxv, err := d.selectMinAndMaxIntValue(tctx, conn, db, tbl, field)
if err != nil {
tctx.L().Info("fallback to sequential dump due to cannot get bounding values. This won't influence the whole dump process",
log.ShortError(err))
return d.dumpWholeTableDirectly(tctx, meta, taskChan, "", orderByClause, 0, 1)
}
tctx.L().Debug("get int bounding values",
zap.String("lower", min.String()),
zap.String("upper", max.String()))
zap.String("lower", minv.String()),
zap.String("upper", maxv.String()))

// every chunk would have eventual adjustments
estimatedChunks := count / conf.Rows
estimatedStep := new(big.Int).Sub(max, min).Uint64()/estimatedChunks + 1
estimatedStep := new(big.Int).Sub(maxv, minv).Uint64()/estimatedChunks + 1
bigEstimatedStep := new(big.Int).SetUint64(estimatedStep)
cutoff := new(big.Int).Set(min)
cutoff := new(big.Int).Set(minv)
totalChunks := estimatedChunks
if estimatedStep == 1 {
totalChunks = new(big.Int).Sub(max, min).Uint64() + 1
totalChunks = new(big.Int).Sub(maxv, minv).Uint64() + 1
}

selectField, selectLen := meta.SelectedField(), meta.SelectedLen()
Expand All @@ -800,7 +800,7 @@ func (d *Dumper) concurrentDumpTable(tctx *tcontext.Context, conn *BaseConn, met
if conf.Where == "" {
nullValueCondition = fmt.Sprintf("`%s` IS NULL OR ", escapeString(field))
}
for max.Cmp(cutoff) >= 0 {
for maxv.Cmp(cutoff) >= 0 {
nextCutOff := new(big.Int).Add(cutoff, bigEstimatedStep)
where := fmt.Sprintf("%s(`%s` >= %d AND `%s` < %d)", nullValueCondition, escapeString(field), cutoff, escapeString(field), nextCutOff)
query := buildSelectQuery(db, tbl, selectField, "", buildWhereCondition(conf, where), orderByClause)
Expand Down Expand Up @@ -847,23 +847,23 @@ func (d *Dumper) selectMinAndMaxIntValue(tctx *tcontext.Context, conn *BaseConn,
return err
}, func() {}, query)
if err != nil {
return zero, zero, errors.Annotatef(err, "can't get min/max values to split chunks, query: %s", query)
return zero, zero, errors.Annotatef(err, "can't get min/maxv values to split chunks, query: %s", query)
}
if !smax.Valid || !smin.Valid {
// found no data
return zero, zero, errors.Errorf("no invalid min/max value found in query %s", query)
}

max := new(big.Int)
min := new(big.Int)
maxv := new(big.Int)
minv := new(big.Int)
var ok bool
if max, ok = max.SetString(smax.String, 10); !ok {
if maxv, ok = maxv.SetString(smax.String, 10); !ok {
return zero, zero, errors.Errorf("fail to convert max value %s in query %s", smax.String, query)
}
if min, ok = min.SetString(smin.String, 10); !ok {
if minv, ok = minv.SetString(smin.String, 10); !ok {
return zero, zero, errors.Errorf("fail to convert min value %s in query %s", smin.String, query)
}
return min, max, nil
return minv, maxv, nil
}

func (d *Dumper) concurrentDumpTiDBTables(tctx *tcontext.Context, conn *BaseConn, meta TableMeta, taskChan chan<- Task) error {
Expand Down
22 changes: 11 additions & 11 deletions pkg/autoid_service/autoid.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ type autoIDValue struct {
}

func (alloc *autoIDValue) alloc4Unsigned(ctx context.Context, store kv.Storage, dbID, tblID int64, isUnsigned bool,
n uint64, increment, offset int64) (min int64, max int64, err error) {
n uint64, increment, offset int64) (minv, maxv int64, err error) {
// Check offset rebase if necessary.
if uint64(offset-1) > uint64(alloc.base) {
if err := alloc.rebase4Unsigned(ctx, store, dbID, tblID, uint64(offset-1)); err != nil {
Expand Down Expand Up @@ -122,17 +122,17 @@ func (alloc *autoIDValue) alloc4Unsigned(ctx context.Context, store kv.Storage,
zap.Int64("to end", newEnd))
alloc.end = newEnd
}
min = alloc.base
minv = alloc.base
// Use uint64 n directly.
alloc.base = int64(uint64(alloc.base) + uint64(n1))
return min, alloc.base, nil
return minv, alloc.base, nil
}

func (alloc *autoIDValue) alloc4Signed(ctx context.Context,
store kv.Storage,
dbID, tblID int64,
isUnsigned bool,
n uint64, increment, offset int64) (min int64, max int64, err error) {
n uint64, increment, offset int64) (minv, maxv int64, err error) {
// Check offset rebase if necessary.
if offset-1 > alloc.base {
if err := alloc.rebase4Signed(ctx, store, dbID, tblID, offset-1); err != nil {
Expand Down Expand Up @@ -198,9 +198,9 @@ func (alloc *autoIDValue) alloc4Signed(ctx context.Context,
zap.Int64("to end", newEnd))
alloc.end = newEnd
}
min = alloc.base
minv = alloc.base
alloc.base += n1
return min, alloc.base, nil
return minv, alloc.base, nil
}

func (alloc *autoIDValue) rebase4Unsigned(ctx context.Context,
Expand Down Expand Up @@ -502,20 +502,20 @@ func (s *Service) allocAutoID(ctx context.Context, req *autoid.AutoIDRequest) (*
}, nil
}

var min, max int64
var minv, maxv int64
var err error
if req.IsUnsigned {
min, max, err = val.alloc4Unsigned(ctx, s.store, req.DbID, req.TblID, req.IsUnsigned, req.N, req.Increment, req.Offset)
minv, maxv, err = val.alloc4Unsigned(ctx, s.store, req.DbID, req.TblID, req.IsUnsigned, req.N, req.Increment, req.Offset)
} else {
min, max, err = val.alloc4Signed(ctx, s.store, req.DbID, req.TblID, req.IsUnsigned, req.N, req.Increment, req.Offset)
minv, maxv, err = val.alloc4Signed(ctx, s.store, req.DbID, req.TblID, req.IsUnsigned, req.N, req.Increment, req.Offset)
}

if err != nil {
return &autoid.AutoIDResponse{Errmsg: []byte(err.Error())}, nil
}
return &autoid.AutoIDResponse{
Min: min,
Max: max,
Min: minv,
Max: maxv,
}, nil
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/autoid_service/autoid_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ type autoIDResp struct {
*testing.T
}

func (resp autoIDResp) check(min, max int64) {
func (resp autoIDResp) check(minv, maxv int64) {
require.NoError(resp.T, resp.error)
require.Equal(resp.T, resp.AutoIDResponse, &autoid.AutoIDResponse{Min: min, Max: max})
require.Equal(resp.T, resp.AutoIDResponse, &autoid.AutoIDResponse{Min: minv, Max: maxv})
}

func (resp autoIDResp) checkErrmsg() {
Expand Down Expand Up @@ -102,12 +102,12 @@ type dest struct {
tblID int64
}

func checkCurrValue(t *testing.T, cli autoid.AutoIDAllocClient, to dest, min, max int64) {
func checkCurrValue(t *testing.T, cli autoid.AutoIDAllocClient, to dest, minv, maxv int64) {
req := &autoid.AutoIDRequest{DbID: to.dbID, TblID: to.tblID, N: 0, KeyspaceID: uint32(tikv.NullspaceID)}
ctx := context.Background()
resp, err := cli.AllocAutoID(ctx, req)
require.NoError(t, err)
require.Equal(t, resp, &autoid.AutoIDResponse{Min: min, Max: max})
require.Equal(t, resp, &autoid.AutoIDResponse{Min: minv, Max: maxv})
}

func autoIDRequest(t *testing.T, cli autoid.AutoIDAllocClient, to dest, unsigned bool, n uint64, more ...int64) autoIDResp {
Expand Down
6 changes: 3 additions & 3 deletions pkg/ddl/job_worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -686,9 +686,9 @@ func writeBinlog(binlogCli *pumpcli.PumpsClient, txn kv.Transaction, job *model.
}
}

func chooseLeaseTime(t, max time.Duration) time.Duration {
if t == 0 || t > max {
return max
func chooseLeaseTime(t, maxv time.Duration) time.Duration {
if t == 0 || t > maxv {
return maxv
}
return t
}
Expand Down
Loading

0 comments on commit 68b81aa

Please sign in to comment.