diff --git a/lightning/backend/sql2kv.go b/lightning/backend/sql2kv.go index 115b2732d..4e18ab269 100644 --- a/lightning/backend/sql2kv.go +++ b/lightning/backend/sql2kv.go @@ -164,12 +164,13 @@ func (kvcodec *tableKVEncoder) Encode( for i, col := range cols { j := columnPermutation[i] + isAutoIncCol := mysql.HasAutoIncrementFlag(col.Flag) if j >= 0 && j < len(row) { value, err = table.CastValue(kvcodec.se, row[j], col.ToInfo()) if err == nil { value, err = col.HandleBadNull(value, kvcodec.se.vars.StmtCtx) } - } else if mysql.HasAutoIncrementFlag(col.Flag) { + } else if isAutoIncCol { // we still need a conversion, e.g. to catch overflow with a TINYINT column. value, err = table.CastValue(kvcodec.se, types.NewIntDatum(rowID), col.ToInfo()) } else { @@ -179,6 +180,9 @@ func (kvcodec *tableKVEncoder) Encode( return nil, logKVConvertFailed(logger, row, j, col.ToInfo(), err) } record = append(record, value) + if isAutoIncCol { + kvcodec.tbl.RebaseAutoID(kvcodec.se, value.GetInt64(), false) + } } if !kvcodec.tbl.Meta().PKIsHandle { @@ -192,6 +196,7 @@ func (kvcodec *tableKVEncoder) Encode( return nil, logKVConvertFailed(logger, row, j, extraHandleColumnInfo, err) } record = append(record, value) + kvcodec.tbl.RebaseAutoID(kvcodec.se, value.GetInt64(), false) } _, err = kvcodec.tbl.AddRecord(kvcodec.se, record) diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index fa8127b10..77d512fa3 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -38,10 +38,10 @@ import ( "go.uber.org/zap" "modernc.org/mathutil" + kv "github.com/pingcap/tidb-lightning/lightning/backend" . "github.com/pingcap/tidb-lightning/lightning/checkpoints" "github.com/pingcap/tidb-lightning/lightning/common" "github.com/pingcap/tidb-lightning/lightning/config" - kv "github.com/pingcap/tidb-lightning/lightning/backend" "github.com/pingcap/tidb-lightning/lightning/log" "github.com/pingcap/tidb-lightning/lightning/metric" "github.com/pingcap/tidb-lightning/lightning/mydump" @@ -646,11 +646,6 @@ func (t *TableRestore) restoreTable( // rebase the allocator so it exceeds the number of rows. cp.AllocBase = mathutil.MaxInt64(cp.AllocBase, t.tableInfo.Core.AutoIncID) - for _, engine := range cp.Engines { - for _, chunk := range engine.Chunks { - cp.AllocBase = mathutil.MaxInt64(cp.AllocBase, chunk.Chunk.RowIDMax) - } - } t.alloc.Rebase(t.tableInfo.ID, cp.AllocBase, false) rc.saveCpCh <- saveCp{ tableName: t.tableName, diff --git a/tests/tool_1472/config.toml b/tests/tool_1472/config.toml new file mode 100644 index 000000000..13021b873 --- /dev/null +++ b/tests/tool_1472/config.toml @@ -0,0 +1,16 @@ +[lightning] +file = "/tmp/lightning_test_result/lightning.log" + +[tikv-importer] +addr = "127.0.0.1:8808" + +[mydumper] +data-source-dir = "tests/tool_1472/data" + +[tidb] +host = "127.0.0.1" +port = 4000 +user = "root" +status-port = 10080 +pd-addr = "127.0.0.1:2379" +log-level = "error" diff --git a/tests/tool_1472/data/EE1472-schema-create.sql b/tests/tool_1472/data/EE1472-schema-create.sql new file mode 100644 index 000000000..88617459d --- /dev/null +++ b/tests/tool_1472/data/EE1472-schema-create.sql @@ -0,0 +1 @@ +create database `EE1472`; diff --git a/tests/tool_1472/data/EE1472.notpk-schema.sql b/tests/tool_1472/data/EE1472.notpk-schema.sql new file mode 100644 index 000000000..8cc80840f --- /dev/null +++ b/tests/tool_1472/data/EE1472.notpk-schema.sql @@ -0,0 +1,5 @@ +create table `notpk` ( + a int primary key, + b tinyint auto_increment, + key(a) +); diff --git a/tests/tool_1472/data/EE1472.notpk.1.sql b/tests/tool_1472/data/EE1472.notpk.1.sql new file mode 100644 index 000000000..3b8464134 --- /dev/null +++ b/tests/tool_1472/data/EE1472.notpk.1.sql @@ -0,0 +1,8 @@ +insert into `notpk` values (1111, 6); +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. diff --git a/tests/tool_1472/data/EE1472.notpk.2.sql b/tests/tool_1472/data/EE1472.notpk.2.sql new file mode 100644 index 000000000..ef42b75e9 --- /dev/null +++ b/tests/tool_1472/data/EE1472.notpk.2.sql @@ -0,0 +1,8 @@ +insert into `notpk` values (2222, 9); +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. diff --git a/tests/tool_1472/data/EE1472.pk-schema.sql b/tests/tool_1472/data/EE1472.pk-schema.sql new file mode 100644 index 000000000..187f4edfd --- /dev/null +++ b/tests/tool_1472/data/EE1472.pk-schema.sql @@ -0,0 +1,3 @@ +create table `pk` ( + a tinyint primary key auto_increment +); diff --git a/tests/tool_1472/data/EE1472.pk.1.sql b/tests/tool_1472/data/EE1472.pk.1.sql new file mode 100644 index 000000000..412f35dee --- /dev/null +++ b/tests/tool_1472/data/EE1472.pk.1.sql @@ -0,0 +1,8 @@ +insert into `pk` values (3); +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. diff --git a/tests/tool_1472/data/EE1472.pk.2.sql b/tests/tool_1472/data/EE1472.pk.2.sql new file mode 100644 index 000000000..52d434888 --- /dev/null +++ b/tests/tool_1472/data/EE1472.pk.2.sql @@ -0,0 +1,8 @@ +insert into `pk` values (4); +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. +-- include some comments to inflate the file size. diff --git a/tests/tool_1472/run.sh b/tests/tool_1472/run.sh new file mode 100755 index 000000000..c0a1a747d --- /dev/null +++ b/tests/tool_1472/run.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +# This test verifies if TOOL-1420 is fixed. +# It involves column names not in lower-case. + +set -eu + +run_sql 'drop database if exists EE1472;' +run_lightning + +run_sql 'insert into EE1472.pk values ();' +run_sql 'select count(a), max(a) from EE1472.pk;' +check_contains 'count(a): 3' +check_contains 'max(a): 5' + +run_sql 'insert into EE1472.notpk (a) values (3333);' +run_sql 'select b from EE1472.notpk where a = 3333;' +check_contains 'b: 10'