backend: dynamically calculate the maximum auto-inc ID

lightning/backend/sql2kv.go

@@ -164,12 +164,13 @@ func (kvcodec *tableKVEncoder) Encode(
 
 	for i, col := range cols {
 		j := columnPermutation[i]
+		isAutoIncCol := mysql.HasAutoIncrementFlag(col.Flag)
 		if j >= 0 && j < len(row) {
 			value, err = table.CastValue(kvcodec.se, row[j], col.ToInfo())
 			if err == nil {
 				value, err = col.HandleBadNull(value, kvcodec.se.vars.StmtCtx)
 			}
-		} else if mysql.HasAutoIncrementFlag(col.Flag) {
+		} else if isAutoIncCol {
 			// we still need a conversion, e.g. to catch overflow with a TINYINT column.
 			value, err = table.CastValue(kvcodec.se, types.NewIntDatum(rowID), col.ToInfo())
 		} else {
@@ -179,6 +180,9 @@ func (kvcodec *tableKVEncoder) Encode(
 			return nil, logKVConvertFailed(logger, row, j, col.ToInfo(), err)
 		}
 		record = append(record, value)
+		if isAutoIncCol {
+			kvcodec.tbl.RebaseAutoID(kvcodec.se, value.GetInt64(), false)
+		}
 	}
 
 	if !kvcodec.tbl.Meta().PKIsHandle {
@@ -192,6 +196,7 @@ func (kvcodec *tableKVEncoder) Encode(
 			return nil, logKVConvertFailed(logger, row, j, extraHandleColumnInfo, err)
 		}
 		record = append(record, value)
+		kvcodec.tbl.RebaseAutoID(kvcodec.se, value.GetInt64(), false)
 	}
 
 	_, err = kvcodec.tbl.AddRecord(kvcodec.se, record)

lightning/backend/sql2kv_test.go

@@ -66,8 +66,9 @@ func (mockTable) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...*tab
 func (s *kvSuite) TestEncode(c *C) {
 	c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)}
 	cols := []*model.ColumnInfo{c1}
-	tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false}
-	tbl := tables.MockTableFromMeta(tblInfo)
+	tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic}
+	tbl, err := tables.TableFromMeta(NewPanickingAllocator(0), tblInfo)
+	c.Assert(err, IsNil)
 
 	logger := log.Logger{Logger: zap.NewNop()}
 	rows := []types.Datum{

lightning/restore/restore.go

@@ -38,10 +38,10 @@ import (
 	"go.uber.org/zap"
 	"modernc.org/mathutil"
 
+	kv "github.com/pingcap/tidb-lightning/lightning/backend"
 	. "github.com/pingcap/tidb-lightning/lightning/checkpoints"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/config"
-	kv "github.com/pingcap/tidb-lightning/lightning/backend"
 	"github.com/pingcap/tidb-lightning/lightning/log"
 	"github.com/pingcap/tidb-lightning/lightning/metric"
 	"github.com/pingcap/tidb-lightning/lightning/mydump"
@@ -646,11 +646,6 @@ func (t *TableRestore) restoreTable(
 
 		// rebase the allocator so it exceeds the number of rows.
 		cp.AllocBase = mathutil.MaxInt64(cp.AllocBase, t.tableInfo.Core.AutoIncID)
-		for _, engine := range cp.Engines {
-			for _, chunk := range engine.Chunks {
-				cp.AllocBase = mathutil.MaxInt64(cp.AllocBase, chunk.Chunk.RowIDMax)
-			}
-		}
 		t.alloc.Rebase(t.tableInfo.ID, cp.AllocBase, false)
 		rc.saveCpCh <- saveCp{
 			tableName: t.tableName,

tests/tool_1472/config.toml

@@ -0,0 +1,16 @@
+[lightning]
+file = "/tmp/lightning_test_result/lightning.log"
+
+[tikv-importer]
+addr = "127.0.0.1:8808"
+
+[mydumper]
+data-source-dir = "tests/tool_1472/data"
+
+[tidb]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+status-port = 10080
+pd-addr = "127.0.0.1:2379"
+log-level = "error"

tests/tool_1472/data/EE1472-schema-create.sql

@@ -0,0 +1 @@
+create database `EE1472`;

tests/tool_1472/data/EE1472.notpk-schema.sql

@@ -0,0 +1,5 @@
+create table `notpk` (
+    a int primary key,
+    b tinyint auto_increment,
+    key(a)
+);

tests/tool_1472/data/EE1472.notpk.1.sql

@@ -0,0 +1,8 @@
+insert into `notpk` values (1111, 6);
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.

tests/tool_1472/data/EE1472.notpk.2.sql

@@ -0,0 +1,8 @@
+insert into `notpk` values (2222, 9);
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.

tests/tool_1472/data/EE1472.pk-schema.sql

@@ -0,0 +1,3 @@
+create table `pk` (
+    a tinyint primary key auto_increment
+);

tests/tool_1472/data/EE1472.pk.1.sql

@@ -0,0 +1,8 @@
+insert into `pk` values (3);
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.

tests/tool_1472/data/EE1472.pk.2.sql

@@ -0,0 +1,8 @@
+insert into `pk` values (4);
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.
+-- include some comments to inflate the file size.

tests/tool_1472/run.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+#
+# Copyright 2019 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This test verifies if TOOL-1420 is fixed.
+# It involves column names not in lower-case.
+
+set -eu
+
+run_sql 'drop database if exists EE1472;'
+run_lightning
+
+run_sql 'insert into EE1472.pk values ();'
+run_sql 'select count(a), max(a) from EE1472.pk;'
+check_contains 'count(a): 3'
+check_contains 'max(a): 5'
+
+run_sql 'insert into EE1472.notpk (a) values (3333);'
+run_sql 'select b from EE1472.notpk where a = 3333;'
+check_contains 'b: 10'