Merge branch 'master' into raw_backup

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,29 @@
+---
+name: "🐛 Bug Report"
+about: Something isn't working as expected
+title: ''
+labels: 'bug'
+---
+
+Please answer these questions before submitting your issue. Thanks!
+
+1. What did you do?
+If possible, provide a recipe for reproducing the error.
+
+
+2. What did you expect to see?
+
+
+
+3. What did you see instead?
+
+
+
+4. What version of BR and TiDB/TiKV/PD are you using?
+
+<!--
+br -V
+tidb-server -V
+tikv-server -V
+pd-server -V
+-->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,19 @@
+---
+name: "🚀 Feature Request"
+about: I have a suggestion
+labels: enhancement
+---
+
+## Feature Request
+
+### Describe your feature request related problem:
+<!-- A description of what the problem is. -->
+
+### Describe the feature you'd like:
+<!-- A description of what you want to happen. -->
+
+### Describe alternatives you've considered:
+<!-- A description of any alternative solutions or features you've considered. -->
+
+### Teachability, Documentation, Adoption, Migration Strategy:
+<!-- If you can, explain some scenarios how users might use this, or situations in which it would be helpful. Any API designs, mockups, or diagrams are also helpful. -->

.github/pull_request_template.md

@@ -0,0 +1,37 @@
+<!--
+Thank you for working on BR! Please read BR's [CONTRIBUTING](https://github.com/pingcap/br/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
+-->
+
+### What problem does this PR solve? <!--add issue link with summary if exists-->
+
+
+### What is changed and how it works?
+
+
+### Check List <!--REMOVE the items that are not applicable-->
+
+Tests <!-- At least one of them must be included. -->
+
+ - Unit test
+ - Integration test
+ - Manual test (add detailed scripts or steps below)
+ - No code
+
+Code changes
+
+ - Has exported function/method change
+ - Has exported variable/fields change
+ - Has interface methods change
+ - Has persistent data change
+
+Side effects
+
+ - Possible performance regression
+ - Increased code complexity
+ - Breaking backward compatibility
+
+Related changes
+
+ - Need to cherry-pick to the release branch
+ - Need to update the documentation
+ - Need to be included in the release note

.golangci.yml

@@ -9,7 +9,8 @@ issues:
       text: "Potential HTTP request made with variable url"
       linters:
         - gosec
-    - path: .go
-      text: "Use of weak random number generator"
+    # TODO Remove it.
+    - path: split_client.go
+      text: "SA1019:"
       linters:
-        - gosec
+        - staticcheck

CHANGELOG.md

@@ -0,0 +1,18 @@
+# BR (Backup and Restore) Change Log
+All notable changes to this project are documented in this file.
+See also,
+- [TiDB Changelog](https://github.com/pingcap/tidb/blob/master/CHANGELOG.md),
+- [TiKV Changelog](https://github.com/tikv/tikv/blob/master/CHANGELOG.md),
+- [PD Changelog](https://github.com/pingcap/pd/blob/master/CHANGELOG.md).
+
+## [3.1.0-beta.1] - 2020-01-10
+
+- Fix the inaccurate backup progress information [#127](https://github.com/pingcap/br/pull/127)
+- Improve the performance of splitting Regions [#122](https://github.com/pingcap/br/pull/122)
+- Add the backup and restore feature for partitioned tables [#137](https://github.com/pingcap/br/pull/137)
+- Add the feature of automatically scheduling PD schedulers [#123](https://github.com/pingcap/br/pull/123)
+- Fix the issue that data is overwritten after non `PKIsHandle` tables are restored [#139](https://github.com/pingcap/br/pull/139)
+
+## [3.1.0-beta] - 2019-12-20
+
+Initial release of the distributed backup and restore tool

Makefile

@@ -26,6 +26,8 @@ build_for_integration_test:
 		-o bin/br.test
 	# build key locker
 	GO111MODULE=on go build -race -o bin/locker tests/br_key_locked/*.go
+	# build gc
+	GO111MODULE=on go build -race -o bin/gc tests/br_z_gc_safepoint/*.go
 
 test:
 	GO111MODULE=on go test -race -tags leak ./...

cmd/backup.go

@@ -18,10 +18,12 @@ import (
 )
 
 const (
-	flagBackupTimeago     = "timeago"
-	flagBackupRateLimit   = "ratelimit"
-	flagBackupConcurrency = "concurrency"
-	flagBackupChecksum    = "checksum"
+	flagBackupTimeago       = "timeago"
+	flagBackupRateLimit     = "ratelimit"
+	flagBackupRateLimitUnit = "ratelimit-unit"
+	flagBackupConcurrency   = "concurrency"
+	flagBackupChecksum      = "checksum"
+	flagLastBackupTS        = "lastbackupts"
 )
 
 type backupContext struct {
@@ -44,6 +46,13 @@ func defineBackupFlags(flagSet *pflag.FlagSet) {
 		flagBackupConcurrency, "", 4, "The size of thread pool on each node that execute the backup task")
 	flagSet.BoolP(flagBackupChecksum, "", true,
 		"Run checksum after backup")
+	flagSet.Uint64P(flagLastBackupTS, "", 0, "the last time backup ts")
+	_ = flagSet.MarkHidden(flagLastBackupTS)
+
+	// Test only flag.
+	flagSet.Uint64P(
+		flagBackupRateLimitUnit, "", utils.MB, "The unit of rate limit of the backup task")
+	_ = flagSet.MarkHidden(flagBackupRateLimitUnit)
 }
 
 func runBackup(flagSet *pflag.FlagSet, cmdName string, bc backupContext) error {
@@ -65,6 +74,11 @@ func runBackup(flagSet *pflag.FlagSet, cmdName string, bc backupContext) error {
 	if err != nil {
 		return err
 	}
+	ratelimitUnit, err := flagSet.GetUint64(flagBackupRateLimitUnit)
+	if err != nil {
+		return err
+	}
+	ratelimit *= ratelimitUnit
 
 	concurrency, err := flagSet.GetUint32(flagBackupConcurrency)
 	if err != nil {
@@ -80,6 +94,11 @@ func runBackup(flagSet *pflag.FlagSet, cmdName string, bc backupContext) error {
 		return err
 	}
 
+	lastBackupTS, err := flagSet.GetUint64(flagLastBackupTS)
+	if err != nil {
+		return nil
+	}
+
 	u, err := storage.ParseBackendFromFlags(flagSet, FlagStorage)
 	if err != nil {
 		return err
@@ -133,7 +152,7 @@ func runBackup(flagSet *pflag.FlagSet, cmdName string, bc backupContext) error {
 	updateCh := utils.StartProgress(
 		ctx, cmdName, int64(approximateRegions), !HasLogFile())
 	err = client.BackupRanges(
-		ctx, ranges, backupTS, ratelimit, concurrency, updateCh, bc.isRawKv, bc.cf)
+		ctx, ranges, lastBackupTS, backupTS, ratelimit, concurrency, updateCh, bc.isRawKv, bc.cf)
 	if err != nil {
 		return err
 	}

cmd/cmd.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/pingcap/br/pkg/conn"
 	"github.com/pingcap/br/pkg/storage"
+	"github.com/pingcap/br/pkg/utils"
 )
 
 var (
@@ -52,12 +53,18 @@ const (
 	FlagSlowLogFile = "slow-log-file"
 
 	flagDatabase = "db"
+	flagTable    = "table"
 
-	flagTable = "table"
+	flagVersion      = "version"
+	flagVersionShort = "V"
 )
 
 // AddFlags adds flags to the given cmd.
 func AddFlags(cmd *cobra.Command) {
+	cmd.Version = utils.BRInfo()
+	cmd.Flags().BoolP(flagVersion, flagVersionShort, false, "Display version information about BR")
+	cmd.SetVersionTemplate("{{printf \"%s\" .Version}}\n")
+
 	cmd.PersistentFlags().StringP(FlagPD, "u", "127.0.0.1:2379", "PD address")
 	cmd.PersistentFlags().String(FlagCA, "", "CA certificate path for TLS connection")
 	cmd.PersistentFlags().String(FlagCert, "", "Certificate path for TLS connection")

cmd/restore.go

@@ -63,15 +63,13 @@ func NewRestoreCommand() *cobra.Command {
 		"Run checksum after restore")
 	command.PersistentFlags().BoolP("online", "", false,
 		"Whether online when restore")
+	// TODO remove hidden flag if it's stable
+	_ = command.PersistentFlags().MarkHidden("online")
 
 	return command
 }
 
 func runRestore(flagSet *flag.FlagSet, cmdName, dbName, tableName string) error {
-	pdAddr, err := flagSet.GetString(FlagPD)
-	if err != nil {
-		return errors.Trace(err)
-	}
 	ctx, cancel := context.WithCancel(GetDefaultContext())
 	defer cancel()
 
@@ -113,6 +111,9 @@ func runRestore(flagSet *flag.FlagSet, cmdName, dbName, tableName string) error
 	case len(dbName) != 0 && len(tableName) == 0:
 		// database restore
 		db := client.GetDatabase(dbName)
+		if db == nil {
+			return errors.Errorf("database %s not found in backup", dbName)
+		}
 		err = client.CreateDatabase(db.Schema)
 		if err != nil {
 			return errors.Trace(err)
@@ -124,19 +125,28 @@ func runRestore(flagSet *flag.FlagSet, cmdName, dbName, tableName string) error
 	case len(dbName) != 0 && len(tableName) != 0:
 		// table restore
 		db := client.GetDatabase(dbName)
+		if db == nil {
+			return errors.Errorf("database %s not found in backup", dbName)
+		}
 		err = client.CreateDatabase(db.Schema)
 		if err != nil {
 			return errors.Trace(err)
 		}
 		table := db.GetTable(tableName)
 		files = table.Files
-		tables = utils.Tables{table}
+		tables = append(tables, table)
 	default:
 		return errors.New("must set db when table was set")
 	}
-
+	var newTS uint64
+	if client.IsIncremental() {
+		newTS, err = client.GetTS(ctx)
+		if err != nil {
+			return err
+		}
+	}
 	summary.CollectInt("restore files", len(files))
-	rewriteRules, newTables, err := client.CreateTables(mgr.GetDomain(), tables)
+	rewriteRules, newTables, err := client.CreateTables(mgr.GetDomain(), tables, newTS)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -159,11 +169,19 @@ func runRestore(flagSet *flag.FlagSet, cmdName, dbName, tableName string) error
 		log.Error("split regions failed", zap.Error(err))
 		return errors.Trace(err)
 	}
-	pdAddrs := strings.Split(pdAddr, ",")
-	err = client.ResetTS(pdAddrs)
-	if err != nil {
-		log.Error("reset pd TS failed", zap.Error(err))
-		return errors.Trace(err)
+
+	if !client.IsIncremental() {
+		var pdAddr string
+		pdAddr, err = flagSet.GetString(FlagPD)
+		if err != nil {
+			return errors.Trace(err)
+		}
+		pdAddrs := strings.Split(pdAddr, ",")
+		err = client.ResetTS(pdAddrs)
+		if err != nil {
+			log.Error("reset pd TS failed", zap.Error(err))
+			return errors.Trace(err)
+		}
 	}
 
 	removedSchedulers, err := RestorePrepareWork(ctx, client, mgr)