From 2d64d2359a88e6d8a13f86dc2771684105b56261 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Sun, 22 Mar 2020 22:44:51 +0800 Subject: [PATCH 1/9] README, docker: add quick start about S3 storage Signed-off-by: Neil Shen --- README.md | 16 +++++++++++++++- docker-compose.yaml | 26 ++++++++++++++++++++++++-- docker/Dockerfile | 9 ++++++--- docker/minio.env | 6 ++++++ 4 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 docker/minio.env diff --git a/README.md b/README.md index 6207d98eb..a05b71e8a 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ go-ycsb load mysql -p workload=core \ mysql -uroot -htidb -P4000 -E -e "SELECT COUNT(*) FROM test.usertable" # Build BR and backup! -make release && \ +make build && \ bin/br backup full --pd pd0:2379 --storage "local:///data/backup/full" \ --log-file "/logs/br_backup.log" @@ -69,6 +69,20 @@ bin/br restore full --pd pd0:2379 --storage "local:///data/backup/full" \ # How many rows do we get again? Expected to be 100000 rows. mysql -uroot -htidb -P4000 -E -e "SELECT COUNT(*) FROM test.usertable" + +# Test S3 compatible storage. +# Create a bucket to save backup. +mc config host add minio $S3_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && \ +mc mb minio/mybucket + +# Backup to S3 compatible storage. +bin/br backup full --pd pd0:2379 --storage "s3://mybucket/full" \ + --s3.endpoint="$S3_ENDPOINT" + +# Drop database and restore! +mysql -uroot -htidb -P4000 -E -e "DROP DATABASE test; SHOW DATABASES;" && \ +bin/br restore full --pd pd0:2379 --storage "s3://mybucket/full" \ + --s3.endpoint="$S3_ENDPOINT" ``` ## Contributing diff --git a/docker-compose.yaml b/docker-compose.yaml index 4d84c67fa..ab6360d6d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,6 @@ --- # Source: tidb-docker-compose/templates/docker-compose.yml -version: '2.1' +version: '3.2' services: control: @@ -10,11 +10,13 @@ services: dockerfile: ./docker/Dockerfile volumes: - ./docker/data:/data - - ./docker/logs:/logs + - ./docker/logs:/tmp command: -c "/usr/bin/tail -f /dev/null" depends_on: - "tidb" restart: on-failure + env_file: + - ./docker/minio.env pd0: image: pingcap/pd:latest @@ -64,6 +66,8 @@ services: # soft: 1000000 # hard: 1000000 restart: on-failure + env_file: + - ./docker/minio.env tikv1: image: pingcap/tikv:latest @@ -87,6 +91,8 @@ services: # soft: 1000000 # hard: 1000000 restart: on-failure + env_file: + - ./docker/minio.env tikv2: image: pingcap/tikv:latest @@ -110,6 +116,8 @@ services: # soft: 1000000 # hard: 1000000 restart: on-failure + env_file: + - ./docker/minio.env tikv3: image: pingcap/tikv:latest @@ -133,6 +141,8 @@ services: # soft: 1000000 # hard: 1000000 restart: on-failure + env_file: + - ./docker/minio.env tikv4: image: pingcap/tikv:latest @@ -156,6 +166,8 @@ services: # soft: 1000000 # hard: 1000000 restart: on-failure + env_file: + - ./docker/minio.env tidb: image: pingcap/tidb:latest @@ -185,6 +197,16 @@ services: # hard: 1000000 restart: on-failure + minio: + image: minio/minio + ports: + - 24927:24927 + volumes: + - ./docker/data/s3:/data/s3 + command: server --address=:24927 /data/s3 + env_file: + - ./docker/minio.env + tidb-vision: image: pingcap/tidb-vision:latest environment: diff --git a/docker/Dockerfile b/docker/Dockerfile index c93d22ab4..14c577fcf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,11 +1,13 @@ -FROM golang:1.13.8-buster as builder - # For loading data to TiDB +FROM golang:1.13.8-buster as go-ycsb-builder WORKDIR /go/src/github.com/pingcap/ RUN git clone https://github.com/pingcap/go-ycsb.git && \ cd go-ycsb && \ make +# For operating minio S3 compatible storage +FROM minio/mc as mc-builder + FROM golang:1.13.8-buster RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -19,6 +21,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /go/src/github.com/pingcap/br COPY . . -COPY --from=builder /go/src/github.com/pingcap/go-ycsb/bin/go-ycsb /go/bin/go-ycsb +COPY --from=go-ycsb-builder /go/src/github.com/pingcap/go-ycsb/bin/go-ycsb /go/bin/go-ycsb +COPY --from=mc-builder /usr/bin/mc /usr/bin/mc ENTRYPOINT ["/bin/bash"] diff --git a/docker/minio.env b/docker/minio.env new file mode 100644 index 000000000..d865b2474 --- /dev/null +++ b/docker/minio.env @@ -0,0 +1,6 @@ +MINIO_ACCESS_KEY=brs3accesskey +MINIO_SECRET_KEY=brs3secretkey +MINIO_BROWSER=off +AWS_ACCESS_KEY_ID=brs3accesskey +AWS_SECRET_ACCESS_KEY=brs3secretkey +S3_ENDPOINT=http://minio:24927 From fe911157d433c46ae8d65d68747aefeaf9cafffc Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Sun, 22 Mar 2020 22:54:11 +0800 Subject: [PATCH 2/9] pkg/summary: make sure to output correct summary Signed-off-by: Neil Shen --- cmd/cmd.go | 3 ++- pkg/summary/collector.go | 15 +++++++++++++-- pkg/summary/collector_test.go | 1 + pkg/summary/summary.go | 5 +++++ pkg/task/backup.go | 3 +++ pkg/task/backup_raw.go | 3 +++ pkg/task/common.go | 2 +- pkg/task/restore.go | 4 ++++ pkg/task/restore_raw.go | 2 ++ 9 files changed, 34 insertions(+), 4 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 3e6abff67..6862d684a 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -4,6 +4,7 @@ package cmd import ( "context" + "fmt" "net/http" "net/http/pprof" "os" @@ -46,7 +47,7 @@ const ( ) func timestampLogFileName() string { - return filepath.Join(os.TempDir(), "br-"+time.Now().Format(time.RFC3339)) + return filepath.Join(os.TempDir(), fmt.Sprintf("br.log.%s", time.Now().Format(time.RFC3339))) } // AddFlags adds flags to the given cmd. diff --git a/pkg/summary/collector.go b/pkg/summary/collector.go index e1d722a18..76dd8a121 100644 --- a/pkg/summary/collector.go +++ b/pkg/summary/collector.go @@ -35,6 +35,8 @@ type LogCollector interface { CollectInt(name string, t int) + SetSuccessStatus(success bool) + Summary(name string) } @@ -43,7 +45,9 @@ type logFunc func(msg string, fields ...zap.Field) var collector LogCollector = newLogCollector(log.Info) // InitCollector initilize global collector instance. -func InitCollector(hasLogFile bool) { +func InitCollector( // revive:disable-line:flag-parameter + hasLogFile bool, +) { logF := log.L().Info if hasLogFile { conf := new(log.Config) @@ -69,6 +73,7 @@ type logCollector struct { failureReasons map[string]error durations map[string]time.Duration ints map[string]int + successStatus bool log logFunc } @@ -134,6 +139,12 @@ func (tc *logCollector) CollectInt(name string, t int) { tc.ints[name] += t } +func (tc *logCollector) SetSuccessStatus(success bool) { + tc.mu.Lock() + defer tc.mu.Unlock() + tc.successStatus = success +} + func (tc *logCollector) Summary(name string) { tc.mu.Lock() defer func() { @@ -162,7 +173,7 @@ func (tc *logCollector) Summary(name string) { logFields = append(logFields, zap.Int(key, val)) } - if len(tc.failureReasons) != 0 { + if len(tc.failureReasons) != 0 || !tc.successStatus { for unitName, reason := range tc.failureReasons { logFields = append(logFields, zap.String("unitName", unitName), zap.Error(reason)) } diff --git a/pkg/summary/collector_test.go b/pkg/summary/collector_test.go index 7dff32dd1..165232f55 100644 --- a/pkg/summary/collector_test.go +++ b/pkg/summary/collector_test.go @@ -30,6 +30,7 @@ func (suit *testCollectorSuite) TestSumDurationInt(c *C) { col.CollectDuration("b", time.Second) col.CollectInt("c", 2) col.CollectInt("c", 2) + col.SetSuccessStatus(true) col.Summary("foo") c.Assert(len(fields), Equals, 3) diff --git a/pkg/summary/summary.go b/pkg/summary/summary.go index 3ffdedf8a..852e936a9 100644 --- a/pkg/summary/summary.go +++ b/pkg/summary/summary.go @@ -29,6 +29,11 @@ func CollectInt(name string, t int) { collector.CollectInt(name, t) } +// SetSuccessStatus sets final success status +func SetSuccessStatus(success bool) { + collector.SetSuccessStatus(success) +} + // Summary outputs summary log func Summary(name string) { collector.Summary(name) diff --git a/pkg/task/backup.go b/pkg/task/backup.go index bf90c6739..fe3021a33 100644 --- a/pkg/task/backup.go +++ b/pkg/task/backup.go @@ -214,6 +214,9 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig if err != nil { return err } + + // Set task summary to success status. + summary.SetSuccessStatus(true) return nil } diff --git a/pkg/task/backup_raw.go b/pkg/task/backup_raw.go index e879523b5..fefcc2cf1 100644 --- a/pkg/task/backup_raw.go +++ b/pkg/task/backup_raw.go @@ -141,5 +141,8 @@ func RunBackupRaw(c context.Context, g glue.Glue, cmdName string, cfg *RawKvConf if err != nil { return err } + + // Set task summary to success status. + summary.SetSuccessStatus(true) return nil } diff --git a/pkg/task/common.go b/pkg/task/common.go index 61186abe1..1f0354d2d 100644 --- a/pkg/task/common.go +++ b/pkg/task/common.go @@ -95,7 +95,7 @@ type Config struct { // DefineCommonFlags defines the flags common to all BRIE commands. func DefineCommonFlags(flags *pflag.FlagSet) { flags.BoolP(flagSendCreds, "c", true, "Whether send credentials to tikv") - flags.StringP(flagStorage, "s", "", `specify the url where backup storage, eg, "s3:///path/to/save"`) + flags.StringP(flagStorage, "s", "", `specify the url where backup storage, eg, "s3:///bucket/to/save"`) flags.StringSliceP(flagPD, "u", []string{"127.0.0.1:2379"}, "PD address") flags.String(flagCA, "", "CA certificate path for TLS connection") flags.String(flagCert, "", "Certificate path for TLS connection") diff --git a/pkg/task/restore.go b/pkg/task/restore.go index c75d53f89..5fffd7ce4 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -251,6 +251,8 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf } updateCh.Close() + // Set task summary to success status. + summary.SetSuccessStatus(true) return nil } @@ -415,5 +417,7 @@ func RunRestoreTiflashReplica(c context.Context, g glue.Glue, cmdName string, cf updateCh.Close() summary.CollectInt("recover tables", len(tables)) + // Set task summary to success status. + summary.SetSuccessStatus(true) return nil } diff --git a/pkg/task/restore_raw.go b/pkg/task/restore_raw.go index ccd6bd17a..ca7504f6f 100644 --- a/pkg/task/restore_raw.go +++ b/pkg/task/restore_raw.go @@ -126,5 +126,7 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR // Restore has finished. updateCh.Close() + // Set task summary to success status. + summary.SetSuccessStatus(true) return nil } From 23265348b689942e22ca9b365d9c57e9593f8784 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Sun, 22 Mar 2020 23:10:13 +0800 Subject: [PATCH 3/9] cmd, tests: log to terminal if BR_LOG_TO_TERM is set Signed-off-by: Neil Shen --- cmd/cmd.go | 17 +++++++++++------ tests/run.sh | 1 + 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 6862d684a..c712c2919 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -26,10 +26,11 @@ import ( ) var ( - initOnce = sync.Once{} - defaultContext context.Context - hasLogFile uint64 - tidbGlue = gluetidb.Glue{} + initOnce = sync.Once{} + defaultContext context.Context + hasLogFile uint64 + tidbGlue = gluetidb.Glue{} + envLogToTermKey = "BR_LOG_TO_TERM" ) const ( @@ -82,11 +83,15 @@ func Init(cmd *cobra.Command) (err error) { if err != nil { return } + _, outputLogToTerm := os.LookupEnv(envLogToTermKey) + if outputLogToTerm { + // Log to term if env `BR_LOG_TO_TERM` is set. + conf.File.Filename = "" + } if len(conf.File.Filename) != 0 { atomic.StoreUint64(&hasLogFile, 1) summary.InitCollector(true) - } else { - cmd.Printf("log file: %s\n", conf.File.Filename) + cmd.Printf("Detial BR log in %s\n", conf.File.Filename) } lg, p, e := log.InitLogger(conf) if e != nil { diff --git a/tests/run.sh b/tests/run.sh index 21d6b27ed..5b1111afd 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -39,5 +39,6 @@ for script in tests/${TEST_NAME-*}/run.sh; do TIKV_ADDR="$TIKV_ADDR" \ PATH="tests/_utils:bin:$PATH" \ TEST_NAME="$(basename "$(dirname "$script")")" \ + BR_LOG_TO_TERM=1 \ bash "$script" done From b22e654ed22d3c4723849acc897191bcddc06c80 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Mon, 23 Mar 2020 10:06:35 +0800 Subject: [PATCH 4/9] Update pkg/task/common.go Co-Authored-By: kennytm --- pkg/task/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/task/common.go b/pkg/task/common.go index 1f0354d2d..e3b48748e 100644 --- a/pkg/task/common.go +++ b/pkg/task/common.go @@ -95,7 +95,7 @@ type Config struct { // DefineCommonFlags defines the flags common to all BRIE commands. func DefineCommonFlags(flags *pflag.FlagSet) { flags.BoolP(flagSendCreds, "c", true, "Whether send credentials to tikv") - flags.StringP(flagStorage, "s", "", `specify the url where backup storage, eg, "s3:///bucket/to/save"`) + flags.StringP(flagStorage, "s", "", `specify the url where backup storage, eg, "s3://bucket/path/prefix"`) flags.StringSliceP(flagPD, "u", []string{"127.0.0.1:2379"}, "PD address") flags.String(flagCA, "", "CA certificate path for TLS connection") flags.String(flagCert, "", "Certificate path for TLS connection") From 3de97466687a1963ad50832c84f900a516c01af9 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Mon, 23 Mar 2020 11:30:58 +0800 Subject: [PATCH 5/9] address comments Signed-off-by: Neil Shen --- cmd/cmd.go | 3 +-- tests/br_full_ddl/run.sh | 3 +++ tests/br_full_index/run.sh | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index c712c2919..87a8aadc9 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -4,7 +4,6 @@ package cmd import ( "context" - "fmt" "net/http" "net/http/pprof" "os" @@ -48,7 +47,7 @@ const ( ) func timestampLogFileName() string { - return filepath.Join(os.TempDir(), fmt.Sprintf("br.log.%s", time.Now().Format(time.RFC3339))) + return filepath.Join(os.TempDir(), "br.log."+time.Now().Format(time.RFC3339)) } // AddFlags adds flags to the given cmd. diff --git a/tests/br_full_ddl/run.sh b/tests/br_full_ddl/run.sh index e50ef1ecf..1127dcb98 100755 --- a/tests/br_full_ddl/run.sh +++ b/tests/br_full_ddl/run.sh @@ -36,7 +36,10 @@ done # backup full echo "backup start..." +# Do not log to terminal +unset BR_LOG_TO_TERM run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 --log-file $LOG +BR_LOG_TO_TERM=1 checksum_count=$(cat $LOG | grep "fast checksum success" | wc -l | xargs) diff --git a/tests/br_full_index/run.sh b/tests/br_full_index/run.sh index 5069035e6..7baeb1edb 100755 --- a/tests/br_full_index/run.sh +++ b/tests/br_full_index/run.sh @@ -36,7 +36,10 @@ done # backup full echo "backup start..." +# Do not log to terminal +unset BR_LOG_TO_TERM run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 --log-file $LOG +BR_LOG_TO_TERM=1 checksum_count=$(cat $LOG | grep "fast checksum success" | wc -l | xargs) From 46befae9382d00872133848443341d5f50fe1780 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Mon, 23 Mar 2020 11:50:02 +0800 Subject: [PATCH 6/9] address comments Signed-off-by: Neil Shen --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a05b71e8a..408b09749 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,8 @@ bin/br restore full --pd pd0:2379 --storage "local:///data/backup/full" \ # How many rows do we get again? Expected to be 100000 rows. mysql -uroot -htidb -P4000 -E -e "SELECT COUNT(*) FROM test.usertable" -# Test S3 compatible storage. -# Create a bucket to save backup. +# Test S3 compatible storage (MinIO). +# Create a bucket to save backup by mc (a MinIO Client). mc config host add minio $S3_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && \ mc mb minio/mybucket From c978538cb550f3d6f91f49f852a18cdb103201f1 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Mon, 23 Mar 2020 21:35:47 +0800 Subject: [PATCH 7/9] tests: disable TLS test Signed-off-by: Neil Shen --- tests/run.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/run.sh b/tests/run.sh index 5b1111afd..a0877a9aa 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -29,6 +29,13 @@ if [ "${1-}" = '--debug' ]; then fi for script in tests/${TEST_NAME-*}/run.sh; do + TEST_NAME="$(basename "$(dirname "$script")")" + if [ $TEST_NAME = "br_tls" ]; then + echo "FIXME enable br_tls tests" + echo "TiKV master (ed71f20f445e10595553d2bf3d1a1eb645b9a61a) aborts when TLS is enabled" + continue + fi + echo "*===== Running test $script... =====*" TEST_DIR="$TEST_DIR" \ PD_ADDR="$PD_ADDR" \ From f6cc11dde99c8c5054333050d5212993a4e78798 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Tue, 24 Mar 2020 18:57:51 +0800 Subject: [PATCH 8/9] Revert "tests: disable TLS test" This reverts commit c978538cb550f3d6f91f49f852a18cdb103201f1. --- tests/run.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/run.sh b/tests/run.sh index a0877a9aa..5b1111afd 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -29,13 +29,6 @@ if [ "${1-}" = '--debug' ]; then fi for script in tests/${TEST_NAME-*}/run.sh; do - TEST_NAME="$(basename "$(dirname "$script")")" - if [ $TEST_NAME = "br_tls" ]; then - echo "FIXME enable br_tls tests" - echo "TiKV master (ed71f20f445e10595553d2bf3d1a1eb645b9a61a) aborts when TLS is enabled" - continue - fi - echo "*===== Running test $script... =====*" TEST_DIR="$TEST_DIR" \ PD_ADDR="$PD_ADDR" \ From 0923891113aafbd41a7dac30d041b0e5a34f98f7 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Wed, 25 Mar 2020 10:30:18 +0800 Subject: [PATCH 9/9] tests: cat log if br fails Signed-off-by: Neil Shen --- tests/br_full_ddl/run.sh | 2 +- tests/br_full_index/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/br_full_ddl/run.sh b/tests/br_full_ddl/run.sh index 1127dcb98..93c5b28fb 100755 --- a/tests/br_full_ddl/run.sh +++ b/tests/br_full_ddl/run.sh @@ -38,7 +38,7 @@ done echo "backup start..." # Do not log to terminal unset BR_LOG_TO_TERM -run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 --log-file $LOG +run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 --log-file $LOG || cat $LOG BR_LOG_TO_TERM=1 checksum_count=$(cat $LOG | grep "fast checksum success" | wc -l | xargs) diff --git a/tests/br_full_index/run.sh b/tests/br_full_index/run.sh index 7baeb1edb..bb2486802 100755 --- a/tests/br_full_index/run.sh +++ b/tests/br_full_index/run.sh @@ -38,7 +38,7 @@ done echo "backup start..." # Do not log to terminal unset BR_LOG_TO_TERM -run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 --log-file $LOG +run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 --log-file $LOG || cat $LOG BR_LOG_TO_TERM=1 checksum_count=$(cat $LOG | grep "fast checksum success" | wc -l | xargs)