Create integration test for S3 storage

Makefile

@@ -49,6 +49,7 @@ integration_test: build build_for_integration_test
 	@which bin/pd-server
 	@which bin/pd-ctl
 	@which bin/go-ycsb
+	@which bin/minio
 	@which bin/br
 	tests/run.sh
 

tests/README.md

@@ -18,6 +18,7 @@ programs.
 
     * `mysql` (the CLI client)
     * `curl`
+    * `s3cmd`
 
 3. The user executing the tests must have permission to create the folder
     `/tmp/backup_restore_test`. All test artifacts will be written into this folder.
@@ -45,4 +46,4 @@ The script should exit with a nonzero error code on failure.
 
 Several convenient commands are provided:
 
-* `run_sql <SQL>` — Executes an SQL query on the TiDB database
+* `run_sql <SQL>` — Executes an SQL query on the TiDB database

tests/_utils/run_services

@@ -90,7 +90,7 @@ start_services() {
     i=0
     while ! curl -o /dev/null -sf "http://$TIDB_IP:10080/status"; do
         i=$((i+1))
-        if [ "$i" -gt 20 ]; then
+        if [ "$i" -gt 50 ]; then
             echo 'Failed to start TiDB'
             exit 1
         fi
@@ -178,7 +178,7 @@ start_services_withTLS() {
             --key $1/certificates/client-key.pem \
             -o /dev/null -sf "https://$TIDB_IP:10080/status"; do
         i=$((i+1))
-        if [ "$i" -gt 20 ]; then
+        if [ "$i" -gt 50 ]; then
             echo 'Failed to start TiDB'
             exit 1
         fi
@@ -197,4 +197,4 @@ start_services_withTLS() {
         fi
         sleep 3
     done
-}
+}

tests/br_s3/run.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+#
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+DB="$TEST_NAME"
+TABLE="usertable"
+DB_COUNT=3
+
+# start the s3 server
+export MINIO_ACCESS_KEY=brs3accesskey
+export MINIO_SECRET_KEY=brs3secretkey
+export MINIO_BROWSER=off
+export AWS_ACCESS_KEY_ID=$MINIO_ACCESS_KEY
+export AWS_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
+export S3_ENDPOINT=127.0.0.1:24927
+rm -rf "$TEST_DIR/$DB"
+mkdir -p "$TEST_DIR/$DB"
+bin/minio server --address $S3_ENDPOINT "$TEST_DIR/$DB" &
+MINIO_PID=$!
+i=0
+while ! curl -o /dev/null -v -s "http://$S3_ENDPOINT/"; do
+    i=$(($i+1))
+    if [ $i -gt 7 ]; then
+        echo 'Failed to start minio'
+        exit 1
+    fi
+    sleep 2
+done
+
+stop_minio() {
+    kill -2 $MINIO_PID
+}
+trap stop_minio EXIT
+
+s3cmd --access_key=$MINIO_ACCESS_KEY --secret_key=$MINIO_SECRET_KEY --host=$S3_ENDPOINT --host-bucket=$S3_ENDPOINT --no-ssl mb s3://mybucket
+
+# Fill in the database
+for i in $(seq $DB_COUNT); do
+    run_sql "CREATE DATABASE $DB${i};"
+    go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
+done
+
+for i in $(seq $DB_COUNT); do
+    row_count_ori[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
+done
+
+# backup full
+echo "backup start..."
+run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB" --s3.endpoint="http://$S3_ENDPOINT"
+
+for i in $(seq $DB_COUNT); do
+    run_sql "DROP DATABASE $DB${i};"
+done
+
+# restore full
+echo "restore start..."
+run_br restore full -s "s3://mybucket/$DB" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT"
+
+for i in $(seq $DB_COUNT); do
+    row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
+done
+
+fail=false
+for i in $(seq $DB_COUNT); do
+    if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then
+        fail=true
+        echo "TEST: [$TEST_NAME] fail on database $DB${i}"
+    fi
+    echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}"
+done
+
+if $fail; then
+    echo "TEST: [$TEST_NAME] failed!"
+    exit 1
+else
+    echo "TEST: [$TEST_NAME] successed!"
+fi
+
+for i in $(seq $DB_COUNT); do
+    run_sql "DROP DATABASE $DB${i};"
+done

tests/br_s3/workload

@@ -0,0 +1,12 @@
+recordcount=5000
+operationcount=0
+workload=core
+
+readallfields=true
+
+readproportion=0
+updateproportion=0
+scanproportion=0
+insertproportion=0
+
+requestdistribution=uniform

tests/download_tools.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+#
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Download tools for running the integration test
+
+set -eu
+
+BIN="$(dirname "$0")/../bin"
+
+if [ "$(uname -s)" != Linux ]; then
+    echo 'Can only automatically download binaries on Linux.'
+    exit 1
+fi
+
+MISSING_TIDB_COMPONENTS=
+for COMPONENT in tidb-server pd-server tikv-server pd-ctl; do
+    if [ ! -e "$BIN/$COMPONENT" ]; then
+        MISSING_TIDB_COMPONENTS="$MISSING_TIDB_COMPONENTS tidb-latest-linux-amd64/bin/$COMPONENT"
+    fi
+done
+
+if [ -n "$MISSING_TIDB_COMPONENTS" ]; then
+    echo "Downloading latest TiDB bundle..."
+    # TODO: the url is going to change from 'latest' to 'nightly' someday.
+    curl -L -f -o "$BIN/tidb.tar.gz" "https://download.pingcap.org/tidb-latest-linux-amd64.tar.gz"
+    tar -x -f "$BIN/tidb.tar.gz" -C "$BIN/" $MISSING_TIDB_COMPONENTS
+    rm "$BIN/tidb.tar.gz"
+    mv "$BIN"/tidb-latest-linux-amd64/bin/* "$BIN/"
+    rmdir "$BIN/tidb-latest-linux-amd64/bin"
+    rmdir "$BIN/tidb-latest-linux-amd64"
+fi
+
+if [ ! -e "$BIN/go-ycsb" ]; then
+    # TODO: replace this once there's a public downloadable release.
+    echo 'go-ycsb is missing. Please build manually following https://github.com/pingcap/go-ycsb#getting-started'
+    exit 1
+fi
+
+if [ ! -e "$BIN/minio" ]; then
+    echo "Downloading minio..."
+    curl -L -f -o "$BIN/minio" "https://dl.min.io/server/minio/release/linux-amd64/minio"
+    chmod a+x "$BIN/minio"
+fi
+
+echo "All binaries are now available."

tests/run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Copyright 2019 PingCAP, Inc.
 #
@@ -28,7 +28,7 @@ if [ "${1-}" = '--debug' ]; then
     read line
 fi
 
-for script in tests/*/run.sh; do
+for script in tests/${TEST_NAME-*}/run.sh; do
     echo "*===== Running test $script... =====*"
     TEST_DIR="$TEST_DIR" \
     PD_ADDR="$PD_ADDR" \
@@ -39,5 +39,5 @@ for script in tests/*/run.sh; do
     TIKV_ADDR="$TIKV_ADDR" \
     PATH="tests/_utils:bin:$PATH" \
     TEST_NAME="$(basename "$(dirname "$script")")" \
-    sh "$script"
+    bash "$script"
 done