diff --git a/.bazelrc b/.bazelrc
index 8339fc80e2d67..61356f086fda5 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -1,4 +1,4 @@
-startup --host_jvm_args=-Xmx5g
+startup --host_jvm_args=-Xmx8g
startup --unlimit_coredumps
run:ci --color=yes
diff --git a/.bazelversion b/.bazelversion
index 84197c89467dd..09b254e90c61e 100644
--- a/.bazelversion
+++ b/.bazelversion
@@ -1 +1 @@
-5.3.2
+6.0.0
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000000000..bde24b721461e
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,14 @@
+version: 2
+updates:
+ - package-ecosystem: "gomod"
+ directory: "/"
+ schedule:
+ interval: "weekly"
+ day: "friday"
+ time: "18:00"
+ timezone: "Asia/Shanghai"
+ allow:
+ - dependency-name: "golang.org/*"
+ - dependency-name: "github.com/golangci/golangci-lint"
+ open-pull-requests-limit: 2
+
diff --git a/.github/licenserc.yml b/.github/licenserc.yml
index e1add7017983b..aec59e4f66d57 100644
--- a/.github/licenserc.yml
+++ b/.github/licenserc.yml
@@ -40,4 +40,5 @@ header:
- "tidb-binlog/proto/go-binlog/secondary_binlog.pb.go"
- "**/*.sql"
- ".bazelversion"
+ - "build/image/.ci_bazel"
comment: on-failure
diff --git a/.github/workflows/integration-test-br-compatibility.yml b/.github/workflows/integration-test-br-compatibility.yml
deleted file mode 100644
index 5df7168467005..0000000000000
--- a/.github/workflows/integration-test-br-compatibility.yml
+++ /dev/null
@@ -1,62 +0,0 @@
-name: BR / Compatibility Test
-
-on:
- push:
- # merged git action
- branches:
- - master
- - "release-[0-9].[0-9]*"
- paths:
- - "br/**"
- - "!**.html"
- - "!**.md"
- - "!CNAME"
- - "!LICENSE"
- - "!br/docs/**"
- - "!br/tests/**"
- - "!br/docker/**"
- # disable pull request only keep the merge action since it is very costly to run those tests
- # pull_request:
-
-concurrency:
- group: ${{ github.ref }}-${{ github.workflow }}
- cancel-in-progress: true
-
-permissions:
- contents: read # to fetch code (actions/checkout)
-
-jobs:
- check:
- runs-on: ubuntu-latest
- timeout-minutes: 25
- steps:
- - uses: actions/checkout@v2
-
- - name: Set up Go
- uses: actions/setup-go@v3
- with:
- go-version-file: 'go.mod'
-
- - name: Generate compatibility test backup data
- timeout-minutes: 15
- run: sh br/compatibility/prepare_backup.sh
-
- - name: Start server
- run: |
- TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml rm -s -v
- TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml build
- TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml up --remove-orphans -d
- TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml exec -T control go mod tidy
- TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml exec -T control make build_br
- TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml exec -T control br/tests/run_compatible.sh run
-
- - name: Collect component log
- if: ${{ failure() }}
- run: |
- tar czvf ${{ github.workspace }}/logs.tar.gz /tmp/br/docker/backup_logs/*
-
- - uses: actions/upload-artifact@v2
- if: ${{ failure() }}
- with:
- name: logs
- path: ${{ github.workspace }}/logs.tar.gz
diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml
deleted file mode 100644
index 94b68e9c95510..0000000000000
--- a/.github/workflows/misc.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-name: misc
-
-on:
- workflow_dispatch:
- pull_request:
- branches:
- - "master"
- - "main"
- - "release-**"
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read # to fetch code (actions/checkout)
-
-jobs:
- check:
- permissions:
- contents: read # to fetch code (actions/checkout)
- pull-requests: write # to comment on pull-requests
-
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - name: Check File Permission
- run: make check-file-perm
- - name: Check License Header
- uses: apache/skywalking-eyes/header@v0.4.0
- with:
- log: info
- token: ${{ secrets.GITHUB_TOKEN }}
- config: .github/licenserc.yml
diff --git a/.gitignore b/.gitignore
index e2cdcd078a0f1..35af372bcccad 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,4 @@ bazel-out
bazel-testlogs
bazel-tidb
.ijwb/
+/oom_record/
diff --git a/DEPS.bzl b/DEPS.bzl
index 4c3e94cca5a14..f46224ad85dab 100644
--- a/DEPS.bzl
+++ b/DEPS.bzl
@@ -5,8 +5,8 @@ def go_deps():
name = "cc_mvdan_gofumpt",
build_file_proto_mode = "disable",
importpath = "mvdan.cc/gofumpt",
- sum = "h1:avhhrOmv0IuvQVK7fvwV91oFSGAk5/6Po8GXTzICeu8=",
- version = "v0.3.1",
+ sum = "h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM=",
+ version = "v0.4.0",
)
go_repository(
name = "cc_mvdan_interfacer",
@@ -26,24 +26,40 @@ def go_deps():
name = "cc_mvdan_unparam",
build_file_proto_mode = "disable",
importpath = "mvdan.cc/unparam",
- sum = "h1:seuXWbRB1qPrS3NQnHmFKLJLtskWyueeIzmLXghMGgk=",
- version = "v0.0.0-20220706161116-678bad134442",
+ sum = "h1:3rvTIIM22r9pvXk+q3swxUQAQOxksVMGK7sml4nG57w=",
+ version = "v0.0.0-20221223090309-7455f1af531d",
)
go_repository(
name = "co_honnef_go_tools",
build_file_proto_mode = "disable_global",
importpath = "honnef.co/go/tools",
- sum = "h1:oDx7VAwstgpYpb3wv0oxiZlxY+foCpRAwY7Vk6XpAgA=",
- version = "v0.3.3",
+ sum = "h1:lyXVV1c8wUBJRKqI8JgIpT8TW1VDagfYYaxbKa/HoL8=",
+ version = "v0.4.0",
+ )
+ go_repository(
+ name = "com_4d63_gocheckcompilerdirectives",
+ build_file_proto_mode = "disable",
+ importpath = "4d63.com/gocheckcompilerdirectives",
+ sum = "h1:AHcMYuw56NPjq/2y615IGg2kYkBdTvOaojYCBcRE7MA=",
+ version = "v1.2.1",
)
+
go_repository(
name = "com_4d63_gochecknoglobals",
build_file_proto_mode = "disable",
importpath = "4d63.com/gochecknoglobals",
- sum = "h1:zeZSRqj5yCg28tCkIV/z/lWbwvNm5qnKVS15PI8nhD0=",
- version = "v0.1.0",
+ sum = "h1:1eiorGsgHOFOuoOiJDy2psSrQbRdIHrlge0IJIkUgDc=",
+ version = "v0.2.1",
+ )
+ go_repository(
+ name = "com_github_abirdcfly_dupword",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/Abirdcfly/dupword",
+ sum = "h1:MxprGjKq3yDBICXDgEEsyGirIXfMYXkLNT/agPsE1tk=",
+ version = "v0.0.9",
)
+
go_repository(
name = "com_github_acarl005_stripansi",
build_file_proto_mode = "disable",
@@ -80,6 +96,14 @@ def go_deps():
sum = "h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=",
version = "v0.0.0-20190924025748-f65c72e2690d",
)
+ go_repository(
+ name = "com_github_aleksi_gocov_xml",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/AlekSi/gocov-xml",
+ sum = "h1:4QctJBgXEkbzeKz6PJy6bt3JSPNSN4I2mITYW+eKUoQ=",
+ version = "v1.0.0",
+ )
+
go_repository(
name = "com_github_alexkohler_prealloc",
build_file_proto_mode = "disable",
@@ -131,6 +155,13 @@ def go_deps():
sum = "h1:PHhrh5ANKFWRBh7TdYmyyq2gyT2lotnvFvvFbylF81Q=",
version = "v0.1.1",
)
+ go_repository(
+ name = "com_github_apache_skywalking_eyes",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/apache/skywalking-eyes",
+ sum = "h1:O13kdRU6FCEZevfD01mdhTgCZLLfPZIQ0GXZrLl7FpQ=",
+ version = "v0.4.0",
+ )
go_repository(
name = "com_github_apache_thrift",
@@ -167,6 +198,14 @@ def go_deps():
sum = "h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to=",
version = "v0.0.0-20180808171621-7fddfc383310",
)
+ go_repository(
+ name = "com_github_armon_go_socks5",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/armon/go-socks5",
+ sum = "h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=",
+ version = "v0.0.0-20160902184237-e75332964ef5",
+ )
+
go_repository(
name = "com_github_ashanbrown_forbidigo",
build_file_proto_mode = "disable",
@@ -189,6 +228,14 @@ def go_deps():
sum = "h1:jLDC9RsNoYMLFlKpB8LdqUnoDdC2yvkS4QbuyPQJ8+M=",
version = "v1.44.48",
)
+ go_repository(
+ name = "com_github_axw_gocov",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/axw/gocov",
+ sum = "h1:YsqYR66hUmilVr23tu8USgnJIJvnwh3n7j5zRn7x4LU=",
+ version = "v1.0.0",
+ )
+
go_repository(
name = "com_github_aymerick_raymond",
build_file_proto_mode = "disable_global",
@@ -268,6 +315,13 @@ def go_deps():
sum = "h1:tYoz1OeRpx3dJZlh9T4dQt4kAndcmpl+VNdzbSgFC/0=",
version = "v0.0.0-20160505134755-913427a1d5e8",
)
+ go_repository(
+ name = "com_github_bitly_go_simplejson",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/bitly/go-simplejson",
+ sum = "h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y=",
+ version = "v0.5.0",
+ )
go_repository(
name = "com_github_bketelsen_crypt",
@@ -298,6 +352,14 @@ def go_deps():
sum = "h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M=",
version = "v0.8.0",
)
+ go_repository(
+ name = "com_github_bmatcuk_doublestar_v2",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/bmatcuk/doublestar/v2",
+ sum = "h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI=",
+ version = "v2.0.4",
+ )
+
go_repository(
name = "com_github_bombsimon_wsl_v3",
build_file_proto_mode = "disable",
@@ -305,6 +367,14 @@ def go_deps():
sum = "h1:Mka/+kRLoQJq7g2rggtgQsjuI/K5Efd87WX96EWFxjM=",
version = "v3.3.0",
)
+ go_repository(
+ name = "com_github_breeswish_gin_jwt_v2",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/breeswish/gin-jwt/v2",
+ sum = "h1:KLE/YeX+9FNaGVW5MtImRVPhjDpfpgJhvkuYWBmOYbo=",
+ version = "v2.6.4-jwt-patch",
+ )
+
go_repository(
name = "com_github_breml_bidichk",
build_file_proto_mode = "disable",
@@ -324,8 +394,8 @@ def go_deps():
name = "com_github_burntsushi_toml",
build_file_proto_mode = "disable_global",
importpath = "github.com/BurntSushi/toml",
- sum = "h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=",
- version = "v1.2.0",
+ sum = "h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=",
+ version = "v1.2.1",
)
go_repository(
name = "com_github_burntsushi_xgb",
@@ -341,6 +411,13 @@ def go_deps():
sum = "h1:QvrO2QF2+/Cx1WA/vETCIYBKtRjc30vesdoPUNo1EbY=",
version = "v0.1.1",
)
+ go_repository(
+ name = "com_github_cakturk_go_netstat",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/cakturk/go-netstat",
+ sum = "h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g=",
+ version = "v0.0.0-20200220111822-e5b49efee7a5",
+ )
go_repository(
name = "com_github_carlmjohnson_flagext",
@@ -356,6 +433,13 @@ def go_deps():
sum = "h1:7vXVw3g7XE+Vnj0A9TmFGtMeP4oZQ5ZzpPvKhLFa80E=",
version = "v2.0.0+incompatible",
)
+ go_repository(
+ name = "com_github_cenkalti_backoff_v4",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/cenkalti/backoff/v4",
+ sum = "h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs=",
+ version = "v4.0.2",
+ )
go_repository(
name = "com_github_census_instrumentation_opencensus_proto",
@@ -382,8 +466,8 @@ def go_deps():
name = "com_github_cespare_xxhash_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/cespare/xxhash/v2",
- sum = "h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=",
- version = "v2.1.2",
+ sum = "h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=",
+ version = "v2.2.0",
)
go_repository(
name = "com_github_charithe_durationcheck",
@@ -396,8 +480,8 @@ def go_deps():
name = "com_github_chavacava_garif",
build_file_proto_mode = "disable",
importpath = "github.com/chavacava/garif",
- sum = "h1:E7LT642ysztPWE0dfz43cWOvMiF42DyTRC+eZIaO4yI=",
- version = "v0.0.0-20220630083739-93517212f375",
+ sum = "h1:cy5GCEZLUCshCGCRRUjxHrDUqkB4l5cuUt3ShEckQEo=",
+ version = "v0.0.0-20221024190013-b3ef35877348",
)
go_repository(
@@ -442,6 +526,14 @@ def go_deps():
sum = "h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=",
version = "v0.3.4",
)
+ go_repository(
+ name = "com_github_cloudfoundry_gosigar",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/cloudfoundry/gosigar",
+ sum = "h1:gIc08FbB3QPb+nAQhINIK/qhf5REKkY0FTGgRGXkcVc=",
+ version = "v1.3.6",
+ )
+
go_repository(
name = "com_github_cloudykit_fastprinter",
build_file_proto_mode = "disable_global",
@@ -657,8 +749,8 @@ def go_deps():
name = "com_github_curioswitch_go_reassign",
build_file_proto_mode = "disable",
importpath = "github.com/curioswitch/go-reassign",
- sum = "h1:ekM07+z+VFT560Exz4mTv0/s1yU9gem6CJc/tlYpkmI=",
- version = "v0.1.2",
+ sum = "h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo=",
+ version = "v0.2.0",
)
go_repository(
@@ -686,8 +778,8 @@ def go_deps():
name = "com_github_daixiang0_gci",
build_file_proto_mode = "disable",
importpath = "github.com/daixiang0/gci",
- sum = "h1:wUAqXChk8HbwXn8AfxD9DYSCp9Bpz1L3e6Q4Roe+q9E=",
- version = "v0.6.3",
+ sum = "h1:t8XZ0vK6l0pwPoOmoGyqW2NwQlvbpAQNVvu/GRBgykM=",
+ version = "v0.9.0",
)
go_repository(
@@ -752,8 +844,8 @@ def go_deps():
name = "com_github_dgraph_io_ristretto",
build_file_proto_mode = "disable_global",
importpath = "github.com/dgraph-io/ristretto",
- sum = "h1:Wrc3UKTS+cffkOx0xRGFC+ZesNuTfn0ThvEC72N0krk=",
- version = "v0.1.1-0.20220403145359-8e850b710d6d",
+ sum = "h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=",
+ version = "v0.1.1",
)
go_repository(
name = "com_github_dgrijalva_jwt_go",
@@ -792,6 +884,14 @@ def go_deps():
sum = "h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=",
version = "v1.2.0",
)
+ go_repository(
+ name = "com_github_dnephin_pflag",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/dnephin/pflag",
+ sum = "h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk=",
+ version = "v1.0.7",
+ )
+
go_repository(
name = "com_github_docker_go_units",
build_file_proto_mode = "disable_global",
@@ -848,6 +948,21 @@ def go_deps():
sum = "h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk=",
version = "v1.0.0",
)
+ go_repository(
+ name = "com_github_elazarl_goproxy",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/elazarl/goproxy",
+ sum = "h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=",
+ version = "v0.0.0-20180725130230-947c36da3153",
+ )
+ go_repository(
+ name = "com_github_elliotchance_pie_v2",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/elliotchance/pie/v2",
+ sum = "h1:KEVAAzxYxTyFs4hvebFZVzBdEo3YeMzl2HYDWn+P3F4=",
+ version = "v2.1.0",
+ )
+
go_repository(
name = "com_github_emirpasic_gods",
build_file_proto_mode = "disable",
@@ -860,8 +975,8 @@ def go_deps():
name = "com_github_envoyproxy_go_control_plane",
build_file_proto_mode = "disable_global",
importpath = "github.com/envoyproxy/go-control-plane",
- sum = "h1:fP+fF0up6oPY49OrjPrhIJ8yQfdIM85NXMLkMg1EXVs=",
- version = "v0.9.10-0.20210907150352-cf90f659a021",
+ sum = "h1:xvqufLtNVwAhN8NMyWklVgxnWohi+wtMGQMhtxexlm0=",
+ version = "v0.10.2-0.20220325020618-49ff273808a1",
)
go_repository(
name = "com_github_envoyproxy_protoc_gen_validate",
@@ -933,8 +1048,8 @@ def go_deps():
name = "com_github_fatih_color",
build_file_proto_mode = "disable_global",
importpath = "github.com/fatih/color",
- sum = "h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=",
- version = "v1.13.0",
+ sum = "h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=",
+ version = "v1.14.1",
)
go_repository(
name = "com_github_fatih_structs",
@@ -998,15 +1113,15 @@ def go_deps():
name = "com_github_frankban_quicktest",
build_file_proto_mode = "disable_global",
importpath = "github.com/frankban/quicktest",
- sum = "h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=",
- version = "v1.11.3",
+ sum = "h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=",
+ version = "v1.14.3",
)
go_repository(
name = "com_github_fsnotify_fsnotify",
build_file_proto_mode = "disable_global",
importpath = "github.com/fsnotify/fsnotify",
- sum = "h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=",
- version = "v1.5.4",
+ sum = "h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=",
+ version = "v1.6.0",
)
go_repository(
name = "com_github_fsouza_fake_gcs_server",
@@ -1058,6 +1173,21 @@ def go_deps():
sum = "h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=",
version = "v1.0.0",
)
+ go_repository(
+ name = "com_github_gin_contrib_cors",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/gin-contrib/cors",
+ sum = "h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_github_gin_contrib_gzip",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/gin-contrib/gzip",
+ sum = "h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc=",
+ version = "v0.0.1",
+ )
+
go_repository(
name = "com_github_gin_contrib_sse",
build_file_proto_mode = "disable_global",
@@ -1083,8 +1213,15 @@ def go_deps():
name = "com_github_go_critic_go_critic",
build_file_proto_mode = "disable",
importpath = "github.com/go-critic/go-critic",
- sum = "h1:tucuG1pvOyYgpBIrVxw0R6gwO42lNa92Aq3VaDoIs+E=",
- version = "v0.6.4",
+ sum = "h1:fDaR/5GWURljXwF8Eh31T2GZNz9X4jeboS912mWF8Uo=",
+ version = "v0.6.5",
+ )
+ go_repository(
+ name = "com_github_go_echarts_go_echarts",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/go-echarts/go-echarts",
+ sum = "h1:n181E4iXwj4zrU9VYmdM2m8dyhERt2w9k9YhHqdp6A8=",
+ version = "v1.0.0",
)
go_repository(
@@ -1127,8 +1264,8 @@ def go_deps():
name = "com_github_go_kit_log",
build_file_proto_mode = "disable_global",
importpath = "github.com/go-kit/log",
- sum = "h1:7i2K3eKTos3Vc0enKCfnVcgHh2olr/MyfboYq7cAcFw=",
- version = "v0.2.0",
+ sum = "h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=",
+ version = "v0.2.1",
)
go_repository(
name = "com_github_go_logfmt_logfmt",
@@ -1137,6 +1274,14 @@ def go_deps():
sum = "h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=",
version = "v0.5.1",
)
+ go_repository(
+ name = "com_github_go_logr_logr",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/go-logr/logr",
+ sum = "h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=",
+ version = "v1.2.3",
+ )
+
go_repository(
name = "com_github_go_martini_martini",
build_file_proto_mode = "disable_global",
@@ -1151,12 +1296,69 @@ def go_deps():
sum = "h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=",
version = "v1.2.6",
)
+ go_repository(
+ name = "com_github_go_openapi_jsonpointer",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/go-openapi/jsonpointer",
+ sum = "h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=",
+ version = "v0.19.5",
+ )
+ go_repository(
+ name = "com_github_go_openapi_jsonreference",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/go-openapi/jsonreference",
+ sum = "h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=",
+ version = "v0.19.6",
+ )
+ go_repository(
+ name = "com_github_go_openapi_spec",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/go-openapi/spec",
+ sum = "h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=",
+ version = "v0.20.4",
+ )
+ go_repository(
+ name = "com_github_go_openapi_swag",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/go-openapi/swag",
+ sum = "h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=",
+ version = "v0.19.15",
+ )
+ go_repository(
+ name = "com_github_go_playground_locales",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/go-playground/locales",
+ sum = "h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=",
+ version = "v0.14.0",
+ )
+ go_repository(
+ name = "com_github_go_playground_universal_translator",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/go-playground/universal-translator",
+ sum = "h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=",
+ version = "v0.18.0",
+ )
+ go_repository(
+ name = "com_github_go_playground_validator_v10",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/go-playground/validator/v10",
+ sum = "h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0=",
+ version = "v10.10.0",
+ )
+ go_repository(
+ name = "com_github_go_resty_resty_v2",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/go-resty/resty/v2",
+ sum = "h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4=",
+ version = "v2.6.0",
+ )
+
go_repository(
name = "com_github_go_sql_driver_mysql",
build_file_proto_mode = "disable_global",
importpath = "github.com/go-sql-driver/mysql",
- sum = "h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=",
- version = "v1.6.0",
+ sum = "h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=",
+ version = "v1.7.0",
)
go_repository(
name = "com_github_go_stack_stack",
@@ -1183,15 +1385,15 @@ def go_deps():
name = "com_github_go_toolsmith_astcopy",
build_file_proto_mode = "disable",
importpath = "github.com/go-toolsmith/astcopy",
- sum = "h1:l09oBhAPyV74kLJ3ZO31iBU8htZGTwr9LTjuMCyL8go=",
- version = "v1.0.1",
+ sum = "h1:YnWf5Rnh1hUudj11kei53kI57quN/VH6Hp1n+erozn0=",
+ version = "v1.0.2",
)
go_repository(
name = "com_github_go_toolsmith_astequal",
build_file_proto_mode = "disable",
importpath = "github.com/go-toolsmith/astequal",
- sum = "h1:+XvaV8zNxua+9+Oa4AHmgmpo4RYAbwr/qjNppLfX2yM=",
- version = "v1.0.2",
+ sum = "h1:+LVdyRatFS+XO78SGV4I3TCEA0AC7fKEGma+fH+674o=",
+ version = "v1.0.3",
)
go_repository(
name = "com_github_go_toolsmith_astfmt",
@@ -1225,8 +1427,8 @@ def go_deps():
name = "com_github_go_xmlfmt_xmlfmt",
build_file_proto_mode = "disable",
importpath = "github.com/go-xmlfmt/xmlfmt",
- sum = "h1:khEcpUM4yFcxg4/FHQWkvVRmgijNXRfzkIDHh23ggEo=",
- version = "v0.0.0-20191208150333-d5b6f63a941b",
+ sum = "h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80U=",
+ version = "v1.1.2",
)
go_repository(
name = "com_github_gobwas_glob",
@@ -1257,6 +1459,14 @@ def go_deps():
sum = "h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=",
version = "v1.0.2",
)
+ go_repository(
+ name = "com_github_goccy_go_graphviz",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/goccy/go-graphviz",
+ sum = "h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ=",
+ version = "v0.0.9",
+ )
+
go_repository(
name = "com_github_goccy_go_json",
build_file_proto_mode = "disable",
@@ -1330,6 +1540,14 @@ def go_deps():
sum = "h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=",
version = "v0.0.0-20210331224755-41bb18bfe9da",
)
+ go_repository(
+ name = "com_github_golang_jwt_jwt",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/golang-jwt/jwt",
+ sum = "h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=",
+ version = "v3.2.1+incompatible",
+ )
+
go_repository(
name = "com_github_golang_mock",
build_file_proto_mode = "disable_global",
@@ -1381,15 +1599,15 @@ def go_deps():
name = "com_github_golangci_gofmt",
build_file_proto_mode = "disable",
importpath = "github.com/golangci/gofmt",
- sum = "h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=",
- version = "v0.0.0-20190930125516-244bba706f1a",
+ sum = "h1:amWTbTGqOZ71ruzrdA+Nx5WA3tV1N0goTspwmKCQvBY=",
+ version = "v0.0.0-20220901101216-f2edd75033f2",
)
go_repository(
name = "com_github_golangci_golangci_lint",
build_file_proto_mode = "disable",
importpath = "github.com/golangci/golangci-lint",
- sum = "h1:I8WHOavragDttlLHtSraHn/h39C+R60bEQ5NoGcHQr8=",
- version = "v1.49.0",
+ sum = "h1:M1bpDymgdaPKNzPwQdebCGki/nzvVkr2f/eUfk9C9oU=",
+ version = "v1.51.0",
)
go_repository(
name = "com_github_golangci_gosec",
@@ -1417,8 +1635,8 @@ def go_deps():
name = "com_github_golangci_misspell",
build_file_proto_mode = "disable",
importpath = "github.com/golangci/misspell",
- sum = "h1:pLzmVdl3VxTOncgzHcvLOKirdvcx/TydsClUQXTehjo=",
- version = "v0.3.5",
+ sum = "h1:KtVB/hTK4bbL/S6bs64rYyk8adjmh1BygbBiaAiX+a0=",
+ version = "v0.4.0",
)
go_repository(
@@ -1457,19 +1675,35 @@ def go_deps():
sum = "h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=",
version = "v1.1.2",
)
+ go_repository(
+ name = "com_github_google_gnostic",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/google/gnostic",
+ sum = "h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=",
+ version = "v0.5.7-v3refs",
+ )
+
go_repository(
name = "com_github_google_go_cmp",
build_file_proto_mode = "disable_global",
importpath = "github.com/google/go-cmp",
- sum = "h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=",
- version = "v0.5.8",
+ sum = "h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=",
+ version = "v0.5.9",
+ )
+ go_repository(
+ name = "com_github_google_go_github_v33",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/google/go-github/v33",
+ sum = "h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM=",
+ version = "v33.0.0",
)
+
go_repository(
name = "com_github_google_go_querystring",
build_file_proto_mode = "disable_global",
importpath = "github.com/google/go-querystring",
- sum = "h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=",
- version = "v1.0.0",
+ sum = "h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=",
+ version = "v1.1.0",
)
go_repository(
name = "com_github_google_gofuzz",
@@ -1478,6 +1712,14 @@ def go_deps():
sum = "h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=",
version = "v1.0.0",
)
+ go_repository(
+ name = "com_github_google_licensecheck",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/google/licensecheck",
+ sum = "h1:QoxgoDkaeC4nFrtGN1jV7IPmDCHFNIVh54e5hSt6sPs=",
+ version = "v0.3.1",
+ )
+
go_repository(
name = "com_github_google_martian",
build_file_proto_mode = "disable_global",
@@ -1506,6 +1748,14 @@ def go_deps():
sum = "h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=",
version = "v0.1.0",
)
+ go_repository(
+ name = "com_github_google_shlex",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/google/shlex",
+ sum = "h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=",
+ version = "v0.0.0-20191202100458-e7afc7fbc510",
+ )
+
go_repository(
name = "com_github_google_uuid",
build_file_proto_mode = "disable_global",
@@ -1513,12 +1763,20 @@ def go_deps():
sum = "h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=",
version = "v1.3.0",
)
+ go_repository(
+ name = "com_github_googleapis_enterprise_certificate_proxy",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/googleapis/enterprise-certificate-proxy",
+ sum = "h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs=",
+ version = "v0.2.0",
+ )
+
go_repository(
name = "com_github_googleapis_gax_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/googleapis/gax-go/v2",
- sum = "h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE=",
- version = "v2.2.0",
+ sum = "h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=",
+ version = "v2.7.0",
)
go_repository(
name = "com_github_googleapis_gnostic",
@@ -1527,6 +1785,14 @@ def go_deps():
sum = "h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=",
version = "v0.2.0",
)
+ go_repository(
+ name = "com_github_googleapis_go_type_adapters",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/googleapis/go-type-adapters",
+ sum = "h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA=",
+ version = "v1.0.0",
+ )
+
go_repository(
name = "com_github_gophercloud_gophercloud",
build_file_proto_mode = "disable",
@@ -1661,6 +1927,13 @@ def go_deps():
sum = "h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=",
version = "v0.0.0-20180507213350-8e809c8a8645",
)
+ go_repository(
+ name = "com_github_gtank_cryptopasta",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/gtank/cryptopasta",
+ sum = "h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0=",
+ version = "v0.0.0-20170601214702-1f550f6f2f69",
+ )
go_repository(
name = "com_github_hashicorp_consul_api",
@@ -1817,6 +2090,14 @@ def go_deps():
sum = "h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=",
version = "v1.0.0",
)
+ go_repository(
+ name = "com_github_huandu_xstrings",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/huandu/xstrings",
+ sum = "h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs=",
+ version = "v1.3.1",
+ )
+
go_repository(
name = "com_github_hydrogen18_memlistener",
build_file_proto_mode = "disable_global",
@@ -1838,6 +2119,14 @@ def go_deps():
sum = "h1:uGg2frlt3IcT7kbV6LEp5ONv4vmoO2FW4qSO+my/aoM=",
version = "v0.0.0-20210905161508-09a460cdf81d",
)
+ go_repository(
+ name = "com_github_imdario_mergo",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/imdario/mergo",
+ sum = "h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=",
+ version = "v0.3.11",
+ )
+
go_repository(
name = "com_github_imkira_go_interpol",
build_file_proto_mode = "disable_global",
@@ -1849,8 +2138,8 @@ def go_deps():
name = "com_github_inconshreveable_mousetrap",
build_file_proto_mode = "disable_global",
importpath = "github.com/inconshreveable/mousetrap",
- sum = "h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=",
- version = "v1.0.0",
+ sum = "h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=",
+ version = "v1.0.1",
)
go_repository(
name = "com_github_influxdata_influxdb",
@@ -1980,6 +2269,21 @@ def go_deps():
sum = "h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs=",
version = "v1.1.1",
)
+ go_repository(
+ name = "com_github_jinzhu_inflection",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/jinzhu/inflection",
+ sum = "h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=",
+ version = "v1.0.0",
+ )
+ go_repository(
+ name = "com_github_jinzhu_now",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/jinzhu/now",
+ sum = "h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=",
+ version = "v1.1.2",
+ )
+
go_repository(
name = "com_github_jirfag_go_printf_func_name",
build_file_proto_mode = "disable",
@@ -2002,6 +2306,14 @@ def go_deps():
sum = "h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=",
version = "v1.5.1",
)
+ go_repository(
+ name = "com_github_joho_godotenv",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/joho/godotenv",
+ sum = "h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=",
+ version = "v1.4.0",
+ )
+
go_repository(
name = "com_github_joho_sqltocsv",
build_file_proto_mode = "disable_global",
@@ -2030,6 +2342,21 @@ def go_deps():
sum = "h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=",
version = "v0.2.2",
)
+ go_repository(
+ name = "com_github_joomcode_errorx",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/joomcode/errorx",
+ sum = "h1:CalpDWz14ZHd68fIqluJasJosAewpz2TFaJALrUxjrk=",
+ version = "v1.0.1",
+ )
+ go_repository(
+ name = "com_github_josharian_intern",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/josharian/intern",
+ sum = "h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=",
+ version = "v1.0.0",
+ )
+
go_repository(
name = "com_github_jpillora_backoff",
build_file_proto_mode = "disable_global",
@@ -2101,6 +2428,14 @@ def go_deps():
sum = "h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0=",
version = "v1.0.3-0.20190309125859-24315acbbda5",
)
+ go_repository(
+ name = "com_github_junk1tm_musttag",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/junk1tm/musttag",
+ sum = "h1:I8UHQkDj2u/MClcGU8PbMoYwhykiSQFEbXKKMjixPyk=",
+ version = "v0.4.3",
+ )
+
go_repository(
name = "com_github_k0kubun_colorstring",
build_file_proto_mode = "disable_global",
@@ -2144,8 +2479,8 @@ def go_deps():
patches = [
"//build/patches:com_github_kisielk_errcheck.patch",
],
- sum = "h1:uGQ9xI8/pgc9iOoCe7kWQgRE6SBTrCGmTSf0LrEtY7c=",
- version = "v1.6.2",
+ sum = "h1:dEKh+GLHcWm2oN34nMvDzn1sqI0i0WxPvrgiJA5JuM8=",
+ version = "v1.6.3",
)
go_repository(
name = "com_github_kisielk_gotool",
@@ -2154,12 +2489,20 @@ def go_deps():
sum = "h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=",
version = "v1.0.0",
)
+ go_repository(
+ name = "com_github_kkhaike_contextcheck",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/kkHAIKE/contextcheck",
+ sum = "h1:l4pNvrb8JSwRd51ojtcOxOeHJzHek+MtOyXbaR0uvmw=",
+ version = "v1.1.3",
+ )
+
go_repository(
name = "com_github_klauspost_compress",
build_file_proto_mode = "disable_global",
importpath = "github.com/klauspost/compress",
- sum = "h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=",
- version = "v1.15.1",
+ sum = "h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0=",
+ version = "v1.15.13",
)
go_repository(
name = "com_github_klauspost_cpuid",
@@ -2225,13 +2568,20 @@ def go_deps():
sum = "h1:FCKYMF1OF2+RveWlABsdnmsvJrei5aoyZoaGS+Ugg8g=",
version = "v1.0.6",
)
+ go_repository(
+ name = "com_github_kylebanks_depth",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/KyleBanks/depth",
+ sum = "h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=",
+ version = "v1.2.1",
+ )
go_repository(
name = "com_github_kyoh86_exportloopref",
build_file_proto_mode = "disable",
importpath = "github.com/kyoh86/exportloopref",
- sum = "h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77LoN/M=",
- version = "v0.1.8",
+ sum = "h1:1Z0bcmTypkL3Q4k+IDHMWTcnCliEZcaPiIe0/ymEyhQ=",
+ version = "v0.1.11",
)
go_repository(
@@ -2259,15 +2609,23 @@ def go_deps():
name = "com_github_ldez_tagliatelle",
build_file_proto_mode = "disable",
importpath = "github.com/ldez/tagliatelle",
- sum = "h1:3BqVVlReVUZwafJUwQ+oxbx2BEX2vUG4Yu/NOfMiKiM=",
- version = "v0.3.1",
+ sum = "h1:sylp7d9kh6AdXN2DpVGHBRb5guTVAgOxqNGhbqc4b1c=",
+ version = "v0.4.0",
+ )
+ go_repository(
+ name = "com_github_leodido_go_urn",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/leodido/go-urn",
+ sum = "h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=",
+ version = "v1.2.1",
)
+
go_repository(
name = "com_github_leonklingele_grouper",
build_file_proto_mode = "disable",
importpath = "github.com/leonklingele/grouper",
- sum = "h1:tC2y/ygPbMFSBOs3DcyaEMKnnwH7eYKzohOtRrf0SAg=",
- version = "v1.1.0",
+ sum = "h1:suWXRU57D4/Enn6pXR0QVqqWWrnJ9Osrz+5rjt8ivzU=",
+ version = "v1.1.1",
)
go_repository(
name = "com_github_lestrrat_go_blackmagic",
@@ -2349,6 +2707,22 @@ def go_deps():
sum = "h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=",
version = "v1.8.6",
)
+ go_repository(
+ name = "com_github_mailru_easyjson",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/mailru/easyjson",
+ sum = "h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=",
+ version = "v0.7.6",
+ )
+
+ go_repository(
+ name = "com_github_maratori_testableexamples",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/maratori/testableexamples",
+ sum = "h1:dU5alXRrD8WKSjOUnmJZuzdxWOEQ57+7s93SLMxb2vI=",
+ version = "v1.0.0",
+ )
+
go_repository(
name = "com_github_maratori_testpackage",
build_file_proto_mode = "disable",
@@ -2356,6 +2730,14 @@ def go_deps():
sum = "h1:GJY4wlzQhuBusMF1oahQCBtUV/AQ/k69IZ68vxaac2Q=",
version = "v1.1.0",
)
+ go_repository(
+ name = "com_github_masterminds_goutils",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/Masterminds/goutils",
+ sum = "h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=",
+ version = "v1.1.1",
+ )
+
go_repository(
name = "com_github_masterminds_semver",
build_file_proto_mode = "disable",
@@ -2363,6 +2745,21 @@ def go_deps():
sum = "h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=",
version = "v1.5.0",
)
+ go_repository(
+ name = "com_github_masterminds_semver_v3",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/Masterminds/semver/v3",
+ sum = "h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=",
+ version = "v3.1.1",
+ )
+ go_repository(
+ name = "com_github_masterminds_sprig_v3",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/Masterminds/sprig/v3",
+ sum = "h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=",
+ version = "v3.2.2",
+ )
+
go_repository(
name = "com_github_matoous_godox",
build_file_proto_mode = "disable",
@@ -2382,8 +2779,8 @@ def go_deps():
name = "com_github_mattn_go_isatty",
build_file_proto_mode = "disable_global",
importpath = "github.com/mattn/go-isatty",
- sum = "h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=",
- version = "v0.0.16",
+ sum = "h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=",
+ version = "v0.0.17",
)
go_repository(
name = "com_github_mattn_go_runewidth",
@@ -2392,6 +2789,21 @@ def go_deps():
sum = "h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=",
version = "v0.0.14",
)
+ go_repository(
+ name = "com_github_mattn_go_shellwords",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/mattn/go-shellwords",
+ sum = "h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=",
+ version = "v1.0.12",
+ )
+ go_repository(
+ name = "com_github_mattn_go_sqlite3",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/mattn/go-sqlite3",
+ sum = "h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=",
+ version = "v1.14.9",
+ )
+
go_repository(
name = "com_github_mattn_goveralls",
build_file_proto_mode = "disable_global",
@@ -2403,8 +2815,8 @@ def go_deps():
name = "com_github_matttproud_golang_protobuf_extensions",
build_file_proto_mode = "disable_global",
importpath = "github.com/matttproud/golang_protobuf_extensions",
- sum = "h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=",
- version = "v1.0.1",
+ sum = "h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=",
+ version = "v1.0.4",
)
go_repository(
name = "com_github_maxatome_go_testdeep",
@@ -2448,8 +2860,8 @@ def go_deps():
name = "com_github_mgechev_revive",
build_file_proto_mode = "disable",
importpath = "github.com/mgechev/revive",
- sum = "h1:+2Hd/S8oO2H0Ikq2+egtNwQsVhAeELHjxjIUFX5ajLI=",
- version = "v1.2.4",
+ sum = "h1:UF9AR8pOAuwNmhXj2odp4mxv9Nx2qUIwVz8ZsU+Mbec=",
+ version = "v1.2.5",
)
go_repository(
@@ -2466,6 +2878,14 @@ def go_deps():
sum = "h1:oN9gL93BkuPrer2rehDbDx86k4zbYJEnMP6Krh82nh0=",
version = "v1.1.10",
)
+ go_repository(
+ name = "com_github_minio_sio",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/minio/sio",
+ sum = "h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus=",
+ version = "v0.3.0",
+ )
+
go_repository(
name = "com_github_mitchellh_cli",
build_file_proto_mode = "disable_global",
@@ -2473,6 +2893,14 @@ def go_deps():
sum = "h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y=",
version = "v1.0.0",
)
+ go_repository(
+ name = "com_github_mitchellh_copystructure",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/mitchellh/copystructure",
+ sum = "h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=",
+ version = "v1.0.0",
+ )
+
go_repository(
name = "com_github_mitchellh_go_homedir",
build_file_proto_mode = "disable_global",
@@ -2531,6 +2959,13 @@ def go_deps():
sum = "h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE=",
version = "v1.0.1",
)
+ go_repository(
+ name = "com_github_moby_spdystream",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/moby/spdystream",
+ sum = "h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=",
+ version = "v0.2.0",
+ )
go_repository(
name = "com_github_modern_go_concurrent",
@@ -2583,6 +3018,14 @@ def go_deps():
sum = "h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=",
version = "v0.0.0-20190716064945-2f068394615f",
)
+ go_repository(
+ name = "com_github_mxk_go_flowrate",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/mxk/go-flowrate",
+ sum = "h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=",
+ version = "v0.0.0-20140419014527-cca7078d478f",
+ )
+
go_repository(
name = "com_github_nakabonne_nestif",
build_file_proto_mode = "disable",
@@ -2652,8 +3095,8 @@ def go_deps():
name = "com_github_nishanths_exhaustive",
build_file_proto_mode = "disable",
importpath = "github.com/nishanths/exhaustive",
- sum = "h1:0QKNascWv9qIHY7zRoZSxeRr6kuk5aAT3YXLTiDmjTo=",
- version = "v0.8.1",
+ sum = "h1:TzssWan6orBiLYVqewCG8faud9qlFntJE30ACpzmGME=",
+ version = "v0.9.5",
)
go_repository(
@@ -2663,13 +3106,20 @@ def go_deps():
sum = "h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk=",
version = "v0.2.2",
)
+ go_repository(
+ name = "com_github_nunnatsa_ginkgolinter",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/nunnatsa/ginkgolinter",
+ sum = "h1:j4mzqx1hkE75mXHs3AUlWghZBi59c3GDWXp6zzcI+kE=",
+ version = "v0.7.1",
+ )
go_repository(
name = "com_github_nxadm_tail",
build_file_proto_mode = "disable_global",
importpath = "github.com/nxadm/tail",
- sum = "h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=",
- version = "v1.4.4",
+ sum = "h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=",
+ version = "v1.4.8",
)
go_repository(
name = "com_github_oklog_run",
@@ -2686,6 +3136,14 @@ def go_deps():
sum = "h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=",
version = "v1.3.1",
)
+ go_repository(
+ name = "com_github_oleiade_reflections",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/oleiade/reflections",
+ sum = "h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM=",
+ version = "v1.0.1",
+ )
+
go_repository(
name = "com_github_olekukonko_tablewriter",
build_file_proto_mode = "disable_global",
@@ -2711,22 +3169,22 @@ def go_deps():
name = "com_github_onsi_ginkgo_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/onsi/ginkgo/v2",
- sum = "h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ=",
- version = "v2.0.0",
+ sum = "h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=",
+ version = "v2.4.0",
)
go_repository(
name = "com_github_onsi_gomega",
build_file_proto_mode = "disable_global",
importpath = "github.com/onsi/gomega",
- sum = "h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=",
- version = "v1.18.1",
+ sum = "h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=",
+ version = "v1.23.0",
)
go_repository(
name = "com_github_openpeedeep_depguard",
build_file_proto_mode = "disable",
importpath = "github.com/OpenPeeDeeP/depguard",
- sum = "h1:pjK9nLPS1FwQYGGpPxoMYpe7qACHOhAWQMQzV71i49o=",
- version = "v1.1.0",
+ sum = "h1:TSUznLjvp/4IUP+OQ0t/4jF4QUyxIcVX8YnghZdunyA=",
+ version = "v1.1.1",
)
go_repository(
@@ -2805,8 +3263,8 @@ def go_deps():
name = "com_github_pelletier_go_toml_v2",
build_file_proto_mode = "disable",
importpath = "github.com/pelletier/go-toml/v2",
- sum = "h1:+jQXlF3scKIcSEKkdHzXhCTDLPFi5r1wnK6yPS+49Gw=",
- version = "v2.0.2",
+ sum = "h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg=",
+ version = "v2.0.5",
)
go_repository(
name = "com_github_peterbourgon_g2s",
@@ -2819,8 +3277,8 @@ def go_deps():
name = "com_github_petermattis_goid",
build_file_proto_mode = "disable",
importpath = "github.com/petermattis/goid",
- sum = "h1:rUMC+oZ89Om6l9wvUNjzI0ZrKrSnXzV+opsgAohYUNc=",
- version = "v0.0.0-20170504144140-0ded85884ba5",
+ sum = "h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo=",
+ version = "v0.0.0-20211229010228-4d14c490ee36",
)
go_repository(
@@ -2838,6 +3296,14 @@ def go_deps():
sum = "h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=",
version = "v0.0.0-20180830031419-95f893ade6f2",
)
+ go_repository(
+ name = "com_github_phf_go_queue",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/phf/go-queue",
+ sum = "h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g=",
+ version = "v0.0.0-20170504031614-9abe38d0371d",
+ )
+
go_repository(
name = "com_github_pierrec_lz4",
build_file_proto_mode = "disable_global",
@@ -2849,29 +3315,37 @@ def go_deps():
name = "com_github_pingcap_badger",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/badger",
- sum = "h1:MKVFZuqFvAMiDtv3AbihOQ6rY5IE8LWflI1BuZ/hF0Y=",
- version = "v1.5.1-0.20220314162537-ab58fbf40580",
+ sum = "h1:AEcvKyVM8CUII3bYzgz8haFXtGiqcrtXW1csu/5UELY=",
+ version = "v1.5.1-0.20230103063557-828f39b09b6d",
)
go_repository(
name = "com_github_pingcap_check",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/check",
- sum = "h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0=",
- version = "v0.0.0-20191107115940-caf2b9e6ccf4",
+ sum = "h1:R8gStypOBmpnHEx1qi//SaqxJVI4inOqljg/Aj5/390=",
+ version = "v0.0.0-20200212061837-5e12011dc712",
+ )
+ go_repository(
+ name = "com_github_pingcap_errcode",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/pingcap/errcode",
+ sum = "h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg=",
+ version = "v0.3.0",
)
+
go_repository(
name = "com_github_pingcap_errors",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/errors",
- sum = "h1:3Dm0DWeQlwV8LbpQxP2tojHhxd9aY59KI+QN0ns6bBo=",
- version = "v0.11.5-0.20220729040631-518f63d66278",
+ sum = "h1:m5ZsBa5o/0CkzZXfXLaThzKuR85SnHHetqBCpzQ30h8=",
+ version = "v0.11.5-0.20221009092201-b66cddb77c32",
)
go_repository(
name = "com_github_pingcap_failpoint",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/failpoint",
- sum = "h1:kJolJWbyadVeL8RKBlqmXQR7FRKPsIeU85TUYyhbhiQ=",
- version = "v0.0.0-20220423142525-ae43b7f4e5c3",
+ sum = "h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ=",
+ version = "v0.0.0-20220801062533-2eaa32854a6c",
)
go_repository(
name = "com_github_pingcap_fn",
@@ -2891,15 +3365,15 @@ def go_deps():
name = "com_github_pingcap_kvproto",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/kvproto",
- sum = "h1:ho5XUD8DVCnkpEj8oiTR57FXDTXnH6znyLe0gyrtzKk=",
- version = "v0.0.0-20221103025916-e7e21f0e9cd9",
+ sum = "h1:QcC52K9hhsP6eVmQBnSMI/b8TiOUVztbaeduTXspmeQ=",
+ version = "v0.0.0-20230206112125-0561adc37543",
)
go_repository(
name = "com_github_pingcap_log",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/log",
- sum = "h1:T7e5Low0BU2ZazI2dz2mh3W1qv+w8wtvq1YR8DneA0c=",
- version = "v1.1.1-0.20221110065318-21a4942860b3",
+ sum = "h1:crhkw6DD+07Bg1wYhW5Piw+kYNKZqFQqfC2puUf6gMI=",
+ version = "v1.1.1-0.20221116035753-734d527bc87c",
)
go_repository(
name = "com_github_pingcap_sysutil",
@@ -2908,12 +3382,20 @@ def go_deps():
sum = "h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM=",
version = "v0.0.0-20220114020952-ea68d2dbf5b4",
)
+ go_repository(
+ name = "com_github_pingcap_tidb_dashboard",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/pingcap/tidb-dashboard",
+ sum = "h1:FUdoQ6zWktVjIWLokNeulEcqIzGn6TnoOjdS9bQcFUo=",
+ version = "v0.0.0-20221201151320-ea3ee6971f2e",
+ )
+
go_repository(
name = "com_github_pingcap_tipb",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/tipb",
- sum = "h1:Yoo8j5xQGxjlsC3yt0ndsiAz0WZXED9rzsKmEN0U0DY=",
- version = "v0.0.0-20221020071514-cd933387bcb5",
+ sum = "h1:j5sw2YZY7QfgIFZEoUcn1P5cYflms1PCVVS96i+IQiI=",
+ version = "v0.0.0-20230119054146-c6b7a5a1623b",
)
go_repository(
name = "com_github_pkg_browser",
@@ -2947,8 +3429,8 @@ def go_deps():
name = "com_github_polyfloyd_go_errorlint",
build_file_proto_mode = "disable",
importpath = "github.com/polyfloyd/go-errorlint",
- sum = "h1:kp1yvHflYhTmw5m3MmBy8SCyQkKPjwDthVuMH0ug6Yk=",
- version = "v1.0.2",
+ sum = "h1:ZevdyEGxDoHAMQUVvdTT06hnYuKULe8TQkOmIYx6s1c=",
+ version = "v1.0.6",
)
go_repository(
@@ -2969,29 +3451,29 @@ def go_deps():
name = "com_github_prometheus_client_golang",
build_file_proto_mode = "disable_global",
importpath = "github.com/prometheus/client_golang",
- sum = "h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU=",
- version = "v1.13.0",
+ sum = "h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=",
+ version = "v1.14.0",
)
go_repository(
name = "com_github_prometheus_client_model",
build_file_proto_mode = "disable_global",
importpath = "github.com/prometheus/client_model",
- sum = "h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=",
- version = "v0.2.0",
+ sum = "h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=",
+ version = "v0.3.0",
)
go_repository(
name = "com_github_prometheus_common",
build_file_proto_mode = "disable_global",
importpath = "github.com/prometheus/common",
- sum = "h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=",
- version = "v0.37.0",
+ sum = "h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI=",
+ version = "v0.39.0",
)
go_repository(
name = "com_github_prometheus_procfs",
build_file_proto_mode = "disable_global",
importpath = "github.com/prometheus/procfs",
- sum = "h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=",
- version = "v0.8.0",
+ sum = "h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=",
+ version = "v0.9.0",
)
go_repository(
name = "com_github_prometheus_prometheus",
@@ -3008,26 +3490,41 @@ def go_deps():
sum = "h1:w1tAGxsBMLkuGrFMhqgcCeBkM5d1YI24udArs+aASuQ=",
version = "v0.8.0",
)
+ go_repository(
+ name = "com_github_puerkitobio_purell",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/PuerkitoBio/purell",
+ sum = "h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=",
+ version = "v1.1.1",
+ )
+ go_repository(
+ name = "com_github_puerkitobio_urlesc",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/PuerkitoBio/urlesc",
+ sum = "h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=",
+ version = "v0.0.0-20170810143723-de5bf2ad4578",
+ )
+
go_repository(
name = "com_github_quasilyte_go_ruleguard",
build_file_proto_mode = "disable",
importpath = "github.com/quasilyte/go-ruleguard",
- sum = "h1:cDdoaSbQg11LXPDQqiCK54QmQXsEQQCTIgdcpeULGSI=",
- version = "v0.3.17",
+ sum = "h1:sd+abO1PEI9fkYennwzHn9kl3nqP6M5vE7FiOzZ+5CE=",
+ version = "v0.3.18",
)
go_repository(
name = "com_github_quasilyte_go_ruleguard_dsl",
build_file_proto_mode = "disable",
importpath = "github.com/quasilyte/go-ruleguard/dsl",
- sum = "h1:vNkC6fC6qMLzCOGbnIHOd5ixUGgTbp3Z4fGnUgULlDA=",
- version = "v0.3.21",
+ sum = "h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=",
+ version = "v0.3.22",
)
go_repository(
name = "com_github_quasilyte_gogrep",
build_file_proto_mode = "disable",
importpath = "github.com/quasilyte/gogrep",
- sum = "h1:PDWGei+Rf2bBiuZIbZmM20J2ftEy9IeUCHA8HbQqed8=",
- version = "v0.0.0-20220120141003-628d8b3623b5",
+ sum = "h1:6Gtn2i04RD0gVyYf2/IUMTIs+qYleBt4zxDqkLTcu4U=",
+ version = "v0.0.0-20220828223005-86e4605de09f",
)
go_repository(
name = "com_github_quasilyte_regex_syntax",
@@ -3058,16 +3555,20 @@ def go_deps():
sum = "h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=",
version = "v0.0.0-20200410134404-eec4a21b6bb0",
)
+ go_repository(
+ name = "com_github_renekroon_ttlcache_v2",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/ReneKroon/ttlcache/v2",
+ sum = "h1:qZnUjRKIrbKHH6vF5T7Y9Izn5ObfTZfyYpGhvz2BKPo=",
+ version = "v2.3.0",
+ )
+
go_repository(
name = "com_github_rivo_uniseg",
build_file_proto_mode = "disable_global",
importpath = "github.com/rivo/uniseg",
- patch_args = ["-p1"],
- patches = [
- "//build/patches:com_github_rivo_uniseg.patch",
- ],
- sum = "h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=",
- version = "v0.4.2",
+ sum = "h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=",
+ version = "v0.4.3",
)
go_repository(
name = "com_github_rlmcpherson_s3gof3r",
@@ -3091,6 +3592,14 @@ def go_deps():
sum = "h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=",
version = "v1.6.1",
)
+ go_repository(
+ name = "com_github_rs_cors",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/rs/cors",
+ sum = "h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=",
+ version = "v1.7.0",
+ )
+
go_repository(
name = "com_github_rubyist_circuitbreaker",
build_file_proto_mode = "disable",
@@ -3117,8 +3626,8 @@ def go_deps():
name = "com_github_ryancurrah_gomodguard",
build_file_proto_mode = "disable",
importpath = "github.com/ryancurrah/gomodguard",
- sum = "h1:CpMSDKan0LtNGGhPrvupAoLeObRFjND8/tU1rEOtBp4=",
- version = "v1.2.4",
+ sum = "h1:q15RT/pd6UggBXVBuLps8BXRvl5GPBcwVA7BJHMLuTw=",
+ version = "v1.3.0",
)
go_repository(
name = "com_github_ryanrolds_sqlclosecheck",
@@ -3154,8 +3663,8 @@ def go_deps():
name = "com_github_sasha_s_go_deadlock",
build_file_proto_mode = "disable",
importpath = "github.com/sasha-s/go-deadlock",
- sum = "h1:yVBZEAirqhDYAc7xftf/swe8eHcg63jqfwdqN8KSoR8=",
- version = "v0.0.0-20161201235124-341000892f3d",
+ sum = "h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y=",
+ version = "v0.2.0",
)
go_repository(
name = "com_github_sashamelentyev_interfacebloat",
@@ -3168,8 +3677,8 @@ def go_deps():
name = "com_github_sashamelentyev_usestdlibvars",
build_file_proto_mode = "disable",
importpath = "github.com/sashamelentyev/usestdlibvars",
- sum = "h1:uObNudVEEHf6JbOJy5bgKJloA1bWjxR9fwgNFpPzKnI=",
- version = "v1.13.0",
+ sum = "h1:GQGlReyL9Ek8DdJmwtwhHbhwHnuPfsKaprpjnrPcjxc=",
+ version = "v1.21.1",
)
go_repository(
@@ -3198,8 +3707,8 @@ def go_deps():
name = "com_github_securego_gosec_v2",
build_file_proto_mode = "disable",
importpath = "github.com/securego/gosec/v2",
- sum = "h1:7mU32qn2dyC81MH9L2kefnQyRMUarfDER3iQyMHcjYM=",
- version = "v2.13.1",
+ sum = "h1:U1hfs0oBackChXA72plCYVA4cOlQ4gO+209dHiSNZbI=",
+ version = "v2.14.0",
)
go_repository(
@@ -3216,13 +3725,20 @@ def go_deps():
sum = "h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=",
version = "v0.0.0-20160112020656-b6b7b6733b8c",
)
+ go_repository(
+ name = "com_github_shirou_gopsutil",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/shirou/gopsutil",
+ sum = "h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8=",
+ version = "v3.21.3+incompatible",
+ )
go_repository(
name = "com_github_shirou_gopsutil_v3",
build_file_proto_mode = "disable_global",
importpath = "github.com/shirou/gopsutil/v3",
- sum = "h1:flKnuCMfUUrO+oAvwAd6GKZgnPzr098VA/UJ14nhJd4=",
- version = "v3.22.7",
+ sum = "h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs=",
+ version = "v3.22.12",
)
go_repository(
name = "com_github_shopify_goreferrer",
@@ -3249,8 +3765,8 @@ def go_deps():
name = "com_github_shopspring_decimal",
build_file_proto_mode = "disable",
importpath = "github.com/shopspring/decimal",
- sum = "h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=",
- version = "v0.0.0-20180709203117-cd690d0c9e24",
+ sum = "h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=",
+ version = "v1.2.0",
)
go_repository(
@@ -3307,8 +3823,15 @@ def go_deps():
name = "com_github_sivchari_tenv",
build_file_proto_mode = "disable",
importpath = "github.com/sivchari/tenv",
- sum = "h1:d4laZMBK6jpe5PWepxlV9S+LC0yXqvYHiq8E6ceoVVE=",
- version = "v1.7.0",
+ sum = "h1:PSpuD4bu6fSmtWMxSGWcvqUUgIn7k3yOJhOIzVWn8Ak=",
+ version = "v1.7.1",
+ )
+ go_repository(
+ name = "com_github_smallnest_chanx",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/smallnest/chanx",
+ sum = "h1:Txo4SXVJq/OgEjwgkWoxkMoTjGlcrgsQE/XSghjmu0w=",
+ version = "v0.0.0-20221229104322-eb4c998d2072",
)
go_repository(
@@ -3343,16 +3866,16 @@ def go_deps():
name = "com_github_sourcegraph_go_diff",
build_file_proto_mode = "disable",
importpath = "github.com/sourcegraph/go-diff",
- sum = "h1:hmA1LzxW0n1c3Q4YbrFgg4P99GSnebYa3x8gr0HZqLQ=",
- version = "v0.6.1",
+ sum = "h1:FEIBISvqa2IsyC4KQQBQ1Ef2QvweGUgEIjCdE3gz+zs=",
+ version = "v0.6.2-0.20221031073116-7ef5f68ebea1",
)
go_repository(
name = "com_github_spaolacci_murmur3",
build_file_proto_mode = "disable_global",
importpath = "github.com/spaolacci/murmur3",
- sum = "h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=",
- version = "v0.0.0-20180118202830-f09979ecbc72",
+ sum = "h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=",
+ version = "v1.1.0",
)
go_repository(
name = "com_github_spf13_afero",
@@ -3372,8 +3895,8 @@ def go_deps():
name = "com_github_spf13_cobra",
build_file_proto_mode = "disable_global",
importpath = "github.com/spf13/cobra",
- sum = "h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=",
- version = "v1.5.0",
+ sum = "h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=",
+ version = "v1.6.1",
)
go_repository(
name = "com_github_spf13_jwalterweatherman",
@@ -3396,6 +3919,14 @@ def go_deps():
sum = "h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ=",
version = "v1.12.0",
)
+ go_repository(
+ name = "com_github_spkg_bom",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/spkg/bom",
+ sum = "h1:S939THe0ukL5WcTGiGqkgtaW5JW+O6ITaIlpJXTYY64=",
+ version = "v1.0.0",
+ )
+
go_repository(
name = "com_github_ssgreg_nlreturn_v2",
build_file_proto_mode = "disable",
@@ -3430,23 +3961,45 @@ def go_deps():
name = "com_github_stretchr_objx",
build_file_proto_mode = "disable_global",
importpath = "github.com/stretchr/objx",
- sum = "h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=",
- version = "v0.4.0",
+ sum = "h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=",
+ version = "v0.5.0",
)
go_repository(
name = "com_github_stretchr_testify",
build_file_proto_mode = "disable_global",
importpath = "github.com/stretchr/testify",
- sum = "h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=",
- version = "v1.8.0",
+ sum = "h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=",
+ version = "v1.8.1",
)
go_repository(
name = "com_github_subosito_gotenv",
build_file_proto_mode = "disable_global",
importpath = "github.com/subosito/gotenv",
- sum = "h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs=",
- version = "v1.4.0",
+ sum = "h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=",
+ version = "v1.4.1",
+ )
+ go_repository(
+ name = "com_github_swaggo_files",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/swaggo/files",
+ sum = "h1:PyYN9JH5jY9j6av01SpfRMb+1DWg/i3MbGOKPxJ2wjM=",
+ version = "v0.0.0-20190704085106-630677cd5c14",
+ )
+ go_repository(
+ name = "com_github_swaggo_http_swagger",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/swaggo/http-swagger",
+ sum = "h1:lUPlXKqgbqT2SVg2Y+eT9mu5wbqMnG+i/+Q9nK7C0Rs=",
+ version = "v0.0.0-20200308142732-58ac5e232fba",
+ )
+ go_repository(
+ name = "com_github_swaggo_swag",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/swaggo/swag",
+ sum = "h1:3pZSSCQ//gAH88lfmxM3Cd1+JCsxV8Md6f36b9hrZ5s=",
+ version = "v1.8.3",
)
+
go_repository(
name = "com_github_sylvia7788_contextcheck",
build_file_proto_mode = "disable",
@@ -3454,6 +4007,21 @@ def go_deps():
sum = "h1:o2EZgVPyMKE/Mtoqym61DInKEjwEbsmyoxg3VrmjNO4=",
version = "v1.0.6",
)
+ go_repository(
+ name = "com_github_syndtr_goleveldb",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/syndtr/goleveldb",
+ sum = "h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=",
+ version = "v1.0.1-0.20190318030020-c3a204f8e965",
+ )
+ go_repository(
+ name = "com_github_t_yuki_gocover_cobertura",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/t-yuki/gocover-cobertura",
+ sum = "h1:+aPplBwWcHBo6q9xrfWdMrT9o4kltkmmvpemgIjep/8=",
+ version = "v0.0.0-20180217150009-aaee18c8195c",
+ )
+
go_repository(
name = "com_github_tdakkota_asciicheck",
build_file_proto_mode = "disable",
@@ -3483,6 +4051,13 @@ def go_deps():
sum = "h1:BVoBIqAf/2QdbFmSwAWnaIqDivZdOV0ZRwEm6jivLKw=",
version = "v1.4.11",
)
+ go_repository(
+ name = "com_github_thoas_go_funk",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/thoas/go-funk",
+ sum = "h1:JP9tKSvnpFVclYgDM0Is7FD9M4fhPvqA0s0BsXmzSRQ=",
+ version = "v0.8.0",
+ )
go_repository(
name = "com_github_tiancaiamao_appdash",
@@ -3491,27 +4066,58 @@ def go_deps():
sum = "h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ=",
version = "v0.0.0-20181126055449-889f96f722a2",
)
+ go_repository(
+ name = "com_github_tiancaiamao_gp",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/tiancaiamao/gp",
+ sum = "h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=",
+ version = "v0.0.0-20221230034425-4025bc8a4d4a",
+ )
+ go_repository(
+ name = "com_github_tidwall_gjson",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/tidwall/gjson",
+ sum = "h1:hqzS9wAHMO+KVBBkLxYdkEeeFHuqr95GfClRLKlgK0E=",
+ version = "v1.9.3",
+ )
+
go_repository(
name = "com_github_tikv_client_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/client-go/v2",
- sum = "h1:nFVdyTXcQYZwQQCdSJcFI1vBFyzG1hVuZ39MAK6wqK4=",
- version = "v2.0.3-0.20221108030801-9c0835c80eba",
+ sum = "h1:kiPiEBubzB2Ef2yPfoItQpjhGepcZSlRyqrXnfs92pQ=",
+ version = "v2.0.6-0.20230209044022-95ebf6a86301",
+ )
+ go_repository(
+ name = "com_github_tikv_pd",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/tikv/pd",
+ sum = "h1:iY/RztOIZ2nTbINUiLGsSv3SUGoEiub1GN0SKVKHJYg=",
+ version = "v1.1.0-beta.0.20230202094356-18df271ce57f",
)
+
go_repository(
name = "com_github_tikv_pd_client",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/pd/client",
- sum = "h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc=",
- version = "v0.0.0-20221031025758-80f0d8ca4d07",
+ sum = "h1:hauBQBHSyrUxAI0zvkTiBKd472c+Iy+aY0Jd+b9VOJ8=",
+ version = "v0.0.0-20230209034200-6d23a31c24be",
)
go_repository(
name = "com_github_timakin_bodyclose",
build_file_proto_mode = "disable",
importpath = "github.com/timakin/bodyclose",
- sum = "h1:kl4KhGNsJIbDHS9/4U9yQo1UcPQM0kOMJHn29EoH/Ro=",
- version = "v0.0.0-20210704033933-f49887972144",
+ sum = "h1:MV6KaVu/hzByHP0UvJ4HcMGE/8a6A4Rggc/0wx2AvJo=",
+ version = "v0.0.0-20221125081123-e39cf3fc478e",
+ )
+ go_repository(
+ name = "com_github_timonwong_loggercheck",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/timonwong/loggercheck",
+ sum = "h1:ecACo9fNiHxX4/Bc02rW2+kaJIAMAes7qJ7JKxt0EZI=",
+ version = "v0.9.3",
)
+
go_repository(
name = "com_github_timonwong_logrlint",
build_file_proto_mode = "disable",
@@ -3524,15 +4130,15 @@ def go_deps():
name = "com_github_tklauser_go_sysconf",
build_file_proto_mode = "disable_global",
importpath = "github.com/tklauser/go-sysconf",
- sum = "h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=",
- version = "v0.3.10",
+ sum = "h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=",
+ version = "v0.3.11",
)
go_repository(
name = "com_github_tklauser_numcpus",
build_file_proto_mode = "disable_global",
importpath = "github.com/tklauser/numcpus",
- sum = "h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=",
- version = "v0.4.0",
+ sum = "h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=",
+ version = "v0.6.0",
)
go_repository(
name = "com_github_tmc_grpc_websocket_proxy",
@@ -3545,15 +4151,15 @@ def go_deps():
name = "com_github_tomarrell_wrapcheck_v2",
build_file_proto_mode = "disable",
importpath = "github.com/tomarrell/wrapcheck/v2",
- sum = "h1:3dI6YNcrJTQ/CJQ6M/DUkc0gnqYSIk6o0rChn9E/D0M=",
- version = "v2.6.2",
+ sum = "h1:J/F8DbSKJC83bAvC6FoZaRjZiZ/iKoueSdrEkmGeacA=",
+ version = "v2.7.0",
)
go_repository(
name = "com_github_tommy_muehle_go_mnd_v2",
build_file_proto_mode = "disable",
importpath = "github.com/tommy-muehle/go-mnd/v2",
- sum = "h1:iAj0a8e6+dXSL7Liq0aXPox36FiN1dBbjA6lt9fl65s=",
- version = "v2.5.0",
+ sum = "h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw=",
+ version = "v2.5.1",
)
go_repository(
@@ -3605,6 +4211,20 @@ def go_deps():
sum = "h1:hh+/cpIcopyMYbZNVov9iSxvJU3OYQg78Sfaqzi/CzI=",
version = "v0.0.5",
)
+ go_repository(
+ name = "com_github_unrolled_render",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/unrolled/render",
+ sum = "h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY=",
+ version = "v1.0.1",
+ )
+ go_repository(
+ name = "com_github_urfave_cli_v2",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/urfave/cli/v2",
+ sum = "h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=",
+ version = "v2.3.0",
+ )
go_repository(
name = "com_github_urfave_negroni",
@@ -3672,6 +4292,28 @@ def go_deps():
sum = "h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=",
version = "v1.2.0",
)
+ go_repository(
+ name = "com_github_vividcortex_mysqlerr",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/VividCortex/mysqlerr",
+ sum = "h1:5pZ2TZA+YnzPgzBfiUWGqWmKDVNBdrkf9g+DNe1Tiq8=",
+ version = "v1.0.0",
+ )
+ go_repository(
+ name = "com_github_vmihailenco_msgpack_v5",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/vmihailenco/msgpack/v5",
+ sum = "h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=",
+ version = "v5.3.5",
+ )
+ go_repository(
+ name = "com_github_vmihailenco_tagparser_v2",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/vmihailenco/tagparser/v2",
+ sum = "h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=",
+ version = "v2.0.0",
+ )
+
go_repository(
name = "com_github_wangjohn_quickselect",
build_file_proto_mode = "disable_global",
@@ -3714,6 +4356,14 @@ def go_deps():
sum = "h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=",
version = "v1.2.0",
)
+ go_repository(
+ name = "com_github_xeoncross_go_aesctr_with_hmac",
+ build_file_proto_mode = "disable",
+ importpath = "github.com/Xeoncross/go-aesctr-with-hmac",
+ sum = "h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o=",
+ version = "v0.0.0-20200623134604-12b17a7ff502",
+ )
+
go_repository(
name = "com_github_xiang90_probing",
build_file_proto_mode = "disable_global",
@@ -3812,110 +4462,932 @@ def go_deps():
name = "com_google_cloud_go",
build_file_proto_mode = "disable_global",
importpath = "cloud.google.com/go",
- sum = "h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y=",
- version = "v0.100.2",
- )
- go_repository(
- name = "com_google_cloud_go_bigquery",
- build_file_proto_mode = "disable_global",
- importpath = "cloud.google.com/go/bigquery",
- sum = "h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA=",
- version = "v1.8.0",
+ sum = "h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y=",
+ version = "v0.105.0",
)
go_repository(
- name = "com_google_cloud_go_compute",
- build_file_proto_mode = "disable_global",
- importpath = "cloud.google.com/go/compute",
- sum = "h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM=",
+ name = "com_google_cloud_go_accessapproval",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/accessapproval",
+ sum = "h1:/nTivgnV/n1CaAeo+ekGexTYUsKEU9jUVkoY5359+3Q=",
version = "v1.5.0",
)
go_repository(
- name = "com_google_cloud_go_datastore",
- build_file_proto_mode = "disable_global",
- importpath = "cloud.google.com/go/datastore",
- sum = "h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ=",
- version = "v1.1.0",
+ name = "com_google_cloud_go_accesscontextmanager",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/accesscontextmanager",
+ sum = "h1:CFhNhU7pcD11cuDkQdrE6PQJgv0EXNKNv06jIzbLlCU=",
+ version = "v1.4.0",
)
go_repository(
- name = "com_google_cloud_go_firestore",
- build_file_proto_mode = "disable_global",
- importpath = "cloud.google.com/go/firestore",
- sum = "h1:9x7Bx0A9R5/M9jibeJeZWqjeVEIxYW9fZYqB9a70/bY=",
- version = "v1.1.0",
+ name = "com_google_cloud_go_aiplatform",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/aiplatform",
+ sum = "h1:DBi3Jk9XjCJ4pkkLM4NqKgj3ozUL1wq4l+d3/jTGXAI=",
+ version = "v1.27.0",
)
go_repository(
- name = "com_google_cloud_go_iam",
- build_file_proto_mode = "disable_global",
- importpath = "cloud.google.com/go/iam",
- sum = "h1:4CapQyNFjiksks1/x7jsvsygFPhihslYk5GptIrlX68=",
- version = "v0.1.1",
+ name = "com_google_cloud_go_analytics",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/analytics",
+ sum = "h1:NKw6PpQi6V1O+KsjuTd+bhip9d0REYu4NevC45vtGp8=",
+ version = "v0.12.0",
)
go_repository(
- name = "com_google_cloud_go_pubsub",
- build_file_proto_mode = "disable_global",
- importpath = "cloud.google.com/go/pubsub",
- sum = "h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU=",
- version = "v1.3.1",
+ name = "com_google_cloud_go_apigateway",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/apigateway",
+ sum = "h1:IIoXKR7FKrEAQhMTz5hK2wiDz2WNFHS7eVr/L1lE/rM=",
+ version = "v1.4.0",
)
go_repository(
- name = "com_google_cloud_go_storage",
- build_file_proto_mode = "disable_global",
- importpath = "cloud.google.com/go/storage",
- sum = "h1:HwnT2u2D309SFDHQII6m18HlrCi3jAXhUMTLOWXYH14=",
- version = "v1.21.0",
+ name = "com_google_cloud_go_apigeeconnect",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/apigeeconnect",
+ sum = "h1:AONoTYJviyv1vS4IkvWzq69gEVdvHx35wKXc+e6wjZQ=",
+ version = "v1.4.0",
)
go_repository(
- name = "com_shuralyov_dmitri_gpu_mtl",
- build_file_proto_mode = "disable_global",
- importpath = "dmitri.shuralyov.com/gpu/mtl",
- sum = "h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY=",
- version = "v0.0.0-20190408044501-666a987793e9",
+ name = "com_google_cloud_go_appengine",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/appengine",
+ sum = "h1:lmG+O5oaR9xNwaRBwE2XoMhwQHsHql5IoiGr1ptdDwU=",
+ version = "v1.5.0",
)
go_repository(
- name = "com_sourcegraph_sourcegraph_appdash",
- build_file_proto_mode = "disable_global",
- importpath = "sourcegraph.com/sourcegraph/appdash",
- sum = "h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM=",
- version = "v0.0.0-20190731080439-ebfcffb1b5c0",
+ name = "com_google_cloud_go_area120",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/area120",
+ sum = "h1:TCMhwWEWhCn8d44/Zs7UCICTWje9j3HuV6nVGMjdpYw=",
+ version = "v0.6.0",
)
go_repository(
- name = "com_sourcegraph_sourcegraph_appdash_data",
- build_file_proto_mode = "disable_global",
- importpath = "sourcegraph.com/sourcegraph/appdash-data",
- sum = "h1:e1sMhtVq9AfcEy8AXNb8eSg6gbzfdpYhoNqnPJa+GzI=",
- version = "v0.0.0-20151005221446-73f23eafcf67",
+ name = "com_google_cloud_go_artifactregistry",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/artifactregistry",
+ sum = "h1:3d0LRAU1K6vfqCahhl9fx2oGHcq+s5gftdix4v8Ibrc=",
+ version = "v1.9.0",
)
go_repository(
- name = "com_stathat_c_consistent",
+ name = "com_google_cloud_go_asset",
build_file_proto_mode = "disable",
- importpath = "stathat.com/c/consistent",
- sum = "h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=",
- version = "v1.0.0",
+ importpath = "cloud.google.com/go/asset",
+ sum = "h1:aCrlaLGJWTODJX4G56ZYzJefITKEWNfbjjtHSzWpxW0=",
+ version = "v1.10.0",
)
-
go_repository(
- name = "in_gopkg_alecthomas_kingpin_v2",
- build_file_proto_mode = "disable_global",
- importpath = "gopkg.in/alecthomas/kingpin.v2",
- sum = "h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=",
- version = "v2.2.6",
+ name = "com_google_cloud_go_assuredworkloads",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/assuredworkloads",
+ sum = "h1:hhIdCOowsT1GG5eMCIA0OwK6USRuYTou/1ZeNxCSRtA=",
+ version = "v1.9.0",
)
go_repository(
- name = "in_gopkg_check_v1",
- build_file_proto_mode = "disable_global",
- importpath = "gopkg.in/check.v1",
- sum = "h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=",
- version = "v1.0.0-20201130134442-10cb98267c6c",
+ name = "com_google_cloud_go_automl",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/automl",
+ sum = "h1:BMioyXSbg7d7xLibn47cs0elW6RT780IUWr42W8rp2Q=",
+ version = "v1.8.0",
)
go_repository(
- name = "in_gopkg_errgo_v2",
- build_file_proto_mode = "disable_global",
- importpath = "gopkg.in/errgo.v2",
- sum = "h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=",
- version = "v2.1.0",
+ name = "com_google_cloud_go_baremetalsolution",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/baremetalsolution",
+ sum = "h1:g9KO6SkakcYPcc/XjAzeuUrEOXlYPnMpuiaywYaGrmQ=",
+ version = "v0.4.0",
)
go_repository(
- name = "in_gopkg_fsnotify_fsnotify_v1",
+ name = "com_google_cloud_go_batch",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/batch",
+ sum = "h1:1jvEBY55OH4Sd2FxEXQfxGExFWov1A/IaRe+Z5Z71Fw=",
+ version = "v0.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_beyondcorp",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/beyondcorp",
+ sum = "h1:w+4kThysgl0JiKshi2MKDCg2NZgOyqOI0wq2eBZyrzA=",
+ version = "v0.3.0",
+ )
+
+ go_repository(
+ name = "com_google_cloud_go_bigquery",
+ build_file_proto_mode = "disable_global",
+ importpath = "cloud.google.com/go/bigquery",
+ sum = "h1:Wi4dITi+cf9VYp4VH2T9O41w0kCW0uQTELq2Z6tukN0=",
+ version = "v1.44.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_billing",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/billing",
+ sum = "h1:Xkii76HWELHwBtkQVZvqmSo9GTr0O+tIbRNnMcGdlg4=",
+ version = "v1.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_binaryauthorization",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/binaryauthorization",
+ sum = "h1:pL70vXWn9TitQYXBWTK2abHl2JHLwkFRjYw6VflRqEA=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_certificatemanager",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/certificatemanager",
+ sum = "h1:tzbR4UHBbgsewMWUD93JHi8EBi/gHBoSAcY1/sThFGk=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_channel",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/channel",
+ sum = "h1:pNuUlZx0Jb0Ts9P312bmNMuH5IiFWIR4RUtLb70Ke5s=",
+ version = "v1.9.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_cloudbuild",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/cloudbuild",
+ sum = "h1:TAAmCmAlOJ4uNBu6zwAjwhyl/7fLHHxIEazVhr3QBbQ=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_clouddms",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/clouddms",
+ sum = "h1:UhzHIlgFfMr6luVYVNydw/pl9/U5kgtjCMJHnSvoVws=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_cloudtasks",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/cloudtasks",
+ sum = "h1:faUiUgXjW8yVZ7XMnKHKm1WE4OldPBUWWfIRN/3z1dc=",
+ version = "v1.8.0",
+ )
+
+ go_repository(
+ name = "com_google_cloud_go_compute",
+ build_file_proto_mode = "disable_global",
+ importpath = "cloud.google.com/go/compute",
+ sum = "h1:AYrLkB8NPdDRslNp4Jxmzrhdr03fUAIDbiGFjLWowoU=",
+ version = "v1.13.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_compute_metadata",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/compute/metadata",
+ sum = "h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=",
+ version = "v0.2.1",
+ )
+ go_repository(
+ name = "com_google_cloud_go_contactcenterinsights",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/contactcenterinsights",
+ sum = "h1:tTQLI/ZvguUf9Hv+36BkG2+/PeC8Ol1q4pBW+tgCx0A=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_container",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/container",
+ sum = "h1:nbEK/59GyDRKKlo1SqpohY1TK8LmJ2XNcvS9Gyom2A0=",
+ version = "v1.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_containeranalysis",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/containeranalysis",
+ sum = "h1:2824iym832ljKdVpCBnpqm5K94YT/uHTVhNF+dRTXPI=",
+ version = "v0.6.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_datacatalog",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/datacatalog",
+ sum = "h1:6kZ4RIOW/uT7QWC5SfPfq/G8sYzr/v+UOmOAxy4Z1TE=",
+ version = "v1.8.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_dataflow",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/dataflow",
+ sum = "h1:CW3541Fm7KPTyZjJdnX6NtaGXYFn5XbFC5UcjgALKvU=",
+ version = "v0.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_dataform",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/dataform",
+ sum = "h1:vLwowLF2ZB5J5gqiZCzv076lDI/Rd7zYQQFu5XO1PSg=",
+ version = "v0.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_datafusion",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/datafusion",
+ sum = "h1:j5m2hjWovTZDTQak4MJeXAR9yN7O+zMfULnjGw/OOLg=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_datalabeling",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/datalabeling",
+ sum = "h1:dp8jOF21n/7jwgo/uuA0RN8hvLcKO4q6s/yvwevs2ZM=",
+ version = "v0.6.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_dataplex",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/dataplex",
+ sum = "h1:cNxeA2DiWliQGi21kPRqnVeQ5xFhNoEjPRt1400Pm8Y=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_dataproc",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/dataproc",
+ sum = "h1:gVOqNmElfa6n/ccG/QDlfurMWwrK3ezvy2b2eDoCmS0=",
+ version = "v1.8.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_dataqna",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/dataqna",
+ sum = "h1:gx9jr41ytcA3dXkbbd409euEaWtofCVXYBvJz3iYm18=",
+ version = "v0.6.0",
+ )
+
+ go_repository(
+ name = "com_google_cloud_go_datastore",
+ build_file_proto_mode = "disable_global",
+ importpath = "cloud.google.com/go/datastore",
+ sum = "h1:4siQRf4zTiAVt/oeH4GureGkApgb2vtPQAtOmhpqQwE=",
+ version = "v1.10.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_datastream",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/datastream",
+ sum = "h1:PgIgbhedBtYBU6POGXFMn2uSl9vpqubc3ewTNdcU8Mk=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_deploy",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/deploy",
+ sum = "h1:kI6dxt8Ml0is/x7YZjLveTvR7YPzXAUD/8wQZ2nH5zA=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_dialogflow",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/dialogflow",
+ sum = "h1:HYHVOkoxQ9bSfNIelSZYNAtUi4CeSrCnROyOsbOqPq8=",
+ version = "v1.19.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_dlp",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/dlp",
+ sum = "h1:9I4BYeJSVKoSKgjr70fLdRDumqcUeVmHV4fd5f9LR6Y=",
+ version = "v1.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_documentai",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/documentai",
+ sum = "h1:jfq09Fdjtnpnmt/MLyf6A3DM3ynb8B2na0K+vSXvpFM=",
+ version = "v1.10.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_domains",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/domains",
+ sum = "h1:pu3JIgC1rswIqi5romW0JgNO6CTUydLYX8zyjiAvO1c=",
+ version = "v0.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_edgecontainer",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/edgecontainer",
+ sum = "h1:hd6J2n5dBBRuAqnNUEsKWrp6XNPKsaxwwIyzOPZTokk=",
+ version = "v0.2.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_errorreporting",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/errorreporting",
+ sum = "h1:kj1XEWMu8P0qlLhm3FwcaFsUvXChV/OraZwA70trRR0=",
+ version = "v0.3.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_essentialcontacts",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/essentialcontacts",
+ sum = "h1:b6csrQXCHKQmfo9h3dG/pHyoEh+fQG1Yg78a53LAviY=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_eventarc",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/eventarc",
+ sum = "h1:AgCqrmMMIcel5WWKkzz5EkCUKC3Rl5LNMMYsS+LvsI0=",
+ version = "v1.8.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_filestore",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/filestore",
+ sum = "h1:yjKOpzvqtDmL5AXbKttLc8j0hL20kuC1qPdy5HPcxp0=",
+ version = "v1.4.0",
+ )
+
+ go_repository(
+ name = "com_google_cloud_go_firestore",
+ build_file_proto_mode = "disable_global",
+ importpath = "cloud.google.com/go/firestore",
+ sum = "h1:IBlRyxgGySXu5VuW0RgGFlTtLukSnNkpDiEOMkQkmpA=",
+ version = "v1.9.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_functions",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/functions",
+ sum = "h1:35tgv1fQOtvKqH/uxJMzX3w6usneJ0zXpsFr9KAVhNE=",
+ version = "v1.9.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_gaming",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/gaming",
+ sum = "h1:97OAEQtDazAJD7yh/kvQdSCQuTKdR0O+qWAJBZJ4xiA=",
+ version = "v1.8.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_gkebackup",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/gkebackup",
+ sum = "h1:4K+jiv4ocqt1niN8q5Imd8imRoXBHTrdnJVt/uFFxF4=",
+ version = "v0.3.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_gkeconnect",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/gkeconnect",
+ sum = "h1:zAcvDa04tTnGdu6TEZewaLN2tdMtUOJJ7fEceULjguA=",
+ version = "v0.6.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_gkehub",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/gkehub",
+ sum = "h1:JTcTaYQRGsVm+qkah7WzHb6e9sf1C0laYdRPn9aN+vg=",
+ version = "v0.10.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_gkemulticloud",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/gkemulticloud",
+ sum = "h1:8F1NhJj8ucNj7lK51UZMtAjSWTgP1zO18XF6vkfiPPU=",
+ version = "v0.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_grafeas",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/grafeas",
+ sum = "h1:CYjC+xzdPvbV65gi6Dr4YowKcmLo045pm18L0DhdELM=",
+ version = "v0.2.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_gsuiteaddons",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/gsuiteaddons",
+ sum = "h1:TGT2oGmO5q3VH6SjcrlgPUWI0njhYv4kywLm6jag0to=",
+ version = "v1.4.0",
+ )
+
+ go_repository(
+ name = "com_google_cloud_go_iam",
+ build_file_proto_mode = "disable_global",
+ importpath = "cloud.google.com/go/iam",
+ sum = "h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk=",
+ version = "v0.8.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_iap",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/iap",
+ sum = "h1:BGEXovwejOCt1zDk8hXq0bOhhRu9haXKWXXXp2B4wBM=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_ids",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/ids",
+ sum = "h1:LncHK4HHucb5Du310X8XH9/ICtMwZ2PCfK0ScjWiJoY=",
+ version = "v1.2.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_iot",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/iot",
+ sum = "h1:Y9+oZT9jD4GUZzORXTU45XsnQrhxmDT+TFbPil6pRVQ=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_kms",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/kms",
+ sum = "h1:OWRZzrPmOZUzurjI2FBGtgY2mB1WaJkqhw6oIwSj0Yg=",
+ version = "v1.6.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_language",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/language",
+ sum = "h1:3Wa+IUMamL4JH3Zd3cDZUHpwyqplTACt6UZKRD2eCL4=",
+ version = "v1.8.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_lifesciences",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/lifesciences",
+ sum = "h1:tIqhivE2LMVYkX0BLgG7xL64oNpDaFFI7teunglt1tI=",
+ version = "v0.6.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_logging",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/logging",
+ sum = "h1:ZBsZK+JG+oCDT+vaxwqF2egKNRjz8soXiS6Xv79benI=",
+ version = "v1.6.1",
+ )
+ go_repository(
+ name = "com_google_cloud_go_longrunning",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/longrunning",
+ sum = "h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=",
+ version = "v0.3.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_managedidentities",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/managedidentities",
+ sum = "h1:3Kdajn6X25yWQFhFCErmKSYTSvkEd3chJROny//F1A0=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_maps",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/maps",
+ sum = "h1:kLReRbclTgJefw2fcCbdLPLhPj0U6UUWN10ldG8sdOU=",
+ version = "v0.1.0",
+ )
+
+ go_repository(
+ name = "com_google_cloud_go_mediatranslation",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/mediatranslation",
+ sum = "h1:qAJzpxmEX+SeND10Y/4868L5wfZpo4Y3BIEnIieP4dk=",
+ version = "v0.6.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_memcache",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/memcache",
+ sum = "h1:yLxUzJkZVSH2kPaHut7k+7sbIBFpvSh1LW9qjM2JDjA=",
+ version = "v1.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_metastore",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/metastore",
+ sum = "h1:3KcShzqWdqxrDEXIBWpYJpOOrgpDj+HlBi07Grot49Y=",
+ version = "v1.8.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_monitoring",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/monitoring",
+ sum = "h1:c9riaGSPQ4dUKWB+M1Fl0N+iLxstMbCktdEwYSPGDvA=",
+ version = "v1.8.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_networkconnectivity",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/networkconnectivity",
+ sum = "h1:BVdIKaI68bihnXGdCVL89Jsg9kq2kg+II30fjVqo62E=",
+ version = "v1.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_networkmanagement",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/networkmanagement",
+ sum = "h1:mDHA3CDW00imTvC5RW6aMGsD1bH+FtKwZm/52BxaiMg=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_networksecurity",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/networksecurity",
+ sum = "h1:qDEX/3sipg9dS5JYsAY+YvgTjPR63cozzAWop8oZS94=",
+ version = "v0.6.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_notebooks",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/notebooks",
+ sum = "h1:AC8RPjNvel3ExgXjO1YOAz+teg9+j+89TNxa7pIZfww=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_optimization",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/optimization",
+ sum = "h1:7PxOq9VTT7TMib/6dMoWpMvWS2E4dJEvtYzjvBreaec=",
+ version = "v1.2.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_orchestration",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/orchestration",
+ sum = "h1:39d6tqvNjd/wsSub1Bn4cEmrYcet5Ur6xpaN+SxOxtY=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_orgpolicy",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/orgpolicy",
+ sum = "h1:erF5PHqDZb6FeFrUHiYj2JK2BMhsk8CyAg4V4amJ3rE=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_osconfig",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/osconfig",
+ sum = "h1:NO0RouqCOM7M2S85Eal6urMSSipWwHU8evzwS+siqUI=",
+ version = "v1.10.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_oslogin",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/oslogin",
+ sum = "h1:pKGDPfeZHDybtw48WsnVLjoIPMi9Kw62kUE5TXCLCN4=",
+ version = "v1.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_phishingprotection",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/phishingprotection",
+ sum = "h1:OrwHLSRSZyaiOt3tnY33dsKSedxbMzsXvqB21okItNQ=",
+ version = "v0.6.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_policytroubleshooter",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/policytroubleshooter",
+ sum = "h1:NQklJuOUoz1BPP+Epjw81COx7IISWslkZubz/1i0UN8=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_privatecatalog",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/privatecatalog",
+ sum = "h1:Vz86uiHCtNGm1DeC32HeG2VXmOq5JRYA3VRPf8ZEcSg=",
+ version = "v0.6.0",
+ )
+
+ go_repository(
+ name = "com_google_cloud_go_pubsub",
+ build_file_proto_mode = "disable_global",
+ importpath = "cloud.google.com/go/pubsub",
+ sum = "h1:q+J/Nfr6Qx4RQeu3rJcnN48SNC0qzlYzSeqkPq93VHs=",
+ version = "v1.27.1",
+ )
+ go_repository(
+ name = "com_google_cloud_go_pubsublite",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/pubsublite",
+ sum = "h1:iqrD8vp3giTb7hI1q4TQQGj77cj8zzgmMPsTZtLnprM=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_recaptchaenterprise",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/recaptchaenterprise",
+ sum = "h1:u6EznTGzIdsyOsvm+Xkw0aSuKFXQlyjGE9a4exk6iNQ=",
+ version = "v1.3.1",
+ )
+ go_repository(
+ name = "com_google_cloud_go_recaptchaenterprise_v2",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/recaptchaenterprise/v2",
+ sum = "h1:UqzFfb/WvhwXGDF1eQtdHLrmni+iByZXY4h3w9Kdyv8=",
+ version = "v2.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_recommendationengine",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/recommendationengine",
+ sum = "h1:6w+WxPf2LmUEqX0YyvfCoYb8aBYOcbIV25Vg6R0FLGw=",
+ version = "v0.6.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_recommender",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/recommender",
+ sum = "h1:9kMZQGeYfcOD/RtZfcNKGKtoex3DdoB4zRgYU/WaIwE=",
+ version = "v1.8.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_redis",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/redis",
+ sum = "h1:/zTwwBKIAD2DEWTrXZp8WD9yD/gntReF/HkPssVYd0U=",
+ version = "v1.10.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_resourcemanager",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/resourcemanager",
+ sum = "h1:NDao6CHMwEZIaNsdWy+tuvHaavNeGP06o1tgrR0kLvU=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_resourcesettings",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/resourcesettings",
+ sum = "h1:eTzOwB13WrfF0kuzG2ZXCfB3TLunSHBur4s+HFU6uSM=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_retail",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/retail",
+ sum = "h1:N9fa//ecFUOEPsW/6mJHfcapPV0wBSwIUwpVZB7MQ3o=",
+ version = "v1.11.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_run",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/run",
+ sum = "h1:AWPuzU7Xtaj3Jf+QarDWIs6AJ5hM1VFQ+F6Q+VZ6OT4=",
+ version = "v0.3.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_scheduler",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/scheduler",
+ sum = "h1:K/mxOewgHGeKuATUJNGylT75Mhtjmx1TOkKukATqMT8=",
+ version = "v1.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_secretmanager",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/secretmanager",
+ sum = "h1:xE6uXljAC1kCR8iadt9+/blg1fvSbmenlsDN4fT9gqw=",
+ version = "v1.9.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_security",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/security",
+ sum = "h1:KSKzzJMyUoMRQzcz7azIgqAUqxo7rmQ5rYvimMhikqg=",
+ version = "v1.10.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_securitycenter",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/securitycenter",
+ sum = "h1:QTVtk/Reqnx2bVIZtJKm1+mpfmwRwymmNvlaFez7fQY=",
+ version = "v1.16.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_servicecontrol",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/servicecontrol",
+ sum = "h1:ImIzbOu6y4jL6ob65I++QzvqgFaoAKgHOG+RU9/c4y8=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_servicedirectory",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/servicedirectory",
+ sum = "h1:f7M8IMcVzO3T425AqlZbP3yLzeipsBHtRza8vVFYMhQ=",
+ version = "v1.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_servicemanagement",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/servicemanagement",
+ sum = "h1:TpkCO5M7dhKSy1bKUD9o/sSEW/U1Gtx7opA1fsiMx0c=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_serviceusage",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/serviceusage",
+ sum = "h1:b0EwJxPJLpavSljMQh0RcdHsUrr5DQ+Nelt/3BAs5ro=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_shell",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/shell",
+ sum = "h1:b1LFhFBgKsG252inyhtmsUUZwchqSz3WTvAIf3JFo4g=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_spanner",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/spanner",
+ sum = "h1:NvdTpRwf7DTegbfFdPjAWyD7bOVu0VeMqcvR9aCQCAc=",
+ version = "v1.41.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_speech",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/speech",
+ sum = "h1:yK0ocnFH4Wsf0cMdUyndJQ/hPv02oTJOxzi6AgpBy4s=",
+ version = "v1.9.0",
+ )
+
+ go_repository(
+ name = "com_google_cloud_go_storage",
+ build_file_proto_mode = "disable_global",
+ importpath = "cloud.google.com/go/storage",
+ sum = "h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ=",
+ version = "v1.27.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_storagetransfer",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/storagetransfer",
+ sum = "h1:fUe3OydbbvHcAYp07xY+2UpH4AermGbmnm7qdEj3tGE=",
+ version = "v1.6.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_talent",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/talent",
+ sum = "h1:MrekAGxLqAeAol4Sc0allOVqUGO8j+Iim8NMvpiD7tM=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_texttospeech",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/texttospeech",
+ sum = "h1:ccPiHgTewxgyAeCWgQWvZvrLmbfQSFABTMAfrSPLPyY=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_tpu",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/tpu",
+ sum = "h1:ztIdKoma1Xob2qm6QwNh4Xi9/e7N3IfvtwG5AcNsj1g=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_trace",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/trace",
+ sum = "h1:qO9eLn2esajC9sxpqp1YKX37nXC3L4BfGnPS0Cx9dYo=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_translate",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/translate",
+ sum = "h1:AOYOH3MspzJ/bH1YXzB+xTE8fMpn3mwhLjugwGXvMPI=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_video",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/video",
+ sum = "h1:ttlvO4J5c1VGq6FkHqWPD/aH6PfdxujHt+muTJlW1Zk=",
+ version = "v1.9.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_videointelligence",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/videointelligence",
+ sum = "h1:RPFgVVXbI2b5vnrciZjtsUgpNKVtHO/WIyXUhEfuMhA=",
+ version = "v1.9.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_vision",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/vision",
+ sum = "h1:/CsSTkbmO9HC8iQpxbK8ATms3OQaX3YQUeTMGCxlaK4=",
+ version = "v1.2.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_vision_v2",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/vision/v2",
+ sum = "h1:TQHxRqvLMi19azwm3qYuDbEzZWmiKJNTpGbkNsfRCik=",
+ version = "v2.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_vmmigration",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/vmmigration",
+ sum = "h1:A2Tl2ZmwMRpvEmhV2ibISY85fmQR+Y5w9a0PlRz5P3s=",
+ version = "v1.3.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_vmwareengine",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/vmwareengine",
+ sum = "h1:JMPZaOT/gIUxVlTqSl/QQ32Y2k+r0stNeM1NSqhVP9o=",
+ version = "v0.1.0",
+ )
+
+ go_repository(
+ name = "com_google_cloud_go_vpcaccess",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/vpcaccess",
+ sum = "h1:woHXXtnW8b9gLFdWO9HLPalAddBQ9V4LT+1vjKwR3W8=",
+ version = "v1.5.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_webrisk",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/webrisk",
+ sum = "h1:ypSnpGlJnZSXbN9a13PDmAYvVekBLnGKxQ3Q9SMwnYY=",
+ version = "v1.7.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_websecurityscanner",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/websecurityscanner",
+ sum = "h1:y7yIFg/h/mO+5Y5aCOtVAnpGUOgqCH5rXQ2Oc8Oq2+g=",
+ version = "v1.4.0",
+ )
+ go_repository(
+ name = "com_google_cloud_go_workflows",
+ build_file_proto_mode = "disable",
+ importpath = "cloud.google.com/go/workflows",
+ sum = "h1:7Chpin9p50NTU8Tb7qk+I11U/IwVXmDhEoSsdccvInE=",
+ version = "v1.9.0",
+ )
+
+ go_repository(
+ name = "com_shuralyov_dmitri_gpu_mtl",
+ build_file_proto_mode = "disable_global",
+ importpath = "dmitri.shuralyov.com/gpu/mtl",
+ sum = "h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY=",
+ version = "v0.0.0-20190408044501-666a987793e9",
+ )
+ go_repository(
+ name = "com_sourcegraph_sourcegraph_appdash",
+ build_file_proto_mode = "disable_global",
+ importpath = "sourcegraph.com/sourcegraph/appdash",
+ sum = "h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM=",
+ version = "v0.0.0-20190731080439-ebfcffb1b5c0",
+ )
+ go_repository(
+ name = "com_sourcegraph_sourcegraph_appdash_data",
+ build_file_proto_mode = "disable_global",
+ importpath = "sourcegraph.com/sourcegraph/appdash-data",
+ sum = "h1:e1sMhtVq9AfcEy8AXNb8eSg6gbzfdpYhoNqnPJa+GzI=",
+ version = "v0.0.0-20151005221446-73f23eafcf67",
+ )
+ go_repository(
+ name = "com_stathat_c_consistent",
+ build_file_proto_mode = "disable",
+ importpath = "stathat.com/c/consistent",
+ sum = "h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=",
+ version = "v1.0.0",
+ )
+
+ go_repository(
+ name = "in_gopkg_alecthomas_kingpin_v2",
+ build_file_proto_mode = "disable_global",
+ importpath = "gopkg.in/alecthomas/kingpin.v2",
+ sum = "h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=",
+ version = "v2.2.6",
+ )
+ go_repository(
+ name = "in_gopkg_check_v1",
+ build_file_proto_mode = "disable_global",
+ importpath = "gopkg.in/check.v1",
+ sum = "h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=",
+ version = "v1.0.0-20201130134442-10cb98267c6c",
+ )
+ go_repository(
+ name = "in_gopkg_errgo_v2",
+ build_file_proto_mode = "disable_global",
+ importpath = "gopkg.in/errgo.v2",
+ sum = "h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=",
+ version = "v2.1.0",
+ )
+ go_repository(
+ name = "in_gopkg_fsnotify_fsnotify_v1",
build_file_proto_mode = "disable",
importpath = "gopkg.in/fsnotify/fsnotify.v1",
sum = "h1:2fkCHbPQZNYRAyRyIV9VX0bpRkxIorlQDiYRmufHnhA=",
@@ -3955,8 +5427,8 @@ def go_deps():
name = "in_gopkg_ini_v1",
build_file_proto_mode = "disable_global",
importpath = "gopkg.in/ini.v1",
- sum = "h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI=",
- version = "v1.66.6",
+ sum = "h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=",
+ version = "v1.67.0",
)
go_repository(
name = "in_gopkg_jcmturner_aescts_v1",
@@ -4004,8 +5476,8 @@ def go_deps():
name = "in_gopkg_natefinch_lumberjack_v2",
build_file_proto_mode = "disable_global",
importpath = "gopkg.in/natefinch/lumberjack.v2",
- sum = "h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=",
- version = "v2.0.0",
+ sum = "h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=",
+ version = "v2.2.1",
)
go_repository(
name = "in_gopkg_resty_v1",
@@ -4042,6 +5514,14 @@ def go_deps():
sum = "h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=",
version = "v1.3.6",
)
+ go_repository(
+ name = "io_etcd_go_etcd",
+ build_file_proto_mode = "disable",
+ importpath = "go.etcd.io/etcd",
+ sum = "h1:fqmtdYQlwZ/vKWSz5amW+a4cnjg23ojz5iL7rjf08Wg=",
+ version = "v0.5.0-alpha.5.0.20220915004622-85b640cee793",
+ )
+
go_repository(
name = "io_etcd_go_etcd_api_v3",
build_file_proto_mode = "disable",
@@ -4092,10 +5572,6 @@ def go_deps():
name = "io_etcd_go_etcd_raft_v3",
build_file_proto_mode = "disable_global",
importpath = "go.etcd.io/etcd/raft/v3",
- patch_args = ["-p1"],
- patches = [
- "//build/patches:io_etcd_go_etcd_raft_v3.patch",
- ],
sum = "h1:uCC37qOXqBvKqTGHGyhASsaCsnTuJugl1GvneJNwHWo=",
version = "v3.5.2",
)
@@ -4113,6 +5589,28 @@ def go_deps():
sum = "h1:uk7/uMGVebpBDl+roivowHt6gJ5Fnqwik3syDkoSKdo=",
version = "v3.5.2",
)
+ go_repository(
+ name = "io_gorm_driver_mysql",
+ build_file_proto_mode = "disable",
+ importpath = "gorm.io/driver/mysql",
+ sum = "h1:mA0XRPjIKi4bkE9nv+NKs6qj6QWOchqUSdWOcpd3x1E=",
+ version = "v1.0.6",
+ )
+ go_repository(
+ name = "io_gorm_driver_sqlite",
+ build_file_proto_mode = "disable",
+ importpath = "gorm.io/driver/sqlite",
+ sum = "h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=",
+ version = "v1.1.4",
+ )
+ go_repository(
+ name = "io_gorm_gorm",
+ build_file_proto_mode = "disable",
+ importpath = "gorm.io/gorm",
+ sum = "h1:INieZtn4P2Pw6xPJ8MzT0G4WUOsHq3RhfuDF1M6GW0E=",
+ version = "v1.21.9",
+ )
+
go_repository(
name = "io_k8s_api",
build_file_proto_mode = "disable",
@@ -4141,6 +5639,14 @@ def go_deps():
sum = "h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE=",
version = "v0.3.0",
)
+ go_repository(
+ name = "io_k8s_klog_v2",
+ build_file_proto_mode = "disable",
+ importpath = "k8s.io/klog/v2",
+ sum = "h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=",
+ version = "v2.80.1",
+ )
+
go_repository(
name = "io_k8s_kube_openapi",
build_file_proto_mode = "disable",
@@ -4148,13 +5654,27 @@ def go_deps():
sum = "h1:tHgpQvrWaYfrnC8G4N0Oszw5HHCsZxKilDi2R7HuCSM=",
version = "v0.0.0-20180629012420-d83b052f768a",
)
+ go_repository(
+ name = "io_k8s_sigs_json",
+ build_file_proto_mode = "disable",
+ importpath = "sigs.k8s.io/json",
+ sum = "h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=",
+ version = "v0.0.0-20220713155537-f223a00ba0e2",
+ )
+ go_repository(
+ name = "io_k8s_sigs_structured_merge_diff_v4",
+ build_file_proto_mode = "disable",
+ importpath = "sigs.k8s.io/structured-merge-diff/v4",
+ sum = "h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=",
+ version = "v4.2.3",
+ )
go_repository(
name = "io_k8s_sigs_yaml",
build_file_proto_mode = "disable_global",
importpath = "sigs.k8s.io/yaml",
- sum = "h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=",
- version = "v1.2.0",
+ sum = "h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=",
+ version = "v1.3.0",
)
go_repository(
name = "io_k8s_utils",
@@ -4163,6 +5683,13 @@ def go_deps():
sum = "h1:8r+l4bNWjRlsFYlQJnKJ2p7s1YQPj4XyXiJVqDHRx7c=",
version = "v0.0.0-20190308190857-21c4ce38f2a7",
)
+ go_repository(
+ name = "io_moul_zapgorm2",
+ build_file_proto_mode = "disable",
+ importpath = "moul.io/zapgorm2",
+ sum = "h1:qwAlMBYf+qJkJ7PAzJl4oCe6eS6QGiKAXUPeis0+RBE=",
+ version = "v1.1.0",
+ )
go_repository(
name = "io_opencensus_go",
@@ -4289,8 +5816,8 @@ def go_deps():
name = "org_golang_google_api",
build_file_proto_mode = "disable_global",
importpath = "google.golang.org/api",
- sum = "h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE=",
- version = "v0.74.0",
+ sum = "h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ=",
+ version = "v0.103.0",
)
go_repository(
name = "org_golang_google_appengine",
@@ -4303,15 +5830,15 @@ def go_deps():
name = "org_golang_google_genproto",
build_file_proto_mode = "disable_global",
importpath = "google.golang.org/genproto",
- sum = "h1:0m9wktIpOxGw+SSKmydXWB3Z3GTfcPP6+q75HCQa6HI=",
- version = "v0.0.0-20220324131243-acbaeb5b85eb",
+ sum = "h1:vArvWooPH749rNHpBGgVl+U9B9dATjiEhJzcWGlovNs=",
+ version = "v0.0.0-20230202175211-008b39050e57",
)
go_repository(
name = "org_golang_google_grpc",
build_file_proto_mode = "disable_global",
importpath = "google.golang.org/grpc",
- sum = "h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=",
- version = "v1.45.0",
+ sum = "h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ=",
+ version = "v1.52.3",
)
go_repository(
name = "org_golang_google_grpc_cmd_protoc_gen_go_grpc",
@@ -4331,22 +5858,22 @@ def go_deps():
name = "org_golang_x_crypto",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/crypto",
- sum = "h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=",
- version = "v0.1.0",
+ sum = "h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=",
+ version = "v0.5.0",
)
go_repository(
name = "org_golang_x_exp",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/exp",
- sum = "h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=",
- version = "v0.0.0-20220722155223-a9213eeb770e",
+ sum = "h1:SkwG94eNiiYJhbeDE018Grw09HIN/KB9NlRmZsrzfWs=",
+ version = "v0.0.0-20221023144134-a1e5550cf13e",
)
go_repository(
name = "org_golang_x_exp_typeparams",
build_file_proto_mode = "disable",
importpath = "golang.org/x/exp/typeparams",
- sum = "h1:+W8Qf4iJtMGKkyAygcKohjxTk4JPsL9DpzApJ22m5Ic=",
- version = "v0.0.0-20220613132600-b0d781184e0d",
+ sum = "h1:Jw5wfR+h9mnIYH+OtGT2im5wV1YGGDora5vTv/aa5bE=",
+ version = "v0.0.0-20221208152030-732eee02a75a",
)
go_repository(
@@ -4374,71 +5901,71 @@ def go_deps():
name = "org_golang_x_mod",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/mod",
- sum = "h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=",
- version = "v0.6.0-dev.0.20220419223038-86c51ed26bb4",
+ sum = "h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA=",
+ version = "v0.7.0",
)
go_repository(
name = "org_golang_x_net",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/net",
- sum = "h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=",
- version = "v0.1.0",
+ sum = "h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=",
+ version = "v0.6.0",
)
go_repository(
name = "org_golang_x_oauth2",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/oauth2",
- sum = "h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=",
- version = "v0.0.0-20220411215720-9780585627b5",
+ sum = "h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=",
+ version = "v0.5.0",
)
go_repository(
name = "org_golang_x_sync",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/sync",
- sum = "h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=",
- version = "v0.0.0-20220722155255-886fb9371eb4",
+ sum = "h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=",
+ version = "v0.1.0",
)
go_repository(
name = "org_golang_x_sys",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/sys",
- sum = "h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=",
- version = "v0.1.0",
+ sum = "h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=",
+ version = "v0.5.0",
)
go_repository(
name = "org_golang_x_term",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/term",
- sum = "h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=",
- version = "v0.1.0",
+ sum = "h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=",
+ version = "v0.5.0",
)
go_repository(
name = "org_golang_x_text",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/text",
- sum = "h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=",
- version = "v0.4.0",
+ sum = "h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=",
+ version = "v0.7.0",
)
go_repository(
name = "org_golang_x_time",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/time",
- sum = "h1:M73Iuj3xbbb9Uk1DYhzydthsj6oOd6l9bpuFcNoUvTs=",
- version = "v0.0.0-20220224211638-0e9765cccd65",
+ sum = "h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=",
+ version = "v0.3.0",
)
go_repository(
name = "org_golang_x_tools",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/tools",
- sum = "h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=",
- version = "v0.1.12",
+ sum = "h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4=",
+ version = "v0.5.0",
)
go_repository(
name = "org_golang_x_xerrors",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/xerrors",
- sum = "h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U=",
- version = "v0.0.0-20220411194840-2f41105eb62f",
+ sum = "h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=",
+ version = "v0.0.0-20220907171357-04be3eba64a2",
)
go_repository(
name = "org_gonum_v1_gonum",
@@ -4545,6 +6072,21 @@ def go_deps():
sum = "h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0=",
version = "v1.4.0",
)
+ go_repository(
+ name = "org_uber_go_dig",
+ build_file_proto_mode = "disable",
+ importpath = "go.uber.org/dig",
+ sum = "h1:pJTDXKEhRqBI8W7rU7kwT5EgyRZuSMVSFcZolOvKK9U=",
+ version = "v1.9.0",
+ )
+ go_repository(
+ name = "org_uber_go_fx",
+ build_file_proto_mode = "disable",
+ importpath = "go.uber.org/fx",
+ sum = "h1:+1+3Cz9M0dFMPy9SW9XUIUHye8bnPUm7q7DroNGWYG4=",
+ version = "v1.12.0",
+ )
+
go_repository(
name = "org_uber_go_goleak",
build_file_proto_mode = "disable_global",
@@ -4556,8 +6098,8 @@ def go_deps():
name = "org_uber_go_multierr",
build_file_proto_mode = "disable_global",
importpath = "go.uber.org/multierr",
- sum = "h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=",
- version = "v1.8.0",
+ sum = "h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=",
+ version = "v1.9.0",
)
go_repository(
name = "org_uber_go_tools",
@@ -4570,6 +6112,20 @@ def go_deps():
name = "org_uber_go_zap",
build_file_proto_mode = "disable_global",
importpath = "go.uber.org/zap",
- sum = "h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=",
- version = "v1.23.0",
+ sum = "h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=",
+ version = "v1.24.0",
+ )
+ go_repository(
+ name = "tools_gotest_gotestsum",
+ build_file_proto_mode = "disable",
+ importpath = "gotest.tools/gotestsum",
+ sum = "h1:RwpqwwFKBAa2h+F6pMEGpE707Edld0etUD3GhqqhDNc=",
+ version = "v1.7.0",
+ )
+ go_repository(
+ name = "tools_gotest_v3",
+ build_file_proto_mode = "disable",
+ importpath = "gotest.tools/v3",
+ sum = "h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=",
+ version = "v3.0.3",
)
diff --git a/Dockerfile b/Dockerfile
index 8416ef542a3d3..f3dae2519f53a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,16 +13,24 @@
# limitations under the License.
# Builder image
-FROM alpine:edge as builder
+FROM rockylinux:9 as builder
-ADD . https://raw.githubusercontent.com/njhallett/apk-fastest-mirror/c4ca44caef3385d830fea34df2dbc2ba4a17e021/apk-fastest-mirror.sh ./proxy
-RUN sh ./proxy/apk-fastest-mirror.sh -t 50 && apk add --no-cache git build-base go
+ENV GOLANG_VERSION 1.19.3
+ENV ARCH amd64
+ENV GOLANG_DOWNLOAD_URL https://dl.google.com/go/go$GOLANG_VERSION.linux-$ARCH.tar.gz
+ENV GOPATH /go
+ENV GOROOT /usr/local/go
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+RUN yum update -y && yum groupinstall 'Development Tools' -y \
+ && curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
+ && tar -C /usr/local -xzf golang.tar.gz \
+ && rm golang.tar.gz
COPY . /tidb
ARG GOPROXY
RUN export GOPROXY=${GOPROXY} && cd /tidb && make server
-FROM alpine:latest
+FROM rockylinux:9-minimal
COPY --from=builder /tidb/bin/tidb-server /tidb-server
diff --git a/Makefile b/Makefile
index a01f8c7e7634d..e5da194e75313 100644
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,7 @@ dev: checklist check explaintest gogenerate br_unit_test test_part_parser_dev ut
# Install the check tools.
check-setup:tools/bin/revive
-check: parser_yacc check-parallel lint tidy testSuite errdoc check-bazel-prepare
+check: parser_yacc check-parallel lint tidy testSuite errdoc license check-bazel-prepare
fmt:
@echo "gofmt (simplify)"
@@ -56,6 +56,12 @@ lint:tools/bin/revive
@echo "linting"
@tools/bin/revive -formatter friendly -config tools/check/revive.toml $(FILES_TIDB_TESTS)
+license:
+ bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG) \
+ --run_under="cd $(CURDIR) && " \
+ @com_github_apache_skywalking_eyes//cmd/license-eye:license-eye --run_under="cd $(CURDIR) && " -- -c ./.github/licenserc.yml header check
+
+
tidy:
@echo "go mod tidy"
./tools/check/check-tidy.sh
@@ -106,7 +112,7 @@ explaintest: server_check
@cd cmd/explaintest && ./run-tests.sh -s ../../bin/tidb-server
ddltest:
- @cd cmd/ddltest && $(GO) test -o ../../bin/ddltest -c
+ @cd cmd/ddltest && $(GO) test --tags=deadllock,intest -o ../../bin/ddltest -c
CLEAN_UT_BINARY := find . -name '*.test.bin'| xargs rm
@@ -417,12 +423,12 @@ bazel_test: failpoint-enable bazel_ci_prepare
bazel_coverage_test: failpoint-enable bazel_ci_prepare
- bazel $(BAZEL_GLOBAL_CONFIG) coverage $(BAZEL_CMD_CONFIG) \
- --build_event_json_file=bazel_1.json --@io_bazel_rules_go//go/config:cover_format=go_cover \
+ bazel $(BAZEL_GLOBAL_CONFIG) coverage $(BAZEL_CMD_CONFIG) --test_keep_going=false \
+ --@io_bazel_rules_go//go/config:cover_format=go_cover --define gotags=deadlock,intest \
-- //... -//cmd/... -//tests/graceshutdown/... \
-//tests/globalkilltest/... -//tests/readonlytest/... -//br/pkg/task:task_test -//tests/realtikvtest/...
- bazel $(BAZEL_GLOBAL_CONFIG) coverage $(BAZEL_CMD_CONFIG) \
- --build_event_json_file=bazel_2.json --@io_bazel_rules_go//go/config:cover_format=go_cover --define gotags=featuretag \
+ bazel $(BAZEL_GLOBAL_CONFIG) coverage $(BAZEL_CMD_CONFIG) --test_keep_going=false \
+ --@io_bazel_rules_go//go/config:cover_format=go_cover --define gotags=deadlock,intest,distributereorg \
-- //... -//cmd/... -//tests/graceshutdown/... \
-//tests/globalkilltest/... -//tests/readonlytest/... -//br/pkg/task:task_test -//tests/realtikvtest/...
@@ -455,27 +461,27 @@ bazel_golangcilinter:
-- run $$($(PACKAGE_DIRECTORIES)) --config ./.golangci.yaml
bazel_brietest: failpoint-enable bazel_ci_prepare
- bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv \
+ bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \
-- //tests/realtikvtest/brietest/...
bazel_pessimistictest: failpoint-enable bazel_ci_prepare
- bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv \
+ bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \
-- //tests/realtikvtest/pessimistictest/...
bazel_sessiontest: failpoint-enable bazel_ci_prepare
- bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv \
+ bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \
-- //tests/realtikvtest/sessiontest/...
bazel_statisticstest: failpoint-enable bazel_ci_prepare
- bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv \
+ bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \
-- //tests/realtikvtest/statisticstest/...
bazel_txntest: failpoint-enable bazel_ci_prepare
- bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv \
+ bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \
-- //tests/realtikvtest/txntest/...
bazel_addindextest: failpoint-enable bazel_ci_prepare
- bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv \
+ bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \
-- //tests/realtikvtest/addindextest/...
bazel_lint: bazel_prepare
diff --git a/Makefile.common b/Makefile.common
index e1ff465336c59..3b7caa149f260 100644
--- a/Makefile.common
+++ b/Makefile.common
@@ -109,7 +109,7 @@ DUMPLING_LDFLAGS += -X "github.com/pingcap/tidb/dumpling/cli.GitHash=$(shell git
DUMPLING_LDFLAGS += -X "github.com/pingcap/tidb/dumpling/cli.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)"
DUMPLING_LDFLAGS += -X "github.com/pingcap/tidb/dumpling/cli.GoVersion=$(shell go version)"
-DUMPLING_GOBUILD := CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '$(DUMPLING_LDFLAGS)'
+DUMPLING_GOBUILD := CGO_ENABLED=1 GO111MODULE=on go build -trimpath -ldflags '$(DUMPLING_LDFLAGS)'
DUMPLING_GOTEST := CGO_ENABLED=1 GO111MODULE=on go test -ldflags '$(DUMPLING_LDFLAGS)'
TEST_COVERAGE_DIR := "test_coverage"
diff --git a/README.md b/README.md
index 74a81008de990..0bed69fe5a1bf 100644
--- a/README.md
+++ b/README.md
@@ -1,26 +1,25 @@
-
+
[](https://github.com/pingcap/tidb/blob/master/LICENSE)
[](https://golang.org/)
-[](https://travis-ci.org/pingcap/tidb)
+[](https://prow.tidb.net/?repo=pingcap%2Ftidb&type=postsubmit)
[](https://goreportcard.com/report/github.com/pingcap/tidb)
[](https://github.com/pingcap/tidb/releases)
[](https://github.com/pingcap/tidb/releases)
-[](https://circleci.com/gh/pingcap/tidb)
[](https://codecov.io/gh/pingcap/tidb)
[](https://godoc.org/github.com/pingcap/tidb)
## What is TiDB?
-TiDB ("Ti" stands for Titanium) is an open-source NewSQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability.
+TiDB (/’taɪdiːbi:/, "Ti" stands for Titanium) is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability.
- [Key features](https://docs.pingcap.com/tidb/stable/overview#key-features)
- [Architecture](#architecture)
-- [MySQL Compatibility](https://docs.pingcap.com/tidb/stable/mysql-compatibility)
+- [MySQL compatibility](https://docs.pingcap.com/tidb/stable/mysql-compatibility)
-For more details and latest updates, see [TiDB docs](https://docs.pingcap.com/tidb/stable) and [release notes](https://docs.pingcap.com/tidb/dev/release-notes).
+For more details and latest updates, see [TiDB documentation](https://docs.pingcap.com/tidb/stable) and [release notes](https://docs.pingcap.com/tidb/dev/release-notes).
-For future plans, see [TiDB Roadmap](roadmap.md).
+For future plans, see the [TiDB roadmap](roadmap.md).
## Quick start
@@ -38,40 +37,42 @@ See [TiDB Quick Start Guide](https://docs.pingcap.com/tidb/stable/quick-start-wi
### Start developing TiDB
-See [Get Started](https://pingcap.github.io/tidb-dev-guide/get-started/introduction.html) chapter of [TiDB Dev Guide](https://pingcap.github.io/tidb-dev-guide/index.html).
+See the [Get Started](https://pingcap.github.io/tidb-dev-guide/get-started/introduction.html) chapter of [TiDB Development Guide](https://pingcap.github.io/tidb-dev-guide/index.html).
## Community
-You can join these groups and chats to discuss and ask TiDB related questions:
+You can join the following groups or channels to discuss or ask questions about TiDB, and to keep yourself informed of the latest TiDB updates:
-- [TiDB Internals Forum](https://internals.tidb.io/)
-- [Slack Channel](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-tidb)
-- [TiDB User Group Forum (Chinese)](https://asktug.com)
-
-In addition, you may enjoy following:
-
-- [@PingCAP](https://twitter.com/PingCAP) on Twitter
-- Question tagged [#tidb on StackOverflow](https://stackoverflow.com/questions/tagged/tidb)
-- The PingCAP Team [English Blog](https://en.pingcap.com/blog) and [Chinese Blog](https://pingcap.com/blog-cn/)
+- Seek help when you use TiDB
+ - TiDB Forum: [English](https://ask.pingcap.com/), [Chinese](https://asktug.com)
+ - Slack channels: [#everyone](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-tidb) (English), [#tidb-japan](https://slack.tidb.io/invite?team=tidb-community&channel=tidb-japan&ref=github-tidb) (Japanese)
+ - [Stack Overflow](https://stackoverflow.com/questions/tagged/tidb) (questions tagged with #tidb)
+- Discuss TiDB's implementation and design
+ - [TiDB Internals forum](https://internals.tidb.io/)
+- Get the latest TiDB news or updates
+ - Follow [@PingCAP](https://twitter.com/PingCAP) on Twitter
+ - Read the PingCAP [English Blog](https://www.pingcap.com/blog/?from=en) or [Chinese Blog](https://cn.pingcap.com/blog/)
For support, please contact [PingCAP](http://bit.ly/contact_us_via_github).
## Contributing
-The [community repository](https://github.com/pingcap/community) hosts all information about the TiDB community, including how to contribute to TiDB, how TiDB community is governed, how special interest groups are organized, etc.
+The [community repository](https://github.com/pingcap/community) hosts all information about the TiDB community, including [how to contribute](https://github.com/pingcap/community/blob/master/contributors/README.md) to TiDB, how the TiDB community is governed, how [teams](https://github.com/pingcap/community/blob/master/teams/README.md) are organized.
+
+Contributions are welcomed and greatly appreciated. You can get started with one of the [good first issues](https://github.com/pingcap/tidb/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) or [help wanted issues](https://github.com/pingcap/tidb/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22). For more details on typical contribution workflows, see [Contribute to TiDB](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/introduction.html). For more contributing information about where to start, click the contributor icon below.
[
](https://github.com/pingcap/tidb-map/blob/master/maps/contribution-map.md#tidb-is-an-open-source-distributed-htap-database-compatible-with-the-mysql-protocol)
-Contributions are welcomed and greatly appreciated. All the contributors are welcomed to claim your reward by filing this [form](https://forms.pingcap.com/f/tidb-contribution-swag). See [Contribution to TiDB](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/introduction.html) for details on typical contribution workflows. For more contributing information, click on the contributor icon above.
+Every contributor is welcome to claim your contribution swag by filling in and submitting this [form](https://forms.pingcap.com/f/tidb-contribution-swag).
## Case studies
-- [English](https://pingcap.com/case-studies)
-- [简体中文](https://pingcap.com/cases-cn/)
+- [Case studies in English](https://www.pingcap.com/customers/)
+- [中文用户案例](https://cn.pingcap.com/case/)
## Architecture
-
+
## License
diff --git a/WORKSPACE b/WORKSPACE
index e6df7760f5d5e..6fce0d77d8da1 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -2,23 +2,23 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
http_archive(
name = "io_bazel_rules_go",
- sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa",
+ sha256 = "dd926a88a564a9246713a9c00b35315f54cbd46b31a26d5d8fb264c07045f05d",
urls = [
- "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
- "https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
+ "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.38.1/rules_go-v0.38.1.zip",
+ "https://github.com/bazelbuild/rules_go/releases/download/v0.38.1/rules_go-v0.38.1.zip",
],
)
http_archive(
name = "bazel_gazelle",
- sha256 = "501deb3d5695ab658e82f6f6f549ba681ea3ca2a5fb7911154b5aa45596183fa",
+ sha256 = "ecba0f04f96b4960a5b250c8e8eeec42281035970aa8852dda73098274d14a1d",
urls = [
- "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.26.0/bazel-gazelle-v0.26.0.tar.gz",
- "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.26.0/bazel-gazelle-v0.26.0.tar.gz",
+ "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.29.0/bazel-gazelle-v0.29.0.tar.gz",
+ "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.29.0/bazel-gazelle-v0.29.0.tar.gz",
],
)
-load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+load("@io_bazel_rules_go//go:deps.bzl", "go_download_sdk", "go_register_toolchains", "go_rules_dependencies")
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
load("//:DEPS.bzl", "go_deps")
@@ -27,9 +27,19 @@ go_deps()
go_rules_dependencies()
+go_download_sdk(
+ name = "go_sdk",
+ urls = [
+ "http://ats.apps.svc/golang/{}",
+ "http://bazel-cache.pingcap.net:8080/golang/{}",
+ "https://mirrors.aliyun.com/golang/{}",
+ "https://dl.google.com/go/{}",
+ ],
+ version = "1.19.5",
+)
+
go_register_toolchains(
nogo = "@//build:tidb_nogo",
- version = "1.19.3",
)
gazelle_dependencies()
@@ -48,3 +58,23 @@ http_archive(
load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
protobuf_deps()
+
+http_archive(
+ name = "remote_java_tools",
+ sha256 = "5cd59ea6bf938a1efc1e11ea562d37b39c82f76781211b7cd941a2346ea8484d",
+ urls = [
+ "http://ats.apps.svc/bazel_java_tools/releases/java/v11.9/java_tools-v11.9.zip",
+ "https://mirror.bazel.build/bazel_java_tools/releases/java/v11.9/java_tools-v11.9.zip",
+ "https://github.com/bazelbuild/java_tools/releases/download/java_v11.9/java_tools-v11.9.zip",
+ ],
+)
+
+http_archive(
+ name = "remote_java_tools_linux",
+ sha256 = "512582cac5b7ea7974a77b0da4581b21f546c9478f206eedf54687eeac035989",
+ urls = [
+ "http://ats.apps.svc/bazel_java_tools/releases/java/v11.9/java_tools_linux-v11.9.zip",
+ "https://mirror.bazel.build/bazel_java_tools/releases/java/v11.9/java_tools_linux-v11.9.zip",
+ "https://github.com/bazelbuild/java_tools/releases/download/java_v11.9/java_tools_linux-v11.9.zip",
+ ],
+)
diff --git a/autoid_service/BUILD.bazel b/autoid_service/BUILD.bazel
index df3d361d412ed..6b206586fc6bb 100644
--- a/autoid_service/BUILD.bazel
+++ b/autoid_service/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "autoid_service",
@@ -9,11 +9,14 @@ go_library(
"//config",
"//kv",
"//meta",
+ "//meta/autoid",
"//metrics",
"//owner",
+ "//parser/model",
"//util/logutil",
"//util/mathutil",
"@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/autoid",
"@io_etcd_go_etcd_client_v3//:client",
"@org_golang_google_grpc//:grpc",
@@ -21,3 +24,20 @@ go_library(
"@org_uber_go_zap//:zap",
],
)
+
+go_test(
+ name = "autoid_service_test",
+ timeout = "short",
+ srcs = ["autoid_test.go"],
+ embed = [":autoid_service"],
+ flaky = True,
+ deps = [
+ "//parser/model",
+ "//testkit",
+ "@com_github_pingcap_kvproto//pkg/autoid",
+ "@com_github_stretchr_testify//require",
+ "@io_etcd_go_etcd_tests_v3//integration",
+ "@org_golang_google_grpc//:grpc",
+ "@org_golang_google_grpc//credentials/insecure",
+ ],
+)
diff --git a/autoid_service/autoid.go b/autoid_service/autoid.go
index d20c78fd06098..aa6c487cb0b48 100644
--- a/autoid_service/autoid.go
+++ b/autoid_service/autoid.go
@@ -22,12 +22,15 @@ import (
"time"
"github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/autoid"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
+ autoid1 "github.com/pingcap/tidb/meta/autoid"
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/owner"
+ "github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/mathutil"
clientv3 "go.etcd.io/etcd/client/v3"
@@ -76,7 +79,7 @@ func (alloc *autoIDValue) alloc4Unsigned(ctx context.Context, store kv.Storage,
ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
- idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).RowID()
+ idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).IncrementID(model.TableInfoVersion5)
var err1 error
newBase, err1 = idAcc.Get()
if err1 != nil {
@@ -137,7 +140,7 @@ func (alloc *autoIDValue) alloc4Signed(ctx context.Context,
ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
- idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).RowID()
+ idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).IncrementID(model.TableInfoVersion5)
var err1 error
newBase, err1 = idAcc.Get()
if err1 != nil {
@@ -188,7 +191,7 @@ func (alloc *autoIDValue) rebase4Unsigned(ctx context.Context,
startTime := time.Now()
ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
- idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).RowID()
+ idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).IncrementID(model.TableInfoVersion5)
currentEnd, err1 := idAcc.Get()
if err1 != nil {
return err1
@@ -221,7 +224,7 @@ func (alloc *autoIDValue) rebase4Signed(ctx context.Context, store kv.Storage, d
var newBase, newEnd int64
ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
- idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).RowID()
+ idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).IncrementID(model.TableInfoVersion5)
currentEnd, err1 := idAcc.Get()
if err1 != nil {
return err1
@@ -251,6 +254,7 @@ type Service struct {
func New(selfAddr string, etcdAddr []string, store kv.Storage, tlsConfig *tls.Config) *Service {
cfg := config.GetGlobalConfig()
etcdLogCfg := zap.NewProductionConfig()
+
cli, err := clientv3.New(clientv3.Config{
LogConfig: &etcdLogCfg,
Endpoints: etcdAddr,
@@ -268,9 +272,12 @@ func New(selfAddr string, etcdAddr []string, store kv.Storage, tlsConfig *tls.Co
if err != nil {
panic(err)
}
+ return newWithCli(selfAddr, cli, store)
+}
+func newWithCli(selfAddr string, cli *clientv3.Client, store kv.Storage) *Service {
l := owner.NewOwnerManager(context.Background(), cli, "autoid", selfAddr, autoIDLeaderPath)
- err = l.CampaignOwner()
+ err := l.CampaignOwner()
if err != nil {
panic(err)
}
@@ -297,7 +304,7 @@ func (m *mockClient) Rebase(ctx context.Context, in *autoid.RebaseRequest, opts
var global = make(map[string]*mockClient)
// MockForTest is used for testing, the UT test and unistore use this.
-func MockForTest(store kv.Storage) *mockClient {
+func MockForTest(store kv.Storage) autoid.AutoIDAllocClient {
uuid := store.UUID()
ret, ok := global[uuid]
if !ok {
@@ -400,16 +407,44 @@ func (s *Service) getAlloc(dbID, tblID int64, isUnsigned bool) *autoIDValue {
func (s *Service) allocAutoID(ctx context.Context, req *autoid.AutoIDRequest) (*autoid.AutoIDResponse, error) {
if s.leaderShip != nil && !s.leaderShip.IsOwner() {
+ logutil.BgLogger().Info("[autoid service] Alloc AutoID fail, not leader")
return nil, errors.New("not leader")
}
+ failpoint.Inject("mockErr", func(val failpoint.Value) {
+ if val.(bool) {
+ failpoint.Return(nil, errors.New("mock reload failed"))
+ }
+ })
+
val := s.getAlloc(req.DbID, req.TblID, req.IsUnsigned)
- if req.N == 0 && val.base != 0 {
- base := val.base
+ if req.N == 0 {
+ if val.base != 0 {
+ return &autoid.AutoIDResponse{
+ Min: val.base,
+ Max: val.base,
+ }, nil
+ }
+ // This item is not initialized, get the data from remote.
+ var currentEnd int64
+ ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
+ err := kv.RunInNewTxn(ctx, s.store, true, func(ctx context.Context, txn kv.Transaction) error {
+ idAcc := meta.NewMeta(txn).GetAutoIDAccessors(req.DbID, req.TblID).RowID()
+ var err1 error
+ currentEnd, err1 = idAcc.Get()
+ if err1 != nil {
+ return err1
+ }
+ val.end = currentEnd
+ return nil
+ })
+ if err != nil {
+ return &autoid.AutoIDResponse{Errmsg: []byte(err.Error())}, nil
+ }
return &autoid.AutoIDResponse{
- Min: base,
- Max: base,
+ Min: currentEnd,
+ Max: currentEnd,
}, nil
}
@@ -424,16 +459,19 @@ func (s *Service) allocAutoID(ctx context.Context, req *autoid.AutoIDRequest) (*
min, max, err = val.alloc4Signed(ctx, s.store, req.DbID, req.TblID, req.IsUnsigned, req.N, req.Increment, req.Offset)
}
+ if err != nil {
+ return &autoid.AutoIDResponse{Errmsg: []byte(err.Error())}, nil
+ }
return &autoid.AutoIDResponse{
Min: min,
Max: max,
- }, err
+ }, nil
}
func (alloc *autoIDValue) forceRebase(ctx context.Context, store kv.Storage, dbID, tblID, requiredBase int64, isUnsigned bool) error {
ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
- idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).RowID()
+ idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).IncrementID(model.TableInfoVersion5)
currentEnd, err1 := idAcc.Get()
if err1 != nil {
return err1
@@ -459,6 +497,7 @@ func (alloc *autoIDValue) forceRebase(ctx context.Context, store kv.Storage, dbI
// req.N = 0 is handled specially, it is used to return the current auto ID value.
func (s *Service) Rebase(ctx context.Context, req *autoid.RebaseRequest) (*autoid.RebaseResponse, error) {
if s.leaderShip != nil && !s.leaderShip.IsOwner() {
+ logutil.BgLogger().Info("[autoid service] Rebase() fail, not leader")
return nil, errors.New("not leader")
}
@@ -466,7 +505,7 @@ func (s *Service) Rebase(ctx context.Context, req *autoid.RebaseRequest) (*autoi
if req.Force {
err := val.forceRebase(ctx, s.store, req.DbID, req.TblID, req.Base, req.IsUnsigned)
if err != nil {
- return nil, errors.Trace(err)
+ return &autoid.RebaseResponse{Errmsg: []byte(err.Error())}, nil
}
}
@@ -476,5 +515,12 @@ func (s *Service) Rebase(ctx context.Context, req *autoid.RebaseRequest) (*autoi
} else {
err = val.rebase4Signed(ctx, s.store, req.DbID, req.TblID, req.Base)
}
- return &autoid.RebaseResponse{}, err
+ if err != nil {
+ return &autoid.RebaseResponse{Errmsg: []byte(err.Error())}, nil
+ }
+ return &autoid.RebaseResponse{}, nil
+}
+
+func init() {
+ autoid1.MockForTest = MockForTest
}
diff --git a/autoid_service/autoid_test.go b/autoid_service/autoid_test.go
new file mode 100644
index 0000000000000..df2722309cf6e
--- /dev/null
+++ b/autoid_service/autoid_test.go
@@ -0,0 +1,202 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package autoid
+
+import (
+ "context"
+ "fmt"
+ "math"
+ "net"
+ "testing"
+ "time"
+
+ "github.com/pingcap/kvproto/pkg/autoid"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/stretchr/testify/require"
+ "go.etcd.io/etcd/tests/v3/integration"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+)
+
+type autoIDResp struct {
+ *autoid.AutoIDResponse
+ error
+ *testing.T
+}
+
+func (resp autoIDResp) check(min, max int64) {
+ require.NoError(resp.T, resp.error)
+ require.Equal(resp.T, resp.AutoIDResponse, &autoid.AutoIDResponse{Min: min, Max: max})
+}
+
+func (resp autoIDResp) checkErrmsg() {
+ require.NoError(resp.T, resp.error)
+ require.True(resp.T, len(resp.GetErrmsg()) > 0)
+}
+
+type rebaseResp struct {
+ *autoid.RebaseResponse
+ error
+ *testing.T
+}
+
+func (resp rebaseResp) check(msg string) {
+ require.NoError(resp.T, resp.error)
+ require.Equal(resp.T, string(resp.RebaseResponse.GetErrmsg()), msg)
+}
+
+func TestAPI(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ cli := MockForTest(store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t (id int key auto_increment);")
+ is := dom.InfoSchema()
+ dbInfo, ok := is.SchemaByName(model.NewCIStr("test"))
+ require.True(t, ok)
+
+ tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+ require.NoError(t, err)
+ tbInfo := tbl.Meta()
+
+ ctx := context.Background()
+ checkCurrValue := func(t *testing.T, cli autoid.AutoIDAllocClient, min, max int64) {
+ req := &autoid.AutoIDRequest{DbID: dbInfo.ID, TblID: tbInfo.ID, N: 0}
+ resp, err := cli.AllocAutoID(ctx, req)
+ require.NoError(t, err)
+ require.Equal(t, resp, &autoid.AutoIDResponse{Min: min, Max: max})
+ }
+ autoIDRequest := func(t *testing.T, cli autoid.AutoIDAllocClient, unsigned bool, n uint64, more ...int64) autoIDResp {
+ increment := int64(1)
+ offset := int64(1)
+ if len(more) >= 1 {
+ increment = more[0]
+ }
+ if len(more) >= 2 {
+ offset = more[1]
+ }
+ req := &autoid.AutoIDRequest{DbID: dbInfo.ID, TblID: tbInfo.ID, IsUnsigned: unsigned, N: n, Increment: increment, Offset: offset}
+ resp, err := cli.AllocAutoID(ctx, req)
+ return autoIDResp{resp, err, t}
+ }
+ rebaseRequest := func(t *testing.T, cli autoid.AutoIDAllocClient, unsigned bool, n int64, force ...struct{}) rebaseResp {
+ req := &autoid.RebaseRequest{
+ DbID: dbInfo.ID,
+ TblID: tbInfo.ID,
+ Base: n,
+ IsUnsigned: unsigned,
+ Force: len(force) > 0,
+ }
+ resp, err := cli.Rebase(ctx, req)
+ return rebaseResp{resp, err, t}
+ }
+ var force = struct{}{}
+
+ // basic auto id operation
+ autoIDRequest(t, cli, false, 1).check(0, 1)
+ autoIDRequest(t, cli, false, 10).check(1, 11)
+ checkCurrValue(t, cli, 11, 11)
+ autoIDRequest(t, cli, false, 128).check(11, 139)
+ autoIDRequest(t, cli, false, 1, 10, 5).check(139, 145)
+
+ // basic rebase operation
+ rebaseRequest(t, cli, false, 666).check("")
+ autoIDRequest(t, cli, false, 1).check(666, 667)
+
+ rebaseRequest(t, cli, false, 6666).check("")
+ autoIDRequest(t, cli, false, 1).check(6666, 6667)
+
+ // rebase will not decrease the value without 'force'
+ rebaseRequest(t, cli, false, 44).check("")
+ checkCurrValue(t, cli, 6667, 6667)
+ rebaseRequest(t, cli, false, 44, force).check("")
+ checkCurrValue(t, cli, 44, 44)
+
+ // max increase 1
+ rebaseRequest(t, cli, false, math.MaxInt64, force).check("")
+ checkCurrValue(t, cli, math.MaxInt64, math.MaxInt64)
+ autoIDRequest(t, cli, false, 1).checkErrmsg()
+
+ rebaseRequest(t, cli, true, 0, force).check("")
+ checkCurrValue(t, cli, 0, 0)
+ autoIDRequest(t, cli, true, 1).check(0, 1)
+ autoIDRequest(t, cli, true, 10).check(1, 11)
+ autoIDRequest(t, cli, true, 128).check(11, 139)
+ autoIDRequest(t, cli, true, 1, 10, 5).check(139, 145)
+
+ // max increase 1
+ rebaseRequest(t, cli, true, math.MaxInt64).check("")
+ checkCurrValue(t, cli, math.MaxInt64, math.MaxInt64)
+ autoIDRequest(t, cli, true, 1).check(math.MaxInt64, math.MinInt64)
+ autoIDRequest(t, cli, true, 1).check(math.MinInt64, math.MinInt64+1)
+
+ rebaseRequest(t, cli, true, -1).check("")
+ checkCurrValue(t, cli, -1, -1)
+ autoIDRequest(t, cli, true, 1).check(-1, 0)
+}
+
+func TestGRPC(t *testing.T) {
+ integration.BeforeTestExternal(t)
+ store := testkit.CreateMockStore(t)
+ cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+ defer cluster.Terminate(t)
+ etcdCli := cluster.RandClient()
+
+ var addr string
+ var listener net.Listener
+ for port := 10080; ; port++ {
+ var err error
+ addr = fmt.Sprintf("127.0.0.1:%d", port)
+ listener, err = net.Listen("tcp", addr)
+ if err == nil {
+ break
+ }
+ }
+ defer listener.Close()
+
+ service := newWithCli(addr, etcdCli, store)
+ defer service.Close()
+
+ var i int
+ for !service.leaderShip.IsOwner() {
+ time.Sleep(100 * time.Millisecond)
+ i++
+ if i >= 20 {
+ break
+ }
+ }
+ require.Less(t, i, 20)
+
+ grpcServer := grpc.NewServer()
+ autoid.RegisterAutoIDAllocServer(grpcServer, service)
+ go func() {
+ grpcServer.Serve(listener)
+ }()
+ defer grpcServer.Stop()
+
+ grpcConn, err := grpc.Dial("127.0.0.1:10080", grpc.WithTransportCredentials(insecure.NewCredentials()))
+ require.NoError(t, err)
+ cli := autoid.NewAutoIDAllocClient(grpcConn)
+ _, err = cli.AllocAutoID(context.Background(), &autoid.AutoIDRequest{
+ DbID: 0,
+ TblID: 0,
+ N: 1,
+ Increment: 1,
+ Offset: 1,
+ IsUnsigned: false,
+ })
+ require.NoError(t, err)
+}
diff --git a/bindinfo/BUILD.bazel b/bindinfo/BUILD.bazel
index 93b55bfab21c2..64a11f86d5556 100644
--- a/bindinfo/BUILD.bazel
+++ b/bindinfo/BUILD.bazel
@@ -34,7 +34,7 @@ go_library(
"//util/memory",
"//util/parser",
"//util/sqlexec",
- "//util/stmtsummary",
+ "//util/stmtsummary/v2:stmtsummary",
"//util/table-filter",
"//util/timeutil",
"@org_golang_x_exp//maps",
diff --git a/bindinfo/bind_cache.go b/bindinfo/bind_cache.go
index 8ce69deedd840..fe67cbbbf9f43 100644
--- a/bindinfo/bind_cache.go
+++ b/bindinfo/bind_cache.go
@@ -146,6 +146,23 @@ func (c *bindCache) GetBindRecord(hash, normdOrigSQL, db string) *BindRecord {
return nil
}
+// GetBindRecordBySQLDigest gets the BindRecord from the cache.
+// The return value is not read-only, but it shouldn't be changed in the caller functions.
+// The function is thread-safe.
+func (c *bindCache) GetBindRecordBySQLDigest(sqlDigest string) (*BindRecord, error) {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+ bindings := c.get(bindCacheKey(sqlDigest))
+ if len(bindings) > 1 {
+ // currently, we only allow one binding for a sql
+ return nil, errors.New("more than 1 binding matched")
+ }
+ if len(bindings) == 0 || len(bindings[0].Bindings) == 0 {
+ return nil, errors.New("can't find any binding for '" + sqlDigest + "'")
+ }
+ return bindings[0], nil
+}
+
// GetAllBindRecords return all the bindRecords from the bindCache.
// The return value is not read-only, but it shouldn't be changed in the caller functions.
// The function is thread-safe.
diff --git a/bindinfo/bind_record.go b/bindinfo/bind_record.go
index 63517d91ac189..50e8e0ba20784 100644
--- a/bindinfo/bind_record.go
+++ b/bindinfo/bind_record.go
@@ -54,6 +54,8 @@ const (
Evolve = "evolve"
// Builtin indicates the binding is a builtin record for internal locking purpose. It is also the status for the builtin binding.
Builtin = "builtin"
+ // History indicate the binding is created from statement summary by plan digest
+ History = "history"
)
// Binding stores the basic bind hint info.
@@ -71,7 +73,9 @@ type Binding struct {
// Hint is the parsed hints, it is used to bind hints to stmt node.
Hint *hint.HintsSet `json:"-"`
// ID is the string form of Hint. It would be non-empty only when the status is `Using` or `PendingVerify`.
- ID string `json:"-"`
+ ID string `json:"-"`
+ SQLDigest string
+ PlanDigest string
}
func (b *Binding) isSame(rb *Binding) bool {
@@ -111,6 +115,17 @@ type BindRecord struct {
Bindings []Binding
}
+// Copy get the copy of bindRecord
+func (br *BindRecord) Copy() *BindRecord {
+ nbr := &BindRecord{
+ OriginalSQL: br.OriginalSQL,
+ Db: br.Db,
+ }
+ nbr.Bindings = make([]Binding, len(br.Bindings))
+ copy(nbr.Bindings, br.Bindings)
+ return nbr
+}
+
// HasEnabledBinding checks if there are any enabled bindings in bind record.
func (br *BindRecord) HasEnabledBinding() bool {
for _, binding := range br.Bindings {
diff --git a/bindinfo/bind_test.go b/bindinfo/bind_test.go
index 07e29923a9790..cb8e62687df8f 100644
--- a/bindinfo/bind_test.go
+++ b/bindinfo/bind_test.go
@@ -29,6 +29,8 @@ import (
"github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/util"
+ utilparser "github.com/pingcap/tidb/util/parser"
+ "github.com/pingcap/tidb/util/stmtsummary"
"github.com/stretchr/testify/require"
)
@@ -1241,3 +1243,148 @@ func TestGCBindRecord(t *testing.T) {
tk.MustQuery("show global bindings").Check(testkit.Rows())
tk.MustQuery("select status from mysql.bind_info where original_sql = 'select * from `test` . `t` where `a` = ?'").Check(testkit.Rows())
}
+
+func TestBindSQLDigest(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(pk int primary key, a int, b int, key(a), key(b))")
+
+ cases := []struct {
+ origin string
+ hint string
+ }{
+ // agg hints
+ {"select count(1) from t", "select /*+ hash_agg() */ count(1) from t"},
+ {"select count(1) from t", "select /*+ stream_agg() */ count(1) from t"},
+ // join hints
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ merge_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ tidb_smj(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ hash_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ tidb_hj(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ inl_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ tidb_inlj(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ inl_hash_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ // index hints
+ {"select * from t", "select * from t use index(primary)"},
+ {"select * from t", "select /*+ use_index(primary) */ * from t"},
+ {"select * from t", "select * from t use index(a)"},
+ {"select * from t", "select /*+ use_index(a) */ * from t use index(a)"},
+ {"select * from t", "select * from t use index(b)"},
+ {"select * from t", "select /*+ use_index(b) */ * from t use index(b)"},
+ {"select a, b from t where a=1 or b=1", "select /*+ use_index_merge(t, a, b) */ a, b from t where a=1 or b=1"},
+ {"select * from t where a=1", "select /*+ ignore_index(t, a) */ * from t where a=1"},
+ // push-down hints
+ {"select * from t limit 10", "select /*+ limit_to_cop() */ * from t limit 10"},
+ {"select a, count(*) from t group by a", "select /*+ agg_to_cop() */ a, count(*) from t group by a"},
+ // index-merge hints
+ {"select a, b from t where a>1 or b>1", "select /*+ no_index_merge() */ a, b from t where a>1 or b>1"},
+ {"select a, b from t where a>1 or b>1", "select /*+ use_index_merge(t, a, b) */ a, b from t where a>1 or b>1"},
+ // runtime hints
+ {"select * from t", "select /*+ memory_quota(1024 MB) */ * from t"},
+ {"select * from t", "select /*+ max_execution_time(1000) */ * from t"},
+ // storage hints
+ {"select * from t", "select /*+ read_from_storage(tikv[t]) */ * from t"},
+ // others
+ {"select t1.a, t1.b from t t1 where t1.a in (select t2.a from t t2)", "select /*+ use_toja(true) */ t1.a, t1.b from t t1 where t1.a in (select t2.a from t t2)"},
+ }
+ for _, c := range cases {
+ stmtsummary.StmtSummaryByDigestMap.Clear()
+ utilCleanBindingEnv(tk, dom)
+ sql := "create global binding for " + c.origin + " using " + c.hint
+ tk.MustExec(sql)
+ res := tk.MustQuery(`show global bindings`).Rows()
+ require.Equal(t, len(res[0]), 11)
+
+ parser4binding := parser.New()
+ originNode, err := parser4binding.ParseOneStmt(c.origin, "utf8mb4", "utf8mb4_general_ci")
+ require.NoError(t, err)
+ _, sqlDigestWithDB := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(originNode, "test", c.origin))
+ require.Equal(t, res[0][9], sqlDigestWithDB.String())
+ }
+}
+
+func TestDropBindBySQLDigest(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(pk int primary key, a int, b int, key(a), key(b))")
+
+ cases := []struct {
+ origin string
+ hint string
+ }{
+ // agg hints
+ {"select count(1) from t", "select /*+ hash_agg() */ count(1) from t"},
+ {"select count(1) from t", "select /*+ stream_agg() */ count(1) from t"},
+ // join hints
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ merge_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ tidb_smj(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ hash_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ tidb_hj(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ inl_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ tidb_inlj(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ {"select * from t t1, t t2 where t1.a=t2.a", "select /*+ inl_hash_join(t1, t2) */ * from t t1, t t2 where t1.a=t2.a"},
+ // index hints
+ {"select * from t", "select * from t use index(primary)"},
+ {"select * from t", "select /*+ use_index(primary) */ * from t"},
+ {"select * from t", "select * from t use index(a)"},
+ {"select * from t", "select /*+ use_index(a) */ * from t use index(a)"},
+ {"select * from t", "select * from t use index(b)"},
+ {"select * from t", "select /*+ use_index(b) */ * from t use index(b)"},
+ {"select a, b from t where a=1 or b=1", "select /*+ use_index_merge(t, a, b) */ a, b from t where a=1 or b=1"},
+ {"select * from t where a=1", "select /*+ ignore_index(t, a) */ * from t where a=1"},
+ // push-down hints
+ {"select * from t limit 10", "select /*+ limit_to_cop() */ * from t limit 10"},
+ {"select a, count(*) from t group by a", "select /*+ agg_to_cop() */ a, count(*) from t group by a"},
+ // index-merge hints
+ {"select a, b from t where a>1 or b>1", "select /*+ no_index_merge() */ a, b from t where a>1 or b>1"},
+ {"select a, b from t where a>1 or b>1", "select /*+ use_index_merge(t, a, b) */ a, b from t where a>1 or b>1"},
+ // runtime hints
+ {"select * from t", "select /*+ memory_quota(1024 MB) */ * from t"},
+ {"select * from t", "select /*+ max_execution_time(1000) */ * from t"},
+ // storage hints
+ {"select * from t", "select /*+ read_from_storage(tikv[t]) */ * from t"},
+ // others
+ {"select t1.a, t1.b from t t1 where t1.a in (select t2.a from t t2)", "select /*+ use_toja(true) */ t1.a, t1.b from t t1 where t1.a in (select t2.a from t t2)"},
+ }
+
+ h := dom.BindHandle()
+ // global scope
+ for _, c := range cases {
+ utilCleanBindingEnv(tk, dom)
+ sql := "create global binding for " + c.origin + " using " + c.hint
+ tk.MustExec(sql)
+ h.ReloadBindings()
+ res := tk.MustQuery(`show global bindings`).Rows()
+
+ require.Equal(t, len(res), 1)
+ require.Equal(t, len(res[0]), 11)
+ drop := fmt.Sprintf("drop global binding for sql digest '%s'", res[0][9])
+ tk.MustExec(drop)
+ require.NoError(t, h.GCBindRecord())
+ h.ReloadBindings()
+ tk.MustQuery("show global bindings").Check(testkit.Rows())
+ }
+
+ // session scope
+ for _, c := range cases {
+ utilCleanBindingEnv(tk, dom)
+ sql := "create binding for " + c.origin + " using " + c.hint
+ tk.MustExec(sql)
+ res := tk.MustQuery(`show bindings`).Rows()
+
+ require.Equal(t, len(res), 1)
+ require.Equal(t, len(res[0]), 11)
+ drop := fmt.Sprintf("drop binding for sql digest '%s'", res[0][9])
+ tk.MustExec(drop)
+ require.NoError(t, h.GCBindRecord())
+ tk.MustQuery("show bindings").Check(testkit.Rows())
+ }
+
+ // exception cases
+ tk.MustGetErrMsg(fmt.Sprintf("drop binding for sql digest '%s'", "1"), "can't find any binding for '1'")
+ tk.MustGetErrMsg(fmt.Sprintf("drop binding for sql digest '%s'", ""), "sql digest is empty")
+}
diff --git a/bindinfo/capture_test.go b/bindinfo/capture_test.go
index bff6b01045c0b..21b31c883e582 100644
--- a/bindinfo/capture_test.go
+++ b/bindinfo/capture_test.go
@@ -22,9 +22,11 @@ import (
"github.com/pingcap/tidb/bindinfo"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/domain"
+ "github.com/pingcap/tidb/parser"
"github.com/pingcap/tidb/parser/auth"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/testkit"
+ utilparser "github.com/pingcap/tidb/util/parser"
"github.com/pingcap/tidb/util/stmtsummary"
"github.com/stretchr/testify/require"
"go.opencensus.io/stats/view"
@@ -397,7 +399,7 @@ func TestConcurrentCapture(t *testing.T) {
// Simulate an existing binding generated by concurrent CREATE BINDING, which has not been synchronized to current tidb-server yet.
// Actually, it is more common to be generated by concurrent baseline capture, I use Manual just for simpler test verification.
tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t`', 'select * from `test` . `t`', '', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustQuery("select original_sql, source from mysql.bind_info where source != 'builtin'").Check(testkit.Rows(
"select * from `test` . `t` manual",
))
@@ -432,8 +434,9 @@ func TestUpdateSubqueryCapture(t *testing.T) {
rows := tk.MustQuery("show global bindings").Rows()
require.Len(t, rows, 1)
bindSQL := "UPDATE /*+ hash_join(@`upd_1` `test`.`t1`), use_index(@`upd_1` `test`.`t1` `idx_b`), use_index(@`sel_1` `test`.`t2` ), use_index(@`sel_2` `test`.`t2` )*/ `test`.`t1` SET `b`=1 WHERE `b` = 2 AND (`a` IN (SELECT `a` FROM `test`.`t2` WHERE `b` = 1) OR `c` IN (SELECT `a` FROM `test`.`t2` WHERE `b` = 1))"
+ originSQL := "UPDATE `test`.`t1` SET `b`=1 WHERE `b` = 2 AND (`a` IN (SELECT `a` FROM `test`.`t2` WHERE `b` = 1) OR `c` IN (SELECT `a` FROM `test`.`t2` WHERE `b` = 1))"
require.Equal(t, bindSQL, rows[0][1])
- tk.MustExec(bindSQL)
+ tk.MustExec(originSQL)
require.Len(t, tk.Session().GetSessionVars().StmtCtx.GetWarnings(), 0)
}
@@ -1011,5 +1014,11 @@ func TestCaptureHints(t *testing.T) {
res := tk.MustQuery(`show global bindings`).Rows()
require.Equal(t, len(res), 1) // this query is captured, and
require.True(t, strings.Contains(res[0][1].(string), capCase.hint)) // the binding contains the expected hint
+ // test sql digest
+ parser4binding := parser.New()
+ originNode, err := parser4binding.ParseOneStmt(capCase.query, "utf8mb4", "utf8mb4_general_ci")
+ require.NoError(t, err)
+ _, sqlDigestWithDB := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(originNode, "test", capCase.query))
+ require.Equal(t, res[0][9], sqlDigestWithDB.String())
}
}
diff --git a/bindinfo/handle.go b/bindinfo/handle.go
index f4a9d372e9d85..8501dcfc9939e 100644
--- a/bindinfo/handle.go
+++ b/bindinfo/handle.go
@@ -41,7 +41,7 @@ import (
"github.com/pingcap/tidb/util/logutil"
utilparser "github.com/pingcap/tidb/util/parser"
"github.com/pingcap/tidb/util/sqlexec"
- "github.com/pingcap/tidb/util/stmtsummary"
+ stmtsummaryv2 "github.com/pingcap/tidb/util/stmtsummary/v2"
tablefilter "github.com/pingcap/tidb/util/table-filter"
"github.com/pingcap/tidb/util/timeutil"
"go.uber.org/zap"
@@ -146,7 +146,7 @@ func (h *BindHandle) Update(fullLoad bool) (err error) {
ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBindInfo)
// No need to acquire the session context lock for ExecRestrictedSQL, it
// uses another background session.
- rows, _, err := exec.ExecRestrictedSQL(ctx, nil, `SELECT original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source
+ rows, _, err := exec.ExecRestrictedSQL(ctx, nil, `SELECT original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest
FROM mysql.bind_info WHERE update_time > %? ORDER BY update_time, create_time`, updateTime)
if err != nil {
@@ -261,7 +261,7 @@ func (h *BindHandle) CreateBindRecord(sctx sessionctx.Context, record *BindRecor
record.Bindings[i].UpdateTime = now
// Insert the BindRecord to the storage.
- _, err = exec.ExecuteInternal(ctx, `INSERT INTO mysql.bind_info VALUES (%?,%?, %?, %?, %?, %?, %?, %?, %?)`,
+ _, err = exec.ExecuteInternal(ctx, `INSERT INTO mysql.bind_info VALUES (%?,%?, %?, %?, %?, %?, %?, %?, %?, %?, %?)`,
record.OriginalSQL,
record.Bindings[i].BindSQL,
record.Db,
@@ -271,6 +271,8 @@ func (h *BindHandle) CreateBindRecord(sctx sessionctx.Context, record *BindRecor
record.Bindings[i].Charset,
record.Bindings[i].Collation,
record.Bindings[i].Source,
+ record.Bindings[i].SQLDigest,
+ record.Bindings[i].PlanDigest,
)
if err != nil {
return err
@@ -349,8 +351,18 @@ func (h *BindHandle) AddBindRecord(sctx sessionctx.Context, record *BindRecord)
}
record.Bindings[i].UpdateTime = now
+ if record.Bindings[i].SQLDigest == "" {
+ parser4binding := parser.New()
+ var originNode ast.StmtNode
+ originNode, err = parser4binding.ParseOneStmt(record.OriginalSQL, record.Bindings[i].Charset, record.Bindings[i].Collation)
+ if err != nil {
+ return err
+ }
+ _, sqlDigestWithDB := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(originNode, record.Db, record.OriginalSQL))
+ record.Bindings[i].SQLDigest = sqlDigestWithDB.String()
+ }
// Insert the BindRecord to the storage.
- _, err = exec.ExecuteInternal(ctx, `INSERT INTO mysql.bind_info VALUES (%?, %?, %?, %?, %?, %?, %?, %?, %?)`,
+ _, err = exec.ExecuteInternal(ctx, `INSERT INTO mysql.bind_info VALUES (%?, %?, %?, %?, %?, %?, %?, %?, %?, %?, %?)`,
record.OriginalSQL,
record.Bindings[i].BindSQL,
record.Db,
@@ -360,6 +372,8 @@ func (h *BindHandle) AddBindRecord(sctx sessionctx.Context, record *BindRecord)
record.Bindings[i].Charset,
record.Bindings[i].Collation,
record.Bindings[i].Source,
+ record.Bindings[i].SQLDigest,
+ record.Bindings[i].PlanDigest,
)
if err != nil {
return err
@@ -423,6 +437,15 @@ func (h *BindHandle) DropBindRecord(originalSQL, db string, binding *Binding) (d
return h.sctx.Context.GetSessionVars().StmtCtx.AffectedRows(), nil
}
+// DropBindRecordByDigest drop BindRecord to the storage and BindRecord int the cache.
+func (h *BindHandle) DropBindRecordByDigest(sqlDigest string) (deletedRows uint64, err error) {
+ oldRecord, err := h.GetBindRecordBySQLDigest(sqlDigest)
+ if err != nil {
+ return 0, err
+ }
+ return h.DropBindRecord(oldRecord.OriginalSQL, strings.ToLower(oldRecord.Db), nil)
+}
+
// SetBindRecordStatus set a BindRecord's status to the storage and bind cache.
func (h *BindHandle) SetBindRecordStatus(originalSQL string, binding *Binding, newStatus string) (ok bool, err error) {
h.bindInfo.Lock()
@@ -510,6 +533,15 @@ func (h *BindHandle) SetBindRecordStatus(originalSQL string, binding *Binding, n
return
}
+// SetBindRecordStatusByDigest set a BindRecord's status to the storage and bind cache.
+func (h *BindHandle) SetBindRecordStatusByDigest(newStatus, sqlDigest string) (ok bool, err error) {
+ oldRecord, err := h.GetBindRecordBySQLDigest(sqlDigest)
+ if err != nil {
+ return false, err
+ }
+ return h.SetBindRecordStatus(oldRecord.OriginalSQL, nil, newStatus)
+}
+
// GCBindRecord physically removes the deleted bind records in mysql.bind_info.
func (h *BindHandle) GCBindRecord() (err error) {
h.bindInfo.Lock()
@@ -644,6 +676,11 @@ func (h *BindHandle) GetBindRecord(hash, normdOrigSQL, db string) *BindRecord {
return h.bindInfo.Load().(*bindCache).GetBindRecord(hash, normdOrigSQL, db)
}
+// GetBindRecordBySQLDigest returns the BindRecord of the sql digest.
+func (h *BindHandle) GetBindRecordBySQLDigest(sqlDigest string) (*BindRecord, error) {
+ return h.bindInfo.Load().(*bindCache).GetBindRecordBySQLDigest(sqlDigest)
+}
+
// GetAllBindRecord returns all bind records in cache.
func (h *BindHandle) GetAllBindRecord() (bindRecords []*BindRecord) {
return h.bindInfo.Load().(*bindCache).GetAllBindRecords()
@@ -680,6 +717,8 @@ func (h *BindHandle) newBindRecord(row chunk.Row) (string, *BindRecord, error) {
Charset: row.GetString(6),
Collation: row.GetString(7),
Source: row.GetString(8),
+ SQLDigest: row.GetString(9),
+ PlanDigest: row.GetString(10),
}
bindRecord := &BindRecord{
OriginalSQL: row.GetString(0),
@@ -853,7 +892,7 @@ func (h *BindHandle) CaptureBaselines() {
parser4Capture := parser.New()
captureFilter := h.extractCaptureFilterFromStorage()
emptyCaptureFilter := captureFilter.isEmpty()
- bindableStmts := stmtsummary.StmtSummaryByDigestMap.GetMoreThanCntBindableStmt(captureFilter.frequency)
+ bindableStmts := stmtsummaryv2.GetMoreThanCntBindableStmt(captureFilter.frequency)
for _, bindableStmt := range bindableStmts {
stmt, err := parser4Capture.ParseOneStmt(bindableStmt.Query, bindableStmt.Charset, bindableStmt.Collation)
if err != nil {
@@ -900,6 +939,7 @@ func (h *BindHandle) CaptureBaselines() {
Charset: charset,
Collation: collation,
Source: Capture,
+ SQLDigest: digest.String(),
}
// We don't need to pass the `sctx` because the BindSQL has been validated already.
err = h.CreateBindRecord(nil, &BindRecord{OriginalSQL: normalizedSQL, Db: dbName, Bindings: []Binding{binding}})
@@ -940,12 +980,12 @@ func getHintsForSQL(sctx sessionctx.Context, sql string) (string, error) {
}
// GenerateBindSQL generates binding sqls from stmt node and plan hints.
-func GenerateBindSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string, captured bool, defaultDB string) string {
+func GenerateBindSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string, skipCheckIfHasParam bool, defaultDB string) string {
// If would be nil for very simple cases such as point get, we do not need to evolve for them.
if planHint == "" {
return ""
}
- if !captured {
+ if !skipCheckIfHasParam {
paramChecker := ¶mMarkerChecker{}
stmtNode.Accept(paramChecker)
// We need to evolve on current sql, but we cannot restore values for paramMarkers yet,
@@ -1121,6 +1161,7 @@ func (h *BindHandle) getRunningDuration(sctx sessionctx.Context, db, sql string,
}
ctx, cancelFunc := context.WithCancel(ctx)
timer := time.NewTimer(maxTime)
+ defer timer.Stop()
resultChan := make(chan error)
startTime := time.Now()
go runSQL(ctx, sctx, sql, resultChan)
diff --git a/bindinfo/handle_test.go b/bindinfo/handle_test.go
index 01ff0628feb5c..ffea398781f8b 100644
--- a/bindinfo/handle_test.go
+++ b/bindinfo/handle_test.go
@@ -107,7 +107,7 @@ func TestBindingLastUpdateTimeWithInvalidBind(t *testing.T) {
require.Equal(t, updateTime0, "0000-00-00 00:00:00")
tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t`', 'select * from `test` . `t` use index(`idx`)', 'test', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int)")
@@ -137,8 +137,9 @@ func TestBindParse(t *testing.T) {
charset := "utf8mb4"
collation := "utf8mb4_bin"
source := bindinfo.Manual
- sql := fmt.Sprintf(`INSERT INTO mysql.bind_info(original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source) VALUES ('%s', '%s', '%s', '%s', NOW(), NOW(),'%s', '%s', '%s')`,
- originSQL, bindSQL, defaultDb, status, charset, collation, source)
+ mockDigest := "0f644e22c38ecc71d4592c52df127df7f86b6ca7f7c0ee899113b794578f9396"
+ sql := fmt.Sprintf(`INSERT INTO mysql.bind_info(original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest) VALUES ('%s', '%s', '%s', '%s', NOW(), NOW(),'%s', '%s', '%s', '%s', '%s')`,
+ originSQL, bindSQL, defaultDb, status, charset, collation, source, mockDigest, mockDigest)
tk.MustExec(sql)
bindHandle := bindinfo.NewBindHandle(tk.Session())
err := bindHandle.Update(true)
@@ -221,7 +222,7 @@ func TestEvolveInvalidBindings(t *testing.T) {
tk.MustExec("create global binding for select * from t where a > 10 using select /*+ USE_INDEX(t) */ * from t where a > 10")
// Manufacture a rejected binding by hacking mysql.bind_info.
tk.MustExec("insert into mysql.bind_info values('select * from test . t where a > ?', 'SELECT /*+ USE_INDEX(t,idx_a) */ * FROM test.t WHERE a > 10', 'test', 'rejected', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustQuery("select bind_sql, status from mysql.bind_info where source != 'builtin'").Sort().Check(testkit.Rows(
"SELECT /*+ USE_INDEX(`t` )*/ * FROM `test`.`t` WHERE `a` > 10 enabled",
"SELECT /*+ USE_INDEX(t,idx_a) */ * FROM test.t WHERE a > 10 rejected",
@@ -242,6 +243,8 @@ func TestEvolveInvalidBindings(t *testing.T) {
require.Equal(t, "SELECT /*+ USE_INDEX(t,idx_a) */ * FROM test.t WHERE a > 10", rows[1][1])
status = rows[1][3].(string)
require.True(t, status == bindinfo.Enabled || status == bindinfo.Rejected)
+ _, sqlDigestWithDB := parser.NormalizeDigest("select * from test.t where a > 10") // test sqlDigest if exists after add columns to mysql.bind_info
+ require.Equal(t, rows[0][9], sqlDigestWithDB.String())
}
func TestSetBindingStatus(t *testing.T) {
@@ -319,9 +322,9 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) {
// Simulate creating bindings on other machines
tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'enabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
dom.BindHandle().Clear()
tk.MustExec("set binding disabled for select * from t where a > 10")
tk.MustExec("admin reload bindings")
@@ -334,9 +337,9 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) {
// Simulate creating bindings on other machines
tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'disabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
dom.BindHandle().Clear()
tk.MustExec("set binding enabled for select * from t where a > 10")
tk.MustExec("admin reload bindings")
diff --git a/bindinfo/session_handle.go b/bindinfo/session_handle.go
index e6baebe3ea960..27b8168ef8e17 100644
--- a/bindinfo/session_handle.go
+++ b/bindinfo/session_handle.go
@@ -33,13 +33,12 @@ import (
// SessionHandle is used to handle all session sql bind operations.
type SessionHandle struct {
- ch *bindCache
- parser *parser.Parser
+ ch *bindCache
}
// NewSessionBindHandle creates a new SessionBindHandle.
-func NewSessionBindHandle(parser *parser.Parser) *SessionHandle {
- sessionHandle := &SessionHandle{parser: parser}
+func NewSessionBindHandle() *SessionHandle {
+ sessionHandle := &SessionHandle{}
sessionHandle.ch = newBindCache()
return sessionHandle
}
@@ -98,11 +97,25 @@ func (h *SessionHandle) DropBindRecord(originalSQL, db string, binding *Binding)
return nil
}
+// DropBindRecordByDigest drop BindRecord in the cache.
+func (h *SessionHandle) DropBindRecordByDigest(sqlDigest string) error {
+ oldRecord, err := h.GetBindRecordBySQLDigest(sqlDigest)
+ if err != nil {
+ return err
+ }
+ return h.DropBindRecord(oldRecord.OriginalSQL, strings.ToLower(oldRecord.Db), nil)
+}
+
// GetBindRecord return the BindMeta of the (normdOrigSQL,db) if BindMeta exist.
func (h *SessionHandle) GetBindRecord(hash, normdOrigSQL, db string) *BindRecord {
return h.ch.GetBindRecord(hash, normdOrigSQL, db)
}
+// GetBindRecordBySQLDigest return all BindMeta corresponding to sqlDigest.
+func (h *SessionHandle) GetBindRecordBySQLDigest(sqlDigest string) (*BindRecord, error) {
+ return h.ch.GetBindRecordBySQLDigest(sqlDigest)
+}
+
// GetAllBindRecord return all session bind info.
func (h *SessionHandle) GetAllBindRecord() (bindRecords []*BindRecord) {
return h.ch.GetAllBindRecords()
diff --git a/bindinfo/session_handle_test.go b/bindinfo/session_handle_test.go
index a60f8ff41cd12..a9b05c03eda18 100644
--- a/bindinfo/session_handle_test.go
+++ b/bindinfo/session_handle_test.go
@@ -219,7 +219,7 @@ func TestBaselineDBLowerCase(t *testing.T) {
// Simulate existing bindings with upper case default_db.
tk.MustExec("insert into mysql.bind_info values('select * from `spm` . `t`', 'select * from `spm` . `t`', 'SPM', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustQuery("select original_sql, default_db from mysql.bind_info where original_sql = 'select * from `spm` . `t`'").Check(testkit.Rows(
"select * from `spm` . `t` SPM",
))
@@ -237,7 +237,7 @@ func TestBaselineDBLowerCase(t *testing.T) {
utilCleanBindingEnv(tk, dom)
// Simulate existing bindings with upper case default_db.
tk.MustExec("insert into mysql.bind_info values('select * from `spm` . `t`', 'select * from `spm` . `t`', 'SPM', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustQuery("select original_sql, default_db from mysql.bind_info where original_sql = 'select * from `spm` . `t`'").Check(testkit.Rows(
"select * from `spm` . `t` SPM",
))
@@ -274,13 +274,13 @@ func TestShowGlobalBindings(t *testing.T) {
require.Len(t, rows, 0)
// Simulate existing bindings in the mysql.bind_info.
tk.MustExec("insert into mysql.bind_info values('select * from `spm` . `t`', 'select * from `spm` . `t` USE INDEX (`a`)', 'SPM', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustExec("insert into mysql.bind_info values('select * from `spm` . `t0`', 'select * from `spm` . `t0` USE INDEX (`a`)', 'SPM', 'enabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustExec("insert into mysql.bind_info values('select * from `spm` . `t`', 'select /*+ use_index(`t` `a`)*/ * from `spm` . `t`', 'SPM', 'enabled', '2000-01-03 09:00:00', '2000-01-03 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustExec("insert into mysql.bind_info values('select * from `spm` . `t0`', 'select /*+ use_index(`t0` `a`)*/ * from `spm` . `t0`', 'SPM', 'enabled', '2000-01-04 09:00:00', '2000-01-04 09:00:00', '', '','" +
- bindinfo.Manual + "')")
+ bindinfo.Manual + "', '', '')")
tk.MustExec("admin reload bindings")
rows = tk.MustQuery("show global bindings").Rows()
require.Len(t, rows, 4)
@@ -521,3 +521,14 @@ func TestPreparedStmt(t *testing.T) {
require.Len(t, tk.Session().GetSessionVars().StmtCtx.IndexNames, 1)
require.Equal(t, "t:idx_c", tk.Session().GetSessionVars().StmtCtx.IndexNames[0])
}
+
+func TestSetVarBinding(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (a int, b varchar(20))")
+ tk.MustExec("insert into t1 values (1, '111111111111111')")
+ tk.MustExec("insert into t1 values (2, '222222222222222')")
+ tk.MustExec("create binding for select group_concat(b) from test.t1 using select /*+ SET_VAR(group_concat_max_len = 4) */ group_concat(b) from test.t1 ;")
+ tk.MustQuery("select group_concat(b) from test.t1").Check(testkit.Rows("1111"))
+}
diff --git a/br/COMPATIBILITY_TEST.md b/br/COMPATIBILITY_TEST.md
index b5580835baee8..44984ebcd2bfa 100644
--- a/br/COMPATIBILITY_TEST.md
+++ b/br/COMPATIBILITY_TEST.md
@@ -3,7 +3,7 @@
## Background
We had some incompatibility issues in the past, which made BR cannot restore backed up data in some situations.
-So we need a test workflow to check the compatiblity.
+So we need a test workflow to check the compatibility.
## Goal
diff --git a/br/cmd/br/restore.go b/br/cmd/br/restore.go
index 5f91bee91c6a9..e826df0e59e77 100644
--- a/br/cmd/br/restore.go
+++ b/br/cmd/br/restore.go
@@ -199,6 +199,5 @@ func newStreamRestoreCommand() *cobra.Command {
}
task.DefineFilterFlags(command, filterOutSysAndMemTables, true)
task.DefineStreamRestoreFlags(command)
- command.Hidden = true
return command
}
diff --git a/br/cmd/tidb-lightning-ctl/main.go b/br/cmd/tidb-lightning-ctl/main.go
index 4dc70af929083..b0d20ae813734 100644
--- a/br/cmd/tidb-lightning-ctl/main.go
+++ b/br/cmd/tidb-lightning-ctl/main.go
@@ -88,7 +88,7 @@ func run() error {
if err != nil {
return err
}
- if err = cfg.TiDB.Security.RegisterMySQL(); err != nil {
+ if err = cfg.TiDB.Security.BuildTLSConfig(); err != nil {
return err
}
diff --git a/br/pkg/aws/BUILD.bazel b/br/pkg/aws/BUILD.bazel
index 6f33637abc5f3..2930569987d8b 100644
--- a/br/pkg/aws/BUILD.bazel
+++ b/br/pkg/aws/BUILD.bazel
@@ -23,7 +23,13 @@ go_library(
go_test(
name = "aws_test",
+ timeout = "short",
srcs = ["ebs_test.go"],
embed = [":aws"],
- deps = ["@com_github_stretchr_testify//require"],
+ flaky = True,
+ deps = [
+ "@com_github_aws_aws_sdk_go//aws",
+ "@com_github_aws_aws_sdk_go//service/ec2",
+ "@com_github_stretchr_testify//require",
+ ],
)
diff --git a/br/pkg/aws/ebs.go b/br/pkg/aws/ebs.go
index 9ded291e1f9b9..fb96b95578ffb 100644
--- a/br/pkg/aws/ebs.go
+++ b/br/pkg/aws/ebs.go
@@ -307,17 +307,9 @@ func (e *EC2Session) WaitVolumesCreated(volumeIDMap map[string]string, progress
return 0, errors.Trace(err)
}
- var unfinishedVolumes []*string
- for _, volume := range resp.Volumes {
- if *volume.State == ec2.VolumeStateAvailable {
- log.Info("volume is available", zap.String("id", *volume.SnapshotId))
- totalVolumeSize += *volume.Size
- progress.Inc()
- } else {
- log.Debug("volume creating...", zap.Stringer("volume", volume))
- unfinishedVolumes = append(unfinishedVolumes, volume.SnapshotId)
- }
- }
+ createdVolumeSize, unfinishedVolumes := e.HandleDescribeVolumesResponse(resp)
+ progress.IncBy(int64(len(pendingVolumes) - len(unfinishedVolumes)))
+ totalVolumeSize += createdVolumeSize
pendingVolumes = unfinishedVolumes
}
log.Info("all pending volume are created.")
@@ -357,3 +349,20 @@ func (e *EC2Session) DeleteVolumes(volumeIDMap map[string]string) {
func ec2Tag(key, val string) *ec2.Tag {
return &ec2.Tag{Key: &key, Value: &val}
}
+
+func (e *EC2Session) HandleDescribeVolumesResponse(resp *ec2.DescribeVolumesOutput) (int64, []*string) {
+ totalVolumeSize := int64(0)
+
+ var unfinishedVolumes []*string
+ for _, volume := range resp.Volumes {
+ if *volume.State == ec2.VolumeStateAvailable {
+ log.Info("volume is available", zap.String("id", *volume.VolumeId))
+ totalVolumeSize += *volume.Size
+ } else {
+ log.Debug("volume creating...", zap.Stringer("volume", volume))
+ unfinishedVolumes = append(unfinishedVolumes, volume.VolumeId)
+ }
+ }
+
+ return totalVolumeSize, unfinishedVolumes
+}
diff --git a/br/pkg/aws/ebs_test.go b/br/pkg/aws/ebs_test.go
index 695f31a73c477..d7f3be2a4a4a1 100644
--- a/br/pkg/aws/ebs_test.go
+++ b/br/pkg/aws/ebs_test.go
@@ -16,26 +16,63 @@ package aws
import (
"testing"
+ awsapi "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/ec2"
"github.com/stretchr/testify/require"
)
func TestEC2SessionExtractSnapProgress(t *testing.T) {
- strPtr := func(s string) *string {
- return &s
- }
tests := []struct {
str *string
want int64
}{
{nil, 0},
- {strPtr("12.12%"), 12},
- {strPtr("44.99%"), 44},
- {strPtr(" 89.89% "), 89},
- {strPtr("100%"), 100},
- {strPtr("111111%"), 100},
+ {awsapi.String("12.12%"), 12},
+ {awsapi.String("44.99%"), 44},
+ {awsapi.String(" 89.89% "), 89},
+ {awsapi.String("100%"), 100},
+ {awsapi.String("111111%"), 100},
}
e := &EC2Session{}
for _, tt := range tests {
require.Equal(t, tt.want, e.extractSnapProgress(tt.str))
}
}
+
+func createVolume(snapshotId string, volumeId string, state string) *ec2.Volume {
+ return &ec2.Volume{
+ Attachments: nil,
+ AvailabilityZone: awsapi.String("us-west-2"),
+ CreateTime: nil,
+ Encrypted: awsapi.Bool(true),
+ FastRestored: awsapi.Bool(true),
+ Iops: awsapi.Int64(3000),
+ KmsKeyId: nil,
+ MultiAttachEnabled: awsapi.Bool(true),
+ OutpostArn: awsapi.String("arn:12342"),
+ Size: awsapi.Int64(1),
+ SnapshotId: awsapi.String(snapshotId),
+ State: awsapi.String(state),
+ Tags: nil,
+ Throughput: nil,
+ VolumeId: awsapi.String(volumeId),
+ VolumeType: awsapi.String("gp3"),
+ }
+}
+func TestHandleDescribeVolumesResponse(t *testing.T) {
+ curentVolumesStates := &ec2.DescribeVolumesOutput{
+ NextToken: awsapi.String("fake token"),
+ Volumes: []*ec2.Volume{
+ createVolume("snap-0873674883", "vol-98768979", "available"),
+ createVolume("snap-0873674883", "vol-98768979", "creating"),
+ createVolume("snap-0873674883", "vol-98768979", "available"),
+ createVolume("snap-0873674883", "vol-98768979", "available"),
+ createVolume("snap-0873674883", "vol-98768979", "available"),
+ },
+ }
+
+ e := &EC2Session{}
+ createdVolumeSize, unfinishedVolumes := e.HandleDescribeVolumesResponse(curentVolumesStates)
+ require.Equal(t, int64(4), createdVolumeSize)
+ require.Equal(t, 1, len(unfinishedVolumes))
+}
diff --git a/br/pkg/backup/BUILD.bazel b/br/pkg/backup/BUILD.bazel
index c8cad292f4607..9f8e01f96007e 100644
--- a/br/pkg/backup/BUILD.bazel
+++ b/br/pkg/backup/BUILD.bazel
@@ -12,6 +12,7 @@ go_library(
importpath = "github.com/pingcap/tidb/br/pkg/backup",
visibility = ["//visibility:public"],
deps = [
+ "//br/pkg/checkpoint",
"//br/pkg/checksum",
"//br/pkg/conn",
"//br/pkg/conn/util",
@@ -28,7 +29,6 @@ go_library(
"//distsql",
"//kv",
"//meta",
- "//meta/autoid",
"//parser/model",
"//statistics/handle",
"//util",
diff --git a/br/pkg/backup/client.go b/br/pkg/backup/client.go
index 6ce3d24b0e209..7718d6d33986c 100644
--- a/br/pkg/backup/client.go
+++ b/br/pkg/backup/client.go
@@ -3,11 +3,14 @@
package backup
import (
+ "bytes"
"context"
+ "encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
+ "math/rand"
"os"
"strings"
"sync"
@@ -21,6 +24,7 @@ import (
"github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/checkpoint"
"github.com/pingcap/tidb/br/pkg/conn"
connutil "github.com/pingcap/tidb/br/pkg/conn/util"
berrors "github.com/pingcap/tidb/br/pkg/errors"
@@ -36,7 +40,6 @@ import (
"github.com/pingcap/tidb/distsql"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
- "github.com/pingcap/tidb/meta/autoid"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/codec"
@@ -90,18 +93,31 @@ type Client struct {
backend *backuppb.StorageBackend
apiVersion kvrpcpb.APIVersion
+ cipher *backuppb.CipherInfo
+ checkpointMeta *checkpoint.CheckpointMetadata
+ checkpointRunner *checkpoint.CheckpointRunner
+
gcTTL int64
}
// NewBackupClient returns a new backup client.
-func NewBackupClient(ctx context.Context, mgr ClientMgr) (*Client, error) {
+func NewBackupClient(ctx context.Context, mgr ClientMgr) *Client {
log.Info("new backup client")
pdClient := mgr.GetPDClient()
clusterID := pdClient.GetClusterID(ctx)
return &Client{
clusterID: clusterID,
mgr: mgr,
- }, nil
+
+ cipher: nil,
+ checkpointMeta: nil,
+ checkpointRunner: nil,
+ }
+}
+
+// SetCipher for checkpoint to encrypt sst file's metadata
+func (bc *Client) SetCipher(cipher *backuppb.CipherInfo) {
+ bc.cipher = cipher
}
// GetTS gets a new timestamp from PD.
@@ -120,6 +136,11 @@ func (bc *Client) GetTS(ctx context.Context, duration time.Duration, ts uint64)
backupTS uint64
err error
)
+
+ if bc.checkpointMeta != nil {
+ log.Info("reuse checkpoint BackupTS", zap.Uint64("backup-ts", bc.checkpointMeta.BackupTS))
+ return bc.checkpointMeta.BackupTS, nil
+ }
if ts > 0 {
backupTS = ts
} else {
@@ -160,6 +181,15 @@ func (bc *Client) SetLockFile(ctx context.Context) error {
"This file exists to remind other backup jobs won't use this path"))
}
+// GetSafePointID get the gc-safe-point's service-id from either checkpoint or immediate generation
+func (bc *Client) GetSafePointID() string {
+ if bc.checkpointMeta != nil {
+ log.Info("reuse the checkpoint gc-safepoint service id", zap.String("service-id", bc.checkpointMeta.GCServiceId))
+ return bc.checkpointMeta.GCServiceId
+ }
+ return utils.MakeSafePointID()
+}
+
// SetGCTTL set gcTTL for client.
func (bc *Client) SetGCTTL(ttl int64) {
if ttl <= 0 {
@@ -204,13 +234,145 @@ func (bc *Client) SetStorageAndCheckNotInUse(
"there may be some backup files in the path already, "+
"please specify a correct backup directory!", bc.storage.URI()+"/"+metautil.MetaFile)
}
- err = CheckBackupStorageIsLocked(ctx, bc.storage)
+ // use checkpoint mode if checkpoint meta exists
+ exist, err = bc.storage.FileExists(ctx, checkpoint.CheckpointMetaPath)
if err != nil {
- return err
+ return errors.Annotatef(err, "error occurred when checking %s file", checkpoint.CheckpointMetaPath)
}
+
+ // if there is no checkpoint meta, then checkpoint mode is not used
+ // or it is the first execution
+ if exist {
+ // load the config's hash to keep the config unchanged.
+ log.Info("load the checkpoint meta, so the existence of lockfile is allowed.")
+ bc.checkpointMeta, err = checkpoint.LoadCheckpointMetadata(ctx, bc.storage)
+ if err != nil {
+ return errors.Annotatef(err, "error occurred when loading %s file", checkpoint.CheckpointMetaPath)
+ }
+ } else {
+ err = CheckBackupStorageIsLocked(ctx, bc.storage)
+ if err != nil {
+ return err
+ }
+ }
+
return nil
}
+// CheckCheckpoint check whether the configs are the same
+func (bc *Client) CheckCheckpoint(hash []byte) error {
+ if bc.checkpointMeta != nil && !bytes.Equal(bc.checkpointMeta.ConfigHash, hash) {
+ return errors.Annotatef(berrors.ErrInvalidArgument, "failed to backup to %v, "+
+ "because the checkpoint mode is used, "+
+ "but the hashs of the configs are not the same. Please check the config",
+ bc.storage.URI(),
+ )
+ }
+
+ // first execution or not using checkpoint mode yet
+ // or using the same config can pass the check
+ return nil
+}
+
+func (bc *Client) GetCheckpointRunner() *checkpoint.CheckpointRunner {
+ return bc.checkpointRunner
+}
+
+// StartCheckpointMeta will
+// 1. saves the initial status into the external storage;
+// 2. load the checkpoint data from external storage
+// 3. start checkpoint runner
+func (bc *Client) StartCheckpointRunner(
+ ctx context.Context,
+ cfgHash []byte,
+ backupTS uint64,
+ ranges []rtree.Range,
+ safePointID string,
+ progressCallBack func(ProgressUnit),
+) (err error) {
+ if bc.checkpointMeta == nil {
+ bc.checkpointMeta = &checkpoint.CheckpointMetadata{
+ GCServiceId: safePointID,
+ ConfigHash: cfgHash,
+ BackupTS: backupTS,
+ Ranges: ranges,
+ }
+
+ // sync the checkpoint meta to the external storage at first
+ if err := checkpoint.SaveCheckpointMetadata(ctx, bc.storage, bc.checkpointMeta); err != nil {
+ return errors.Trace(err)
+ }
+ } else {
+ // otherwise, the checkpoint meta is loaded from the external storage,
+ // no need to save it again
+ // besides, there are exist checkpoint data need to be loaded before start checkpoint runner
+ bc.checkpointMeta.CheckpointDataMap, err = bc.loadCheckpointRanges(ctx, progressCallBack)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
+
+ bc.checkpointRunner, err = checkpoint.StartCheckpointRunner(ctx, bc.storage, bc.cipher, bc.mgr.GetPDClient())
+ return errors.Trace(err)
+}
+
+func (bc *Client) WaitForFinishCheckpoint(ctx context.Context) {
+ if bc.checkpointRunner != nil {
+ bc.checkpointRunner.WaitForFinish(ctx)
+ }
+}
+
+// GetProgressRange loads the checkpoint(finished) sub-ranges of the current range, and calculate its incompleted sub-ranges.
+func (bc *Client) GetProgressRange(r rtree.Range) (*rtree.ProgressRange, error) {
+ // use groupKey to distinguish different ranges
+ groupKey := base64.URLEncoding.EncodeToString(r.StartKey)
+ if bc.checkpointMeta != nil && len(bc.checkpointMeta.CheckpointDataMap) > 0 {
+ rangeTree, exists := bc.checkpointMeta.CheckpointDataMap[groupKey]
+ if exists {
+ incomplete := rangeTree.GetIncompleteRange(r.StartKey, r.EndKey)
+ delete(bc.checkpointMeta.CheckpointDataMap, groupKey)
+ return &rtree.ProgressRange{
+ Res: rangeTree,
+ Incomplete: incomplete,
+ Origin: r,
+ GroupKey: groupKey,
+ }, nil
+ }
+ }
+
+ // the origin range are not recorded in checkpoint
+ // return the default progress range
+ return &rtree.ProgressRange{
+ Res: rtree.NewRangeTree(),
+ Incomplete: []rtree.Range{
+ r,
+ },
+ Origin: r,
+ GroupKey: groupKey,
+ }, nil
+}
+
+// LoadCheckpointRange loads the checkpoint(finished) sub-ranges of the current range, and calculate its incompleted sub-ranges.
+func (bc *Client) loadCheckpointRanges(ctx context.Context, progressCallBack func(ProgressUnit)) (map[string]rtree.RangeTree, error) {
+ rangeDataMap := make(map[string]rtree.RangeTree)
+
+ pastDureTime, err := checkpoint.WalkCheckpointFile(ctx, bc.storage, bc.cipher, func(groupKey string, rg *rtree.Range) {
+ rangeTree, exists := rangeDataMap[groupKey]
+ if !exists {
+ rangeTree = rtree.NewRangeTree()
+ rangeDataMap[groupKey] = rangeTree
+ }
+ rangeTree.Put(rg.StartKey, rg.EndKey, rg.Files)
+ progressCallBack(RegionUnit)
+ })
+
+ // we should adjust start-time of the summary to `pastDureTime` earlier
+ log.Info("past cost time", zap.Duration("cost", pastDureTime))
+ summary.AdjustStartTimeToEarlierTime(pastDureTime)
+
+ return rangeDataMap, errors.Trace(err)
+}
+
// SetStorage sets ExternalStorage for client.
func (bc *Client) SetStorage(
ctx context.Context,
@@ -239,6 +401,22 @@ func (bc *Client) SetApiVersion(v kvrpcpb.APIVersion) {
bc.apiVersion = v
}
+// Client.BuildBackupRangeAndSchema calls BuildBackupRangeAndSchema,
+// if the checkpoint mode is used, return the ranges from checkpoint meta
+func (bc *Client) BuildBackupRangeAndSchema(
+ storage kv.Storage,
+ tableFilter filter.Filter,
+ backupTS uint64,
+ isFullBackup bool,
+) ([]rtree.Range, *Schemas, []*backuppb.PlacementPolicy, error) {
+ if bc.checkpointMeta == nil {
+ return BuildBackupRangeAndSchema(storage, tableFilter, backupTS, isFullBackup, true)
+ }
+ _, schemas, policies, err := BuildBackupRangeAndSchema(storage, tableFilter, backupTS, isFullBackup, false)
+ schemas.SetCheckpointChecksum(bc.checkpointMeta.CheckpointChecksum)
+ return bc.checkpointMeta.Ranges, schemas, policies, errors.Trace(err)
+}
+
// CheckBackupStorageIsLocked checks whether backups is locked.
// which means we found other backup progress already write
// some data files into the same backup directory or cloud prefix.
@@ -252,7 +430,7 @@ func CheckBackupStorageIsLocked(ctx context.Context, s storage.ExternalStorage)
// should return error to break the walkDir when found lock file and other .sst files.
if strings.HasSuffix(path, ".sst") {
return errors.Annotatef(berrors.ErrInvalidArgument, "backup lock file and sst file exist in %v, "+
- "there are some backup files in the path already, "+
+ "there are some backup files in the path already, but hasn't checkpoint metadata, "+
"please specify a correct backup directory!", s.URI()+"/"+metautil.LockFile)
}
return nil
@@ -290,10 +468,12 @@ func appendRanges(tbl *model.TableInfo, tblID int64) ([]kv.KeyRange, error) {
ranges = ranger.FullIntRange(false)
}
+ retRanges := make([]kv.KeyRange, 0, 1+len(tbl.Indices))
kvRanges, err := distsql.TableHandleRangesToKVRanges(nil, []int64{tblID}, tbl.IsCommonHandle, ranges, nil)
if err != nil {
return nil, errors.Trace(err)
}
+ retRanges = kvRanges.AppendSelfTo(retRanges)
for _, index := range tbl.Indices {
if index.State != model.StatePublic {
@@ -304,9 +484,9 @@ func appendRanges(tbl *model.TableInfo, tblID int64) ([]kv.KeyRange, error) {
if err != nil {
return nil, errors.Trace(err)
}
- kvRanges = append(kvRanges, idxRanges...)
+ retRanges = idxRanges.AppendSelfTo(retRanges)
}
- return kvRanges, nil
+ return retRanges, nil
}
// BuildBackupRangeAndSchema gets KV range and schema of tables.
@@ -317,6 +497,7 @@ func BuildBackupRangeAndSchema(
tableFilter filter.Filter,
backupTS uint64,
isFullBackup bool,
+ buildRange bool,
) ([]rtree.Range, *Schemas, []*backuppb.PlacementPolicy, error) {
snapshot := storage.GetSnapshot(kv.NewVersion(backupTS))
m := meta.NewSnapshotMeta(snapshot)
@@ -348,7 +529,7 @@ func BuildBackupRangeAndSchema(
for _, dbInfo := range dbs {
// skip system databases
- if !tableFilter.MatchSchema(dbInfo.Name.O) || util.IsMemDB(dbInfo.Name.L) {
+ if !tableFilter.MatchSchema(dbInfo.Name.O) || util.IsMemDB(dbInfo.Name.L) || utils.IsTemplateSysDB(dbInfo.Name) {
continue
}
@@ -375,29 +556,26 @@ func BuildBackupRangeAndSchema(
continue
}
- logger := log.With(
+ logger := log.L().With(
zap.String("db", dbInfo.Name.O),
zap.String("table", tableInfo.Name.O),
)
- tblVer := autoid.AllocOptionTableInfoVersion(tableInfo.Version)
- idAlloc := autoid.NewAllocator(storage, dbInfo.ID, tableInfo.ID, false, autoid.RowIDAllocType, tblVer)
- seqAlloc := autoid.NewAllocator(storage, dbInfo.ID, tableInfo.ID, false, autoid.SequenceType, tblVer)
- randAlloc := autoid.NewAllocator(storage, dbInfo.ID, tableInfo.ID, false, autoid.AutoRandomType, tblVer)
+ autoIDAccess := m.GetAutoIDAccessors(dbInfo.ID, tableInfo.ID)
var globalAutoID int64
switch {
case tableInfo.IsSequence():
- globalAutoID, err = seqAlloc.NextGlobalAutoID()
+ globalAutoID, err = autoIDAccess.SequenceValue().Get()
case tableInfo.IsView() || !utils.NeedAutoID(tableInfo):
// no auto ID for views or table without either rowID nor auto_increment ID.
default:
- globalAutoID, err = idAlloc.NextGlobalAutoID()
+ globalAutoID, err = autoIDAccess.RowID().Get()
}
if err != nil {
return nil, nil, nil, errors.Trace(err)
}
- tableInfo.AutoIncID = globalAutoID
+ tableInfo.AutoIncID = globalAutoID + 1
if !isFullBackup {
// according to https://github.com/pingcap/tidb/issues/32290.
// ignore placement policy when not in full backup
@@ -410,11 +588,11 @@ func BuildBackupRangeAndSchema(
if tableInfo.PKIsHandle && tableInfo.ContainsAutoRandomBits() {
// this table has auto_random id, we need backup and rebase in restoration
var globalAutoRandID int64
- globalAutoRandID, err = randAlloc.NextGlobalAutoID()
+ globalAutoRandID, err = autoIDAccess.RandomID().Get()
if err != nil {
return nil, nil, nil, errors.Trace(err)
}
- tableInfo.AutoRandID = globalAutoRandID
+ tableInfo.AutoRandID = globalAutoRandID + 1
logger.Debug("change table AutoRandID",
zap.Int64("AutoRandID", globalAutoRandID))
}
@@ -433,15 +611,17 @@ func BuildBackupRangeAndSchema(
backupSchemas.AddSchema(dbInfo, tableInfo)
- tableRanges, err := BuildTableRanges(tableInfo)
- if err != nil {
- return nil, nil, nil, errors.Trace(err)
- }
- for _, r := range tableRanges {
- ranges = append(ranges, rtree.Range{
- StartKey: r.StartKey,
- EndKey: r.EndKey,
- })
+ if buildRange {
+ tableRanges, err := BuildTableRanges(tableInfo)
+ if err != nil {
+ return nil, nil, nil, errors.Trace(err)
+ }
+ for _, r := range tableRanges {
+ ranges = append(ranges, rtree.Range{
+ StartKey: r.StartKey,
+ EndKey: r.EndKey,
+ })
+ }
}
}
}
@@ -579,6 +759,7 @@ func (bc *Client) BackupRanges(
ranges []rtree.Range,
request backuppb.BackupRequest,
concurrency uint,
+ replicaReadLabel map[string]string,
metaWriter *metautil.MetaWriter,
progressCallBack func(ProgressUnit),
) error {
@@ -602,10 +783,13 @@ func (bc *Client) BackupRanges(
id := id
req := request
req.StartKey, req.EndKey = r.StartKey, r.EndKey
-
+ pr, err := bc.GetProgressRange(r)
+ if err != nil {
+ return errors.Trace(err)
+ }
workerPool.ApplyOnErrorGroup(eg, func() error {
elctx := logutil.ContextWithField(ectx, logutil.RedactAny("range-sn", id))
- err := bc.BackupRange(elctx, req, metaWriter, progressCallBack)
+ err := bc.BackupRange(elctx, req, replicaReadLabel, pr, metaWriter, progressCallBack)
if err != nil {
// The error due to context cancel, stack trace is meaningless, the stack shall be suspended (also clear)
if errors.Cause(err) == context.Canceled {
@@ -616,6 +800,7 @@ func (bc *Client) BackupRanges(
return nil
})
}
+
return eg.Wait()
}
@@ -623,7 +808,9 @@ func (bc *Client) BackupRanges(
// Returns an array of files backed up.
func (bc *Client) BackupRange(
ctx context.Context,
- req backuppb.BackupRequest,
+ request backuppb.BackupRequest,
+ replicaReadLabel map[string]string,
+ progressRange *rtree.ProgressRange,
metaWriter *metautil.MetaWriter,
progressCallBack func(ProgressUnit),
) (err error) {
@@ -631,54 +818,92 @@ func (bc *Client) BackupRange(
defer func() {
elapsed := time.Since(start)
logutil.CL(ctx).Info("backup range completed",
- logutil.Key("startKey", req.StartKey), logutil.Key("endKey", req.EndKey),
+ logutil.Key("startKey", progressRange.Origin.StartKey), logutil.Key("endKey", progressRange.Origin.EndKey),
zap.Duration("take", elapsed))
- key := "range start:" + hex.EncodeToString(req.StartKey) + " end:" + hex.EncodeToString(req.EndKey)
+ key := "range start:" + hex.EncodeToString(progressRange.Origin.StartKey) + " end:" + hex.EncodeToString(progressRange.Origin.EndKey)
if err != nil {
summary.CollectFailureUnit(key, err)
}
}()
logutil.CL(ctx).Info("backup range started",
- logutil.Key("startKey", req.StartKey), logutil.Key("endKey", req.EndKey),
- zap.Uint64("rateLimit", req.RateLimit),
- zap.Uint32("concurrency", req.Concurrency))
+ logutil.Key("startKey", progressRange.Origin.StartKey), logutil.Key("endKey", progressRange.Origin.EndKey),
+ zap.Uint64("rateLimit", request.RateLimit),
+ zap.Uint32("concurrency", request.Concurrency))
- var allStores []*metapb.Store
- allStores, err = conn.GetAllTiKVStoresWithRetry(ctx, bc.mgr.GetPDClient(), connutil.SkipTiFlash)
+ allStores, err := conn.GetAllTiKVStoresWithRetry(ctx, bc.mgr.GetPDClient(), connutil.SkipTiFlash)
if err != nil {
return errors.Trace(err)
}
+ var targetStores []*metapb.Store
+ targetStoreIds := make(map[uint64]struct{})
+ if len(replicaReadLabel) == 0 {
+ targetStores = allStores // send backup push down request to all stores
+ } else {
+ for _, store := range allStores {
+ for _, label := range store.Labels {
+ if val, ok := replicaReadLabel[label.Key]; ok && val == label.Value {
+ targetStores = append(targetStores, store) // send backup push down request to stores that match replica read label
+ targetStoreIds[store.GetId()] = struct{}{} // record store id for fine grained backup
+ }
+ }
+ }
+ }
+ if len(replicaReadLabel) > 0 && len(targetStores) == 0 {
+ return errors.Errorf("no store matches replica read label: %v", replicaReadLabel)
+ }
logutil.CL(ctx).Info("backup push down started")
- push := newPushDown(bc.mgr, len(allStores))
- results, err := push.pushBackup(ctx, req, allStores, progressCallBack)
- if err != nil {
- return errors.Trace(err)
+ // either the `incomplete` is origin range itself,
+ // or the `incomplete` is sub-ranges split by checkpoint of origin range
+ if len(progressRange.Incomplete) > 0 {
+ // don't make the origin request dirty,
+ // since fineGrainedBackup need to use it.
+ req := request
+ if len(progressRange.Incomplete) > 1 {
+ subRanges := make([]*kvrpcpb.KeyRange, 0, len(progressRange.Incomplete))
+ for _, r := range progressRange.Incomplete {
+ subRanges = append(subRanges, &kvrpcpb.KeyRange{
+ StartKey: r.StartKey,
+ EndKey: r.EndKey,
+ })
+ }
+ req.SubRanges = subRanges
+ } else {
+ // compatible with older version of TiKV
+ req.StartKey = progressRange.Incomplete[0].StartKey
+ req.EndKey = progressRange.Incomplete[0].EndKey
+ }
+
+ push := newPushDown(bc.mgr, len(targetStores))
+ err = push.pushBackup(ctx, req, progressRange, targetStores, bc.checkpointRunner, progressCallBack)
+ if err != nil {
+ return errors.Trace(err)
+ }
}
- logutil.CL(ctx).Info("backup push down completed", zap.Int("small-range-count", results.Len()))
+ logutil.CL(ctx).Info("backup push down completed", zap.Int("small-range-count", progressRange.Res.Len()))
// Find and backup remaining ranges.
// TODO: test fine grained backup.
- if err := bc.fineGrainedBackup(ctx, req, results, progressCallBack); err != nil {
+ if err := bc.fineGrainedBackup(ctx, request, targetStoreIds, progressRange, progressCallBack); err != nil {
return errors.Trace(err)
}
// update progress of range unit
progressCallBack(RangeUnit)
- if req.IsRawKv {
+ if request.IsRawKv {
logutil.CL(ctx).Info("raw ranges backed up",
- logutil.Key("startKey", req.StartKey),
- logutil.Key("endKey", req.EndKey),
- zap.String("cf", req.Cf))
+ logutil.Key("startKey", progressRange.Origin.StartKey),
+ logutil.Key("endKey", progressRange.Origin.EndKey),
+ zap.String("cf", request.Cf))
} else {
logutil.CL(ctx).Info("transactional range backup completed",
- zap.Reflect("StartTS", req.StartVersion),
- zap.Reflect("EndTS", req.EndVersion))
+ zap.Reflect("StartTS", request.StartVersion),
+ zap.Reflect("EndTS", request.EndVersion))
}
var ascendErr error
- results.Ascend(func(i btree.Item) bool {
+ progressRange.Res.Ascend(func(i btree.Item) bool {
r := i.(*rtree.Range)
for _, f := range r.Files {
summary.CollectSuccessUnit(summary.TotalKV, 1, f.TotalKvs)
@@ -697,12 +922,12 @@ func (bc *Client) BackupRange(
}
// Check if there are duplicated files.
- checkDupFiles(&results)
+ checkDupFiles(&progressRange.Res)
return nil
}
-func (bc *Client) findRegionLeader(ctx context.Context, key []byte, isRawKv bool) (*metapb.Peer, error) {
+func (bc *Client) findTargetPeer(ctx context.Context, key []byte, isRawKv bool, targetStoreIds map[uint64]struct{}) (*metapb.Peer, error) {
// Keys are saved in encoded format in TiKV, so the key must be encoded
// in order to find the correct region.
key = codec.EncodeBytesExt([]byte{}, key, isRawKv)
@@ -710,27 +935,47 @@ func (bc *Client) findRegionLeader(ctx context.Context, key []byte, isRawKv bool
// better backoff.
region, err := bc.mgr.GetPDClient().GetRegion(ctx, key)
if err != nil || region == nil {
- log.Error("find leader failed", zap.Error(err), zap.Reflect("region", region))
+ log.Error("find region failed", zap.Error(err), zap.Reflect("region", region))
time.Sleep(time.Millisecond * time.Duration(100*i))
continue
}
- if region.Leader != nil {
- log.Info("find leader",
- zap.Reflect("Leader", region.Leader), logutil.Key("key", key))
- return region.Leader, nil
+ if len(targetStoreIds) == 0 {
+ if region.Leader != nil {
+ log.Info("find leader",
+ zap.Reflect("Leader", region.Leader), logutil.Key("key", key))
+ return region.Leader, nil
+ }
+ } else {
+ candidates := make([]*metapb.Peer, 0, len(region.Meta.Peers))
+ for _, peer := range region.Meta.Peers {
+ if _, ok := targetStoreIds[peer.StoreId]; ok {
+ candidates = append(candidates, peer)
+ }
+ }
+ if len(candidates) > 0 {
+ peer := candidates[rand.Intn(len(candidates))]
+ log.Info("find target peer for backup",
+ zap.Reflect("Peer", peer), logutil.Key("key", key))
+ return peer, nil
+ }
}
- log.Warn("no region found", logutil.Key("key", key))
- time.Sleep(time.Millisecond * time.Duration(100*i))
+
+ log.Warn("fail to find a target peer", logutil.Key("key", key))
+ time.Sleep(time.Millisecond * time.Duration(1000*i))
continue
}
- log.Error("can not find leader", logutil.Key("key", key))
- return nil, errors.Annotatef(berrors.ErrBackupNoLeader, "can not find leader")
+ log.Error("can not find a valid target peer", logutil.Key("key", key))
+ if len(targetStoreIds) == 0 {
+ return nil, errors.Annotatef(berrors.ErrBackupNoLeader, "can not find a valid leader for key %s", key)
+ }
+ return nil, errors.Errorf("can not find a valid target peer for key %s", key)
}
func (bc *Client) fineGrainedBackup(
ctx context.Context,
req backuppb.BackupRequest,
- rangeTree rtree.RangeTree,
+ targetStoreIds map[uint64]struct{},
+ pr *rtree.ProgressRange,
progressCallBack func(ProgressUnit),
) error {
if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
@@ -757,7 +1002,7 @@ func (bc *Client) fineGrainedBackup(
bo := tikv.NewBackoffer(ctx, backupFineGrainedMaxBackoff)
for {
// Step1, check whether there is any incomplete range
- incomplete := rangeTree.GetIncompleteRange(req.StartKey, req.EndKey)
+ incomplete := pr.Res.GetIncompleteRange(req.StartKey, req.EndKey)
if len(incomplete) == 0 {
return nil
}
@@ -780,7 +1025,7 @@ func (bc *Client) fineGrainedBackup(
for rg := range retry {
subReq := req
subReq.StartKey, subReq.EndKey = rg.StartKey, rg.EndKey
- backoffMs, err := bc.handleFineGrained(ctx, boFork, subReq, respCh)
+ backoffMs, err := bc.handleFineGrained(ctx, boFork, subReq, targetStoreIds, respCh)
if err != nil {
errCh <- err
return
@@ -825,7 +1070,18 @@ func (bc *Client) fineGrainedBackup(
logutil.Key("fine-grained-range-start", resp.StartKey),
logutil.Key("fine-grained-range-end", resp.EndKey),
)
- rangeTree.Put(resp.StartKey, resp.EndKey, resp.Files)
+ if bc.checkpointRunner != nil {
+ if err := bc.checkpointRunner.Append(
+ ctx,
+ pr.GroupKey,
+ resp.StartKey,
+ resp.EndKey,
+ resp.Files,
+ ); err != nil {
+ return errors.Annotate(err, "failed to flush checkpoint when fineGrainedBackup")
+ }
+ }
+ pr.Res.Put(resp.StartKey, resp.EndKey, resp.Files)
apiVersion := resp.ApiVersion
bc.SetApiVersion(apiVersion)
@@ -921,13 +1177,14 @@ func (bc *Client) handleFineGrained(
ctx context.Context,
bo *tikv.Backoffer,
req backuppb.BackupRequest,
+ targetStoreIds map[uint64]struct{},
respCh chan<- *backuppb.BackupResponse,
) (int, error) {
- leader, pderr := bc.findRegionLeader(ctx, req.StartKey, req.IsRawKv)
+ targetPeer, pderr := bc.findTargetPeer(ctx, req.StartKey, req.IsRawKv, targetStoreIds)
if pderr != nil {
return 0, errors.Trace(pderr)
}
- storeID := leader.GetStoreId()
+ storeID := targetPeer.GetStoreId()
lockResolver := bc.mgr.GetLockResolver()
client, err := bc.mgr.GetBackupClient(ctx, storeID)
if err != nil {
diff --git a/br/pkg/backup/client_test.go b/br/pkg/backup/client_test.go
index 76d885e04a201..592416e8ec03c 100644
--- a/br/pkg/backup/client_test.go
+++ b/br/pkg/backup/client_test.go
@@ -57,8 +57,7 @@ func createBackupSuite(t *testing.T) *testBackup {
mockMgr := &conn.Mgr{PdController: &pdutil.PdController{}}
mockMgr.SetPDClient(s.mockPDClient)
mockMgr.SetHTTP([]string{"test"}, nil)
- s.backupClient, err = backup.NewBackupClient(s.ctx, mockMgr)
- require.NoError(t, err)
+ s.backupClient = backup.NewBackupClient(s.ctx, mockMgr)
s.cluster, err = mock.NewCluster()
require.NoError(t, err)
diff --git a/br/pkg/backup/push.go b/br/pkg/backup/push.go
index 45c2b9acca01c..ed929a12895a4 100644
--- a/br/pkg/backup/push.go
+++ b/br/pkg/backup/push.go
@@ -13,6 +13,7 @@ import (
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/errorpb"
"github.com/pingcap/kvproto/pkg/metapb"
+ "github.com/pingcap/tidb/br/pkg/checkpoint"
berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/br/pkg/redact"
@@ -54,9 +55,11 @@ func newPushDown(mgr ClientMgr, capacity int) *pushDown {
func (push *pushDown) pushBackup(
ctx context.Context,
req backuppb.BackupRequest,
+ pr *rtree.ProgressRange,
stores []*metapb.Store,
+ checkpointRunner *checkpoint.CheckpointRunner,
progressCallBack func(ProgressUnit),
-) (rtree.RangeTree, error) {
+) error {
if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
span1 := span.Tracer().StartSpan("pushDown.pushBackup", opentracing.ChildOf(span.Context()))
defer span1.Finish()
@@ -64,10 +67,9 @@ func (push *pushDown) pushBackup(
}
// Push down backup tasks to all tikv instances.
- res := rtree.NewRangeTree()
failpoint.Inject("noop-backup", func(_ failpoint.Value) {
logutil.CL(ctx).Warn("skipping normal backup, jump to fine-grained backup, meow :3", logutil.Key("start-key", req.StartKey), logutil.Key("end-key", req.EndKey))
- failpoint.Return(res, nil)
+ failpoint.Return(nil)
})
wg := new(sync.WaitGroup)
@@ -84,7 +86,7 @@ func (push *pushDown) pushBackup(
// BR should be able to backup even some of stores disconnected.
// The regions managed by this store can be retried at fine-grained backup then.
logutil.CL(lctx).Warn("fail to connect store, skipping", zap.Error(err))
- return res, nil
+ return nil
}
wg.Add(1)
go func() {
@@ -125,7 +127,7 @@ func (push *pushDown) pushBackup(
store := respAndStore.GetStore()
if !ok {
// Finished.
- return res, nil
+ return nil
}
failpoint.Inject("backup-timeout-error", func(val failpoint.Value) {
msg := val.(string)
@@ -165,7 +167,19 @@ func (push *pushDown) pushBackup(
})
if resp.GetError() == nil {
// None error means range has been backuped successfully.
- res.Put(
+ if checkpointRunner != nil {
+ if err := checkpointRunner.Append(
+ ctx,
+ pr.GroupKey,
+ resp.StartKey,
+ resp.EndKey,
+ resp.Files,
+ ); err != nil {
+ // the error is only from flush operator
+ return errors.Annotate(err, "failed to flush checkpoint")
+ }
+ }
+ pr.Res.Put(
resp.GetStartKey(), resp.GetEndKey(), resp.GetFiles())
// Update progress
@@ -181,7 +195,7 @@ func (push *pushDown) pushBackup(
case *backuppb.Error_ClusterIdError:
logutil.CL(ctx).Error("backup occur cluster ID error", zap.Reflect("error", v))
- return res, errors.Annotatef(berrors.ErrKVClusterIDMismatch, "%v", errPb)
+ return errors.Annotatef(berrors.ErrKVClusterIDMismatch, "%v", errPb)
default:
if utils.MessageIsRetryableStorageError(errPb.GetMsg()) {
logutil.CL(ctx).Warn("backup occur storage error", zap.String("error", errPb.GetMsg()))
@@ -204,20 +218,19 @@ func (push *pushDown) pushBackup(
if len(errMsg) <= 0 {
errMsg = errPb.Msg
}
- return res, errors.Annotatef(berrors.ErrKVStorage, "error happen in store %v at %s: %s %s",
+ return errors.Annotatef(berrors.ErrKVStorage, "error happen in store %v at %s: %s",
store.GetId(),
redact.String(store.GetAddress()),
- req.StorageBackend.String(),
errMsg,
)
}
}
case err := <-push.errCh:
if !berrors.Is(err, berrors.ErrFailedToConnect) {
- return res, errors.Annotatef(err, "failed to backup range [%s, %s)", redact.Key(req.StartKey), redact.Key(req.EndKey))
+ return errors.Annotatef(err, "failed to backup range [%s, %s)", redact.Key(req.StartKey), redact.Key(req.EndKey))
}
logutil.CL(ctx).Warn("skipping disconnected stores", logutil.ShortError(err))
- return res, nil
+ return nil
}
}
}
diff --git a/br/pkg/backup/schema.go b/br/pkg/backup/schema.go
index 066043c224064..19910af1c9b9b 100644
--- a/br/pkg/backup/schema.go
+++ b/br/pkg/backup/schema.go
@@ -12,6 +12,7 @@ import (
"github.com/pingcap/errors"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/checkpoint"
"github.com/pingcap/tidb/br/pkg/checksum"
"github.com/pingcap/tidb/br/pkg/glue"
"github.com/pingcap/tidb/br/pkg/logutil"
@@ -44,14 +45,22 @@ type schemaInfo struct {
type Schemas struct {
// name -> schema
schemas map[string]*schemaInfo
+
+ // checkpoint: table id -> checksum
+ checkpointChecksum map[int64]*checkpoint.ChecksumItem
}
func NewBackupSchemas() *Schemas {
return &Schemas{
- schemas: make(map[string]*schemaInfo),
+ schemas: make(map[string]*schemaInfo),
+ checkpointChecksum: nil,
}
}
+func (ss *Schemas) SetCheckpointChecksum(checkpointChecksum map[int64]*checkpoint.ChecksumItem) {
+ ss.checkpointChecksum = checkpointChecksum
+}
+
func (ss *Schemas) AddSchema(
dbInfo *model.DBInfo, tableInfo *model.TableInfo,
) {
@@ -73,6 +82,7 @@ func (ss *Schemas) AddSchema(
func (ss *Schemas) BackupSchemas(
ctx context.Context,
metaWriter *metautil.MetaWriter,
+ checkpointRunner *checkpoint.CheckpointRunner,
store kv.Storage,
statsHandle *handle.Handle,
backupTS uint64,
@@ -100,25 +110,52 @@ func (ss *Schemas) BackupSchemas(
schema.dbInfo.Name = utils.TemporaryDBName(schema.dbInfo.Name.O)
}
+ var checksum *checkpoint.ChecksumItem
+ var exists bool = false
+ if ss.checkpointChecksum != nil && schema.tableInfo != nil {
+ checksum, exists = ss.checkpointChecksum[schema.tableInfo.ID]
+ }
workerPool.ApplyOnErrorGroup(errg, func() error {
if schema.tableInfo != nil {
- logger := log.With(
+ logger := log.L().With(
zap.String("db", schema.dbInfo.Name.O),
zap.String("table", schema.tableInfo.Name.O),
)
if !skipChecksum {
logger.Info("Calculate table checksum start")
- start := time.Now()
- err := schema.calculateChecksum(ectx, store.GetClient(), backupTS, copConcurrency)
- if err != nil {
- return errors.Trace(err)
+ if exists && checksum != nil {
+ schema.crc64xor = checksum.Crc64xor
+ schema.totalKvs = checksum.TotalKvs
+ schema.totalBytes = checksum.TotalBytes
+ logger.Info("Calculate table checksum completed (from checkpoint)",
+ zap.Uint64("Crc64Xor", schema.crc64xor),
+ zap.Uint64("TotalKvs", schema.totalKvs),
+ zap.Uint64("TotalBytes", schema.totalBytes))
+ } else {
+ start := time.Now()
+ err := schema.calculateChecksum(ectx, store.GetClient(), backupTS, copConcurrency)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ calculateCost := time.Since(start)
+ var flushCost time.Duration
+ if checkpointRunner != nil {
+ // if checkpoint runner is running and the checksum is not from checkpoint
+ // then flush the checksum by the checkpoint runner
+ startFlush := time.Now()
+ if err = checkpointRunner.FlushChecksum(ctx, schema.tableInfo.ID, schema.crc64xor, schema.totalKvs, schema.totalBytes, calculateCost.Seconds()); err != nil {
+ return errors.Trace(err)
+ }
+ flushCost = time.Since(startFlush)
+ }
+ logger.Info("Calculate table checksum completed",
+ zap.Uint64("Crc64Xor", schema.crc64xor),
+ zap.Uint64("TotalKvs", schema.totalKvs),
+ zap.Uint64("TotalBytes", schema.totalBytes),
+ zap.Duration("calculate-take", calculateCost),
+ zap.Duration("flush-take", flushCost))
}
- logger.Info("Calculate table checksum completed",
- zap.Uint64("Crc64Xor", schema.crc64xor),
- zap.Uint64("TotalKvs", schema.totalKvs),
- zap.Uint64("TotalBytes", schema.totalBytes),
- zap.Duration("take", time.Since(start)))
}
if statsHandle != nil {
if err := schema.dumpStatsToJSON(statsHandle); err != nil {
diff --git a/br/pkg/backup/schema_test.go b/br/pkg/backup/schema_test.go
index bed9d834d2e10..08d560bf03c25 100644
--- a/br/pkg/backup/schema_test.go
+++ b/br/pkg/backup/schema_test.go
@@ -108,7 +108,7 @@ func TestBuildBackupRangeAndSchema(t *testing.T) {
testFilter, err := filter.Parse([]string{"test.t1"})
require.NoError(t, err)
_, backupSchemas, _, err := backup.BuildBackupRangeAndSchema(
- m.Storage, testFilter, math.MaxUint64, false)
+ m.Storage, testFilter, math.MaxUint64, false, true)
require.NoError(t, err)
require.NotNil(t, backupSchemas)
@@ -116,7 +116,7 @@ func TestBuildBackupRangeAndSchema(t *testing.T) {
fooFilter, err := filter.Parse([]string{"foo.t1"})
require.NoError(t, err)
_, backupSchemas, _, err = backup.BuildBackupRangeAndSchema(
- m.Storage, fooFilter, math.MaxUint64, false)
+ m.Storage, fooFilter, math.MaxUint64, false, true)
require.NoError(t, err)
require.Nil(t, backupSchemas)
@@ -125,7 +125,7 @@ func TestBuildBackupRangeAndSchema(t *testing.T) {
noFilter, err := filter.Parse([]string{"*.*", "!mysql.*"})
require.NoError(t, err)
_, backupSchemas, _, err = backup.BuildBackupRangeAndSchema(
- m.Storage, noFilter, math.MaxUint64, false)
+ m.Storage, noFilter, math.MaxUint64, false, true)
require.NoError(t, err)
require.NotNil(t, backupSchemas)
@@ -137,7 +137,7 @@ func TestBuildBackupRangeAndSchema(t *testing.T) {
var policies []*backuppb.PlacementPolicy
_, backupSchemas, policies, err = backup.BuildBackupRangeAndSchema(
- m.Storage, testFilter, math.MaxUint64, false)
+ m.Storage, testFilter, math.MaxUint64, false, true)
require.NoError(t, err)
require.Equal(t, 1, backupSchemas.Len())
// we expect no policies collected, because it's not full backup.
@@ -151,7 +151,7 @@ func TestBuildBackupRangeAndSchema(t *testing.T) {
metaWriter := metautil.NewMetaWriter(es, metautil.MetaFileSize, false, "", &cipher)
ctx := context.Background()
err = backupSchemas.BackupSchemas(
- ctx, metaWriter, m.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
+ ctx, metaWriter, nil, m.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
require.Equal(t, int64(1), updateCh.get())
require.NoError(t, err)
err = metaWriter.FlushBackupMeta(ctx)
@@ -170,7 +170,7 @@ func TestBuildBackupRangeAndSchema(t *testing.T) {
tk.MustExec("insert into t2 values (11);")
_, backupSchemas, policies, err = backup.BuildBackupRangeAndSchema(
- m.Storage, noFilter, math.MaxUint64, true)
+ m.Storage, noFilter, math.MaxUint64, true, true)
require.NoError(t, err)
require.Equal(t, 2, backupSchemas.Len())
// we expect the policy fivereplicas collected in full backup.
@@ -180,7 +180,7 @@ func TestBuildBackupRangeAndSchema(t *testing.T) {
es2 := GetRandomStorage(t)
metaWriter2 := metautil.NewMetaWriter(es2, metautil.MetaFileSize, false, "", &cipher)
err = backupSchemas.BackupSchemas(
- ctx, metaWriter2, m.Storage, nil, math.MaxUint64, 2, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
+ ctx, metaWriter2, nil, m.Storage, nil, math.MaxUint64, 2, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
require.Equal(t, int64(2), updateCh.get())
require.NoError(t, err)
err = metaWriter2.FlushBackupMeta(ctx)
@@ -219,7 +219,7 @@ func TestBuildBackupRangeAndSchemaWithBrokenStats(t *testing.T) {
f, err := filter.Parse([]string{"test.t3"})
require.NoError(t, err)
- _, backupSchemas, _, err := backup.BuildBackupRangeAndSchema(m.Storage, f, math.MaxUint64, false)
+ _, backupSchemas, _, err := backup.BuildBackupRangeAndSchema(m.Storage, f, math.MaxUint64, false, true)
require.NoError(t, err)
require.Equal(t, 1, backupSchemas.Len())
@@ -234,7 +234,7 @@ func TestBuildBackupRangeAndSchemaWithBrokenStats(t *testing.T) {
metaWriter := metautil.NewMetaWriter(es, metautil.MetaFileSize, false, "", &cipher)
ctx := context.Background()
err = backupSchemas.BackupSchemas(
- ctx, metaWriter, m.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
+ ctx, metaWriter, nil, m.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
require.NoError(t, err)
err = metaWriter.FlushBackupMeta(ctx)
require.NoError(t, err)
@@ -253,7 +253,7 @@ func TestBuildBackupRangeAndSchemaWithBrokenStats(t *testing.T) {
// recover the statistics.
tk.MustExec("analyze table t3;")
- _, backupSchemas, _, err = backup.BuildBackupRangeAndSchema(m.Storage, f, math.MaxUint64, false)
+ _, backupSchemas, _, err = backup.BuildBackupRangeAndSchema(m.Storage, f, math.MaxUint64, false, true)
require.NoError(t, err)
require.Equal(t, 1, backupSchemas.Len())
@@ -262,7 +262,7 @@ func TestBuildBackupRangeAndSchemaWithBrokenStats(t *testing.T) {
es2 := GetRandomStorage(t)
metaWriter2 := metautil.NewMetaWriter(es2, metautil.MetaFileSize, false, "", &cipher)
err = backupSchemas.BackupSchemas(
- ctx, metaWriter2, m.Storage, statsHandle, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
+ ctx, metaWriter2, nil, m.Storage, statsHandle, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
require.NoError(t, err)
err = metaWriter2.FlushBackupMeta(ctx)
require.NoError(t, err)
@@ -294,7 +294,7 @@ func TestBackupSchemasForSystemTable(t *testing.T) {
f, err := filter.Parse([]string{"mysql.systable*"})
require.NoError(t, err)
- _, backupSchemas, _, err := backup.BuildBackupRangeAndSchema(m.Storage, f, math.MaxUint64, false)
+ _, backupSchemas, _, err := backup.BuildBackupRangeAndSchema(m.Storage, f, math.MaxUint64, false, true)
require.NoError(t, err)
require.Equal(t, systemTablesCount, backupSchemas.Len())
@@ -305,7 +305,7 @@ func TestBackupSchemasForSystemTable(t *testing.T) {
updateCh := new(simpleProgress)
metaWriter2 := metautil.NewMetaWriter(es2, metautil.MetaFileSize, false, "", &cipher)
- err = backupSchemas.BackupSchemas(ctx, metaWriter2, m.Storage, nil,
+ err = backupSchemas.BackupSchemas(ctx, metaWriter2, nil, m.Storage, nil,
math.MaxUint64, 1, variable.DefChecksumTableConcurrency, true, updateCh)
require.NoError(t, err)
err = metaWriter2.FlushBackupMeta(ctx)
diff --git a/br/pkg/checkpoint/BUILD.bazel b/br/pkg/checkpoint/BUILD.bazel
new file mode 100644
index 0000000000000..e8e0f0bd536d5
--- /dev/null
+++ b/br/pkg/checkpoint/BUILD.bazel
@@ -0,0 +1,37 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "checkpoint",
+ srcs = ["checkpoint.go"],
+ importpath = "github.com/pingcap/tidb/br/pkg/checkpoint",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//br/pkg/logutil",
+ "//br/pkg/metautil",
+ "//br/pkg/rtree",
+ "//br/pkg/storage",
+ "//br/pkg/summary",
+ "//br/pkg/utils",
+ "@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_kvproto//pkg/brpb",
+ "@com_github_pingcap_log//:log",
+ "@com_github_tikv_client_go_v2//oracle",
+ "@org_uber_go_zap//:zap",
+ ],
+)
+
+go_test(
+ name = "checkpoint_test",
+ timeout = "short",
+ srcs = ["checkpoint_test.go"],
+ flaky = True,
+ deps = [
+ ":checkpoint",
+ "//br/pkg/rtree",
+ "//br/pkg/storage",
+ "@com_github_pingcap_kvproto//pkg/brpb",
+ "@com_github_pingcap_kvproto//pkg/encryptionpb",
+ "@com_github_stretchr_testify//require",
+ "@com_github_tikv_client_go_v2//oracle",
+ ],
+)
diff --git a/br/pkg/checkpoint/checkpoint.go b/br/pkg/checkpoint/checkpoint.go
new file mode 100644
index 0000000000000..0af1cefdf2594
--- /dev/null
+++ b/br/pkg/checkpoint/checkpoint.go
@@ -0,0 +1,759 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package checkpoint
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "math/rand"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/pingcap/errors"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/metautil"
+ "github.com/pingcap/tidb/br/pkg/rtree"
+ "github.com/pingcap/tidb/br/pkg/storage"
+ "github.com/pingcap/tidb/br/pkg/summary"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/tikv/client-go/v2/oracle"
+ "go.uber.org/zap"
+)
+
+const (
+ CheckpointMetaPath = "checkpoint.meta"
+ CheckpointDir = "/checkpoints"
+
+ CheckpointDataDir = CheckpointDir + "/data"
+ CheckpointChecksumDir = CheckpointDir + "/checksum"
+ CheckpointLockPath = CheckpointDir + "/checkpoint.lock"
+)
+
+const MaxChecksumTotalCost float64 = 60.0
+
+const tickDurationForFlush = 30 * time.Second
+
+const tickDurationForLock = 4 * time.Minute
+
+const lockTimeToLive = 5 * time.Minute
+
+type CheckpointMessage struct {
+ // start-key of the origin range
+ GroupKey string
+
+ Group *rtree.Range
+}
+
+// A Checkpoint Range File is like this:
+//
+// ChecksumData
+// +----------------+ RangeGroupData RangeGroups
+// | DureTime | +--------------------------+ encrypted +-------------+
+// | RangeGroupData-+---> | RangeGroupsEncriptedData-+----------> | GroupKey |
+// | RangeGroupData | | Checksum | | Range |
+// | ... | | CipherIv | | ... |
+// | RangeGroupData | | Size | | Range |
+// +----------------+ +--------------------------+ +-------------+
+
+type RangeGroups struct {
+ GroupKey string `json:"group-key"`
+ Groups []*rtree.Range `json:"groups"`
+}
+
+type RangeGroupData struct {
+ RangeGroupsEncriptedData []byte
+ Checksum []byte
+ CipherIv []byte
+
+ Size int
+}
+
+type CheckpointData struct {
+ DureTime time.Duration `json:"dure-time"`
+ RangeGroupMetas []*RangeGroupData `json:"range-group-metas"`
+}
+
+// A Checkpoint Checksum File is like this:
+//
+// ChecksumInfo ChecksumItems ChecksumItem
+// +-------------+ +--------------+ +--------------+
+// | Content---+-> | ChecksumItem-+---> | TableID |
+// | Checksum | | ChecksumItem | | Crc64xor |
+// +-------------+ | ... | | TotalKvs |
+// | ChecksumItem | | TotalBytes |
+// +--------------+ +--------------+
+
+type ChecksumItem struct {
+ TableID int64 `json:"table-id"`
+ Crc64xor uint64 `json:"crc64-xor"`
+ TotalKvs uint64 `json:"total-kvs"`
+ TotalBytes uint64 `json:"total-bytes"`
+}
+
+type ChecksumItems struct {
+ Items []*ChecksumItem `json:"checksum-items"`
+}
+
+type ChecksumInfo struct {
+ Content []byte `json:"content"`
+ Checksum []byte `json:"checksum"`
+}
+
+type ChecksumRunner struct {
+ sync.Mutex
+
+ checksumItems ChecksumItems
+
+ // when the total time cost is large than the threshold,
+ // begin to flush checksum
+ totalCost float64
+
+ err error
+ wg sync.WaitGroup
+ workerPool utils.WorkerPool
+}
+
+func NewChecksumRunner() *ChecksumRunner {
+ return &ChecksumRunner{
+ workerPool: *utils.NewWorkerPool(4, "checksum flush worker"),
+ }
+}
+
+func (cr *ChecksumRunner) RecordError(err error) {
+ cr.Lock()
+ cr.err = err
+ cr.Unlock()
+}
+
+// FlushChecksum save the checksum in the memory temporarily
+// and flush to the external storage if checksum take much time
+func (cr *ChecksumRunner) FlushChecksum(
+ ctx context.Context,
+ s storage.ExternalStorage,
+ tableID int64,
+ crc64xor uint64,
+ totalKvs uint64,
+ totalBytes uint64,
+ timeCost float64,
+) error {
+ checksumItem := &ChecksumItem{
+ TableID: tableID,
+ Crc64xor: crc64xor,
+ TotalKvs: totalKvs,
+ TotalBytes: totalBytes,
+ }
+ var toBeFlushedChecksumItems *ChecksumItems = nil
+ cr.Lock()
+ if cr.err != nil {
+ err := cr.err
+ cr.Unlock()
+ return err
+ }
+ if cr.checksumItems.Items == nil {
+ // reset the checksumInfo
+ cr.totalCost = 0
+ cr.checksumItems.Items = make([]*ChecksumItem, 0)
+ }
+ cr.totalCost += timeCost
+ cr.checksumItems.Items = append(cr.checksumItems.Items, checksumItem)
+ if cr.totalCost > MaxChecksumTotalCost {
+ toBeFlushedChecksumItems = &ChecksumItems{
+ Items: cr.checksumItems.Items,
+ }
+ cr.checksumItems.Items = nil
+ }
+ cr.Unlock()
+
+ // now lock is free
+ if toBeFlushedChecksumItems == nil {
+ return nil
+ }
+
+ // create a goroutine to flush checksumInfo to external storage
+ cr.wg.Add(1)
+ cr.workerPool.Apply(func() {
+ defer cr.wg.Done()
+
+ content, err := json.Marshal(toBeFlushedChecksumItems)
+ if err != nil {
+ cr.RecordError(err)
+ return
+ }
+
+ checksum := sha256.Sum256(content)
+ checksumInfo := &ChecksumInfo{
+ Content: content,
+ Checksum: checksum[:],
+ }
+
+ data, err := json.Marshal(checksumInfo)
+ if err != nil {
+ cr.RecordError(err)
+ return
+ }
+
+ fname := fmt.Sprintf("%s/t%d_and__", CheckpointChecksumDir, tableID)
+ err = s.WriteFile(ctx, fname, data)
+ if err != nil {
+ cr.RecordError(err)
+ return
+ }
+ })
+ return nil
+}
+
+type GlobalTimer interface {
+ GetTS(context.Context) (int64, int64, error)
+}
+
+type CheckpointRunner struct {
+ lockId uint64
+
+ meta map[string]*RangeGroups
+
+ checksumRunner *ChecksumRunner
+
+ storage storage.ExternalStorage
+ cipher *backuppb.CipherInfo
+ timer GlobalTimer
+
+ appendCh chan *CheckpointMessage
+ metaCh chan map[string]*RangeGroups
+ lockCh chan struct{}
+ errCh chan error
+
+ wg sync.WaitGroup
+}
+
+// only for test
+func StartCheckpointRunnerForTest(ctx context.Context, storage storage.ExternalStorage, cipher *backuppb.CipherInfo, tick time.Duration, timer GlobalTimer) (*CheckpointRunner, error) {
+ runner := &CheckpointRunner{
+ meta: make(map[string]*RangeGroups),
+
+ checksumRunner: NewChecksumRunner(),
+
+ storage: storage,
+ cipher: cipher,
+ timer: timer,
+
+ appendCh: make(chan *CheckpointMessage),
+ metaCh: make(chan map[string]*RangeGroups),
+ lockCh: make(chan struct{}),
+ errCh: make(chan error, 1),
+ }
+
+ err := runner.initialLock(ctx)
+ if err != nil {
+ return nil, errors.Annotate(err, "Failed to initialize checkpoint lock.")
+ }
+ runner.startCheckpointLoop(ctx, tick, tick)
+ return runner, nil
+}
+
+func StartCheckpointRunner(ctx context.Context, storage storage.ExternalStorage, cipher *backuppb.CipherInfo, timer GlobalTimer) (*CheckpointRunner, error) {
+ runner := &CheckpointRunner{
+ meta: make(map[string]*RangeGroups),
+
+ checksumRunner: NewChecksumRunner(),
+
+ storage: storage,
+ cipher: cipher,
+ timer: timer,
+
+ appendCh: make(chan *CheckpointMessage),
+ metaCh: make(chan map[string]*RangeGroups),
+ lockCh: make(chan struct{}),
+ errCh: make(chan error, 1),
+ }
+
+ err := runner.initialLock(ctx)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ runner.startCheckpointLoop(ctx, tickDurationForFlush, tickDurationForLock)
+ return runner, nil
+}
+
+func (r *CheckpointRunner) FlushChecksum(ctx context.Context, tableID int64, crc64xor uint64, totalKvs uint64, totalBytes uint64, timeCost float64) error {
+ return r.checksumRunner.FlushChecksum(ctx, r.storage, tableID, crc64xor, totalKvs, totalBytes, timeCost)
+}
+
+func (r *CheckpointRunner) Append(
+ ctx context.Context,
+ groupKey string,
+ startKey []byte,
+ endKey []byte,
+ files []*backuppb.File,
+) error {
+ select {
+ case <-ctx.Done():
+ return nil
+ case err := <-r.errCh:
+ return err
+ case r.appendCh <- &CheckpointMessage{
+ GroupKey: groupKey,
+ Group: &rtree.Range{
+ StartKey: startKey,
+ EndKey: endKey,
+ Files: files,
+ },
+ }:
+ return nil
+ }
+}
+
+// Note: Cannot be parallel with `Append` function
+func (r *CheckpointRunner) WaitForFinish(ctx context.Context) {
+ // can not append anymore
+ close(r.appendCh)
+ // wait the range flusher exit
+ r.wg.Wait()
+ // wait the checksum flusher exit
+ r.checksumRunner.wg.Wait()
+ // remove the checkpoint lock
+ err := r.storage.DeleteFile(ctx, CheckpointLockPath)
+ if err != nil {
+ log.Warn("failed to remove the checkpoint lock", zap.Error(err))
+ }
+}
+
+// Send the meta to the flush goroutine, and reset the CheckpointRunner's meta
+func (r *CheckpointRunner) flushMeta(ctx context.Context, errCh chan error) error {
+ meta := r.meta
+ r.meta = make(map[string]*RangeGroups)
+ // do flush
+ select {
+ case <-ctx.Done():
+ case err := <-errCh:
+ return err
+ case r.metaCh <- meta:
+ }
+ return nil
+}
+
+func (r *CheckpointRunner) setLock(ctx context.Context, errCh chan error) error {
+ select {
+ case <-ctx.Done():
+ case err := <-errCh:
+ return err
+ case r.lockCh <- struct{}{}:
+ }
+ return nil
+}
+
+// start a goroutine to flush the meta, which is sent from `checkpoint looper`, to the external storage
+func (r *CheckpointRunner) startCheckpointRunner(ctx context.Context, wg *sync.WaitGroup) chan error {
+ errCh := make(chan error, 1)
+ wg.Add(1)
+ flushWorker := func(ctx context.Context, errCh chan error) {
+ defer wg.Done()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case meta, ok := <-r.metaCh:
+ if !ok {
+ log.Info("stop checkpoint flush worker")
+ return
+ }
+ if err := r.doFlush(ctx, meta); err != nil {
+ errCh <- err
+ return
+ }
+ case _, ok := <-r.lockCh:
+ if !ok {
+ log.Info("stop checkpoint flush worker")
+ return
+ }
+ if err := r.updateLock(ctx); err != nil {
+ errCh <- errors.Annotate(err, "Failed to update checkpoint lock.")
+ return
+ }
+ }
+ }
+ }
+
+ go flushWorker(ctx, errCh)
+ return errCh
+}
+
+func (r *CheckpointRunner) sendError(err error) {
+ select {
+ case r.errCh <- err:
+ default:
+ log.Error("errCh is blocked", logutil.ShortError(err))
+ }
+ r.checksumRunner.RecordError(err)
+}
+
+func (r *CheckpointRunner) startCheckpointLoop(ctx context.Context, tickDurationForFlush, tickDurationForLock time.Duration) {
+ r.wg.Add(1)
+ checkpointLoop := func(ctx context.Context) {
+ defer r.wg.Done()
+ cctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ var wg sync.WaitGroup
+ errCh := r.startCheckpointRunner(cctx, &wg)
+ flushTicker := time.NewTicker(tickDurationForFlush)
+ defer flushTicker.Stop()
+ lockTicker := time.NewTicker(tickDurationForLock)
+ defer lockTicker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-lockTicker.C:
+ if err := r.setLock(ctx, errCh); err != nil {
+ r.sendError(err)
+ return
+ }
+ case <-flushTicker.C:
+ if err := r.flushMeta(ctx, errCh); err != nil {
+ r.sendError(err)
+ return
+ }
+ case msg, ok := <-r.appendCh:
+ if !ok {
+ log.Info("stop checkpoint runner")
+ if err := r.flushMeta(ctx, errCh); err != nil {
+ r.sendError(err)
+ }
+ // close the channel to flush worker
+ // and wait it to consumes all the metas
+ close(r.metaCh)
+ close(r.lockCh)
+ wg.Wait()
+ return
+ }
+ groups, exist := r.meta[msg.GroupKey]
+ if !exist {
+ groups = &RangeGroups{
+ GroupKey: msg.GroupKey,
+ Groups: make([]*rtree.Range, 0),
+ }
+ r.meta[msg.GroupKey] = groups
+ }
+ groups.Groups = append(groups.Groups, msg.Group)
+ case err := <-errCh:
+ // pass flush worker's error back
+ r.sendError(err)
+ return
+ }
+ }
+ }
+
+ go checkpointLoop(ctx)
+}
+
+// flush the meta to the external storage
+func (r *CheckpointRunner) doFlush(ctx context.Context, meta map[string]*RangeGroups) error {
+ if len(meta) == 0 {
+ return nil
+ }
+
+ checkpointData := &CheckpointData{
+ DureTime: summary.NowDureTime(),
+ RangeGroupMetas: make([]*RangeGroupData, 0, len(meta)),
+ }
+
+ var fname []byte = nil
+
+ for _, group := range meta {
+ if len(group.Groups) == 0 {
+ continue
+ }
+
+ // use the first item's group-key and sub-range-key as the filename
+ if len(fname) == 0 {
+ fname = append(append([]byte(group.GroupKey), '.', '.'), group.Groups[0].StartKey...)
+ }
+
+ // Flush the metaFile to storage
+ content, err := json.Marshal(group)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ encryptBuff, iv, err := metautil.Encrypt(content, r.cipher)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ checksum := sha256.Sum256(content)
+
+ checkpointData.RangeGroupMetas = append(checkpointData.RangeGroupMetas, &RangeGroupData{
+ RangeGroupsEncriptedData: encryptBuff,
+ Checksum: checksum[:],
+ Size: len(content),
+ CipherIv: iv,
+ })
+ }
+
+ if len(checkpointData.RangeGroupMetas) > 0 {
+ data, err := json.Marshal(checkpointData)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ checksum := sha256.Sum256(fname)
+ checksumEncoded := base64.URLEncoding.EncodeToString(checksum[:])
+ path := fmt.Sprintf("%s/%s_%d.cpt", CheckpointDataDir, checksumEncoded, rand.Uint64())
+ if err := r.storage.WriteFile(ctx, path, data); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ return nil
+}
+
+type CheckpointLock struct {
+ LockId uint64 `json:"lock-id"`
+ ExpireAt int64 `json:"expire-at"`
+}
+
+// get ts with retry
+func (r *CheckpointRunner) getTS(ctx context.Context) (int64, int64, error) {
+ var (
+ p int64 = 0
+ l int64 = 0
+ retry int = 0
+ )
+ errRetry := utils.WithRetry(ctx, func() error {
+ var err error
+ p, l, err = r.timer.GetTS(ctx)
+ if err != nil {
+ retry++
+ log.Info("failed to get ts", zap.Int("retry", retry), zap.Error(err))
+ return err
+ }
+
+ return nil
+ }, utils.NewPDReqBackoffer())
+
+ return p, l, errors.Trace(errRetry)
+}
+
+// flush the lock to the external storage
+func (r *CheckpointRunner) flushLock(ctx context.Context, p int64) error {
+ lock := &CheckpointLock{
+ LockId: r.lockId,
+ ExpireAt: p + lockTimeToLive.Milliseconds(),
+ }
+ log.Info("start to flush the checkpoint lock", zap.Int64("lock-at", p), zap.Int64("expire-at", lock.ExpireAt))
+ data, err := json.Marshal(lock)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ err = r.storage.WriteFile(ctx, CheckpointLockPath, data)
+ return errors.Trace(err)
+}
+
+// check whether this lock belongs to this BR
+func (r *CheckpointRunner) checkLockFile(ctx context.Context, now int64) error {
+ data, err := r.storage.ReadFile(ctx, CheckpointLockPath)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ lock := &CheckpointLock{}
+ err = json.Unmarshal(data, lock)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if lock.ExpireAt <= now {
+ if lock.LockId > r.lockId {
+ return errors.Errorf("There are another BR(%d) running after but setting lock before this one(%d). "+
+ "Please check whether the BR is running. If not, you can retry.", lock.LockId, r.lockId)
+ }
+ if lock.LockId == r.lockId {
+ log.Warn("The lock has expired.", zap.Int64("expire-at(ms)", lock.ExpireAt), zap.Int64("now(ms)", now))
+ }
+ } else if lock.LockId != r.lockId {
+ return errors.Errorf("The existing lock will expire in %d seconds. "+
+ "There may be another BR(%d) running. If not, you can wait for the lock to expire, or delete the file `%s%s` manually.",
+ (lock.ExpireAt-now)/1000, lock.LockId, strings.TrimRight(r.storage.URI(), "/"), CheckpointLockPath)
+ }
+
+ return nil
+}
+
+// generate a new lock and flush the lock to the external storage
+func (r *CheckpointRunner) updateLock(ctx context.Context) error {
+ p, _, err := r.getTS(ctx)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if err = r.checkLockFile(ctx, p); err != nil {
+ return errors.Trace(err)
+ }
+ return errors.Trace(r.flushLock(ctx, p))
+}
+
+// Attempt to initialize the lock. Need to stop the backup when there is an unexpired locks.
+func (r *CheckpointRunner) initialLock(ctx context.Context) error {
+ p, l, err := r.getTS(ctx)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ r.lockId = oracle.ComposeTS(p, l)
+ exist, err := r.storage.FileExists(ctx, CheckpointLockPath)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if exist {
+ if err := r.checkLockFile(ctx, p); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ if err = r.flushLock(ctx, p); err != nil {
+ return errors.Trace(err)
+ }
+
+ // wait for 3 seconds to check whether the lock file is overwritten by another BR
+ time.Sleep(3 * time.Second)
+ err = r.checkLockFile(ctx, p)
+ return errors.Trace(err)
+}
+
+// walk the whole checkpoint range files and retrieve the metadatat of backed up ranges
+// and return the total time cost in the past executions
+func WalkCheckpointFile(ctx context.Context, s storage.ExternalStorage, cipher *backuppb.CipherInfo, fn func(groupKey string, rg *rtree.Range)) (time.Duration, error) {
+ // records the total time cost in the past executions
+ var pastDureTime time.Duration = 0
+ err := s.WalkDir(ctx, &storage.WalkOption{SubDir: CheckpointDataDir}, func(path string, size int64) error {
+ if strings.HasSuffix(path, ".cpt") {
+ content, err := s.ReadFile(ctx, path)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ checkpointData := &CheckpointData{}
+ if err = json.Unmarshal(content, checkpointData); err != nil {
+ return errors.Trace(err)
+ }
+
+ if checkpointData.DureTime > pastDureTime {
+ pastDureTime = checkpointData.DureTime
+ }
+ for _, meta := range checkpointData.RangeGroupMetas {
+ decryptContent, err := metautil.Decrypt(meta.RangeGroupsEncriptedData, cipher, meta.CipherIv)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ checksum := sha256.Sum256(decryptContent)
+ if !bytes.Equal(meta.Checksum, checksum[:]) {
+ log.Error("checkpoint checksum info's checksum mismatch, skip it",
+ zap.ByteString("expect", meta.Checksum),
+ zap.ByteString("got", checksum[:]),
+ )
+ continue
+ }
+
+ group := &RangeGroups{}
+ if err = json.Unmarshal(decryptContent, group); err != nil {
+ return errors.Trace(err)
+ }
+
+ for _, g := range group.Groups {
+ fn(group.GroupKey, g)
+ }
+ }
+ }
+ return nil
+ })
+
+ return pastDureTime, errors.Trace(err)
+}
+
+type CheckpointMetadata struct {
+ GCServiceId string `json:"gc-service-id"`
+ ConfigHash []byte `json:"config-hash"`
+ BackupTS uint64 `json:"backup-ts"`
+ Ranges []rtree.Range `json:"ranges"`
+
+ CheckpointChecksum map[int64]*ChecksumItem `json:"-"`
+ CheckpointDataMap map[string]rtree.RangeTree `json:"-"`
+}
+
+// load checkpoint metadata from the external storage
+func LoadCheckpointMetadata(ctx context.Context, s storage.ExternalStorage) (*CheckpointMetadata, error) {
+ data, err := s.ReadFile(ctx, CheckpointMetaPath)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ m := &CheckpointMetadata{}
+ err = json.Unmarshal(data, m)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ m.CheckpointChecksum, err = loadCheckpointChecksum(ctx, s)
+ return m, errors.Trace(err)
+}
+
+// walk the whole checkpoint checksum files and retrieve checksum information of tables calculated
+func loadCheckpointChecksum(ctx context.Context, s storage.ExternalStorage) (map[int64]*ChecksumItem, error) {
+ checkpointChecksum := make(map[int64]*ChecksumItem)
+
+ err := s.WalkDir(ctx, &storage.WalkOption{SubDir: CheckpointChecksumDir}, func(path string, size int64) error {
+ data, err := s.ReadFile(ctx, path)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ info := &ChecksumInfo{}
+ err = json.Unmarshal(data, info)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ checksum := sha256.Sum256(info.Content)
+ if !bytes.Equal(info.Checksum, checksum[:]) {
+ log.Error("checkpoint checksum info's checksum mismatch, skip it",
+ zap.ByteString("expect", info.Checksum),
+ zap.ByteString("got", checksum[:]),
+ )
+ return nil
+ }
+
+ items := &ChecksumItems{}
+ err = json.Unmarshal(info.Content, items)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ for _, c := range items.Items {
+ checkpointChecksum[c.TableID] = c
+ }
+ return nil
+ })
+ return checkpointChecksum, errors.Trace(err)
+}
+
+// save the checkpoint metadata into the external storage
+func SaveCheckpointMetadata(ctx context.Context, s storage.ExternalStorage, meta *CheckpointMetadata) error {
+ data, err := json.Marshal(meta)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ err = s.WriteFile(ctx, CheckpointMetaPath, data)
+ return errors.Trace(err)
+}
diff --git a/br/pkg/checkpoint/checkpoint_test.go b/br/pkg/checkpoint/checkpoint_test.go
new file mode 100644
index 0000000000000..29d8b5aa993ac
--- /dev/null
+++ b/br/pkg/checkpoint/checkpoint_test.go
@@ -0,0 +1,229 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package checkpoint_test
+
+import (
+ "context"
+ "encoding/json"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/kvproto/pkg/encryptionpb"
+ "github.com/pingcap/tidb/br/pkg/checkpoint"
+ "github.com/pingcap/tidb/br/pkg/rtree"
+ "github.com/pingcap/tidb/br/pkg/storage"
+ "github.com/stretchr/testify/require"
+ "github.com/tikv/client-go/v2/oracle"
+)
+
+func TestCheckpointMeta(t *testing.T) {
+ ctx := context.Background()
+ base := t.TempDir()
+ s, err := storage.NewLocalStorage(base)
+ require.NoError(t, err)
+
+ checkpointMeta := &checkpoint.CheckpointMetadata{
+ ConfigHash: []byte("123456"),
+ BackupTS: 123456,
+ }
+
+ err = checkpoint.SaveCheckpointMetadata(ctx, s, checkpointMeta)
+ require.NoError(t, err)
+
+ checkpointMeta2, err := checkpoint.LoadCheckpointMetadata(ctx, s)
+ require.NoError(t, err)
+ require.Equal(t, checkpointMeta.ConfigHash, checkpointMeta2.ConfigHash)
+ require.Equal(t, checkpointMeta.BackupTS, checkpointMeta2.BackupTS)
+}
+
+type mockTimer struct {
+ p int64
+ l int64
+}
+
+func NewMockTimer(p, l int64) *mockTimer {
+ return &mockTimer{p: p, l: l}
+}
+
+func (t *mockTimer) GetTS(ctx context.Context) (int64, int64, error) {
+ return t.p, t.l, nil
+}
+
+func TestCheckpointRunner(t *testing.T) {
+ ctx := context.Background()
+ base := t.TempDir()
+ s, err := storage.NewLocalStorage(base)
+ require.NoError(t, err)
+ os.MkdirAll(base+checkpoint.CheckpointDataDir, 0o755)
+ os.MkdirAll(base+checkpoint.CheckpointChecksumDir, 0o755)
+
+ cipher := &backuppb.CipherInfo{
+ CipherType: encryptionpb.EncryptionMethod_AES256_CTR,
+ CipherKey: []byte("01234567890123456789012345678901"),
+ }
+ checkpointRunner, err := checkpoint.StartCheckpointRunnerForTest(ctx, s, cipher, 5*time.Second, NewMockTimer(10, 10))
+ require.NoError(t, err)
+
+ data := map[string]struct {
+ StartKey string
+ EndKey string
+ Name string
+ Name2 string
+ }{
+ "a": {
+ StartKey: "a",
+ EndKey: "b",
+ Name: "c",
+ Name2: "d",
+ },
+ "A": {
+ StartKey: "A",
+ EndKey: "B",
+ Name: "C",
+ Name2: "D",
+ },
+ "1": {
+ StartKey: "1",
+ EndKey: "2",
+ Name: "3",
+ Name2: "4",
+ },
+ }
+
+ data2 := map[string]struct {
+ StartKey string
+ EndKey string
+ Name string
+ Name2 string
+ }{
+ "+": {
+ StartKey: "+",
+ EndKey: "-",
+ Name: "*",
+ Name2: "/",
+ },
+ }
+
+ for _, d := range data {
+ err = checkpointRunner.Append(ctx, "a", []byte(d.StartKey), []byte(d.EndKey), []*backuppb.File{
+ {Name: d.Name},
+ {Name: d.Name2},
+ })
+ require.NoError(t, err)
+ }
+
+ checkpointRunner.FlushChecksum(ctx, 1, 1, 1, 1, checkpoint.MaxChecksumTotalCost-20.0)
+ checkpointRunner.FlushChecksum(ctx, 2, 2, 2, 2, 40.0)
+ // now the checksum is flushed, because the total time cost is larger than `MaxChecksumTotalCost`
+ checkpointRunner.FlushChecksum(ctx, 3, 3, 3, 3, checkpoint.MaxChecksumTotalCost-20.0)
+ time.Sleep(6 * time.Second)
+ // the checksum has not been flushed even though after 6 seconds,
+ // because the total time cost is less than `MaxChecksumTotalCost`
+ checkpointRunner.FlushChecksum(ctx, 4, 4, 4, 4, 40.0)
+
+ for _, d := range data2 {
+ err = checkpointRunner.Append(ctx, "+", []byte(d.StartKey), []byte(d.EndKey), []*backuppb.File{
+ {Name: d.Name},
+ {Name: d.Name2},
+ })
+ require.NoError(t, err)
+ }
+
+ checkpointRunner.WaitForFinish(ctx)
+
+ checker := func(groupKey string, resp *rtree.Range) {
+ require.NotNil(t, resp)
+ d, ok := data[string(resp.StartKey)]
+ if !ok {
+ d, ok = data2[string(resp.StartKey)]
+ require.True(t, ok)
+ }
+ require.Equal(t, d.StartKey, string(resp.StartKey))
+ require.Equal(t, d.EndKey, string(resp.EndKey))
+ require.Equal(t, d.Name, resp.Files[0].Name)
+ require.Equal(t, d.Name2, resp.Files[1].Name)
+ }
+
+ _, err = checkpoint.WalkCheckpointFile(ctx, s, cipher, checker)
+ require.NoError(t, err)
+
+ checkpointMeta := &checkpoint.CheckpointMetadata{
+ ConfigHash: []byte("123456"),
+ BackupTS: 123456,
+ }
+
+ err = checkpoint.SaveCheckpointMetadata(ctx, s, checkpointMeta)
+ require.NoError(t, err)
+ meta, err := checkpoint.LoadCheckpointMetadata(ctx, s)
+ require.NoError(t, err)
+
+ var i int64
+ for i = 1; i <= 4; i++ {
+ require.Equal(t, meta.CheckpointChecksum[i].Crc64xor, uint64(i))
+ }
+
+ // only 2 checksum files exists, they are t2_and__ and t4_and__
+ count := 0
+ err = s.WalkDir(ctx, &storage.WalkOption{SubDir: checkpoint.CheckpointChecksumDir}, func(s string, i int64) error {
+ count += 1
+ if !strings.Contains(s, "t2") {
+ require.True(t, strings.Contains(s, "t4"))
+ }
+ return nil
+ })
+ require.NoError(t, err)
+ require.Equal(t, count, 2)
+}
+
+func getLockData(p, l int64) ([]byte, error) {
+ lock := checkpoint.CheckpointLock{
+ LockId: oracle.ComposeTS(p, l),
+ ExpireAt: p + 10,
+ }
+ return json.Marshal(lock)
+}
+
+func TestCheckpointRunnerLock(t *testing.T) {
+ ctx := context.Background()
+ base := t.TempDir()
+ s, err := storage.NewLocalStorage(base)
+ require.NoError(t, err)
+ os.MkdirAll(base+checkpoint.CheckpointDataDir, 0o755)
+ os.MkdirAll(base+checkpoint.CheckpointChecksumDir, 0o755)
+
+ cipher := &backuppb.CipherInfo{
+ CipherType: encryptionpb.EncryptionMethod_AES256_CTR,
+ CipherKey: []byte("01234567890123456789012345678901"),
+ }
+
+ data, err := getLockData(10, 20)
+ require.NoError(t, err)
+ err = s.WriteFile(ctx, checkpoint.CheckpointLockPath, data)
+ require.NoError(t, err)
+
+ _, err = checkpoint.StartCheckpointRunnerForTest(ctx, s, cipher, 5*time.Second, NewMockTimer(10, 10))
+ require.Error(t, err)
+
+ runner, err := checkpoint.StartCheckpointRunnerForTest(ctx, s, cipher, 5*time.Second, NewMockTimer(30, 10))
+ require.NoError(t, err)
+
+ _, err = checkpoint.StartCheckpointRunnerForTest(ctx, s, cipher, 5*time.Second, NewMockTimer(40, 10))
+ require.Error(t, err)
+
+ runner.WaitForFinish(ctx)
+}
diff --git a/br/pkg/checksum/executor.go b/br/pkg/checksum/executor.go
index c30ae49fccdca..300caf706ef25 100644
--- a/br/pkg/checksum/executor.go
+++ b/br/pkg/checksum/executor.go
@@ -27,6 +27,9 @@ type ExecutorBuilder struct {
oldTable *metautil.Table
concurrency uint
+
+ oldKeyspace []byte
+ newKeyspace []byte
}
// NewExecutorBuilder returns a new executor builder.
@@ -51,9 +54,26 @@ func (builder *ExecutorBuilder) SetConcurrency(conc uint) *ExecutorBuilder {
return builder
}
+func (builder *ExecutorBuilder) SetOldKeyspace(keyspace []byte) *ExecutorBuilder {
+ builder.oldKeyspace = keyspace
+ return builder
+}
+
+func (builder *ExecutorBuilder) SetNewKeyspace(keyspace []byte) *ExecutorBuilder {
+ builder.newKeyspace = keyspace
+ return builder
+}
+
// Build builds a checksum executor.
func (builder *ExecutorBuilder) Build() (*Executor, error) {
- reqs, err := buildChecksumRequest(builder.table, builder.oldTable, builder.ts, builder.concurrency)
+ reqs, err := buildChecksumRequest(
+ builder.table,
+ builder.oldTable,
+ builder.ts,
+ builder.concurrency,
+ builder.oldKeyspace,
+ builder.newKeyspace,
+ )
if err != nil {
return nil, errors.Trace(err)
}
@@ -65,6 +85,8 @@ func buildChecksumRequest(
oldTable *metautil.Table,
startTS uint64,
concurrency uint,
+ oldKeyspace []byte,
+ newKeyspace []byte,
) ([]*kv.Request, error) {
var partDefs []model.PartitionDefinition
if part := newTable.Partition; part != nil {
@@ -76,7 +98,7 @@ func buildChecksumRequest(
if oldTable != nil {
oldTableID = oldTable.Info.ID
}
- rs, err := buildRequest(newTable, newTable.ID, oldTable, oldTableID, startTS, concurrency)
+ rs, err := buildRequest(newTable, newTable.ID, oldTable, oldTableID, startTS, concurrency, oldKeyspace, newKeyspace)
if err != nil {
return nil, errors.Trace(err)
}
@@ -91,7 +113,7 @@ func buildChecksumRequest(
}
}
}
- rs, err := buildRequest(newTable, partDef.ID, oldTable, oldPartID, startTS, concurrency)
+ rs, err := buildRequest(newTable, partDef.ID, oldTable, oldPartID, startTS, concurrency, oldKeyspace, newKeyspace)
if err != nil {
return nil, errors.Trace(err)
}
@@ -108,9 +130,11 @@ func buildRequest(
oldTableID int64,
startTS uint64,
concurrency uint,
+ oldKeyspace []byte,
+ newKeyspace []byte,
) ([]*kv.Request, error) {
reqs := make([]*kv.Request, 0)
- req, err := buildTableRequest(tableInfo, tableID, oldTable, oldTableID, startTS, concurrency)
+ req, err := buildTableRequest(tableInfo, tableID, oldTable, oldTableID, startTS, concurrency, oldKeyspace, newKeyspace)
if err != nil {
return nil, errors.Trace(err)
}
@@ -139,7 +163,7 @@ func buildRequest(
}
}
req, err = buildIndexRequest(
- tableID, indexInfo, oldTableID, oldIndexInfo, startTS, concurrency)
+ tableID, indexInfo, oldTableID, oldIndexInfo, startTS, concurrency, oldKeyspace, newKeyspace)
if err != nil {
return nil, errors.Trace(err)
}
@@ -156,12 +180,14 @@ func buildTableRequest(
oldTableID int64,
startTS uint64,
concurrency uint,
+ oldKeyspace []byte,
+ newKeyspace []byte,
) (*kv.Request, error) {
var rule *tipb.ChecksumRewriteRule
if oldTable != nil {
rule = &tipb.ChecksumRewriteRule{
- OldPrefix: tablecodec.GenTableRecordPrefix(oldTableID),
- NewPrefix: tablecodec.GenTableRecordPrefix(tableID),
+ OldPrefix: append(append([]byte{}, oldKeyspace...), tablecodec.GenTableRecordPrefix(oldTableID)...),
+ NewPrefix: append(append([]byte{}, newKeyspace...), tablecodec.GenTableRecordPrefix(tableID)...),
}
}
@@ -195,12 +221,14 @@ func buildIndexRequest(
oldIndexInfo *model.IndexInfo,
startTS uint64,
concurrency uint,
+ oldKeyspace []byte,
+ newKeyspace []byte,
) (*kv.Request, error) {
var rule *tipb.ChecksumRewriteRule
if oldIndexInfo != nil {
rule = &tipb.ChecksumRewriteRule{
- OldPrefix: tablecodec.EncodeTableIndexPrefix(oldTableID, oldIndexInfo.ID),
- NewPrefix: tablecodec.EncodeTableIndexPrefix(tableID, indexInfo.ID),
+ OldPrefix: append(append([]byte{}, oldKeyspace...), tablecodec.EncodeTableIndexPrefix(oldTableID, oldIndexInfo.ID)...),
+ NewPrefix: append(append([]byte{}, newKeyspace...), tablecodec.EncodeTableIndexPrefix(tableID, indexInfo.ID)...),
}
}
checksum := &tipb.ChecksumRequest{
diff --git a/br/pkg/checksum/executor_test.go b/br/pkg/checksum/executor_test.go
index adcaed9c314f9..876103bc055a2 100644
--- a/br/pkg/checksum/executor_test.go
+++ b/br/pkg/checksum/executor_test.go
@@ -104,7 +104,7 @@ func TestChecksum(t *testing.T) {
first = false
ranges, err := backup.BuildTableRanges(tableInfo3)
require.NoError(t, err)
- require.Equalf(t, ranges[:1], req.KeyRanges, "%v", req.KeyRanges)
+ require.Equalf(t, ranges[:1], req.KeyRanges.FirstPartitionRange(), "%v", req.KeyRanges.FirstPartitionRange())
}
return nil
}))
diff --git a/br/pkg/config/BUILD.bazel b/br/pkg/config/BUILD.bazel
index 942fd3462a7b9..610787d039894 100644
--- a/br/pkg/config/BUILD.bazel
+++ b/br/pkg/config/BUILD.bazel
@@ -15,8 +15,10 @@ go_library(
go_test(
name = "config_test",
+ timeout = "short",
srcs = ["ebs_test.go"],
data = ["ebs_backup.json"],
embed = [":config"],
+ flaky = True,
deps = ["@com_github_stretchr_testify//require"],
)
diff --git a/br/pkg/conn/BUILD.bazel b/br/pkg/conn/BUILD.bazel
index fc88f174394f3..da06e516f37ac 100644
--- a/br/pkg/conn/BUILD.bazel
+++ b/br/pkg/conn/BUILD.bazel
@@ -13,6 +13,7 @@ go_library(
"//br/pkg/pdutil",
"//br/pkg/utils",
"//br/pkg/version",
+ "//config",
"//domain",
"//kv",
"@com_github_docker_go_units//:go-units",
diff --git a/br/pkg/conn/conn.go b/br/pkg/conn/conn.go
index 5adbe0a33ab1c..187bc5ef1db79 100644
--- a/br/pkg/conn/conn.go
+++ b/br/pkg/conn/conn.go
@@ -26,6 +26,7 @@ import (
"github.com/pingcap/tidb/br/pkg/pdutil"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/br/pkg/version"
+ "github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/kv"
"github.com/tikv/client-go/v2/oracle"
@@ -174,7 +175,8 @@ func NewMgr(
}
// Disable GC because TiDB enables GC already.
- storage, err := g.Open(fmt.Sprintf("tikv://%s?disableGC=true", pdAddrs), securityOption)
+ path := fmt.Sprintf("tikv://%s?disableGC=true&keyspaceName=%s", pdAddrs, config.GetGlobalKeyspaceName())
+ storage, err := g.Open(path, securityOption)
if err != nil {
return nil, errors.Trace(err)
}
@@ -191,7 +193,6 @@ func NewMgr(
return nil, errors.Trace(err)
}
// we must check tidb(tikv version) any time after concurrent ddl feature implemented in v6.2.
- // when tidb < 6.2 we need set EnableConcurrentDDL false to make ddl works.
// we will keep this check until 7.0, which allow the breaking changes.
// NOTE: must call it after domain created!
// FIXME: remove this check in v7.0
@@ -281,7 +282,8 @@ func (mgr *Mgr) GetTS(ctx context.Context) (uint64, error) {
}
// GetMergeRegionSizeAndCount returns the tikv config `coprocessor.region-split-size` and `coprocessor.region-split-key`.
-func (mgr *Mgr) GetMergeRegionSizeAndCount(ctx context.Context, client *http.Client) (uint64, uint64, error) {
+// returns the default config when failed.
+func (mgr *Mgr) GetMergeRegionSizeAndCount(ctx context.Context, client *http.Client) (uint64, uint64) {
regionSplitSize := DefaultMergeRegionSizeBytes
regionSplitKeys := DefaultMergeRegionKeyCount
type coprocessor struct {
@@ -310,9 +312,10 @@ func (mgr *Mgr) GetMergeRegionSizeAndCount(ctx context.Context, client *http.Cli
return nil
})
if err != nil {
- return 0, 0, errors.Trace(err)
+ log.Warn("meet error when getting config from TiKV; using default", logutil.ShortError(err))
+ return DefaultMergeRegionSizeBytes, DefaultMergeRegionKeyCount
}
- return regionSplitSize, regionSplitKeys, nil
+ return regionSplitSize, regionSplitKeys
}
// GetConfigFromTiKV get configs from all alive tikv stores.
diff --git a/br/pkg/conn/conn_test.go b/br/pkg/conn/conn_test.go
index 01ce8bc08203e..fc822fac123d9 100644
--- a/br/pkg/conn/conn_test.go
+++ b/br/pkg/conn/conn_test.go
@@ -292,6 +292,38 @@ func TestGetMergeRegionSizeAndCount(t *testing.T) {
regionSplitSize: DefaultMergeRegionSizeBytes,
regionSplitKeys: DefaultMergeRegionKeyCount,
},
+ {
+ stores: []*metapb.Store{
+ {
+ Id: 1,
+ State: metapb.StoreState_Up,
+ Labels: []*metapb.StoreLabel{
+ {
+ Key: "engine",
+ Value: "tiflash",
+ },
+ },
+ },
+ {
+ Id: 2,
+ State: metapb.StoreState_Up,
+ Labels: []*metapb.StoreLabel{
+ {
+ Key: "engine",
+ Value: "tikv",
+ },
+ },
+ },
+ },
+ content: []string{
+ "",
+ // Assuming the TiKV has failed due to some reason.
+ "",
+ },
+ // no tikv detected in this case
+ regionSplitSize: DefaultMergeRegionSizeBytes,
+ regionSplitKeys: DefaultMergeRegionKeyCount,
+ },
{
stores: []*metapb.Store{
{
@@ -388,8 +420,7 @@ func TestGetMergeRegionSizeAndCount(t *testing.T) {
httpCli := mockServer.Client()
mgr := &Mgr{PdController: &pdutil.PdController{}}
mgr.PdController.SetPDClient(pdCli)
- rs, rk, err := mgr.GetMergeRegionSizeAndCount(ctx, httpCli)
- require.NoError(t, err)
+ rs, rk := mgr.GetMergeRegionSizeAndCount(ctx, httpCli)
require.Equal(t, ca.regionSplitSize, rs)
require.Equal(t, ca.regionSplitKeys, rk)
mockServer.Close()
diff --git a/br/pkg/errors/errors.go b/br/pkg/errors/errors.go
index 07e9fb6317cb9..2b7d76e28d795 100644
--- a/br/pkg/errors/errors.go
+++ b/br/pkg/errors/errors.go
@@ -83,8 +83,9 @@ var (
ErrStorageInvalidPermission = errors.Normalize("external storage permission", errors.RFCCodeText("BR:ExternalStorage:ErrStorageInvalidPermission"))
// Snapshot restore
- ErrRestoreTotalKVMismatch = errors.Normalize("restore total tikvs mismatch", errors.RFCCodeText("BR:EBS:ErrRestoreTotalKVMismatch"))
- ErrRestoreInvalidPeer = errors.Normalize("restore met a invalid peer", errors.RFCCodeText("BR:EBS:ErrRestoreInvalidPeer"))
+ ErrRestoreTotalKVMismatch = errors.Normalize("restore total tikvs mismatch", errors.RFCCodeText("BR:EBS:ErrRestoreTotalKVMismatch"))
+ ErrRestoreInvalidPeer = errors.Normalize("restore met a invalid peer", errors.RFCCodeText("BR:EBS:ErrRestoreInvalidPeer"))
+ ErrRestoreRegionWithoutPeer = errors.Normalize("restore met a region without any peer", errors.RFCCodeText("BR:EBS:ErrRestoreRegionWithoutPeer"))
// Errors reported from TiKV.
ErrKVStorage = errors.Normalize("tikv storage occur I/O error", errors.RFCCodeText("BR:KV:ErrKVStorage"))
diff --git a/br/pkg/errors/errors_test.go b/br/pkg/errors/errors_test.go
index a6f4c412280cc..fab655ba60c7e 100644
--- a/br/pkg/errors/errors_test.go
+++ b/br/pkg/errors/errors_test.go
@@ -22,3 +22,9 @@ func TestIsContextCanceled(t *testing.T) {
require.True(t, berrors.IsContextCanceled(&url.Error{Err: context.Canceled}))
require.True(t, berrors.IsContextCanceled(&url.Error{Err: context.DeadlineExceeded}))
}
+
+func TestEqual(t *testing.T) {
+ err := errors.Annotate(berrors.ErrPDBatchScanRegion, "test error equla")
+ r := berrors.ErrPDBatchScanRegion.Equal(err)
+ require.True(t, r)
+}
diff --git a/br/pkg/gluetidb/glue.go b/br/pkg/gluetidb/glue.go
index 5483380d03e65..abe239f7723dc 100644
--- a/br/pkg/gluetidb/glue.go
+++ b/br/pkg/gluetidb/glue.go
@@ -61,6 +61,10 @@ type tidbSession struct {
// GetDomain implements glue.Glue.
func (Glue) GetDomain(store kv.Storage) (*domain.Domain, error) {
+ initStatsSe, err := session.CreateSession(store)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
se, err := session.CreateSession(store)
if err != nil {
return nil, errors.Trace(err)
@@ -69,8 +73,12 @@ func (Glue) GetDomain(store kv.Storage) (*domain.Domain, error) {
if err != nil {
return nil, errors.Trace(err)
}
+ err = session.InitMDLVariable(store)
+ if err != nil {
+ return nil, err
+ }
// create stats handler for backup and restore.
- err = dom.UpdateTableStatsLoop(se)
+ err = dom.UpdateTableStatsLoop(se, initStatsSe)
if err != nil {
return nil, errors.Trace(err)
}
@@ -132,6 +140,10 @@ func (g Glue) UseOneShotSession(store kv.Storage, closeDomain bool, fn func(glue
if err != nil {
return errors.Trace(err)
}
+ if err = session.InitMDLVariable(store); err != nil {
+ return errors.Trace(err)
+ }
+
// because domain was created during the whole program exists.
// and it will register br info to info syncer.
// we'd better close it as soon as possible.
@@ -203,11 +215,36 @@ func (gs *tidbSession) CreatePlacementPolicy(ctx context.Context, policy *model.
return d.CreatePlacementPolicyWithInfo(gs.se, policy, ddl.OnExistIgnore)
}
+// SplitBatchCreateTable provide a way to split batch into small batch when batch size is large than 6 MB.
+// The raft entry has limit size of 6 MB, a batch of CreateTables may hit this limitation
+// TODO: shall query string be set for each split batch create, it looks does not matter if we set once for all.
+func (gs *tidbSession) SplitBatchCreateTable(schema model.CIStr, info []*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error {
+ var err error
+ d := domain.GetDomain(gs.se).DDL()
+ if err = d.BatchCreateTableWithInfo(gs.se, schema, info, append(cs, ddl.OnExistIgnore)...); kv.ErrEntryTooLarge.Equal(err) {
+ if len(info) == 1 {
+ return err
+ }
+ mid := len(info) / 2
+ err = gs.SplitBatchCreateTable(schema, info[:mid])
+ if err != nil {
+ return err
+ }
+ err = gs.SplitBatchCreateTable(schema, info[mid:])
+ if err != nil {
+ return err
+ }
+ return nil
+ }
+ return err
+}
+
// CreateTables implements glue.BatchCreateTableSession.
func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error {
- d := domain.GetDomain(gs.se).DDL()
var dbName model.CIStr
+ // Disable foreign key check when batch create tables.
+ gs.se.GetSessionVars().ForeignKeyChecks = false
for db, tablesInDB := range tables {
dbName = model.NewCIStr(db)
queryBuilder := strings.Builder{}
@@ -231,8 +268,7 @@ func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*mo
cloneTables = append(cloneTables, table)
}
gs.se.SetValue(sessionctx.QueryString, queryBuilder.String())
- err := d.BatchCreateTableWithInfo(gs.se, dbName, cloneTables, append(cs, ddl.OnExistIgnore)...)
- if err != nil {
+ if err := gs.SplitBatchCreateTable(dbName, cloneTables); err != nil {
//It is possible to failure when TiDB does not support model.ActionCreateTables.
//In this circumstance, BatchCreateTableWithInfo returns errno.ErrInvalidDDLJob,
//we fall back to old way that creating table one by one
@@ -252,6 +288,8 @@ func (gs *tidbSession) CreateTable(ctx context.Context, dbName model.CIStr, tabl
return errors.Trace(err)
}
gs.se.SetValue(sessionctx.QueryString, query)
+ // Disable foreign key check when batch create tables.
+ gs.se.GetSessionVars().ForeignKeyChecks = false
// Clone() does not clone partitions yet :(
table = table.Clone()
if table.Partition != nil {
@@ -303,7 +341,8 @@ func (gs *tidbSession) showCreatePlacementPolicy(policy *model.PolicyInfo) strin
// mockSession is used for test.
type mockSession struct {
- se session.Session
+ se session.Session
+ globalVars map[string]string
}
// GetSessionCtx implements glue.Glue
@@ -368,12 +407,16 @@ func (s *mockSession) Close() {
// GetGlobalVariables implements glue.Session.
func (s *mockSession) GetGlobalVariable(name string) (string, error) {
- return "true", nil
+ if ret, ok := s.globalVars[name]; ok {
+ return ret, nil
+ }
+ return "True", nil
}
// MockGlue only used for test
type MockGlue struct {
- se session.Session
+ se session.Session
+ GlobalVars map[string]string
}
func (m *MockGlue) SetSession(se session.Session) {
@@ -388,7 +431,8 @@ func (*MockGlue) GetDomain(store kv.Storage) (*domain.Domain, error) {
// CreateSession implements glue.Glue.
func (m *MockGlue) CreateSession(store kv.Storage) (glue.Session, error) {
glueSession := &mockSession{
- se: m.se,
+ se: m.se,
+ globalVars: m.GlobalVars,
}
return glueSession, nil
}
diff --git a/br/pkg/lightning/BUILD.bazel b/br/pkg/lightning/BUILD.bazel
index 6b3c5f8e3ce31..fc646195c6e2b 100644
--- a/br/pkg/lightning/BUILD.bazel
+++ b/br/pkg/lightning/BUILD.bazel
@@ -28,7 +28,10 @@ go_library(
"//br/pkg/version/build",
"//expression",
"//planner/core",
+ "//util",
"//util/promutil",
+ "@com_github_go_sql_driver_mysql//:mysql",
+ "@com_github_google_uuid//:uuid",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/import_sstpb",
@@ -37,6 +40,7 @@ go_library(
"@com_github_prometheus_client_golang//prometheus/promhttp",
"@com_github_shurcool_httpgzip//:httpgzip",
"@org_golang_x_exp//slices",
+ "@org_uber_go_atomic//:atomic",
"@org_uber_go_zap//:zap",
"@org_uber_go_zap//zapcore",
],
diff --git a/br/pkg/lightning/backend/kv/BUILD.bazel b/br/pkg/lightning/backend/kv/BUILD.bazel
index ea2cfefc2440e..b0da8a0e7deb4 100644
--- a/br/pkg/lightning/backend/kv/BUILD.bazel
+++ b/br/pkg/lightning/backend/kv/BUILD.bazel
@@ -46,13 +46,14 @@ go_test(
name = "kv_test",
timeout = "short",
srcs = [
+ "session_internal_test.go",
"session_test.go",
"sql2kv_test.go",
],
+ embed = [":kv"],
flaky = True,
race = "on",
deps = [
- ":kv",
"//br/pkg/lightning/common",
"//br/pkg/lightning/log",
"//br/pkg/lightning/verification",
@@ -69,6 +70,7 @@ go_test(
"//tablecodec",
"//types",
"//util/mock",
+ "@com_github_docker_go_units//:go-units",
"@com_github_stretchr_testify//require",
"@org_uber_go_zap//:zap",
"@org_uber_go_zap//zapcore",
diff --git a/br/pkg/lightning/backend/kv/allocator.go b/br/pkg/lightning/backend/kv/allocator.go
index 02f46ea8c7e36..14703e1143a45 100644
--- a/br/pkg/lightning/backend/kv/allocator.go
+++ b/br/pkg/lightning/backend/kv/allocator.go
@@ -34,6 +34,7 @@ type panickingAllocator struct {
func NewPanickingAllocators(base int64) autoid.Allocators {
sharedBase := &base
return autoid.NewAllocators(
+ false,
&panickingAllocator{base: sharedBase, ty: autoid.RowIDAllocType},
&panickingAllocator{base: sharedBase, ty: autoid.AutoIncrementType},
&panickingAllocator{base: sharedBase, ty: autoid.AutoRandomType},
diff --git a/br/pkg/lightning/backend/kv/session.go b/br/pkg/lightning/backend/kv/session.go
index 1cc261b677fe4..a8c5b5970cdf8 100644
--- a/br/pkg/lightning/backend/kv/session.go
+++ b/br/pkg/lightning/backend/kv/session.go
@@ -38,6 +38,8 @@ import (
"go.uber.org/zap"
)
+const maxAvailableBufSize int = 20
+
// invalidIterator is a trimmed down Iterator type which is invalid.
type invalidIterator struct {
kv.Iterator
@@ -92,6 +94,12 @@ func (mb *kvMemBuf) Recycle(buf *bytesBuf) {
buf.idx = 0
buf.cap = len(buf.buf)
mb.Lock()
+ if len(mb.availableBufs) >= maxAvailableBufSize {
+ // too many byte buffers, evict one byte buffer and continue
+ evictedByteBuf := mb.availableBufs[0]
+ evictedByteBuf.destroy()
+ mb.availableBufs = mb.availableBufs[1:]
+ }
mb.availableBufs = append(mb.availableBufs, buf)
mb.Unlock()
}
@@ -99,8 +107,20 @@ func (mb *kvMemBuf) Recycle(buf *bytesBuf) {
func (mb *kvMemBuf) AllocateBuf(size int) {
mb.Lock()
size = mathutil.Max(units.MiB, int(utils.NextPowerOfTwo(int64(size)))*2)
- if len(mb.availableBufs) > 0 && mb.availableBufs[0].cap >= size {
- mb.buf = mb.availableBufs[0]
+ var (
+ existingBuf *bytesBuf
+ existingBufIdx int
+ )
+ for i, buf := range mb.availableBufs {
+ if buf.cap >= size {
+ existingBuf = buf
+ existingBufIdx = i
+ break
+ }
+ }
+ if existingBuf != nil {
+ mb.buf = existingBuf
+ mb.availableBufs[existingBufIdx] = mb.availableBufs[0]
mb.availableBufs = mb.availableBufs[1:]
} else {
mb.buf = newBytesBuf(size)
diff --git a/br/pkg/lightning/backend/kv/session_internal_test.go b/br/pkg/lightning/backend/kv/session_internal_test.go
new file mode 100644
index 0000000000000..97ebd8cc82d1b
--- /dev/null
+++ b/br/pkg/lightning/backend/kv/session_internal_test.go
@@ -0,0 +1,126 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kv
+
+import (
+ "testing"
+
+ "github.com/docker/go-units"
+ "github.com/stretchr/testify/require"
+)
+
+func TestKVMemBufInterweaveAllocAndRecycle(t *testing.T) {
+ type testCase struct {
+ AllocSizes []int
+ FinalAvailableByteBufCaps []int
+ }
+ for _, tc := range []testCase{
+ {
+ AllocSizes: []int{
+ 1 * units.MiB,
+ 2 * units.MiB,
+ 3 * units.MiB,
+ 4 * units.MiB,
+ 5 * units.MiB,
+ },
+ // [2] => [2,4] => [2,4,8] => [4,2,8] => [4,2,8,16]
+ FinalAvailableByteBufCaps: []int{
+ 4 * units.MiB,
+ 2 * units.MiB,
+ 8 * units.MiB,
+ 16 * units.MiB,
+ },
+ },
+ {
+ AllocSizes: []int{
+ 5 * units.MiB,
+ 4 * units.MiB,
+ 3 * units.MiB,
+ 2 * units.MiB,
+ 1 * units.MiB,
+ },
+ // [16] => [16] => [16] => [16] => [16]
+ FinalAvailableByteBufCaps: []int{16 * units.MiB},
+ },
+ {
+ AllocSizes: []int{5, 4, 3, 2, 1},
+ // [1] => [1] => [1] => [1] => [1]
+ FinalAvailableByteBufCaps: []int{1 * units.MiB},
+ },
+ {
+ AllocSizes: []int{
+ 1 * units.MiB,
+ 2 * units.MiB,
+ 3 * units.MiB,
+ 2 * units.MiB,
+ 1 * units.MiB,
+ 5 * units.MiB,
+ },
+ // [2] => [2,4] => [2,4,8] => [2,8,4] => [8,4,2] => [8,4,2,16]
+ FinalAvailableByteBufCaps: []int{
+ 8 * units.MiB,
+ 4 * units.MiB,
+ 2 * units.MiB,
+ 16 * units.MiB,
+ },
+ },
+ } {
+ testKVMemBuf := &kvMemBuf{}
+ for _, allocSize := range tc.AllocSizes {
+ testKVMemBuf.AllocateBuf(allocSize)
+ testKVMemBuf.Recycle(testKVMemBuf.buf)
+ }
+ require.Equal(t, len(tc.FinalAvailableByteBufCaps), len(testKVMemBuf.availableBufs))
+ for i, bb := range testKVMemBuf.availableBufs {
+ require.Equal(t, tc.FinalAvailableByteBufCaps[i], bb.cap)
+ }
+ }
+}
+
+func TestKVMemBufBatchAllocAndRecycle(t *testing.T) {
+ type testCase struct {
+ AllocSizes []int
+ FinalAvailableByteBufCaps []int
+ }
+ testKVMemBuf := &kvMemBuf{}
+ bBufs := []*bytesBuf{}
+ for i := 0; i < maxAvailableBufSize; i++ {
+ testKVMemBuf.AllocateBuf(1 * units.MiB)
+ bBufs = append(bBufs, testKVMemBuf.buf)
+ }
+ for i := 0; i < maxAvailableBufSize; i++ {
+ testKVMemBuf.AllocateBuf(2 * units.MiB)
+ bBufs = append(bBufs, testKVMemBuf.buf)
+ }
+ for _, bb := range bBufs {
+ testKVMemBuf.Recycle(bb)
+ }
+ require.Equal(t, maxAvailableBufSize, len(testKVMemBuf.availableBufs))
+ for _, bb := range testKVMemBuf.availableBufs {
+ require.Equal(t, 4*units.MiB, bb.cap)
+ }
+ bBufs = bBufs[:0]
+ for i := 0; i < maxAvailableBufSize; i++ {
+ testKVMemBuf.AllocateBuf(1 * units.MiB)
+ bb := testKVMemBuf.buf
+ require.Equal(t, 4*units.MiB, bb.cap)
+ bBufs = append(bBufs, bb)
+ require.Equal(t, maxAvailableBufSize-i-1, len(testKVMemBuf.availableBufs))
+ }
+ for _, bb := range bBufs {
+ testKVMemBuf.Recycle(bb)
+ }
+ require.Equal(t, maxAvailableBufSize, len(testKVMemBuf.availableBufs))
+}
diff --git a/br/pkg/lightning/backend/kv/sql2kv.go b/br/pkg/lightning/backend/kv/sql2kv.go
index 6cebb1e29e329..9ad552ef5f340 100644
--- a/br/pkg/lightning/backend/kv/sql2kv.go
+++ b/br/pkg/lightning/backend/kv/sql2kv.go
@@ -169,7 +169,7 @@ func collectGeneratedColumns(se *session, meta *model.TableInfo, cols []*table.C
var genCols []genCol
for i, col := range cols {
if col.GeneratedExpr != nil {
- expr, err := expression.RewriteAstExpr(se, col.GeneratedExpr, schema, names)
+ expr, err := expression.RewriteAstExpr(se, col.GeneratedExpr, schema, names, false)
if err != nil {
return nil, err
}
diff --git a/br/pkg/lightning/backend/local/BUILD.bazel b/br/pkg/lightning/backend/local/BUILD.bazel
index 6e2b5e9a1c43c..42fbadabc9e11 100644
--- a/br/pkg/lightning/backend/local/BUILD.bazel
+++ b/br/pkg/lightning/backend/local/BUILD.bazel
@@ -69,6 +69,8 @@ go_library(
"@org_golang_google_grpc//backoff",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//credentials",
+ "@org_golang_google_grpc//credentials/insecure",
+ "@org_golang_google_grpc//encoding/gzip",
"@org_golang_google_grpc//keepalive",
"@org_golang_google_grpc//status",
"@org_golang_x_exp//slices",
@@ -94,7 +96,7 @@ go_test(
],
embed = [":local"],
flaky = True,
- shard_count = 20,
+ shard_count = 40,
deps = [
"//br/pkg/errors",
"//br/pkg/lightning/backend",
@@ -103,12 +105,14 @@ go_test(
"//br/pkg/lightning/glue",
"//br/pkg/lightning/log",
"//br/pkg/lightning/mydump",
+ "//br/pkg/lightning/worker",
"//br/pkg/membuf",
"//br/pkg/mock",
"//br/pkg/pdutil",
"//br/pkg/restore/split",
"//br/pkg/utils",
"//ddl",
+ "//keyspace",
"//kv",
"//parser",
"//parser/ast",
diff --git a/br/pkg/lightning/backend/local/duplicate.go b/br/pkg/lightning/backend/local/duplicate.go
index b2858a8456f36..fe6cd110a026c 100644
--- a/br/pkg/lightning/backend/local/duplicate.go
+++ b/br/pkg/lightning/backend/local/duplicate.go
@@ -211,7 +211,7 @@ func physicalTableIDs(tableInfo *model.TableInfo) []int64 {
}
// tableHandleKeyRanges returns all key ranges associated with the tableInfo.
-func tableHandleKeyRanges(tableInfo *model.TableInfo) ([]tidbkv.KeyRange, error) {
+func tableHandleKeyRanges(tableInfo *model.TableInfo) (*tidbkv.KeyRanges, error) {
ranges := ranger.FullIntRange(false)
if tableInfo.IsCommonHandle {
ranges = ranger.FullRange()
@@ -221,18 +221,9 @@ func tableHandleKeyRanges(tableInfo *model.TableInfo) ([]tidbkv.KeyRange, error)
}
// tableIndexKeyRanges returns all key ranges associated with the tableInfo and indexInfo.
-func tableIndexKeyRanges(tableInfo *model.TableInfo, indexInfo *model.IndexInfo) ([]tidbkv.KeyRange, error) {
+func tableIndexKeyRanges(tableInfo *model.TableInfo, indexInfo *model.IndexInfo) (*tidbkv.KeyRanges, error) {
tableIDs := physicalTableIDs(tableInfo)
- //nolint: prealloc
- var keyRanges []tidbkv.KeyRange
- for _, tid := range tableIDs {
- partitionKeysRanges, err := distsql.IndexRangesToKVRanges(nil, tid, indexInfo.ID, ranger.FullRange(), nil)
- if err != nil {
- return nil, errors.Trace(err)
- }
- keyRanges = append(keyRanges, partitionKeysRanges...)
- }
- return keyRanges, nil
+ return distsql.IndexRangesToKVRangesForTables(nil, tableIDs, indexInfo.ID, ranger.FullRange(), nil)
}
// DupKVStream is a streaming interface for collecting duplicate key-value pairs.
@@ -396,6 +387,7 @@ type DuplicateManager struct {
tableName string
splitCli split.SplitClient
tikvCli *tikv.KVStore
+ tikvCodec tikv.Codec
errorMgr *errormanager.ErrorManager
decoder *kv.TableKVDecoder
logger log.Logger
@@ -410,6 +402,7 @@ func NewDuplicateManager(
tableName string,
splitCli split.SplitClient,
tikvCli *tikv.KVStore,
+ tikvCodec tikv.Codec,
errMgr *errormanager.ErrorManager,
sessOpts *kv.SessionOptions,
concurrency int,
@@ -426,6 +419,7 @@ func NewDuplicateManager(
tableName: tableName,
splitCli: splitCli,
tikvCli: tikvCli,
+ tikvCodec: tikvCodec,
errorMgr: errMgr,
decoder: decoder,
logger: logger,
@@ -448,6 +442,10 @@ func (m *DuplicateManager) RecordDataConflictError(ctx context.Context, stream D
if err != nil {
return errors.Trace(err)
}
+ key, err = m.tikvCodec.DecodeKey(key)
+ if err != nil {
+ return errors.Trace(err)
+ }
m.hasDupe.Store(true)
h, err := m.decoder.DecodeHandleFromRowKey(key)
@@ -513,6 +511,10 @@ func (m *DuplicateManager) RecordIndexConflictError(ctx context.Context, stream
if err != nil {
return errors.Trace(err)
}
+ key, err = m.tikvCodec.DecodeKey(key)
+ if err != nil {
+ return errors.Trace(err)
+ }
m.hasDupe.Store(true)
h, err := m.decoder.DecodeHandleFromIndex(indexInfo, key, val)
@@ -561,14 +563,23 @@ func (m *DuplicateManager) buildDupTasks() ([]dupTask, error) {
if err != nil {
return nil, errors.Trace(err)
}
- tasks := make([]dupTask, 0, len(keyRanges))
- for _, kr := range keyRanges {
- tableID := tablecodec.DecodeTableID(kr.StartKey)
- tasks = append(tasks, dupTask{
- KeyRange: kr,
- tableID: tableID,
- })
+ tasks := make([]dupTask, 0, keyRanges.TotalRangeNum()*(1+len(m.tbl.Meta().Indices)))
+ putToTaskFunc := func(ranges []tidbkv.KeyRange, indexInfo *model.IndexInfo) {
+ if len(ranges) == 0 {
+ return
+ }
+ tid := tablecodec.DecodeTableID(ranges[0].StartKey)
+ for _, r := range ranges {
+ tasks = append(tasks, dupTask{
+ KeyRange: r,
+ tableID: tid,
+ indexInfo: indexInfo,
+ })
+ }
}
+ keyRanges.ForEachPartition(func(ranges []tidbkv.KeyRange) {
+ putToTaskFunc(ranges, nil)
+ })
for _, indexInfo := range m.tbl.Meta().Indices {
if indexInfo.State != model.StatePublic {
continue
@@ -577,14 +588,14 @@ func (m *DuplicateManager) buildDupTasks() ([]dupTask, error) {
if err != nil {
return nil, errors.Trace(err)
}
- for _, kr := range keyRanges {
- tableID := tablecodec.DecodeTableID(kr.StartKey)
- tasks = append(tasks, dupTask{
- KeyRange: kr,
- tableID: tableID,
- indexInfo: indexInfo,
- })
- }
+ keyRanges.ForEachPartition(func(ranges []tidbkv.KeyRange) {
+ putToTaskFunc(ranges, indexInfo)
+ })
+ }
+
+ // Encode all the tasks
+ for i := range tasks {
+ tasks[i].StartKey, tasks[i].EndKey = m.tikvCodec.EncodeRange(tasks[i].StartKey, tasks[i].EndKey)
}
return tasks, nil
}
@@ -598,15 +609,19 @@ func (m *DuplicateManager) buildIndexDupTasks() ([]dupTask, error) {
if err != nil {
return nil, errors.Trace(err)
}
- tasks := make([]dupTask, 0, len(keyRanges))
- for _, kr := range keyRanges {
- tableID := tablecodec.DecodeTableID(kr.StartKey)
- tasks = append(tasks, dupTask{
- KeyRange: kr,
- tableID: tableID,
- indexInfo: indexInfo,
- })
- }
+ tasks := make([]dupTask, 0, keyRanges.TotalRangeNum())
+ keyRanges.ForEachPartition(func(ranges []tidbkv.KeyRange) {
+ if len(ranges) == 0 {
+ return
+ }
+ tid := tablecodec.DecodeTableID(ranges[0].StartKey)
+ for _, r := range ranges {
+ tasks = append(tasks, dupTask{
+ KeyRange: r,
+ tableID: tid,
+ })
+ }
+ })
return tasks, nil
}
return nil, nil
diff --git a/br/pkg/lightning/backend/local/duplicate_test.go b/br/pkg/lightning/backend/local/duplicate_test.go
index d1db76aae92f8..7c7d8a6182e25 100644
--- a/br/pkg/lightning/backend/local/duplicate_test.go
+++ b/br/pkg/lightning/backend/local/duplicate_test.go
@@ -22,6 +22,7 @@ import (
"github.com/pingcap/tidb/br/pkg/lightning/backend/local"
"github.com/pingcap/tidb/br/pkg/lightning/log"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/keyspace"
"github.com/pingcap/tidb/parser"
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/model"
@@ -52,7 +53,7 @@ func TestBuildDupTask(t *testing.T) {
{&lkv.SessionOptions{IndexID: info.Indices[1].ID}, false},
}
for _, tc := range testCases {
- dupMgr, err := local.NewDuplicateManager(tbl, "t", nil, nil, nil,
+ dupMgr, err := local.NewDuplicateManager(tbl, "t", nil, nil, keyspace.CodecV1, nil,
tc.sessOpt, 4, atomic.NewBool(false), log.FromContext(context.Background()))
require.NoError(t, err)
tasks, err := local.BuildDuplicateTaskForTest(dupMgr)
diff --git a/br/pkg/lightning/backend/local/engine.go b/br/pkg/lightning/backend/local/engine.go
index 32d004654233f..745454a372b18 100644
--- a/br/pkg/lightning/backend/local/engine.go
+++ b/br/pkg/lightning/backend/local/engine.go
@@ -32,6 +32,7 @@ import (
"github.com/google/btree"
"github.com/google/uuid"
"github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/br/pkg/lightning/backend"
"github.com/pingcap/tidb/br/pkg/lightning/backend/kv"
"github.com/pingcap/tidb/br/pkg/lightning/checkpoints"
@@ -42,6 +43,7 @@ import (
"github.com/pingcap/tidb/br/pkg/membuf"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/util/hack"
+ "github.com/tikv/client-go/v2/tikv"
"go.uber.org/atomic"
"go.uber.org/zap"
"golang.org/x/exp/slices"
@@ -123,6 +125,8 @@ type Engine struct {
config backend.LocalEngineConfig
tableInfo *checkpoints.TidbTableInfo
+ dupDetectOpt dupDetectOpt
+
// total size of SST files waiting to be ingested
pendingFileSize atomic.Int64
@@ -981,7 +985,33 @@ func (e *Engine) newKVIter(ctx context.Context, opts *pebble.IterOptions) Iter {
zap.String("table", common.UniqueTable(e.tableInfo.DB, e.tableInfo.Name)),
zap.Int64("tableID", e.tableInfo.ID),
zap.Stringer("engineUUID", e.UUID))
- return newDupDetectIter(ctx, e.db, e.keyAdapter, opts, e.duplicateDB, logger)
+ return newDupDetectIter(ctx, e.db, e.keyAdapter, opts, e.duplicateDB, logger, e.dupDetectOpt)
+}
+
+func (e *Engine) getFirstAndLastKey(lowerBound, upperBound []byte) ([]byte, []byte, error) {
+ opt := &pebble.IterOptions{
+ LowerBound: lowerBound,
+ UpperBound: upperBound,
+ }
+
+ iter := e.newKVIter(context.Background(), opt)
+ //nolint: errcheck
+ defer iter.Close()
+ // Needs seek to first because NewIter returns an iterator that is unpositioned
+ hasKey := iter.First()
+ if iter.Error() != nil {
+ return nil, nil, errors.Annotate(iter.Error(), "failed to read the first key")
+ }
+ if !hasKey {
+ return nil, nil, nil
+ }
+ firstKey := append([]byte{}, iter.Key()...)
+ iter.Last()
+ if iter.Error() != nil {
+ return nil, nil, errors.Annotate(iter.Error(), "failed to seek to the last key")
+ }
+ lastKey := append([]byte{}, iter.Key()...)
+ return firstKey, lastKey, nil
}
type sstMeta struct {
@@ -1017,6 +1047,8 @@ type Writer struct {
batchSize int64
lastMetaSeq int32
+
+ tikvCodec tikv.Codec
}
func (w *Writer) appendRowsSorted(kvs []common.KvPair) error {
@@ -1099,6 +1131,10 @@ func (w *Writer) AppendRows(ctx context.Context, tableName string, columnNames [
return errorEngineClosed
}
+ for i := range kvs {
+ kvs[i].Key = w.tikvCodec.EncodeKey(kvs[i].Key)
+ }
+
w.Lock()
defer w.Unlock()
@@ -1189,6 +1225,15 @@ func (w *Writer) flushKVs(ctx context.Context) error {
if err != nil {
return errors.Trace(err)
}
+
+ failpoint.Inject("orphanWriterGoRoutine", func() {
+ _ = common.KillMySelf()
+ // mimic we meet context cancel error when `addSST`
+ <-ctx.Done()
+ time.Sleep(5 * time.Second)
+ failpoint.Return(errors.Trace(ctx.Err()))
+ })
+
err = w.addSST(ctx, meta)
if err != nil {
return errors.Trace(err)
diff --git a/br/pkg/lightning/backend/local/engine_test.go b/br/pkg/lightning/backend/local/engine_test.go
index c7ffe04b95285..2b935f30bb0c2 100644
--- a/br/pkg/lightning/backend/local/engine_test.go
+++ b/br/pkg/lightning/backend/local/engine_test.go
@@ -31,8 +31,17 @@ import (
"github.com/stretchr/testify/require"
)
-func TestIngestSSTWithClosedEngine(t *testing.T) {
+func makePebbleDB(t *testing.T, opt *pebble.Options) (*pebble.DB, string) {
dir := t.TempDir()
+ db, err := pebble.Open(path.Join(dir, "test"), opt)
+ require.NoError(t, err)
+ tmpPath := filepath.Join(dir, "test.sst")
+ err = os.Mkdir(tmpPath, 0o755)
+ require.NoError(t, err)
+ return db, tmpPath
+}
+
+func TestIngestSSTWithClosedEngine(t *testing.T) {
opt := &pebble.Options{
MemTableSize: 1024 * 1024,
MaxConcurrentCompactions: 16,
@@ -41,11 +50,7 @@ func TestIngestSSTWithClosedEngine(t *testing.T) {
DisableWAL: true,
ReadOnly: false,
}
- db, err := pebble.Open(filepath.Join(dir, "test"), opt)
- require.NoError(t, err)
- tmpPath := filepath.Join(dir, "test.sst")
- err = os.Mkdir(tmpPath, 0o755)
- require.NoError(t, err)
+ db, tmpPath := makePebbleDB(t, opt)
_, engineUUID := backend.MakeUUID("ww", 0)
engineCtx, cancel := context.WithCancel(context.Background())
@@ -84,3 +89,37 @@ func TestIngestSSTWithClosedEngine(t *testing.T) {
},
}), errorEngineClosed)
}
+
+func TestGetFirstAndLastKey(t *testing.T) {
+ db, tmpPath := makePebbleDB(t, nil)
+ f := &Engine{
+ db: db,
+ sstDir: tmpPath,
+ }
+ err := db.Set([]byte("a"), []byte("a"), nil)
+ require.NoError(t, err)
+ err = db.Set([]byte("c"), []byte("c"), nil)
+ require.NoError(t, err)
+ err = db.Set([]byte("e"), []byte("e"), nil)
+ require.NoError(t, err)
+
+ first, last, err := f.getFirstAndLastKey(nil, nil)
+ require.NoError(t, err)
+ require.Equal(t, []byte("a"), first)
+ require.Equal(t, []byte("e"), last)
+
+ first, last, err = f.getFirstAndLastKey([]byte("b"), []byte("d"))
+ require.NoError(t, err)
+ require.Equal(t, []byte("c"), first)
+ require.Equal(t, []byte("c"), last)
+
+ first, last, err = f.getFirstAndLastKey([]byte("b"), []byte("f"))
+ require.NoError(t, err)
+ require.Equal(t, []byte("c"), first)
+ require.Equal(t, []byte("e"), last)
+
+ first, last, err = f.getFirstAndLastKey([]byte("y"), []byte("z"))
+ require.NoError(t, err)
+ require.Nil(t, first)
+ require.Nil(t, last)
+}
diff --git a/br/pkg/lightning/backend/local/iterator.go b/br/pkg/lightning/backend/local/iterator.go
index e2cb3a447cfbb..29ed50b743773 100644
--- a/br/pkg/lightning/backend/local/iterator.go
+++ b/br/pkg/lightning/backend/local/iterator.go
@@ -21,6 +21,7 @@ import (
"github.com/cockroachdb/pebble"
sst "github.com/pingcap/kvproto/pkg/import_sstpb"
+ "github.com/pingcap/tidb/br/pkg/lightning/common"
"github.com/pingcap/tidb/br/pkg/lightning/log"
"github.com/pingcap/tidb/br/pkg/logutil"
"go.uber.org/multierr"
@@ -82,6 +83,11 @@ type dupDetectIter struct {
writeBatch *pebble.Batch
writeBatchSize int64
logger log.Logger
+ option dupDetectOpt
+}
+
+type dupDetectOpt struct {
+ reportErrOnDup bool
}
func (d *dupDetectIter) Seek(key []byte) bool {
@@ -149,6 +155,14 @@ func (d *dupDetectIter) Next() bool {
d.curVal = append(d.curVal[:0], d.iter.Value()...)
return true
}
+ if d.option.reportErrOnDup {
+ dupKey := make([]byte, len(d.curKey))
+ dupVal := make([]byte, len(d.iter.Value()))
+ copy(dupKey, d.curKey)
+ copy(dupVal, d.curVal)
+ d.err = common.ErrFoundDuplicateKeys.FastGenByArgs(dupKey, dupVal)
+ return false
+ }
if !recordFirst {
d.record(d.curRawKey, d.curKey, d.curVal)
recordFirst = true
@@ -192,7 +206,7 @@ func (d *dupDetectIter) OpType() sst.Pair_OP {
var _ Iter = &dupDetectIter{}
func newDupDetectIter(ctx context.Context, db *pebble.DB, keyAdapter KeyAdapter,
- opts *pebble.IterOptions, dupDB *pebble.DB, logger log.Logger) *dupDetectIter {
+ opts *pebble.IterOptions, dupDB *pebble.DB, logger log.Logger, dupOpt dupDetectOpt) *dupDetectIter {
newOpts := &pebble.IterOptions{TableFilter: opts.TableFilter}
if len(opts.LowerBound) > 0 {
newOpts.LowerBound = keyAdapter.Encode(nil, opts.LowerBound, math.MinInt64)
@@ -206,6 +220,7 @@ func newDupDetectIter(ctx context.Context, db *pebble.DB, keyAdapter KeyAdapter,
keyAdapter: keyAdapter,
writeBatch: dupDB.NewBatch(),
logger: logger,
+ option: dupOpt,
}
}
diff --git a/br/pkg/lightning/backend/local/iterator_test.go b/br/pkg/lightning/backend/local/iterator_test.go
index 3abb6fbc3b06c..c183963443bae 100644
--- a/br/pkg/lightning/backend/local/iterator_test.go
+++ b/br/pkg/lightning/backend/local/iterator_test.go
@@ -122,7 +122,7 @@ func TestDupDetectIterator(t *testing.T) {
dupDB, err := pebble.Open(filepath.Join(storeDir, "duplicates"), &pebble.Options{})
require.NoError(t, err)
var iter Iter
- iter = newDupDetectIter(context.Background(), db, keyAdapter, &pebble.IterOptions{}, dupDB, log.L())
+ iter = newDupDetectIter(context.Background(), db, keyAdapter, &pebble.IterOptions{}, dupDB, log.L(), dupDetectOpt{})
sort.Slice(pairs, func(i, j int) bool {
key1 := keyAdapter.Encode(nil, pairs[i].Key, pairs[i].RowID)
key2 := keyAdapter.Encode(nil, pairs[j].Key, pairs[j].RowID)
@@ -217,7 +217,7 @@ func TestDupDetectIterSeek(t *testing.T) {
dupDB, err := pebble.Open(filepath.Join(storeDir, "duplicates"), &pebble.Options{})
require.NoError(t, err)
- iter := newDupDetectIter(context.Background(), db, keyAdapter, &pebble.IterOptions{}, dupDB, log.L())
+ iter := newDupDetectIter(context.Background(), db, keyAdapter, &pebble.IterOptions{}, dupDB, log.L(), dupDetectOpt{})
require.True(t, iter.Seek([]byte{1, 2, 3, 1}))
require.Equal(t, pairs[1].Val, iter.Value())
diff --git a/br/pkg/lightning/backend/local/local.go b/br/pkg/lightning/backend/local/local.go
index 317124d0b8d19..cd2a7dd44c5cc 100644
--- a/br/pkg/lightning/backend/local/local.go
+++ b/br/pkg/lightning/backend/local/local.go
@@ -20,6 +20,7 @@ import (
"fmt"
"io"
"math"
+ "net"
"os"
"path/filepath"
"strings"
@@ -69,12 +70,15 @@ import (
"go.uber.org/atomic"
"go.uber.org/multierr"
"go.uber.org/zap"
+ "golang.org/x/exp/slices"
"golang.org/x/sync/errgroup"
"golang.org/x/time/rate"
"google.golang.org/grpc"
"google.golang.org/grpc/backoff"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/encoding/gzip"
"google.golang.org/grpc/keepalive"
"google.golang.org/grpc/status"
)
@@ -91,6 +95,7 @@ const (
gRPCKeepAliveTime = 10 * time.Minute
gRPCKeepAliveTimeout = 5 * time.Minute
gRPCBackOffMaxDelay = 10 * time.Minute
+ writeStallSleepTime = 10 * time.Second
// The max ranges count in a batch to split and scatter.
maxBatchSplitRanges = 4096
@@ -127,18 +132,25 @@ type ImportClientFactory interface {
}
type importClientFactoryImpl struct {
- conns *common.GRPCConns
- splitCli split.SplitClient
- tls *common.TLS
- tcpConcurrency int
+ conns *common.GRPCConns
+ splitCli split.SplitClient
+ tls *common.TLS
+ tcpConcurrency int
+ compressionType config.CompressionType
}
-func newImportClientFactoryImpl(splitCli split.SplitClient, tls *common.TLS, tcpConcurrency int) *importClientFactoryImpl {
+func newImportClientFactoryImpl(
+ splitCli split.SplitClient,
+ tls *common.TLS,
+ tcpConcurrency int,
+ compressionType config.CompressionType,
+) *importClientFactoryImpl {
return &importClientFactoryImpl{
- conns: common.NewGRPCConns(),
- splitCli: splitCli,
- tls: tls,
- tcpConcurrency: tcpConcurrency,
+ conns: common.NewGRPCConns(),
+ splitCli: splitCli,
+ tls: tls,
+ tcpConcurrency: tcpConcurrency,
+ compressionType: compressionType,
}
}
@@ -147,11 +159,14 @@ func (f *importClientFactoryImpl) makeConn(ctx context.Context, storeID uint64)
if err != nil {
return nil, errors.Trace(err)
}
- opt := grpc.WithInsecure()
+ var opts []grpc.DialOption
if f.tls.TLSConfig() != nil {
- opt = grpc.WithTransportCredentials(credentials.NewTLS(f.tls.TLSConfig()))
+ opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(f.tls.TLSConfig())))
+ } else {
+ opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
}
ctx, cancel := context.WithTimeout(ctx, dialTimeout)
+ defer cancel()
bfConf := backoff.DefaultConfig
bfConf.MaxDelay = gRPCBackOffMaxDelay
@@ -160,10 +175,7 @@ func (f *importClientFactoryImpl) makeConn(ctx context.Context, storeID uint64)
if addr == "" {
addr = store.GetAddress()
}
- conn, err := grpc.DialContext(
- ctx,
- addr,
- opt,
+ opts = append(opts,
grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}),
grpc.WithKeepaliveParams(keepalive.ClientParameters{
Time: gRPCKeepAliveTime,
@@ -171,7 +183,26 @@ func (f *importClientFactoryImpl) makeConn(ctx context.Context, storeID uint64)
PermitWithoutStream: true,
}),
)
- cancel()
+ switch f.compressionType {
+ case config.CompressionNone:
+ // do nothing
+ case config.CompressionGzip:
+ opts = append(opts, grpc.WithDefaultCallOptions(grpc.UseCompressor(gzip.Name)))
+ default:
+ return nil, common.ErrInvalidConfig.GenWithStack("unsupported compression type %s", f.compressionType)
+ }
+
+ failpoint.Inject("LoggingImportBytes", func() {
+ opts = append(opts, grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) {
+ conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", target)
+ if err != nil {
+ return nil, err
+ }
+ return &loggingConn{Conn: conn}, nil
+ }))
+ })
+
+ conn, err := grpc.DialContext(ctx, addr, opts...)
if err != nil {
return nil, errors.Trace(err)
}
@@ -197,6 +228,15 @@ func (f *importClientFactoryImpl) Close() {
f.conns.Close()
}
+type loggingConn struct {
+ net.Conn
+}
+
+func (c loggingConn) Write(b []byte) (int, error) {
+ log.L().Debug("import write", zap.Int("bytes", len(b)))
+ return c.Conn.Write(b)
+}
+
// Range record start and end key for localStoreDir.DB
// so we can write it to tikv in streaming
type Range struct {
@@ -346,12 +386,13 @@ func checkTiFlashVersion(ctx context.Context, g glue.Glue, checkCtx *backend.Che
type local struct {
engines sync.Map // sync version of map[uuid.UUID]*Engine
- pdCtl *pdutil.PdController
- splitCli split.SplitClient
- tikvCli *tikvclient.KVStore
- tls *common.TLS
- pdAddr string
- g glue.Glue
+ pdCtl *pdutil.PdController
+ splitCli split.SplitClient
+ tikvCli *tikvclient.KVStore
+ tls *common.TLS
+ pdAddr string
+ g glue.Glue
+ tikvCodec tikvclient.Codec
localStoreDir string
@@ -369,6 +410,7 @@ type local struct {
checkTiKVAvaliable bool
duplicateDetection bool
+ duplicateDetectOpt dupDetectOpt
duplicateDB *pebble.DB
keyAdapter KeyAdapter
errorMgr *errormanager.ErrorManager
@@ -381,6 +423,12 @@ type local struct {
encBuilder backend.EncodingBuilder
targetInfoGetter backend.TargetInfoGetter
+
+ // When TiKV is in normal mode, ingesting too many SSTs will cause TiKV write stall.
+ // To avoid this, we should check write stall before ingesting SSTs. Note that, we
+ // must check both leader node and followers in client side, because followers will
+ // not check write stall as long as ingest command is accepted by leader.
+ shouldCheckWriteStall bool
}
func openDuplicateDB(storeDir string) (*pebble.DB, error) {
@@ -394,6 +442,13 @@ func openDuplicateDB(storeDir string) (*pebble.DB, error) {
return pebble.Open(dbPath, opts)
}
+var (
+ // RunInTest indicates whether the current process is running in test.
+ RunInTest bool
+ // LastAlloc is the last ID allocator.
+ LastAlloc manual.Allocator
+)
+
// NewLocalBackend creates new connections to tikv.
func NewLocalBackend(
ctx context.Context,
@@ -402,6 +457,7 @@ func NewLocalBackend(
g glue.Glue,
maxOpenFiles int,
errorMgr *errormanager.ErrorManager,
+ keyspaceName string,
) (backend.Backend, error) {
localFile := cfg.TikvImporter.SortedKVDir
rangeConcurrency := cfg.TikvImporter.RangeConcurrency
@@ -443,13 +499,24 @@ func NewLocalBackend(
if err != nil {
return backend.MakeBackend(nil), common.ErrCreateKVClient.Wrap(err).GenWithStackByArgs()
}
- rpcCli := tikvclient.NewRPCClient(tikvclient.WithSecurity(tls.ToTiKVSecurityConfig()))
- pdCliForTiKV := &tikvclient.CodecPDClient{Client: pdCtl.GetPDClient()}
+
+ var pdCliForTiKV *tikvclient.CodecPDClient
+ if keyspaceName == "" {
+ pdCliForTiKV = tikvclient.NewCodecPDClient(tikvclient.ModeTxn, pdCtl.GetPDClient())
+ } else {
+ pdCliForTiKV, err = tikvclient.NewCodecPDClientWithKeyspace(tikvclient.ModeTxn, pdCtl.GetPDClient(), keyspaceName)
+ if err != nil {
+ return backend.MakeBackend(nil), common.ErrCreatePDClient.Wrap(err).GenWithStackByArgs()
+ }
+ }
+
+ tikvCodec := pdCliForTiKV.GetCodec()
+ rpcCli := tikvclient.NewRPCClient(tikvclient.WithSecurity(tls.ToTiKVSecurityConfig()), tikvclient.WithCodec(tikvCodec))
tikvCli, err := tikvclient.NewKVStore("lightning-local-backend", pdCliForTiKV, spkv, rpcCli)
if err != nil {
return backend.MakeBackend(nil), common.ErrCreateKVClient.Wrap(err).GenWithStackByArgs()
}
- importClientFactory := newImportClientFactoryImpl(splitCli, tls, rangeConcurrency)
+ importClientFactory := newImportClientFactoryImpl(splitCli, tls, rangeConcurrency, cfg.TikvImporter.CompressKVPairs)
duplicateDetection := cfg.TikvImporter.DuplicateResolution != config.DupeResAlgNone
keyAdapter := KeyAdapter(noopKeyAdapter{})
if duplicateDetection {
@@ -461,14 +528,20 @@ func NewLocalBackend(
} else {
writeLimiter = noopStoreWriteLimiter{}
}
+ alloc := manual.Allocator{}
+ if RunInTest {
+ alloc.RefCnt = new(atomic.Int64)
+ LastAlloc = alloc
+ }
local := &local{
- engines: sync.Map{},
- pdCtl: pdCtl,
- splitCli: splitCli,
- tikvCli: tikvCli,
- tls: tls,
- pdAddr: cfg.TiDB.PdAddr,
- g: g,
+ engines: sync.Map{},
+ pdCtl: pdCtl,
+ splitCli: splitCli,
+ tikvCli: tikvCli,
+ tls: tls,
+ pdAddr: cfg.TiDB.PdAddr,
+ g: g,
+ tikvCodec: tikvCodec,
localStoreDir: localFile,
rangeConcurrency: worker.NewPool(ctx, rangeConcurrency, "range"),
@@ -481,16 +554,18 @@ func NewLocalBackend(
engineMemCacheSize: int(cfg.TikvImporter.EngineMemCacheSize),
localWriterMemCacheSize: int64(cfg.TikvImporter.LocalWriterMemCacheSize),
duplicateDetection: duplicateDetection,
+ duplicateDetectOpt: dupDetectOpt{duplicateDetection && cfg.TikvImporter.DuplicateResolution == config.DupeResAlgErr},
checkTiKVAvaliable: cfg.App.CheckRequirements,
duplicateDB: duplicateDB,
keyAdapter: keyAdapter,
errorMgr: errorMgr,
importClientFactory: importClientFactory,
- bufferPool: membuf.NewPool(membuf.WithAllocator(manual.Allocator{})),
+ bufferPool: membuf.NewPool(membuf.WithAllocator(alloc)),
writeLimiter: writeLimiter,
logger: log.FromContext(ctx),
encBuilder: NewEncodingBuilder(ctx),
targetInfoGetter: NewTargetInfoGetter(tls, g, cfg.TiDB.PdAddr),
+ shouldCheckWriteStall: cfg.Cron.SwitchMode.Duration == 0,
}
if m, ok := metric.FromContext(ctx); ok {
local.metrics = m
@@ -784,6 +859,7 @@ func (local *local) OpenEngine(ctx context.Context, cfg *backend.EngineConfig, e
config: engineCfg,
tableInfo: cfg.TableInfo,
duplicateDetection: local.duplicateDetection,
+ dupDetectOpt: local.duplicateDetectOpt,
duplicateDB: local.duplicateDB,
errorMgr: local.errorMgr,
keyAdapter: local.keyAdapter,
@@ -834,6 +910,7 @@ func (local *local) CloseEngine(ctx context.Context, cfg *backend.EngineConfig,
tableInfo: cfg.TableInfo,
keyAdapter: local.keyAdapter,
duplicateDetection: local.duplicateDetection,
+ dupDetectOpt: local.duplicateDetectOpt,
duplicateDB: local.duplicateDB,
errorMgr: local.errorMgr,
logger: log.FromContext(ctx),
@@ -878,9 +955,15 @@ type rangeStats struct {
totalBytes int64
}
+type tikvWriteResult struct {
+ sstMeta []*sst.SSTMeta
+ finishedRange Range
+ rangeStats rangeStats
+}
+
// WriteToTiKV writer engine key-value pairs to tikv and return the sst meta generated by tikv.
// we don't need to do cleanup for the pairs written to tikv if encounters an error,
-// tikv will takes the responsibility to do so.
+// tikv will take the responsibility to do so.
func (local *local) WriteToTiKV(
ctx context.Context,
engine *Engine,
@@ -888,9 +971,9 @@ func (local *local) WriteToTiKV(
start, end []byte,
regionSplitSize int64,
regionSplitKeys int64,
-) ([]*sst.SSTMeta, Range, rangeStats, error) {
+) (*tikvWriteResult, error) {
failpoint.Inject("WriteToTiKVNotEnoughDiskSpace", func(_ failpoint.Value) {
- failpoint.Return(nil, Range{}, rangeStats{},
+ failpoint.Return(nil,
errors.Errorf("The available disk of TiKV (%s) only left %d, and capacity is %d", "", 0, 0))
})
if local.checkTiKVAvaliable {
@@ -906,7 +989,7 @@ func (local *local) WriteToTiKV(
// The available disk percent of TiKV
ratio := store.Status.Available * 100 / store.Status.Capacity
if ratio < 10 {
- return nil, Range{}, rangeStats{}, errors.Errorf("The available disk of TiKV (%s) only left %d, and capacity is %d",
+ return nil, errors.Errorf("The available disk of TiKV (%s) only left %d, and capacity is %d",
store.Store.Address, store.Status.Available, store.Status.Capacity)
}
}
@@ -919,29 +1002,20 @@ func (local *local) WriteToTiKV(
}
begin := time.Now()
regionRange := intersectRange(region.Region, Range{start: start, end: end})
- opt := &pebble.IterOptions{LowerBound: regionRange.start, UpperBound: regionRange.end}
- iter := engine.newKVIter(ctx, opt)
- //nolint: errcheck
- defer iter.Close()
-
stats := rangeStats{}
- iter.First()
- if iter.Error() != nil {
- return nil, Range{}, stats, errors.Annotate(iter.Error(), "failed to read the first key")
+ firstKey, lastKey, err := engine.getFirstAndLastKey(regionRange.start, regionRange.end)
+ if err != nil {
+ return nil, errors.Trace(err)
}
- if !iter.Valid() {
+ if firstKey == nil {
log.FromContext(ctx).Info("keys within region is empty, skip ingest", logutil.Key("start", start),
logutil.Key("regionStart", region.Region.StartKey), logutil.Key("end", end),
logutil.Key("regionEnd", region.Region.EndKey))
- return nil, regionRange, stats, nil
- }
- firstKey := codec.EncodeBytes([]byte{}, iter.Key())
- iter.Last()
- if iter.Error() != nil {
- return nil, Range{}, stats, errors.Annotate(iter.Error(), "failed to seek to the last key")
+ return &tikvWriteResult{sstMeta: nil, finishedRange: regionRange, rangeStats: stats}, nil
}
- lastKey := codec.EncodeBytes([]byte{}, iter.Key())
+ firstKey = codec.EncodeBytes([]byte{}, firstKey)
+ lastKey = codec.EncodeBytes([]byte{}, lastKey)
u := uuid.New()
meta := &sst.SSTMeta{
@@ -952,6 +1026,7 @@ func (local *local) WriteToTiKV(
Start: firstKey,
End: lastKey,
},
+ ApiVersion: local.tikvCodec.GetAPIVersion(),
}
leaderID := region.Leader.GetId()
@@ -961,12 +1036,12 @@ func (local *local) WriteToTiKV(
for _, peer := range region.Region.GetPeers() {
cli, err := local.getImportClient(ctx, peer.StoreId)
if err != nil {
- return nil, Range{}, stats, err
+ return nil, errors.Trace(err)
}
wstream, err := cli.Write(ctx)
if err != nil {
- return nil, Range{}, stats, errors.Trace(err)
+ return nil, errors.Trace(err)
}
// Bind uuid for this write request
@@ -976,7 +1051,7 @@ func (local *local) WriteToTiKV(
},
}
if err = wstream.Send(req); err != nil {
- return nil, Range{}, stats, errors.Trace(err)
+ return nil, errors.Trace(err)
}
req.Chunk = &sst.WriteRequest_Batch{
Batch: &sst.WriteBatch{
@@ -1017,6 +1092,11 @@ func (local *local) WriteToTiKV(
return nil
}
+ opt := &pebble.IterOptions{LowerBound: regionRange.start, UpperBound: regionRange.end}
+ iter := engine.newKVIter(ctx, opt)
+ //nolint: errcheck
+ defer iter.Close()
+
for iter.First(); iter.Valid(); iter.Next() {
kvSize := int64(len(iter.Key()) + len(iter.Value()))
// here we reuse the `*sst.Pair`s to optimize object allocation
@@ -1037,7 +1117,7 @@ func (local *local) WriteToTiKV(
if count >= local.batchWriteKVPairs || size >= flushLimit {
if err := flushKVs(); err != nil {
- return nil, Range{}, stats, err
+ return nil, errors.Trace(err)
}
count = 0
size = 0
@@ -1049,12 +1129,12 @@ func (local *local) WriteToTiKV(
}
if iter.Error() != nil {
- return nil, Range{}, stats, errors.Trace(iter.Error())
+ return nil, errors.Trace(iter.Error())
}
if count > 0 {
if err := flushKVs(); err != nil {
- return nil, Range{}, stats, err
+ return nil, errors.Trace(err)
}
count = 0
size = 0
@@ -1065,10 +1145,10 @@ func (local *local) WriteToTiKV(
for i, wStream := range clients {
resp, closeErr := wStream.CloseAndRecv()
if closeErr != nil {
- return nil, Range{}, stats, errors.Trace(closeErr)
+ return nil, errors.Trace(closeErr)
}
if resp.Error != nil {
- return nil, Range{}, stats, errors.New(resp.Error.Message)
+ return nil, errors.New(resp.Error.Message)
}
if leaderID == region.Region.Peers[i].GetId() {
leaderPeerMetas = resp.Metas
@@ -1081,7 +1161,7 @@ func (local *local) WriteToTiKV(
log.FromContext(ctx).Warn("write to tikv no leader", logutil.Region(region.Region), logutil.Leader(region.Leader),
zap.Uint64("leader_id", leaderID), logutil.SSTMeta(meta),
zap.Int64("kv_pairs", totalCount), zap.Int64("total_bytes", size))
- return nil, Range{}, stats, errors.Errorf("write to tikv with no leader returned, region '%d', leader: %d",
+ return nil, errors.Errorf("write to tikv with no leader returned, region '%d', leader: %d",
region.Region.Id, leaderID)
}
@@ -1103,7 +1183,11 @@ func (local *local) WriteToTiKV(
stats.count = totalCount
stats.totalBytes = totalSize
- return leaderPeerMetas, finishedRange, stats, nil
+ return &tikvWriteResult{
+ sstMeta: leaderPeerMetas,
+ finishedRange: finishedRange,
+ rangeStats: stats,
+ }, nil
}
func (local *local) Ingest(ctx context.Context, metas []*sst.SSTMeta, region *split.RegionInfo) (*sst.IngestResponse, error) {
@@ -1134,6 +1218,16 @@ func (local *local) Ingest(ctx context.Context, metas []*sst.SSTMeta, region *sp
return resp, errors.Trace(err)
}
+ if local.shouldCheckWriteStall {
+ writeStall, resp, err := local.checkWriteStall(ctx, region)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if writeStall {
+ return resp, nil
+ }
+ }
+
req := &sst.MultiIngestRequest{
Context: reqCtx,
Ssts: metas,
@@ -1142,6 +1236,25 @@ func (local *local) Ingest(ctx context.Context, metas []*sst.SSTMeta, region *sp
return resp, errors.Trace(err)
}
+func (local *local) checkWriteStall(ctx context.Context, region *split.RegionInfo) (bool, *sst.IngestResponse, error) {
+ for _, peer := range region.Region.GetPeers() {
+ cli, err := local.getImportClient(ctx, peer.StoreId)
+ if err != nil {
+ return false, nil, errors.Trace(err)
+ }
+ // currently we use empty MultiIngestRequest to check if TiKV is busy.
+ // If in future the rate limit feature contains more metrics we can switch to use it.
+ resp, err := cli.MultiIngest(ctx, &sst.MultiIngestRequest{})
+ if err != nil {
+ return false, nil, errors.Trace(err)
+ }
+ if resp.Error != nil && resp.Error.ServerIsBusy != nil {
+ return true, resp, nil
+ }
+ }
+ return false, nil, nil
+}
+
func splitRangeBySizeProps(fullRange Range, sizeProps *sizeProperties, sizeLimit int64, keysLimit int64) []Range {
ranges := make([]Range, 0, sizeProps.totalSize/uint64(sizeLimit))
curSize := uint64(0)
@@ -1178,29 +1291,14 @@ func splitRangeBySizeProps(fullRange Range, sizeProps *sizeProperties, sizeLimit
}
func (local *local) readAndSplitIntoRange(ctx context.Context, engine *Engine, regionSplitSize int64, regionSplitKeys int64) ([]Range, error) {
- iter := engine.newKVIter(ctx, &pebble.IterOptions{})
- //nolint: errcheck
- defer iter.Close()
-
- iterError := func(e string) error {
- err := iter.Error()
- if err != nil {
- return errors.Annotate(err, e)
- }
- return errors.New(e)
- }
-
- var firstKey, lastKey []byte
- if iter.First() {
- firstKey = append([]byte{}, iter.Key()...)
- } else {
- return nil, iterError("could not find first pair")
+ firstKey, lastKey, err := engine.getFirstAndLastKey(nil, nil)
+ if err != nil {
+ return nil, err
}
- if iter.Last() {
- lastKey = append([]byte{}, iter.Key()...)
- } else {
- return nil, iterError("could not find last pair")
+ if firstKey == nil {
+ return nil, errors.New("could not find first pair")
}
+
endKey := nextKey(lastKey)
engineFileTotalSize := engine.TotalSize.Load()
@@ -1230,45 +1328,27 @@ func (local *local) readAndSplitIntoRange(ctx context.Context, engine *Engine, r
}
func (local *local) writeAndIngestByRange(
- ctxt context.Context,
+ ctx context.Context,
engine *Engine,
start, end []byte,
regionSplitSize int64,
regionSplitKeys int64,
) error {
- ito := &pebble.IterOptions{
- LowerBound: start,
- UpperBound: end,
- }
-
- iter := engine.newKVIter(ctxt, ito)
- //nolint: errcheck
- defer iter.Close()
- // Needs seek to first because NewIter returns an iterator that is unpositioned
- hasKey := iter.First()
- if iter.Error() != nil {
- return errors.Annotate(iter.Error(), "failed to read the first key")
+ pairStart, pairEnd, err := engine.getFirstAndLastKey(start, end)
+ if err != nil {
+ return err
}
- if !hasKey {
- log.FromContext(ctxt).Info("There is no pairs in iterator",
+ if pairStart == nil {
+ log.FromContext(ctx).Info("There is no pairs in iterator",
logutil.Key("start", start),
logutil.Key("end", end))
engine.finishedRanges.add(Range{start: start, end: end})
return nil
}
- pairStart := append([]byte{}, iter.Key()...)
- iter.Last()
- if iter.Error() != nil {
- return errors.Annotate(iter.Error(), "failed to seek to the last key")
- }
- pairEnd := append([]byte{}, iter.Key()...)
var regions []*split.RegionInfo
- var err error
- ctx, cancel := context.WithCancel(ctxt)
- defer cancel()
-WriteAndIngest:
+ScanWriteIngest:
for retry := 0; retry < maxRetryTimes; {
if retry != 0 {
select {
@@ -1284,7 +1364,7 @@ WriteAndIngest:
log.FromContext(ctx).Warn("scan region failed", log.ShortError(err), zap.Int("region_len", len(regions)),
logutil.Key("startKey", startKey), logutil.Key("endKey", endKey), zap.Int("retry", retry))
retry++
- continue WriteAndIngest
+ continue ScanWriteIngest
}
for _, region := range regions {
@@ -1309,7 +1389,7 @@ WriteAndIngest:
}
log.FromContext(ctx).Info("retry write and ingest kv pairs", logutil.Key("startKey", pairStart),
logutil.Key("endKey", end), log.ShortError(err), zap.Int("retry", retry))
- continue WriteAndIngest
+ continue ScanWriteIngest
}
}
@@ -1325,6 +1405,7 @@ const (
retryNone retryType = iota
retryWrite
retryIngest
+ retryBusyIngest
)
func (local *local) isRetryableImportTiKVError(err error) bool {
@@ -1340,6 +1421,11 @@ func (local *local) isRetryableImportTiKVError(err error) bool {
return common.IsRetryableError(err)
}
+// writeAndIngestPairs writes the kv pairs in the range [start, end) to the peers
+// of the region, and then send the ingest command to do RocksDB ingest.
+// when return nil, it does not mean the whole task success. The success ranges is
+// recorded in the engine.finishedRanges.
+// TODO: regionSplitSize and regionSplitKeys can be a member of Engine, no need to pass it in every function.
func (local *local) writeAndIngestPairs(
ctx context.Context,
engine *Engine,
@@ -1349,13 +1435,10 @@ func (local *local) writeAndIngestPairs(
regionSplitKeys int64,
) error {
var err error
-
+ var writeResult *tikvWriteResult
loopWrite:
for i := 0; i < maxRetryTimes; i++ {
- var metas []*sst.SSTMeta
- var finishedRange Range
- var rangeStats rangeStats
- metas, finishedRange, rangeStats, err = local.WriteToTiKV(ctx, engine, region, start, end, regionSplitSize, regionSplitKeys)
+ writeResult, err = local.WriteToTiKV(ctx, engine, region, start, end, regionSplitSize, regionSplitKeys)
if err != nil {
if !local.isRetryableImportTiKVError(err) {
return err
@@ -1364,6 +1447,7 @@ loopWrite:
log.FromContext(ctx).Warn("write to tikv failed", log.ShortError(err), zap.Int("retry", i))
continue loopWrite
}
+ metas, finishedRange, rangeStats := writeResult.sstMeta, writeResult.finishedRange, writeResult.rangeStats
if len(metas) == 0 {
return nil
@@ -1430,6 +1514,7 @@ loopWrite:
// ingest next meta
break
}
+
switch retryTy {
case retryNone:
log.FromContext(ctx).Warn("ingest failed noretry", log.ShortError(err), logutil.SSTMetas(ingestMetas),
@@ -1442,25 +1527,30 @@ loopWrite:
case retryIngest:
region = newRegion
continue
+ case retryBusyIngest:
+ log.FromContext(ctx).Warn("meet tikv busy when ingest", log.ShortError(err), logutil.SSTMetas(ingestMetas),
+ logutil.Region(region.Region))
+ // ImportEngine will continue on this unfinished range
+ return nil
}
}
}
- if err != nil {
- log.FromContext(ctx).Warn("write and ingest region, will retry import full range", log.ShortError(err),
- logutil.Region(region.Region), logutil.Key("start", start),
- logutil.Key("end", end))
- } else {
+ if err == nil {
engine.importedKVSize.Add(rangeStats.totalBytes)
engine.importedKVCount.Add(rangeStats.count)
engine.finishedRanges.add(finishedRange)
if local.metrics != nil {
local.metrics.BytesCounter.WithLabelValues(metric.BytesStateImported).Add(float64(rangeStats.totalBytes))
}
+ return nil
}
+
+ log.FromContext(ctx).Warn("write and ingest region, will retry import full range", log.ShortError(err),
+ logutil.Region(region.Region), logutil.Key("start", start),
+ logutil.Key("end", end))
return errors.Trace(err)
}
-
return errors.Trace(err)
}
@@ -1603,6 +1693,9 @@ func (local *local) ImportEngine(ctx context.Context, engineUUID uuid.UUID, regi
// the table when table is created.
needSplit := len(unfinishedRanges) > 1 || lfTotalSize > regionSplitSize || lfLength > regionSplitKeys
// split region by given ranges
+ failpoint.Inject("failToSplit", func(_ failpoint.Value) {
+ needSplit = true
+ })
for i := 0; i < maxRetryTimes; i++ {
err = local.SplitAndScatterRegionInBatches(ctx, unfinishedRanges, lf.tableInfo, needSplit, regionSplitSize, maxBatchSplitRanges)
if err == nil || common.IsContextCanceledError(err) {
@@ -1638,7 +1731,7 @@ func (local *local) CollectLocalDuplicateRows(ctx context.Context, tbl table.Tab
}()
atomicHasDupe := atomic.NewBool(false)
- duplicateManager, err := NewDuplicateManager(tbl, tableName, local.splitCli, local.tikvCli,
+ duplicateManager, err := NewDuplicateManager(tbl, tableName, local.splitCli, local.tikvCli, local.tikvCodec,
local.errorMgr, opts, local.dupeConcurrency, atomicHasDupe, log.FromContext(ctx))
if err != nil {
return false, errors.Trace(err)
@@ -1656,7 +1749,7 @@ func (local *local) CollectRemoteDuplicateRows(ctx context.Context, tbl table.Ta
}()
atomicHasDupe := atomic.NewBool(false)
- duplicateManager, err := NewDuplicateManager(tbl, tableName, local.splitCli, local.tikvCli,
+ duplicateManager, err := NewDuplicateManager(tbl, tableName, local.splitCli, local.tikvCli, local.tikvCodec,
local.errorMgr, opts, local.dupeConcurrency, atomicHasDupe, log.FromContext(ctx))
if err != nil {
return false, errors.Trace(err)
@@ -1690,13 +1783,18 @@ func (local *local) ResolveDuplicateRows(ctx context.Context, tbl table.Table, t
return err
}
+ tableIDs := physicalTableIDs(tbl.Meta())
+ keyInTable := func(key []byte) bool {
+ return slices.Contains(tableIDs, tablecodec.DecodeTableID(key))
+ }
+
errLimiter := rate.NewLimiter(1, 1)
pool := utils.NewWorkerPool(uint(local.dupeConcurrency), "resolve duplicate rows")
err = local.errorMgr.ResolveAllConflictKeys(
ctx, tableName, pool,
func(ctx context.Context, handleRows [][2][]byte) error {
for {
- err := local.deleteDuplicateRows(ctx, logger, handleRows, decoder)
+ err := local.deleteDuplicateRows(ctx, logger, handleRows, decoder, keyInTable)
if err == nil {
return nil
}
@@ -1719,7 +1817,13 @@ func (local *local) ResolveDuplicateRows(ctx context.Context, tbl table.Table, t
return errors.Trace(err)
}
-func (local *local) deleteDuplicateRows(ctx context.Context, logger *log.Task, handleRows [][2][]byte, decoder *kv.TableKVDecoder) (err error) {
+func (local *local) deleteDuplicateRows(
+ ctx context.Context,
+ logger *log.Task,
+ handleRows [][2][]byte,
+ decoder *kv.TableKVDecoder,
+ keyInTable func(key []byte) bool,
+) (err error) {
// Starts a Delete transaction.
txn, err := local.tikvCli.Begin()
if err != nil {
@@ -1744,6 +1848,12 @@ func (local *local) deleteDuplicateRows(ctx context.Context, logger *log.Task, h
// (if the number of duplicates is small this should fit entirely in memory)
// (Txn's MemBuf's bufferSizeLimit is currently infinity)
for _, handleRow := range handleRows {
+ // Skip the row key if it's not in the table.
+ // This can happen if the table has been recreated or truncated,
+ // and the duplicate key is from the old table.
+ if !keyInTable(handleRow[0]) {
+ continue
+ }
logger.Debug("[resolve-dupe] found row to resolve",
logutil.Key("handle", handleRow[0]),
logutil.Key("row", handleRow[1]))
@@ -1853,16 +1963,17 @@ func (local *local) LocalWriter(ctx context.Context, cfg *backend.LocalWriterCon
return nil, errors.Errorf("could not find engine for %s", engineUUID.String())
}
engine := e.(*Engine)
- return openLocalWriter(cfg, engine, local.localWriterMemCacheSize, local.bufferPool.NewBuffer())
+ return openLocalWriter(cfg, engine, local.tikvCodec, local.localWriterMemCacheSize, local.bufferPool.NewBuffer())
}
-func openLocalWriter(cfg *backend.LocalWriterConfig, engine *Engine, cacheSize int64, kvBuffer *membuf.Buffer) (*Writer, error) {
+func openLocalWriter(cfg *backend.LocalWriterConfig, engine *Engine, tikvCodec tikvclient.Codec, cacheSize int64, kvBuffer *membuf.Buffer) (*Writer, error) {
w := &Writer{
engine: engine,
memtableSizeLimit: cacheSize,
kvBuffer: kvBuffer,
isKVSorted: cfg.IsKVSorted,
isWriteBatchSorted: true,
+ tikvCodec: tikvCodec,
}
// pre-allocate a long enough buffer to avoid a lot of runtime.growslice
// this can help save about 3% of CPU.
@@ -1959,7 +2070,7 @@ func (local *local) isIngestRetryable(
}
return retryWrite, newRegion, common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
case errPb.ServerIsBusy != nil:
- return retryNone, nil, common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
+ return retryBusyIngest, nil, common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
case errPb.RegionNotFound != nil:
return retryNone, nil, common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
case errPb.ReadIndexNotReady != nil:
@@ -1992,7 +2103,8 @@ func nextKey(key []byte) []byte {
// in tikv <= 4.x, tikv will truncate the row key, so we should fetch the next valid row key
// See: https://github.com/tikv/tikv/blob/f7f22f70e1585d7ca38a59ea30e774949160c3e8/components/raftstore/src/coprocessor/split_observer.rs#L36-L41
- if tablecodec.IsRecordKey(key) {
+ // we only do this for IntHandle, which is checked by length
+ if tablecodec.IsRecordKey(key) && len(key) == tablecodec.RecordRowKeyLen {
tableID, handle, _ := tablecodec.DecodeRecordKey(key)
nextHandle := handle.Next()
// int handle overflow, use the next table prefix as nextKey
@@ -2002,7 +2114,7 @@ func nextKey(key []byte) []byte {
return tablecodec.EncodeRowKeyWithHandle(tableID, nextHandle)
}
- // if key is an index, directly append a 0x00 to the key.
+ // for index key and CommonHandle, directly append a 0x00 to the key.
res := make([]byte, 0, len(key)+1)
res = append(res, key...)
res = append(res, 0)
diff --git a/br/pkg/lightning/backend/local/local_test.go b/br/pkg/lightning/backend/local/local_test.go
index 1a399552becf9..0294cd7c5f05e 100644
--- a/br/pkg/lightning/backend/local/local_test.go
+++ b/br/pkg/lightning/backend/local/local_test.go
@@ -18,10 +18,10 @@ import (
"bytes"
"context"
"encoding/binary"
+ "fmt"
"io"
"math"
"math/rand"
- "os"
"path/filepath"
"sort"
"strings"
@@ -42,10 +42,12 @@ import (
"github.com/pingcap/tidb/br/pkg/lightning/backend/kv"
"github.com/pingcap/tidb/br/pkg/lightning/common"
"github.com/pingcap/tidb/br/pkg/lightning/log"
+ "github.com/pingcap/tidb/br/pkg/lightning/worker"
"github.com/pingcap/tidb/br/pkg/membuf"
"github.com/pingcap/tidb/br/pkg/pdutil"
"github.com/pingcap/tidb/br/pkg/restore/split"
"github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/keyspace"
tidbkv "github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/tablecodec"
@@ -113,10 +115,21 @@ func TestNextKey(t *testing.T) {
require.NoError(t, err)
nextHdl, err := tidbkv.NewCommonHandle(nextKeyBytes)
require.NoError(t, err)
- expectNextKey := []byte(tablecodec.EncodeRowKeyWithHandle(1, nextHdl))
- require.Equal(t, expectNextKey, nextKey(key))
+ nextValidKey := []byte(tablecodec.EncodeRowKeyWithHandle(1, nextHdl))
+ // nextKey may return a key that can't be decoded, but it must not be larger than the valid next key.
+ require.True(t, bytes.Compare(nextKey(key), nextValidKey) <= 0, "datums: %v", datums)
}
+ // a special case that when len(string datum) % 8 == 7, nextKey twice should not panic.
+ keyBytes, err := codec.EncodeKey(stmtCtx, nil, types.NewStringDatum("1234567"))
+ require.NoError(t, err)
+ h, err := tidbkv.NewCommonHandle(keyBytes)
+ require.NoError(t, err)
+ key = tablecodec.EncodeRowKeyWithHandle(1, h)
+ nextOnce := nextKey(key)
+ // should not panic
+ _ = nextKey(nextOnce)
+
// dIAAAAAAAAD/PV9pgAAAAAD/AAABA4AAAAD/AAAAAQOAAAD/AAAAAAEAAAD8
// a index key with: table: 61, index: 1, int64: 1, int64: 1
a := []byte{116, 128, 0, 0, 0, 0, 0, 0, 255, 61, 95, 105, 128, 0, 0, 0, 0, 255, 0, 0, 1, 3, 128, 0, 0, 0, 255, 0, 0, 0, 1, 3, 128, 0, 0, 255, 0, 0, 0, 0, 1, 0, 0, 0, 252}
@@ -237,8 +250,6 @@ func TestRangeProperties(t *testing.T) {
}
func TestRangePropertiesWithPebble(t *testing.T) {
- dir := t.TempDir()
-
sizeDistance := uint64(500)
keysDistance := uint64(20)
opt := &pebble.Options{
@@ -259,8 +270,7 @@ func TestRangePropertiesWithPebble(t *testing.T) {
},
},
}
- db, err := pebble.Open(filepath.Join(dir, "test"), opt)
- require.NoError(t, err)
+ db, _ := makePebbleDB(t, opt)
defer db.Close()
// local collector
@@ -277,7 +287,7 @@ func TestRangePropertiesWithPebble(t *testing.T) {
key := make([]byte, 8)
valueLen := rand.Intn(50)
binary.BigEndian.PutUint64(key, uint64(i*100+j))
- err = wb.Set(key, value[:valueLen], writeOpt)
+ err := wb.Set(key, value[:valueLen], writeOpt)
require.NoError(t, err)
err = collector.Add(pebble.InternalKey{UserKey: key, Trailer: pebble.InternalKeyKindSet}, value[:valueLen])
require.NoError(t, err)
@@ -304,7 +314,6 @@ func TestRangePropertiesWithPebble(t *testing.T) {
}
func testLocalWriter(t *testing.T, needSort bool, partitialSort bool) {
- dir := t.TempDir()
opt := &pebble.Options{
MemTableSize: 1024 * 1024,
MaxConcurrentCompactions: 16,
@@ -313,12 +322,8 @@ func testLocalWriter(t *testing.T, needSort bool, partitialSort bool) {
DisableWAL: true,
ReadOnly: false,
}
- db, err := pebble.Open(filepath.Join(dir, "test"), opt)
- require.NoError(t, err)
+ db, tmpPath := makePebbleDB(t, opt)
defer db.Close()
- tmpPath := filepath.Join(dir, "test.sst")
- err = os.Mkdir(tmpPath, 0o755)
- require.NoError(t, err)
_, engineUUID := backend.MakeUUID("ww", 0)
engineCtx, cancel := context.WithCancel(context.Background())
@@ -339,7 +344,7 @@ func testLocalWriter(t *testing.T, needSort bool, partitialSort bool) {
pool := membuf.NewPool()
defer pool.Destroy()
kvBuffer := pool.NewBuffer()
- w, err := openLocalWriter(&backend.LocalWriterConfig{IsKVSorted: sorted}, f, 1024, kvBuffer)
+ w, err := openLocalWriter(&backend.LocalWriterConfig{IsKVSorted: sorted}, f, keyspace.CodecV1, 1024, kvBuffer)
require.NoError(t, err)
ctx := context.Background()
@@ -564,7 +569,6 @@ func (i testIngester) ingest([]*sstMeta) error {
}
func TestLocalIngestLoop(t *testing.T) {
- dir := t.TempDir()
opt := &pebble.Options{
MemTableSize: 1024 * 1024,
MaxConcurrentCompactions: 16,
@@ -573,18 +577,14 @@ func TestLocalIngestLoop(t *testing.T) {
DisableWAL: true,
ReadOnly: false,
}
- db, err := pebble.Open(filepath.Join(dir, "test"), opt)
- require.NoError(t, err)
+ db, tmpPath := makePebbleDB(t, opt)
defer db.Close()
- tmpPath := filepath.Join(dir, "test.sst")
- err = os.Mkdir(tmpPath, 0o755)
- require.NoError(t, err)
_, engineUUID := backend.MakeUUID("ww", 0)
engineCtx, cancel := context.WithCancel(context.Background())
f := Engine{
db: db,
UUID: engineUUID,
- sstDir: "",
+ sstDir: tmpPath,
ctx: engineCtx,
cancel: cancel,
sstMetasChan: make(chan metaOrFlush, 64),
@@ -637,7 +637,7 @@ func TestLocalIngestLoop(t *testing.T) {
wg.Wait()
f.mutex.RLock()
- err = f.flushEngineWithoutLock(engineCtx)
+ err := f.flushEngineWithoutLock(engineCtx)
require.NoError(t, err)
f.mutex.RUnlock()
@@ -732,7 +732,6 @@ func TestFilterOverlapRange(t *testing.T) {
}
func testMergeSSTs(t *testing.T, kvs [][]common.KvPair, meta *sstMeta) {
- dir := t.TempDir()
opt := &pebble.Options{
MemTableSize: 1024 * 1024,
MaxConcurrentCompactions: 16,
@@ -741,12 +740,8 @@ func testMergeSSTs(t *testing.T, kvs [][]common.KvPair, meta *sstMeta) {
DisableWAL: true,
ReadOnly: false,
}
- db, err := pebble.Open(filepath.Join(dir, "test"), opt)
- require.NoError(t, err)
+ db, tmpPath := makePebbleDB(t, opt)
defer db.Close()
- tmpPath := filepath.Join(dir, "test.sst")
- err = os.Mkdir(tmpPath, 0o755)
- require.NoError(t, err)
_, engineUUID := backend.MakeUUID("ww", 0)
engineCtx, cancel := context.WithCancel(context.Background())
@@ -837,49 +832,90 @@ func TestMergeSSTsDuplicated(t *testing.T) {
type mockPdClient struct {
pd.Client
- stores []*metapb.Store
+ stores []*metapb.Store
+ regions []*pd.Region
}
func (c *mockPdClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) ([]*metapb.Store, error) {
return c.stores, nil
}
+func (c *mockPdClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*pd.Region, error) {
+ return c.regions, nil
+}
+
type mockGrpcErr struct{}
func (e mockGrpcErr) GRPCStatus() *status.Status {
- return status.New(codes.Unimplemented, "unimplmented")
+ return status.New(codes.Unimplemented, "unimplemented")
}
func (e mockGrpcErr) Error() string {
- return "unimplmented"
+ return "unimplemented"
}
type mockImportClient struct {
sst.ImportSSTClient
store *metapb.Store
+ resp *sst.IngestResponse
err error
retry int
cnt int
multiIngestCheckFn func(s *metapb.Store) bool
+ apiInvokeRecorder map[string][]uint64
+}
+
+func newMockImportClient() *mockImportClient {
+ return &mockImportClient{
+ multiIngestCheckFn: func(s *metapb.Store) bool {
+ return true
+ },
+ }
}
func (c *mockImportClient) MultiIngest(context.Context, *sst.MultiIngestRequest, ...grpc.CallOption) (*sst.IngestResponse, error) {
defer func() {
c.cnt++
}()
- if c.cnt < c.retry && c.err != nil {
- return nil, c.err
+ if c.apiInvokeRecorder != nil {
+ c.apiInvokeRecorder["MultiIngest"] = append(c.apiInvokeRecorder["MultiIngest"], c.store.GetId())
+ }
+ if c.cnt < c.retry && (c.err != nil || c.resp != nil) {
+ return c.resp, c.err
}
if !c.multiIngestCheckFn(c.store) {
return nil, mockGrpcErr{}
}
- return nil, nil
+ return &sst.IngestResponse{}, nil
+}
+
+type mockWriteClient struct {
+ sst.ImportSST_WriteClient
+ writeResp *sst.WriteResponse
+}
+
+func (m mockWriteClient) Send(request *sst.WriteRequest) error {
+ return nil
+}
+
+func (m mockWriteClient) CloseAndRecv() (*sst.WriteResponse, error) {
+ return m.writeResp, nil
+}
+
+func (c *mockImportClient) Write(ctx context.Context, opts ...grpc.CallOption) (sst.ImportSST_WriteClient, error) {
+ if c.apiInvokeRecorder != nil {
+ c.apiInvokeRecorder["Write"] = append(c.apiInvokeRecorder["Write"], c.store.GetId())
+ }
+ return mockWriteClient{writeResp: &sst.WriteResponse{Metas: []*sst.SSTMeta{
+ {}, {}, {},
+ }}}, nil
}
type mockImportClientFactory struct {
- stores []*metapb.Store
- createClientFn func(store *metapb.Store) sst.ImportSSTClient
+ stores []*metapb.Store
+ createClientFn func(store *metapb.Store) sst.ImportSSTClient
+ apiInvokeRecorder map[string][]uint64
}
func (f *mockImportClientFactory) Create(_ context.Context, storeID uint64) (sst.ImportSSTClient, error) {
@@ -888,7 +924,7 @@ func (f *mockImportClientFactory) Create(_ context.Context, storeID uint64) (sst
return f.createClientFn(store), nil
}
}
- return nil, errors.New("store not found")
+ return nil, fmt.Errorf("store %d not found", storeID)
}
func (f *mockImportClientFactory) Close() {}
@@ -1220,3 +1256,78 @@ func TestLocalIsRetryableTiKVWriteError(t *testing.T) {
require.True(t, l.isRetryableImportTiKVError(io.EOF))
require.True(t, l.isRetryableImportTiKVError(errors.Trace(io.EOF)))
}
+
+func TestCheckPeersBusy(t *testing.T) {
+ ctx := context.Background()
+ pdCli := &mockPdClient{}
+ pdCtl := &pdutil.PdController{}
+ pdCtl.SetPDClient(pdCli)
+
+ keys := [][]byte{[]byte(""), []byte("a"), []byte("b"), []byte("")}
+ splitCli := initTestSplitClient3Replica(keys, nil)
+ apiInvokeRecorder := map[string][]uint64{}
+ serverIsBusyResp := &sst.IngestResponse{
+ Error: &errorpb.Error{
+ ServerIsBusy: &errorpb.ServerIsBusy{},
+ }}
+
+ createTimeStore12 := 0
+ local := &local{
+ pdCtl: pdCtl,
+ splitCli: splitCli,
+ importClientFactory: &mockImportClientFactory{
+ stores: []*metapb.Store{
+ // region ["", "a") is not used, skip (1, 2, 3)
+ {Id: 11}, {Id: 12}, {Id: 13}, // region ["a", "b")
+ {Id: 21}, {Id: 22}, {Id: 23}, // region ["b", "")
+ },
+ createClientFn: func(store *metapb.Store) sst.ImportSSTClient {
+ importCli := newMockImportClient()
+ importCli.store = store
+ importCli.apiInvokeRecorder = apiInvokeRecorder
+ if store.Id == 12 {
+ createTimeStore12++
+ // the second time to checkWriteStall
+ if createTimeStore12 == 2 {
+ importCli.retry = 1
+ importCli.resp = serverIsBusyResp
+ }
+ }
+ return importCli
+ },
+ },
+ logger: log.L(),
+ ingestConcurrency: worker.NewPool(ctx, 1, "ingest"),
+ writeLimiter: noopStoreWriteLimiter{},
+ bufferPool: membuf.NewPool(),
+ supportMultiIngest: true,
+ shouldCheckWriteStall: true,
+ tikvCodec: keyspace.CodecV1,
+ }
+
+ db, tmpPath := makePebbleDB(t, nil)
+ _, engineUUID := backend.MakeUUID("ww", 0)
+ engineCtx, cancel := context.WithCancel(context.Background())
+ f := &Engine{
+ db: db,
+ UUID: engineUUID,
+ sstDir: tmpPath,
+ ctx: engineCtx,
+ cancel: cancel,
+ sstMetasChan: make(chan metaOrFlush, 64),
+ keyAdapter: noopKeyAdapter{},
+ logger: log.L(),
+ }
+ err := f.db.Set([]byte("a"), []byte("a"), nil)
+ require.NoError(t, err)
+ err = f.db.Set([]byte("b"), []byte("b"), nil)
+ require.NoError(t, err)
+ err = local.writeAndIngestByRange(ctx, f, []byte("a"), []byte("c"), 0, 0)
+ require.NoError(t, err)
+
+ require.Equal(t, []uint64{11, 12, 13, 21, 22, 23}, apiInvokeRecorder["Write"])
+ // store 12 has a follower busy, so it will break the workflow for region (11, 12, 13)
+ require.Equal(t, []uint64{11, 12, 21, 22, 23, 21}, apiInvokeRecorder["MultiIngest"])
+ // region (11, 12, 13) has key range ["a", "b"), it's not finished.
+ require.Equal(t, []Range{{start: []byte("b"), end: []byte("c")}}, f.finishedRanges.ranges)
+}
diff --git a/br/pkg/lightning/backend/local/localhelper.go b/br/pkg/lightning/backend/local/localhelper.go
index 8b5819175430b..cd8a27b9bda48 100644
--- a/br/pkg/lightning/backend/local/localhelper.go
+++ b/br/pkg/lightning/backend/local/localhelper.go
@@ -27,6 +27,7 @@ import (
"github.com/docker/go-units"
"github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
sst "github.com/pingcap/kvproto/pkg/import_sstpb"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/kvproto/pkg/pdpb"
@@ -379,7 +380,14 @@ func fetchTableRegionSizeStats(ctx context.Context, db *sql.DB, tableID int64) (
return stats, errors.Trace(err)
}
-func (local *local) BatchSplitRegions(ctx context.Context, region *split.RegionInfo, keys [][]byte) (*split.RegionInfo, []*split.RegionInfo, error) {
+func (local *local) BatchSplitRegions(
+ ctx context.Context,
+ region *split.RegionInfo,
+ keys [][]byte,
+) (*split.RegionInfo, []*split.RegionInfo, error) {
+ failpoint.Inject("failToSplit", func(_ failpoint.Value) {
+ failpoint.Return(nil, nil, errors.New("retryable error"))
+ })
region, newRegions, err := local.splitCli.BatchSplitRegionsWithOrigin(ctx, region, keys)
if err != nil {
return nil, nil, errors.Annotatef(err, "batch split regions failed")
diff --git a/br/pkg/lightning/backend/local/localhelper_test.go b/br/pkg/lightning/backend/local/localhelper_test.go
index 6cbf7f2f14808..023fade304fae 100644
--- a/br/pkg/lightning/backend/local/localhelper_test.go
+++ b/br/pkg/lightning/backend/local/localhelper_test.go
@@ -47,7 +47,7 @@ func init() {
splitRetryTimes = 2
}
-type testClient struct {
+type testSplitClient struct {
mu sync.RWMutex
stores map[uint64]*metapb.Store
regions map[uint64]*split.RegionInfo
@@ -57,17 +57,17 @@ type testClient struct {
hook clientHook
}
-func newTestClient(
+func newTestSplitClient(
stores map[uint64]*metapb.Store,
regions map[uint64]*split.RegionInfo,
nextRegionID uint64,
hook clientHook,
-) *testClient {
+) *testSplitClient {
regionsInfo := &pdtypes.RegionTree{}
for _, regionInfo := range regions {
regionsInfo.SetRegion(pdtypes.NewRegionInfo(regionInfo.Region, regionInfo.Leader))
}
- return &testClient{
+ return &testSplitClient{
stores: stores,
regions: regions,
regionsInfo: regionsInfo,
@@ -77,17 +77,17 @@ func newTestClient(
}
// ScatterRegions scatters regions in a batch.
-func (c *testClient) ScatterRegions(ctx context.Context, regionInfo []*split.RegionInfo) error {
+func (c *testSplitClient) ScatterRegions(ctx context.Context, regionInfo []*split.RegionInfo) error {
return nil
}
-func (c *testClient) GetAllRegions() map[uint64]*split.RegionInfo {
+func (c *testSplitClient) GetAllRegions() map[uint64]*split.RegionInfo {
c.mu.RLock()
defer c.mu.RUnlock()
return c.regions
}
-func (c *testClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) {
+func (c *testSplitClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) {
c.mu.RLock()
defer c.mu.RUnlock()
store, ok := c.stores[storeID]
@@ -97,19 +97,18 @@ func (c *testClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Stor
return store, nil
}
-func (c *testClient) GetRegion(ctx context.Context, key []byte) (*split.RegionInfo, error) {
+func (c *testSplitClient) GetRegion(ctx context.Context, key []byte) (*split.RegionInfo, error) {
c.mu.RLock()
defer c.mu.RUnlock()
for _, region := range c.regions {
- if bytes.Compare(key, region.Region.StartKey) >= 0 &&
- (len(region.Region.EndKey) == 0 || bytes.Compare(key, region.Region.EndKey) < 0) {
+ if bytes.Compare(key, region.Region.StartKey) >= 0 && beforeEnd(key, region.Region.EndKey) {
return region, nil
}
}
return nil, errors.Errorf("region not found: key=%s", string(key))
}
-func (c *testClient) GetRegionByID(ctx context.Context, regionID uint64) (*split.RegionInfo, error) {
+func (c *testSplitClient) GetRegionByID(ctx context.Context, regionID uint64) (*split.RegionInfo, error) {
c.mu.RLock()
defer c.mu.RUnlock()
region, ok := c.regions[regionID]
@@ -119,7 +118,7 @@ func (c *testClient) GetRegionByID(ctx context.Context, regionID uint64) (*split
return region, nil
}
-func (c *testClient) SplitRegion(
+func (c *testSplitClient) SplitRegion(
ctx context.Context,
regionInfo *split.RegionInfo,
key []byte,
@@ -130,7 +129,7 @@ func (c *testClient) SplitRegion(
splitKey := codec.EncodeBytes([]byte{}, key)
for _, region := range c.regions {
if bytes.Compare(splitKey, region.Region.StartKey) >= 0 &&
- (len(region.Region.EndKey) == 0 || bytes.Compare(splitKey, region.Region.EndKey) < 0) {
+ beforeEnd(splitKey, region.Region.EndKey) {
target = region
}
}
@@ -159,7 +158,7 @@ func (c *testClient) SplitRegion(
return newRegion, nil
}
-func (c *testClient) BatchSplitRegionsWithOrigin(
+func (c *testSplitClient) BatchSplitRegionsWithOrigin(
ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte,
) (*split.RegionInfo, []*split.RegionInfo, error) {
c.mu.Lock()
@@ -234,24 +233,24 @@ func (c *testClient) BatchSplitRegionsWithOrigin(
return target, newRegions, err
}
-func (c *testClient) BatchSplitRegions(
+func (c *testSplitClient) BatchSplitRegions(
ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte,
) ([]*split.RegionInfo, error) {
_, newRegions, err := c.BatchSplitRegionsWithOrigin(ctx, regionInfo, keys)
return newRegions, err
}
-func (c *testClient) ScatterRegion(ctx context.Context, regionInfo *split.RegionInfo) error {
+func (c *testSplitClient) ScatterRegion(ctx context.Context, regionInfo *split.RegionInfo) error {
return nil
}
-func (c *testClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) {
+func (c *testSplitClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) {
return &pdpb.GetOperatorResponse{
Header: new(pdpb.ResponseHeader),
}, nil
}
-func (c *testClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*split.RegionInfo, error) {
+func (c *testSplitClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*split.RegionInfo, error) {
if c.hook != nil {
key, endKey, limit = c.hook.BeforeScanRegions(ctx, key, endKey, limit)
}
@@ -272,19 +271,19 @@ func (c *testClient) ScanRegions(ctx context.Context, key, endKey []byte, limit
return regions, err
}
-func (c *testClient) GetPlacementRule(ctx context.Context, groupID, ruleID string) (r pdtypes.Rule, err error) {
+func (c *testSplitClient) GetPlacementRule(ctx context.Context, groupID, ruleID string) (r pdtypes.Rule, err error) {
return
}
-func (c *testClient) SetPlacementRule(ctx context.Context, rule pdtypes.Rule) error {
+func (c *testSplitClient) SetPlacementRule(ctx context.Context, rule pdtypes.Rule) error {
return nil
}
-func (c *testClient) DeletePlacementRule(ctx context.Context, groupID, ruleID string) error {
+func (c *testSplitClient) DeletePlacementRule(ctx context.Context, groupID, ruleID string) error {
return nil
}
-func (c *testClient) SetStoresLabel(ctx context.Context, stores []uint64, labelKey, labelValue string) error {
+func (c *testSplitClient) SetStoresLabel(ctx context.Context, stores []uint64, labelKey, labelValue string) error {
return nil
}
@@ -305,7 +304,7 @@ func cloneRegion(region *split.RegionInfo) *split.RegionInfo {
// For keys ["", "aay", "bba", "bbh", "cca", ""], the key ranges of
// regions are [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, ).
-func initTestClient(keys [][]byte, hook clientHook) *testClient {
+func initTestSplitClient(keys [][]byte, hook clientHook) *testSplitClient {
peers := make([]*metapb.Peer, 1)
peers[0] = &metapb.Peer{
Id: 1,
@@ -329,13 +328,56 @@ func initTestClient(keys [][]byte, hook clientHook) *testClient {
EndKey: endKey,
RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1},
},
+ Leader: peers[0],
}
}
stores := make(map[uint64]*metapb.Store)
stores[1] = &metapb.Store{
Id: 1,
}
- return newTestClient(stores, regions, uint64(len(keys)), hook)
+ return newTestSplitClient(stores, regions, uint64(len(keys)), hook)
+}
+
+// initTestSplitClient3Replica will create a client that each region has 3 replicas, and their IDs and StoreIDs are
+// (1, 2, 3), (11, 12, 13), ...
+// For keys ["", "aay", "bba", "bbh", "cca", ""], the key ranges of
+// region ranges are [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, ).
+func initTestSplitClient3Replica(keys [][]byte, hook clientHook) *testSplitClient {
+ regions := make(map[uint64]*split.RegionInfo)
+ stores := make(map[uint64]*metapb.Store)
+ for i := uint64(1); i < uint64(len(keys)); i++ {
+ startKey := keys[i-1]
+ if len(startKey) != 0 {
+ startKey = codec.EncodeBytes([]byte{}, startKey)
+ }
+ endKey := keys[i]
+ if len(endKey) != 0 {
+ endKey = codec.EncodeBytes([]byte{}, endKey)
+ }
+ baseID := (i-1)*10 + 1
+ peers := make([]*metapb.Peer, 3)
+ for j := 0; j < 3; j++ {
+ peers[j] = &metapb.Peer{
+ Id: baseID + uint64(j),
+ StoreId: baseID + uint64(j),
+ }
+ }
+
+ regions[baseID] = &split.RegionInfo{
+ Region: &metapb.Region{
+ Id: baseID,
+ Peers: peers,
+ StartKey: startKey,
+ EndKey: endKey,
+ RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1},
+ },
+ Leader: peers[0],
+ }
+ stores[baseID] = &metapb.Store{
+ Id: baseID,
+ }
+ }
+ return newTestSplitClient(stores, regions, uint64(len(keys)), hook)
}
func checkRegionRanges(t *testing.T, regions []*split.RegionInfo, keys [][]byte) {
@@ -376,7 +418,7 @@ func (h *noopHook) AfterScanRegions(res []*split.RegionInfo, err error) ([]*spli
type batchSplitHook interface {
setup(t *testing.T) func()
- check(t *testing.T, cli *testClient)
+ check(t *testing.T, cli *testSplitClient)
}
type defaultHook struct{}
@@ -392,7 +434,7 @@ func (d defaultHook) setup(t *testing.T) func() {
}
}
-func (d defaultHook) check(t *testing.T, cli *testClient) {
+func (d defaultHook) check(t *testing.T, cli *testSplitClient) {
// so with a batch split size of 4, there will be 7 time batch split
// 1. region: [aay, bba), keys: [b, ba, bb]
// 2. region: [bbh, cca), keys: [bc, bd, be, bf]
@@ -414,7 +456,7 @@ func doTestBatchSplitRegionByRanges(ctx context.Context, t *testing.T, hook clie
defer deferFunc()
keys := [][]byte{[]byte(""), []byte("aay"), []byte("bba"), []byte("bbh"), []byte("cca"), []byte("")}
- client := initTestClient(keys, hook)
+ client := initTestSplitClient(keys, hook)
local := &local{
splitCli: client,
g: glue.NewExternalTiDBGlue(nil, mysql.ModeNone),
@@ -479,7 +521,7 @@ func (h batchSizeHook) setup(t *testing.T) func() {
}
}
-func (h batchSizeHook) check(t *testing.T, cli *testClient) {
+func (h batchSizeHook) check(t *testing.T, cli *testSplitClient) {
// so with a batch split key size of 6, there will be 9 time batch split
// 1. region: [aay, bba), keys: [b, ba, bb]
// 2. region: [bbh, cca), keys: [bc, bd, be]
@@ -583,7 +625,7 @@ func TestSplitAndScatterRegionInBatches(t *testing.T) {
defer deferFunc()
keys := [][]byte{[]byte(""), []byte("a"), []byte("b"), []byte("")}
- client := initTestClient(keys, nil)
+ client := initTestSplitClient(keys, nil)
local := &local{
splitCli: client,
g: glue.NewExternalTiDBGlue(nil, mysql.ModeNone),
@@ -670,7 +712,7 @@ func doTestBatchSplitByRangesWithClusteredIndex(t *testing.T, hook clientHook) {
keys = append(keys, key)
}
keys = append(keys, tableEndKey, []byte(""))
- client := initTestClient(keys, hook)
+ client := initTestSplitClient(keys, hook)
local := &local{
splitCli: client,
g: glue.NewExternalTiDBGlue(nil, mysql.ModeNone),
diff --git a/br/pkg/lightning/checkpoints/checkpoints.go b/br/pkg/lightning/checkpoints/checkpoints.go
index 13817e28eb668..d20134660de9c 100644
--- a/br/pkg/lightning/checkpoints/checkpoints.go
+++ b/br/pkg/lightning/checkpoints/checkpoints.go
@@ -262,6 +262,29 @@ func (ccp *ChunkCheckpoint) DeepCopy() *ChunkCheckpoint {
}
}
+func (ccp *ChunkCheckpoint) UnfinishedSize() int64 {
+ if ccp.FileMeta.Compression == mydump.CompressionNone {
+ return ccp.Chunk.EndOffset - ccp.Chunk.Offset
+ }
+ return ccp.FileMeta.FileSize - ccp.Chunk.RealOffset
+}
+
+func (ccp *ChunkCheckpoint) TotalSize() int64 {
+ if ccp.FileMeta.Compression == mydump.CompressionNone {
+ return ccp.Chunk.EndOffset - ccp.Key.Offset
+ }
+ // TODO: compressed file won't be split into chunks, so using FileSize as TotalSize is ok
+ // change this when we support split compressed file into chunks
+ return ccp.FileMeta.FileSize
+}
+
+func (ccp *ChunkCheckpoint) FinishedSize() int64 {
+ if ccp.FileMeta.Compression == mydump.CompressionNone {
+ return ccp.Chunk.Offset - ccp.Key.Offset
+ }
+ return ccp.Chunk.RealOffset - ccp.Key.Offset
+}
+
type EngineCheckpoint struct {
Status CheckpointStatus
Chunks []*ChunkCheckpoint // a sorted array
diff --git a/br/pkg/lightning/common/BUILD.bazel b/br/pkg/lightning/common/BUILD.bazel
index a5245356b2715..3bd871276d733 100644
--- a/br/pkg/lightning/common/BUILD.bazel
+++ b/br/pkg/lightning/common/BUILD.bazel
@@ -33,10 +33,51 @@ go_library(
"@org_golang_google_grpc//:grpc",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//credentials",
+ "@org_golang_google_grpc//credentials/insecure",
"@org_golang_google_grpc//status",
- "@org_golang_x_sys//unix",
"@org_uber_go_zap//:zap",
- ],
+ ] + select({
+ "@io_bazel_rules_go//go/platform:aix": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:android": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:darwin": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:dragonfly": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:freebsd": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:illumos": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:ios": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:js": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:linux": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:netbsd": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:openbsd": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:plan9": [
+ "@org_golang_x_sys//unix",
+ ],
+ "@io_bazel_rules_go//go/platform:solaris": [
+ "@org_golang_x_sys//unix",
+ ],
+ "//conditions:default": [],
+ }),
)
go_test(
diff --git a/br/pkg/lightning/common/errors.go b/br/pkg/lightning/common/errors.go
index c2fc3dbaa901f..14645217636a4 100644
--- a/br/pkg/lightning/common/errors.go
+++ b/br/pkg/lightning/common/errors.go
@@ -51,6 +51,7 @@ var (
ErrCheckTableEmpty = errors.Normalize("check table empty error", errors.RFCCodeText("Lightning:PreCheck:ErrCheckTableEmpty"))
ErrCheckCSVHeader = errors.Normalize("check csv header error", errors.RFCCodeText("Lightning:PreCheck:ErrCheckCSVHeader"))
ErrCheckDataSource = errors.Normalize("check data source error", errors.RFCCodeText("Lightning:PreCheck:ErrCheckDataSource"))
+ ErrCheckCDCPiTR = errors.Normalize("check TiCDC/PiTR task error", errors.RFCCodeText("Lightning:PreCheck:ErrCheckCDCPiTR"))
ErrOpenCheckpoint = errors.Normalize("open checkpoint error", errors.RFCCodeText("Lightning:Checkpoint:ErrOpenCheckpoint"))
ErrReadCheckpoint = errors.Normalize("read checkpoint error", errors.RFCCodeText("Lightning:Checkpoint:ErrReadCheckpoint"))
@@ -96,6 +97,7 @@ var (
ErrInvalidMetaStatus = errors.Normalize("invalid meta status: '%s'", errors.RFCCodeText("Lightning:Restore:ErrInvalidMetaStatus"))
ErrTableIsChecksuming = errors.Normalize("table '%s' is checksuming", errors.RFCCodeText("Lightning:Restore:ErrTableIsChecksuming"))
ErrResolveDuplicateRows = errors.Normalize("resolve duplicate rows error on table '%s'", errors.RFCCodeText("Lightning:Restore:ErrResolveDuplicateRows"))
+ ErrFoundDuplicateKeys = errors.Normalize("found duplicate key '%s', value '%s'", errors.RFCCodeText("Lightning:Restore:ErrFoundDuplicateKey"))
)
type withStack struct {
diff --git a/br/pkg/lightning/common/security.go b/br/pkg/lightning/common/security.go
index a48abc48c2c54..8d0c98223d18a 100644
--- a/br/pkg/lightning/common/security.go
+++ b/br/pkg/lightning/common/security.go
@@ -28,6 +28,7 @@ import (
pd "github.com/tikv/pd/client"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
)
type TLS struct {
@@ -104,7 +105,7 @@ func (tc *TLS) ToGRPCDialOption() grpc.DialOption {
if tc.inner != nil {
return grpc.WithTransportCredentials(credentials.NewTLS(tc.inner))
}
- return grpc.WithInsecure()
+ return grpc.WithTransportCredentials(insecure.NewCredentials())
}
// WrapListener places a TLS layer on top of the existing listener.
diff --git a/br/pkg/lightning/common/util.go b/br/pkg/lightning/common/util.go
index 679ba6cc5d48b..b9bdf564403de 100644
--- a/br/pkg/lightning/common/util.go
+++ b/br/pkg/lightning/common/util.go
@@ -16,6 +16,7 @@ package common
import (
"context"
+ "crypto/tls"
"database/sql"
"encoding/base64"
"encoding/json"
@@ -47,14 +48,16 @@ const (
// MySQLConnectParam records the parameters needed to connect to a MySQL database.
type MySQLConnectParam struct {
- Host string
- Port int
- User string
- Password string
- SQLMode string
- MaxAllowedPacket uint64
- TLS string
- Vars map[string]string
+ Host string
+ Port int
+ User string
+ Password string
+ SQLMode string
+ MaxAllowedPacket uint64
+ TLSConfig *tls.Config
+ AllowFallbackToPlaintext bool
+ Net string
+ Vars map[string]string
}
func (param *MySQLConnectParam) ToDriverConfig() *mysql.Config {
@@ -64,11 +67,16 @@ func (param *MySQLConnectParam) ToDriverConfig() *mysql.Config {
cfg.User = param.User
cfg.Passwd = param.Password
cfg.Net = "tcp"
+ if param.Net != "" {
+ cfg.Net = param.Net
+ }
cfg.Addr = net.JoinHostPort(param.Host, strconv.Itoa(param.Port))
cfg.Params["charset"] = "utf8mb4"
cfg.Params["sql_mode"] = fmt.Sprintf("'%s'", param.SQLMode)
cfg.MaxAllowedPacket = int(param.MaxAllowedPacket)
- cfg.TLSConfig = param.TLS
+
+ cfg.TLS = param.TLSConfig
+ cfg.AllowFallbackToPlaintext = param.AllowFallbackToPlaintext
for k, v := range param.Vars {
cfg.Params[k] = fmt.Sprintf("'%s'", v)
diff --git a/br/pkg/lightning/config/BUILD.bazel b/br/pkg/lightning/config/BUILD.bazel
index 6d2cb32a9ccc7..b035b506aebf2 100644
--- a/br/pkg/lightning/config/BUILD.bazel
+++ b/br/pkg/lightning/config/BUILD.bazel
@@ -24,8 +24,9 @@ go_library(
"@com_github_carlmjohnson_flagext//:flagext",
"@com_github_docker_go_units//:go-units",
"@com_github_go_sql_driver_mysql//:mysql",
- "@com_github_google_uuid//:uuid",
"@com_github_pingcap_errors//:errors",
+ "@org_golang_google_grpc//:grpc",
+ "@org_golang_google_grpc//keepalive",
"@org_uber_go_atomic//:atomic",
"@org_uber_go_zap//:zap",
],
diff --git a/br/pkg/lightning/config/config.go b/br/pkg/lightning/config/config.go
index fdbe962433806..d535478c5f4c4 100644
--- a/br/pkg/lightning/config/config.go
+++ b/br/pkg/lightning/config/config.go
@@ -16,6 +16,7 @@ package config
import (
"context"
+ "crypto/tls"
"encoding/json"
"fmt"
"math"
@@ -32,7 +33,6 @@ import (
"github.com/BurntSushi/toml"
"github.com/docker/go-units"
gomysql "github.com/go-sql-driver/mysql"
- "github.com/google/uuid"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/br/pkg/lightning/common"
"github.com/pingcap/tidb/br/pkg/lightning/log"
@@ -135,6 +135,9 @@ type DBStore struct {
IndexSerialScanConcurrency int `toml:"index-serial-scan-concurrency" json:"index-serial-scan-concurrency"`
ChecksumTableConcurrency int `toml:"checksum-table-concurrency" json:"checksum-table-concurrency"`
Vars map[string]string `toml:"-" json:"vars"`
+
+ IOTotalBytes *atomic.Uint64 `toml:"-" json:"-"`
+ UUID string `toml:"-" json:"-"`
}
type Config struct {
@@ -340,32 +343,64 @@ type MaxError struct {
// In TiDB backend, this also includes all possible SQL errors raised from INSERT,
// such as unique key conflict when `on-duplicate` is set to `error`.
// When tolerated, the row causing the error will be skipped, and adds 1 to the counter.
+ // The default value is zero, which means that such errors are not tolerated.
Type atomic.Int64 `toml:"type" json:"type"`
// Conflict is the maximum number of unique key conflicts in local backend accepted.
// When tolerated, every pair of conflict adds 1 to the counter.
// Those pairs will NOT be deleted from the target. Conflict resolution is performed separately.
- // TODO Currently this is hard-coded to infinity.
- Conflict atomic.Int64 `toml:"conflict" json:"-"`
+ // The default value is max int64, which means conflict errors will be recorded as much as possible.
+ // Sometime the actual number of conflict record logged will be greater than the value configured here,
+ // because conflict error data are recorded batch by batch.
+ // If the limit is reached in a single batch, the entire batch of records will be persisted before an error is reported.
+ Conflict atomic.Int64 `toml:"conflict" json:"conflict"`
}
func (cfg *MaxError) UnmarshalTOML(v interface{}) error {
+ defaultValMap := map[string]int64{
+ "syntax": 0,
+ "charset": math.MaxInt64,
+ "type": 0,
+ "conflict": math.MaxInt64,
+ }
+ // set default value first
+ cfg.Syntax.Store(defaultValMap["syntax"])
+ cfg.Charset.Store(defaultValMap["charset"])
+ cfg.Type.Store(defaultValMap["type"])
+ cfg.Conflict.Store(defaultValMap["conflict"])
switch val := v.(type) {
case int64:
// ignore val that is smaller than 0
- if val < 0 {
- val = 0
+ if val >= 0 {
+ // only set type error
+ cfg.Type.Store(val)
}
- cfg.Syntax.Store(0)
- cfg.Charset.Store(math.MaxInt64)
- cfg.Type.Store(val)
- cfg.Conflict.Store(math.MaxInt64)
return nil
case map[string]interface{}:
- // TODO support stuff like `max-error = { charset = 1000, type = 1000 }` if proved useful.
+ // support stuff like `max-error = { charset = 1000, type = 1000 }`.
+ getVal := func(k string, v interface{}) int64 {
+ defaultVal, ok := defaultValMap[k]
+ if !ok {
+ return 0
+ }
+ iVal, ok := v.(int64)
+ if !ok || iVal < 0 {
+ return defaultVal
+ }
+ return iVal
+ }
+ for k, v := range val {
+ switch k {
+ case "type":
+ cfg.Type.Store(getVal(k, v))
+ case "conflict":
+ cfg.Conflict.Store(getVal(k, v))
+ }
+ }
+ return nil
default:
+ return errors.Errorf("invalid max-error '%v', should be an integer or a map of string:int64", v)
}
- return errors.Errorf("invalid max-error '%v', should be an integer", v)
}
// DuplicateResolutionAlgorithm is the config type of how to resolve duplicates.
@@ -381,6 +416,10 @@ const (
// DupeResAlgRemove records all duplicate records like the 'record' algorithm and remove all information related to the
// duplicated rows. Users need to analyze the lightning_task_info.conflict_error_v1 table to add back the correct rows.
DupeResAlgRemove
+
+ // DupeResAlgErr reports an error and stops the import process.
+ // Note: this value is only used for internal.
+ DupeResAlgErr
)
func (dra *DuplicateResolutionAlgorithm) UnmarshalTOML(v interface{}) error {
@@ -429,6 +468,58 @@ func (dra DuplicateResolutionAlgorithm) String() string {
}
}
+// CompressionType is the config type of compression algorithm.
+type CompressionType int
+
+const (
+ // CompressionNone means no compression.
+ CompressionNone CompressionType = iota
+ // CompressionGzip means gzip compression.
+ CompressionGzip
+)
+
+func (t *CompressionType) UnmarshalTOML(v interface{}) error {
+ if val, ok := v.(string); ok {
+ return t.FromStringValue(val)
+ }
+ return errors.Errorf("invalid compression-type '%v', please choose valid option between ['gzip']", v)
+}
+
+func (t CompressionType) MarshalText() ([]byte, error) {
+ return []byte(t.String()), nil
+}
+
+func (t *CompressionType) FromStringValue(s string) error {
+ switch strings.ToLower(s) {
+ case "":
+ *t = CompressionNone
+ case "gz", "gzip":
+ *t = CompressionGzip
+ default:
+ return errors.Errorf("invalid compression-type '%s', please choose valid option between ['gzip']", s)
+ }
+ return nil
+}
+
+func (t *CompressionType) MarshalJSON() ([]byte, error) {
+ return []byte(`"` + t.String() + `"`), nil
+}
+
+func (t *CompressionType) UnmarshalJSON(data []byte) error {
+ return t.FromStringValue(strings.Trim(string(data), `"`))
+}
+
+func (t CompressionType) String() string {
+ switch t {
+ case CompressionGzip:
+ return "gzip"
+ case CompressionNone:
+ return ""
+ default:
+ panic(fmt.Sprintf("invalid compression type '%d'", t))
+ }
+}
+
// PostRestore has some options which will be executed after kv restored.
type PostRestore struct {
Checksum PostOpLevel `toml:"checksum" json:"checksum"`
@@ -440,14 +531,18 @@ type PostRestore struct {
type CSVConfig struct {
// Separator, Delimiter and Terminator should all be in utf8mb4 encoding.
- Separator string `toml:"separator" json:"separator"`
- Delimiter string `toml:"delimiter" json:"delimiter"`
- Terminator string `toml:"terminator" json:"terminator"`
- Null string `toml:"null" json:"null"`
- Header bool `toml:"header" json:"header"`
- TrimLastSep bool `toml:"trim-last-separator" json:"trim-last-separator"`
- NotNull bool `toml:"not-null" json:"not-null"`
- BackslashEscape bool `toml:"backslash-escape" json:"backslash-escape"`
+ Separator string `toml:"separator" json:"separator"`
+ Delimiter string `toml:"delimiter" json:"delimiter"`
+ Terminator string `toml:"terminator" json:"terminator"`
+ Null string `toml:"null" json:"null"`
+ Header bool `toml:"header" json:"header"`
+ HeaderSchemaMatch bool `toml:"header-schema-match" json:"header-schema-match"`
+ TrimLastSep bool `toml:"trim-last-separator" json:"trim-last-separator"`
+ NotNull bool `toml:"not-null" json:"not-null"`
+ BackslashEscape bool `toml:"backslash-escape" json:"backslash-escape"`
+ // hide these options for lightning configuration file, they can only be used by LOAD DATA
+ // https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-field-line-handling
+ StartingBy string `toml:"-" json:"-"`
}
type MydumperRuntime struct {
@@ -540,6 +635,7 @@ type TikvImporter struct {
OnDuplicate string `toml:"on-duplicate" json:"on-duplicate"`
MaxKVPairs int `toml:"max-kv-pairs" json:"max-kv-pairs"`
SendKVPairs int `toml:"send-kv-pairs" json:"send-kv-pairs"`
+ CompressKVPairs CompressionType `toml:"compress-kv-pairs" json:"compress-kv-pairs"`
RegionSplitSize ByteSize `toml:"region-split-size" json:"region-split-size"`
RegionSplitKeys int `toml:"region-split-keys" json:"region-split-keys"`
SortedKVDir string `toml:"sorted-kv-dir" json:"sorted-kv-dir"`
@@ -575,9 +671,8 @@ type Security struct {
// RedactInfoLog indicates that whether enabling redact log
RedactInfoLog bool `toml:"redact-info-log" json:"redact-info-log"`
- // TLSConfigName is used to set tls config for lightning in DM, so we don't expose this field to user
- // DM may running many lightning instances at same time, so we need to set different tls config name for each lightning
- TLSConfigName string `toml:"-" json:"-"`
+ TLSConfig *tls.Config `toml:"-" json:"-"`
+ AllowFallbackToPlaintext bool `toml:"-" json:"-"`
// When DM/engine uses lightning as a library, it can directly pass in the content
CABytes []byte `toml:"-" json:"-"`
@@ -585,10 +680,9 @@ type Security struct {
KeyBytes []byte `toml:"-" json:"-"`
}
-// RegisterMySQL registers the TLS config with name "cluster" or security.TLSConfigName
-// for use in `sql.Open()`. This method is goroutine-safe.
-func (sec *Security) RegisterMySQL() error {
- if sec == nil {
+// BuildTLSConfig builds the tls config which is used by SQL drier later.
+func (sec *Security) BuildTLSConfig() error {
+ if sec == nil || sec.TLSConfig != nil {
return nil
}
@@ -601,21 +695,10 @@ func (sec *Security) RegisterMySQL() error {
if err != nil {
return errors.Trace(err)
}
- if tlsConfig != nil {
- // error happens only when the key coincides with the built-in names.
- _ = gomysql.RegisterTLSConfig(sec.TLSConfigName, tlsConfig)
- }
+ sec.TLSConfig = tlsConfig
return nil
}
-// DeregisterMySQL deregisters the TLS config with security.TLSConfigName
-func (sec *Security) DeregisterMySQL() {
- if sec == nil || len(sec.CAPath) == 0 {
- return
- }
- gomysql.DeregisterTLSConfig(sec.TLSConfigName)
-}
-
// A duration which can be deserialized from a TOML string.
// Implemented as https://github.com/BurntSushi/toml#using-the-encodingtextunmarshaler-interface
type Duration struct {
@@ -714,13 +797,14 @@ func NewConfig() *Config {
Mydumper: MydumperRuntime{
ReadBlockSize: ReadBlockSize,
CSV: CSVConfig{
- Separator: ",",
- Delimiter: `"`,
- Header: true,
- NotNull: false,
- Null: `\N`,
- BackslashEscape: true,
- TrimLastSep: false,
+ Separator: ",",
+ Delimiter: `"`,
+ Header: true,
+ HeaderSchemaMatch: true,
+ NotNull: false,
+ Null: `\N`,
+ BackslashEscape: true,
+ TrimLastSep: false,
},
StrictFormat: false,
MaxRegionSize: MaxRegionSize,
@@ -808,8 +892,16 @@ func (cfg *Config) LoadFromTOML(data []byte) error {
unusedGlobalKeyStrs[key.String()] = struct{}{}
}
+iterateUnusedKeys:
for _, key := range unusedConfigKeys {
keyStr := key.String()
+ switch keyStr {
+ // these keys are not counted as decoded by toml decoder, but actually they are decoded,
+ // because the corresponding unmarshal logic handles these key's decoding in a custom way
+ case "lightning.max-error.type",
+ "lightning.max-error.conflict":
+ continue iterateUnusedKeys
+ }
if _, found := unusedGlobalKeyStrs[keyStr]; found {
bothUnused = append(bothUnused, keyStr)
} else {
@@ -1136,18 +1228,27 @@ func (cfg *Config) AdjustCheckPoint() {
switch cfg.Checkpoint.Driver {
case CheckpointDriverMySQL:
param := common.MySQLConnectParam{
- Host: cfg.TiDB.Host,
- Port: cfg.TiDB.Port,
- User: cfg.TiDB.User,
- Password: cfg.TiDB.Psw,
- SQLMode: mysql.DefaultSQLMode,
- MaxAllowedPacket: defaultMaxAllowedPacket,
- TLS: cfg.TiDB.TLS,
+ Host: cfg.TiDB.Host,
+ Port: cfg.TiDB.Port,
+ User: cfg.TiDB.User,
+ Password: cfg.TiDB.Psw,
+ SQLMode: mysql.DefaultSQLMode,
+ MaxAllowedPacket: defaultMaxAllowedPacket,
+ TLSConfig: cfg.TiDB.Security.TLSConfig,
+ AllowFallbackToPlaintext: cfg.TiDB.Security.AllowFallbackToPlaintext,
}
cfg.Checkpoint.MySQLParam = ¶m
case CheckpointDriverFile:
cfg.Checkpoint.DSN = "/tmp/" + cfg.Checkpoint.Schema + ".pb"
}
+ } else {
+ // try to remove allowAllFiles
+ mysqlCfg, err := gomysql.ParseDSN(cfg.Checkpoint.DSN)
+ if err != nil {
+ return
+ }
+ mysqlCfg.AllowAllFiles = false
+ cfg.Checkpoint.DSN = mysqlCfg.FormatDSN()
}
}
@@ -1180,22 +1281,22 @@ func (cfg *Config) CheckAndAdjustSecurity() error {
}
switch cfg.TiDB.TLS {
- case "":
- if len(cfg.TiDB.Security.CAPath) > 0 || len(cfg.TiDB.Security.CABytes) > 0 ||
- len(cfg.TiDB.Security.CertPath) > 0 || len(cfg.TiDB.Security.CertBytes) > 0 ||
- len(cfg.TiDB.Security.KeyPath) > 0 || len(cfg.TiDB.Security.KeyBytes) > 0 {
- if cfg.TiDB.Security.TLSConfigName == "" {
- cfg.TiDB.Security.TLSConfigName = uuid.NewString() // adjust this the default value
+ case "skip-verify", "preferred":
+ if cfg.TiDB.Security.TLSConfig == nil {
+ /* #nosec G402 */
+ cfg.TiDB.Security.TLSConfig = &tls.Config{
+ MinVersion: tls.VersionTLS10,
+ InsecureSkipVerify: true,
+ NextProtos: []string{"h2", "http/1.1"}, // specify `h2` to let Go use HTTP/2.
}
- cfg.TiDB.TLS = cfg.TiDB.Security.TLSConfigName
- } else {
- cfg.TiDB.TLS = "false"
+ cfg.TiDB.Security.AllowFallbackToPlaintext = true
}
case "cluster":
if len(cfg.Security.CAPath) == 0 {
return common.ErrInvalidConfig.GenWithStack("cannot set `tidb.tls` to 'cluster' without a [security] section")
}
- case "false", "skip-verify", "preferred":
+ case "", "false":
+ cfg.TiDB.TLS = "false"
return nil
default:
return common.ErrInvalidConfig.GenWithStack("unsupported `tidb.tls` config %s", cfg.TiDB.TLS)
diff --git a/br/pkg/lightning/config/config_test.go b/br/pkg/lightning/config/config_test.go
index e74094a6b9066..98635ba4674e9 100644
--- a/br/pkg/lightning/config/config_test.go
+++ b/br/pkg/lightning/config/config_test.go
@@ -19,6 +19,7 @@ import (
"context"
"flag"
"fmt"
+ "math"
"net"
"net/http"
"net/http/httptest"
@@ -279,31 +280,34 @@ func TestAdjustWillBatchImportRatioInvalid(t *testing.T) {
}
func TestAdjustSecuritySection(t *testing.T) {
- uuidHolder := ""
testCases := []struct {
- input string
- expectedCA string
- expectedTLS string
+ input string
+ expectedCA string
+ hasTLS bool
+ fallback2NoTLS bool
}{
{
- input: ``,
- expectedCA: "",
- expectedTLS: "false",
+ input: ``,
+ expectedCA: "",
+ hasTLS: false,
+ fallback2NoTLS: false,
},
{
input: `
[security]
`,
- expectedCA: "",
- expectedTLS: "false",
+ expectedCA: "",
+ hasTLS: false,
+ fallback2NoTLS: false,
},
{
input: `
[security]
ca-path = "/path/to/ca.pem"
`,
- expectedCA: "/path/to/ca.pem",
- expectedTLS: uuidHolder,
+ expectedCA: "/path/to/ca.pem",
+ hasTLS: false,
+ fallback2NoTLS: false,
},
{
input: `
@@ -311,8 +315,9 @@ func TestAdjustSecuritySection(t *testing.T) {
ca-path = "/path/to/ca.pem"
[tidb.security]
`,
- expectedCA: "",
- expectedTLS: "false",
+ expectedCA: "",
+ hasTLS: false,
+ fallback2NoTLS: false,
},
{
input: `
@@ -321,8 +326,9 @@ func TestAdjustSecuritySection(t *testing.T) {
[tidb.security]
ca-path = "/path/to/ca2.pem"
`,
- expectedCA: "/path/to/ca2.pem",
- expectedTLS: uuidHolder,
+ expectedCA: "/path/to/ca2.pem",
+ hasTLS: false,
+ fallback2NoTLS: false,
},
{
input: `
@@ -330,8 +336,9 @@ func TestAdjustSecuritySection(t *testing.T) {
[tidb.security]
ca-path = "/path/to/ca2.pem"
`,
- expectedCA: "/path/to/ca2.pem",
- expectedTLS: uuidHolder,
+ expectedCA: "/path/to/ca2.pem",
+ hasTLS: false,
+ fallback2NoTLS: false,
},
{
input: `
@@ -340,8 +347,20 @@ func TestAdjustSecuritySection(t *testing.T) {
tls = "skip-verify"
[tidb.security]
`,
- expectedCA: "",
- expectedTLS: "skip-verify",
+ expectedCA: "",
+ hasTLS: true,
+ fallback2NoTLS: true,
+ },
+ {
+ input: `
+ [security]
+ [tidb]
+ tls = "false"
+ [tidb.security]
+ `,
+ expectedCA: "",
+ hasTLS: false,
+ fallback2NoTLS: false,
},
}
@@ -357,19 +376,18 @@ func TestAdjustSecuritySection(t *testing.T) {
err = cfg.Adjust(context.Background())
require.NoError(t, err, comment)
require.Equal(t, tc.expectedCA, cfg.TiDB.Security.CAPath, comment)
- if tc.expectedTLS == uuidHolder {
- require.NotEmpty(t, cfg.TiDB.TLS, comment)
+ if tc.hasTLS {
+ require.NotNil(t, cfg.TiDB.Security.TLSConfig, comment)
} else {
- require.Equal(t, tc.expectedTLS, cfg.TiDB.TLS, comment)
+ require.Nil(t, cfg.TiDB.Security.TLSConfig, comment)
}
+ require.Equal(t, tc.fallback2NoTLS, cfg.TiDB.Security.AllowFallbackToPlaintext, comment)
}
// test different tls config name
cfg := config.NewConfig()
assignMinimalLegalValue(cfg)
cfg.Security.CAPath = "/path/to/ca.pem"
- cfg.Security.TLSConfigName = "tidb-tls"
require.NoError(t, cfg.Adjust(context.Background()))
- require.Equal(t, cfg.TiDB.TLS, cfg.TiDB.Security.TLSConfigName)
}
func TestInvalidCSV(t *testing.T) {
@@ -544,6 +562,126 @@ func TestDurationUnmarshal(t *testing.T) {
require.Regexp(t, "time: unknown unit .?x.? in duration .?13x20s.?", err.Error())
}
+func TestMaxErrorUnmarshal(t *testing.T) {
+ type testCase struct {
+ TOMLStr string
+ ExpectedValues map[string]int64
+ ExpectErrStr string
+ CaseName string
+ }
+ for _, tc := range []*testCase{
+ {
+ TOMLStr: `max-error = 123`,
+ ExpectedValues: map[string]int64{
+ "syntax": 0,
+ "charset": math.MaxInt64,
+ "type": 123,
+ "conflict": math.MaxInt64,
+ },
+ CaseName: "Normal_Int",
+ },
+ {
+ TOMLStr: `max-error = -123`,
+ ExpectedValues: map[string]int64{
+ "syntax": 0,
+ "charset": math.MaxInt64,
+ "type": 0,
+ "conflict": math.MaxInt64,
+ },
+ CaseName: "Abnormal_Negative_Int",
+ },
+ {
+ TOMLStr: `max-error = "abcde"`,
+ ExpectErrStr: "invalid max-error 'abcde', should be an integer or a map of string:int64",
+ CaseName: "Abnormal_String",
+ },
+ {
+ TOMLStr: `[max-error]
+syntax = 1
+charset = 2
+type = 3
+conflict = 4
+`,
+ ExpectedValues: map[string]int64{
+ "syntax": 0,
+ "charset": math.MaxInt64,
+ "type": 3,
+ "conflict": 4,
+ },
+ CaseName: "Normal_Map_All_Set",
+ },
+ {
+ TOMLStr: `[max-error]
+conflict = 1000
+`,
+ ExpectedValues: map[string]int64{
+ "syntax": 0,
+ "charset": math.MaxInt64,
+ "type": 0,
+ "conflict": 1000,
+ },
+ CaseName: "Normal_Map_Partial_Set",
+ },
+ {
+ TOMLStr: `max-error = { conflict = 1000, type = 123 }`,
+ ExpectedValues: map[string]int64{
+ "syntax": 0,
+ "charset": math.MaxInt64,
+ "type": 123,
+ "conflict": 1000,
+ },
+ CaseName: "Normal_OneLineMap_Partial_Set",
+ },
+ {
+ TOMLStr: `[max-error]
+conflict = 1000
+not_exist = 123
+`,
+ ExpectedValues: map[string]int64{
+ "syntax": 0,
+ "charset": math.MaxInt64,
+ "type": 0,
+ "conflict": 1000,
+ },
+ CaseName: "Normal_Map_Partial_Set_Invalid_Key",
+ },
+ {
+ TOMLStr: `[max-error]
+conflict = 1000
+type = -123
+`,
+ ExpectedValues: map[string]int64{
+ "syntax": 0,
+ "charset": math.MaxInt64,
+ "type": 0,
+ "conflict": 1000,
+ },
+ CaseName: "Normal_Map_Partial_Set_Invalid_Value",
+ },
+ {
+ TOMLStr: `[max-error]
+conflict = 1000
+type = abc
+`,
+ ExpectErrStr: `toml: line 3 (last key "max-error.type"): expected value but found "abc" instead`,
+ CaseName: "Normal_Map_Partial_Set_Invalid_ValueType",
+ },
+ } {
+ targetLightningCfg := new(config.Lightning)
+ err := toml.Unmarshal([]byte(tc.TOMLStr), targetLightningCfg)
+ if len(tc.ExpectErrStr) > 0 {
+ require.Errorf(t, err, "test case: %s", tc.CaseName)
+ require.Equalf(t, tc.ExpectErrStr, err.Error(), "test case: %s", tc.CaseName)
+ } else {
+ require.NoErrorf(t, err, "test case: %s", tc.CaseName)
+ require.Equalf(t, tc.ExpectedValues["syntax"], targetLightningCfg.MaxError.Syntax.Load(), "test case: %s", tc.CaseName)
+ require.Equalf(t, tc.ExpectedValues["charset"], targetLightningCfg.MaxError.Charset.Load(), "test case: %s", tc.CaseName)
+ require.Equalf(t, tc.ExpectedValues["type"], targetLightningCfg.MaxError.Type.Load(), "test case: %s", tc.CaseName)
+ require.Equalf(t, tc.ExpectedValues["conflict"], targetLightningCfg.MaxError.Conflict.Load(), "test case: %s", tc.CaseName)
+ }
+ }
+}
+
func TestDurationMarshalJSON(t *testing.T) {
duration := config.Duration{}
err := duration.UnmarshalText([]byte("13m20s"))
@@ -626,7 +764,7 @@ func TestLoadConfig(t *testing.T) {
err = taskCfg.Adjust(context.Background())
require.NoError(t, err)
equivalentDSN := taskCfg.Checkpoint.MySQLParam.ToDriverConfig().FormatDSN()
- expectedDSN := "guest:12345@tcp(172.16.30.11:4001)/?tls=false&maxAllowedPacket=67108864&charset=utf8mb4&sql_mode=%27ONLY_FULL_GROUP_BY%2CSTRICT_TRANS_TABLES%2CNO_ZERO_IN_DATE%2CNO_ZERO_DATE%2CERROR_FOR_DIVISION_BY_ZERO%2CNO_AUTO_CREATE_USER%2CNO_ENGINE_SUBSTITUTION%27"
+ expectedDSN := "guest:12345@tcp(172.16.30.11:4001)/?maxAllowedPacket=67108864&charset=utf8mb4&sql_mode=%27ONLY_FULL_GROUP_BY%2CSTRICT_TRANS_TABLES%2CNO_ZERO_IN_DATE%2CNO_ZERO_DATE%2CERROR_FOR_DIVISION_BY_ZERO%2CNO_AUTO_CREATE_USER%2CNO_ENGINE_SUBSTITUTION%27"
require.Equal(t, expectedDSN, equivalentDSN)
result := taskCfg.String()
@@ -783,6 +921,17 @@ func TestAdjustDiskQuota(t *testing.T) {
require.Equal(t, int64(0), int64(cfg.TikvImporter.DiskQuota))
}
+func TestRemoveAllowAllFiles(t *testing.T) {
+ cfg := config.NewConfig()
+ assignMinimalLegalValue(cfg)
+ ctx := context.Background()
+
+ cfg.Checkpoint.Driver = config.CheckpointDriverMySQL
+ cfg.Checkpoint.DSN = "guest:12345@tcp(172.16.30.11:4001)/?tls=false&allowAllFiles=true&charset=utf8mb4"
+ require.NoError(t, cfg.Adjust(ctx))
+ require.Equal(t, "guest:12345@tcp(172.16.30.11:4001)/?tls=false&charset=utf8mb4", cfg.Checkpoint.DSN)
+}
+
func TestDataCharacterSet(t *testing.T) {
testCases := []struct {
input string
@@ -995,3 +1144,17 @@ func TestCreateSeveralConfigsWithDifferentFilters(t *testing.T) {
))
require.True(t, common.StringSliceEqual(config.GetDefaultFilter(), originalDefaultCfg))
}
+
+func TestCompressionType(t *testing.T) {
+ var ct config.CompressionType
+ require.NoError(t, ct.FromStringValue(""))
+ require.Equal(t, config.CompressionNone, ct)
+ require.NoError(t, ct.FromStringValue("gzip"))
+ require.Equal(t, config.CompressionGzip, ct)
+ require.NoError(t, ct.FromStringValue("gz"))
+ require.Equal(t, config.CompressionGzip, ct)
+ require.EqualError(t, ct.FromStringValue("zstd"), "invalid compression-type 'zstd', please choose valid option between ['gzip']")
+
+ require.Equal(t, "", config.CompressionNone.String())
+ require.Equal(t, "gzip", config.CompressionGzip.String())
+}
diff --git a/br/pkg/lightning/config/const.go b/br/pkg/lightning/config/const.go
index 23a38ac41117d..767b7a3a06c89 100644
--- a/br/pkg/lightning/config/const.go
+++ b/br/pkg/lightning/config/const.go
@@ -15,7 +15,11 @@
package config
import (
+ "time"
+
"github.com/docker/go-units"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/keepalive"
)
const (
@@ -34,3 +38,11 @@ const (
DefaultBatchSize ByteSize = 100 * units.GiB
)
+
+var (
+ DefaultGrpcKeepaliveParams = grpc.WithKeepaliveParams(keepalive.ClientParameters{
+ Time: 1 * time.Minute,
+ Timeout: 2 * time.Minute,
+ PermitWithoutStream: false,
+ })
+)
diff --git a/br/pkg/lightning/errormanager/errormanager.go b/br/pkg/lightning/errormanager/errormanager.go
index 43035716d729c..4085226063d38 100644
--- a/br/pkg/lightning/errormanager/errormanager.go
+++ b/br/pkg/lightning/errormanager/errormanager.go
@@ -40,9 +40,10 @@ const (
CREATE SCHEMA IF NOT EXISTS %s;
`
- syntaxErrorTableName = "syntax_error_v1"
- typeErrorTableName = "type_error_v1"
- conflictErrorTableName = "conflict_error_v1"
+ syntaxErrorTableName = "syntax_error_v1"
+ typeErrorTableName = "type_error_v1"
+ // ConflictErrorTableName is the table name for duplicate detection.
+ ConflictErrorTableName = "conflict_error_v1"
createSyntaxErrorTable = `
CREATE TABLE IF NOT EXISTS %s.` + syntaxErrorTableName + ` (
@@ -69,7 +70,7 @@ const (
`
createConflictErrorTable = `
- CREATE TABLE IF NOT EXISTS %s.` + conflictErrorTableName + ` (
+ CREATE TABLE IF NOT EXISTS %s.` + ConflictErrorTableName + ` (
task_id bigint NOT NULL,
create_time datetime(6) NOT NULL DEFAULT now(6),
table_name varchar(261) NOT NULL,
@@ -91,7 +92,7 @@ const (
`
insertIntoConflictErrorData = `
- INSERT INTO %s.` + conflictErrorTableName + `
+ INSERT INTO %s.` + ConflictErrorTableName + `
(task_id, table_name, index_name, key_data, row_data, raw_key, raw_value, raw_handle, raw_row)
VALUES
`
@@ -99,7 +100,7 @@ const (
sqlValuesConflictErrorData = "(?,?,'PRIMARY',?,?,?,?,raw_key,raw_value)"
insertIntoConflictErrorIndex = `
- INSERT INTO %s.` + conflictErrorTableName + `
+ INSERT INTO %s.` + ConflictErrorTableName + `
(task_id, table_name, index_name, key_data, row_data, raw_key, raw_value, raw_handle, raw_row)
VALUES
`
@@ -108,7 +109,7 @@ const (
selectConflictKeys = `
SELECT _tidb_rowid, raw_handle, raw_row
- FROM %s.` + conflictErrorTableName + `
+ FROM %s.` + ConflictErrorTableName + `
WHERE table_name = ? AND _tidb_rowid >= ? and _tidb_rowid < ?
ORDER BY _tidb_rowid LIMIT ?;
`
@@ -193,7 +194,8 @@ func (em *ErrorManager) RecordTypeError(
if em.remainingError.Type.Dec() < 0 {
threshold := em.configError.Type.Load()
if threshold > 0 {
- encodeErr = errors.Annotatef(encodeErr, "meet errors exceed the max-error.type threshold '%d'",
+ encodeErr = errors.Annotatef(encodeErr,
+ "The number of type errors exceeds the threshold configured by `max-error.type`: '%d'",
em.configError.Type.Load())
}
return encodeErr
@@ -240,17 +242,20 @@ func (em *ErrorManager) RecordDataConflictError(
tableName string,
conflictInfos []DataConflictInfo,
) error {
+ var gerr error
if len(conflictInfos) == 0 {
return nil
}
if em.remainingError.Conflict.Sub(int64(len(conflictInfos))) < 0 {
threshold := em.configError.Conflict.Load()
- return errors.Errorf(" meet errors exceed the max-error.conflict threshold '%d'", threshold)
+ // Still need to record this batch of conflict records, and then return this error at last.
+ // Otherwise, if the max-error.conflict is set a very small value, non of the conflict errors will be recorded
+ gerr = errors.Errorf("The number of conflict errors exceeds the threshold configured by `max-error.conflict`: '%d'", threshold)
}
if em.db == nil {
- return nil
+ return gerr
}
exec := common.SQLWithRetry{
@@ -258,7 +263,7 @@ func (em *ErrorManager) RecordDataConflictError(
Logger: logger,
HideQueryLog: redact.NeedRedact(),
}
- return exec.Transact(ctx, "insert data conflict error record", func(c context.Context, txn *sql.Tx) error {
+ if err := exec.Transact(ctx, "insert data conflict error record", func(c context.Context, txn *sql.Tx) error {
sb := &strings.Builder{}
fmt.Fprintf(sb, insertIntoConflictErrorData, em.schemaEscaped)
var sqlArgs []interface{}
@@ -278,7 +283,10 @@ func (em *ErrorManager) RecordDataConflictError(
}
_, err := txn.ExecContext(c, sb.String(), sqlArgs...)
return err
- })
+ }); err != nil {
+ gerr = err
+ }
+ return gerr
}
func (em *ErrorManager) RecordIndexConflictError(
@@ -289,17 +297,20 @@ func (em *ErrorManager) RecordIndexConflictError(
conflictInfos []DataConflictInfo,
rawHandles, rawRows [][]byte,
) error {
+ var gerr error
if len(conflictInfos) == 0 {
return nil
}
if em.remainingError.Conflict.Sub(int64(len(conflictInfos))) < 0 {
threshold := em.configError.Conflict.Load()
- return errors.Errorf(" meet errors exceed the max-error.conflict threshold %d", threshold)
+ // Still need to record this batch of conflict records, and then return this error at last.
+ // Otherwise, if the max-error.conflict is set a very small value, non of the conflict errors will be recorded
+ gerr = errors.Errorf("The number of conflict errors exceeds the threshold configured by `max-error.conflict`: '%d'", threshold)
}
if em.db == nil {
- return nil
+ return gerr
}
exec := common.SQLWithRetry{
@@ -307,7 +318,7 @@ func (em *ErrorManager) RecordIndexConflictError(
Logger: logger,
HideQueryLog: redact.NeedRedact(),
}
- return exec.Transact(ctx, "insert index conflict error record", func(c context.Context, txn *sql.Tx) error {
+ if err := exec.Transact(ctx, "insert index conflict error record", func(c context.Context, txn *sql.Tx) error {
sb := &strings.Builder{}
fmt.Fprintf(sb, insertIntoConflictErrorIndex, em.schemaEscaped)
var sqlArgs []interface{}
@@ -330,7 +341,10 @@ func (em *ErrorManager) RecordIndexConflictError(
}
_, err := txn.ExecContext(c, sb.String(), sqlArgs...)
return err
- })
+ }); err != nil {
+ gerr = err
+ }
+ return gerr
}
// ResolveAllConflictKeys query all conflicting rows (handle and their
@@ -468,7 +482,7 @@ func (em *ErrorManager) LogErrorDetails() {
em.logger.Warn(fmtErrMsg(errCnt, "data type", ""))
}
if errCnt := em.conflictError(); errCnt > 0 {
- em.logger.Warn(fmtErrMsg(errCnt, "data type", conflictErrorTableName))
+ em.logger.Warn(fmtErrMsg(errCnt, "data type", ConflictErrorTableName))
}
}
@@ -511,7 +525,7 @@ func (em *ErrorManager) Output() string {
}
if errCnt := em.conflictError(); errCnt > 0 {
count++
- t.AppendRow(table.Row{count, "Unique Key Conflict", errCnt, em.fmtTableName(conflictErrorTableName)})
+ t.AppendRow(table.Row{count, "Unique Key Conflict", errCnt, em.fmtTableName(ConflictErrorTableName)})
}
res := "\nImport Data Error Summary: \n"
diff --git a/br/pkg/lightning/lightning.go b/br/pkg/lightning/lightning.go
index 3770f7c8f07a4..6ed4b40335b96 100644
--- a/br/pkg/lightning/lightning.go
+++ b/br/pkg/lightning/lightning.go
@@ -33,6 +33,8 @@ import (
"sync"
"time"
+ "github.com/go-sql-driver/mysql"
+ "github.com/google/uuid"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/import_sstpb"
@@ -53,11 +55,13 @@ import (
"github.com/pingcap/tidb/br/pkg/version/build"
_ "github.com/pingcap/tidb/expression" // get rid of `import cycle`: just init expression.RewriteAstExpr,and called at package `backend.kv`.
_ "github.com/pingcap/tidb/planner/core"
+ "github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/promutil"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/shurcooL/httpgzip"
+ "go.uber.org/atomic"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"golang.org/x/exp/slices"
@@ -77,6 +81,7 @@ type Lightning struct {
promFactory promutil.Factory
promRegistry promutil.Registry
+ metrics *metric.Metrics
cancelLock sync.Mutex
curTask *config.Config
@@ -369,6 +374,36 @@ func (l *Lightning) RunOnceWithOptions(taskCtx context.Context, taskCfg *config.
taskCfg.TaskID = int64(val.(int))
})
+ failpoint.Inject("SetIOTotalBytes", func(_ failpoint.Value) {
+ o.logger.Info("set io total bytes")
+ taskCfg.TiDB.IOTotalBytes = atomic.NewUint64(0)
+ taskCfg.TiDB.UUID = uuid.New().String()
+ go func() {
+ for {
+ time.Sleep(time.Millisecond * 10)
+ log.L().Info("IOTotalBytes", zap.Uint64("IOTotalBytes", taskCfg.TiDB.IOTotalBytes.Load()))
+ }
+ }()
+ })
+ if taskCfg.TiDB.IOTotalBytes != nil {
+ o.logger.Info("found IO total bytes counter")
+ mysql.RegisterDialContext(taskCfg.TiDB.UUID, func(ctx context.Context, addr string) (net.Conn, error) {
+ o.logger.Debug("connection with IO bytes counter")
+ d := &net.Dialer{}
+ conn, err := d.DialContext(ctx, "tcp", addr)
+ if err != nil {
+ return nil, err
+ }
+ tcpConn := conn.(*net.TCPConn)
+ // try https://github.com/go-sql-driver/mysql/blob/bcc459a906419e2890a50fc2c99ea6dd927a88f2/connector.go#L56-L64
+ err = tcpConn.SetKeepAlive(true)
+ if err != nil {
+ o.logger.Warn("set TCP keep alive failed", zap.Error(err))
+ }
+ return util.NewTCPConnWithIOCounter(tcpConn, taskCfg.TiDB.IOTotalBytes), nil
+ })
+ }
+
return l.run(taskCtx, taskCfg, o)
}
@@ -377,6 +412,38 @@ var (
taskCfgRecorderKey = "taskCfgRecorderKey"
)
+func getKeyspaceName(g glue.Glue) (string, error) {
+ db, err := g.GetDB()
+ if err != nil {
+ return "", err
+ }
+ if db == nil {
+ return "", nil
+ }
+
+ rows, err := db.Query("show config where Type = 'tidb' and name = 'keyspace-name'")
+ if err != nil {
+ return "", err
+ }
+ //nolint: errcheck
+ defer rows.Close()
+
+ var (
+ _type string
+ _instance string
+ _name string
+ value string
+ )
+ if rows.Next() {
+ err = rows.Scan(&_type, &_instance, &_name, &value)
+ if err != nil {
+ return "", err
+ }
+ }
+
+ return value, rows.Err()
+}
+
func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, o *options) (err error) {
build.LogInfo(build.Lightning)
o.logger.Info("cfg", zap.Stringer("cfg", taskCfg))
@@ -388,6 +455,7 @@ func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, o *opti
defer func() {
metrics.UnregisterFrom(o.promRegistry)
}()
+ l.metrics = metrics
ctx := metric.NewContext(taskCtx, metrics)
ctx = log.NewContext(ctx, o.logger)
@@ -433,18 +501,21 @@ func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, o *opti
}
})
- if err := taskCfg.TiDB.Security.RegisterMySQL(); err != nil {
+ failpoint.Inject("PrintStatus", func() {
+ defer func() {
+ finished, total := l.Status()
+ o.logger.Warn("PrintStatus Failpoint",
+ zap.Int64("finished", finished),
+ zap.Int64("total", total),
+ zap.Bool("equal", finished == total))
+ }()
+ })
+
+ if err := taskCfg.TiDB.Security.BuildTLSConfig(); err != nil {
return common.ErrInvalidTLSConfig.Wrap(err)
}
- defer func() {
- // deregister TLS config with name "cluster"
- if taskCfg.TiDB.Security == nil {
- return
- }
- taskCfg.TiDB.Security.DeregisterMySQL()
- }()
- // initiation of default glue should be after RegisterMySQL, which is ready to be called after taskCfg.Adjust
+ // initiation of default glue should be after BuildTLSConfig, which is ready to be called after taskCfg.Adjust
// and also put it here could avoid injecting another two SkipRunTask failpoint to caller
g := o.glue
if g == nil {
@@ -502,7 +573,12 @@ func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, o *opti
dbMetas := mdl.GetDatabases()
web.BroadcastInitProgress(dbMetas)
- var procedure *restore.Controller
+ keyspaceName, err := getKeyspaceName(g)
+ if err != nil {
+ o.logger.Error("fail to get keyspace name", zap.Error(err))
+ return errors.Trace(err)
+ }
+ o.logger.Info("acquired keyspace name", zap.String("keyspaceName", keyspaceName))
param := &restore.ControllerParam{
DBMetas: dbMetas,
@@ -512,13 +588,21 @@ func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, o *opti
Glue: g,
CheckpointStorage: o.checkpointStorage,
CheckpointName: o.checkpointName,
+ DupIndicator: o.dupIndicator,
+ KeyspaceName: keyspaceName,
}
+ var procedure *restore.Controller
procedure, err = restore.NewRestoreController(ctx, taskCfg, param)
if err != nil {
o.logger.Error("restore failed", log.ShortError(err))
return errors.Trace(err)
}
+
+ failpoint.Inject("orphanWriterGoRoutine", func() {
+ // don't exit too quickly to expose panic
+ defer time.Sleep(time.Second * 10)
+ })
defer procedure.Close()
err = procedure.Run(ctx)
@@ -544,6 +628,12 @@ func (l *Lightning) Status() (finished int64, total int64) {
return
}
+// Metrics returns the metrics of lightning.
+// it's inited during `run`, so might return nil.
+func (l *Lightning) Metrics() *metric.Metrics {
+ return l.metrics
+}
+
func writeJSONError(w http.ResponseWriter, code int, prefix string, err error) {
type errorResponse struct {
Error string `json:"error"`
diff --git a/br/pkg/lightning/manual/BUILD.bazel b/br/pkg/lightning/manual/BUILD.bazel
index 6d1fc18dd2495..d54902a23c066 100644
--- a/br/pkg/lightning/manual/BUILD.bazel
+++ b/br/pkg/lightning/manual/BUILD.bazel
@@ -10,4 +10,5 @@ go_library(
cgo = True,
importpath = "github.com/pingcap/tidb/br/pkg/lightning/manual",
visibility = ["//visibility:public"],
+ deps = ["@org_uber_go_atomic//:atomic"],
)
diff --git a/br/pkg/lightning/manual/allocator.go b/br/pkg/lightning/manual/allocator.go
index 821eb750c5030..18aa8cc9353c4 100644
--- a/br/pkg/lightning/manual/allocator.go
+++ b/br/pkg/lightning/manual/allocator.go
@@ -14,8 +14,33 @@
package manual
-type Allocator struct{}
+import (
+ "fmt"
-func (Allocator) Alloc(n int) []byte { return New(n) }
+ "go.uber.org/atomic"
+)
-func (Allocator) Free(b []byte) { Free(b) }
+type Allocator struct {
+ RefCnt *atomic.Int64
+}
+
+func (a Allocator) Alloc(n int) []byte {
+ if a.RefCnt != nil {
+ a.RefCnt.Add(1)
+ }
+ return New(n)
+}
+
+func (a Allocator) Free(b []byte) {
+ if a.RefCnt != nil {
+ a.RefCnt.Add(-1)
+ }
+ Free(b)
+}
+
+func (a Allocator) CheckRefCnt() error {
+ if a.RefCnt != nil && a.RefCnt.Load() != 0 {
+ return fmt.Errorf("memory leak detected, refCnt: %d", a.RefCnt.Load())
+ }
+ return nil
+}
diff --git a/br/pkg/lightning/mydump/BUILD.bazel b/br/pkg/lightning/mydump/BUILD.bazel
index dccd93f84e7ce..a4aa1626afc46 100644
--- a/br/pkg/lightning/mydump/BUILD.bazel
+++ b/br/pkg/lightning/mydump/BUILD.bazel
@@ -23,6 +23,7 @@ go_library(
"//br/pkg/lightning/metric",
"//br/pkg/lightning/worker",
"//br/pkg/storage",
+ "//config",
"//parser/mysql",
"//types",
"//util/filter",
@@ -31,6 +32,7 @@ go_library(
"//util/slice",
"//util/table-filter",
"@com_github_pingcap_errors//:errors",
+ "@com_github_spkg_bom//:bom",
"@com_github_xitongsys_parquet_go//parquet",
"@com_github_xitongsys_parquet_go//reader",
"@com_github_xitongsys_parquet_go//source",
diff --git a/br/pkg/lightning/mydump/csv/split_large_file.csv.zst b/br/pkg/lightning/mydump/csv/split_large_file.csv.zst
new file mode 100644
index 0000000000000..9609230bf04a5
Binary files /dev/null and b/br/pkg/lightning/mydump/csv/split_large_file.csv.zst differ
diff --git a/br/pkg/lightning/mydump/csv_parser.go b/br/pkg/lightning/mydump/csv_parser.go
index 96de51bd49c73..d9cd033d70861 100644
--- a/br/pkg/lightning/mydump/csv_parser.go
+++ b/br/pkg/lightning/mydump/csv_parser.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/tidb/br/pkg/lightning/log"
"github.com/pingcap/tidb/br/pkg/lightning/metric"
"github.com/pingcap/tidb/br/pkg/lightning/worker"
+ tidbconfig "github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/mathutil"
)
@@ -33,16 +34,23 @@ var (
errUnterminatedQuotedField = errors.NewNoStackError("syntax error: unterminated quoted field")
errDanglingBackslash = errors.NewNoStackError("syntax error: no character after backslash")
errUnexpectedQuoteField = errors.NewNoStackError("syntax error: cannot have consecutive fields without separator")
+ // LargestEntryLimit is the max size for reading file to buf
+ LargestEntryLimit int
)
+func init() {
+ LargestEntryLimit = tidbconfig.MaxTxnEntrySizeLimit
+}
+
// CSVParser is basically a copy of encoding/csv, but special-cased for MySQL-like input.
type CSVParser struct {
blockParser
cfg *config.CSVConfig
- comma []byte
- quote []byte
- newLine []byte
+ comma []byte
+ quote []byte
+ newLine []byte
+ startingBy []byte
charsetConvertor *CharsetConvertor
// These variables are used with IndexAnyByte to search a byte slice for the
@@ -113,6 +121,12 @@ func NewCSVParser(
}
unquoteStopSet = append(unquoteStopSet, newLineStopSet...)
+ if len(cfg.StartingBy) > 0 {
+ if strings.Contains(cfg.StartingBy, terminator) {
+ return nil, errors.New("starting-by cannot contain (line) terminator")
+ }
+ }
+
escFlavor := backslashEscapeFlavorNone
if cfg.BackslashEscape {
escFlavor = backslashEscapeFlavorMySQL
@@ -131,6 +145,7 @@ func NewCSVParser(
comma: []byte(separator),
quote: []byte(delimiter),
newLine: []byte(terminator),
+ startingBy: []byte(cfg.StartingBy),
escFlavor: escFlavor,
quoteByteSet: makeByteSet(quoteStopSet),
unquoteByteSet: makeByteSet(unquoteStopSet),
@@ -336,6 +351,9 @@ func (parser *CSVParser) readUntil(chars *byteSet) ([]byte, byte, error) {
var buf []byte
for {
buf = append(buf, parser.buf...)
+ if len(buf) > LargestEntryLimit {
+ return buf, 0, errors.New("size of row cannot exceed the max value of txn-entry-size-limit")
+ }
parser.buf = nil
if err := parser.readBlock(); err != nil || len(parser.buf) == 0 {
if err == nil {
@@ -360,11 +378,43 @@ func (parser *CSVParser) readRecord(dst []string) ([]string, error) {
isEmptyLine := true
whitespaceLine := true
+ foundStartingByThisLine := false
prevToken := csvTokenNewLine
var firstToken csvToken
outside:
for {
+ // we should drop
+ // 1. the whole line if it does not contain startingBy
+ // 2. any character before startingBy
+ // since we have checked startingBy does not contain terminator, we can
+ // split at terminator to check the substring contains startingBy. Even
+ // if the terminator is inside a quoted field which means it's not the
+ // end of a line, the substring can still be dropped by rule 2.
+ if len(parser.startingBy) > 0 && !foundStartingByThisLine {
+ oldPos := parser.pos
+ content, _, err := parser.ReadUntilTerminator()
+ if err != nil {
+ if !(errors.Cause(err) == io.EOF) {
+ return nil, err
+ }
+ if len(content) == 0 {
+ return nil, err
+ }
+ // if we reached EOF, we should still check the content contains
+ // startingBy and try to put back and parse it.
+ }
+ idx := bytes.Index(content, parser.startingBy)
+ if idx == -1 {
+ continue
+ }
+ foundStartingByThisLine = true
+ content = content[idx+len(parser.startingBy):]
+ content = append(content, parser.newLine...)
+ parser.buf = append(content, parser.buf...)
+ parser.pos = oldPos + int64(idx+len(parser.startingBy))
+ }
+
content, firstByte, err := parser.readUntil(&parser.unquoteByteSet)
if len(content) > 0 {
@@ -405,6 +455,7 @@ outside:
}
whitespaceLine = false
case csvTokenNewLine:
+ foundStartingByThisLine = false
// new line = end of record (ignore empty lines)
prevToken = firstToken
if isEmptyLine {
@@ -447,9 +498,18 @@ outside:
func (parser *CSVParser) readQuotedField() error {
for {
+ prevPos := parser.pos
content, terminator, err := parser.readUntil(&parser.quoteByteSet)
- err = parser.replaceEOF(err, errUnterminatedQuotedField)
if err != nil {
+ if errors.Cause(err) == io.EOF {
+ // return the position of quote to the caller.
+ // because we return an error here, the parser won't
+ // use the `pos` again, so it's safe to modify it here.
+ parser.pos = prevPos - 1
+ // set buf to parser.buf in order to print err log
+ parser.buf = content
+ err = parser.replaceEOF(err, errUnterminatedQuotedField)
+ }
return err
}
parser.recordBuffer = append(parser.recordBuffer, content...)
@@ -547,6 +607,9 @@ func (parser *CSVParser) ReadColumns() error {
if err != nil {
return errors.Trace(err)
}
+ if !parser.cfg.HeaderSchemaMatch {
+ return nil
+ }
parser.columns = make([]string, 0, len(columns))
for _, colName := range columns {
colName, _, err = parser.unescapeString(colName)
@@ -559,17 +622,21 @@ func (parser *CSVParser) ReadColumns() error {
}
// ReadUntilTerminator seeks the file until the terminator token is found, and
-// returns the file offset beyond the terminator.
-// This function is used in strict-format dividing a CSV file.
-func (parser *CSVParser) ReadUntilTerminator() (int64, error) {
+// returns
+// - the content before terminator
+// - the file offset beyond the terminator
+// - error
+// Note that the terminator string pattern may be the content of a field, which
+// means it's inside quotes. Caller should make sure to handle this case.
+func (parser *CSVParser) ReadUntilTerminator() ([]byte, int64, error) {
for {
- _, firstByte, err := parser.readUntil(&parser.newLineByteSet)
+ content, firstByte, err := parser.readUntil(&parser.newLineByteSet)
if err != nil {
- return 0, err
+ return content, 0, err
}
parser.skipBytes(1)
if ok, err := parser.tryReadNewLine(firstByte); ok || err != nil {
- return parser.pos, err
+ return content, parser.pos, err
}
}
}
diff --git a/br/pkg/lightning/mydump/csv_parser_test.go b/br/pkg/lightning/mydump/csv_parser_test.go
index 2696a6909c96c..8980ce221fe75 100644
--- a/br/pkg/lightning/mydump/csv_parser_test.go
+++ b/br/pkg/lightning/mydump/csv_parser_test.go
@@ -1,6 +1,7 @@
package mydump_test
import (
+ "bytes"
"context"
"encoding/csv"
"fmt"
@@ -54,6 +55,35 @@ func runTestCasesCSV(t *testing.T, cfg *config.MydumperRuntime, blockBufSize int
}
}
+func runTestCasesCSVIgnoreNLines(t *testing.T, cfg *config.MydumperRuntime, blockBufSize int64, cases []testCase, ignoreNLines int) {
+ for _, tc := range cases {
+ charsetConvertor, err := mydump.NewCharsetConvertor(cfg.DataCharacterSet, cfg.DataInvalidCharReplace)
+ assert.NoError(t, err)
+ parser, err := mydump.NewCSVParser(context.Background(), &cfg.CSV, mydump.NewStringReader(tc.input), blockBufSize, ioWorkers, false, charsetConvertor)
+ assert.NoError(t, err)
+
+ for ignoreNLines > 0 {
+ // IGNORE N LINES will directly find (line) terminator without checking it's inside quotes
+ _, _, err = parser.ReadUntilTerminator()
+ if errors.Cause(err) == io.EOF {
+ assert.Len(t, tc.expected, 0, "input = %q", tc.input)
+ return
+ }
+ assert.NoError(t, err)
+ ignoreNLines--
+ }
+
+ for i, row := range tc.expected {
+ comment := fmt.Sprintf("input = %q, row = %d", tc.input, i+1)
+ e := parser.ReadRow()
+ assert.NoErrorf(t, e, "input = %q, row = %d, error = %s", tc.input, i+1, errors.ErrorStack(e))
+ assert.Equal(t, int64(i)+1, parser.LastRow().RowID, comment)
+ assert.Equal(t, row, parser.LastRow().Row, comment)
+ }
+ assert.ErrorIsf(t, errors.Cause(parser.ReadRow()), io.EOF, "input = %q", tc.input)
+ }
+}
+
func runFailingTestCasesCSV(t *testing.T, cfg *config.MydumperRuntime, blockBufSize int64, cases []string) {
for _, tc := range cases {
charsetConvertor, err := mydump.NewCharsetConvertor(cfg.DataCharacterSet, cfg.DataInvalidCharReplace)
@@ -451,12 +481,13 @@ func TestSyntaxErrorCSV(t *testing.T) {
func TestTSV(t *testing.T) {
cfg := config.CSVConfig{
- Separator: "\t",
- Delimiter: "",
- BackslashEscape: false,
- NotNull: false,
- Null: "",
- Header: true,
+ Separator: "\t",
+ Delimiter: "",
+ BackslashEscape: false,
+ NotNull: false,
+ Null: "",
+ Header: true,
+ HeaderSchemaMatch: true,
}
parser, err := mydump.NewCSVParser(context.Background(), &cfg, mydump.NewStringReader(`a b c d e f
@@ -547,6 +578,7 @@ func TestCsvWithWhiteSpaceLine(t *testing.T) {
require.Nil(t, parser.Close())
cfg.Header = true
+ cfg.HeaderSchemaMatch = true
data = " \r\na,b,c\r\n0,,abc\r\n"
parser, err = mydump.NewCSVParser(context.Background(), &cfg, mydump.NewStringReader(data), int64(config.ReadBlockSize), ioWorkers, true, nil)
require.NoError(t, err)
@@ -579,6 +611,7 @@ func TestEmpty(t *testing.T) {
// Try again with headers.
cfg.Header = true
+ cfg.HeaderSchemaMatch = true
parser, err = mydump.NewCSVParser(context.Background(), &cfg, mydump.NewStringReader(""), int64(config.ReadBlockSize), ioWorkers, true, nil)
require.NoError(t, err)
@@ -680,6 +713,29 @@ func TestConsecutiveFields(t *testing.T) {
})
}
+func TestTooLargeRow(t *testing.T) {
+ cfg := config.MydumperRuntime{
+ CSV: config.CSVConfig{
+ Separator: ",",
+ Delimiter: `"`,
+ },
+ }
+ var testCase bytes.Buffer
+ testCase.WriteString("a,b,c,d")
+ // WARN: will take up 10KB memory here.
+ mydump.LargestEntryLimit = 10 * 1024
+ for i := 0; i < mydump.LargestEntryLimit; i++ {
+ testCase.WriteByte('d')
+ }
+ charsetConvertor, err := mydump.NewCharsetConvertor(cfg.DataCharacterSet, cfg.DataInvalidCharReplace)
+ require.NoError(t, err)
+ parser, err := mydump.NewCSVParser(context.Background(), &cfg.CSV, mydump.NewStringReader(testCase.String()), int64(config.ReadBlockSize), ioWorkers, false, charsetConvertor)
+ require.NoError(t, err)
+ e := parser.ReadRow()
+ require.Error(t, e)
+ require.Contains(t, e.Error(), "size of row cannot exceed the max value of txn-entry-size-limit")
+}
+
func TestSpecialChars(t *testing.T) {
cfg := config.MydumperRuntime{
CSV: config.CSVConfig{Separator: ",", Delimiter: `"`},
@@ -911,6 +967,211 @@ func TestTerminator(t *testing.T) {
runTestCasesCSV(t, &cfg, 1, testCases)
}
+func TestStartingBy(t *testing.T) {
+ cfg := config.MydumperRuntime{
+ CSV: config.CSVConfig{
+ Separator: ",",
+ Delimiter: `"`,
+ Terminator: "\n",
+ StartingBy: "xxx",
+ },
+ }
+ testCases := []testCase{
+ {
+ input: `xxx"abc",1
+something xxx"def",2
+"ghi",3`,
+ expected: [][]types.Datum{
+ {types.NewStringDatum("abc"), types.NewStringDatum("1")},
+ {types.NewStringDatum("def"), types.NewStringDatum("2")},
+ },
+ },
+ }
+ runTestCasesCSV(t, &cfg, 1, testCases)
+
+ testCases = []testCase{
+ {
+ input: `xxxabc,1
+something xxxdef,2
+ghi,3
+"bad syntax"aaa`,
+ expected: [][]types.Datum{
+ {types.NewStringDatum("abc"), types.NewStringDatum("1")},
+ {types.NewStringDatum("def"), types.NewStringDatum("2")},
+ },
+ },
+ }
+ runTestCasesCSV(t, &cfg, 1, testCases)
+
+ // test that special characters appears before StartingBy, and StartingBy only takes effect after once
+
+ testCases = []testCase{
+ {
+ input: `xxx"abc",1
+something xxxdef,2
+"ghi",3
+"yyy"xxx"yyy",4
+"yyy",5,xxxyyy,5
+qwe,zzzxxxyyy,6
+"yyyxxx"yyyxxx",7
+yyy",5,xxxxxx,8
+`,
+ expected: [][]types.Datum{
+ {types.NewStringDatum("abc"), types.NewStringDatum("1")},
+ {types.NewStringDatum("def"), types.NewStringDatum("2")},
+ {types.NewStringDatum("yyy"), types.NewStringDatum("4")},
+ {types.NewStringDatum("yyy"), types.NewStringDatum("5")},
+ {types.NewStringDatum("yyy"), types.NewStringDatum("6")},
+ {types.NewStringDatum("yyyxxx"), types.NewStringDatum("7")},
+ {types.NewStringDatum("xxx"), types.NewStringDatum("8")},
+ },
+ },
+ }
+ runTestCasesCSV(t, &cfg, 1, testCases)
+
+ // test StartingBy contains special characters
+
+ cfg = config.MydumperRuntime{
+ CSV: config.CSVConfig{
+ Separator: ",",
+ Delimiter: `"`,
+ Terminator: "\n",
+ StartingBy: "x,xx",
+ },
+ }
+ testCases = []testCase{
+ {
+ input: `x,xx"abc",1
+something x,xxdef,2
+"ghi",3
+"yyy"xxx"yyy",4
+"yyy",5,xxxyyy,5
+qwe,zzzxxxyyy,6
+"yyyxxx"yyyxxx",7
+yyy",5,xx,xxxx,8`,
+ expected: [][]types.Datum{
+ {types.NewStringDatum("abc"), types.NewStringDatum("1")},
+ {types.NewStringDatum("def"), types.NewStringDatum("2")},
+ {types.NewStringDatum("xx"), types.NewStringDatum("8")},
+ },
+ },
+ }
+ runTestCasesCSV(t, &cfg, 1, testCases)
+
+ cfg = config.MydumperRuntime{
+ CSV: config.CSVConfig{
+ Separator: ",",
+ Delimiter: `"`,
+ Terminator: "\n",
+ StartingBy: `x"xx`,
+ },
+ }
+ testCases = []testCase{
+ {
+ input: `x"xx"abc",1
+something x"xxdef,2
+"ghi",3
+"yyy"xxx"yyy",4
+"yyy",5,xxxyyy,5
+qwe,zzzxxxyyy,6
+"yyyxxx"yyyxxx",7
+yyy",5,xx"xxxx,8
+`,
+ expected: [][]types.Datum{
+ {types.NewStringDatum("abc"), types.NewStringDatum("1")},
+ {types.NewStringDatum("def"), types.NewStringDatum("2")},
+ {types.NewStringDatum("xx"), types.NewStringDatum("8")},
+ },
+ },
+ }
+ runTestCasesCSV(t, &cfg, 1, testCases)
+
+ cfg = config.MydumperRuntime{
+ CSV: config.CSVConfig{
+ Separator: ",",
+ Delimiter: `"`,
+ Terminator: "\n",
+ StartingBy: "x\nxx",
+ },
+ }
+ _, err := mydump.NewCSVParser(context.Background(), &cfg.CSV, nil, 1, ioWorkers, false, nil)
+ require.ErrorContains(t, err, "starting-by cannot contain (line) terminator")
+}
+
+func TestCallerCanIgnoreNLines(t *testing.T) {
+ cfg := config.MydumperRuntime{
+ CSV: config.CSVConfig{
+ Separator: ",",
+ Delimiter: `"`,
+ Terminator: "\n",
+ },
+ }
+ testCases := []testCase{
+ {
+ input: `1,1
+2,2
+3,3`,
+ expected: [][]types.Datum{
+ {types.NewStringDatum("3"), types.NewStringDatum("3")},
+ },
+ },
+ }
+ runTestCasesCSVIgnoreNLines(t, &cfg, 1, testCases, 2)
+
+ testCases = []testCase{
+ {
+ input: `"bad syntax"1
+"b",2
+"c",3`,
+ expected: [][]types.Datum{
+ {types.NewStringDatum("c"), types.NewStringDatum("3")},
+ },
+ },
+ }
+ runTestCasesCSVIgnoreNLines(t, &cfg, 1, testCases, 2)
+
+ cfg = config.MydumperRuntime{
+ CSV: config.CSVConfig{
+ Separator: ",",
+ Delimiter: `"`,
+ Terminator: "\n",
+ },
+ }
+ testCases = []testCase{
+ {
+ input: `1,1
+2,2
+3,3`,
+ expected: [][]types.Datum{},
+ },
+ }
+ runTestCasesCSVIgnoreNLines(t, &cfg, 1, testCases, 100)
+
+ // test IGNORE N LINES will directly find (line) terminator without checking it's inside quotes
+
+ cfg = config.MydumperRuntime{
+ CSV: config.CSVConfig{
+ Separator: ",",
+ Delimiter: `"`,
+ Terminator: "\n",
+ },
+ }
+ testCases = []testCase{
+ {
+ input: `"a
+",1
+"b
+",2
+"c",3`,
+ expected: [][]types.Datum{
+ {types.NewStringDatum("b\n"), types.NewStringDatum("2")},
+ {types.NewStringDatum("c"), types.NewStringDatum("3")},
+ },
+ },
+ }
+ runTestCasesCSVIgnoreNLines(t, &cfg, 1, testCases, 2)
+}
+
func TestCharsetConversion(t *testing.T) {
cfg := config.MydumperRuntime{
CSV: config.CSVConfig{
@@ -1034,3 +1295,72 @@ func BenchmarkReadRowUsingEncodingCSV(b *testing.B) {
}
require.Equal(b, b.N, rowsCount)
}
+
+func TestHeaderSchemaMatch(t *testing.T) {
+ cfg := config.MydumperRuntime{
+ CSV: config.CSVConfig{
+ Separator: ",",
+ Delimiter: `"`,
+ },
+ }
+
+ inputData := `id,val1,val2,val3
+1,111,aaa,1.0
+2,222,bbb,2.0
+3,333,ccc,3.0
+4,444,ddd,4.0`
+
+ parsedDataPart := [][]types.Datum{
+ {types.NewStringDatum("1"), types.NewStringDatum("111"), types.NewStringDatum("aaa"), types.NewStringDatum("1.0")},
+ {types.NewStringDatum("2"), types.NewStringDatum("222"), types.NewStringDatum("bbb"), types.NewStringDatum("2.0")},
+ {types.NewStringDatum("3"), types.NewStringDatum("333"), types.NewStringDatum("ccc"), types.NewStringDatum("3.0")},
+ {types.NewStringDatum("4"), types.NewStringDatum("444"), types.NewStringDatum("ddd"), types.NewStringDatum("4.0")},
+ }
+
+ type testCase struct {
+ Header bool
+ HeaderSchemaMatch bool
+ ExpectedData [][]types.Datum
+ ExpectedColumns []string
+ }
+
+ for _, tc := range []testCase{
+ {
+ Header: true,
+ HeaderSchemaMatch: true,
+ ExpectedData: parsedDataPart,
+ ExpectedColumns: []string{"id", "val1", "val2", "val3"},
+ },
+ {
+ Header: true,
+ HeaderSchemaMatch: false,
+ ExpectedData: parsedDataPart,
+ ExpectedColumns: nil,
+ },
+ {
+ Header: false,
+ HeaderSchemaMatch: true,
+ ExpectedData: append([][]types.Datum{
+ {types.NewStringDatum("id"), types.NewStringDatum("val1"), types.NewStringDatum("val2"), types.NewStringDatum("val3")},
+ }, parsedDataPart...),
+ ExpectedColumns: nil,
+ },
+ } {
+ comment := fmt.Sprintf("header = %v, header-schema-match = %v", tc.Header, tc.HeaderSchemaMatch)
+ cfg.CSV.Header = tc.Header
+ cfg.CSV.HeaderSchemaMatch = tc.HeaderSchemaMatch
+ charsetConvertor, err := mydump.NewCharsetConvertor(cfg.DataCharacterSet, cfg.DataInvalidCharReplace)
+ assert.NoError(t, err)
+ parser, err := mydump.NewCSVParser(context.Background(), &cfg.CSV, mydump.NewStringReader(inputData), int64(config.ReadBlockSize), ioWorkers, tc.Header, charsetConvertor)
+ assert.NoError(t, err)
+ for i, row := range tc.ExpectedData {
+ comment := fmt.Sprintf("row = %d, header = %v, header-schema-match = %v", i+1, tc.Header, tc.HeaderSchemaMatch)
+ e := parser.ReadRow()
+ assert.NoErrorf(t, e, "row = %d, error = %s", i+1, errors.ErrorStack(e))
+ assert.Equal(t, int64(i)+1, parser.LastRow().RowID, comment)
+ assert.Equal(t, row, parser.LastRow().Row, comment)
+ }
+ assert.ErrorIsf(t, errors.Cause(parser.ReadRow()), io.EOF, comment)
+ assert.Equal(t, tc.ExpectedColumns, parser.Columns(), comment)
+ }
+}
diff --git a/br/pkg/lightning/mydump/loader.go b/br/pkg/lightning/mydump/loader.go
index 98661d3a46f8c..d55bce6f94fc5 100644
--- a/br/pkg/lightning/mydump/loader.go
+++ b/br/pkg/lightning/mydump/loader.go
@@ -16,6 +16,7 @@ package mydump
import (
"context"
+ "io"
"path/filepath"
"sort"
"strings"
@@ -30,6 +31,9 @@ import (
"go.uber.org/zap"
)
+// sampleCompressedFileSize represents how many bytes need to be sampled for compressed files
+const sampleCompressedFileSize = 4 * 1024
+
// MDDatabaseMeta contains some parsed metadata for a database in the source by MyDumper Loader.
type MDDatabaseMeta struct {
Name string
@@ -82,7 +86,9 @@ type SourceFileMeta struct {
Compression Compression
SortKey string
FileSize int64
- ExtendData ExtendColumnData
+ // WARNING: variables below are not persistent
+ ExtendData ExtendColumnData
+ RealSize int64
}
// NewMDTableMeta creates an Mydumper table meta with specified character set.
@@ -386,7 +392,7 @@ func (s *mdLoaderSetup) setup(ctx context.Context) error {
// set a dummy `FileInfo` here without file meta because we needn't restore the table schema
tableMeta, _, _ := s.insertTable(FileInfo{TableName: fileInfo.TableName})
tableMeta.DataFiles = append(tableMeta.DataFiles, fileInfo)
- tableMeta.TotalSize += fileInfo.FileMeta.FileSize
+ tableMeta.TotalSize += fileInfo.FileMeta.RealSize
}
for _, dbMeta := range s.loader.dbs {
@@ -453,7 +459,7 @@ func (s *mdLoaderSetup) constructFileInfo(ctx context.Context, path string, size
info := FileInfo{
TableName: filter.Table{Schema: res.Schema, Name: res.Name},
- FileMeta: SourceFileMeta{Path: path, Type: res.Type, Compression: res.Compression, SortKey: res.Key, FileSize: size},
+ FileMeta: SourceFileMeta{Path: path, Type: res.Type, Compression: res.Compression, SortKey: res.Key, FileSize: size, RealSize: size},
}
if s.loader.shouldSkip(&info.TableName) {
@@ -470,6 +476,15 @@ func (s *mdLoaderSetup) constructFileInfo(ctx context.Context, path string, size
case SourceTypeViewSchema:
s.viewSchemas = append(s.viewSchemas, info)
case SourceTypeSQL, SourceTypeCSV, SourceTypeParquet:
+ if info.FileMeta.Compression != CompressionNone {
+ compressRatio, err2 := SampleFileCompressRatio(ctx, info.FileMeta, s.loader.GetStore())
+ if err2 != nil {
+ logger.Error("[loader] fail to calculate data file compress ratio",
+ zap.String("schema", res.Schema), zap.String("table", res.Name), zap.Stringer("type", res.Type))
+ } else {
+ info.FileMeta.RealSize = int64(compressRatio * float64(info.FileMeta.FileSize))
+ }
+ }
s.tableDatas = append(s.tableDatas, info)
}
@@ -648,3 +663,81 @@ func (l *MDLoader) GetDatabases() []*MDDatabaseMeta {
func (l *MDLoader) GetStore() storage.ExternalStorage {
return l.store
}
+
+func calculateFileBytes(ctx context.Context,
+ dataFile string,
+ compressType storage.CompressType,
+ store storage.ExternalStorage,
+ offset int64) (tot int, pos int64, err error) {
+ bytes := make([]byte, sampleCompressedFileSize)
+ reader, err := store.Open(ctx, dataFile)
+ if err != nil {
+ return 0, 0, errors.Trace(err)
+ }
+ defer reader.Close()
+
+ compressReader, err := storage.NewLimitedInterceptReader(reader, compressType, offset)
+ if err != nil {
+ return 0, 0, errors.Trace(err)
+ }
+
+ readBytes := func() error {
+ n, err2 := compressReader.Read(bytes)
+ if err2 != nil && errors.Cause(err2) != io.EOF && errors.Cause(err) != io.ErrUnexpectedEOF {
+ return err2
+ }
+ tot += n
+ return err2
+ }
+
+ if offset == 0 {
+ err = readBytes()
+ if err != nil && errors.Cause(err) != io.EOF && errors.Cause(err) != io.ErrUnexpectedEOF {
+ return 0, 0, err
+ }
+ pos, err = compressReader.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return 0, 0, errors.Trace(err)
+ }
+ return tot, pos, nil
+ }
+
+ for {
+ err = readBytes()
+ if err != nil {
+ break
+ }
+ }
+ if err != nil && errors.Cause(err) != io.EOF && errors.Cause(err) != io.ErrUnexpectedEOF {
+ return 0, 0, errors.Trace(err)
+ }
+ return tot, offset, nil
+}
+
+// SampleFileCompressRatio samples the compress ratio of the compressed file.
+func SampleFileCompressRatio(ctx context.Context, fileMeta SourceFileMeta, store storage.ExternalStorage) (float64, error) {
+ if fileMeta.Compression == CompressionNone {
+ return 1, nil
+ }
+ compressType, err := ToStorageCompressType(fileMeta.Compression)
+ if err != nil {
+ return 0, err
+ }
+ // We use the following method to sample the compress ratio of the first few bytes of the file.
+ // 1. read first time aiming to find a valid compressed file offset. If we continue read now, the compress reader will
+ // request more data from file reader buffer them in its memory. We can't compute an accurate compress ratio.
+ // 2. we use a second reading and limit the file reader only read n bytes(n is the valid position we find in the first reading).
+ // Then we read all the data out from the compress reader. The data length m we read out is the uncompressed data length.
+ // Use m/n to compute the compress ratio.
+ // read first time, aims to find a valid end pos in compressed file
+ _, pos, err := calculateFileBytes(ctx, fileMeta.Path, compressType, store, 0)
+ if err != nil {
+ return 0, err
+ }
+ // read second time, original reader ends at first time's valid pos, compute sample data compress ratio
+ tot, pos, err := calculateFileBytes(ctx, fileMeta.Path, compressType, store, pos)
+ if err != nil {
+ return 0, err
+ }
+ return float64(tot) / float64(pos), nil
+}
diff --git a/br/pkg/lightning/mydump/loader_test.go b/br/pkg/lightning/mydump/loader_test.go
index da910c70bedc0..58236d7b626f5 100644
--- a/br/pkg/lightning/mydump/loader_test.go
+++ b/br/pkg/lightning/mydump/loader_test.go
@@ -15,6 +15,8 @@
package mydump_test
import (
+ "bytes"
+ "compress/gzip"
"context"
"fmt"
"os"
@@ -1053,3 +1055,34 @@ func TestExternalDataRoutes(t *testing.T) {
require.Equal(t, expectedExtendVals[i], fileInfo.FileMeta.ExtendData.Values)
}
}
+
+func TestSampleFileCompressRatio(t *testing.T) {
+ s := newTestMydumpLoaderSuite(t)
+ store, err := storage.NewLocalStorage(s.sourceDir)
+ require.NoError(t, err)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ byteArray := make([]byte, 0, 4096)
+ bf := bytes.NewBuffer(byteArray)
+ compressWriter := gzip.NewWriter(bf)
+ csvData := []byte("aaaa\n")
+ for i := 0; i < 1000; i++ {
+ _, err = compressWriter.Write(csvData)
+ require.NoError(t, err)
+ }
+ err = compressWriter.Flush()
+ require.NoError(t, err)
+
+ fileName := "test_1.t1.csv.gz"
+ err = store.WriteFile(ctx, fileName, bf.Bytes())
+ require.NoError(t, err)
+
+ ratio, err := md.SampleFileCompressRatio(ctx, md.SourceFileMeta{
+ Path: fileName,
+ Compression: md.CompressionGZ,
+ }, store)
+ require.NoError(t, err)
+ require.InDelta(t, ratio, 5000.0/float64(bf.Len()), 1e-5)
+}
diff --git a/br/pkg/lightning/mydump/parquet_parser.go b/br/pkg/lightning/mydump/parquet_parser.go
index e7ac2baa6d80f..a1b612903c5e8 100644
--- a/br/pkg/lightning/mydump/parquet_parser.go
+++ b/br/pkg/lightning/mydump/parquet_parser.go
@@ -351,6 +351,12 @@ func (pp *ParquetParser) SetPos(pos int64, rowID int64) error {
return nil
}
+// RealPos implements the Parser interface.
+// For parquet it's equal to Pos().
+func (pp *ParquetParser) RealPos() (int64, error) {
+ return pp.curStart + int64(pp.curIndex), nil
+}
+
// Close closes the parquet file of the parser.
// It implements the Parser interface.
func (pp *ParquetParser) Close() error {
diff --git a/br/pkg/lightning/mydump/parser.go b/br/pkg/lightning/mydump/parser.go
index 73f84424bf5e3..0ac82ce189d71 100644
--- a/br/pkg/lightning/mydump/parser.go
+++ b/br/pkg/lightning/mydump/parser.go
@@ -32,6 +32,7 @@ import (
"github.com/pingcap/tidb/br/pkg/lightning/worker"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/types"
+ "github.com/spkg/bom"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
@@ -94,6 +95,7 @@ type ChunkParser struct {
type Chunk struct {
Offset int64
EndOffset int64
+ RealOffset int64
PrevRowIDMax int64
RowIDMax int64
Columns []string
@@ -126,6 +128,7 @@ const (
type Parser interface {
Pos() (pos int64, rowID int64)
SetPos(pos int64, rowID int64) error
+ RealPos() (int64, error)
Close() error
ReadRow() error
LastRow() Row
@@ -175,6 +178,11 @@ func (parser *blockParser) SetPos(pos int64, rowID int64) error {
return nil
}
+// RealPos gets the read position of current reader.
+func (parser *blockParser) RealPos() (int64, error) {
+ return parser.reader.Seek(0, io.SeekCurrent)
+}
+
// Pos returns the current file offset.
// Attention: for compressed sql/csv files, pos is the position in uncompressed files
func (parser *blockParser) Pos() (pos int64, lastRowID int64) {
@@ -278,7 +286,13 @@ func (parser *blockParser) readBlock() error {
parser.remainBuf.Write(parser.buf)
parser.appendBuf.Reset()
parser.appendBuf.Write(parser.remainBuf.Bytes())
- parser.appendBuf.Write(parser.blockBuf[:n])
+ blockData := parser.blockBuf[:n]
+ if parser.pos == 0 {
+ bomCleanedData := bom.Clean(blockData)
+ parser.pos += int64(n - len(bomCleanedData))
+ blockData = bomCleanedData
+ }
+ parser.appendBuf.Write(blockData)
parser.buf = parser.appendBuf.Bytes()
if parser.metrics != nil {
parser.metrics.ChunkParserReadBlockSecondsHistogram.Observe(time.Since(startTime).Seconds())
diff --git a/br/pkg/lightning/mydump/reader.go b/br/pkg/lightning/mydump/reader.go
index 2988c3675dfa9..3735e97cb48ee 100644
--- a/br/pkg/lightning/mydump/reader.go
+++ b/br/pkg/lightning/mydump/reader.go
@@ -26,6 +26,7 @@ import (
"github.com/pingcap/tidb/br/pkg/lightning/log"
"github.com/pingcap/tidb/br/pkg/lightning/worker"
"github.com/pingcap/tidb/br/pkg/storage"
+ "github.com/spkg/bom"
"go.uber.org/zap"
"golang.org/x/text/encoding/simplifiedchinese"
)
@@ -70,13 +71,20 @@ func decodeCharacterSet(data []byte, characterSet string) ([]byte, error) {
// ExportStatement exports the SQL statement in the schema file.
func ExportStatement(ctx context.Context, store storage.ExternalStorage, sqlFile FileInfo, characterSet string) ([]byte, error) {
+ if sqlFile.FileMeta.Compression != CompressionNone {
+ compressType, err := ToStorageCompressType(sqlFile.FileMeta.Compression)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ store = storage.WithCompression(store, compressType)
+ }
fd, err := store.Open(ctx, sqlFile.FileMeta.Path)
if err != nil {
return nil, errors.Trace(err)
}
defer fd.Close()
- br := bufio.NewReader(fd)
+ br := bufio.NewReader(bom.NewReader(fd))
data := make([]byte, 0, sqlFile.FileMeta.FileSize+1)
buffer := make([]byte, 0, sqlFile.FileMeta.FileSize+1)
diff --git a/br/pkg/lightning/mydump/reader_test.go b/br/pkg/lightning/mydump/reader_test.go
index e7506ea869782..1f67f2c31c43a 100644
--- a/br/pkg/lightning/mydump/reader_test.go
+++ b/br/pkg/lightning/mydump/reader_test.go
@@ -15,6 +15,7 @@
package mydump_test
import (
+ "compress/gzip"
"context"
"errors"
"os"
@@ -173,3 +174,28 @@ func TestExportStatementHandleNonEOFError(t *testing.T) {
_, err := ExportStatement(ctx, mockStorage, f, "auto")
require.Contains(t, err.Error(), "read error")
}
+
+func TestExportStatementCompressed(t *testing.T) {
+ dir := t.TempDir()
+ file, err := os.Create(filepath.Join(dir, "tidb_lightning_test_reader"))
+ require.NoError(t, err)
+ defer os.Remove(file.Name())
+
+ store, err := storage.NewLocalStorage(dir)
+ require.NoError(t, err)
+
+ gzipFile := gzip.NewWriter(file)
+ _, err = gzipFile.Write([]byte("CREATE DATABASE whatever;"))
+ require.NoError(t, err)
+ err = gzipFile.Close()
+ require.NoError(t, err)
+ stat, err := file.Stat()
+ require.NoError(t, err)
+ err = file.Close()
+ require.NoError(t, err)
+
+ f := FileInfo{FileMeta: SourceFileMeta{Path: stat.Name(), FileSize: stat.Size(), Compression: CompressionGZ}}
+ data, err := ExportStatement(context.TODO(), store, f, "auto")
+ require.NoError(t, err)
+ require.Equal(t, []byte("CREATE DATABASE whatever;"), data)
+}
diff --git a/br/pkg/lightning/mydump/region.go b/br/pkg/lightning/mydump/region.go
index 8562acc2867b3..d57a2c4742c73 100644
--- a/br/pkg/lightning/mydump/region.go
+++ b/br/pkg/lightning/mydump/region.go
@@ -34,6 +34,11 @@ const (
tableRegionSizeWarningThreshold int64 = 1024 * 1024 * 1024
// the increment ratio of large CSV file size threshold by `region-split-size`
largeCSVLowerThresholdRation = 10
+ // TableFileSizeINF for compressed size, for lightning 10TB is a relatively big value and will strongly affect efficiency
+ // It's used to make sure compressed files can be read until EOF. Because we can't get the exact decompressed size of the compressed files.
+ TableFileSizeINF = 10 * 1024 * tableRegionSizeWarningThreshold
+ // CompressSizeFactor is used to adjust compressed data size
+ CompressSizeFactor = 5
)
// TableRegion contains information for a table region during import.
@@ -292,25 +297,41 @@ func MakeSourceFileRegion(
return regions, subFileSizes, err
}
+ fileSize := fi.FileMeta.FileSize
+ rowIDMax := fileSize / divisor
+ // for compressed files, suggest the compress ratio is 1% to calculate the rowIDMax.
+ // set fileSize to INF to make sure compressed files can be read until EOF. Because we can't get the exact size of the compressed files.
+ if fi.FileMeta.Compression != CompressionNone {
+ // RealSize the estimated file size. There are some cases that the first few bytes of this compressed file
+ // has smaller compress ratio than the whole compressed file. So we still need to multiply this factor to
+ // make sure the rowIDMax computation is correct.
+ rowIDMax = fi.FileMeta.RealSize * CompressSizeFactor / divisor
+ fileSize = TableFileSizeINF
+ }
tableRegion := &TableRegion{
DB: meta.DB,
Table: meta.Name,
FileMeta: fi.FileMeta,
Chunk: Chunk{
Offset: 0,
- EndOffset: fi.FileMeta.FileSize,
+ EndOffset: fileSize,
+ RealOffset: 0,
PrevRowIDMax: 0,
- RowIDMax: fi.FileMeta.FileSize / divisor,
+ RowIDMax: rowIDMax,
},
}
- if tableRegion.Size() > tableRegionSizeWarningThreshold {
+ regionSize := tableRegion.Size()
+ if fi.FileMeta.Compression != CompressionNone {
+ regionSize = fi.FileMeta.RealSize
+ }
+ if regionSize > tableRegionSizeWarningThreshold {
log.FromContext(ctx).Warn(
"file is too big to be processed efficiently; we suggest splitting it at 256 MB each",
zap.String("file", fi.FileMeta.Path),
- zap.Int64("size", dataFileSize))
+ zap.Int64("size", regionSize))
}
- return []*TableRegion{tableRegion}, []float64{float64(fi.FileMeta.FileSize)}, nil
+ return []*TableRegion{tableRegion}, []float64{float64(fi.FileMeta.RealSize)}, nil
}
// because parquet files can't seek efficiently, there is no benefit in split.
@@ -383,7 +404,9 @@ func SplitLargeFile(
if err = parser.ReadColumns(); err != nil {
return 0, nil, nil, err
}
- columns = parser.Columns()
+ if cfg.Mydumper.CSV.HeaderSchemaMatch {
+ columns = parser.Columns()
+ }
startOffset, _ = parser.Pos()
endOffset = startOffset + maxRegionSize
if endOffset > dataFile.FileMeta.FileSize {
@@ -410,7 +433,7 @@ func SplitLargeFile(
if err = parser.SetPos(endOffset, prevRowIDMax); err != nil {
return 0, nil, nil, err
}
- pos, err := parser.ReadUntilTerminator()
+ _, pos, err := parser.ReadUntilTerminator()
if err != nil {
if !errors.ErrorEqual(err, io.EOF) {
return 0, nil, nil, err
diff --git a/br/pkg/lightning/mydump/region_test.go b/br/pkg/lightning/mydump/region_test.go
index 0830d378f47ff..362ff8603c7f9 100644
--- a/br/pkg/lightning/mydump/region_test.go
+++ b/br/pkg/lightning/mydump/region_test.go
@@ -174,13 +174,14 @@ func TestMakeSourceFileRegion(t *testing.T) {
ReadBlockSize: config.ReadBlockSize,
MaxRegionSize: 1,
CSV: config.CSVConfig{
- Separator: ",",
- Delimiter: "",
- Header: true,
- TrimLastSep: false,
- NotNull: false,
- Null: "NULL",
- BackslashEscape: true,
+ Separator: ",",
+ Delimiter: "",
+ Header: true,
+ HeaderSchemaMatch: true,
+ TrimLastSep: false,
+ NotNull: false,
+ Null: "NULL",
+ BackslashEscape: true,
},
StrictFormat: true,
Filter: []string{"*.*"},
@@ -199,7 +200,6 @@ func TestMakeSourceFileRegion(t *testing.T) {
store, err := storage.NewLocalStorage(".")
assert.NoError(t, err)
- // test - no compression
fileInfo.FileMeta.Compression = CompressionNone
regions, _, err := MakeSourceFileRegion(ctx, meta, fileInfo, colCnt, cfg, ioWorkers, store)
assert.NoError(t, err)
@@ -217,10 +217,66 @@ func TestMakeSourceFileRegion(t *testing.T) {
assert.NoError(t, err)
assert.Len(t, regions, 1)
assert.Equal(t, int64(0), regions[0].Chunk.Offset)
- assert.Equal(t, fileInfo.FileMeta.FileSize, regions[0].Chunk.EndOffset)
+ assert.Equal(t, TableFileSizeINF, regions[0].Chunk.EndOffset)
assert.Len(t, regions[0].Chunk.Columns, 0)
}
+func TestCompressedMakeSourceFileRegion(t *testing.T) {
+ meta := &MDTableMeta{
+ DB: "csv",
+ Name: "large_csv_file",
+ }
+ cfg := &config.Config{
+ Mydumper: config.MydumperRuntime{
+ ReadBlockSize: config.ReadBlockSize,
+ MaxRegionSize: 1,
+ CSV: config.CSVConfig{
+ Separator: ",",
+ Delimiter: "",
+ Header: true,
+ HeaderSchemaMatch: true,
+ TrimLastSep: false,
+ NotNull: false,
+ Null: "NULL",
+ BackslashEscape: true,
+ },
+ StrictFormat: true,
+ Filter: []string{"*.*"},
+ },
+ }
+ filePath := "./csv/split_large_file.csv.zst"
+ dataFileInfo, err := os.Stat(filePath)
+ require.NoError(t, err)
+ fileSize := dataFileInfo.Size()
+
+ fileInfo := FileInfo{FileMeta: SourceFileMeta{
+ Path: filePath,
+ Type: SourceTypeCSV,
+ Compression: CompressionZStd,
+ FileSize: fileSize,
+ }}
+ colCnt := 3
+
+ ctx := context.Background()
+ ioWorkers := worker.NewPool(ctx, 4, "io")
+ store, err := storage.NewLocalStorage(".")
+ assert.NoError(t, err)
+ compressRatio, err := SampleFileCompressRatio(ctx, fileInfo.FileMeta, store)
+ require.NoError(t, err)
+ fileInfo.FileMeta.RealSize = int64(compressRatio * float64(fileInfo.FileMeta.FileSize))
+
+ regions, sizes, err := MakeSourceFileRegion(ctx, meta, fileInfo, colCnt, cfg, ioWorkers, store)
+ assert.NoError(t, err)
+ assert.Len(t, regions, 1)
+ assert.Equal(t, int64(0), regions[0].Chunk.Offset)
+ assert.Equal(t, int64(0), regions[0].Chunk.RealOffset)
+ assert.Equal(t, TableFileSizeINF, regions[0].Chunk.EndOffset)
+ rowIDMax := fileInfo.FileMeta.RealSize * CompressSizeFactor / int64(colCnt)
+ assert.Equal(t, rowIDMax, regions[0].Chunk.RowIDMax)
+ assert.Len(t, regions[0].Chunk.Columns, 0)
+ assert.Equal(t, fileInfo.FileMeta.RealSize, int64(sizes[0]))
+}
+
func TestSplitLargeFile(t *testing.T) {
meta := &MDTableMeta{
DB: "csv",
@@ -230,13 +286,14 @@ func TestSplitLargeFile(t *testing.T) {
Mydumper: config.MydumperRuntime{
ReadBlockSize: config.ReadBlockSize,
CSV: config.CSVConfig{
- Separator: ",",
- Delimiter: "",
- Header: true,
- TrimLastSep: false,
- NotNull: false,
- Null: "NULL",
- BackslashEscape: true,
+ Separator: ",",
+ Delimiter: "",
+ Header: true,
+ HeaderSchemaMatch: true,
+ TrimLastSep: false,
+ NotNull: false,
+ Null: "NULL",
+ BackslashEscape: true,
},
StrictFormat: true,
Filter: []string{"*.*"},
@@ -288,13 +345,14 @@ func TestSplitLargeFileNoNewLineAtEOF(t *testing.T) {
Mydumper: config.MydumperRuntime{
ReadBlockSize: config.ReadBlockSize,
CSV: config.CSVConfig{
- Separator: ",",
- Delimiter: "",
- Header: true,
- TrimLastSep: false,
- NotNull: false,
- Null: "NULL",
- BackslashEscape: true,
+ Separator: ",",
+ Delimiter: "",
+ Header: true,
+ HeaderSchemaMatch: true,
+ TrimLastSep: false,
+ NotNull: false,
+ Null: "NULL",
+ BackslashEscape: true,
},
StrictFormat: true,
Filter: []string{"*.*"},
@@ -393,13 +451,14 @@ func TestSplitLargeFileOnlyOneChunk(t *testing.T) {
Mydumper: config.MydumperRuntime{
ReadBlockSize: config.ReadBlockSize,
CSV: config.CSVConfig{
- Separator: ",",
- Delimiter: "",
- Header: true,
- TrimLastSep: false,
- NotNull: false,
- Null: "NULL",
- BackslashEscape: true,
+ Separator: ",",
+ Delimiter: "",
+ Header: true,
+ HeaderSchemaMatch: true,
+ TrimLastSep: false,
+ NotNull: false,
+ Null: "NULL",
+ BackslashEscape: true,
},
StrictFormat: true,
Filter: []string{"*.*"},
diff --git a/br/pkg/lightning/mydump/router.go b/br/pkg/lightning/mydump/router.go
index bdc2a922f12f7..bf0ccba834fe0 100644
--- a/br/pkg/lightning/mydump/router.go
+++ b/br/pkg/lightning/mydump/router.go
@@ -134,7 +134,7 @@ func parseCompressionType(t string) (Compression, error) {
return CompressionGZ, nil
case "lz4":
return CompressionLZ4, nil
- case "zstd":
+ case "zstd", "zst":
return CompressionZStd, nil
case "xz":
return CompressionXZ, nil
@@ -324,6 +324,9 @@ func (p regexRouterParser) Parse(r *config.FileRouteRule, logger log.Logger) (*R
if err != nil {
return err
}
+ if result.Type == SourceTypeParquet && compression != CompressionNone {
+ return errors.Errorf("can't support whole compressed parquet file, should compress parquet files by choosing correct parquet compress writer, path: %s", r.Path)
+ }
result.Compression = compression
return nil
})
diff --git a/br/pkg/lightning/mydump/router_test.go b/br/pkg/lightning/mydump/router_test.go
index 4e3d8a4215a0d..ab97769e30ce8 100644
--- a/br/pkg/lightning/mydump/router_test.go
+++ b/br/pkg/lightning/mydump/router_test.go
@@ -292,3 +292,21 @@ func TestRouteWithPath(t *testing.T) {
require.NoError(t, err)
require.Nil(t, res)
}
+
+func TestRouteWithCompressedParquet(t *testing.T) {
+ fileName := "myschema.my_table.000.parquet.gz"
+ rule := &config.FileRouteRule{
+ Pattern: `(?i)^(?:[^/]*/)*([^/.]+)\.(.*?)(?:\.([0-9]+))?\.(sql|csv|parquet)(?:\.(\w+))?$`,
+ Schema: "$1",
+ Table: "$2",
+ Type: "$4",
+ Key: "$3",
+ Compression: "$5",
+ Unescape: true,
+ }
+ r := *rule
+ router, err := NewFileRouter([]*config.FileRouteRule{&r}, log.L())
+ require.NoError(t, err)
+ _, err = router.Route(fileName)
+ require.Error(t, err)
+}
diff --git a/br/pkg/lightning/restore/BUILD.bazel b/br/pkg/lightning/restore/BUILD.bazel
index 175e69e22d02f..647c4cea4d191 100644
--- a/br/pkg/lightning/restore/BUILD.bazel
+++ b/br/pkg/lightning/restore/BUILD.bazel
@@ -39,11 +39,13 @@ go_library(
"//br/pkg/pdutil",
"//br/pkg/redact",
"//br/pkg/storage",
+ "//br/pkg/streamhelper",
"//br/pkg/utils",
"//br/pkg/version",
"//br/pkg/version/build",
"//ddl",
"//errno",
+ "//keyspace",
"//kv",
"//meta/autoid",
"//parser",
@@ -77,6 +79,8 @@ go_library(
"@com_github_pingcap_tipb//go-tipb",
"@com_github_tikv_client_go_v2//oracle",
"@com_github_tikv_pd_client//:client",
+ "@io_etcd_go_etcd_client_v3//:client",
+ "@org_golang_google_grpc//:grpc",
"@org_golang_x_exp//maps",
"@org_golang_x_exp//slices",
"@org_golang_x_sync//errgroup",
@@ -124,6 +128,7 @@ go_test(
"//br/pkg/lightning/worker",
"//br/pkg/mock",
"//br/pkg/storage",
+ "//br/pkg/streamhelper",
"//br/pkg/version/build",
"//ddl",
"//errno",
@@ -158,6 +163,8 @@ go_test(
"@com_github_tikv_pd_client//:client",
"@com_github_xitongsys_parquet_go//writer",
"@com_github_xitongsys_parquet_go_source//buffer",
+ "@io_etcd_go_etcd_client_v3//:client",
+ "@io_etcd_go_etcd_tests_v3//integration",
"@org_uber_go_atomic//:atomic",
"@org_uber_go_zap//:zap",
],
diff --git a/br/pkg/lightning/restore/check_info.go b/br/pkg/lightning/restore/check_info.go
index cc4b3b734ebaa..aab9e5ebacef5 100644
--- a/br/pkg/lightning/restore/check_info.go
+++ b/br/pkg/lightning/restore/check_info.go
@@ -155,3 +155,10 @@ func (rc *Controller) checkSourceSchema(ctx context.Context) error {
}
return rc.doPreCheckOnItem(ctx, CheckSourceSchemaValid)
}
+
+func (rc *Controller) checkCDCPiTR(ctx context.Context) error {
+ if rc.cfg.TikvImporter.Backend == config.BackendTiDB {
+ return nil
+ }
+ return rc.doPreCheckOnItem(ctx, CheckTargetUsingCDCPITR)
+}
diff --git a/br/pkg/lightning/restore/check_info_test.go b/br/pkg/lightning/restore/check_info_test.go
index 3a8a666699164..36903ab93b22c 100644
--- a/br/pkg/lightning/restore/check_info_test.go
+++ b/br/pkg/lightning/restore/check_info_test.go
@@ -493,11 +493,11 @@ func TestCheckTableEmpty(t *testing.T) {
require.NoError(t, err)
mock.MatchExpectationsInOrder(false)
targetInfoGetter.targetDBGlue = glue.NewExternalTiDBGlue(db, mysql.ModeNone)
- mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl1` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl1` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(sqlmock.NewRows([]string{""}).RowError(0, sql.ErrNoRows))
- mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl2` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl2` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(sqlmock.NewRows([]string{""}).RowError(0, sql.ErrNoRows))
- mock.ExpectQuery("SELECT 1 FROM `test2`.`tbl1` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test2`.`tbl1` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(sqlmock.NewRows([]string{""}).RowError(0, sql.ErrNoRows))
rc.checkTemplate = NewSimpleTemplate()
err = rc.checkTableEmpty(ctx)
@@ -510,13 +510,13 @@ func TestCheckTableEmpty(t *testing.T) {
targetInfoGetter.targetDBGlue = glue.NewExternalTiDBGlue(db, mysql.ModeNone)
mock.MatchExpectationsInOrder(false)
// test auto retry retryable error
- mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl1` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl1` USE INDEX\\(\\) LIMIT 1").
WillReturnError(&gmysql.MySQLError{Number: errno.ErrPDServerTimeout})
- mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl1` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl1` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(sqlmock.NewRows([]string{""}).RowError(0, sql.ErrNoRows))
- mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl2` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl2` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(sqlmock.NewRows([]string{""}).RowError(0, sql.ErrNoRows))
- mock.ExpectQuery("SELECT 1 FROM `test2`.`tbl1` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test2`.`tbl1` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(sqlmock.NewRows([]string{""}).AddRow(1))
rc.checkTemplate = NewSimpleTemplate()
err = rc.checkTableEmpty(ctx)
@@ -532,11 +532,11 @@ func TestCheckTableEmpty(t *testing.T) {
require.NoError(t, err)
targetInfoGetter.targetDBGlue = glue.NewExternalTiDBGlue(db, mysql.ModeNone)
mock.MatchExpectationsInOrder(false)
- mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl1` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl1` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(sqlmock.NewRows([]string{""}).AddRow(1))
- mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl2` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl2` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(sqlmock.NewRows([]string{""}).RowError(0, sql.ErrNoRows))
- mock.ExpectQuery("SELECT 1 FROM `test2`.`tbl1` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test2`.`tbl1` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(sqlmock.NewRows([]string{""}).AddRow(1))
rc.checkTemplate = NewSimpleTemplate()
err = rc.checkTableEmpty(ctx)
@@ -576,7 +576,7 @@ func TestCheckTableEmpty(t *testing.T) {
require.NoError(t, err)
targetInfoGetter.targetDBGlue = glue.NewExternalTiDBGlue(db, mysql.ModeNone)
// only need to check the one that is not in checkpoint
- mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl2` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test1`.`tbl2` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(sqlmock.NewRows([]string{""}).RowError(0, sql.ErrNoRows))
err = rc.checkTableEmpty(ctx)
require.NoError(t, err)
diff --git a/br/pkg/lightning/restore/checksum.go b/br/pkg/lightning/restore/checksum.go
index 71b02801dc2dc..b30fe14e01fc1 100644
--- a/br/pkg/lightning/restore/checksum.go
+++ b/br/pkg/lightning/restore/checksum.go
@@ -374,7 +374,7 @@ func newGCTTLManager(pdClient pd.Client) gcTTLManager {
func (m *gcTTLManager) addOneJob(ctx context.Context, table string, ts uint64) error {
// start gc ttl loop if not started yet.
- if m.started.CAS(false, true) {
+ if m.started.CompareAndSwap(false, true) {
m.start(ctx)
}
m.lock.Lock()
diff --git a/br/pkg/lightning/restore/get_pre_info.go b/br/pkg/lightning/restore/get_pre_info.go
index 287d59c6145a4..4273ff708a89b 100644
--- a/br/pkg/lightning/restore/get_pre_info.go
+++ b/br/pkg/lightning/restore/get_pre_info.go
@@ -189,7 +189,12 @@ func (g *TargetInfoGetterImpl) IsTableEmpty(ctx context.Context, schemaName stri
}
var dump int
err = exec.QueryRow(ctx, "check table empty",
- fmt.Sprintf("SELECT 1 FROM %s LIMIT 1", common.UniqueTable(schemaName, tableName)),
+ // Here we use the `USE INDEX()` hint to skip fetch the record from index.
+ // In Lightning, if previous importing is halted half-way, it is possible that
+ // the data is partially imported, but the index data has not been imported.
+ // In this situation, if no hint is added, the SQL executor might fetch the record from index,
+ // which is empty. This will result in missing check.
+ fmt.Sprintf("SELECT 1 FROM %s USE INDEX() LIMIT 1", common.UniqueTable(schemaName, tableName)),
&dump,
)
@@ -444,15 +449,7 @@ func (p *PreRestoreInfoGetterImpl) ReadFirstNRowsByTableName(ctx context.Context
// ReadFirstNRowsByFileMeta reads the first N rows of an data file.
// It implements the PreRestoreInfoGetter interface.
func (p *PreRestoreInfoGetterImpl) ReadFirstNRowsByFileMeta(ctx context.Context, dataFileMeta mydump.SourceFileMeta, n int) ([]string, [][]types.Datum, error) {
- var (
- reader storage.ReadSeekCloser
- err error
- )
- if dataFileMeta.Type == mydump.SourceTypeParquet {
- reader, err = mydump.OpenParquetReader(ctx, p.srcStorage, dataFileMeta.Path, dataFileMeta.FileSize)
- } else {
- reader, err = p.srcStorage.Open(ctx, dataFileMeta.Path)
- }
+ reader, err := openReader(ctx, dataFileMeta, p.srcStorage)
if err != nil {
return nil, nil, errors.Trace(err)
}
@@ -590,13 +587,7 @@ func (p *PreRestoreInfoGetterImpl) sampleDataFromTable(
return resultIndexRatio, isRowOrdered, nil
}
sampleFile := tableMeta.DataFiles[0].FileMeta
- var reader storage.ReadSeekCloser
- var err error
- if sampleFile.Type == mydump.SourceTypeParquet {
- reader, err = mydump.OpenParquetReader(ctx, p.srcStorage, sampleFile.Path, sampleFile.FileSize)
- } else {
- reader, err = p.srcStorage.Open(ctx, sampleFile.Path)
- }
+ reader, err := openReader(ctx, sampleFile, p.srcStorage)
if err != nil {
return 0.0, false, errors.Trace(err)
}
diff --git a/br/pkg/lightning/restore/get_pre_info_test.go b/br/pkg/lightning/restore/get_pre_info_test.go
index 8ea57d023c679..71c2810d0b60e 100644
--- a/br/pkg/lightning/restore/get_pre_info_test.go
+++ b/br/pkg/lightning/restore/get_pre_info_test.go
@@ -14,6 +14,8 @@
package restore
import (
+ "bytes"
+ "compress/gzip"
"context"
"database/sql"
"fmt"
@@ -24,6 +26,7 @@ import (
mysql_sql_driver "github.com/go-sql-driver/mysql"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/br/pkg/lightning/config"
+ "github.com/pingcap/tidb/br/pkg/lightning/mydump"
"github.com/pingcap/tidb/br/pkg/lightning/restore/mock"
ropts "github.com/pingcap/tidb/br/pkg/lightning/restore/opts"
"github.com/pingcap/tidb/errno"
@@ -412,6 +415,118 @@ INSERT INTO db01.tbl01 (ival, sval) VALUES (444, 'ddd');`
require.Equal(t, theDataInfo.ExpectFirstRowDatums, rowDatums)
}
+func compressGz(t *testing.T, data []byte) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ w := gzip.NewWriter(&buf)
+ _, err := w.Write(data)
+ require.NoError(t, err)
+ require.NoError(t, w.Close())
+ return buf.Bytes()
+}
+
+func TestGetPreInfoReadCompressedFirstRow(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ var (
+ testCSVData01 = []byte(`ival,sval
+111,"aaa"
+222,"bbb"
+`)
+ testSQLData01 = []byte(`INSERT INTO db01.tbl01 (ival, sval) VALUES (333, 'ccc');
+INSERT INTO db01.tbl01 (ival, sval) VALUES (444, 'ddd');`)
+ )
+
+ test1CSVCompressed := compressGz(t, testCSVData01)
+ test1SQLCompressed := compressGz(t, testSQLData01)
+
+ testDataInfos := []struct {
+ FileName string
+ Data []byte
+ FirstN int
+ CSVConfig *config.CSVConfig
+ ExpectFirstRowDatums [][]types.Datum
+ ExpectColumns []string
+ }{
+ {
+ FileName: "/db01/tbl01/data.001.csv.gz",
+ Data: test1CSVCompressed,
+ FirstN: 1,
+ ExpectFirstRowDatums: [][]types.Datum{
+ {
+ types.NewStringDatum("111"),
+ types.NewStringDatum("aaa"),
+ },
+ },
+ ExpectColumns: []string{"ival", "sval"},
+ },
+ {
+ FileName: "/db01/tbl01/data.001.sql.gz",
+ Data: test1SQLCompressed,
+ FirstN: 1,
+ ExpectFirstRowDatums: [][]types.Datum{
+ {
+ types.NewUintDatum(333),
+ types.NewStringDatum("ccc"),
+ },
+ },
+ ExpectColumns: []string{"ival", "sval"},
+ },
+ }
+
+ tbl01SchemaBytes := []byte("CREATE TABLE db01.tbl01(id INTEGER PRIMARY KEY AUTO_INCREMENT, ival INTEGER, sval VARCHAR(64));")
+ tbl01SchemaBytesCompressed := compressGz(t, tbl01SchemaBytes)
+
+ tblMockSourceData := &mock.MockTableSourceData{
+ DBName: "db01",
+ TableName: "tbl01",
+ SchemaFile: &mock.MockSourceFile{
+ FileName: "/db01/tbl01/tbl01.schema.sql.gz",
+ Data: tbl01SchemaBytesCompressed,
+ },
+ DataFiles: []*mock.MockSourceFile{},
+ }
+ for _, testInfo := range testDataInfos {
+ tblMockSourceData.DataFiles = append(tblMockSourceData.DataFiles, &mock.MockSourceFile{
+ FileName: testInfo.FileName,
+ Data: testInfo.Data,
+ })
+ }
+ mockDataMap := map[string]*mock.MockDBSourceData{
+ "db01": {
+ Name: "db01",
+ Tables: map[string]*mock.MockTableSourceData{
+ "tbl01": tblMockSourceData,
+ },
+ },
+ }
+ mockSrc, err := mock.NewMockImportSource(mockDataMap)
+ require.Nil(t, err)
+ mockTarget := mock.NewMockTargetInfo()
+ cfg := config.NewConfig()
+ cfg.TikvImporter.Backend = config.BackendLocal
+ ig, err := NewPreRestoreInfoGetter(cfg, mockSrc.GetAllDBFileMetas(), mockSrc.GetStorage(), mockTarget, nil, nil)
+ require.NoError(t, err)
+
+ cfg.Mydumper.CSV.Header = true
+ tblMeta := mockSrc.GetDBMetaMap()["db01"].Tables[0]
+ for i, dataFile := range tblMeta.DataFiles {
+ theDataInfo := testDataInfos[i]
+ dataFile.FileMeta.Compression = mydump.CompressionGZ
+ cols, rowDatums, err := ig.ReadFirstNRowsByFileMeta(ctx, dataFile.FileMeta, theDataInfo.FirstN)
+ require.Nil(t, err)
+ t.Logf("%v, %v", cols, rowDatums)
+ require.Equal(t, theDataInfo.ExpectColumns, cols)
+ require.Equal(t, theDataInfo.ExpectFirstRowDatums, rowDatums)
+ }
+
+ theDataInfo := testDataInfos[0]
+ cols, rowDatums, err := ig.ReadFirstNRowsByTableName(ctx, "db01", "tbl01", theDataInfo.FirstN)
+ require.NoError(t, err)
+ require.Equal(t, theDataInfo.ExpectColumns, cols)
+ require.Equal(t, theDataInfo.ExpectFirstRowDatums, rowDatums)
+}
+
func TestGetPreInfoSampleSource(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -497,6 +612,100 @@ func TestGetPreInfoSampleSource(t *testing.T) {
}
}
+func TestGetPreInfoSampleSourceCompressed(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ dataFileName := "/db01/tbl01/tbl01.data.001.csv.gz"
+ schemaFileData := []byte("CREATE TABLE db01.tbl01 (id INTEGER PRIMARY KEY AUTO_INCREMENT, ival INTEGER, sval VARCHAR(64));")
+ schemaFileDataCompressed := compressGz(t, schemaFileData)
+ mockDataMap := map[string]*mock.MockDBSourceData{
+ "db01": {
+ Name: "db01",
+ Tables: map[string]*mock.MockTableSourceData{
+ "tbl01": {
+ DBName: "db01",
+ TableName: "tbl01",
+ SchemaFile: &mock.MockSourceFile{
+ FileName: "/db01/tbl01/tbl01.schema.sql.gz",
+ Data: schemaFileDataCompressed,
+ },
+ DataFiles: []*mock.MockSourceFile{
+ {
+ FileName: dataFileName,
+ Data: []byte(nil),
+ },
+ },
+ },
+ },
+ },
+ }
+ mockSrc, err := mock.NewMockImportSource(mockDataMap)
+ require.Nil(t, err)
+ mockTarget := mock.NewMockTargetInfo()
+ cfg := config.NewConfig()
+ cfg.TikvImporter.Backend = config.BackendLocal
+ ig, err := NewPreRestoreInfoGetter(cfg, mockSrc.GetAllDBFileMetas(), mockSrc.GetStorage(), mockTarget, nil, nil, ropts.WithIgnoreDBNotExist(true))
+ require.NoError(t, err)
+
+ mdDBMeta := mockSrc.GetAllDBFileMetas()[0]
+ mdTblMeta := mdDBMeta.Tables[0]
+ dbInfos, err := ig.GetAllTableStructures(ctx)
+ require.NoError(t, err)
+
+ data := [][]byte{
+ []byte(`id,ival,sval
+1,111,"aaa"
+2,222,"bbb"
+`),
+ []byte(`sval,ival,id
+"aaa",111,1
+"bbb",222,2
+`),
+ []byte(`id,ival,sval
+2,222,"bbb"
+1,111,"aaa"
+`),
+ []byte(`sval,ival,id
+"aaa",111,2
+"bbb",222,1
+`),
+ }
+ compressedData := make([][]byte, 0, 4)
+ for _, d := range data {
+ compressedData = append(compressedData, compressGz(t, d))
+ }
+
+ subTests := []struct {
+ Data []byte
+ ExpectIsOrdered bool
+ }{
+ {
+ Data: compressedData[0],
+ ExpectIsOrdered: true,
+ },
+ {
+ Data: compressedData[1],
+ ExpectIsOrdered: true,
+ },
+ {
+ Data: compressedData[2],
+ ExpectIsOrdered: false,
+ },
+ {
+ Data: compressedData[3],
+ ExpectIsOrdered: false,
+ },
+ }
+ for _, subTest := range subTests {
+ require.NoError(t, mockSrc.GetStorage().WriteFile(ctx, dataFileName, subTest.Data))
+ sampledIndexRatio, isRowOrderedFromSample, err := ig.sampleDataFromTable(ctx, "db01", mdTblMeta, dbInfos["db01"].Tables["tbl01"].Core, nil, defaultImportantVariables)
+ require.NoError(t, err)
+ t.Logf("%v, %v", sampledIndexRatio, isRowOrderedFromSample)
+ require.Greater(t, sampledIndexRatio, 1.0)
+ require.Equal(t, subTest.ExpectIsOrdered, isRowOrderedFromSample)
+ }
+}
+
func TestGetPreInfoEstimateSourceSize(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -553,7 +762,7 @@ func TestGetPreInfoIsTableEmpty(t *testing.T) {
require.NoError(t, err)
require.Equal(t, lnConfig, targetGetter.cfg)
- mock.ExpectQuery("SELECT 1 FROM `test_db`.`test_tbl` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test_db`.`test_tbl` USE INDEX\\(\\) LIMIT 1").
WillReturnError(&mysql_sql_driver.MySQLError{
Number: errno.ErrNoSuchTable,
Message: "Table 'test_db.test_tbl' doesn't exist",
@@ -563,7 +772,7 @@ func TestGetPreInfoIsTableEmpty(t *testing.T) {
require.NotNil(t, pIsEmpty)
require.Equal(t, true, *pIsEmpty)
- mock.ExpectQuery("SELECT 1 FROM `test_db`.`test_tbl` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test_db`.`test_tbl` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(
sqlmock.NewRows([]string{"1"}).
RowError(0, sql.ErrNoRows),
@@ -573,7 +782,7 @@ func TestGetPreInfoIsTableEmpty(t *testing.T) {
require.NotNil(t, pIsEmpty)
require.Equal(t, true, *pIsEmpty)
- mock.ExpectQuery("SELECT 1 FROM `test_db`.`test_tbl` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test_db`.`test_tbl` USE INDEX\\(\\) LIMIT 1").
WillReturnRows(
sqlmock.NewRows([]string{"1"}).AddRow(1),
)
@@ -582,7 +791,7 @@ func TestGetPreInfoIsTableEmpty(t *testing.T) {
require.NotNil(t, pIsEmpty)
require.Equal(t, false, *pIsEmpty)
- mock.ExpectQuery("SELECT 1 FROM `test_db`.`test_tbl` LIMIT 1").
+ mock.ExpectQuery("SELECT 1 FROM `test_db`.`test_tbl` USE INDEX\\(\\) LIMIT 1").
WillReturnError(errors.New("some dummy error"))
_, err = targetGetter.IsTableEmpty(ctx, "test_db", "test_tbl")
require.Error(t, err)
diff --git a/br/pkg/lightning/restore/meta_manager.go b/br/pkg/lightning/restore/meta_manager.go
index 2d9875ad56960..7f1546d552aa7 100644
--- a/br/pkg/lightning/restore/meta_manager.go
+++ b/br/pkg/lightning/restore/meta_manager.go
@@ -87,7 +87,7 @@ type tableMetaMgr interface {
UpdateTableStatus(ctx context.Context, status metaStatus) error
UpdateTableBaseChecksum(ctx context.Context, checksum *verify.KVChecksum) error
CheckAndUpdateLocalChecksum(ctx context.Context, checksum *verify.KVChecksum, hasLocalDupes bool) (
- needChecksum bool, needRemoteDupe bool, baseTotalChecksum *verify.KVChecksum, err error)
+ otherHasDupe bool, needRemoteDupe bool, baseTotalChecksum *verify.KVChecksum, err error)
FinishTable(ctx context.Context) error
}
@@ -370,7 +370,7 @@ func (m *dbTableMetaMgr) UpdateTableStatus(ctx context.Context, status metaStatu
}
func (m *dbTableMetaMgr) CheckAndUpdateLocalChecksum(ctx context.Context, checksum *verify.KVChecksum, hasLocalDupes bool) (
- needChecksum bool, needRemoteDupe bool, baseTotalChecksum *verify.KVChecksum, err error,
+ otherHasDupe bool, needRemoteDupe bool, baseTotalChecksum *verify.KVChecksum, err error,
) {
conn, err := m.session.Conn(ctx)
if err != nil {
@@ -393,7 +393,7 @@ func (m *dbTableMetaMgr) CheckAndUpdateLocalChecksum(ctx context.Context, checks
taskHasDuplicates bool
)
newStatus := metaStatusChecksuming
- needChecksum = true
+ otherHasDupe = false
needRemoteDupe = true
err = exec.Transact(ctx, "checksum pre-check", func(ctx context.Context, tx *sql.Tx) error {
rows, err := tx.QueryContext(
@@ -423,9 +423,7 @@ func (m *dbTableMetaMgr) CheckAndUpdateLocalChecksum(ctx context.Context, checks
return err
}
- if taskHasDuplicates {
- needChecksum = false
- }
+ otherHasDupe = otherHasDupe || taskHasDuplicates
// skip finished meta
if status >= metaStatusFinished {
@@ -436,7 +434,6 @@ func (m *dbTableMetaMgr) CheckAndUpdateLocalChecksum(ctx context.Context, checks
if status >= metaStatusChecksuming {
newStatus = status
needRemoteDupe = status == metaStatusChecksuming
- needChecksum = needChecksum && needRemoteDupe
return nil
}
@@ -445,7 +442,6 @@ func (m *dbTableMetaMgr) CheckAndUpdateLocalChecksum(ctx context.Context, checks
if status < metaStatusChecksuming {
newStatus = metaStatusChecksumSkipped
- needChecksum = false
needRemoteDupe = false
break
} else if status == metaStatusChecksuming {
@@ -475,12 +471,13 @@ func (m *dbTableMetaMgr) CheckAndUpdateLocalChecksum(ctx context.Context, checks
return false, false, nil, err
}
- if needChecksum {
+ if !otherHasDupe && needRemoteDupe {
ck := verify.MakeKVChecksum(totalBytes, totalKvs, totalChecksum)
baseTotalChecksum = &ck
}
log.FromContext(ctx).Info("check table checksum", zap.String("table", m.tr.tableName),
- zap.Bool("checksum", needChecksum), zap.String("new_status", newStatus.String()))
+ zap.Bool("otherHasDupe", otherHasDupe), zap.Bool("needRemoteDupe", needRemoteDupe),
+ zap.String("new_status", newStatus.String()))
return
}
@@ -1073,7 +1070,7 @@ func (m noopTableMetaMgr) UpdateTableBaseChecksum(ctx context.Context, checksum
}
func (m noopTableMetaMgr) CheckAndUpdateLocalChecksum(ctx context.Context, checksum *verify.KVChecksum, hasLocalDupes bool) (bool, bool, *verify.KVChecksum, error) {
- return true, true, &verify.KVChecksum{}, nil
+ return false, true, &verify.KVChecksum{}, nil
}
func (m noopTableMetaMgr) FinishTable(ctx context.Context) error {
diff --git a/br/pkg/lightning/restore/mock/mock.go b/br/pkg/lightning/restore/mock/mock.go
index f43e6c022673e..24e287f11c5f0 100644
--- a/br/pkg/lightning/restore/mock/mock.go
+++ b/br/pkg/lightning/restore/mock/mock.go
@@ -77,14 +77,19 @@ func NewMockImportSource(dbSrcDataMap map[string]*MockDBSourceData) (*MockImport
tblMeta := mydump.NewMDTableMeta("binary")
tblMeta.DB = dbName
tblMeta.Name = tblName
+ compression := mydump.CompressionNone
+ if strings.HasSuffix(tblData.SchemaFile.FileName, ".gz") {
+ compression = mydump.CompressionGZ
+ }
tblMeta.SchemaFile = mydump.FileInfo{
TableName: filter.Table{
Schema: dbName,
Name: tblName,
},
FileMeta: mydump.SourceFileMeta{
- Path: tblData.SchemaFile.FileName,
- Type: mydump.SourceTypeTableSchema,
+ Path: tblData.SchemaFile.FileName,
+ Type: mydump.SourceTypeTableSchema,
+ Compression: compression,
},
}
tblMeta.DataFiles = []mydump.FileInfo{}
@@ -106,14 +111,20 @@ func NewMockImportSource(dbSrcDataMap map[string]*MockDBSourceData) (*MockImport
FileMeta: mydump.SourceFileMeta{
Path: tblDataFile.FileName,
FileSize: int64(fileSize),
+ RealSize: int64(fileSize),
},
}
+ fileName := tblDataFile.FileName
+ if strings.HasSuffix(fileName, ".gz") {
+ fileName = strings.TrimSuffix(tblDataFile.FileName, ".gz")
+ fileInfo.FileMeta.Compression = mydump.CompressionGZ
+ }
switch {
- case strings.HasSuffix(tblDataFile.FileName, ".csv"):
+ case strings.HasSuffix(fileName, ".csv"):
fileInfo.FileMeta.Type = mydump.SourceTypeCSV
- case strings.HasSuffix(tblDataFile.FileName, ".sql"):
+ case strings.HasSuffix(fileName, ".sql"):
fileInfo.FileMeta.Type = mydump.SourceTypeSQL
- case strings.HasSuffix(tblDataFile.FileName, ".parquet"):
+ case strings.HasSuffix(fileName, ".parquet"):
fileInfo.FileMeta.Type = mydump.SourceTypeParquet
default:
return nil, errors.Errorf("unsupported file type: %s", tblDataFile.FileName)
diff --git a/br/pkg/lightning/restore/precheck.go b/br/pkg/lightning/restore/precheck.go
index 7dc578053492d..f078fe50f473c 100644
--- a/br/pkg/lightning/restore/precheck.go
+++ b/br/pkg/lightning/restore/precheck.go
@@ -25,6 +25,7 @@ const (
CheckTargetClusterVersion CheckItemID = "CHECK_TARGET_CLUSTER_VERSION"
CheckLocalDiskPlacement CheckItemID = "CHECK_LOCAL_DISK_PLACEMENT"
CheckLocalTempKVDir CheckItemID = "CHECK_LOCAL_TEMP_KV_DIR"
+ CheckTargetUsingCDCPITR CheckItemID = "CHECK_TARGET_USING_CDC_PITR"
)
type CheckResult struct {
@@ -138,7 +139,9 @@ func (b *PrecheckItemBuilder) BuildPrecheckItem(checkID CheckItemID) (PrecheckIt
case CheckLocalDiskPlacement:
return NewLocalDiskPlacementCheckItem(b.cfg), nil
case CheckLocalTempKVDir:
- return NewLocalTempKVDirCheckItem(b.cfg, b.preInfoGetter), nil
+ return NewLocalTempKVDirCheckItem(b.cfg, b.preInfoGetter, b.dbMetas), nil
+ case CheckTargetUsingCDCPITR:
+ return NewCDCPITRCheckItem(b.cfg), nil
default:
return nil, errors.Errorf("unsupported check item: %v", checkID)
}
diff --git a/br/pkg/lightning/restore/precheck_impl.go b/br/pkg/lightning/restore/precheck_impl.go
index 64e15d5aa828a..3dcd5cce973f9 100644
--- a/br/pkg/lightning/restore/precheck_impl.go
+++ b/br/pkg/lightning/restore/precheck_impl.go
@@ -14,13 +14,16 @@
package restore
import (
+ "bytes"
"context"
+ "encoding/json"
"fmt"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync"
+ "time"
"github.com/docker/go-units"
"github.com/pingcap/errors"
@@ -32,6 +35,7 @@ import (
"github.com/pingcap/tidb/br/pkg/lightning/log"
"github.com/pingcap/tidb/br/pkg/lightning/mydump"
"github.com/pingcap/tidb/br/pkg/storage"
+ "github.com/pingcap/tidb/br/pkg/streamhelper"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/store/pdtypes"
@@ -40,9 +44,11 @@ import (
"github.com/pingcap/tidb/util/engine"
"github.com/pingcap/tidb/util/mathutil"
"github.com/pingcap/tidb/util/set"
+ clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"golang.org/x/exp/slices"
"golang.org/x/sync/errgroup"
+ "google.golang.org/grpc"
)
type clusterResourceCheckItem struct {
@@ -428,7 +434,7 @@ func (ci *largeFileCheckItem) Check(ctx context.Context) (*CheckResult, error) {
for _, db := range ci.dbMetas {
for _, t := range db.Tables {
for _, f := range t.DataFiles {
- if f.FileMeta.FileSize > defaultCSVSize {
+ if f.FileMeta.RealSize > defaultCSVSize {
theResult.Message = fmt.Sprintf("large csv: %s file exists and it will slow down import performance", f.FileMeta.Path)
theResult.Passed = false
}
@@ -478,12 +484,14 @@ func (ci *localDiskPlacementCheckItem) Check(ctx context.Context) (*CheckResult,
type localTempKVDirCheckItem struct {
cfg *config.Config
preInfoGetter PreRestoreInfoGetter
+ dbMetas []*mydump.MDDatabaseMeta
}
-func NewLocalTempKVDirCheckItem(cfg *config.Config, preInfoGetter PreRestoreInfoGetter) PrecheckItem {
+func NewLocalTempKVDirCheckItem(cfg *config.Config, preInfoGetter PreRestoreInfoGetter, dbMetas []*mydump.MDDatabaseMeta) PrecheckItem {
return &localTempKVDirCheckItem{
cfg: cfg,
preInfoGetter: preInfoGetter,
+ dbMetas: dbMetas,
}
}
@@ -491,10 +499,28 @@ func (ci *localTempKVDirCheckItem) GetCheckItemID() CheckItemID {
return CheckLocalTempKVDir
}
+func (ci *localTempKVDirCheckItem) hasCompressedFiles() bool {
+ for _, dbMeta := range ci.dbMetas {
+ for _, tbMeta := range dbMeta.Tables {
+ for _, file := range tbMeta.DataFiles {
+ if file.FileMeta.Compression != mydump.CompressionNone {
+ return true
+ }
+ }
+ }
+ }
+ return false
+}
+
func (ci *localTempKVDirCheckItem) Check(ctx context.Context) (*CheckResult, error) {
+ severity := Critical
+ // for cases that have compressed files, the estimated size may not be accurate, set severity to Warn to avoid failure
+ if ci.hasCompressedFiles() {
+ severity = Warn
+ }
theResult := &CheckResult{
Item: ci.GetCheckItemID(),
- Severity: Critical,
+ Severity: severity,
}
storageSize, err := common.GetStorageSize(ci.cfg.TikvImporter.SortedKVDir)
if err != nil {
@@ -672,6 +698,182 @@ func (ci *checkpointCheckItem) checkpointIsValid(ctx context.Context, tableInfo
return msgs, nil
}
+// CDCPITRCheckItem check downstream has enabled CDC or PiTR. It's exposed to let
+// caller override the Instruction message.
+type CDCPITRCheckItem struct {
+ cfg *config.Config
+ Instruction string
+ // used in test
+ etcdCli *clientv3.Client
+}
+
+// NewCDCPITRCheckItem creates a checker to check downstream has enabled CDC or PiTR.
+func NewCDCPITRCheckItem(cfg *config.Config) PrecheckItem {
+ return &CDCPITRCheckItem{
+ cfg: cfg,
+ Instruction: "local backend is not compatible with them. Please switch to tidb backend then try again.",
+ }
+}
+
+// GetCheckItemID implements PrecheckItem interface.
+func (ci *CDCPITRCheckItem) GetCheckItemID() CheckItemID {
+ return CheckTargetUsingCDCPITR
+}
+
+func dialEtcdWithCfg(ctx context.Context, cfg *config.Config) (*clientv3.Client, error) {
+ cfg2, err := cfg.ToTLS()
+ if err != nil {
+ return nil, err
+ }
+ tlsConfig := cfg2.TLSConfig()
+
+ return clientv3.New(clientv3.Config{
+ TLS: tlsConfig,
+ Endpoints: []string{cfg.TiDB.PdAddr},
+ AutoSyncInterval: 30 * time.Second,
+ DialTimeout: 5 * time.Second,
+ DialOptions: []grpc.DialOption{
+ config.DefaultGrpcKeepaliveParams,
+ grpc.WithBlock(),
+ grpc.WithReturnConnectionError(),
+ },
+ Context: ctx,
+ })
+}
+
+// Check implements PrecheckItem interface.
+func (ci *CDCPITRCheckItem) Check(ctx context.Context) (*CheckResult, error) {
+ theResult := &CheckResult{
+ Item: ci.GetCheckItemID(),
+ Severity: Critical,
+ }
+
+ if ci.cfg.TikvImporter.Backend != config.BackendLocal {
+ theResult.Passed = true
+ theResult.Message = "TiDB Lightning is not using local backend, skip this check"
+ return theResult, nil
+ }
+
+ if ci.etcdCli == nil {
+ var err error
+ ci.etcdCli, err = dialEtcdWithCfg(ctx, ci.cfg)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ //nolint: errcheck
+ defer ci.etcdCli.Close()
+ }
+
+ errorMsg := make([]string, 0, 2)
+
+ pitrCli := streamhelper.NewMetaDataClient(ci.etcdCli)
+ tasks, err := pitrCli.GetAllTasks(ctx)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if len(tasks) > 0 {
+ names := make([]string, 0, len(tasks))
+ for _, task := range tasks {
+ names = append(names, task.Info.GetName())
+ }
+ errorMsg = append(errorMsg, fmt.Sprintf("found PiTR log streaming task(s): %v,", names))
+ }
+
+ // check etcd KV of CDC >= v6.2
+ cdcPrefix := "/tidb/cdc/"
+ changefeedPath := []byte("/changefeed/info/")
+
+ nameSet := make(map[string][]string, 1)
+ resp, err := ci.etcdCli.Get(ctx, cdcPrefix, clientv3.WithPrefix())
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ for _, kv := range resp.Kvs {
+ // example: /tidb/cdc///changefeed/info/
+ k := kv.Key[len(cdcPrefix):]
+ clusterAndNamespace, changefeedID, found := bytes.Cut(k, changefeedPath)
+ if !found {
+ continue
+ }
+ if !isActiveCDCChangefeed(kv.Value) {
+ continue
+ }
+
+ nameSet[string(clusterAndNamespace)] = append(nameSet[string(clusterAndNamespace)], string(changefeedID))
+ }
+ if len(nameSet) == 0 {
+ // check etcd KV of CDC <= v6.1
+ cdcPrefixV61 := "/tidb/cdc/changefeed/info/"
+ resp, err = ci.etcdCli.Get(ctx, cdcPrefixV61, clientv3.WithPrefix())
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ for _, kv := range resp.Kvs {
+ // example: /tidb/cdc/changefeed/info/
+ k := kv.Key[len(cdcPrefixV61):]
+ if len(k) == 0 {
+ continue
+ }
+ if !isActiveCDCChangefeed(kv.Value) {
+ continue
+ }
+
+ nameSet[""] = append(nameSet[""], string(k))
+ }
+ }
+
+ if len(nameSet) > 0 {
+ var changefeedMsgBuf strings.Builder
+ changefeedMsgBuf.WriteString("found CDC changefeed(s): ")
+ isFirst := true
+ for clusterID, captureIDs := range nameSet {
+ if !isFirst {
+ changefeedMsgBuf.WriteString(", ")
+ }
+ isFirst = false
+ changefeedMsgBuf.WriteString("cluster/namespace: ")
+ changefeedMsgBuf.WriteString(clusterID)
+ changefeedMsgBuf.WriteString(" changefeed(s): ")
+ changefeedMsgBuf.WriteString(fmt.Sprintf("%v", captureIDs))
+ }
+ changefeedMsgBuf.WriteString(",")
+ errorMsg = append(errorMsg, changefeedMsgBuf.String())
+ }
+
+ if len(errorMsg) > 0 {
+ errorMsg = append(errorMsg, ci.Instruction)
+ theResult.Passed = false
+ theResult.Message = strings.Join(errorMsg, "\n")
+ } else {
+ theResult.Passed = true
+ theResult.Message = "no CDC or PiTR task found"
+ }
+
+ return theResult, nil
+}
+
+type onlyState struct {
+ State string `json:"state"`
+}
+
+func isActiveCDCChangefeed(jsonBytes []byte) bool {
+ s := onlyState{}
+ err := json.Unmarshal(jsonBytes, &s)
+ if err != nil {
+ // maybe a compatible issue, skip this key
+ log.L().Error("unmarshal etcd value failed when check CDC changefeed, will skip this key",
+ zap.ByteString("value", jsonBytes),
+ zap.Error(err))
+ return false
+ }
+ switch s.State {
+ case "normal", "stopped", "error":
+ return true
+ default:
+ return false
+ }
+}
+
type schemaCheckItem struct {
cfg *config.Config
preInfoGetter PreRestoreInfoGetter
diff --git a/br/pkg/lightning/restore/precheck_impl_test.go b/br/pkg/lightning/restore/precheck_impl_test.go
index 88f3cf8f9a30b..9d10c34b76d13 100644
--- a/br/pkg/lightning/restore/precheck_impl_test.go
+++ b/br/pkg/lightning/restore/precheck_impl_test.go
@@ -24,7 +24,11 @@ import (
"github.com/pingcap/tidb/br/pkg/lightning/log"
"github.com/pingcap/tidb/br/pkg/lightning/restore/mock"
ropts "github.com/pingcap/tidb/br/pkg/lightning/restore/opts"
+ "github.com/pingcap/tidb/br/pkg/storage"
+ "github.com/pingcap/tidb/br/pkg/streamhelper"
"github.com/stretchr/testify/suite"
+ clientv3 "go.etcd.io/etcd/client/v3"
+ "go.etcd.io/etcd/tests/v3/integration"
)
type precheckImplSuite struct {
@@ -377,7 +381,7 @@ func (s *precheckImplSuite) TestLocalTempKVDirCheckBasic() {
defer cancel()
s.cfg.TikvImporter.SortedKVDir = "/tmp/"
- ci = NewLocalTempKVDirCheckItem(s.cfg, s.preInfoGetter)
+ ci = NewLocalTempKVDirCheckItem(s.cfg, s.preInfoGetter, s.mockSrc.GetAllDBFileMetas())
s.Require().Equal(CheckLocalTempKVDir, ci.GetCheckItemID())
result, err = ci.Check(ctx)
s.Require().NoError(err)
@@ -396,7 +400,7 @@ func (s *precheckImplSuite) TestLocalTempKVDirCheckBasic() {
},
)
s.Require().NoError(s.setMockImportData(testMockSrcData))
- ci = NewLocalTempKVDirCheckItem(s.cfg, s.preInfoGetter)
+ ci = NewLocalTempKVDirCheckItem(s.cfg, s.preInfoGetter, s.mockSrc.GetAllDBFileMetas())
s.Require().Equal(CheckLocalTempKVDir, ci.GetCheckItemID())
result, err = ci.Check(ctx)
s.Require().NoError(err)
@@ -581,3 +585,99 @@ func (s *precheckImplSuite) TestTableEmptyCheckBasic() {
s.T().Logf("check result message: %s", result.Message)
s.Require().False(result.Passed)
}
+
+func (s *precheckImplSuite) TestCDCPITRCheckItem() {
+ integration.BeforeTestExternal(s.T())
+ testEtcdCluster := integration.NewClusterV3(s.T(), &integration.ClusterConfig{Size: 1})
+ defer testEtcdCluster.Terminate(s.T())
+
+ ctx := context.Background()
+ cfg := &config.Config{
+ TikvImporter: config.TikvImporter{
+ Backend: config.BackendLocal,
+ },
+ }
+ ci := NewCDCPITRCheckItem(cfg)
+ checker := ci.(*CDCPITRCheckItem)
+ checker.etcdCli = testEtcdCluster.RandClient()
+ result, err := ci.Check(ctx)
+ s.Require().NoError(err)
+ s.Require().NotNil(result)
+ s.Require().Equal(ci.GetCheckItemID(), result.Item)
+ s.Require().Equal(Critical, result.Severity)
+ s.Require().True(result.Passed)
+ s.Require().Equal("no CDC or PiTR task found", result.Message)
+
+ cli := testEtcdCluster.RandClient()
+ brCli := streamhelper.NewMetaDataClient(cli)
+ backend, _ := storage.ParseBackend("noop://", nil)
+ taskInfo, err := streamhelper.NewTaskInfo("br_name").
+ FromTS(1).
+ UntilTS(1000).
+ WithTableFilter("*.*", "!mysql").
+ ToStorage(backend).
+ Check()
+ s.Require().NoError(err)
+ err = brCli.PutTask(ctx, *taskInfo)
+ s.Require().NoError(err)
+ checkEtcdPut := func(key string, vals ...string) {
+ val := ""
+ if len(vals) == 1 {
+ val = vals[0]
+ }
+ _, err := cli.Put(ctx, key, val)
+ s.Require().NoError(err)
+ }
+ // TiCDC >= v6.2
+ checkEtcdPut("/tidb/cdc/default/__cdc_meta__/capture/3ecd5c98-0148-4086-adfd-17641995e71f")
+ checkEtcdPut("/tidb/cdc/default/__cdc_meta__/meta/meta-version")
+ checkEtcdPut("/tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count")
+ checkEtcdPut("/tidb/cdc/default/__cdc_meta__/owner/22318498f4dd6639")
+ checkEtcdPut(
+ "/tidb/cdc/default/default/changefeed/info/test",
+ `{"upstream-id":7195826648407968958,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306?time-zone=","create-time":"2023-02-03T15:23:34.773768+08:00","start-ts":439198420741652483,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":true,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"none","enable-partition-separator":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"storage":""},"scheduler":{"region-per-span":0}},"state":"normal","error":null,"creator-version":"v6.5.0-master-dirty"}`,
+ )
+ checkEtcdPut(
+ "/tidb/cdc/default/default/changefeed/info/test-1",
+ `{"upstream-id":7195826648407968958,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306?time-zone=","create-time":"2023-02-03T15:23:34.773768+08:00","start-ts":439198420741652483,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":true,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"none","enable-partition-separator":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"storage":""},"scheduler":{"region-per-span":0}},"state":"failed","error":null,"creator-version":"v6.5.0-master-dirty"}`,
+ )
+ checkEtcdPut("/tidb/cdc/default/default/changefeed/status/test")
+ checkEtcdPut("/tidb/cdc/default/default/changefeed/status/test-1")
+ checkEtcdPut("/tidb/cdc/default/default/task/position/3ecd5c98-0148-4086-adfd-17641995e71f/test-1")
+ checkEtcdPut("/tidb/cdc/default/default/upstream/7168358383033671922")
+
+ result, err = ci.Check(ctx)
+ s.Require().NoError(err)
+ s.Require().False(result.Passed)
+ s.Require().Equal("found PiTR log streaming task(s): [br_name],\n"+
+ "found CDC changefeed(s): cluster/namespace: default/default changefeed(s): [test],\n"+
+ "local backend is not compatible with them. Please switch to tidb backend then try again.",
+ result.Message)
+
+ _, err = cli.Delete(ctx, "/tidb/cdc/", clientv3.WithPrefix())
+ s.Require().NoError(err)
+
+ // TiCDC <= v6.1
+ checkEtcdPut("/tidb/cdc/capture/f14cb04d-5ba1-410e-a59b-ccd796920e9d")
+ checkEtcdPut(
+ "/tidb/cdc/changefeed/info/test",
+ `{"upstream-id":7195826648407968958,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306?time-zone=","create-time":"2023-02-03T15:23:34.773768+08:00","start-ts":439198420741652483,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":true,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"none","enable-partition-separator":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"storage":""},"scheduler":{"region-per-span":0}},"state":"stopped","error":null,"creator-version":"v6.5.0-master-dirty"}`,
+ )
+ checkEtcdPut("/tidb/cdc/job/test")
+ checkEtcdPut("/tidb/cdc/owner/223184ad80a88b0b")
+ checkEtcdPut("/tidb/cdc/task/position/f14cb04d-5ba1-410e-a59b-ccd796920e9d/test")
+
+ result, err = ci.Check(ctx)
+ s.Require().NoError(err)
+ s.Require().False(result.Passed)
+ s.Require().Equal("found PiTR log streaming task(s): [br_name],\n"+
+ "found CDC changefeed(s): cluster/namespace: changefeed(s): [test],\n"+
+ "local backend is not compatible with them. Please switch to tidb backend then try again.",
+ result.Message)
+
+ checker.cfg.TikvImporter.Backend = config.BackendTiDB
+ result, err = ci.Check(ctx)
+ s.Require().NoError(err)
+ s.Require().True(result.Passed)
+ s.Require().Equal("TiDB Lightning is not using local backend, skip this check", result.Message)
+}
diff --git a/br/pkg/lightning/restore/restore.go b/br/pkg/lightning/restore/restore.go
index 0a0e05b45ac5d..1032419ab777f 100644
--- a/br/pkg/lightning/restore/restore.go
+++ b/br/pkg/lightning/restore/restore.go
@@ -54,6 +54,7 @@ import (
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/br/pkg/version"
"github.com/pingcap/tidb/br/pkg/version/build"
+ "github.com/pingcap/tidb/keyspace"
tidbkv "github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta/autoid"
"github.com/pingcap/tidb/parser/model"
@@ -227,12 +228,23 @@ type Controller struct {
diskQuotaState atomic.Int32
compactState atomic.Int32
status *LightningStatus
+ dupIndicator *atomic.Bool
preInfoGetter PreRestoreInfoGetter
precheckItemBuilder *PrecheckItemBuilder
+
+ keyspaceName string
}
+// LightningStatus provides the finished bytes and total bytes of the current task.
+// It should keep the value after restart from checkpoint.
+// When it is tidb backend, FinishedFileSize can be counted after chunk data is
+// restored to tidb. When it is local backend it's counted after whole engine is
+// imported.
+// TotalFileSize may be an estimated value, so when the task is finished, it may
+// not equal to FinishedFileSize.
type LightningStatus struct {
+ backend string
FinishedFileSize atomic.Int64
TotalFileSize atomic.Int64
}
@@ -255,6 +267,10 @@ type ControllerParam struct {
CheckpointStorage storage.ExternalStorage
// when CheckpointStorage is not nil, save file checkpoint to it with this name
CheckpointName string
+ // DupIndicator can expose the duplicate detection result to the caller
+ DupIndicator *atomic.Bool
+ // Keyspace name
+ KeyspaceName string
}
func NewRestoreController(
@@ -342,7 +358,7 @@ func NewRestoreControllerWithPauser(
}
}
- backend, err = local.NewLocalBackend(ctx, tls, cfg, p.Glue, maxOpenFiles, errorMgr)
+ backend, err = local.NewLocalBackend(ctx, tls, cfg, p.Glue, maxOpenFiles, errorMgr, p.KeyspaceName)
if err != nil {
return nil, common.NormalizeOrWrapErr(common.ErrUnknown, err)
}
@@ -353,6 +369,7 @@ func NewRestoreControllerWithPauser(
default:
return nil, common.ErrUnknownBackend.GenWithStackByArgs(cfg.TikvImporter.Backend)
}
+ p.Status.backend = cfg.TikvImporter.Backend
var metaBuilder metaMgrBuilder
isSSTImport := cfg.TikvImporter.Backend == config.BackendLocal
@@ -421,9 +438,12 @@ func NewRestoreControllerWithPauser(
errorMgr: errorMgr,
status: p.Status,
taskMgr: nil,
+ dupIndicator: p.DupIndicator,
preInfoGetter: preInfoGetter,
precheckItemBuilder: preCheckBuilder,
+
+ keyspaceName: p.KeyspaceName,
}
return rc, nil
@@ -925,7 +945,7 @@ func (rc *Controller) estimateChunkCountIntoMetrics(ctx context.Context) error {
if _, ok := fileChunks[c.Key.Path]; !ok {
fileChunks[c.Key.Path] = 0.0
}
- remainChunkCnt := float64(c.Chunk.EndOffset-c.Chunk.Offset) / float64(c.Chunk.EndOffset-c.Key.Offset)
+ remainChunkCnt := float64(c.UnfinishedSize()) / float64(c.TotalSize())
fileChunks[c.Key.Path] += remainChunkCnt
}
}
@@ -940,7 +960,8 @@ func (rc *Controller) estimateChunkCountIntoMetrics(ctx context.Context) error {
}
if fileMeta.FileMeta.Type == mydump.SourceTypeCSV {
cfg := rc.cfg.Mydumper
- if fileMeta.FileMeta.FileSize > int64(cfg.MaxRegionSize) && cfg.StrictFormat && !cfg.CSV.Header {
+ if fileMeta.FileMeta.FileSize > int64(cfg.MaxRegionSize) && cfg.StrictFormat &&
+ !cfg.CSV.Header && fileMeta.FileMeta.Compression == mydump.CompressionNone {
estimatedChunkCount += math.Round(float64(fileMeta.FileMeta.FileSize) / float64(cfg.MaxRegionSize))
} else {
estimatedChunkCount++
@@ -1486,7 +1507,7 @@ func (rc *Controller) restoreTables(ctx context.Context) (finalErr error) {
// Disable GC because TiDB enables GC already.
kvStore, err = driver.TiKVDriver{}.OpenWithOptions(
- fmt.Sprintf("tikv://%s?disableGC=true", rc.cfg.TiDB.PdAddr),
+ fmt.Sprintf("tikv://%s?disableGC=true&keyspaceName=%s", rc.cfg.TiDB.PdAddr, rc.keyspaceName),
driver.WithSecurity(rc.tls.ToTiKVSecurityConfig()),
)
if err != nil {
@@ -1606,7 +1627,7 @@ func (rc *Controller) restoreTables(ctx context.Context) (finalErr error) {
} else {
for _, eng := range cp.Engines {
for _, chunk := range eng.Chunks {
- totalDataSizeToRestore += chunk.Chunk.EndOffset - chunk.Chunk.Offset
+ totalDataSizeToRestore += chunk.UnfinishedSize()
}
}
}
@@ -1832,7 +1853,7 @@ func (rc *Controller) fullCompact(ctx context.Context) error {
// wait until any existing level-1 compact to complete first.
task := log.FromContext(ctx).Begin(zap.InfoLevel, "wait for completion of existing level 1 compaction")
- for !rc.compactState.CAS(compactStateIdle, compactStateDoing) {
+ for !rc.compactState.CompareAndSwap(compactStateIdle, compactStateDoing) {
time.Sleep(100 * time.Millisecond)
}
task.End(zap.ErrorLevel, nil)
@@ -1892,7 +1913,7 @@ func (rc *Controller) switchTiKVMode(ctx context.Context, mode sstpb.SwitchMode)
}
func (rc *Controller) enforceDiskQuota(ctx context.Context) {
- if !rc.diskQuotaState.CAS(diskQuotaStateIdle, diskQuotaStateChecking) {
+ if !rc.diskQuotaState.CompareAndSwap(diskQuotaStateIdle, diskQuotaStateChecking) {
// do not run multiple the disk quota check / import simultaneously.
// (we execute the lock check in background to avoid blocking the cron thread)
return
@@ -2127,6 +2148,10 @@ func (rc *Controller) preCheckRequirements(ctx context.Context) error {
return common.ErrCheckClusterRegion.Wrap(err).GenWithStackByArgs()
}
}
+ // even if checkpoint exists, we still need to make sure CDC/PiTR task is not running.
+ if err := rc.checkCDCPiTR(ctx); err != nil {
+ return common.ErrCheckCDCPiTR.Wrap(err).GenWithStackByArgs()
+ }
}
}
@@ -2190,23 +2215,7 @@ func newChunkRestore(
) (*chunkRestore, error) {
blockBufSize := int64(cfg.Mydumper.ReadBlockSize)
- var (
- reader storage.ReadSeekCloser
- compressType storage.CompressType
- err error
- )
- switch {
- case chunk.FileMeta.Type == mydump.SourceTypeParquet:
- reader, err = mydump.OpenParquetReader(ctx, store, chunk.FileMeta.Path, chunk.FileMeta.FileSize)
- case chunk.FileMeta.Compression != mydump.CompressionNone:
- compressType, err = mydump.ToStorageCompressType(chunk.FileMeta.Compression)
- if err != nil {
- break
- }
- reader, err = storage.WithCompression(store, compressType).Open(ctx, chunk.FileMeta.Path)
- default:
- reader, err = store.Open(ctx, chunk.FileMeta.Path)
- }
+ reader, err := openReader(ctx, chunk.FileMeta, store)
if err != nil {
return nil, errors.Trace(err)
}
@@ -2298,6 +2307,8 @@ type deliveredKVs struct {
columns []string
offset int64
rowID int64
+
+ realOffset int64 // indicates file reader's current position, only used for compressed files
}
type deliverResult struct {
@@ -2326,14 +2337,25 @@ func (cr *chunkRestore) deliverLoop(
dataSynced := true
hasMoreKVs := true
+ var startRealOffset, currRealOffset int64 // save to 0 at first
+
for hasMoreKVs {
- var dataChecksum, indexChecksum verify.KVChecksum
+ c := keyspace.CodecV1
+ if t.kvStore != nil {
+ c = t.kvStore.GetCodec()
+ }
+ var (
+ dataChecksum = verify.NewKVChecksumWithKeyspace(c)
+ indexChecksum = verify.NewKVChecksumWithKeyspace(c)
+ )
var columns []string
var kvPacket []deliveredKVs
// init these two field as checkpoint current value, so even if there are no kv pairs delivered,
// chunk checkpoint should stay the same
startOffset := cr.chunk.Chunk.Offset
currOffset := startOffset
+ startRealOffset = cr.chunk.Chunk.RealOffset
+ currRealOffset = startRealOffset
rowID := cr.chunk.Chunk.PrevRowIDMax
populate:
@@ -2348,12 +2370,14 @@ func (cr *chunkRestore) deliverLoop(
if p.kvs == nil {
// This is the last message.
currOffset = p.offset
+ currRealOffset = p.realOffset
hasMoreKVs = false
break populate
}
- p.kvs.ClassifyAndAppend(&dataKVs, &dataChecksum, &indexKVs, &indexChecksum)
+ p.kvs.ClassifyAndAppend(&dataKVs, dataChecksum, &indexKVs, indexChecksum)
columns = p.columns
currOffset = p.offset
+ currRealOffset = p.realOffset
rowID = p.rowID
}
case <-ctx.Done():
@@ -2417,9 +2441,10 @@ func (cr *chunkRestore) deliverLoop(
// No need to apply a lock since this is the only thread updating `cr.chunk.**`.
// In local mode, we should write these checkpoints after engine flushed.
lastOffset := cr.chunk.Chunk.Offset
- cr.chunk.Checksum.Add(&dataChecksum)
- cr.chunk.Checksum.Add(&indexChecksum)
+ cr.chunk.Checksum.Add(dataChecksum)
+ cr.chunk.Checksum.Add(indexChecksum)
cr.chunk.Chunk.Offset = currOffset
+ cr.chunk.Chunk.RealOffset = currRealOffset
cr.chunk.Chunk.PrevRowIDMax = rowID
if m, ok := metric.FromContext(ctx); ok {
@@ -2427,11 +2452,21 @@ func (cr *chunkRestore) deliverLoop(
// comes from chunk.Chunk.Offset. so it shouldn't happen that currOffset - startOffset < 0.
// but we met it one time, but cannot reproduce it now, we add this check to make code more robust
// TODO: reproduce and find the root cause and fix it completely
- if currOffset >= startOffset {
- m.BytesCounter.WithLabelValues(metric.BytesStateRestored).Add(float64(currOffset - startOffset))
+ var lowOffset, highOffset int64
+ if cr.chunk.FileMeta.Compression != mydump.CompressionNone {
+ lowOffset, highOffset = startRealOffset, currRealOffset
} else {
- deliverLogger.Warn("offset go back", zap.Int64("curr", currOffset),
- zap.Int64("start", startOffset))
+ lowOffset, highOffset = startOffset, currOffset
+ }
+ delta := highOffset - lowOffset
+ if delta >= 0 {
+ m.BytesCounter.WithLabelValues(metric.BytesStateRestored).Add(float64(delta))
+ if rc.status != nil && rc.status.backend == config.BackendTiDB {
+ rc.status.FinishedFileSize.Add(delta)
+ }
+ } else {
+ deliverLogger.Warn("offset go back", zap.Int64("curr", highOffset),
+ zap.Int64("start", lowOffset))
}
}
@@ -2441,6 +2476,11 @@ func (cr *chunkRestore) deliverLoop(
}
failpoint.Inject("SlowDownWriteRows", func() {
deliverLogger.Warn("Slowed down write rows")
+ finished := rc.status.FinishedFileSize.Load()
+ total := rc.status.TotalFileSize.Load()
+ deliverLogger.Warn("PrintStatus Failpoint",
+ zap.Int64("finished", finished),
+ zap.Int64("total", total))
})
failpoint.Inject("FailAfterWriteRows", nil)
// TODO: for local backend, we may save checkpoint more frequently, e.g. after written
@@ -2607,14 +2647,22 @@ func (cr *chunkRestore) encodeLoop(
canDeliver := false
kvPacket := make([]deliveredKVs, 0, maxKvPairsCnt)
curOffset := offset
- var newOffset, rowID int64
+ var newOffset, rowID, realOffset int64
var kvSize uint64
+ var realOffsetErr error
outLoop:
for !canDeliver {
readDurStart := time.Now()
err = cr.parser.ReadRow()
columnNames := cr.parser.Columns()
newOffset, rowID = cr.parser.Pos()
+ if cr.chunk.FileMeta.Compression != mydump.CompressionNone {
+ realOffset, realOffsetErr = cr.parser.RealPos()
+ if realOffsetErr != nil {
+ logger.Warn("fail to get data engine RealPos, progress may not be accurate",
+ log.ShortError(realOffsetErr), zap.String("file", cr.chunk.FileMeta.Path))
+ }
+ }
switch errors.Cause(err) {
case nil:
@@ -2676,7 +2724,8 @@ func (cr *chunkRestore) encodeLoop(
continue
}
- kvPacket = append(kvPacket, deliveredKVs{kvs: kvs, columns: filteredColumns, offset: newOffset, rowID: rowID})
+ kvPacket = append(kvPacket, deliveredKVs{kvs: kvs, columns: filteredColumns, offset: newOffset,
+ rowID: rowID, realOffset: realOffset})
kvSize += kvs.Size()
failpoint.Inject("mock-kv-size", func(val failpoint.Value) {
kvSize += uint64(val.(int))
@@ -2708,7 +2757,7 @@ func (cr *chunkRestore) encodeLoop(
}
}
- err = send([]deliveredKVs{{offset: cr.chunk.Chunk.EndOffset}})
+ err = send([]deliveredKVs{{offset: cr.chunk.Chunk.EndOffset, realOffset: cr.chunk.FileMeta.FileSize}})
return
}
@@ -2771,3 +2820,20 @@ func (cr *chunkRestore) restore(
}
return errors.Trace(firstErr(encodeErr, deliverErr))
}
+
+func openReader(ctx context.Context, fileMeta mydump.SourceFileMeta, store storage.ExternalStorage) (
+ reader storage.ReadSeekCloser, err error) {
+ switch {
+ case fileMeta.Type == mydump.SourceTypeParquet:
+ reader, err = mydump.OpenParquetReader(ctx, store, fileMeta.Path, fileMeta.FileSize)
+ case fileMeta.Compression != mydump.CompressionNone:
+ compressType, err2 := mydump.ToStorageCompressType(fileMeta.Compression)
+ if err2 != nil {
+ return nil, err2
+ }
+ reader, err = storage.WithCompression(store, compressType).Open(ctx, fileMeta.Path)
+ default:
+ reader, err = store.Open(ctx, fileMeta.Path)
+ }
+ return
+}
diff --git a/br/pkg/lightning/restore/table_restore.go b/br/pkg/lightning/restore/table_restore.go
index 11038d62195ea..a3562cb436a5c 100644
--- a/br/pkg/lightning/restore/table_restore.go
+++ b/br/pkg/lightning/restore/table_restore.go
@@ -235,10 +235,12 @@ func (tr *TableRestore) restoreEngines(pCtx context.Context, rc *Controller, cp
// data-engines that need to be restore or import. Otherwise, all data-engines should
// be finished already.
+ handleDataEngineThisRun := false
idxEngineCfg := &backend.EngineConfig{
TableInfo: tr.tableInfo,
}
if indexEngineCp.Status < checkpoints.CheckpointStatusClosed {
+ handleDataEngineThisRun = true
indexWorker := rc.indexWorkers.Apply()
defer rc.indexWorkers.Recycle(indexWorker)
@@ -248,7 +250,7 @@ func (tr *TableRestore) restoreEngines(pCtx context.Context, rc *Controller, cp
if !common.TableHasAutoRowID(tr.tableInfo.Core) {
idxCnt--
}
- threshold := estimateCompactionThreshold(cp, int64(idxCnt))
+ threshold := estimateCompactionThreshold(tr.tableMeta.DataFiles, cp, int64(idxCnt))
idxEngineCfg.Local = &backend.LocalEngineConfig{
Compact: threshold > 0,
CompactConcurrency: 4,
@@ -327,9 +329,9 @@ func (tr *TableRestore) restoreEngines(pCtx context.Context, rc *Controller, cp
dataWorker := rc.closedEngineLimit.Apply()
defer rc.closedEngineLimit.Recycle(dataWorker)
err = tr.importEngine(ctx, dataClosedEngine, rc, eid, ecp)
- if rc.status != nil {
+ if rc.status != nil && rc.status.backend == config.BackendLocal {
for _, chunk := range ecp.Chunks {
- rc.status.FinishedFileSize.Add(chunk.Chunk.EndOffset - chunk.Key.Offset)
+ rc.status.FinishedFileSize.Add(chunk.TotalSize())
}
}
}
@@ -339,7 +341,7 @@ func (tr *TableRestore) restoreEngines(pCtx context.Context, rc *Controller, cp
}(restoreWorker, engineID, engine)
} else {
for _, chunk := range engine.Chunks {
- rc.status.FinishedFileSize.Add(chunk.Chunk.EndOffset - chunk.Key.Offset)
+ rc.status.FinishedFileSize.Add(chunk.TotalSize())
}
}
}
@@ -370,11 +372,31 @@ func (tr *TableRestore) restoreEngines(pCtx context.Context, rc *Controller, cp
return errors.Trace(restoreErr)
}
+ // if data engine is handled in previous run and we continue importing from checkpoint
+ if !handleDataEngineThisRun {
+ for _, engine := range cp.Engines {
+ for _, chunk := range engine.Chunks {
+ rc.status.FinishedFileSize.Add(chunk.Chunk.EndOffset - chunk.Key.Offset)
+ }
+ }
+ }
+
if cp.Status < checkpoints.CheckpointStatusIndexImported {
var err error
if indexEngineCp.Status < checkpoints.CheckpointStatusImported {
+ failpoint.Inject("FailBeforeStartImportingIndexEngine", func() {
+ errMsg := "fail before importing index KV data"
+ tr.logger.Warn(errMsg)
+ failpoint.Return(errors.New(errMsg))
+ })
err = tr.importKV(ctx, closedIndexEngine, rc, indexEngineID)
failpoint.Inject("FailBeforeIndexEngineImported", func() {
+ finished := rc.status.FinishedFileSize.Load()
+ total := rc.status.TotalFileSize.Load()
+ tr.logger.Warn("print lightning status",
+ zap.Int64("finished", finished),
+ zap.Int64("total", total),
+ zap.Bool("equal", finished == total))
panic("forcing failure due to FailBeforeIndexEngineImported")
})
}
@@ -406,6 +428,11 @@ func (tr *TableRestore) restoreEngine(
if err != nil {
return closedEngine, errors.Trace(err)
}
+ if rc.status != nil && rc.status.backend == config.BackendTiDB {
+ for _, chunk := range cp.Chunks {
+ rc.status.FinishedFileSize.Add(chunk.Chunk.EndOffset - chunk.Key.Offset)
+ }
+ }
return closedEngine, nil
}
@@ -474,7 +501,11 @@ func (tr *TableRestore) restoreEngine(
metrics, _ := metric.FromContext(ctx)
// Restore table data
+ChunkLoop:
for chunkIndex, chunk := range cp.Chunks {
+ if rc.status != nil && rc.status.backend == config.BackendTiDB {
+ rc.status.FinishedFileSize.Add(chunk.Chunk.Offset - chunk.Key.Offset)
+ }
if chunk.Chunk.Offset >= chunk.Chunk.EndOffset {
continue
}
@@ -494,9 +525,15 @@ func (tr *TableRestore) restoreEngine(
}
checkFlushLock.Unlock()
+ failpoint.Inject("orphanWriterGoRoutine", func() {
+ if chunkIndex > 0 {
+ <-pCtx.Done()
+ }
+ })
+
select {
case <-pCtx.Done():
- return nil, pCtx.Err()
+ break ChunkLoop
default:
}
@@ -516,7 +553,7 @@ func (tr *TableRestore) restoreEngine(
}
var remainChunkCnt float64
if chunk.Chunk.Offset < chunk.Chunk.EndOffset {
- remainChunkCnt = float64(chunk.Chunk.EndOffset-chunk.Chunk.Offset) / float64(chunk.Chunk.EndOffset-chunk.Key.Offset)
+ remainChunkCnt = float64(chunk.UnfinishedSize()) / float64(chunk.TotalSize())
if metrics != nil {
metrics.ChunkCounter.WithLabelValues(metric.ChunkStatePending).Add(remainChunkCnt)
}
@@ -585,13 +622,18 @@ func (tr *TableRestore) restoreEngine(
}
wg.Wait()
+ select {
+ case <-pCtx.Done():
+ return nil, pCtx.Err()
+ default:
+ }
// Report some statistics into the log for debugging.
totalKVSize := uint64(0)
totalSQLSize := int64(0)
for _, chunk := range cp.Chunks {
totalKVSize += chunk.Checksum.SumSize()
- totalSQLSize += chunk.Chunk.EndOffset - chunk.Chunk.Offset
+ totalSQLSize += chunk.UnfinishedSize()
}
err = chunkErr.Get()
@@ -675,7 +717,7 @@ func (tr *TableRestore) importEngine(
}
// 2. perform a level-1 compact if idling.
- if rc.cfg.PostRestore.Level1Compact && rc.compactState.CAS(compactStateIdle, compactStateDoing) {
+ if rc.cfg.PostRestore.Level1Compact && rc.compactState.CompareAndSwap(compactStateIdle, compactStateDoing) {
go func() {
// we ignore level-1 compact failure since it is not fatal.
// no need log the error, it is done in (*Importer).Compact already.
@@ -764,11 +806,19 @@ func (tr *TableRestore) postProcess(
}
hasDupe = hasLocalDupe
}
+ failpoint.Inject("SlowDownCheckDupe", func(v failpoint.Value) {
+ sec := v.(int)
+ tr.logger.Warn("start to sleep several seconds before checking other dupe",
+ zap.Int("seconds", sec))
+ time.Sleep(time.Duration(sec) * time.Second)
+ })
- needChecksum, needRemoteDupe, baseTotalChecksum, err := metaMgr.CheckAndUpdateLocalChecksum(ctx, &localChecksum, hasDupe)
+ otherHasDupe, needRemoteDupe, baseTotalChecksum, err := metaMgr.CheckAndUpdateLocalChecksum(ctx, &localChecksum, hasDupe)
if err != nil {
return false, err
}
+ needChecksum := !otherHasDupe && needRemoteDupe
+ hasDupe = hasDupe || otherHasDupe
if needRemoteDupe && rc.cfg.TikvImporter.DuplicateResolution != config.DupeResAlgNone {
opts := &kv.SessionOptions{
@@ -782,12 +832,19 @@ func (tr *TableRestore) postProcess(
}
hasDupe = hasDupe || hasRemoteDupe
- if err = rc.backend.ResolveDuplicateRows(ctx, tr.encTable, tr.tableName, rc.cfg.TikvImporter.DuplicateResolution); err != nil {
- tr.logger.Error("resolve remote duplicate keys failed", log.ShortError(err))
- return false, err
+ if hasDupe {
+ if err = rc.backend.ResolveDuplicateRows(ctx, tr.encTable, tr.tableName, rc.cfg.TikvImporter.DuplicateResolution); err != nil {
+ tr.logger.Error("resolve remote duplicate keys failed", log.ShortError(err))
+ return false, err
+ }
}
}
+ if rc.dupIndicator != nil {
+ tr.logger.Debug("set dupIndicator", zap.Bool("has-duplicate", hasDupe))
+ rc.dupIndicator.CompareAndSwap(false, hasDupe)
+ }
+
nextStage := checkpoints.CheckpointStatusChecksummed
if rc.cfg.PostRestore.Checksum != config.OpLevelOff && !hasDupe && needChecksum {
if cp.Checksum.SumKVS() > 0 || baseTotalChecksum.SumKVS() > 0 {
@@ -1015,15 +1072,23 @@ func (tr *TableRestore) analyzeTable(ctx context.Context, g glue.SQLExecutor) er
// Try to limit the total SST files number under 500. But size compress 32GB SST files cost about 20min,
// we set the upper bound to 32GB to avoid too long compression time.
// factor is the non-clustered(1 for data engine and number of non-clustered index count for index engine).
-func estimateCompactionThreshold(cp *checkpoints.TableCheckpoint, factor int64) int64 {
+func estimateCompactionThreshold(files []mydump.FileInfo, cp *checkpoints.TableCheckpoint, factor int64) int64 {
totalRawFileSize := int64(0)
var lastFile string
+ fileSizeMap := make(map[string]int64, len(files))
+ for _, file := range files {
+ fileSizeMap[file.FileMeta.Path] = file.FileMeta.RealSize
+ }
+
for _, engineCp := range cp.Engines {
for _, chunk := range engineCp.Chunks {
if chunk.FileMeta.Path == lastFile {
continue
}
- size := chunk.FileMeta.FileSize
+ size, ok := fileSizeMap[chunk.FileMeta.Path]
+ if !ok {
+ size = chunk.FileMeta.FileSize
+ }
if chunk.FileMeta.Type == mydump.SourceTypeParquet {
// parquet file is compressed, thus estimates with a factor of 2
size *= 2
diff --git a/br/pkg/lightning/restore/table_restore_test.go b/br/pkg/lightning/restore/table_restore_test.go
index 17fb97e346e36..0f6d87892e329 100644
--- a/br/pkg/lightning/restore/table_restore_test.go
+++ b/br/pkg/lightning/restore/table_restore_test.go
@@ -129,6 +129,7 @@ func (s *tableRestoreSuiteBase) setupSuite(t *testing.T) {
Type: mydump.SourceTypeSQL,
SortKey: strconv.Itoa(i),
FileSize: 37,
+ RealSize: 37,
},
})
}
@@ -144,6 +145,7 @@ func (s *tableRestoreSuiteBase) setupSuite(t *testing.T) {
Type: mydump.SourceTypeCSV,
SortKey: "99",
FileSize: 14,
+ RealSize: 14,
},
})
@@ -304,6 +306,7 @@ func (s *tableRestoreSuite) TestPopulateChunks() {
// set csv header to true, this will cause check columns fail
s.cfg.Mydumper.CSV.Header = true
+ s.cfg.Mydumper.CSV.HeaderSchemaMatch = true
s.cfg.Mydumper.StrictFormat = true
regionSize := s.cfg.Mydumper.MaxRegionSize
s.cfg.Mydumper.MaxRegionSize = 5
@@ -427,7 +430,7 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader() {
require.NoError(s.T(), err)
fakeDataFiles = append(fakeDataFiles, mydump.FileInfo{
TableName: filter.Table{Schema: "db", Name: "table"},
- FileMeta: mydump.SourceFileMeta{Path: csvName, Type: mydump.SourceTypeCSV, SortKey: fmt.Sprintf("%02d", i), FileSize: int64(len(str))},
+ FileMeta: mydump.SourceFileMeta{Path: csvName, Type: mydump.SourceTypeCSV, SortKey: fmt.Sprintf("%02d", i), FileSize: int64(len(str)), RealSize: int64(len(str))},
})
total += len(str)
}
@@ -453,6 +456,7 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader() {
cfg.Mydumper.MaxRegionSize = 40
cfg.Mydumper.CSV.Header = true
+ cfg.Mydumper.CSV.HeaderSchemaMatch = true
cfg.Mydumper.StrictFormat = true
rc := &Controller{cfg: cfg, ioWorkers: worker.NewPool(context.Background(), 1, "io"), store: store}
@@ -1349,6 +1353,7 @@ func (s *tableRestoreSuite) TestCheckHasLargeCSV() {
{
FileMeta: mydump.SourceFileMeta{
FileSize: 1 * units.TiB,
+ RealSize: 1 * units.TiB,
Path: "/testPath",
},
},
@@ -2132,13 +2137,14 @@ func (s *tableRestoreSuite) TestSchemaIsValid() {
Mydumper: config.MydumperRuntime{
ReadBlockSize: config.ReadBlockSize,
CSV: config.CSVConfig{
- Separator: ",",
- Delimiter: `"`,
- Header: ca.hasHeader,
- NotNull: false,
- Null: `\N`,
- BackslashEscape: true,
- TrimLastSep: false,
+ Separator: ",",
+ Delimiter: `"`,
+ Header: ca.hasHeader,
+ HeaderSchemaMatch: true,
+ NotNull: false,
+ Null: `\N`,
+ BackslashEscape: true,
+ TrimLastSep: false,
},
IgnoreColumns: ca.ignoreColumns,
},
@@ -2167,13 +2173,14 @@ func (s *tableRestoreSuite) TestGBKEncodedSchemaIsValid() {
DataCharacterSet: "gb18030",
DataInvalidCharReplace: string(utf8.RuneError),
CSV: config.CSVConfig{
- Separator: "，",
- Delimiter: `"`,
- Header: true,
- NotNull: false,
- Null: `\N`,
- BackslashEscape: true,
- TrimLastSep: false,
+ Separator: "，",
+ Delimiter: `"`,
+ Header: true,
+ HeaderSchemaMatch: true,
+ NotNull: false,
+ Null: `\N`,
+ BackslashEscape: true,
+ TrimLastSep: false,
},
IgnoreColumns: nil,
},
diff --git a/br/pkg/lightning/restore/tidb.go b/br/pkg/lightning/restore/tidb.go
index 0e114bc035a56..98c780e65dc98 100644
--- a/br/pkg/lightning/restore/tidb.go
+++ b/br/pkg/lightning/restore/tidb.go
@@ -66,13 +66,15 @@ type TiDBManager struct {
func DBFromConfig(ctx context.Context, dsn config.DBStore) (*sql.DB, error) {
param := common.MySQLConnectParam{
- Host: dsn.Host,
- Port: dsn.Port,
- User: dsn.User,
- Password: dsn.Psw,
- SQLMode: dsn.StrSQLMode,
- MaxAllowedPacket: dsn.MaxAllowedPacket,
- TLS: dsn.TLS,
+ Host: dsn.Host,
+ Port: dsn.Port,
+ User: dsn.User,
+ Password: dsn.Psw,
+ SQLMode: dsn.StrSQLMode,
+ MaxAllowedPacket: dsn.MaxAllowedPacket,
+ TLSConfig: dsn.Security.TLSConfig,
+ AllowFallbackToPlaintext: dsn.Security.AllowFallbackToPlaintext,
+ Net: dsn.UUID,
}
db, err := param.Connect()
@@ -93,8 +95,10 @@ func DBFromConfig(ctx context.Context, dsn config.DBStore) (*sql.DB, error) {
"tidb_opt_write_row_id": "1",
// always set auto-commit to ON
"autocommit": "1",
- // alway set transaction mode to optimistic
+ // always set transaction mode to optimistic
"tidb_txn_mode": "optimistic",
+ // disable foreign key checks
+ "foreign_key_checks": "0",
}
if dsn.Vars != nil {
@@ -141,47 +145,6 @@ func (timgr *TiDBManager) Close() {
timgr.db.Close()
}
-func InitSchema(ctx context.Context, g glue.Glue, database string, tablesSchema map[string]string) error {
- logger := log.FromContext(ctx).With(zap.String("db", database))
- sqlExecutor := g.GetSQLExecutor()
-
- var createDatabase strings.Builder
- createDatabase.WriteString("CREATE DATABASE IF NOT EXISTS ")
- common.WriteMySQLIdentifier(&createDatabase, database)
- err := sqlExecutor.ExecuteWithLog(ctx, createDatabase.String(), "create database", logger)
- if err != nil {
- return errors.Trace(err)
- }
-
- task := logger.Begin(zap.InfoLevel, "create tables")
- var sqlCreateStmts []string
-loopCreate:
- for tbl, sqlCreateTable := range tablesSchema {
- task.Debug("create table", zap.String("schema", sqlCreateTable))
-
- sqlCreateStmts, err = createIfNotExistsStmt(g.GetParser(), sqlCreateTable, database, tbl)
- if err != nil {
- break
- }
-
- // TODO: maybe we should put these createStems into a transaction
- for _, s := range sqlCreateStmts {
- err = sqlExecutor.ExecuteWithLog(
- ctx,
- s,
- "create table",
- logger.With(zap.String("table", common.UniqueTable(database, tbl))),
- )
- if err != nil {
- break loopCreate
- }
- }
- }
- task.End(zap.ErrorLevel, err)
-
- return errors.Trace(err)
-}
-
func createIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName string) ([]string, error) {
stmts, _, err := p.ParseSQL(createTable)
if err != nil {
@@ -189,7 +152,7 @@ func createIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName string
}
var res strings.Builder
- ctx := format.NewRestoreCtx(format.DefaultRestoreFlags|format.RestoreTiDBSpecialComment, &res)
+ ctx := format.NewRestoreCtx(format.DefaultRestoreFlags|format.RestoreTiDBSpecialComment|format.RestoreWithTTLEnableOff, &res)
retStmts := make([]string, 0, len(stmts))
for _, stmt := range stmts {
@@ -197,6 +160,9 @@ func createIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName string
case *ast.CreateDatabaseStmt:
node.Name = model.NewCIStr(dbName)
node.IfNotExists = true
+ case *ast.DropDatabaseStmt:
+ node.Name = model.NewCIStr(dbName)
+ node.IfExists = true
case *ast.CreateTableStmt:
node.Table.Schema = model.NewCIStr(dbName)
node.Table.Name = model.NewCIStr(tblName)
diff --git a/br/pkg/lightning/restore/tidb_test.go b/br/pkg/lightning/restore/tidb_test.go
index 9b204b2da22b1..a3710d822d2dd 100644
--- a/br/pkg/lightning/restore/tidb_test.go
+++ b/br/pkg/lightning/restore/tidb_test.go
@@ -165,97 +165,6 @@ func TestCreateTableIfNotExistsStmt(t *testing.T) {
`, "m"))
}
-func TestInitSchema(t *testing.T) {
- s := newTiDBSuite(t)
- ctx := context.Background()
-
- s.mockDB.
- ExpectExec("CREATE DATABASE IF NOT EXISTS `db`").
- WillReturnResult(sqlmock.NewResult(1, 1))
- s.mockDB.
- ExpectExec("\\QCREATE TABLE IF NOT EXISTS `db`.`t1` (`a` INT PRIMARY KEY,`b` VARCHAR(200));\\E").
- WillReturnResult(sqlmock.NewResult(2, 1))
- s.mockDB.
- ExpectExec("\\QSET @@SESSION.`FOREIGN_KEY_CHECKS`=0;\\E").
- WillReturnResult(sqlmock.NewResult(0, 0))
- s.mockDB.
- ExpectExec("\\QCREATE TABLE IF NOT EXISTS `db`.`t2` (`xx` TEXT) AUTO_INCREMENT = 11203;\\E").
- WillReturnResult(sqlmock.NewResult(2, 1))
- s.mockDB.
- ExpectClose()
-
- s.mockDB.MatchExpectationsInOrder(false) // maps are unordered.
- err := InitSchema(ctx, s.tiGlue, "db", map[string]string{
- "t1": "create table t1 (a int primary key, b varchar(200));",
- "t2": "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;CREATE TABLE `db`.`t2` (xx TEXT) AUTO_INCREMENT=11203;",
- })
- s.mockDB.MatchExpectationsInOrder(true)
- require.NoError(t, err)
-}
-
-func TestInitSchemaSyntaxError(t *testing.T) {
- s := newTiDBSuite(t)
- ctx := context.Background()
-
- s.mockDB.
- ExpectExec("CREATE DATABASE IF NOT EXISTS `db`").
- WillReturnResult(sqlmock.NewResult(1, 1))
- s.mockDB.
- ExpectClose()
-
- err := InitSchema(ctx, s.tiGlue, "db", map[string]string{
- "t1": "create table `t1` with invalid syntax;",
- })
- require.Error(t, err)
-}
-
-func TestInitSchemaErrorLost(t *testing.T) {
- s := newTiDBSuite(t)
- ctx := context.Background()
-
- s.mockDB.
- ExpectExec("CREATE DATABASE IF NOT EXISTS `db`").
- WillReturnResult(sqlmock.NewResult(1, 1))
-
- s.mockDB.
- ExpectExec("CREATE TABLE IF NOT EXISTS.*").
- WillReturnError(&mysql.MySQLError{
- Number: tmysql.ErrTooBigFieldlength,
- Message: "Column length too big",
- })
-
- s.mockDB.
- ExpectClose()
-
- err := InitSchema(ctx, s.tiGlue, "db", map[string]string{
- "t1": "create table `t1` (a int);",
- "t2": "create table t2 (a int primary key, b varchar(200));",
- })
- require.Regexp(t, ".*Column length too big.*", err.Error())
-}
-
-func TestInitSchemaUnsupportedSchemaError(t *testing.T) {
- s := newTiDBSuite(t)
- ctx := context.Background()
-
- s.mockDB.
- ExpectExec("CREATE DATABASE IF NOT EXISTS `db`").
- WillReturnResult(sqlmock.NewResult(1, 1))
- s.mockDB.
- ExpectExec("CREATE TABLE IF NOT EXISTS `db`.`t1`.*").
- WillReturnError(&mysql.MySQLError{
- Number: tmysql.ErrTooBigFieldlength,
- Message: "Column length too big",
- })
- s.mockDB.
- ExpectClose()
-
- err := InitSchema(ctx, s.tiGlue, "db", map[string]string{
- "t1": "create table `t1` (a VARCHAR(999999999));",
- })
- require.Regexp(t, ".*Column length too big.*", err.Error())
-}
-
func TestDropTable(t *testing.T) {
s := newTiDBSuite(t)
ctx := context.Background()
diff --git a/br/pkg/lightning/run_options.go b/br/pkg/lightning/run_options.go
index a7b5b90770c02..169c2e47088dd 100644
--- a/br/pkg/lightning/run_options.go
+++ b/br/pkg/lightning/run_options.go
@@ -19,6 +19,7 @@ import (
"github.com/pingcap/tidb/br/pkg/lightning/log"
"github.com/pingcap/tidb/br/pkg/storage"
"github.com/pingcap/tidb/util/promutil"
+ "go.uber.org/atomic"
"go.uber.org/zap"
)
@@ -30,6 +31,7 @@ type options struct {
promFactory promutil.Factory
promRegistry promutil.Registry
logger log.Logger
+ dupIndicator *atomic.Bool
}
type Option func(*options)
@@ -81,3 +83,10 @@ func WithLogger(logger *zap.Logger) Option {
o.logger = log.Logger{Logger: logger}
}
}
+
+// WithDupIndicator sets a *bool to indicate duplicate detection has found duplicate data.
+func WithDupIndicator(b *atomic.Bool) Option {
+ return func(o *options) {
+ o.dupIndicator = b
+ }
+}
diff --git a/br/pkg/lightning/tikv/BUILD.bazel b/br/pkg/lightning/tikv/BUILD.bazel
index 596aa52075758..48758bfedaacf 100644
--- a/br/pkg/lightning/tikv/BUILD.bazel
+++ b/br/pkg/lightning/tikv/BUILD.bazel
@@ -7,6 +7,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//br/pkg/lightning/common",
+ "//br/pkg/lightning/config",
"//br/pkg/lightning/log",
"//br/pkg/pdutil",
"//br/pkg/version",
diff --git a/br/pkg/lightning/tikv/tikv.go b/br/pkg/lightning/tikv/tikv.go
index 8d2d797d322d1..53c06cc6102f6 100644
--- a/br/pkg/lightning/tikv/tikv.go
+++ b/br/pkg/lightning/tikv/tikv.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/kvproto/pkg/debugpb"
"github.com/pingcap/kvproto/pkg/import_sstpb"
"github.com/pingcap/tidb/br/pkg/lightning/common"
+ "github.com/pingcap/tidb/br/pkg/lightning/config"
"github.com/pingcap/tidb/br/pkg/lightning/log"
"github.com/pingcap/tidb/br/pkg/pdutil"
"github.com/pingcap/tidb/br/pkg/version"
@@ -88,7 +89,7 @@ func withTiKVConnection(ctx context.Context, tls *common.TLS, tikvAddr string, a
// Connect to the ImportSST service on the given TiKV node.
// The connection is needed for executing `action` and will be tear down
// when this function exits.
- conn, err := grpc.DialContext(ctx, tikvAddr, tls.ToGRPCDialOption())
+ conn, err := grpc.DialContext(ctx, tikvAddr, tls.ToGRPCDialOption(), config.DefaultGrpcKeepaliveParams)
if err != nil {
return errors.Trace(err)
}
@@ -172,7 +173,8 @@ var fetchModeRegexp = regexp.MustCompile(`\btikv_config_rocksdb\{cf="default",na
// FetchMode obtains the import mode status of the TiKV node.
func FetchMode(ctx context.Context, tls *common.TLS, tikvAddr string) (import_sstpb.SwitchMode, error) {
- conn, err := grpc.DialContext(ctx, tikvAddr, tls.ToGRPCDialOption())
+ conn, err := grpc.DialContext(ctx, tikvAddr, tls.ToGRPCDialOption(),
+ config.DefaultGrpcKeepaliveParams)
if err != nil {
return 0, err
}
diff --git a/br/pkg/lightning/verification/BUILD.bazel b/br/pkg/lightning/verification/BUILD.bazel
index 9f308aeb81e11..122e4b0e8535f 100644
--- a/br/pkg/lightning/verification/BUILD.bazel
+++ b/br/pkg/lightning/verification/BUILD.bazel
@@ -7,6 +7,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//br/pkg/lightning/common",
+ "@com_github_tikv_client_go_v2//tikv",
"@org_uber_go_zap//zapcore",
],
)
diff --git a/br/pkg/lightning/verification/checksum.go b/br/pkg/lightning/verification/checksum.go
index 0a44177f80fe4..9eb2130074eec 100644
--- a/br/pkg/lightning/verification/checksum.go
+++ b/br/pkg/lightning/verification/checksum.go
@@ -19,15 +19,18 @@ import (
"hash/crc64"
"github.com/pingcap/tidb/br/pkg/lightning/common"
+ "github.com/tikv/client-go/v2/tikv"
"go.uber.org/zap/zapcore"
)
var ecmaTable = crc64.MakeTable(crc64.ECMA)
type KVChecksum struct {
- bytes uint64
- kvs uint64
- checksum uint64
+ base uint64
+ prefixLen int
+ bytes uint64
+ kvs uint64
+ checksum uint64
}
func NewKVChecksum(checksum uint64) *KVChecksum {
@@ -36,6 +39,14 @@ func NewKVChecksum(checksum uint64) *KVChecksum {
}
}
+func NewKVChecksumWithKeyspace(k tikv.Codec) *KVChecksum {
+ ks := k.GetKeyspace()
+ return &KVChecksum{
+ base: crc64.Update(0, ecmaTable, ks),
+ prefixLen: len(ks),
+ }
+}
+
func MakeKVChecksum(bytes uint64, kvs uint64, checksum uint64) KVChecksum {
return KVChecksum{
bytes: bytes,
@@ -45,10 +56,10 @@ func MakeKVChecksum(bytes uint64, kvs uint64, checksum uint64) KVChecksum {
}
func (c *KVChecksum) UpdateOne(kv common.KvPair) {
- sum := crc64.Update(0, ecmaTable, kv.Key)
+ sum := crc64.Update(c.base, ecmaTable, kv.Key)
sum = crc64.Update(sum, ecmaTable, kv.Val)
- c.bytes += uint64(len(kv.Key) + len(kv.Val))
+ c.bytes += uint64(c.prefixLen + len(kv.Key) + len(kv.Val))
c.kvs++
c.checksum ^= sum
}
@@ -62,11 +73,12 @@ func (c *KVChecksum) Update(kvs []common.KvPair) {
)
for _, pair := range kvs {
- sum = crc64.Update(0, ecmaTable, pair.Key)
+ sum = crc64.Update(c.base, ecmaTable, pair.Key)
sum = crc64.Update(sum, ecmaTable, pair.Val)
checksum ^= sum
kvNum++
- bytes += (len(pair.Key) + len(pair.Val))
+ bytes += c.prefixLen
+ bytes += len(pair.Key) + len(pair.Val)
}
c.bytes += uint64(bytes)
diff --git a/br/pkg/lightning/web/BUILD.bazel b/br/pkg/lightning/web/BUILD.bazel
index 842eb48fb3dd3..93cb28cdaf0ce 100644
--- a/br/pkg/lightning/web/BUILD.bazel
+++ b/br/pkg/lightning/web/BUILD.bazel
@@ -4,7 +4,7 @@ go_library(
name = "web",
srcs = [
"progress.go",
- "res.go",
+ "res.go", #keep
"res_vfsdata.go",
],
importpath = "github.com/pingcap/tidb/br/pkg/lightning/web",
diff --git a/br/pkg/lightning/web/progress.go b/br/pkg/lightning/web/progress.go
index 8a3412087b94f..d5f3494a14040 100644
--- a/br/pkg/lightning/web/progress.go
+++ b/br/pkg/lightning/web/progress.go
@@ -64,7 +64,7 @@ func (cpm *checkpointsMap) update(diffs map[string]*checkpoints.TableCheckpointD
for _, engine := range cp.Engines {
for _, chunk := range engine.Chunks {
if engine.Status >= checkpoints.CheckpointStatusAllWritten {
- tw += chunk.Chunk.EndOffset - chunk.Key.Offset
+ tw += chunk.TotalSize()
} else {
tw += chunk.Chunk.Offset - chunk.Key.Offset
}
diff --git a/br/pkg/logutil/logging.go b/br/pkg/logutil/logging.go
index 028cfc00e5f43..41b8e135c220f 100644
--- a/br/pkg/logutil/logging.go
+++ b/br/pkg/logutil/logging.go
@@ -306,3 +306,13 @@ func (rng StringifyRange) String() string {
sb.WriteString(")")
return sb.String()
}
+
+// StringifyMany returns an array marshaler for a slice of stringers.
+func StringifyMany[T fmt.Stringer](items []T) zapcore.ArrayMarshaler {
+ return zapcore.ArrayMarshalerFunc(func(ae zapcore.ArrayEncoder) error {
+ for _, item := range items {
+ ae.AppendString(item.String())
+ }
+ return nil
+ })
+}
diff --git a/br/pkg/logutil/rate.go b/br/pkg/logutil/rate.go
index 12ef85732efd7..e3e67ed241bff 100644
--- a/br/pkg/logutil/rate.go
+++ b/br/pkg/logutil/rate.go
@@ -51,5 +51,5 @@ func (r *RateTracer) RateAt(instant time.Time) float64 {
// L make a logger with the current speed.
func (r *RateTracer) L() *zap.Logger {
- return log.With(zap.String("speed", fmt.Sprintf("%.2f ops/s", r.Rate())))
+ return log.L().With(zap.String("speed", fmt.Sprintf("%.2f ops/s", r.Rate())))
}
diff --git a/br/pkg/metautil/metafile.go b/br/pkg/metautil/metafile.go
index 77b3c4de8b6f6..76ac9a248ccf1 100644
--- a/br/pkg/metautil/metafile.go
+++ b/br/pkg/metautil/metafile.go
@@ -170,12 +170,12 @@ type MetaReader struct {
// NewMetaReader creates MetaReader.
func NewMetaReader(
- backpMeta *backuppb.BackupMeta,
+ backupMeta *backuppb.BackupMeta,
storage storage.ExternalStorage,
cipher *backuppb.CipherInfo) *MetaReader {
return &MetaReader{
storage: storage,
- backupMeta: backpMeta,
+ backupMeta: backupMeta,
cipher: cipher,
}
}
diff --git a/br/pkg/restore/BUILD.bazel b/br/pkg/restore/BUILD.bazel
index 2c42717bafe36..35bed1951e9b3 100644
--- a/br/pkg/restore/BUILD.bazel
+++ b/br/pkg/restore/BUILD.bazel
@@ -43,6 +43,7 @@ go_library(
"//br/pkg/summary",
"//br/pkg/utils",
"//br/pkg/utils/iter",
+ "//br/pkg/version",
"//config",
"//ddl",
"//ddl/util",
@@ -57,8 +58,10 @@ go_library(
"//tablecodec",
"//util",
"//util/codec",
+ "//util/collate",
"//util/hack",
"//util/mathutil",
+ "//util/sqlexec",
"//util/table-filter",
"@com_github_emirpasic_gods//maps/treemap",
"@com_github_go_sql_driver_mysql//:mysql",
@@ -85,6 +88,7 @@ go_library(
"@org_golang_google_grpc//backoff",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//credentials",
+ "@org_golang_google_grpc//credentials/insecure",
"@org_golang_google_grpc//keepalive",
"@org_golang_google_grpc//status",
"@org_golang_x_exp//slices",
@@ -113,12 +117,13 @@ go_test(
"search_test.go",
"split_test.go",
"stream_metas_test.go",
+ "systable_restore_test.go",
"util_test.go",
],
embed = [":restore"],
flaky = True,
race = "on",
- shard_count = 20,
+ shard_count = 50,
deps = [
"//br/pkg/backup",
"//br/pkg/conn",
diff --git a/br/pkg/restore/batcher_test.go b/br/pkg/restore/batcher_test.go
index 2c182cb0351cc..808b1868eed16 100644
--- a/br/pkg/restore/batcher_test.go
+++ b/br/pkg/restore/batcher_test.go
@@ -49,7 +49,7 @@ func (sender *drySender) Close() {
}
func waitForSend() {
- time.Sleep(10 * time.Millisecond)
+ time.Sleep(50 * time.Millisecond)
}
func (sender *drySender) Ranges() []rtree.Range {
diff --git a/br/pkg/restore/client.go b/br/pkg/restore/client.go
index 81215e675c8c0..a783fed862d29 100644
--- a/br/pkg/restore/client.go
+++ b/br/pkg/restore/client.go
@@ -20,6 +20,7 @@ import (
"github.com/pingcap/failpoint"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/import_sstpb"
+ "github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/log"
"github.com/pingcap/tidb/br/pkg/backup"
@@ -39,6 +40,7 @@ import (
"github.com/pingcap/tidb/br/pkg/stream"
"github.com/pingcap/tidb/br/pkg/summary"
"github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/version"
"github.com/pingcap/tidb/config"
ddlutil "github.com/pingcap/tidb/ddl/util"
"github.com/pingcap/tidb/domain"
@@ -50,7 +52,9 @@ import (
"github.com/pingcap/tidb/store/pdtypes"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/util/codec"
+ "github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/mathutil"
+ "github.com/pingcap/tidb/util/sqlexec"
filter "github.com/pingcap/tidb/util/table-filter"
"github.com/tikv/client-go/v2/oracle"
pd "github.com/tikv/pd/client"
@@ -61,6 +65,7 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/backoff"
"google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/keepalive"
)
@@ -165,6 +170,9 @@ type Client struct {
// the successfully preallocated table IDs.
preallocedTableIDs *tidalloc.PreallocIDs
+
+ // the rewrite mode of the downloaded SST files in TiKV.
+ rewriteMode RewriteMode
}
// NewRestoreClient returns a new RestoreClient.
@@ -313,6 +321,14 @@ func (rc *Client) GetBatchDdlSize() uint {
return rc.batchDdlSize
}
+func (rc *Client) SetRewriteMode(mode RewriteMode) {
+ rc.rewriteMode = mode
+}
+
+func (rc *Client) GetRewriteMode() RewriteMode {
+ return rc.rewriteMode
+}
+
// Close a client.
func (rc *Client) Close() {
// rc.db can be nil in raw kv mode.
@@ -342,7 +358,7 @@ func (rc *Client) SetStorage(ctx context.Context, backend *backuppb.StorageBacke
func (rc *Client) InitClients(backend *backuppb.StorageBackend, isRawKvMode bool) {
metaClient := split.NewSplitClient(rc.pdClient, rc.tlsConf, isRawKvMode)
importCli := NewImportClient(metaClient, rc.tlsConf, rc.keepaliveConf)
- rc.fileImporter = NewFileImporter(metaClient, importCli, backend, isRawKvMode)
+ rc.fileImporter = NewFileImporter(metaClient, importCli, backend, isRawKvMode, rc.rewriteMode)
}
func (rc *Client) SetRawKVClient(c *RawKVBatchClient) {
@@ -446,7 +462,7 @@ func (rc *Client) GetFilesInRawRange(startKey []byte, endKey []byte, cf string)
// SetConcurrency sets the concurrency of dbs tables files.
func (rc *Client) SetConcurrency(c uint) {
- log.Debug("new worker pool", zap.Uint("currency-count", c))
+ log.Info("new worker pool", zap.Uint("currency-count", c))
rc.workerPool = utils.NewWorkerPool(c, "file")
}
@@ -866,7 +882,7 @@ func (rc *Client) createTablesInWorkerPool(ctx context.Context, dom *domain.Doma
}
})
if err != nil {
- log.Error("create tables fail")
+ log.Error("create tables fail", zap.Error(err))
return err
}
for _, ct := range cts {
@@ -938,7 +954,9 @@ func (rc *Client) CheckSysTableCompatibility(dom *domain.Domain, tables []*metau
return errors.Annotate(berrors.ErrRestoreIncompatibleSys, "missed system table: "+table.Info.Name.O)
}
backupTi := table.Info
- if len(ti.Columns) != len(backupTi.Columns) {
+ // skip checking the number of columns in mysql.user table,
+ // because higher versions of TiDB may add new columns.
+ if len(ti.Columns) != len(backupTi.Columns) && backupTi.Name.L != sysUserTableName {
log.Error("column count mismatch",
zap.Stringer("table", table.Info.Name),
zap.Int("col in cluster", len(ti.Columns)),
@@ -957,6 +975,13 @@ func (rc *Client) CheckSysTableCompatibility(dom *domain.Domain, tables []*metau
col := ti.Columns[i]
backupCol := backupColMap[col.Name.L]
if backupCol == nil {
+ // skip when the backed up mysql.user table is missing columns.
+ if backupTi.Name.L == sysUserTableName {
+ log.Warn("missing column in backup data",
+ zap.Stringer("table", table.Info.Name),
+ zap.String("col", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())))
+ continue
+ }
log.Error("missing column in backup data",
zap.Stringer("table", table.Info.Name),
zap.String("col", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())))
@@ -977,6 +1002,29 @@ func (rc *Client) CheckSysTableCompatibility(dom *domain.Domain, tables []*metau
backupCol.Name, backupCol.FieldType.String())
}
}
+
+ if backupTi.Name.L == sysUserTableName {
+ // check whether the columns of table in cluster are less than the backup data
+ clusterColMap := make(map[string]*model.ColumnInfo)
+ for i := range ti.Columns {
+ col := ti.Columns[i]
+ clusterColMap[col.Name.L] = col
+ }
+ // order can be different
+ for i := range backupTi.Columns {
+ col := backupTi.Columns[i]
+ clusterCol := clusterColMap[col.Name.L]
+ if clusterCol == nil {
+ log.Error("missing column in cluster data",
+ zap.Stringer("table", table.Info.Name),
+ zap.String("col", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())))
+ return errors.Annotatef(berrors.ErrRestoreIncompatibleSys,
+ "missing column in cluster data, table: %s, col: %s %s",
+ table.Info.Name.O,
+ col.Name, col.FieldType.String())
+ }
+ }
+ }
}
return nil
}
@@ -1006,7 +1054,7 @@ func MockCallSetSpeedLimit(ctx context.Context, fakeImportClient ImporterClient,
rc.SetRateLimit(42)
rc.SetConcurrency(concurrency)
rc.hasSpeedLimited = false
- rc.fileImporter = NewFileImporter(nil, fakeImportClient, nil, false)
+ rc.fileImporter = NewFileImporter(nil, fakeImportClient, nil, false, RewriteModeLegacy)
return rc.setSpeedLimit(ctx, rc.rateLimit)
}
@@ -1092,6 +1140,18 @@ func (rc *Client) SplitRanges(ctx context.Context,
return SplitRanges(ctx, rc, ranges, rewriteRules, updateCh, isRawKv)
}
+func (rc *Client) WrapLogFilesIterWithSplitHelper(iter LogIter, rules map[int64]*RewriteRules, g glue.Glue, store kv.Storage) (LogIter, error) {
+ se, err := g.CreateSession(store)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ execCtx := se.GetSessionCtx().(sqlexec.RestrictedSQLExecutor)
+ splitSize, splitKeys := utils.GetRegionSplitInfo(execCtx)
+ log.Info("get split threshold from tikv config", zap.Uint64("split-size", splitSize), zap.Int64("split-keys", splitKeys))
+ client := split.NewSplitClient(rc.GetPDClient(), rc.GetTLSConfig(), false)
+ return NewLogFilesIterWithSplitHelper(iter, rules, client, splitSize, splitKeys), nil
+}
+
// RestoreSSTFiles tries to restore the files.
func (rc *Client) RestoreSSTFiles(
ctx context.Context,
@@ -1134,7 +1194,7 @@ func (rc *Client) RestoreSSTFiles(
zap.Duration("take", time.Since(fileStart)))
updateCh.Inc()
}()
- return rc.fileImporter.ImportSSTFiles(ectx, filesReplica, rewriteRules, rc.cipher, rc.backupMeta.ApiVersion)
+ return rc.fileImporter.ImportSSTFiles(ectx, filesReplica, rewriteRules, rc.cipher, rc.dom.Store().GetCodec().GetAPIVersion())
})
}
@@ -1251,7 +1311,7 @@ func (rc *Client) switchTiKVMode(ctx context.Context, mode import_sstpb.SwitchMo
finalStore := store
rc.workerPool.ApplyOnErrorGroup(eg,
func() error {
- opt := grpc.WithInsecure()
+ opt := grpc.WithTransportCredentials(insecure.NewCredentials())
if rc.tlsConf != nil {
opt = grpc.WithTransportCredentials(credentials.NewTLS(rc.tlsConf))
}
@@ -1363,7 +1423,7 @@ func (rc *Client) execChecksum(
concurrency uint,
loadStatCh chan<- *CreatedTable,
) error {
- logger := log.With(
+ logger := log.L().With(
zap.String("db", tbl.OldTable.DB.Name.O),
zap.String("table", tbl.OldTable.Info.Name.O),
)
@@ -1386,6 +1446,8 @@ func (rc *Client) execChecksum(
exe, err := checksum.NewExecutorBuilder(tbl.Table, startTS).
SetOldTable(tbl.OldTable).
SetConcurrency(concurrency).
+ SetOldKeyspace(tbl.RewriteRule.OldKeyspace).
+ SetNewKeyspace(tbl.RewriteRule.NewKeyspace).
Build()
if err != nil {
return errors.Trace(err)
@@ -1427,7 +1489,7 @@ func (rc *Client) updateMetaAndLoadStats(ctx context.Context, input <-chan *Crea
}
// Not need to return err when failed because of update analysis-meta
- restoreTS, err := rc.GetTS(ctx)
+ restoreTS, err := rc.GetTSWithRetry(ctx)
if err != nil {
log.Error("getTS failed", zap.Error(err))
} else {
@@ -1675,6 +1737,15 @@ func (rc *Client) PreCheckTableTiFlashReplica(
tables []*metautil.Table,
recorder *tiflashrec.TiFlashRecorder,
) error {
+ // For TiDB 6.6, we do not support recover TiFlash replica while enabling API V2.
+ // TODO(iosmanthus): remove this after TiFlash support API V2.
+ if rc.GetDomain().Store().GetCodec().GetAPIVersion() == kvrpcpb.APIVersion_V2 {
+ log.Warn("TiFlash does not support API V2, reset replica count to 0")
+ for _, table := range tables {
+ table.Info.TiFlashReplica = nil
+ }
+ return nil
+ }
tiFlashStoreCount, err := rc.getTiFlashNodeCount(ctx)
if err != nil {
return err
@@ -1807,19 +1878,179 @@ func (rc *Client) FixIndicesOfTable(ctx context.Context, schema string, table *m
return nil
}
+type FilesInRegion struct {
+ defaultSize uint64
+ defaultKVCount int64
+ writeSize uint64
+ writeKVCount int64
+
+ defaultFiles []*backuppb.DataFileInfo
+ writeFiles []*backuppb.DataFileInfo
+ deleteFiles []*backuppb.DataFileInfo
+}
+
+type FilesInTable struct {
+ regionMapFiles map[int64]*FilesInRegion
+}
+
+func ApplyKVFilesWithBatchMethod(
+ ctx context.Context,
+ iter LogIter,
+ batchCount int,
+ batchSize uint64,
+ applyFunc func(files []*backuppb.DataFileInfo, kvCount int64, size uint64),
+) error {
+ var (
+ tableMapFiles = make(map[int64]*FilesInTable)
+ tmpFiles = make([]*backuppb.DataFileInfo, 0, batchCount)
+ tmpSize uint64 = 0
+ tmpKVCount int64 = 0
+ )
+ for r := iter.TryNext(ctx); !r.Finished; r = iter.TryNext(ctx) {
+ if r.Err != nil {
+ return r.Err
+ }
+
+ f := r.Item
+ if f.GetType() == backuppb.FileType_Put && f.GetLength() >= batchSize {
+ applyFunc([]*backuppb.DataFileInfo{f}, f.GetNumberOfEntries(), f.GetLength())
+ continue
+ }
+
+ fit, exist := tableMapFiles[f.TableId]
+ if !exist {
+ fit = &FilesInTable{
+ regionMapFiles: make(map[int64]*FilesInRegion),
+ }
+ tableMapFiles[f.TableId] = fit
+ }
+ fs, exist := fit.regionMapFiles[f.RegionId]
+ if !exist {
+ fs = &FilesInRegion{}
+ fit.regionMapFiles[f.RegionId] = fs
+ }
+
+ if f.GetType() == backuppb.FileType_Delete {
+ if fs.defaultFiles == nil {
+ fs.deleteFiles = make([]*backuppb.DataFileInfo, 0)
+ }
+ fs.deleteFiles = append(fs.deleteFiles, f)
+ } else {
+ if f.GetCf() == stream.DefaultCF {
+ if fs.defaultFiles == nil {
+ fs.defaultFiles = make([]*backuppb.DataFileInfo, 0, batchCount)
+ }
+ fs.defaultFiles = append(fs.defaultFiles, f)
+ fs.defaultSize += f.Length
+ fs.defaultKVCount += f.GetNumberOfEntries()
+ if len(fs.defaultFiles) >= batchCount || fs.defaultSize >= batchSize {
+ applyFunc(fs.defaultFiles, fs.defaultKVCount, fs.defaultSize)
+ fs.defaultFiles = nil
+ fs.defaultSize = 0
+ fs.defaultKVCount = 0
+ }
+ } else {
+ if fs.writeFiles == nil {
+ fs.writeFiles = make([]*backuppb.DataFileInfo, 0, batchCount)
+ }
+ fs.writeFiles = append(fs.writeFiles, f)
+ fs.writeSize += f.GetLength()
+ fs.writeKVCount += f.GetNumberOfEntries()
+ if len(fs.writeFiles) >= batchCount || fs.writeSize >= batchSize {
+ applyFunc(fs.writeFiles, fs.writeKVCount, fs.writeSize)
+ fs.writeFiles = nil
+ fs.writeSize = 0
+ fs.writeKVCount = 0
+ }
+ }
+ }
+ }
+
+ for _, fwt := range tableMapFiles {
+ for _, fs := range fwt.regionMapFiles {
+ if len(fs.defaultFiles) > 0 {
+ applyFunc(fs.defaultFiles, fs.defaultKVCount, fs.defaultSize)
+ }
+ if len(fs.writeFiles) > 0 {
+ applyFunc(fs.writeFiles, fs.writeKVCount, fs.writeSize)
+ }
+ }
+ }
+
+ for _, fwt := range tableMapFiles {
+ for _, fs := range fwt.regionMapFiles {
+ for _, d := range fs.deleteFiles {
+ tmpFiles = append(tmpFiles, d)
+ tmpSize += d.GetLength()
+ tmpKVCount += d.GetNumberOfEntries()
+
+ if len(tmpFiles) >= batchCount || tmpSize >= batchSize {
+ applyFunc(tmpFiles, tmpKVCount, tmpSize)
+ tmpFiles = make([]*backuppb.DataFileInfo, 0, batchCount)
+ tmpSize = 0
+ tmpKVCount = 0
+ }
+ }
+ if len(tmpFiles) > 0 {
+ applyFunc(tmpFiles, tmpKVCount, tmpSize)
+ tmpFiles = make([]*backuppb.DataFileInfo, 0, batchCount)
+ tmpSize = 0
+ tmpKVCount = 0
+ }
+ }
+ }
+
+ return nil
+}
+
+func ApplyKVFilesWithSingelMethod(
+ ctx context.Context,
+ files LogIter,
+ applyFunc func(file []*backuppb.DataFileInfo, kvCount int64, size uint64),
+) error {
+ deleteKVFiles := make([]*backuppb.DataFileInfo, 0)
+
+ for r := files.TryNext(ctx); !r.Finished; r = files.TryNext(ctx) {
+ if r.Err != nil {
+ return r.Err
+ }
+
+ f := r.Item
+ if f.GetType() == backuppb.FileType_Delete {
+ deleteKVFiles = append(deleteKVFiles, f)
+ continue
+ }
+ applyFunc([]*backuppb.DataFileInfo{f}, f.GetNumberOfEntries(), f.GetLength())
+ }
+
+ log.Info("restore delete files", zap.Int("count", len(deleteKVFiles)))
+ for _, file := range deleteKVFiles {
+ f := file
+ applyFunc([]*backuppb.DataFileInfo{f}, f.GetNumberOfEntries(), f.GetLength())
+ }
+
+ return nil
+}
+
func (rc *Client) RestoreKVFiles(
ctx context.Context,
rules map[int64]*RewriteRules,
- files LogIter,
+ iter LogIter,
+ pitrBatchCount uint32,
+ pitrBatchSize uint32,
updateStats func(kvCount uint64, size uint64),
- onProgress func(),
+ onProgress func(cnt int64),
) error {
- var err error
- fileCount := 0
- start := time.Now()
+ var (
+ err error
+ fileCount = 0
+ start = time.Now()
+ supportBatch = version.CheckPITRSupportBatchKVFiles()
+ skipFile = 0
+ )
defer func() {
- elapsed := time.Since(start)
if err == nil {
+ elapsed := time.Since(start)
log.Info("Restore KV files", zap.Duration("take", elapsed))
summary.CollectSuccessUnit("files", fileCount, elapsed)
}
@@ -1832,74 +2063,65 @@ func (rc *Client) RestoreKVFiles(
}
eg, ectx := errgroup.WithContext(ctx)
- skipFile := 0
- deleteFiles := make([]*backuppb.DataFileInfo, 0)
-
- applyFunc := func(file *backuppb.DataFileInfo) {
- // get rewrite rule from table id
- rule, ok := rules[file.TableId]
+ applyFunc := func(files []*backuppb.DataFileInfo, kvCount int64, size uint64) {
+ if len(files) == 0 {
+ return
+ }
+ // get rewrite rule from table id.
+ // because the tableID of files is the same.
+ rule, ok := rules[files[0].TableId]
if !ok {
// TODO handle new created table
// For this version we do not handle new created table after full backup.
// in next version we will perform rewrite and restore meta key to restore new created tables.
// so we can simply skip the file that doesn't have the rule here.
- onProgress()
- summary.CollectInt("FileSkip", 1)
- log.Debug("skip file due to table id not matched", zap.String("file", file.Path), zap.Int64("tableId", file.TableId))
- skipFile++
+ onProgress(int64(len(files)))
+ summary.CollectInt("FileSkip", len(files))
+ log.Debug("skip file due to table id not matched", zap.Int64("table-id", files[0].TableId))
+ skipFile += len(files)
} else {
- rc.workerPool.ApplyOnErrorGroup(eg, func() error {
+ rc.workerPool.ApplyOnErrorGroup(eg, func() (err error) {
fileStart := time.Now()
defer func() {
- onProgress()
- updateStats(uint64(file.NumberOfEntries), file.Length)
- summary.CollectInt("File", 1)
- log.Info("import files done", zap.String("name", file.Path), zap.Duration("take", time.Since(fileStart)))
+ onProgress(int64(len(files)))
+ updateStats(uint64(kvCount), size)
+ summary.CollectInt("File", len(files))
+
+ if err == nil {
+ filenames := make([]string, 0, len(files))
+ for _, f := range files {
+ filenames = append(filenames, f.Path+", ")
+ }
+ log.Info("import files done", zap.Int("batch-count", len(files)), zap.Uint64("batch-size", size),
+ zap.Duration("take", time.Since(fileStart)), zap.Strings("files", filenames))
+ }
}()
- startTS := rc.startTS
- if file.Cf == stream.DefaultCF {
- startTS = rc.shiftStartTS
- }
- return rc.fileImporter.ImportKVFiles(ectx, file, rule, startTS, rc.restoreTS)
+
+ return rc.fileImporter.ImportKVFiles(ectx, files, rule, rc.shiftStartTS, rc.startTS, rc.restoreTS, supportBatch)
})
}
}
- for r := files.TryNext(ctx); !r.Finished; r = files.TryNext(ctx) {
- if r.Err != nil {
- return err
- }
- file := r.Item
- if file.Type == backuppb.FileType_Delete {
- // collect delete type file and apply it later.
- deleteFiles = append(deleteFiles, file)
- continue
+
+ rc.workerPool.ApplyOnErrorGroup(eg, func() error {
+ if supportBatch {
+ err = ApplyKVFilesWithBatchMethod(ectx, iter, int(pitrBatchCount), uint64(pitrBatchSize), applyFunc)
+ } else {
+ err = ApplyKVFilesWithSingelMethod(ectx, iter, applyFunc)
}
- fileReplica := file
- // applyFunc blocks once there aren't enough workers.
- // this would help us don't load too many DML file info.
- applyFunc(fileReplica)
- }
- if len(deleteFiles) > 0 {
- log.Info("restore delete files", zap.Int("count", len(deleteFiles)))
- }
- for _, file := range deleteFiles {
- fileReplica := file
- applyFunc(fileReplica)
+ return errors.Trace(err)
+ })
+
+ if err = eg.Wait(); err != nil {
+ summary.CollectFailureUnit("file", err)
+ log.Error("restore files failed", zap.Error(err))
}
+
log.Info("total skip files due to table id not matched", zap.Int("count", skipFile))
if skipFile > 0 {
log.Debug("table id in full backup storage", zap.Any("tables", rules))
}
- if err = eg.Wait(); err != nil {
- summary.CollectFailureUnit("file", err)
- log.Error(
- "restore files failed",
- zap.Error(err),
- )
- return errors.Trace(err)
- }
- return nil
+ return errors.Trace(err)
}
func (rc *Client) CleanUpKVFiles(
@@ -2399,7 +2621,7 @@ func (rc *Client) RunGCRowsLoader(ctx context.Context) {
func (rc *Client) InsertGCRows(ctx context.Context) error {
close(rc.deleteRangeQueryCh)
rc.deleteRangeQueryWaitGroup.Wait()
- ts, err := rc.GetTS(ctx)
+ ts, err := rc.GetTSWithRetry(ctx)
if err != nil {
return errors.Trace(err)
}
@@ -2433,7 +2655,7 @@ func (rc *Client) SaveSchemas(
schemas := TidyOldSchemas(sr)
schemasConcurrency := uint(mathutil.Min(64, schemas.Len()))
- err := schemas.BackupSchemas(ctx, metaWriter, nil, nil, rc.restoreTS, schemasConcurrency, 0, true, nil)
+ err := schemas.BackupSchemas(ctx, metaWriter, nil, nil, nil, rc.restoreTS, schemasConcurrency, 0, true, nil)
if err != nil {
return errors.Trace(err)
}
@@ -2464,6 +2686,74 @@ func (rc *Client) SetWithSysTable(withSysTable bool) {
rc.withSysTable = withSysTable
}
+func (rc *Client) ResetTiFlashReplicas(ctx context.Context, g glue.Glue, storage kv.Storage) error {
+ dom, err := g.GetDomain(storage)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ info := dom.InfoSchema()
+ allSchema := info.AllSchemas()
+ recorder := tiflashrec.New()
+
+ expectTiFlashStoreCount := uint64(0)
+ needTiFlash := false
+ for _, s := range allSchema {
+ for _, t := range s.Tables {
+ if t.TiFlashReplica != nil {
+ expectTiFlashStoreCount = mathutil.Max(expectTiFlashStoreCount, t.TiFlashReplica.Count)
+ recorder.AddTable(t.ID, *t.TiFlashReplica)
+ needTiFlash = true
+ }
+ }
+ }
+ if !needTiFlash {
+ log.Info("no need to set tiflash replica, since there is no tables enable tiflash replica")
+ return nil
+ }
+ // we wait for ten minutes to wait tiflash starts.
+ // since tiflash only starts when set unmark recovery mode finished.
+ timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+ defer cancel()
+ err = utils.WithRetry(timeoutCtx, func() error {
+ tiFlashStoreCount, err := rc.getTiFlashNodeCount(ctx)
+ log.Info("get tiflash store count for resetting TiFlash Replica",
+ zap.Uint64("count", tiFlashStoreCount))
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if tiFlashStoreCount < expectTiFlashStoreCount {
+ log.Info("still waiting for enough tiflash store start",
+ zap.Uint64("expect", expectTiFlashStoreCount),
+ zap.Uint64("actual", tiFlashStoreCount),
+ )
+ return errors.New("tiflash store count is less than expected")
+ }
+ return nil
+ }, &waitTiFlashBackoffer{
+ Attempts: 30,
+ BaseBackoff: 4 * time.Second,
+ })
+ if err != nil {
+ return err
+ }
+
+ sqls := recorder.GenerateResetAlterTableDDLs(info)
+ log.Info("Generating SQLs for resetting tiflash replica",
+ zap.Strings("sqls", sqls))
+
+ return g.UseOneShotSession(storage, false, func(se glue.Session) error {
+ for _, sql := range sqls {
+ if errExec := se.ExecuteInternal(ctx, sql); errExec != nil {
+ logutil.WarnTerm("Failed to restore tiflash replica config, you may execute the sql restore it manually.",
+ logutil.ShortError(errExec),
+ zap.String("sql", sql),
+ )
+ }
+ }
+ return nil
+ })
+}
+
// MockClient create a fake client used to test.
func MockClient(dbs map[string]*utils.Database) *Client {
return &Client{databases: dbs}
@@ -2495,3 +2785,75 @@ func TidyOldSchemas(sr *stream.SchemasReplace) *backup.Schemas {
}
return schemas
}
+
+func CheckKeyspaceBREnable(ctx context.Context, pdClient pd.Client) error {
+ return version.CheckClusterVersion(ctx, pdClient, version.CheckVersionForKeyspaceBR)
+}
+
+func CheckNewCollationEnable(
+ backupNewCollationEnable string,
+ g glue.Glue,
+ storage kv.Storage,
+ CheckRequirements bool,
+) error {
+ if backupNewCollationEnable == "" {
+ if CheckRequirements {
+ return errors.Annotatef(berrors.ErrUnknown,
+ "the config 'new_collations_enabled_on_first_bootstrap' not found in backupmeta. "+
+ "you can use \"show config WHERE name='new_collations_enabled_on_first_bootstrap';\" to manually check the config. "+
+ "if you ensure the config 'new_collations_enabled_on_first_bootstrap' in backup cluster is as same as restore cluster, "+
+ "use --check-requirements=false to skip this check")
+ }
+ log.Warn("the config 'new_collations_enabled_on_first_bootstrap' is not in backupmeta")
+ return nil
+ }
+
+ se, err := g.CreateSession(storage)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ newCollationEnable, err := se.GetGlobalVariable(utils.GetTidbNewCollationEnabled())
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ if !strings.EqualFold(backupNewCollationEnable, newCollationEnable) {
+ return errors.Annotatef(berrors.ErrUnknown,
+ "the config 'new_collations_enabled_on_first_bootstrap' not match, upstream:%v, downstream: %v",
+ backupNewCollationEnable, newCollationEnable)
+ }
+
+ // collate.newCollationEnabled is set to 1 when the collate package is initialized,
+ // so we need to modify this value according to the config of the cluster
+ // before using the collate package.
+ enabled := newCollationEnable == "True"
+ // modify collate.newCollationEnabled according to the config of the cluster
+ collate.SetNewCollationEnabledForTest(enabled)
+ log.Info("set new_collation_enabled", zap.Bool("new_collation_enabled", enabled))
+ return nil
+}
+
+type waitTiFlashBackoffer struct {
+ Attempts int
+ BaseBackoff time.Duration
+}
+
+// NextBackoff returns a duration to wait before retrying again
+func (b *waitTiFlashBackoffer) NextBackoff(error) time.Duration {
+ bo := b.BaseBackoff
+ b.Attempts--
+ if b.Attempts == 0 {
+ return 0
+ }
+ b.BaseBackoff *= 2
+ if b.BaseBackoff > 32*time.Second {
+ b.BaseBackoff = 32 * time.Second
+ }
+ return bo
+}
+
+// Attempt returns the remain attempt times
+func (b *waitTiFlashBackoffer) Attempt() int {
+ return b.Attempts
+}
diff --git a/br/pkg/restore/client_test.go b/br/pkg/restore/client_test.go
index c43c3c80c2452..ae943a96f276b 100644
--- a/br/pkg/restore/client_test.go
+++ b/br/pkg/restore/client_test.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/tidb/br/pkg/restore/tiflashrec"
"github.com/pingcap/tidb/br/pkg/stream"
"github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/utils/iter"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/types"
@@ -194,32 +195,33 @@ func TestCheckSysTableCompatibility(t *testing.T) {
userTI, err := client.GetTableSchema(cluster.Domain, sysDB, model.NewCIStr("user"))
require.NoError(t, err)
- // column count mismatch
+ // user table in cluster have more columns(success)
mockedUserTI := userTI.Clone()
- mockedUserTI.Columns = mockedUserTI.Columns[:len(mockedUserTI.Columns)-1]
+ userTI.Columns = append(userTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")})
err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
DB: tmpSysDB,
Info: mockedUserTI,
}})
- require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err))
+ require.NoError(t, err)
+ userTI.Columns = userTI.Columns[:len(userTI.Columns)-1]
- // column order mismatch(success)
+ // user table in cluster have less columns(failed)
mockedUserTI = userTI.Clone()
- mockedUserTI.Columns[4], mockedUserTI.Columns[5] = mockedUserTI.Columns[5], mockedUserTI.Columns[4]
+ mockedUserTI.Columns = append(mockedUserTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")})
err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
DB: tmpSysDB,
Info: mockedUserTI,
}})
- require.NoError(t, err)
+ require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err))
- // missing column
+ // column order mismatch(success)
mockedUserTI = userTI.Clone()
- mockedUserTI.Columns[0].Name = model.NewCIStr("new-name")
+ mockedUserTI.Columns[4], mockedUserTI.Columns[5] = mockedUserTI.Columns[5], mockedUserTI.Columns[4]
err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
DB: tmpSysDB,
Info: mockedUserTI,
}})
- require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err))
+ require.NoError(t, err)
// incompatible column type
mockedUserTI = userTI.Clone()
@@ -237,6 +239,19 @@ func TestCheckSysTableCompatibility(t *testing.T) {
Info: mockedUserTI,
}})
require.NoError(t, err)
+
+ // use the mysql.db table to test for column count mismatch.
+ dbTI, err := client.GetTableSchema(cluster.Domain, sysDB, model.NewCIStr("db"))
+ require.NoError(t, err)
+
+ // other system tables in cluster have more columns(failed)
+ mockedDBTI := dbTI.Clone()
+ dbTI.Columns = append(dbTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")})
+ err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
+ DB: tmpSysDB,
+ Info: mockedDBTI,
+ }})
+ require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err))
}
func TestInitFullClusterRestore(t *testing.T) {
@@ -332,18 +347,17 @@ type fakePDClient struct {
pd.Client
stores []*metapb.Store
- notLeader bool
+ notLeader bool
+ retryTimes *int
}
-var retryTimes int
-
func (fpdc fakePDClient) GetAllStores(context.Context, ...pd.GetStoreOption) ([]*metapb.Store, error) {
return append([]*metapb.Store{}, fpdc.stores...), nil
}
func (fpdc fakePDClient) GetTS(ctx context.Context) (int64, int64, error) {
- retryTimes++
- if retryTimes >= 3 { // the mock PD leader switched successfully
+ (*fpdc.retryTimes)++
+ if *fpdc.retryTimes >= 3 { // the mock PD leader switched successfully
fpdc.notLeader = false
}
@@ -355,24 +369,24 @@ func (fpdc fakePDClient) GetTS(ctx context.Context) (int64, int64, error) {
func TestGetTSWithRetry(t *testing.T) {
t.Run("PD leader is healthy:", func(t *testing.T) {
- retryTimes = -1000
- pDClient := fakePDClient{notLeader: false}
+ retryTimes := -1000
+ pDClient := fakePDClient{notLeader: false, retryTimes: &retryTimes}
client := restore.NewRestoreClient(pDClient, nil, defaultKeepaliveCfg, false)
_, err := client.GetTSWithRetry(context.Background())
require.NoError(t, err)
})
t.Run("PD leader failure:", func(t *testing.T) {
- retryTimes = -1000
- pDClient := fakePDClient{notLeader: true}
+ retryTimes := -1000
+ pDClient := fakePDClient{notLeader: true, retryTimes: &retryTimes}
client := restore.NewRestoreClient(pDClient, nil, defaultKeepaliveCfg, false)
_, err := client.GetTSWithRetry(context.Background())
require.Error(t, err)
})
t.Run("PD leader switch successfully", func(t *testing.T) {
- retryTimes = 0
- pDClient := fakePDClient{notLeader: true}
+ retryTimes := 0
+ pDClient := fakePDClient{notLeader: true, retryTimes: &retryTimes}
client := restore.NewRestoreClient(pDClient, nil, defaultKeepaliveCfg, false)
_, err := client.GetTSWithRetry(context.Background())
require.NoError(t, err)
@@ -1096,3 +1110,458 @@ func TestSortMetaKVFiles(t *testing.T) {
require.Equal(t, files[3].Path, "f4")
require.Equal(t, files[4].Path, "f5")
}
+
+func TestApplyKVFilesWithSingelMethod(t *testing.T) {
+ var (
+ totalKVCount int64 = 0
+ totalSize uint64 = 0
+ logs = make([]string, 0)
+ )
+ ds := []*backuppb.DataFileInfo{
+ {
+ Path: "log3",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Delete,
+ },
+ {
+ Path: "log1",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.DefaultCF,
+ Type: backuppb.FileType_Put,
+ }, {
+ Path: "log2",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ },
+ }
+ applyFunc := func(
+ files []*backuppb.DataFileInfo,
+ kvCount int64,
+ size uint64,
+ ) {
+ totalKVCount += kvCount
+ totalSize += size
+ for _, f := range files {
+ logs = append(logs, f.GetPath())
+ }
+ }
+
+ restore.ApplyKVFilesWithSingelMethod(
+ context.TODO(),
+ iter.FromSlice(ds),
+ applyFunc,
+ )
+
+ require.Equal(t, totalKVCount, int64(15))
+ require.Equal(t, totalSize, uint64(300))
+ require.Equal(t, logs, []string{"log1", "log2", "log3"})
+}
+
+func TestApplyKVFilesWithBatchMethod1(t *testing.T) {
+ var (
+ runCount = 0
+ batchCount int = 3
+ batchSize uint64 = 1000
+ totalKVCount int64 = 0
+ logs = make([][]string, 0)
+ )
+ ds := []*backuppb.DataFileInfo{
+ {
+ Path: "log5",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Delete,
+ RegionId: 1,
+ }, {
+ Path: "log3",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ }, {
+ Path: "log4",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ }, {
+ Path: "log1",
+ NumberOfEntries: 5,
+ Length: 800,
+ Cf: stream.DefaultCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ },
+ {
+ Path: "log2",
+ NumberOfEntries: 5,
+ Length: 200,
+ Cf: stream.DefaultCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ },
+ }
+ applyFunc := func(
+ files []*backuppb.DataFileInfo,
+ kvCount int64,
+ size uint64,
+ ) {
+ runCount += 1
+ totalKVCount += kvCount
+ log := make([]string, 0, len(files))
+ for _, f := range files {
+ log = append(log, f.GetPath())
+ }
+ logs = append(logs, log)
+ }
+
+ restore.ApplyKVFilesWithBatchMethod(
+ context.TODO(),
+ iter.FromSlice(ds),
+ batchCount,
+ batchSize,
+ applyFunc,
+ )
+
+ require.Equal(t, runCount, 3)
+ require.Equal(t, totalKVCount, int64(25))
+ require.Equal(t,
+ logs,
+ [][]string{
+ {"log1", "log2"},
+ {"log3", "log4"},
+ {"log5"},
+ },
+ )
+}
+
+func TestApplyKVFilesWithBatchMethod2(t *testing.T) {
+ var (
+ runCount = 0
+ batchCount int = 2
+ batchSize uint64 = 1500
+ totalKVCount int64 = 0
+ logs = make([][]string, 0)
+ )
+ ds := []*backuppb.DataFileInfo{
+ {
+ Path: "log1",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Delete,
+ RegionId: 1,
+ }, {
+ Path: "log2",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ }, {
+ Path: "log3",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ }, {
+ Path: "log4",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ }, {
+ Path: "log5",
+ NumberOfEntries: 5,
+ Length: 800,
+ Cf: stream.DefaultCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ },
+ {
+ Path: "log6",
+ NumberOfEntries: 5,
+ Length: 200,
+ Cf: stream.DefaultCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ },
+ }
+ applyFunc := func(
+ files []*backuppb.DataFileInfo,
+ kvCount int64,
+ size uint64,
+ ) {
+ runCount += 1
+ totalKVCount += kvCount
+ log := make([]string, 0, len(files))
+ for _, f := range files {
+ log = append(log, f.GetPath())
+ }
+ logs = append(logs, log)
+ }
+
+ restore.ApplyKVFilesWithBatchMethod(
+ context.TODO(),
+ iter.FromSlice(ds),
+ batchCount,
+ batchSize,
+ applyFunc,
+ )
+
+ require.Equal(t, runCount, 4)
+ require.Equal(t, totalKVCount, int64(30))
+ require.Equal(t,
+ logs,
+ [][]string{
+ {"log2", "log3"},
+ {"log5", "log6"},
+ {"log4"},
+ {"log1"},
+ },
+ )
+}
+
+func TestApplyKVFilesWithBatchMethod3(t *testing.T) {
+ var (
+ runCount = 0
+ batchCount int = 2
+ batchSize uint64 = 1500
+ totalKVCount int64 = 0
+ logs = make([][]string, 0)
+ )
+ ds := []*backuppb.DataFileInfo{
+ {
+ Path: "log1",
+ NumberOfEntries: 5,
+ Length: 2000,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Delete,
+ RegionId: 1,
+ }, {
+ Path: "log2",
+ NumberOfEntries: 5,
+ Length: 2000,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ }, {
+ Path: "log3",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 1,
+ }, {
+ Path: "log5",
+ NumberOfEntries: 5,
+ Length: 800,
+ Cf: stream.DefaultCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 3,
+ },
+ {
+ Path: "log6",
+ NumberOfEntries: 5,
+ Length: 200,
+ Cf: stream.DefaultCF,
+ Type: backuppb.FileType_Put,
+ RegionId: 3,
+ },
+ }
+ applyFunc := func(
+ files []*backuppb.DataFileInfo,
+ kvCount int64,
+ size uint64,
+ ) {
+ runCount += 1
+ totalKVCount += kvCount
+ log := make([]string, 0, len(files))
+ for _, f := range files {
+ log = append(log, f.GetPath())
+ }
+ logs = append(logs, log)
+ }
+
+ restore.ApplyKVFilesWithBatchMethod(
+ context.TODO(),
+ iter.FromSlice(ds),
+ batchCount,
+ batchSize,
+ applyFunc,
+ )
+
+ require.Equal(t, totalKVCount, int64(25))
+ require.Equal(t,
+ logs,
+ [][]string{
+ {"log2"},
+ {"log5", "log6"},
+ {"log3"},
+ {"log1"},
+ },
+ )
+}
+
+func TestApplyKVFilesWithBatchMethod4(t *testing.T) {
+ var (
+ runCount = 0
+ batchCount int = 2
+ batchSize uint64 = 1500
+ totalKVCount int64 = 0
+ logs = make([][]string, 0)
+ )
+ ds := []*backuppb.DataFileInfo{
+ {
+ Path: "log1",
+ NumberOfEntries: 5,
+ Length: 2000,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Delete,
+ TableId: 1,
+ }, {
+ Path: "log2",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ TableId: 1,
+ }, {
+ Path: "log3",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ TableId: 2,
+ }, {
+ Path: "log4",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.WriteCF,
+ Type: backuppb.FileType_Put,
+ TableId: 1,
+ }, {
+ Path: "log5",
+ NumberOfEntries: 5,
+ Length: 100,
+ Cf: stream.DefaultCF,
+ Type: backuppb.FileType_Put,
+ TableId: 2,
+ },
+ }
+ applyFunc := func(
+ files []*backuppb.DataFileInfo,
+ kvCount int64,
+ size uint64,
+ ) {
+ runCount += 1
+ totalKVCount += kvCount
+ log := make([]string, 0, len(files))
+ for _, f := range files {
+ log = append(log, f.GetPath())
+ }
+ logs = append(logs, log)
+ }
+
+ restore.ApplyKVFilesWithBatchMethod(
+ context.TODO(),
+ iter.FromSlice(ds),
+ batchCount,
+ batchSize,
+ applyFunc,
+ )
+
+ require.Equal(t, runCount, 4)
+ require.Equal(t, totalKVCount, int64(25))
+ require.Equal(t,
+ logs,
+ [][]string{
+ {"log2", "log4"},
+ {"log5"},
+ {"log3"},
+ {"log1"},
+ },
+ )
+}
+
+func TestCheckNewCollationEnable(t *testing.T) {
+ caseList := []struct {
+ backupMeta *backuppb.BackupMeta
+ newCollationEnableInCluster string
+ CheckRequirements bool
+ isErr bool
+ }{
+ {
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "True"},
+ newCollationEnableInCluster: "True",
+ CheckRequirements: true,
+ isErr: false,
+ },
+ {
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "True"},
+ newCollationEnableInCluster: "False",
+ CheckRequirements: true,
+ isErr: true,
+ },
+ {
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "False"},
+ newCollationEnableInCluster: "True",
+ CheckRequirements: true,
+ isErr: true,
+ },
+ {
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "False"},
+ newCollationEnableInCluster: "false",
+ CheckRequirements: true,
+ isErr: false,
+ },
+ {
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "False"},
+ newCollationEnableInCluster: "True",
+ CheckRequirements: false,
+ isErr: true,
+ },
+ {
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "True"},
+ newCollationEnableInCluster: "False",
+ CheckRequirements: false,
+ isErr: true,
+ },
+ {
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: ""},
+ newCollationEnableInCluster: "True",
+ CheckRequirements: false,
+ isErr: false,
+ },
+ {
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: ""},
+ newCollationEnableInCluster: "True",
+ CheckRequirements: true,
+ isErr: true,
+ },
+ }
+
+ for i, ca := range caseList {
+ g := &gluetidb.MockGlue{
+ GlobalVars: map[string]string{"new_collation_enabled": ca.newCollationEnableInCluster},
+ }
+ err := restore.CheckNewCollationEnable(ca.backupMeta.GetNewCollationsEnabled(), g, nil, ca.CheckRequirements)
+
+ t.Logf("[%d] Got Error: %v\n", i, err)
+ if ca.isErr {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ }
+ }
+}
diff --git a/br/pkg/restore/data.go b/br/pkg/restore/data.go
index f7efce83362f5..265126b9411af 100644
--- a/br/pkg/restore/data.go
+++ b/br/pkg/restore/data.go
@@ -4,7 +4,6 @@ package restore
import (
"context"
"io"
- "sync/atomic"
"github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/metapb"
@@ -57,7 +56,7 @@ func RecoverData(ctx context.Context, resolveTS uint64, allStores []*metapb.Stor
return totalRegions, errors.Trace(err)
}
- if err := recovery.PrepareFlashbackToVersion(ctx); err != nil {
+ if err := recovery.PrepareFlashbackToVersion(ctx, resolveTS, restoreTS-1); err != nil {
return totalRegions, errors.Trace(err)
}
@@ -80,14 +79,13 @@ func NewStoreMeta(storeId uint64) StoreMeta {
// for test
type Recovery struct {
- allStores []*metapb.Store
- StoreMetas []StoreMeta
- RecoveryPlan map[uint64][]*recovpb.RecoverRegionRequest
- MaxAllocID uint64
- mgr *conn.Mgr
- progress glue.Progress
- concurrency uint32
- totalFlashbackRegions uint64
+ allStores []*metapb.Store
+ StoreMetas []StoreMeta
+ RecoveryPlan map[uint64][]*recovpb.RecoverRegionRequest
+ MaxAllocID uint64
+ mgr *conn.Mgr
+ progress glue.Progress
+ concurrency uint32
}
func NewRecovery(allStores []*metapb.Store, mgr *conn.Mgr, progress glue.Progress, concurrency uint32) Recovery {
@@ -95,14 +93,13 @@ func NewRecovery(allStores []*metapb.Store, mgr *conn.Mgr, progress glue.Progres
var StoreMetas = make([]StoreMeta, totalStores)
var regionRecovers = make(map[uint64][]*recovpb.RecoverRegionRequest, totalStores)
return Recovery{
- allStores: allStores,
- StoreMetas: StoreMetas,
- RecoveryPlan: regionRecovers,
- MaxAllocID: 0,
- mgr: mgr,
- progress: progress,
- concurrency: concurrency,
- totalFlashbackRegions: 0}
+ allStores: allStores,
+ StoreMetas: StoreMetas,
+ RecoveryPlan: regionRecovers,
+ MaxAllocID: 0,
+ mgr: mgr,
+ progress: progress,
+ concurrency: concurrency}
}
func (recovery *Recovery) newRecoveryClient(ctx context.Context, storeAddr string) (recovpb.RecoverDataClient, *grpc.ClientConn, error) {
@@ -304,40 +301,36 @@ func (recovery *Recovery) WaitApply(ctx context.Context) (err error) {
}
// prepare the region for flashback the data, the purpose is to stop region service, put region in flashback state
-func (recovery *Recovery) PrepareFlashbackToVersion(ctx context.Context) (err error) {
- var totalRegions atomic.Uint64
- totalRegions.Store(0)
-
- handler := func(ctx context.Context, r tikvstore.KeyRange) (rangetask.TaskStat, error) {
- stats, err := ddl.SendPrepareFlashbackToVersionRPC(ctx, recovery.mgr.GetStorage().(tikv.Storage), r)
- totalRegions.Add(uint64(stats.CompletedRegions))
- return stats, err
- }
-
- runner := rangetask.NewRangeTaskRunner("br-flashback-prepare-runner", recovery.mgr.GetStorage().(tikv.Storage), int(recovery.concurrency), handler)
- // Run prepare flashback on the entire TiKV cluster. Empty keys means the range is unbounded.
- err = runner.RunOnRange(ctx, []byte(""), []byte(""))
- if err != nil {
- log.Error("region flashback prepare get error")
- return errors.Trace(err)
- }
+func (recovery *Recovery) PrepareFlashbackToVersion(ctx context.Context, resolveTS uint64, startTS uint64) (err error) {
+ retryErr := utils.WithRetry(
+ ctx,
+ func() error {
+ handler := func(ctx context.Context, r tikvstore.KeyRange) (rangetask.TaskStat, error) {
+ stats, err := ddl.SendPrepareFlashbackToVersionRPC(ctx, recovery.mgr.GetStorage().(tikv.Storage), resolveTS, startTS, r)
+ return stats, err
+ }
- recovery.totalFlashbackRegions = totalRegions.Load()
- log.Info("region flashback prepare complete", zap.Int("regions", runner.CompletedRegions()))
+ runner := rangetask.NewRangeTaskRunner("br-flashback-prepare-runner", recovery.mgr.GetStorage().(tikv.Storage), int(recovery.concurrency), handler)
+ // Run prepare flashback on the entire TiKV cluster. Empty keys means the range is unbounded.
+ err = runner.RunOnRange(ctx, []byte(""), []byte(""))
+ if err != nil {
+ log.Warn("region flashback prepare get error")
+ return errors.Trace(err)
+ }
+ log.Info("region flashback prepare complete", zap.Int("regions", runner.CompletedRegions()))
+ return nil
+ },
+ utils.NewFlashBackBackoffer(),
+ )
- return nil
+ recovery.progress.Inc()
+ return retryErr
}
// flashback the region data to version resolveTS
func (recovery *Recovery) FlashbackToVersion(ctx context.Context, resolveTS uint64, commitTS uint64) (err error) {
- var completedRegions atomic.Uint64
-
- // only know the total progress of tikv, progress is total state of the whole restore flow.
- ratio := int(recovery.totalFlashbackRegions) / len(recovery.allStores)
-
handler := func(ctx context.Context, r tikvstore.KeyRange) (rangetask.TaskStat, error) {
stats, err := ddl.SendFlashbackToVersionRPC(ctx, recovery.mgr.GetStorage().(tikv.Storage), resolveTS, commitTS-1, commitTS, r)
- completedRegions.Add(uint64(stats.CompletedRegions))
return stats, err
}
@@ -352,13 +345,12 @@ func (recovery *Recovery) FlashbackToVersion(ctx context.Context, resolveTS uint
return errors.Trace(err)
}
- recovery.progress.IncBy(int64(completedRegions.Load()) / int64(ratio))
-
log.Info("region flashback complete",
zap.Uint64("resolveTS", resolveTS),
zap.Uint64("commitTS", commitTS),
zap.Int("regions", runner.CompletedRegions()))
+ recovery.progress.Inc()
return nil
}
@@ -372,6 +364,7 @@ type RecoverRegion struct {
// 2. build a leader list for all region during the tikv startup
// 3. get max allocate id
func (recovery *Recovery) MakeRecoveryPlan() error {
+ storeBalanceScore := make(map[uint64]int, len(recovery.allStores))
// Group region peer info by region id. find the max allocateId
// region [id] [peer[0-n]]
var regions = make(map[uint64][]*RecoverRegion, 0)
@@ -410,16 +403,20 @@ func (recovery *Recovery) MakeRecoveryPlan() error {
}
} else {
// Generate normal commands.
- log.Debug("detected valid peer", zap.Uint64("region id", regionId))
- for i, peer := range peers {
- log.Debug("make plan", zap.Uint64("store id", peer.StoreId), zap.Uint64("region id", peer.RegionId))
- plan := &recovpb.RecoverRegionRequest{RegionId: peer.RegionId, AsLeader: i == 0}
- // sorted by log term -> last index -> commit index in a region
- if plan.AsLeader {
- log.Debug("as leader peer", zap.Uint64("store id", peer.StoreId), zap.Uint64("region id", peer.RegionId))
- recovery.RecoveryPlan[peer.StoreId] = append(recovery.RecoveryPlan[peer.StoreId], plan)
- }
+ log.Debug("detected valid region", zap.Uint64("region id", regionId))
+ // calc the leader candidates
+ leaderCandidates, err := LeaderCandidates(peers)
+ if err != nil {
+ log.Warn("region without peer", zap.Uint64("region id", regionId))
+ return errors.Trace(err)
}
+
+ // select the leader base on tikv storeBalanceScore
+ leader := SelectRegionLeader(storeBalanceScore, leaderCandidates)
+ log.Debug("as leader peer", zap.Uint64("store id", leader.StoreId), zap.Uint64("region id", leader.RegionId))
+ plan := &recovpb.RecoverRegionRequest{RegionId: leader.RegionId, AsLeader: true}
+ recovery.RecoveryPlan[leader.StoreId] = append(recovery.RecoveryPlan[leader.StoreId], plan)
+ storeBalanceScore[leader.StoreId] += 1
}
}
return nil
diff --git a/br/pkg/restore/db.go b/br/pkg/restore/db.go
index ae62162c3e890..1f3f5d949e26e 100644
--- a/br/pkg/restore/db.go
+++ b/br/pkg/restore/db.go
@@ -284,7 +284,7 @@ func (db *DB) tableIDAllocFilter() ddl.AllocTableIDIf {
if db.preallocedIDs == nil {
return true
}
- prealloced := db.preallocedIDs.Prealloced(ti.ID)
+ prealloced := db.preallocedIDs.PreallocedFor(ti)
if prealloced {
log.Info("reusing table ID", zap.Stringer("table", ti.Name))
}
@@ -308,6 +308,10 @@ func (db *DB) CreateTables(ctx context.Context, tables []*metautil.Table,
return errors.Trace(err)
}
}
+
+ if ttlInfo := table.Info.TTLInfo; ttlInfo != nil {
+ ttlInfo.Enable = false
+ }
}
if err := batchSession.CreateTables(ctx, m, db.tableIDAllocFilter()); err != nil {
return err
@@ -336,6 +340,10 @@ func (db *DB) CreateTable(ctx context.Context, table *metautil.Table,
}
}
+ if ttlInfo := table.Info.TTLInfo; ttlInfo != nil {
+ ttlInfo.Enable = false
+ }
+
err := db.se.CreateTable(ctx, table.DB.Name, table.Info, db.tableIDAllocFilter())
if err != nil {
log.Error("create table failed",
diff --git a/br/pkg/restore/db_test.go b/br/pkg/restore/db_test.go
index 8801a6af34727..3a5416501e4df 100644
--- a/br/pkg/restore/db_test.go
+++ b/br/pkg/restore/db_test.go
@@ -381,7 +381,7 @@ func TestGetExistedUserDBs(t *testing.T) {
{Name: model.NewCIStr("mysql")},
{Name: model.NewCIStr("test")},
},
- nil, 1)
+ nil, nil, 1)
require.Nil(t, err)
dom.MockInfoCacheAndLoadInfoSchema(builder.Build())
dbs = restore.GetExistedUserDBs(dom)
@@ -393,7 +393,7 @@ func TestGetExistedUserDBs(t *testing.T) {
{Name: model.NewCIStr("test")},
{Name: model.NewCIStr("d1")},
},
- nil, 1)
+ nil, nil, 1)
require.Nil(t, err)
dom.MockInfoCacheAndLoadInfoSchema(builder.Build())
dbs = restore.GetExistedUserDBs(dom)
@@ -409,7 +409,7 @@ func TestGetExistedUserDBs(t *testing.T) {
State: model.StatePublic,
},
},
- nil, 1)
+ nil, nil, 1)
require.Nil(t, err)
dom.MockInfoCacheAndLoadInfoSchema(builder.Build())
dbs = restore.GetExistedUserDBs(dom)
diff --git a/br/pkg/restore/import.go b/br/pkg/restore/import.go
index 0245add57554b..47c25672719c5 100644
--- a/br/pkg/restore/import.go
+++ b/br/pkg/restore/import.go
@@ -6,6 +6,8 @@ import (
"bytes"
"context"
"crypto/tls"
+ "fmt"
+ "math/rand"
"strings"
"sync"
"sync/atomic"
@@ -24,8 +26,10 @@ import (
berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/br/pkg/restore/split"
+ "github.com/pingcap/tidb/br/pkg/stream"
"github.com/pingcap/tidb/br/pkg/summary"
"github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/kv"
pd "github.com/tikv/pd/client"
"go.uber.org/multierr"
"go.uber.org/zap"
@@ -34,6 +38,7 @@ import (
"google.golang.org/grpc/backoff"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/keepalive"
"google.golang.org/grpc/status"
)
@@ -43,6 +48,17 @@ const (
gRPCBackOffMaxDelay = 3 * time.Second
)
+// RewriteMode is a mode flag that tells the TiKV how to handle the rewrite rules.
+type RewriteMode int
+
+const (
+ // RewriteModeLegacy means no rewrite rule is applied.
+ RewriteModeLegacy RewriteMode = iota
+
+ // RewriteModeKeyspace means the rewrite rule could be applied to keyspace.
+ RewriteModeKeyspace
+)
+
// ImporterClient is used to import a file to TiKV.
type ImporterClient interface {
ClearFiles(
@@ -193,7 +209,7 @@ func (ic *importClient) GetImportClient(
if err != nil {
return nil, errors.Trace(err)
}
- opt := grpc.WithInsecure()
+ opt := grpc.WithTransportCredentials(insecure.NewCredentials())
if ic.tlsConf != nil {
opt = grpc.WithTransportCredentials(credentials.NewTLS(ic.tlsConf))
}
@@ -245,6 +261,9 @@ type FileImporter struct {
rawStartKey []byte
rawEndKey []byte
supportMultiIngest bool
+ rewriteMode RewriteMode
+
+ cacheKey string
}
// NewFileImporter returns a new file importClient.
@@ -253,12 +272,15 @@ func NewFileImporter(
importClient ImporterClient,
backend *backuppb.StorageBackend,
isRawKvMode bool,
+ rewriteMode RewriteMode,
) FileImporter {
return FileImporter{
metaClient: metaClient,
backend: backend,
importClient: importClient,
isRawKvMode: isRawKvMode,
+ rewriteMode: rewriteMode,
+ cacheKey: fmt.Sprintf("BR-%s-%d", time.Now().Format("20060102150405"), rand.Int63()),
}
}
@@ -332,14 +354,16 @@ func (importer *FileImporter) getKeyRangeForFiles(
// Import tries to import a file.
func (importer *FileImporter) ImportKVFileForRegion(
ctx context.Context,
- file *backuppb.DataFileInfo,
+ files []*backuppb.DataFileInfo,
rule *RewriteRules,
+ shiftStartTS uint64,
startTS uint64,
restoreTS uint64,
info *split.RegionInfo,
+ supportBatch bool,
) RPCResult {
// Try to download file.
- result := importer.downloadAndApplyKVFile(ctx, file, rule, info, startTS, restoreTS)
+ result := importer.downloadAndApplyKVFile(ctx, files, rule, info, shiftStartTS, startTS, restoreTS, supportBatch)
if !result.OK() {
errDownload := result.Err
for _, e := range multierr.Errors(errDownload) {
@@ -380,39 +404,85 @@ func (importer *FileImporter) ClearFiles(ctx context.Context, pdClient pd.Client
return nil
}
+func FilterFilesByRegion(
+ files []*backuppb.DataFileInfo,
+ ranges []kv.KeyRange,
+ r *split.RegionInfo,
+) ([]*backuppb.DataFileInfo, error) {
+ if len(files) != len(ranges) {
+ return nil, errors.Annotatef(berrors.ErrInvalidArgument,
+ "count of files no equals count of ranges, file-count:%v, ranges-count:%v",
+ len(files), len(ranges))
+ }
+
+ output := make([]*backuppb.DataFileInfo, 0, len(files))
+ if r != nil && r.Region != nil {
+ for i, f := range files {
+ if bytes.Compare(r.Region.StartKey, ranges[i].EndKey) <= 0 &&
+ (len(r.Region.EndKey) == 0 || bytes.Compare(r.Region.EndKey, ranges[i].StartKey) >= 0) {
+ output = append(output, f)
+ }
+ }
+ } else {
+ output = files
+ }
+
+ return output, nil
+}
+
// ImportKVFiles restores the kv events.
func (importer *FileImporter) ImportKVFiles(
ctx context.Context,
- file *backuppb.DataFileInfo,
+ files []*backuppb.DataFileInfo,
rule *RewriteRules,
+ shiftStartTS uint64,
startTS uint64,
restoreTS uint64,
+ supportBatch bool,
) error {
- startTime := time.Now()
- log.Debug("import kv files", zap.String("file", file.Path))
- startKey, endKey, err := GetRewriteEncodedKeys(file, rule)
- if err != nil {
- return errors.Trace(err)
+ var (
+ startKey []byte
+ endKey []byte
+ ranges = make([]kv.KeyRange, len(files))
+ err error
+ )
+
+ if !supportBatch && len(files) > 1 {
+ return errors.Annotatef(berrors.ErrInvalidArgument,
+ "do not support batch apply but files count:%v > 1", len(files))
+ }
+ log.Debug("import kv files", zap.Int("batch file count", len(files)))
+
+ for i, f := range files {
+ ranges[i].StartKey, ranges[i].EndKey, err = GetRewriteEncodedKeys(f, rule)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ if len(startKey) == 0 || bytes.Compare(ranges[i].StartKey, startKey) < 0 {
+ startKey = ranges[i].StartKey
+ }
+ if len(endKey) == 0 || bytes.Compare(ranges[i].EndKey, endKey) > 0 {
+ endKey = ranges[i].EndKey
+ }
}
log.Debug("rewrite file keys",
- zap.String("name", file.Path),
- logutil.Key("startKey", startKey),
- logutil.Key("endKey", endKey))
+ logutil.Key("startKey", startKey), logutil.Key("endKey", endKey))
- // This RetryState will retry 48 time, for 5 min - 6 min.
- rs := utils.InitialRetryState(48, 100*time.Millisecond, 8*time.Second)
+ // This RetryState will retry 45 time, about 10 min.
+ rs := utils.InitialRetryState(45, 100*time.Millisecond, 15*time.Second)
ctl := OverRegionsInRange(startKey, endKey, importer.metaClient, &rs)
err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) RPCResult {
- return importer.ImportKVFileForRegion(ctx, file, rule, startTS, restoreTS, r)
+ subfiles, errFilter := FilterFilesByRegion(files, ranges, r)
+ if errFilter != nil {
+ return RPCResultFromError(errFilter)
+ }
+ if len(subfiles) == 0 {
+ return RPCResultOK()
+ }
+ return importer.ImportKVFileForRegion(ctx, subfiles, rule, shiftStartTS, startTS, restoreTS, r, supportBatch)
})
-
- log.Debug("download and apply file done",
- zap.String("file", file.Path),
- zap.Stringer("take", time.Since(startTime)),
- logutil.Key("fileStart", file.StartKey),
- logutil.Key("fileEnd", file.EndKey),
- )
return errors.Trace(err)
}
@@ -525,7 +595,7 @@ func (importer *FileImporter) download(
if importer.isRawKvMode {
downloadMeta, e = importer.downloadRawKVSST(ctx, regionInfo, f, cipher, apiVersion)
} else {
- downloadMeta, e = importer.downloadSST(ctx, regionInfo, f, rewriteRules, cipher)
+ downloadMeta, e = importer.downloadSST(ctx, regionInfo, f, rewriteRules, cipher, apiVersion)
}
failpoint.Inject("restore-storage-error", func(val failpoint.Value) {
@@ -542,7 +612,7 @@ func (importer *FileImporter) download(
if importer.isRawKvMode {
downloadMeta, e = importer.downloadRawKVSST(ctx, regionInfo, f, nil, apiVersion)
} else {
- downloadMeta, e = importer.downloadSST(ctx, regionInfo, f, rewriteRules, nil)
+ downloadMeta, e = importer.downloadSST(ctx, regionInfo, f, rewriteRules, nil, apiVersion)
}
}
@@ -566,6 +636,7 @@ func (importer *FileImporter) downloadSST(
file *backuppb.File,
rewriteRules *RewriteRules,
cipher *backuppb.CipherInfo,
+ apiVersion kvrpcpb.APIVersion,
) (*import_sstpb.SSTMeta, error) {
uid := uuid.New()
id := uid[:]
@@ -574,21 +645,38 @@ func (importer *FileImporter) downloadSST(
if fileRule == nil {
return nil, errors.Trace(berrors.ErrKVRewriteRuleNotFound)
}
- rule := import_sstpb.RewriteRule{
- OldKeyPrefix: encodeKeyPrefix(fileRule.GetOldKeyPrefix()),
- NewKeyPrefix: encodeKeyPrefix(fileRule.GetNewKeyPrefix()),
+
+ // For the legacy version of TiKV, we need to encode the key prefix, since in the legacy
+ // version, the TiKV will rewrite the key with the encoded prefix without decoding the keys in
+ // the SST file. For the new version of TiKV that support keyspace rewrite, we don't need to
+ // encode the key prefix. The TiKV will decode the keys in the SST file and rewrite the keys
+ // with the plain prefix and encode the keys before writing to SST.
+
+ // for the keyspace rewrite mode
+ rule := *fileRule
+ // for the legacy rewrite mode
+ if importer.rewriteMode == RewriteModeLegacy {
+ rule.OldKeyPrefix = encodeKeyPrefix(fileRule.GetOldKeyPrefix())
+ rule.NewKeyPrefix = encodeKeyPrefix(fileRule.GetNewKeyPrefix())
+ }
+
+ sstMeta, err := GetSSTMetaFromFile(id, file, regionInfo.Region, &rule, importer.rewriteMode)
+ if err != nil {
+ return nil, err
}
- sstMeta := GetSSTMetaFromFile(id, file, regionInfo.Region, &rule)
req := &import_sstpb.DownloadRequest{
- Sst: sstMeta,
+ Sst: *sstMeta,
StorageBackend: importer.backend,
Name: file.GetName(),
RewriteRule: rule,
CipherInfo: cipher,
+ StorageCacheId: importer.cacheKey,
+ // For the older version of TiDB, the request type will be default to `import_sstpb.RequestType_Legacy`
+ RequestType: import_sstpb.DownloadRequestType_Keyspace,
}
log.Debug("download SST",
- logutil.SSTMeta(&sstMeta),
+ logutil.SSTMeta(sstMeta),
logutil.File(file),
logutil.Region(regionInfo.Region),
logutil.Leader(regionInfo.Leader),
@@ -614,7 +702,7 @@ func (importer *FileImporter) downloadSST(
logutil.Region(regionInfo.Region),
logutil.Peer(peer),
logutil.Key("resp-range-start", resp.Range.Start),
- logutil.Key("resp-range-end", resp.Range.Start),
+ logutil.Key("resp-range-end", resp.Range.End),
zap.Bool("resp-isempty", resp.IsEmpty),
zap.Uint32("resp-crc32", resp.Crc32),
)
@@ -629,7 +717,8 @@ func (importer *FileImporter) downloadSST(
downloadResp := atomicResp.Load().(*import_sstpb.DownloadResponse)
sstMeta.Range.Start = TruncateTS(downloadResp.Range.GetStart())
sstMeta.Range.End = TruncateTS(downloadResp.Range.GetEnd())
- return &sstMeta, nil
+ sstMeta.ApiVersion = apiVersion
+ return sstMeta, nil
}
func (importer *FileImporter) downloadRawKVSST(
@@ -643,7 +732,10 @@ func (importer *FileImporter) downloadRawKVSST(
id := uid[:]
// Empty rule
var rule import_sstpb.RewriteRule
- sstMeta := GetSSTMetaFromFile(id, file, regionInfo.Region, &rule)
+ sstMeta, err := GetSSTMetaFromFile(id, file, regionInfo.Region, &rule, RewriteModeLegacy)
+ if err != nil {
+ return nil, err
+ }
// Cut the SST file's range to fit in the restoring range.
if bytes.Compare(importer.rawStartKey, sstMeta.Range.GetStart()) > 0 {
@@ -659,14 +751,15 @@ func (importer *FileImporter) downloadRawKVSST(
}
req := &import_sstpb.DownloadRequest{
- Sst: sstMeta,
+ Sst: *sstMeta,
StorageBackend: importer.backend,
Name: file.GetName(),
RewriteRule: rule,
IsRawKv: true,
CipherInfo: cipher,
+ StorageCacheId: importer.cacheKey,
}
- log.Debug("download SST", logutil.SSTMeta(&sstMeta), logutil.Region(regionInfo.Region))
+ log.Debug("download SST", logutil.SSTMeta(sstMeta), logutil.Region(regionInfo.Region))
var atomicResp atomic.Value
eg, ectx := errgroup.WithContext(ctx)
@@ -697,7 +790,7 @@ func (importer *FileImporter) downloadRawKVSST(
sstMeta.Range.Start = downloadResp.Range.GetStart()
sstMeta.Range.End = downloadResp.Range.GetEnd()
sstMeta.ApiVersion = apiVersion
- return &sstMeta, nil
+ return sstMeta, nil
}
func (importer *FileImporter) ingest(
@@ -801,41 +894,57 @@ func (importer *FileImporter) ingestSSTs(
func (importer *FileImporter) downloadAndApplyKVFile(
ctx context.Context,
- file *backuppb.DataFileInfo,
+ files []*backuppb.DataFileInfo,
rules *RewriteRules,
regionInfo *split.RegionInfo,
+ shiftStartTS uint64,
startTS uint64,
restoreTS uint64,
+ supportBatch bool,
) RPCResult {
leader := regionInfo.Leader
if leader == nil {
return RPCResultFromError(errors.Annotatef(berrors.ErrPDLeaderNotFound,
"region id %d has no leader", regionInfo.Region.Id))
}
- // Get the rewrite rule for the file.
- fileRule := findMatchedRewriteRule(file, rules)
- if fileRule == nil {
- return RPCResultFromError(errors.Annotatef(berrors.ErrKVRewriteRuleNotFound,
- "rewrite rule for file %+v not find (in %+v)", file, rules))
- }
- rule := import_sstpb.RewriteRule{
- OldKeyPrefix: encodeKeyPrefix(fileRule.GetOldKeyPrefix()),
- NewKeyPrefix: encodeKeyPrefix(fileRule.GetNewKeyPrefix()),
- }
- meta := &import_sstpb.KVMeta{
- Name: file.Path,
- Cf: file.Cf,
- RangeOffset: file.RangeOffset,
- Length: file.Length,
- RangeLength: file.RangeLength,
- IsDelete: file.Type == backuppb.FileType_Delete,
- StartSnapshotTs: startTS,
- RestoreTs: restoreTS,
- StartKey: regionInfo.Region.GetStartKey(),
- EndKey: regionInfo.Region.GetEndKey(),
- Sha256: file.GetSha256(),
- CompressionType: file.CompressionType,
+ metas := make([]*import_sstpb.KVMeta, 0, len(files))
+ rewriteRules := make([]*import_sstpb.RewriteRule, 0, len(files))
+
+ for _, file := range files {
+ // Get the rewrite rule for the file.
+ fileRule := findMatchedRewriteRule(file, rules)
+ if fileRule == nil {
+ return RPCResultFromError(errors.Annotatef(berrors.ErrKVRewriteRuleNotFound,
+ "rewrite rule for file %+v not find (in %+v)", file, rules))
+ }
+ rule := import_sstpb.RewriteRule{
+ OldKeyPrefix: encodeKeyPrefix(fileRule.GetOldKeyPrefix()),
+ NewKeyPrefix: encodeKeyPrefix(fileRule.GetNewKeyPrefix()),
+ }
+
+ meta := &import_sstpb.KVMeta{
+ Name: file.Path,
+ Cf: file.Cf,
+ RangeOffset: file.RangeOffset,
+ Length: file.Length,
+ RangeLength: file.RangeLength,
+ IsDelete: file.Type == backuppb.FileType_Delete,
+ StartTs: func() uint64 {
+ if file.Cf == stream.DefaultCF {
+ return shiftStartTS
+ }
+ return startTS
+ }(),
+ RestoreTs: restoreTS,
+ StartKey: regionInfo.Region.GetStartKey(),
+ EndKey: regionInfo.Region.GetEndKey(),
+ Sha256: file.GetSha256(),
+ CompressionType: file.CompressionType,
+ }
+
+ metas = append(metas, meta)
+ rewriteRules = append(rewriteRules, &rule)
}
reqCtx := &kvrpcpb.Context{
@@ -844,12 +953,25 @@ func (importer *FileImporter) downloadAndApplyKVFile(
Peer: leader,
}
- req := &import_sstpb.ApplyRequest{
- Meta: meta,
- StorageBackend: importer.backend,
- RewriteRule: rule,
- Context: reqCtx,
+ var req *import_sstpb.ApplyRequest
+ if supportBatch {
+ req = &import_sstpb.ApplyRequest{
+ Metas: metas,
+ StorageBackend: importer.backend,
+ RewriteRules: rewriteRules,
+ Context: reqCtx,
+ StorageCacheId: importer.cacheKey,
+ }
+ } else {
+ req = &import_sstpb.ApplyRequest{
+ Meta: metas[0],
+ StorageBackend: importer.backend,
+ RewriteRule: *rewriteRules[0],
+ Context: reqCtx,
+ StorageCacheId: importer.cacheKey,
+ }
}
+
log.Debug("apply kv file", logutil.Leader(leader))
resp, err := importer.importClient.ApplyKVFile(ctx, leader.GetStoreId(), req)
if err != nil {
diff --git a/br/pkg/restore/import_retry.go b/br/pkg/restore/import_retry.go
index 7dcdb01a6c765..6f3b9fc1cca53 100644
--- a/br/pkg/restore/import_retry.go
+++ b/br/pkg/restore/import_retry.go
@@ -224,7 +224,8 @@ func (r *RPCResult) StrategyForRetryStoreError() RetryStrategy {
if r.StoreError.GetServerIsBusy() != nil ||
r.StoreError.GetRegionNotInitialized() != nil ||
- r.StoreError.GetNotLeader() != nil {
+ r.StoreError.GetNotLeader() != nil ||
+ r.StoreError.GetServerIsBusy() != nil {
return StrategyFromThisRegion
}
diff --git a/br/pkg/restore/import_retry_test.go b/br/pkg/restore/import_retry_test.go
index d79e2a317a4c0..6f3d8f490ef13 100644
--- a/br/pkg/restore/import_retry_test.go
+++ b/br/pkg/restore/import_retry_test.go
@@ -12,12 +12,15 @@ import (
"time"
"github.com/pingcap/errors"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/errorpb"
"github.com/pingcap/kvproto/pkg/import_sstpb"
"github.com/pingcap/kvproto/pkg/metapb"
+ berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/pingcap/tidb/br/pkg/restore"
"github.com/pingcap/tidb/br/pkg/restore/split"
"github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/store/pdtypes"
"github.com/pingcap/tidb/util/codec"
"github.com/stretchr/testify/require"
@@ -127,6 +130,41 @@ func TestNotLeader(t *testing.T) {
assertRegions(t, meetRegions, "", "aay", "bba", "bbh", "cca", "")
}
+func TestServerIsBusy(t *testing.T) {
+ // region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, )
+ cli := initTestClient(false)
+ rs := utils.InitialRetryState(2, 0, 0)
+ ctl := restore.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
+ ctx := context.Background()
+
+ serverIsBusy := errorpb.Error{
+ Message: "server is busy",
+ ServerIsBusy: &errorpb.ServerIsBusy{
+ Reason: "memory is out",
+ },
+ }
+ // record the regions we didn't touch.
+ meetRegions := []*split.RegionInfo{}
+ // record all regions we meet with id == 2.
+ idEqualsTo2Regions := []*split.RegionInfo{}
+ theFirstRun := true
+ err := ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) restore.RPCResult {
+ if theFirstRun && r.Region.Id == 2 {
+ idEqualsTo2Regions = append(idEqualsTo2Regions, r)
+ theFirstRun = false
+ return restore.RPCResult{
+ StoreError: &serverIsBusy,
+ }
+ }
+ meetRegions = append(meetRegions, r)
+ return restore.RPCResultOK()
+ })
+
+ require.NoError(t, err)
+ assertRegions(t, idEqualsTo2Regions, "aay", "bba")
+ assertRegions(t, meetRegions, "", "aay", "bba", "bbh", "cca", "")
+}
+
func printRegion(name string, infos []*split.RegionInfo) {
fmt.Printf(">>>>> %s <<<<<\n", name)
for _, info := range infos {
@@ -345,3 +383,166 @@ func TestPaginateScanLeader(t *testing.T) {
})
assertRegions(t, collectedRegions, "", "aay", "bba")
}
+
+func TestImportKVFiles(t *testing.T) {
+ var (
+ importer = restore.FileImporter{}
+ ctx = context.Background()
+ shiftStartTS uint64 = 100
+ startTS uint64 = 200
+ restoreTS uint64 = 300
+ )
+
+ err := importer.ImportKVFiles(
+ ctx,
+ []*backuppb.DataFileInfo{
+ {
+ Path: "log3",
+ },
+ {
+ Path: "log1",
+ },
+ },
+ nil,
+ shiftStartTS,
+ startTS,
+ restoreTS,
+ false,
+ )
+ require.True(t, berrors.ErrInvalidArgument.Equal(err))
+}
+
+func TestFilterFilesByRegion(t *testing.T) {
+ files := []*backuppb.DataFileInfo{
+ {
+ Path: "log1",
+ },
+ {
+ Path: "log2",
+ },
+ }
+ ranges := []kv.KeyRange{
+ {
+ StartKey: []byte("1111"),
+ EndKey: []byte("2222"),
+ }, {
+ StartKey: []byte("3333"),
+ EndKey: []byte("4444"),
+ },
+ }
+
+ testCases := []struct {
+ r split.RegionInfo
+ subfiles []*backuppb.DataFileInfo
+ err error
+ }{
+ {
+ r: split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte("0000"),
+ EndKey: []byte("1110"),
+ },
+ },
+ subfiles: []*backuppb.DataFileInfo{},
+ err: nil,
+ },
+ {
+ r: split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte("0000"),
+ EndKey: []byte("1111"),
+ },
+ },
+ subfiles: []*backuppb.DataFileInfo{
+ files[0],
+ },
+ err: nil,
+ },
+ {
+ r: split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte("0000"),
+ EndKey: []byte("2222"),
+ },
+ },
+ subfiles: []*backuppb.DataFileInfo{
+ files[0],
+ },
+ err: nil,
+ },
+ {
+ r: split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte("2222"),
+ EndKey: []byte("3332"),
+ },
+ },
+ subfiles: []*backuppb.DataFileInfo{
+ files[0],
+ },
+ err: nil,
+ },
+ {
+ r: split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte("2223"),
+ EndKey: []byte("3332"),
+ },
+ },
+ subfiles: []*backuppb.DataFileInfo{},
+ err: nil,
+ },
+ {
+ r: split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte("3332"),
+ EndKey: []byte("3333"),
+ },
+ },
+ subfiles: []*backuppb.DataFileInfo{
+ files[1],
+ },
+ err: nil,
+ },
+ {
+ r: split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte("4444"),
+ EndKey: []byte("5555"),
+ },
+ },
+ subfiles: []*backuppb.DataFileInfo{
+ files[1],
+ },
+ err: nil,
+ },
+ {
+ r: split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte("4444"),
+ EndKey: nil,
+ },
+ },
+ subfiles: []*backuppb.DataFileInfo{
+ files[1],
+ },
+ err: nil,
+ },
+ {
+ r: split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte("0000"),
+ EndKey: nil,
+ },
+ },
+ subfiles: files,
+ err: nil,
+ },
+ }
+
+ for _, c := range testCases {
+ subfile, err := restore.FilterFilesByRegion(files, ranges, &c.r)
+ require.Equal(t, err, c.err)
+ require.Equal(t, subfile, c.subfiles)
+ }
+}
diff --git a/br/pkg/restore/log_client.go b/br/pkg/restore/log_client.go
index 7c01897741ba2..cce295090ba02 100644
--- a/br/pkg/restore/log_client.go
+++ b/br/pkg/restore/log_client.go
@@ -101,14 +101,17 @@ func (rc *logFileManager) loadShiftTS(ctx context.Context) error {
if err != nil {
return err
}
- shiftTS.Lock()
- defer shiftTS.Unlock()
+ log.Info("read meta from storage and parse", zap.String("path", path), zap.Uint64("min-ts", m.MinTs),
+ zap.Uint64("max-ts", m.MaxTs), zap.Int32("meta-version", int32(m.MetaVersion)))
ts, ok := UpdateShiftTS(m, rc.startTS, rc.restoreTS)
+ shiftTS.Lock()
if ok && (!shiftTS.exists || shiftTS.value > ts) {
shiftTS.value = ts
shiftTS.exists = true
}
+ shiftTS.Unlock()
+
return nil
})
if err != nil {
diff --git a/br/pkg/restore/prealloc_table_id/BUILD.bazel b/br/pkg/restore/prealloc_table_id/BUILD.bazel
index 8ce80b039178a..b8c69a206ad21 100644
--- a/br/pkg/restore/prealloc_table_id/BUILD.bazel
+++ b/br/pkg/restore/prealloc_table_id/BUILD.bazel
@@ -5,11 +5,15 @@ go_library(
srcs = ["alloc.go"],
importpath = "github.com/pingcap/tidb/br/pkg/restore/prealloc_table_id",
visibility = ["//visibility:public"],
- deps = ["//br/pkg/metautil"],
+ deps = [
+ "//br/pkg/metautil",
+ "//parser/model",
+ ],
)
go_test(
name = "prealloc_table_id_test",
+ timeout = "short",
srcs = ["alloc_test.go"],
flaky = True,
race = "on",
diff --git a/br/pkg/restore/prealloc_table_id/alloc.go b/br/pkg/restore/prealloc_table_id/alloc.go
index 9232ed84a8fc8..8554de5e9891b 100644
--- a/br/pkg/restore/prealloc_table_id/alloc.go
+++ b/br/pkg/restore/prealloc_table_id/alloc.go
@@ -7,6 +7,7 @@ import (
"math"
"github.com/pingcap/tidb/br/pkg/metautil"
+ "github.com/pingcap/tidb/parser/model"
)
const (
@@ -48,6 +49,14 @@ func New(tables []*metautil.Table) *PreallocIDs {
if t.Info.ID > max && t.Info.ID < insaneTableIDThreshold {
max = t.Info.ID
}
+
+ if t.Info.Partition != nil && t.Info.Partition.Definitions != nil {
+ for _, part := range t.Info.Partition.Definitions {
+ if part.ID > max && part.ID < insaneTableIDThreshold {
+ max = part.ID
+ }
+ }
+ }
}
return &PreallocIDs{
end: max + 1,
@@ -86,3 +95,17 @@ func (p *PreallocIDs) Alloc(m Allocator) error {
func (p *PreallocIDs) Prealloced(tid int64) bool {
return p.allocedFrom <= tid && tid < p.end
}
+
+func (p *PreallocIDs) PreallocedFor(ti *model.TableInfo) bool {
+ if !p.Prealloced(ti.ID) {
+ return false
+ }
+ if ti.Partition != nil && ti.Partition.Definitions != nil {
+ for _, part := range ti.Partition.Definitions {
+ if !p.Prealloced(part.ID) {
+ return false
+ }
+ }
+ }
+ return true
+}
diff --git a/br/pkg/restore/prealloc_table_id/alloc_test.go b/br/pkg/restore/prealloc_table_id/alloc_test.go
index 8cf6b95fb070e..c1c3f018a2de8 100644
--- a/br/pkg/restore/prealloc_table_id/alloc_test.go
+++ b/br/pkg/restore/prealloc_table_id/alloc_test.go
@@ -27,6 +27,7 @@ func (t *testAllocator) AdvanceGlobalIDs(n int) (int64, error) {
func TestAllocator(t *testing.T) {
type Case struct {
tableIDs []int64
+ partitions map[int64][]int64
hasAllocatedTo int64
successfullyAllocated []int64
shouldAllocatedTo int64
@@ -57,16 +58,41 @@ func TestAllocator(t *testing.T) {
successfullyAllocated: []int64{5, 6},
shouldAllocatedTo: 7,
},
+ {
+ tableIDs: []int64{1, 2, 5, 6, 7},
+ hasAllocatedTo: 6,
+ successfullyAllocated: []int64{6, 7},
+ shouldAllocatedTo: 13,
+ partitions: map[int64][]int64{
+ 7: {8, 9, 10, 11, 12},
+ },
+ },
+ {
+ tableIDs: []int64{1, 2, 5, 6, 7, 13},
+ hasAllocatedTo: 9,
+ successfullyAllocated: []int64{13},
+ shouldAllocatedTo: 14,
+ partitions: map[int64][]int64{
+ 7: {8, 9, 10, 11, 12},
+ },
+ },
}
run := func(t *testing.T, c Case) {
tables := make([]*metautil.Table, 0, len(c.tableIDs))
for _, id := range c.tableIDs {
- tables = append(tables, &metautil.Table{
+ table := metautil.Table{
Info: &model.TableInfo{
- ID: id,
+ ID: id,
+ Partition: &model.PartitionInfo{},
},
- })
+ }
+ if c.partitions != nil {
+ for _, part := range c.partitions[id] {
+ table.Info.Partition.Definitions = append(table.Info.Partition.Definitions, model.PartitionDefinition{ID: part})
+ }
+ }
+ tables = append(tables, &table)
}
ids := prealloctableid.New(tables)
@@ -74,9 +100,9 @@ func TestAllocator(t *testing.T) {
require.NoError(t, ids.Alloc(&allocator))
allocated := make([]int64, 0, len(c.successfullyAllocated))
- for _, t := range c.tableIDs {
- if ids.Prealloced(t) {
- allocated = append(allocated, t)
+ for _, t := range tables {
+ if ids.PreallocedFor(t.Info) {
+ allocated = append(allocated, t.Info.ID)
}
}
require.ElementsMatch(t, allocated, c.successfullyAllocated)
diff --git a/br/pkg/restore/range.go b/br/pkg/restore/range.go
index 72a76105dd440..6f3fa76325b0c 100644
--- a/br/pkg/restore/range.go
+++ b/br/pkg/restore/range.go
@@ -72,7 +72,9 @@ func SortRanges(ranges []rtree.Range, rewriteRules *RewriteRules) ([]rtree.Range
// RewriteRules contains rules for rewriting keys of tables.
type RewriteRules struct {
- Data []*import_sstpb.RewriteRule
+ Data []*import_sstpb.RewriteRule
+ OldKeyspace []byte
+ NewKeyspace []byte
}
// Append append its argument to this rewrite rules.
diff --git a/br/pkg/restore/split.go b/br/pkg/restore/split.go
index a707d0f086ce9..17e04486587b9 100644
--- a/br/pkg/restore/split.go
+++ b/br/pkg/restore/split.go
@@ -5,12 +5,15 @@ package restore
import (
"bytes"
"context"
+ "sort"
"strconv"
"strings"
+ "sync"
"time"
"github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
sst "github.com/pingcap/kvproto/pkg/import_sstpb"
"github.com/pingcap/kvproto/pkg/pdpb"
"github.com/pingcap/log"
@@ -19,9 +22,12 @@ import (
"github.com/pingcap/tidb/br/pkg/restore/split"
"github.com/pingcap/tidb/br/pkg/rtree"
"github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/utils/iter"
+ "github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/util/codec"
"go.uber.org/multierr"
"go.uber.org/zap"
+ "golang.org/x/sync/errgroup"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
@@ -428,3 +434,426 @@ func replacePrefix(s []byte, rewriteRules *RewriteRules) ([]byte, *sst.RewriteRu
return s, nil
}
+
+type rewriteSplitter struct {
+ rewriteKey []byte
+ tableID int64
+ rule *RewriteRules
+ splitter *split.SplitHelper
+}
+
+type splitHelperIterator struct {
+ tableSplitters []*rewriteSplitter
+}
+
+func (iter *splitHelperIterator) Traverse(fn func(v split.Valued, endKey []byte, rule *RewriteRules) bool) {
+ for _, entry := range iter.tableSplitters {
+ endKey := codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(entry.tableID+1))
+ rule := entry.rule
+ entry.splitter.Traverse(func(v split.Valued) bool {
+ return fn(v, endKey, rule)
+ })
+ }
+}
+
+func NewSplitHelperIteratorForTest(helper *split.SplitHelper, tableID int64, rule *RewriteRules) *splitHelperIterator {
+ return &splitHelperIterator{
+ tableSplitters: []*rewriteSplitter{
+ {
+ tableID: tableID,
+ rule: rule,
+ splitter: helper,
+ },
+ },
+ }
+}
+
+type LogSplitHelper struct {
+ tableSplitter map[int64]*split.SplitHelper
+ rules map[int64]*RewriteRules
+ client split.SplitClient
+ pool *utils.WorkerPool
+ eg *errgroup.Group
+ regionsCh chan []*split.RegionInfo
+
+ splitThreSholdSize uint64
+ splitThreSholdKeys int64
+}
+
+func NewLogSplitHelper(rules map[int64]*RewriteRules, client split.SplitClient, splitSize uint64, splitKeys int64) *LogSplitHelper {
+ return &LogSplitHelper{
+ tableSplitter: make(map[int64]*split.SplitHelper),
+ rules: rules,
+ client: client,
+ pool: utils.NewWorkerPool(128, "split region"),
+ eg: nil,
+
+ splitThreSholdSize: splitSize,
+ splitThreSholdKeys: splitKeys,
+ }
+}
+
+func (helper *LogSplitHelper) iterator() *splitHelperIterator {
+ tableSplitters := make([]*rewriteSplitter, 0, len(helper.tableSplitter))
+ for tableID, splitter := range helper.tableSplitter {
+ delete(helper.tableSplitter, tableID)
+ rewriteRule, exists := helper.rules[tableID]
+ if !exists {
+ log.Info("skip splitting due to no table id matched", zap.Int64("tableID", tableID))
+ continue
+ }
+ newTableID := GetRewriteTableID(tableID, rewriteRule)
+ if newTableID == 0 {
+ log.Warn("failed to get the rewrite table id", zap.Int64("tableID", tableID))
+ continue
+ }
+ tableSplitters = append(tableSplitters, &rewriteSplitter{
+ rewriteKey: codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(newTableID)),
+ tableID: newTableID,
+ rule: rewriteRule,
+ splitter: splitter,
+ })
+ }
+ sort.Slice(tableSplitters, func(i, j int) bool {
+ return bytes.Compare(tableSplitters[i].rewriteKey, tableSplitters[j].rewriteKey) < 0
+ })
+ return &splitHelperIterator{
+ tableSplitters: tableSplitters,
+ }
+}
+
+const splitFileThreshold = 1024 * 1024 // 1 MB
+
+func (helper *LogSplitHelper) skipFile(file *backuppb.DataFileInfo) bool {
+ _, exist := helper.rules[file.TableId]
+ return file.Length < splitFileThreshold || file.IsMeta || !exist
+}
+
+func (helper *LogSplitHelper) Merge(file *backuppb.DataFileInfo) {
+ if helper.skipFile(file) {
+ return
+ }
+ splitHelper, exist := helper.tableSplitter[file.TableId]
+ if !exist {
+ splitHelper = split.NewSplitHelper()
+ helper.tableSplitter[file.TableId] = splitHelper
+ }
+
+ splitHelper.Merge(split.Valued{
+ Key: split.Span{
+ StartKey: file.StartKey,
+ EndKey: file.EndKey,
+ },
+ Value: split.Value{
+ Size: file.Length,
+ Number: file.NumberOfEntries,
+ },
+ })
+}
+
+type splitFunc = func(context.Context, *RegionSplitter, uint64, int64, *split.RegionInfo, []split.Valued) error
+
+func (helper *LogSplitHelper) splitRegionByPoints(
+ ctx context.Context,
+ regionSplitter *RegionSplitter,
+ initialLength uint64,
+ initialNumber int64,
+ region *split.RegionInfo,
+ valueds []split.Valued,
+) error {
+ var (
+ splitPoints [][]byte = make([][]byte, 0)
+ lastKey []byte = region.Region.StartKey
+ length uint64 = initialLength
+ number int64 = initialNumber
+ )
+ for _, v := range valueds {
+ // decode will discard ts behind the key, which results in the same key for consecutive ranges
+ if !bytes.Equal(lastKey, v.GetStartKey()) && (v.Value.Size+length > helper.splitThreSholdSize || v.Value.Number+number > helper.splitThreSholdKeys) {
+ _, rawKey, _ := codec.DecodeBytes(v.GetStartKey(), nil)
+ splitPoints = append(splitPoints, rawKey)
+ length = 0
+ number = 0
+ }
+ lastKey = v.GetStartKey()
+ length += v.Value.Size
+ number += v.Value.Number
+ }
+
+ if len(splitPoints) == 0 {
+ return nil
+ }
+
+ helper.pool.ApplyOnErrorGroup(helper.eg, func() error {
+ newRegions, errSplit := regionSplitter.splitAndScatterRegions(ctx, region, splitPoints)
+ if errSplit != nil {
+ log.Warn("failed to split the scaned region", zap.Error(errSplit))
+ _, startKey, _ := codec.DecodeBytes(region.Region.StartKey, nil)
+ ranges := make([]rtree.Range, 0, len(splitPoints))
+ for _, point := range splitPoints {
+ ranges = append(ranges, rtree.Range{StartKey: startKey, EndKey: point})
+ startKey = point
+ }
+
+ return regionSplitter.Split(ctx, ranges, nil, false, func([][]byte) {})
+ }
+ select {
+ case <-ctx.Done():
+ return nil
+ case helper.regionsCh <- newRegions:
+ }
+ log.Info("split the region", zap.Uint64("region-id", region.Region.Id), zap.Int("split-point-number", len(splitPoints)))
+ return nil
+ })
+ return nil
+}
+
+// GetRewriteTableID gets rewrite table id by the rewrite rule and original table id
+func GetRewriteTableID(tableID int64, rewriteRules *RewriteRules) int64 {
+ tableKey := tablecodec.GenTableRecordPrefix(tableID)
+ rule := matchOldPrefix(tableKey, rewriteRules)
+ if rule == nil {
+ return 0
+ }
+
+ return tablecodec.DecodeTableID(rule.GetNewKeyPrefix())
+}
+
+// SplitPoint selects ranges overlapped with each region, and calls `splitF` to split the region
+func SplitPoint(
+ ctx context.Context,
+ iter *splitHelperIterator,
+ client split.SplitClient,
+ splitF splitFunc,
+) (err error) {
+ // common status
+ var (
+ regionSplitter *RegionSplitter = NewRegionSplitter(client)
+ )
+ // region traverse status
+ var (
+ // the region buffer of each scan
+ regions []*split.RegionInfo = nil
+ regionIndex int = 0
+ )
+ // region split status
+ var (
+ // range span +----------------+------+---+-------------+
+ // region span +------------------------------------+
+ // +initial length+ +end valued+
+ // regionValueds is the ranges array overlapped with `regionInfo`
+ regionValueds []split.Valued = nil
+ // regionInfo is the region to be split
+ regionInfo *split.RegionInfo = nil
+ // intialLength is the length of the part of the first range overlapped with the region
+ initialLength uint64 = 0
+ initialNumber int64 = 0
+ )
+ // range status
+ var (
+ // regionOverCount is the number of regions overlapped with the range
+ regionOverCount uint64 = 0
+ )
+
+ iter.Traverse(func(v split.Valued, endKey []byte, rule *RewriteRules) bool {
+ if v.Value.Number == 0 || v.Value.Size == 0 {
+ return true
+ }
+ var (
+ vStartKey []byte
+ vEndKey []byte
+ )
+ // use `vStartKey` and `vEndKey` to compare with region's key
+ vStartKey, vEndKey, err = GetRewriteEncodedKeys(v, rule)
+ if err != nil {
+ return false
+ }
+ // traverse to the first region overlapped with the range
+ for ; regionIndex < len(regions); regionIndex++ {
+ if bytes.Compare(vStartKey, regions[regionIndex].Region.EndKey) < 0 {
+ break
+ }
+ }
+ // cannot find any regions overlapped with the range
+ // need to scan regions again
+ if regionIndex == len(regions) {
+ regions = nil
+ }
+ regionOverCount = 0
+ for {
+ if regionIndex >= len(regions) {
+ var startKey []byte
+ if len(regions) > 0 {
+ // has traversed over the region buffer, should scan from the last region's end-key of the region buffer
+ startKey = regions[len(regions)-1].Region.EndKey
+ } else {
+ // scan from the range's start-key
+ startKey = vStartKey
+ }
+ // scan at most 64 regions into the region buffer
+ regions, err = split.ScanRegionsWithRetry(ctx, client, startKey, endKey, 64)
+ if err != nil {
+ return false
+ }
+ regionIndex = 0
+ }
+
+ region := regions[regionIndex]
+ // this region must be overlapped with the range
+ regionOverCount++
+ // the region is the last one overlapped with the range,
+ // should split the last recorded region,
+ // and then record this region as the region to be split
+ if bytes.Compare(vEndKey, region.Region.EndKey) < 0 {
+ endLength := v.Value.Size / regionOverCount
+ endNumber := v.Value.Number / int64(regionOverCount)
+ if len(regionValueds) > 0 && regionInfo != region {
+ // add a part of the range as the end part
+ if bytes.Compare(vStartKey, regionInfo.Region.EndKey) < 0 {
+ regionValueds = append(regionValueds, split.NewValued(vStartKey, regionInfo.Region.EndKey, split.Value{Size: endLength, Number: endNumber}))
+ }
+ // try to split the region
+ err = splitF(ctx, regionSplitter, initialLength, initialNumber, regionInfo, regionValueds)
+ if err != nil {
+ return false
+ }
+ regionValueds = make([]split.Valued, 0)
+ }
+ if regionOverCount == 1 {
+ // the region completely contains the range
+ regionValueds = append(regionValueds, split.Valued{
+ Key: split.Span{
+ StartKey: vStartKey,
+ EndKey: vEndKey,
+ },
+ Value: v.Value,
+ })
+ } else {
+ // the region is overlapped with the last part of the range
+ initialLength = endLength
+ initialNumber = endNumber
+ }
+ regionInfo = region
+ // try the next range
+ return true
+ }
+
+ // try the next region
+ regionIndex++
+ }
+ })
+
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if len(regionValueds) > 0 {
+ // try to split the region
+ err = splitF(ctx, regionSplitter, initialLength, initialNumber, regionInfo, regionValueds)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
+
+ return nil
+}
+
+func (helper *LogSplitHelper) Split(ctx context.Context) error {
+ var ectx context.Context
+ var wg sync.WaitGroup
+ helper.eg, ectx = errgroup.WithContext(ctx)
+ helper.regionsCh = make(chan []*split.RegionInfo, 1024)
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ scatterRegions := make([]*split.RegionInfo, 0)
+ receiveNewRegions:
+ for {
+ select {
+ case <-ectx.Done():
+ return
+ case newRegions, ok := <-helper.regionsCh:
+ if !ok {
+ break receiveNewRegions
+ }
+
+ scatterRegions = append(scatterRegions, newRegions...)
+ }
+ }
+
+ startTime := time.Now()
+ regionSplitter := NewRegionSplitter(helper.client)
+ for _, region := range scatterRegions {
+ regionSplitter.waitForScatterRegion(ctx, region)
+ // It is too expensive to stop recovery and wait for a small number of regions
+ // to complete scatter, so the maximum waiting time is reduced to 1 minute.
+ if time.Since(startTime) > time.Minute {
+ break
+ }
+ }
+ }()
+
+ iter := helper.iterator()
+ if err := SplitPoint(ectx, iter, helper.client, helper.splitRegionByPoints); err != nil {
+ return errors.Trace(err)
+ }
+
+ // wait for completion of splitting regions
+ if err := helper.eg.Wait(); err != nil {
+ return errors.Trace(err)
+ }
+
+ // wait for completion of scattering regions
+ close(helper.regionsCh)
+ wg.Wait()
+
+ return nil
+}
+
+type LogFilesIterWithSplitHelper struct {
+ iter LogIter
+ helper *LogSplitHelper
+ buffer []*backuppb.DataFileInfo
+ next int
+}
+
+const SplitFilesBufferSize = 4096
+
+func NewLogFilesIterWithSplitHelper(iter LogIter, rules map[int64]*RewriteRules, client split.SplitClient, splitSize uint64, splitKeys int64) LogIter {
+ return &LogFilesIterWithSplitHelper{
+ iter: iter,
+ helper: NewLogSplitHelper(rules, client, splitSize, splitKeys),
+ buffer: nil,
+ next: 0,
+ }
+}
+
+func (splitIter *LogFilesIterWithSplitHelper) TryNext(ctx context.Context) iter.IterResult[*backuppb.DataFileInfo] {
+ if splitIter.next >= len(splitIter.buffer) {
+ splitIter.buffer = make([]*backuppb.DataFileInfo, 0, SplitFilesBufferSize)
+ for r := splitIter.iter.TryNext(ctx); !r.Finished; r = splitIter.iter.TryNext(ctx) {
+ if r.Err != nil {
+ return r
+ }
+ f := r.Item
+ splitIter.helper.Merge(f)
+ splitIter.buffer = append(splitIter.buffer, f)
+ if len(splitIter.buffer) >= SplitFilesBufferSize {
+ break
+ }
+ }
+ splitIter.next = 0
+ if len(splitIter.buffer) == 0 {
+ return iter.Done[*backuppb.DataFileInfo]()
+ }
+ log.Info("start to split the regions")
+ startTime := time.Now()
+ if err := splitIter.helper.Split(ctx); err != nil {
+ return iter.Throw[*backuppb.DataFileInfo](errors.Trace(err))
+ }
+ log.Info("end to split the regions", zap.Duration("takes", time.Since(startTime)))
+ }
+
+ res := iter.Emit(splitIter.buffer[splitIter.next])
+ splitIter.next += 1
+ return res
+}
diff --git a/br/pkg/restore/split/BUILD.bazel b/br/pkg/restore/split/BUILD.bazel
index 49fbec82c543c..7c91f32cfae90 100644
--- a/br/pkg/restore/split/BUILD.bazel
+++ b/br/pkg/restore/split/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "split",
@@ -6,6 +6,7 @@ go_library(
"client.go",
"region.go",
"split.go",
+ "sum_sorted.go",
],
importpath = "github.com/pingcap/tidb/br/pkg/restore/split",
visibility = ["//visibility:public"],
@@ -13,10 +14,13 @@ go_library(
"//br/pkg/conn/util",
"//br/pkg/errors",
"//br/pkg/httputil",
+ "//br/pkg/lightning/config",
"//br/pkg/logutil",
"//br/pkg/redact",
"//br/pkg/utils",
+ "//kv",
"//store/pdtypes",
+ "@com_github_google_btree//:btree",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/errorpb",
@@ -29,8 +33,20 @@ go_library(
"@org_golang_google_grpc//:grpc",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//credentials",
+ "@org_golang_google_grpc//credentials/insecure",
"@org_golang_google_grpc//status",
"@org_uber_go_multierr//:multierr",
"@org_uber_go_zap//:zap",
],
)
+
+go_test(
+ name = "split_test",
+ timeout = "short",
+ srcs = ["sum_sorted_test.go"],
+ flaky = True,
+ deps = [
+ ":split",
+ "@com_github_stretchr_testify//require",
+ ],
+)
diff --git a/br/pkg/restore/split/client.go b/br/pkg/restore/split/client.go
index fb82135ae2af6..72482a94e87dc 100644
--- a/br/pkg/restore/split/client.go
+++ b/br/pkg/restore/split/client.go
@@ -27,6 +27,7 @@ import (
"github.com/pingcap/tidb/br/pkg/conn/util"
berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/pingcap/tidb/br/pkg/httputil"
+ "github.com/pingcap/tidb/br/pkg/lightning/config"
"github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/store/pdtypes"
pd "github.com/tikv/pd/client"
@@ -35,6 +36,7 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
)
@@ -200,7 +202,9 @@ func (c *pdClient) SplitRegion(ctx context.Context, regionInfo *RegionInfo, key
if err != nil {
return nil, errors.Trace(err)
}
- conn, err := grpc.Dial(store.GetAddress(), grpc.WithInsecure())
+ conn, err := grpc.Dial(store.GetAddress(),
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ config.DefaultGrpcKeepaliveParams)
if err != nil {
return nil, errors.Trace(err)
}
@@ -336,11 +340,12 @@ func sendSplitRegionRequest(ctx context.Context, c *pdClient, regionInfo *Region
if err != nil {
return false, nil, err
}
- opt := grpc.WithInsecure()
+ opt := grpc.WithTransportCredentials(insecure.NewCredentials())
if c.tlsConf != nil {
opt = grpc.WithTransportCredentials(credentials.NewTLS(c.tlsConf))
}
- conn, err := grpc.Dial(store.GetAddress(), opt)
+ conn, err := grpc.Dial(store.GetAddress(), opt,
+ config.DefaultGrpcKeepaliveParams)
if err != nil {
return false, nil, err
}
diff --git a/br/pkg/restore/split/split.go b/br/pkg/restore/split/split.go
index 6af36a400a03f..e06c8ab1c93d5 100644
--- a/br/pkg/restore/split/split.go
+++ b/br/pkg/restore/split/split.go
@@ -18,7 +18,7 @@ import (
)
var (
- ScanRegionAttemptTimes = 60
+ ScanRegionAttemptTimes = 128
)
// Constants for split retry machinery.
@@ -121,6 +121,65 @@ func PaginateScanRegion(
return regions, err
}
+// CheckPartRegionConsistency only checks the continuity of regions and the first region consistency.
+func CheckPartRegionConsistency(startKey, endKey []byte, regions []*RegionInfo) error {
+ // current pd can't guarantee the consistency of returned regions
+ if len(regions) == 0 {
+ return errors.Annotatef(berrors.ErrPDBatchScanRegion, "scan region return empty result, startKey: %s, endKey: %s",
+ redact.Key(startKey), redact.Key(endKey))
+ }
+
+ if bytes.Compare(regions[0].Region.StartKey, startKey) > 0 {
+ return errors.Annotatef(berrors.ErrPDBatchScanRegion, "first region's startKey > startKey, startKey: %s, regionStartKey: %s",
+ redact.Key(startKey), redact.Key(regions[0].Region.StartKey))
+ }
+
+ cur := regions[0]
+ for _, r := range regions[1:] {
+ if !bytes.Equal(cur.Region.EndKey, r.Region.StartKey) {
+ return errors.Annotatef(berrors.ErrPDBatchScanRegion, "region endKey not equal to next region startKey, endKey: %s, startKey: %s",
+ redact.Key(cur.Region.EndKey), redact.Key(r.Region.StartKey))
+ }
+ cur = r
+ }
+
+ return nil
+}
+
+func ScanRegionsWithRetry(
+ ctx context.Context, client SplitClient, startKey, endKey []byte, limit int,
+) ([]*RegionInfo, error) {
+ if len(endKey) != 0 && bytes.Compare(startKey, endKey) > 0 {
+ return nil, errors.Annotatef(berrors.ErrRestoreInvalidRange, "startKey > endKey, startKey: %s, endkey: %s",
+ hex.EncodeToString(startKey), hex.EncodeToString(endKey))
+ }
+
+ var regions []*RegionInfo
+ var err error
+ // we don't need to return multierr. since there only 3 times retry.
+ // in most case 3 times retry have the same error. so we just return the last error.
+ // actually we'd better remove all multierr in br/lightning.
+ // because it's not easy to check multierr equals normal error.
+ // see https://github.com/pingcap/tidb/issues/33419.
+ _ = utils.WithRetry(ctx, func() error {
+ regions, err = client.ScanRegions(ctx, startKey, endKey, limit)
+ if err != nil {
+ err = errors.Annotatef(berrors.ErrPDBatchScanRegion, "scan regions from start-key:%s, err: %s",
+ redact.Key(startKey), err.Error())
+ return err
+ }
+
+ if err = CheckPartRegionConsistency(startKey, endKey, regions); err != nil {
+ log.Warn("failed to scan region, retrying", logutil.ShortError(err))
+ return err
+ }
+
+ return nil
+ }, newScanRegionBackoffer())
+
+ return regions, err
+}
+
type scanRegionBackoffer struct {
attempt int
}
diff --git a/br/pkg/restore/split/sum_sorted.go b/br/pkg/restore/split/sum_sorted.go
new file mode 100644
index 0000000000000..c4e9657900e35
--- /dev/null
+++ b/br/pkg/restore/split/sum_sorted.go
@@ -0,0 +1,204 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+package split
+
+import (
+ "bytes"
+ "fmt"
+
+ "github.com/google/btree"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/kv"
+)
+
+// Value is the value type of stored in the span tree.
+type Value struct {
+ Size uint64
+ Number int64
+}
+
+// join finds the upper bound of two values.
+func join(a, b Value) Value {
+ return Value{
+ Size: a.Size + b.Size,
+ Number: a.Number + b.Number,
+ }
+}
+
+// Span is the type of an adjacent sub key space.
+type Span = kv.KeyRange
+
+// Valued is span binding to a value, which is the entry type of span tree.
+type Valued struct {
+ Key Span
+ Value Value
+}
+
+func NewValued(startKey, endKey []byte, value Value) Valued {
+ return Valued{
+ Key: Span{
+ StartKey: startKey,
+ EndKey: endKey,
+ },
+ Value: value,
+ }
+}
+
+func (v Valued) String() string {
+ return fmt.Sprintf("(%s, %.2f MB, %d)", logutil.StringifyRange(v.Key), float64(v.Value.Size)/1024/1024, v.Value.Number)
+}
+
+func (v Valued) Less(other btree.Item) bool {
+ return bytes.Compare(v.Key.StartKey, other.(Valued).Key.StartKey) < 0
+}
+
+// implement for `AppliedFile`
+func (v Valued) GetStartKey() []byte {
+ return v.Key.StartKey
+}
+
+// implement for `AppliedFile`
+func (v Valued) GetEndKey() []byte {
+ return v.Key.EndKey
+}
+
+// SplitHelper represents a set of valued ranges, which doesn't overlap and union of them all is the full key space.
+type SplitHelper struct {
+ inner *btree.BTree
+}
+
+// NewSplitHelper creates a set of a subset of spans, with the full key space as initial status
+func NewSplitHelper() *SplitHelper {
+ t := btree.New(16)
+ t.ReplaceOrInsert(Valued{Value: Value{Size: 0, Number: 0}, Key: Span{StartKey: []byte(""), EndKey: []byte("")}})
+ return &SplitHelper{inner: t}
+}
+
+func (f *SplitHelper) Merge(val Valued) {
+ if len(val.Key.StartKey) == 0 || len(val.Key.EndKey) == 0 {
+ return
+ }
+ overlaps := make([]Valued, 0, 8)
+ f.overlapped(val.Key, &overlaps)
+ f.mergeWithOverlap(val, overlaps)
+}
+
+// traverse the items in ascend order
+func (f *SplitHelper) Traverse(m func(Valued) bool) {
+ f.inner.Ascend(func(item btree.Item) bool {
+ return m(item.(Valued))
+ })
+}
+
+func (f *SplitHelper) mergeWithOverlap(val Valued, overlapped []Valued) {
+ // There isn't any range overlaps with the input range, perhaps the input range is empty.
+ // do nothing for this case.
+ if len(overlapped) == 0 {
+ return
+ }
+
+ for _, r := range overlapped {
+ f.inner.Delete(r)
+ }
+ // Assert All overlapped ranges are deleted.
+
+ // the new valued item's Value is equally dividedd into `len(overlapped)` shares
+ appendValue := Value{
+ Size: val.Value.Size / uint64(len(overlapped)),
+ Number: val.Value.Number / int64(len(overlapped)),
+ }
+ var (
+ rightTrail *Valued
+ leftTrail *Valued
+ // overlapped ranges +-------------+----------+
+ // new valued item +-------------+
+ // a b c d e
+ // the part [a,b] is `standalone` because it is not overlapped with the new valued item
+ // the part [a,b] and [b,c] are `split` because they are from range [a,c]
+ emitToCollected = func(rng Valued, standalone bool, split bool) {
+ merged := rng.Value
+ if split {
+ merged.Size /= 2
+ merged.Number /= 2
+ }
+ if !standalone {
+ merged = join(appendValue, merged)
+ }
+ rng.Value = merged
+ f.inner.ReplaceOrInsert(rng)
+ }
+ )
+
+ leftmost := overlapped[0]
+ if bytes.Compare(leftmost.Key.StartKey, val.Key.StartKey) < 0 {
+ leftTrail = &Valued{
+ Key: Span{StartKey: leftmost.Key.StartKey, EndKey: val.Key.StartKey},
+ Value: leftmost.Value,
+ }
+ overlapped[0].Key.StartKey = val.Key.StartKey
+ }
+
+ rightmost := overlapped[len(overlapped)-1]
+ if utils.CompareBytesExt(rightmost.Key.EndKey, true, val.Key.EndKey, true) > 0 {
+ rightTrail = &Valued{
+ Key: Span{StartKey: val.Key.EndKey, EndKey: rightmost.Key.EndKey},
+ Value: rightmost.Value,
+ }
+ overlapped[len(overlapped)-1].Key.EndKey = val.Key.EndKey
+ if len(overlapped) == 1 && leftTrail != nil {
+ // (split) (split) (split)
+ // overlapped ranges +-----------------------------+
+ // new valued item +-------------+
+ // a b c d
+ // now the overlapped range should be divided into 3 equal parts
+ // so modify the value to the 2/3x to be compatible with function `emitToCollected`
+ val := Value{
+ Size: rightTrail.Value.Size * 2 / 3,
+ Number: rightTrail.Value.Number * 2 / 3,
+ }
+ leftTrail.Value = val
+ overlapped[0].Value = val
+ rightTrail.Value = val
+ }
+ }
+
+ if leftTrail != nil {
+ emitToCollected(*leftTrail, true, true)
+ }
+
+ for i, rng := range overlapped {
+ split := (i == 0 && leftTrail != nil) || (i == len(overlapped)-1 && rightTrail != nil)
+ emitToCollected(rng, false, split)
+ }
+
+ if rightTrail != nil {
+ emitToCollected(*rightTrail, true, true)
+ }
+}
+
+// overlapped inserts the overlapped ranges of the span into the `result` slice.
+func (f *SplitHelper) overlapped(k Span, result *[]Valued) {
+ var first Span
+ f.inner.DescendLessOrEqual(Valued{Key: k}, func(item btree.Item) bool {
+ first = item.(Valued).Key
+ return false
+ })
+
+ f.inner.AscendGreaterOrEqual(Valued{Key: first}, func(item btree.Item) bool {
+ r := item.(Valued)
+ if !checkOverlaps(r.Key, k) {
+ return false
+ }
+ *result = append(*result, r)
+ return true
+ })
+}
+
+// checkOverlaps checks whether two spans have overlapped part.
+// `ap` should be a finite range
+func checkOverlaps(a, ap Span) bool {
+ if len(a.EndKey) == 0 {
+ return bytes.Compare(ap.EndKey, a.StartKey) > 0
+ }
+ return bytes.Compare(a.StartKey, ap.EndKey) < 0 && bytes.Compare(ap.StartKey, a.EndKey) < 0
+}
diff --git a/br/pkg/restore/split/sum_sorted_test.go b/br/pkg/restore/split/sum_sorted_test.go
new file mode 100644
index 0000000000000..3a3b3db6d90eb
--- /dev/null
+++ b/br/pkg/restore/split/sum_sorted_test.go
@@ -0,0 +1,141 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+package split_test
+
+import (
+ "testing"
+
+ "github.com/pingcap/tidb/br/pkg/restore/split"
+ "github.com/stretchr/testify/require"
+)
+
+func v(s, e string, val split.Value) split.Valued {
+ return split.Valued{
+ Key: split.Span{
+ StartKey: []byte(s),
+ EndKey: []byte(e),
+ },
+ Value: val,
+ }
+}
+
+func mb(b uint64) split.Value {
+ return split.Value{
+ Size: b * 1024 * 1024,
+ Number: int64(b),
+ }
+}
+
+func TestSumSorted(t *testing.T) {
+ cases := []struct {
+ values []split.Valued
+ result []uint64
+ }{
+ {
+ values: []split.Valued{
+ v("a", "f", mb(100)),
+ v("a", "c", mb(200)),
+ v("d", "g", mb(100)),
+ },
+ result: []uint64{0, 250, 25, 75, 50, 0},
+ },
+ {
+ values: []split.Valued{
+ v("a", "f", mb(100)),
+ v("a", "c", mb(200)),
+ v("d", "f", mb(100)),
+ },
+ result: []uint64{0, 250, 25, 125, 0},
+ },
+ {
+ values: []split.Valued{
+ v("a", "f", mb(100)),
+ v("a", "c", mb(200)),
+ v("c", "f", mb(100)),
+ },
+ result: []uint64{0, 250, 150, 0},
+ },
+ {
+ values: []split.Valued{
+ v("a", "f", mb(100)),
+ v("a", "c", mb(200)),
+ v("c", "f", mb(100)),
+ v("da", "db", mb(100)),
+ },
+ result: []uint64{0, 250, 50, 150, 50, 0},
+ },
+ {
+ values: []split.Valued{
+ v("a", "f", mb(100)),
+ v("a", "c", mb(200)),
+ v("c", "f", mb(100)),
+ v("da", "db", mb(100)),
+ v("cb", "db", mb(100)),
+ },
+ result: []uint64{0, 250, 25, 75, 200, 50, 0},
+ },
+ {
+ values: []split.Valued{
+ v("a", "f", mb(100)),
+ v("a", "c", mb(200)),
+ v("c", "f", mb(100)),
+ v("da", "db", mb(100)),
+ v("cb", "f", mb(150)),
+ },
+ result: []uint64{0, 250, 25, 75, 200, 100, 0},
+ },
+ {
+ values: []split.Valued{
+ v("a", "f", mb(100)),
+ v("a", "c", mb(200)),
+ v("c", "f", mb(100)),
+ v("da", "db", mb(100)),
+ v("cb", "df", mb(150)),
+ },
+ result: []uint64{0, 250, 25, 75, 200, 75, 25, 0},
+ },
+ {
+ values: []split.Valued{
+ v("a", "f", mb(100)),
+ v("a", "c", mb(200)),
+ v("c", "f", mb(100)),
+ v("da", "db", mb(100)),
+ v("cb", "df", mb(150)),
+ },
+ result: []uint64{0, 250, 25, 75, 200, 75, 25, 0},
+ },
+ {
+ values: []split.Valued{
+ v("a", "f", mb(100)),
+ v("a", "c", mb(200)),
+ v("c", "f", mb(100)),
+ v("da", "db", mb(100)),
+ v("c", "df", mb(150)),
+ },
+ result: []uint64{0, 250, 100, 200, 75, 25, 0},
+ },
+ {
+ values: []split.Valued{
+ v("a", "f", mb(100)),
+ v("a", "c", mb(200)),
+ v("c", "f", mb(100)),
+ v("da", "db", mb(100)),
+ v("c", "f", mb(150)),
+ },
+ result: []uint64{0, 250, 100, 200, 100, 0},
+ },
+ }
+
+ for _, ca := range cases {
+ full := split.NewSplitHelper()
+ for _, v := range ca.values {
+ full.Merge(v)
+ }
+
+ i := 0
+ full.Traverse(func(v split.Valued) bool {
+ require.Equal(t, mb(ca.result[i]), v.Value)
+ i++
+ return true
+ })
+ }
+}
diff --git a/br/pkg/restore/split_test.go b/br/pkg/restore/split_test.go
index b726a5ec78729..1b560a4e1474d 100644
--- a/br/pkg/restore/split_test.go
+++ b/br/pkg/restore/split_test.go
@@ -5,6 +5,7 @@ package restore_test
import (
"bytes"
"context"
+ "fmt"
"sync"
"testing"
"time"
@@ -22,7 +23,9 @@ import (
"github.com/pingcap/tidb/br/pkg/restore/split"
"github.com/pingcap/tidb/br/pkg/rtree"
"github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/utils/iter"
"github.com/pingcap/tidb/store/pdtypes"
+ "github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/util/codec"
"github.com/stretchr/testify/require"
"go.uber.org/multierr"
@@ -729,3 +732,316 @@ func TestSplitFailed(t *testing.T) {
require.GreaterOrEqual(t, len(r.splitRanges), 2)
require.Len(t, r.restoredFiles, 0)
}
+
+func keyWithTablePrefix(tableID int64, key string) []byte {
+ rawKey := append(tablecodec.GenTableRecordPrefix(tableID), []byte(key)...)
+ return codec.EncodeBytes([]byte{}, rawKey)
+}
+
+func TestSplitPoint(t *testing.T) {
+ ctx := context.Background()
+ var oldTableID int64 = 50
+ var tableID int64 = 100
+ rewriteRules := &restore.RewriteRules{
+ Data: []*import_sstpb.RewriteRule{
+ {
+ OldKeyPrefix: tablecodec.EncodeTablePrefix(oldTableID),
+ NewKeyPrefix: tablecodec.EncodeTablePrefix(tableID),
+ },
+ },
+ }
+
+ // range: b c d e g i
+ // +---+ +---+ +---------+
+ // +-------------+----------+---------+
+ // region: a f h j
+ splitHelper := split.NewSplitHelper()
+ splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "b"), EndKey: keyWithTablePrefix(oldTableID, "c")}, Value: split.Value{Size: 100, Number: 100}})
+ splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "d"), EndKey: keyWithTablePrefix(oldTableID, "e")}, Value: split.Value{Size: 200, Number: 200}})
+ splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "g"), EndKey: keyWithTablePrefix(oldTableID, "i")}, Value: split.Value{Size: 300, Number: 300}})
+ client := NewFakeSplitClient()
+ client.AppendRegion(keyWithTablePrefix(tableID, "a"), keyWithTablePrefix(tableID, "f"))
+ client.AppendRegion(keyWithTablePrefix(tableID, "f"), keyWithTablePrefix(tableID, "h"))
+ client.AppendRegion(keyWithTablePrefix(tableID, "h"), keyWithTablePrefix(tableID, "j"))
+ client.AppendRegion(keyWithTablePrefix(tableID, "j"), keyWithTablePrefix(tableID+1, "a"))
+
+ iter := restore.NewSplitHelperIteratorForTest(splitHelper, tableID, rewriteRules)
+ err := restore.SplitPoint(ctx, iter, client, func(ctx context.Context, rs *restore.RegionSplitter, u uint64, o int64, ri *split.RegionInfo, v []split.Valued) error {
+ require.Equal(t, u, uint64(0))
+ require.Equal(t, o, int64(0))
+ require.Equal(t, ri.Region.StartKey, keyWithTablePrefix(tableID, "a"))
+ require.Equal(t, ri.Region.EndKey, keyWithTablePrefix(tableID, "f"))
+ require.EqualValues(t, v[0].Key.StartKey, keyWithTablePrefix(tableID, "b"))
+ require.EqualValues(t, v[0].Key.EndKey, keyWithTablePrefix(tableID, "c"))
+ require.EqualValues(t, v[1].Key.StartKey, keyWithTablePrefix(tableID, "d"))
+ require.EqualValues(t, v[1].Key.EndKey, keyWithTablePrefix(tableID, "e"))
+ require.Equal(t, len(v), 2)
+ return nil
+ })
+ require.NoError(t, err)
+}
+
+func getCharFromNumber(prefix string, i int) string {
+ c := '1' + (i % 10)
+ b := '1' + (i%100)/10
+ a := '1' + i/100
+ return fmt.Sprintf("%s%c%c%c", prefix, a, b, c)
+}
+
+func TestSplitPoint2(t *testing.T) {
+ ctx := context.Background()
+ var oldTableID int64 = 50
+ var tableID int64 = 100
+ rewriteRules := &restore.RewriteRules{
+ Data: []*import_sstpb.RewriteRule{
+ {
+ OldKeyPrefix: tablecodec.EncodeTablePrefix(oldTableID),
+ NewKeyPrefix: tablecodec.EncodeTablePrefix(tableID),
+ },
+ },
+ }
+
+ // range: b c d e f i j k l n
+ // +---+ +---+ +-----------------+ +----+ +--------+
+ // +---------------+--+.....+----+------------+---------+
+ // region: a g >128 h m o
+ splitHelper := split.NewSplitHelper()
+ splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "b"), EndKey: keyWithTablePrefix(oldTableID, "c")}, Value: split.Value{Size: 100, Number: 100}})
+ splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "d"), EndKey: keyWithTablePrefix(oldTableID, "e")}, Value: split.Value{Size: 200, Number: 200}})
+ splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "f"), EndKey: keyWithTablePrefix(oldTableID, "i")}, Value: split.Value{Size: 300, Number: 300}})
+ splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "j"), EndKey: keyWithTablePrefix(oldTableID, "k")}, Value: split.Value{Size: 200, Number: 200}})
+ splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "l"), EndKey: keyWithTablePrefix(oldTableID, "n")}, Value: split.Value{Size: 200, Number: 200}})
+ client := NewFakeSplitClient()
+ client.AppendRegion(keyWithTablePrefix(tableID, "a"), keyWithTablePrefix(tableID, "g"))
+ client.AppendRegion(keyWithTablePrefix(tableID, "g"), keyWithTablePrefix(tableID, getCharFromNumber("g", 0)))
+ for i := 0; i < 256; i++ {
+ client.AppendRegion(keyWithTablePrefix(tableID, getCharFromNumber("g", i)), keyWithTablePrefix(tableID, getCharFromNumber("g", i+1)))
+ }
+ client.AppendRegion(keyWithTablePrefix(tableID, getCharFromNumber("g", 256)), keyWithTablePrefix(tableID, "h"))
+ client.AppendRegion(keyWithTablePrefix(tableID, "h"), keyWithTablePrefix(tableID, "m"))
+ client.AppendRegion(keyWithTablePrefix(tableID, "m"), keyWithTablePrefix(tableID, "o"))
+ client.AppendRegion(keyWithTablePrefix(tableID, "o"), keyWithTablePrefix(tableID+1, "a"))
+
+ firstSplit := true
+ iter := restore.NewSplitHelperIteratorForTest(splitHelper, tableID, rewriteRules)
+ err := restore.SplitPoint(ctx, iter, client, func(ctx context.Context, rs *restore.RegionSplitter, u uint64, o int64, ri *split.RegionInfo, v []split.Valued) error {
+ if firstSplit {
+ require.Equal(t, u, uint64(0))
+ require.Equal(t, o, int64(0))
+ require.Equal(t, ri.Region.StartKey, keyWithTablePrefix(tableID, "a"))
+ require.Equal(t, ri.Region.EndKey, keyWithTablePrefix(tableID, "g"))
+ require.EqualValues(t, v[0].Key.StartKey, keyWithTablePrefix(tableID, "b"))
+ require.EqualValues(t, v[0].Key.EndKey, keyWithTablePrefix(tableID, "c"))
+ require.EqualValues(t, v[1].Key.StartKey, keyWithTablePrefix(tableID, "d"))
+ require.EqualValues(t, v[1].Key.EndKey, keyWithTablePrefix(tableID, "e"))
+ require.EqualValues(t, v[2].Key.StartKey, keyWithTablePrefix(tableID, "f"))
+ require.EqualValues(t, v[2].Key.EndKey, keyWithTablePrefix(tableID, "g"))
+ require.Equal(t, v[2].Value.Size, uint64(1))
+ require.Equal(t, v[2].Value.Number, int64(1))
+ require.Equal(t, len(v), 3)
+ firstSplit = false
+ } else {
+ require.Equal(t, u, uint64(1))
+ require.Equal(t, o, int64(1))
+ require.Equal(t, ri.Region.StartKey, keyWithTablePrefix(tableID, "h"))
+ require.Equal(t, ri.Region.EndKey, keyWithTablePrefix(tableID, "m"))
+ require.EqualValues(t, v[0].Key.StartKey, keyWithTablePrefix(tableID, "j"))
+ require.EqualValues(t, v[0].Key.EndKey, keyWithTablePrefix(tableID, "k"))
+ require.EqualValues(t, v[1].Key.StartKey, keyWithTablePrefix(tableID, "l"))
+ require.EqualValues(t, v[1].Key.EndKey, keyWithTablePrefix(tableID, "m"))
+ require.Equal(t, v[1].Value.Size, uint64(100))
+ require.Equal(t, v[1].Value.Number, int64(100))
+ require.Equal(t, len(v), 2)
+ }
+ return nil
+ })
+ require.NoError(t, err)
+}
+
+type fakeSplitClient struct {
+ regions []*split.RegionInfo
+}
+
+func NewFakeSplitClient() *fakeSplitClient {
+ return &fakeSplitClient{
+ regions: make([]*split.RegionInfo, 0),
+ }
+}
+
+func (f *fakeSplitClient) AppendRegion(startKey, endKey []byte) {
+ f.regions = append(f.regions, &split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: startKey,
+ EndKey: endKey,
+ },
+ })
+}
+
+func (*fakeSplitClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) {
+ return nil, nil
+}
+func (*fakeSplitClient) GetRegion(ctx context.Context, key []byte) (*split.RegionInfo, error) {
+ return nil, nil
+}
+func (*fakeSplitClient) GetRegionByID(ctx context.Context, regionID uint64) (*split.RegionInfo, error) {
+ return nil, nil
+}
+func (*fakeSplitClient) SplitRegion(ctx context.Context, regionInfo *split.RegionInfo, key []byte) (*split.RegionInfo, error) {
+ return nil, nil
+}
+func (*fakeSplitClient) BatchSplitRegions(ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte) ([]*split.RegionInfo, error) {
+ return nil, nil
+}
+func (*fakeSplitClient) BatchSplitRegionsWithOrigin(ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte) (*split.RegionInfo, []*split.RegionInfo, error) {
+ return nil, nil, nil
+}
+func (*fakeSplitClient) ScatterRegion(ctx context.Context, regionInfo *split.RegionInfo) error {
+ return nil
+}
+func (*fakeSplitClient) ScatterRegions(ctx context.Context, regionInfo []*split.RegionInfo) error {
+ return nil
+}
+func (*fakeSplitClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) {
+ return nil, nil
+}
+func (f *fakeSplitClient) ScanRegions(ctx context.Context, startKey, endKey []byte, limit int) ([]*split.RegionInfo, error) {
+ result := make([]*split.RegionInfo, 0)
+ count := 0
+ for _, rng := range f.regions {
+ if bytes.Compare(rng.Region.StartKey, endKey) <= 0 && bytes.Compare(rng.Region.EndKey, startKey) > 0 {
+ result = append(result, rng)
+ count++
+ }
+ if count >= limit {
+ break
+ }
+ }
+ return result, nil
+}
+func (*fakeSplitClient) GetPlacementRule(ctx context.Context, groupID, ruleID string) (pdtypes.Rule, error) {
+ return pdtypes.Rule{}, nil
+}
+func (*fakeSplitClient) SetPlacementRule(ctx context.Context, rule pdtypes.Rule) error { return nil }
+func (*fakeSplitClient) DeletePlacementRule(ctx context.Context, groupID, ruleID string) error {
+ return nil
+}
+func (*fakeSplitClient) SetStoresLabel(ctx context.Context, stores []uint64, labelKey, labelValue string) error {
+ return nil
+}
+
+func TestGetRewriteTableID(t *testing.T) {
+ var tableID int64 = 76
+ var oldTableID int64 = 80
+ {
+ rewriteRules := &restore.RewriteRules{
+ Data: []*import_sstpb.RewriteRule{
+ {
+ OldKeyPrefix: tablecodec.EncodeTablePrefix(oldTableID),
+ NewKeyPrefix: tablecodec.EncodeTablePrefix(tableID),
+ },
+ },
+ }
+
+ newTableID := restore.GetRewriteTableID(oldTableID, rewriteRules)
+ require.Equal(t, tableID, newTableID)
+ }
+
+ {
+ rewriteRules := &restore.RewriteRules{
+ Data: []*import_sstpb.RewriteRule{
+ {
+ OldKeyPrefix: tablecodec.GenTableRecordPrefix(oldTableID),
+ NewKeyPrefix: tablecodec.GenTableRecordPrefix(tableID),
+ },
+ },
+ }
+
+ newTableID := restore.GetRewriteTableID(oldTableID, rewriteRules)
+ require.Equal(t, tableID, newTableID)
+ }
+}
+
+type mockLogIter struct {
+ next int
+}
+
+func (m *mockLogIter) TryNext(ctx context.Context) iter.IterResult[*backuppb.DataFileInfo] {
+ if m.next > 10000 {
+ return iter.Done[*backuppb.DataFileInfo]()
+ }
+ m.next += 1
+ return iter.Emit(&backuppb.DataFileInfo{
+ StartKey: []byte(fmt.Sprintf("a%d", m.next)),
+ EndKey: []byte("b"),
+ Length: 1024, // 1 KB
+ })
+}
+
+func TestLogFilesIterWithSplitHelper(t *testing.T) {
+ var tableID int64 = 76
+ var oldTableID int64 = 80
+ rewriteRules := &restore.RewriteRules{
+ Data: []*import_sstpb.RewriteRule{
+ {
+ OldKeyPrefix: tablecodec.EncodeTablePrefix(oldTableID),
+ NewKeyPrefix: tablecodec.EncodeTablePrefix(tableID),
+ },
+ },
+ }
+ rewriteRulesMap := map[int64]*restore.RewriteRules{
+ oldTableID: rewriteRules,
+ }
+ mockIter := &mockLogIter{}
+ ctx := context.Background()
+ logIter := restore.NewLogFilesIterWithSplitHelper(mockIter, rewriteRulesMap, NewFakeSplitClient(), 144*1024*1024, 1440000)
+ next := 0
+ for r := logIter.TryNext(ctx); !r.Finished; r = logIter.TryNext(ctx) {
+ require.NoError(t, r.Err)
+ next += 1
+ require.Equal(t, []byte(fmt.Sprintf("a%d", next)), r.Item.StartKey)
+ }
+}
+
+func regionInfo(startKey, endKey string) *split.RegionInfo {
+ return &split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte(startKey),
+ EndKey: []byte(endKey),
+ },
+ }
+}
+
+func TestSplitCheckPartRegionConsistency(t *testing.T) {
+ var (
+ startKey []byte = []byte("a")
+ endKey []byte = []byte("f")
+ err error
+ )
+ err = split.CheckPartRegionConsistency(startKey, endKey, nil)
+ require.Error(t, err)
+ err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
+ regionInfo("b", "c"),
+ })
+ require.Error(t, err)
+ err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
+ regionInfo("a", "c"),
+ regionInfo("d", "e"),
+ })
+ require.Error(t, err)
+ err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
+ regionInfo("a", "c"),
+ regionInfo("c", "d"),
+ })
+ require.NoError(t, err)
+ err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
+ regionInfo("a", "c"),
+ regionInfo("c", "d"),
+ regionInfo("d", "f"),
+ })
+ require.NoError(t, err)
+ err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
+ regionInfo("a", "c"),
+ regionInfo("c", "z"),
+ })
+ require.NoError(t, err)
+}
diff --git a/br/pkg/restore/stream_metas_test.go b/br/pkg/restore/stream_metas_test.go
index 5b75e9de6b3d8..407f5a0154ca3 100644
--- a/br/pkg/restore/stream_metas_test.go
+++ b/br/pkg/restore/stream_metas_test.go
@@ -317,7 +317,7 @@ func TestTruncateSafepointForGCS(t *testing.T) {
CredentialsBlob: "Fake Credentials",
}
- l, err := storage.NewGCSStorageForTest(ctx, gcs, &storage.ExternalStorageOptions{
+ l, err := storage.NewGCSStorage(ctx, gcs, &storage.ExternalStorageOptions{
SendCredentials: false,
CheckPermissions: []storage.Permission{storage.AccessBuckets},
HTTPClient: server.HTTPClient(),
diff --git a/br/pkg/restore/systable_restore.go b/br/pkg/restore/systable_restore.go
index 40e3450c772f2..ac21b0dba7e42 100644
--- a/br/pkg/restore/systable_restore.go
+++ b/br/pkg/restore/systable_restore.go
@@ -19,6 +19,12 @@ import (
"go.uber.org/zap"
)
+const (
+ rootUser = "root"
+ sysUserTableName = "user"
+ cloudAdminUser = "cloud_admin"
+)
+
var statsTables = map[string]struct{}{
"stats_buckets": {},
"stats_extended": {},
@@ -49,14 +55,14 @@ var unRecoverableTable = map[string]struct{}{
// skip clearing or restoring 'cloud_admin'@'%' which is a special
// user on TiDB Cloud
var sysPrivilegeTableMap = map[string]string{
- "user": "not (user = 'cloud_admin' and host = '%')", // since v1.0.0
- "db": "not (user = 'cloud_admin' and host = '%')", // since v1.0.0
- "tables_priv": "not (user = 'cloud_admin' and host = '%')", // since v1.0.0
- "columns_priv": "not (user = 'cloud_admin' and host = '%')", // since v1.0.0
- "default_roles": "not (user = 'cloud_admin' and host = '%')", // since v3.0.0
- "role_edges": "not (to_user = 'cloud_admin' and to_host = '%')", // since v3.0.0
- "global_priv": "not (user = 'cloud_admin' and host = '%')", // since v3.0.8
- "global_grants": "not (user = 'cloud_admin' and host = '%')", // since v5.0.3
+ "user": "(user = '%s' and host = '%%')", // since v1.0.0
+ "db": "(user = '%s' and host = '%%')", // since v1.0.0
+ "tables_priv": "(user = '%s' and host = '%%')", // since v1.0.0
+ "columns_priv": "(user = '%s' and host = '%%')", // since v1.0.0
+ "default_roles": "(user = '%s' and host = '%%')", // since v3.0.0
+ "role_edges": "(to_user = '%s' and to_host = '%%')", // since v3.0.0
+ "global_priv": "(user = '%s' and host = '%%')", // since v3.0.8
+ "global_grants": "(user = '%s' and host = '%%')", // since v5.0.3
}
func isUnrecoverableTable(tableName string) bool {
@@ -69,6 +75,78 @@ func isStatsTable(tableName string) bool {
return ok
}
+func generateResetSQLs(db *database, resetUsers []string) []string {
+ if db.Name.L != mysql.SystemDB {
+ return nil
+ }
+ sqls := make([]string, 0, 10)
+ // we only need reset root password once
+ rootReset := false
+ for tableName := range db.ExistingTables {
+ if sysPrivilegeTableMap[tableName] != "" {
+ for _, name := range resetUsers {
+ if strings.ToLower(name) == rootUser {
+ if !rootReset {
+ updateSQL := fmt.Sprintf("UPDATE %s.%s SET authentication_string='',"+
+ " Shutdown_priv='Y',"+
+ " Config_priv='Y'"+
+ " WHERE USER='root' AND Host='%%';",
+ db.Name.L, sysUserTableName)
+ sqls = append(sqls, updateSQL)
+ rootReset = true
+ } else {
+ continue
+ }
+ } else {
+ /* #nosec G202: SQL string concatenation */
+ whereClause := fmt.Sprintf("WHERE "+sysPrivilegeTableMap[tableName], name)
+ deleteSQL := fmt.Sprintf("DELETE FROM %s %s;",
+ utils.EncloseDBAndTable(db.Name.L, tableName), whereClause)
+ sqls = append(sqls, deleteSQL)
+ }
+ }
+ }
+ }
+ return sqls
+}
+
+// ClearSystemUsers is used for volume-snapshot restoration.
+// because we can not support restore user in some scenarios, for example in cloud.
+// we'd better use this function to drop cloud_admin user after volume-snapshot restore.
+func (rc *Client) ClearSystemUsers(ctx context.Context, resetUsers []string) error {
+ sysDB := mysql.SystemDB
+ db, ok := rc.getDatabaseByName(sysDB)
+ if !ok {
+ log.Warn("target database not exist, aborting", zap.String("database", sysDB))
+ return nil
+ }
+ execSQL := func(sql string) error {
+ // SQLs here only contain table name and database name, seems it is no need to redact them.
+ if err := rc.db.se.Execute(ctx, sql); err != nil {
+ log.Warn("failed to clear system users",
+ zap.Stringer("database", db.Name),
+ zap.String("sql", sql),
+ zap.Error(err),
+ )
+ return berrors.ErrUnknown.Wrap(err).GenWithStack("failed to execute %s", sql)
+ }
+ log.Info("successfully clear system users after restoration",
+ zap.Stringer("database", db.Name),
+ zap.String("sql", sql),
+ )
+ return nil
+ }
+
+ sqls := generateResetSQLs(db, resetUsers)
+ for _, sql := range sqls {
+ log.Info("reset system user for cloud", zap.String("sql", sql))
+ if err := execSQL(sql); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
// RestoreSystemSchemas restores the system schema(i.e. the `mysql` schema).
// Detail see https://github.com/pingcap/br/issues/679#issuecomment-762592254.
func (rc *Client) RestoreSystemSchemas(ctx context.Context, f filter.Filter) {
@@ -201,14 +279,15 @@ func (rc *Client) replaceTemporaryTableToSystable(ctx context.Context, ti *model
}
if db.ExistingTables[tableName] != nil {
- whereClause := ""
+ whereNotClause := ""
if rc.fullClusterRestore && sysPrivilegeTableMap[tableName] != "" {
// cloud_admin is a special user on tidb cloud, need to skip it.
- whereClause = fmt.Sprintf("WHERE %s", sysPrivilegeTableMap[tableName])
+ /* #nosec G202: SQL string concatenation */
+ whereNotClause = fmt.Sprintf("WHERE NOT "+sysPrivilegeTableMap[tableName], cloudAdminUser)
log.Info("full cluster restore, delete existing data",
zap.String("table", tableName), zap.Stringer("schema", db.Name))
deleteSQL := fmt.Sprintf("DELETE FROM %s %s;",
- utils.EncloseDBAndTable(db.Name.L, tableName), whereClause)
+ utils.EncloseDBAndTable(db.Name.L, tableName), whereNotClause)
if err := execSQL(deleteSQL); err != nil {
return err
}
@@ -226,7 +305,7 @@ func (rc *Client) replaceTemporaryTableToSystable(ctx context.Context, ti *model
utils.EncloseDBAndTable(db.Name.L, tableName),
colListStr, colListStr,
utils.EncloseDBAndTable(db.TemporaryName.L, tableName),
- whereClause)
+ whereNotClause)
return execSQL(replaceIntoSQL)
}
diff --git a/br/pkg/restore/systable_restore_test.go b/br/pkg/restore/systable_restore_test.go
new file mode 100644
index 0000000000000..2371f066a43a1
--- /dev/null
+++ b/br/pkg/restore/systable_restore_test.go
@@ -0,0 +1,72 @@
+// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
+
+package restore
+
+import (
+ "regexp"
+ "testing"
+
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/stretchr/testify/require"
+)
+
+func testTableInfo(name string) *model.TableInfo {
+ return &model.TableInfo{
+ Name: model.NewCIStr(name),
+ }
+}
+
+func TestGenerateResetSQL(t *testing.T) {
+ // case #1: ignore non-mysql databases
+ mockDB := &database{
+ ExistingTables: map[string]*model.TableInfo{},
+ Name: model.NewCIStr("non-mysql"),
+ TemporaryName: utils.TemporaryDBName("non-mysql"),
+ }
+ for name := range sysPrivilegeTableMap {
+ mockDB.ExistingTables[name] = testTableInfo(name)
+ }
+ resetUsers := []string{"cloud_admin", "root"}
+ require.Equal(t, 0, len(generateResetSQLs(mockDB, resetUsers)))
+
+ // case #2: ignore non expected table
+ mockDB = &database{
+ ExistingTables: map[string]*model.TableInfo{},
+ Name: model.NewCIStr("mysql"),
+ TemporaryName: utils.TemporaryDBName("mysql"),
+ }
+ for name := range sysPrivilegeTableMap {
+ name += "non_available"
+ mockDB.ExistingTables[name] = testTableInfo(name)
+ }
+ resetUsers = []string{"cloud_admin", "root"}
+ require.Equal(t, 0, len(generateResetSQLs(mockDB, resetUsers)))
+
+ // case #3: only reset cloud admin account
+ for name := range sysPrivilegeTableMap {
+ mockDB.ExistingTables[name] = testTableInfo(name)
+ }
+ resetUsers = []string{"cloud_admin"}
+ sqls := generateResetSQLs(mockDB, resetUsers)
+ require.Equal(t, 8, len(sqls))
+ for _, sql := range sqls {
+ // for cloud_admin we only generate DELETE sql
+ require.Regexp(t, regexp.MustCompile("DELETE*"), sql)
+ }
+
+ // case #4: reset cloud admin/other account
+ resetUsers = []string{"cloud_admin", "cloud_other"}
+ sqls = generateResetSQLs(mockDB, resetUsers)
+ require.Equal(t, 16, len(sqls))
+ for _, sql := range sqls {
+ // for cloud_admin/cloud_other we only generate DELETE sql
+ require.Regexp(t, regexp.MustCompile("DELETE*"), sql)
+ }
+
+ // case #5: reset cloud admin && root account
+ resetUsers = []string{"cloud_admin", "root"}
+ sqls = generateResetSQLs(mockDB, resetUsers)
+ // 8 DELETE sqls for cloud admin and 1 UPDATE sql for root
+ require.Equal(t, 9, len(sqls))
+}
diff --git a/br/pkg/restore/tiflashrec/BUILD.bazel b/br/pkg/restore/tiflashrec/BUILD.bazel
index fbe6ba415cb8c..2025370a8d9d0 100644
--- a/br/pkg/restore/tiflashrec/BUILD.bazel
+++ b/br/pkg/restore/tiflashrec/BUILD.bazel
@@ -19,6 +19,7 @@ go_library(
go_test(
name = "tiflashrec_test",
+ timeout = "short",
srcs = ["tiflash_recorder_test.go"],
flaky = True,
deps = [
diff --git a/br/pkg/restore/tiflashrec/tiflash_recorder.go b/br/pkg/restore/tiflashrec/tiflash_recorder.go
index 31dde982a7b69..84707f05e1f1b 100644
--- a/br/pkg/restore/tiflashrec/tiflash_recorder.go
+++ b/br/pkg/restore/tiflashrec/tiflash_recorder.go
@@ -79,6 +79,46 @@ func (r *TiFlashRecorder) Rewrite(oldID int64, newID int64) {
}
}
+func (r *TiFlashRecorder) GenerateResetAlterTableDDLs(info infoschema.InfoSchema) []string {
+ items := make([]string, 0, len(r.items))
+ r.Iterate(func(id int64, replica model.TiFlashReplicaInfo) {
+ table, ok := info.TableByID(id)
+ if !ok {
+ log.Warn("Table do not exist, skipping", zap.Int64("id", id))
+ return
+ }
+ schema, ok := info.SchemaByTable(table.Meta())
+ if !ok {
+ log.Warn("Schema do not exist, skipping", zap.Int64("id", id), zap.Stringer("table", table.Meta().Name))
+ return
+ }
+ // Currently, we didn't backup tiflash cluster volume during volume snapshot backup,
+ // But the table has replica info after volume restoration.
+ // We should reset it to 0, then set it back. otherwise, it will return error when alter tiflash replica.
+ altTableSpec, err := alterTableSpecOf(replica, true)
+ if err != nil {
+ log.Warn("Failed to generate the alter table spec", logutil.ShortError(err), zap.Any("replica", replica))
+ return
+ }
+ items = append(items, fmt.Sprintf(
+ "ALTER TABLE %s %s",
+ utils.EncloseDBAndTable(schema.Name.O, table.Meta().Name.O),
+ altTableSpec),
+ )
+ altTableSpec, err = alterTableSpecOf(replica, false)
+ if err != nil {
+ log.Warn("Failed to generate the alter table spec", logutil.ShortError(err), zap.Any("replica", replica))
+ return
+ }
+ items = append(items, fmt.Sprintf(
+ "ALTER TABLE %s %s",
+ utils.EncloseDBAndTable(schema.Name.O, table.Meta().Name.O),
+ altTableSpec),
+ )
+ })
+ return items
+}
+
func (r *TiFlashRecorder) GenerateAlterTableDDLs(info infoschema.InfoSchema) []string {
items := make([]string, 0, len(r.items))
r.Iterate(func(id int64, replica model.TiFlashReplicaInfo) {
@@ -92,7 +132,7 @@ func (r *TiFlashRecorder) GenerateAlterTableDDLs(info infoschema.InfoSchema) []s
log.Warn("Schema do not exist, skipping", zap.Int64("id", id), zap.Stringer("table", table.Meta().Name))
return
}
- altTableSpec, err := alterTableSpecOf(replica)
+ altTableSpec, err := alterTableSpecOf(replica, false)
if err != nil {
log.Warn("Failed to generate the alter table spec", logutil.ShortError(err), zap.Any("replica", replica))
return
@@ -106,7 +146,7 @@ func (r *TiFlashRecorder) GenerateAlterTableDDLs(info infoschema.InfoSchema) []s
return items
}
-func alterTableSpecOf(replica model.TiFlashReplicaInfo) (string, error) {
+func alterTableSpecOf(replica model.TiFlashReplicaInfo, reset bool) (string, error) {
spec := &ast.AlterTableSpec{
Tp: ast.AlterTableSetTiFlashReplica,
TiFlashReplica: &ast.TiFlashReplicaSpec{
@@ -114,6 +154,14 @@ func alterTableSpecOf(replica model.TiFlashReplicaInfo) (string, error) {
Labels: replica.LocationLabels,
},
}
+ if reset {
+ spec = &ast.AlterTableSpec{
+ Tp: ast.AlterTableSetTiFlashReplica,
+ TiFlashReplica: &ast.TiFlashReplicaSpec{
+ Count: 0,
+ },
+ }
+ }
buf := bytes.NewBuffer(make([]byte, 0, 32))
restoreCx := format.NewRestoreCtx(
diff --git a/br/pkg/restore/tiflashrec/tiflash_recorder_test.go b/br/pkg/restore/tiflashrec/tiflash_recorder_test.go
index b01272caeddc5..f7316a1ed3133 100644
--- a/br/pkg/restore/tiflashrec/tiflash_recorder_test.go
+++ b/br/pkg/restore/tiflashrec/tiflash_recorder_test.go
@@ -170,3 +170,32 @@ func TestGenSql(t *testing.T) {
"ALTER TABLE `test`.`evils` SET TIFLASH REPLICA 1 LOCATION LABELS 'kIll''; OR DROP DATABASE test --', 'dEaTh with " + `\\"quoting\\"` + "'",
})
}
+
+func TestGenResetSql(t *testing.T) {
+ tInfo := func(id int, name string) *model.TableInfo {
+ return &model.TableInfo{
+ ID: int64(id),
+ Name: model.NewCIStr(name),
+ }
+ }
+ fakeInfo := infoschema.MockInfoSchema([]*model.TableInfo{
+ tInfo(1, "fruits"),
+ tInfo(2, "whisper"),
+ })
+ rec := tiflashrec.New()
+ rec.AddTable(1, model.TiFlashReplicaInfo{
+ Count: 1,
+ })
+ rec.AddTable(2, model.TiFlashReplicaInfo{
+ Count: 2,
+ LocationLabels: []string{"climate"},
+ })
+
+ sqls := rec.GenerateResetAlterTableDDLs(fakeInfo)
+ require.ElementsMatch(t, sqls, []string{
+ "ALTER TABLE `test`.`whisper` SET TIFLASH REPLICA 0",
+ "ALTER TABLE `test`.`whisper` SET TIFLASH REPLICA 2 LOCATION LABELS 'climate'",
+ "ALTER TABLE `test`.`fruits` SET TIFLASH REPLICA 0",
+ "ALTER TABLE `test`.`fruits` SET TIFLASH REPLICA 1",
+ })
+}
diff --git a/br/pkg/restore/util.go b/br/pkg/restore/util.go
index 259d3fa28d888..b29ef6674c74f 100644
--- a/br/pkg/restore/util.go
+++ b/br/pkg/restore/util.go
@@ -196,7 +196,25 @@ func GetSSTMetaFromFile(
file *backuppb.File,
region *metapb.Region,
regionRule *import_sstpb.RewriteRule,
-) import_sstpb.SSTMeta {
+ rewriteMode RewriteMode,
+) (meta *import_sstpb.SSTMeta, err error) {
+ r := *region
+ // If the rewrite mode is for keyspace, then the region bound should be decoded.
+ if rewriteMode == RewriteModeKeyspace {
+ if len(region.GetStartKey()) > 0 {
+ _, r.StartKey, err = codec.DecodeBytes(region.GetStartKey(), nil)
+ if err != nil {
+ return
+ }
+ }
+ if len(region.GetEndKey()) > 0 {
+ _, r.EndKey, err = codec.DecodeBytes(region.GetEndKey(), nil)
+ if err != nil {
+ return
+ }
+ }
+ }
+
// Get the column family of the file by the file name.
var cfName string
if strings.Contains(file.GetName(), defaultCFName) {
@@ -208,8 +226,8 @@ func GetSSTMetaFromFile(
// Here we rewrites the keys to compare with the keys of the region.
rangeStart := regionRule.GetNewKeyPrefix()
// rangeStart = max(rangeStart, region.StartKey)
- if bytes.Compare(rangeStart, region.GetStartKey()) < 0 {
- rangeStart = region.GetStartKey()
+ if bytes.Compare(rangeStart, r.GetStartKey()) < 0 {
+ rangeStart = r.GetStartKey()
}
// Append 10 * 0xff to make sure rangeEnd cover all file key
@@ -219,8 +237,8 @@ func GetSSTMetaFromFile(
suffix := []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
rangeEnd := append(append([]byte{}, regionRule.GetNewKeyPrefix()...), suffix...)
// rangeEnd = min(rangeEnd, region.EndKey)
- if len(region.GetEndKey()) > 0 && bytes.Compare(rangeEnd, region.GetEndKey()) > 0 {
- rangeEnd = region.GetEndKey()
+ if len(r.GetEndKey()) > 0 && bytes.Compare(rangeEnd, r.GetEndKey()) > 0 {
+ rangeEnd = r.GetEndKey()
}
if bytes.Compare(rangeStart, rangeEnd) > 0 {
@@ -235,7 +253,7 @@ func GetSSTMetaFromFile(
logutil.Key("startKey", rangeStart),
logutil.Key("endKey", rangeEnd))
- return import_sstpb.SSTMeta{
+ return &import_sstpb.SSTMeta{
Uuid: id,
CfName: cfName,
Range: &import_sstpb.Range{
@@ -246,7 +264,7 @@ func GetSSTMetaFromFile(
RegionId: region.GetId(),
RegionEpoch: region.GetRegionEpoch(),
CipherIv: file.GetCipherIv(),
- }
+ }, nil
}
// makeDBPool makes a session pool with specficated size by sessionFactory.
@@ -750,3 +768,43 @@ func CheckConsistencyAndValidPeer(regionInfos []*RecoverRegionInfo) (map[uint64]
}
return validPeers, nil
}
+
+// in cloud, since iops and bandwidth limitation, write operator in raft is slow, so raft state (logterm, lastlog, commitlog...) are the same among the peers
+// LeaderCandidates select all peers can be select as a leader during the restore
+func LeaderCandidates(peers []*RecoverRegion) ([]*RecoverRegion, error) {
+ if peers == nil {
+ return nil, errors.Annotatef(berrors.ErrRestoreRegionWithoutPeer,
+ "invalid region range")
+ }
+ candidates := make([]*RecoverRegion, 0, len(peers))
+ // by default, the peers[0] to be assign as a leader, since peers already sorted by leader selection rule
+ leader := peers[0]
+ candidates = append(candidates, leader)
+ for _, peer := range peers[1:] {
+ // qualificated candidate is leader.logterm = candidate.logterm && leader.lastindex = candidate.lastindex && && leader.commitindex = candidate.commitindex
+ if peer.LastLogTerm == leader.LastLogTerm && peer.LastIndex == leader.LastIndex && peer.CommitIndex == leader.CommitIndex {
+ log.Debug("leader candidate", zap.Uint64("store id", peer.StoreId), zap.Uint64("region id", peer.RegionId), zap.Uint64("peer id", peer.PeerId))
+ candidates = append(candidates, peer)
+ }
+ }
+ return candidates, nil
+}
+
+// for region A, has candidate leader x, y, z
+// peer x on store 1 with storeBalanceScore 3
+// peer y on store 3 with storeBalanceScore 2
+// peer z on store 4 with storeBalanceScore 1
+// result: peer z will be select as leader on store 4
+func SelectRegionLeader(storeBalanceScore map[uint64]int, peers []*RecoverRegion) *RecoverRegion {
+ // by default, the peers[0] to be assign as a leader
+ leader := peers[0]
+ minLeaderStore := storeBalanceScore[leader.StoreId]
+ for _, peer := range peers[1:] {
+ log.Debug("leader candidate", zap.Int("score", storeBalanceScore[peer.StoreId]), zap.Int("min-score", minLeaderStore), zap.Uint64("store id", peer.StoreId), zap.Uint64("region id", peer.RegionId), zap.Uint64("peer id", peer.PeerId))
+ if storeBalanceScore[peer.StoreId] < minLeaderStore {
+ minLeaderStore = storeBalanceScore[peer.StoreId]
+ leader = peer
+ }
+ }
+ return leader
+}
diff --git a/br/pkg/restore/util_test.go b/br/pkg/restore/util_test.go
index 44620e9cb4e5c..4af6922efa30a 100644
--- a/br/pkg/restore/util_test.go
+++ b/br/pkg/restore/util_test.go
@@ -52,7 +52,8 @@ func TestGetSSTMetaFromFile(t *testing.T) {
StartKey: []byte("t2abc"),
EndKey: []byte("t3a"),
}
- sstMeta := restore.GetSSTMetaFromFile([]byte{}, file, region, rule)
+ sstMeta, err := restore.GetSSTMetaFromFile([]byte{}, file, region, rule, restore.RewriteModeLegacy)
+ require.Nil(t, err)
require.Equal(t, "t2abc", string(sstMeta.GetRange().GetStart()))
require.Equal(t, "t2\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", string(sstMeta.GetRange().GetEnd()))
}
@@ -460,3 +461,52 @@ func TestCheckConsistencyAndValidPeer(t *testing.T) {
require.Error(t, err)
require.Regexp(t, ".*invalid restore range.*", err.Error())
}
+
+func TestLeaderCandidates(t *testing.T) {
+ //key space is continuous
+ validPeer1 := newPeerMeta(9, 11, 2, []byte(""), []byte("bb"), 2, 1, 0, 0, false)
+ validPeer2 := newPeerMeta(19, 22, 3, []byte("bb"), []byte("cc"), 2, 1, 0, 1, false)
+ validPeer3 := newPeerMeta(29, 30, 1, []byte("cc"), []byte(""), 2, 1, 0, 2, false)
+
+ peers := []*restore.RecoverRegion{
+ validPeer1,
+ validPeer2,
+ validPeer3,
+ }
+
+ candidates, err := restore.LeaderCandidates(peers)
+ require.NoError(t, err)
+ require.Equal(t, 3, len(candidates))
+}
+
+func TestSelectRegionLeader(t *testing.T) {
+ validPeer1 := newPeerMeta(9, 11, 2, []byte(""), []byte("bb"), 2, 1, 0, 0, false)
+ validPeer2 := newPeerMeta(19, 22, 3, []byte("bb"), []byte("cc"), 2, 1, 0, 1, false)
+ validPeer3 := newPeerMeta(29, 30, 1, []byte("cc"), []byte(""), 2, 1, 0, 2, false)
+
+ peers := []*restore.RecoverRegion{
+ validPeer1,
+ validPeer2,
+ validPeer3,
+ }
+ // init store banlance score all is 0
+ storeBalanceScore := make(map[uint64]int, len(peers))
+ leader := restore.SelectRegionLeader(storeBalanceScore, peers)
+ require.Equal(t, validPeer1, leader)
+
+ // change store banlance store
+ storeBalanceScore[2] = 3
+ storeBalanceScore[3] = 2
+ storeBalanceScore[1] = 1
+ leader = restore.SelectRegionLeader(storeBalanceScore, peers)
+ require.Equal(t, validPeer3, leader)
+
+ // one peer
+ peer := []*restore.RecoverRegion{
+ validPeer3,
+ }
+ // init store banlance score all is 0
+ storeScore := make(map[uint64]int, len(peer))
+ leader = restore.SelectRegionLeader(storeScore, peer)
+ require.Equal(t, validPeer3, leader)
+}
diff --git a/br/pkg/rtree/rtree.go b/br/pkg/rtree/rtree.go
index 9f12b22daca75..f17ebf38df510 100644
--- a/br/pkg/rtree/rtree.go
+++ b/br/pkg/rtree/rtree.go
@@ -217,3 +217,10 @@ func (rangeTree *RangeTree) GetIncompleteRange(
}
return incomplete
}
+
+type ProgressRange struct {
+ Res RangeTree
+ Incomplete []Range
+ Origin Range
+ GroupKey string
+}
diff --git a/br/pkg/storage/BUILD.bazel b/br/pkg/storage/BUILD.bazel
index c67a17713b2ca..68b0a8b03aca2 100644
--- a/br/pkg/storage/BUILD.bazel
+++ b/br/pkg/storage/BUILD.bazel
@@ -35,6 +35,7 @@ go_library(
"@com_github_aws_aws_sdk_go//service/s3",
"@com_github_aws_aws_sdk_go//service/s3/s3iface",
"@com_github_aws_aws_sdk_go//service/s3/s3manager",
+ "@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
"@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
"@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:azblob",
"@com_github_golang_snappy//:snappy",
@@ -55,7 +56,7 @@ go_library(
go_test(
name = "storage_test",
- timeout = "short",
+ timeout = "moderate",
srcs = [
"azblob_test.go",
"compress_test.go",
diff --git a/br/pkg/storage/azblob.go b/br/pkg/storage/azblob.go
index c557a79e3ac8f..41d8fa88f559f 100644
--- a/br/pkg/storage/azblob.go
+++ b/br/pkg/storage/azblob.go
@@ -12,6 +12,7 @@ import (
"path"
"strings"
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/google/uuid"
@@ -30,6 +31,16 @@ const (
azblobAccountKey = "azblob.account-key"
)
+const azblobRetryTimes int32 = 5
+
+func getDefaultClientOptions() *azblob.ClientOptions {
+ return &azblob.ClientOptions{
+ Retry: policy.RetryOptions{
+ MaxRetries: azblobRetryTimes,
+ },
+ }
+}
+
// AzblobBackendOptions is the options for Azure Blob storage.
type AzblobBackendOptions struct {
Endpoint string `json:"endpoint" toml:"endpoint"`
@@ -99,7 +110,7 @@ type sharedKeyClientBuilder struct {
}
func (b *sharedKeyClientBuilder) GetServiceClient() (azblob.ServiceClient, error) {
- return azblob.NewServiceClientWithSharedKey(b.serviceURL, b.cred, nil)
+ return azblob.NewServiceClientWithSharedKey(b.serviceURL, b.cred, getDefaultClientOptions())
}
func (b *sharedKeyClientBuilder) GetAccountName() string {
@@ -114,7 +125,7 @@ type tokenClientBuilder struct {
}
func (b *tokenClientBuilder) GetServiceClient() (azblob.ServiceClient, error) {
- return azblob.NewServiceClient(b.serviceURL, b.cred, nil)
+ return azblob.NewServiceClient(b.serviceURL, b.cred, getDefaultClientOptions())
}
func (b *tokenClientBuilder) GetAccountName() string {
@@ -285,7 +296,9 @@ func (s *AzureBlobStorage) ReadFile(ctx context.Context, name string) ([]byte, e
return nil, errors.Annotatef(err, "Failed to download azure blob file, file info: bucket(container)='%s', key='%s'", s.options.Bucket, s.withPrefix(name))
}
defer resp.RawResponse.Body.Close()
- data, err := io.ReadAll(resp.Body(azblob.RetryReaderOptions{}))
+ data, err := io.ReadAll(resp.Body(azblob.RetryReaderOptions{
+ MaxRetryRequests: int(azblobRetryTimes),
+ }))
if err != nil {
return nil, errors.Annotatef(err, "Failed to read azure blob file, file info: bucket(container)='%s', key='%s'", s.options.Bucket, s.withPrefix(name))
}
diff --git a/br/pkg/storage/azblob_test.go b/br/pkg/storage/azblob_test.go
index c099037ea51b2..74ddfa7125699 100644
--- a/br/pkg/storage/azblob_test.go
+++ b/br/pkg/storage/azblob_test.go
@@ -4,9 +4,13 @@ package storage
import (
"context"
+ "fmt"
"io"
+ "net/http"
+ "net/http/httptest"
"os"
"strings"
+ "sync"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
@@ -298,3 +302,52 @@ func TestNewAzblobStorage(t *testing.T) {
require.Equal(t, "http://127.0.0.1:1000", b.serviceURL)
}
}
+
+type fakeClientBuilder struct {
+ Endpoint string
+}
+
+func (b *fakeClientBuilder) GetServiceClient() (azblob.ServiceClient, error) {
+ connStr := fmt.Sprintf("DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=%s/devstoreaccount1;", b.Endpoint)
+ return azblob.NewServiceClientFromConnectionString(connStr, getDefaultClientOptions())
+}
+
+func (b *fakeClientBuilder) GetAccountName() string {
+ return "devstoreaccount1"
+}
+
+func TestDownloadRetry(t *testing.T) {
+ var count int32 = 0
+ var lock sync.Mutex
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ t.Log(r.URL)
+ if strings.Contains(r.URL.String(), "restype=container") {
+ w.WriteHeader(201)
+ return
+ }
+ lock.Lock()
+ count += 1
+ lock.Unlock()
+ header := w.Header()
+ header.Add("Etag", "0x1")
+ header.Add("Content-Length", "5")
+ w.WriteHeader(200)
+ w.Write([]byte("1234567"))
+ }))
+
+ defer server.Close()
+ t.Log(server.URL)
+
+ options := &backuppb.AzureBlobStorage{
+ Bucket: "test",
+ Prefix: "a/b/",
+ }
+
+ ctx := context.Background()
+ builder := &fakeClientBuilder{Endpoint: server.URL}
+ s, err := newAzureBlobStorageWithClientBuilder(ctx, options, builder)
+ require.NoError(t, err)
+ _, err = s.ReadFile(ctx, "c")
+ require.Error(t, err)
+ require.Less(t, azblobRetryTimes, count)
+}
diff --git a/br/pkg/storage/compress.go b/br/pkg/storage/compress.go
index 1d5300cfa8d55..36c07846f3271 100644
--- a/br/pkg/storage/compress.go
+++ b/br/pkg/storage/compress.go
@@ -80,8 +80,10 @@ func (w *withCompression) ReadFile(ctx context.Context, name string) ([]byte, er
return io.ReadAll(compressBf)
}
+// compressReader is a wrapper for compress.Reader
type compressReader struct {
io.Reader
+ io.Seeker
io.Closer
}
@@ -97,11 +99,30 @@ func newInterceptReader(fileReader ExternalFileReader, compressType CompressType
return &compressReader{
Reader: r,
Closer: fileReader,
+ Seeker: fileReader,
}, nil
}
-func (*compressReader) Seek(_ int64, _ int) (int64, error) {
- return int64(0), errors.Annotatef(berrors.ErrStorageInvalidConfig, "compressReader doesn't support Seek now")
+func NewLimitedInterceptReader(fileReader ExternalFileReader, compressType CompressType, n int64) (ExternalFileReader, error) {
+ newFileReader := fileReader
+ if n < 0 {
+ return nil, errors.Annotatef(berrors.ErrStorageInvalidConfig, "compressReader doesn't support negative limit, n: %d", n)
+ } else if n > 0 {
+ newFileReader = &compressReader{
+ Reader: io.LimitReader(fileReader, n),
+ Seeker: fileReader,
+ Closer: fileReader,
+ }
+ }
+ return newInterceptReader(newFileReader, compressType)
+}
+
+func (c *compressReader) Seek(offset int64, whence int) (int64, error) {
+ // only support get original reader's current offset
+ if offset == 0 && whence == io.SeekCurrent {
+ return c.Seeker.Seek(offset, whence)
+ }
+ return int64(0), errors.Annotatef(berrors.ErrStorageInvalidConfig, "compressReader doesn't support Seek now, offset %d, whence %d", offset, whence)
}
func (c *compressReader) Close() error {
diff --git a/br/pkg/storage/gcs.go b/br/pkg/storage/gcs.go
index 063100a52ad59..ac5098ed16973 100644
--- a/br/pkg/storage/gcs.go
+++ b/br/pkg/storage/gcs.go
@@ -90,24 +90,36 @@ func (options *GCSBackendOptions) parseFromFlags(flags *pflag.FlagSet) error {
return nil
}
-type gcsStorage struct {
+// GCSStorage defines some standard operations for BR/Lightning on the GCS storage.
+// It implements the `ExternalStorage` interface.
+type GCSStorage struct {
gcs *backuppb.GCS
bucket *storage.BucketHandle
}
+// GetBucketHandle gets the handle to the GCS API on the bucket.
+func (s *GCSStorage) GetBucketHandle() *storage.BucketHandle {
+ return s.bucket
+}
+
+// GetOptions gets the external storage operations for the GCS.
+func (s *GCSStorage) GetOptions() *backuppb.GCS {
+ return s.gcs
+}
+
// DeleteFile delete the file in storage
-func (s *gcsStorage) DeleteFile(ctx context.Context, name string) error {
+func (s *GCSStorage) DeleteFile(ctx context.Context, name string) error {
object := s.objectName(name)
err := s.bucket.Object(object).Delete(ctx)
return errors.Trace(err)
}
-func (s *gcsStorage) objectName(name string) string {
+func (s *GCSStorage) objectName(name string) string {
return path.Join(s.gcs.Prefix, name)
}
// WriteFile writes data to a file to storage.
-func (s *gcsStorage) WriteFile(ctx context.Context, name string, data []byte) error {
+func (s *GCSStorage) WriteFile(ctx context.Context, name string, data []byte) error {
object := s.objectName(name)
wc := s.bucket.Object(object).NewWriter(ctx)
wc.StorageClass = s.gcs.StorageClass
@@ -120,7 +132,7 @@ func (s *gcsStorage) WriteFile(ctx context.Context, name string, data []byte) er
}
// ReadFile reads the file from the storage and returns the contents.
-func (s *gcsStorage) ReadFile(ctx context.Context, name string) ([]byte, error) {
+func (s *GCSStorage) ReadFile(ctx context.Context, name string) ([]byte, error) {
object := s.objectName(name)
rc, err := s.bucket.Object(object).NewReader(ctx)
if err != nil {
@@ -143,7 +155,7 @@ func (s *gcsStorage) ReadFile(ctx context.Context, name string) ([]byte, error)
}
// FileExists return true if file exists.
-func (s *gcsStorage) FileExists(ctx context.Context, name string) (bool, error) {
+func (s *GCSStorage) FileExists(ctx context.Context, name string) (bool, error) {
object := s.objectName(name)
_, err := s.bucket.Object(object).Attrs(ctx)
if err != nil {
@@ -156,7 +168,7 @@ func (s *gcsStorage) FileExists(ctx context.Context, name string) (bool, error)
}
// Open a Reader by file path.
-func (s *gcsStorage) Open(ctx context.Context, path string) (ExternalFileReader, error) {
+func (s *GCSStorage) Open(ctx context.Context, path string) (ExternalFileReader, error) {
object := s.objectName(path)
handle := s.bucket.Object(object)
@@ -182,7 +194,7 @@ func (s *gcsStorage) Open(ctx context.Context, path string) (ExternalFileReader,
// The first argument is the file path that can be used in `Open`
// function; the second argument is the size in byte of the file determined
// by path.
-func (s *gcsStorage) WalkDir(ctx context.Context, opt *WalkOption, fn func(string, int64) error) error {
+func (s *GCSStorage) WalkDir(ctx context.Context, opt *WalkOption, fn func(string, int64) error) error {
if opt == nil {
opt = &WalkOption{}
}
@@ -221,12 +233,12 @@ func (s *gcsStorage) WalkDir(ctx context.Context, opt *WalkOption, fn func(strin
return nil
}
-func (s *gcsStorage) URI() string {
+func (s *GCSStorage) URI() string {
return "gcs://" + s.gcs.Bucket + "/" + s.gcs.Prefix
}
// Create implements ExternalStorage interface.
-func (s *gcsStorage) Create(ctx context.Context, name string) (ExternalFileWriter, error) {
+func (s *GCSStorage) Create(ctx context.Context, name string) (ExternalFileWriter, error) {
object := s.objectName(name)
wc := s.bucket.Object(object).NewWriter(ctx)
wc.StorageClass = s.gcs.StorageClass
@@ -235,7 +247,7 @@ func (s *gcsStorage) Create(ctx context.Context, name string) (ExternalFileWrite
}
// Rename file name from oldFileName to newFileName.
-func (s *gcsStorage) Rename(ctx context.Context, oldFileName, newFileName string) error {
+func (s *GCSStorage) Rename(ctx context.Context, oldFileName, newFileName string) error {
data, err := s.ReadFile(ctx, oldFileName)
if err != nil {
return errors.Trace(err)
@@ -247,7 +259,8 @@ func (s *gcsStorage) Rename(ctx context.Context, oldFileName, newFileName string
return s.DeleteFile(ctx, oldFileName)
}
-func newGCSStorage(ctx context.Context, gcs *backuppb.GCS, opts *ExternalStorageOptions) (*gcsStorage, error) {
+// NewGCSStorage creates a GCS external storage implementation.
+func NewGCSStorage(ctx context.Context, gcs *backuppb.GCS, opts *ExternalStorageOptions) (*GCSStorage, error) {
var clientOps []option.ClientOption
if opts.NoCredentials {
clientOps = append(clientOps, option.WithoutAuthentication())
@@ -301,12 +314,7 @@ func newGCSStorage(ctx context.Context, gcs *backuppb.GCS, opts *ExternalStorage
// so we need find sst in slash directory
gcs.Prefix += "//"
}
- return &gcsStorage{gcs: gcs, bucket: bucket}, nil
-}
-
-// only for unit test
-func NewGCSStorageForTest(ctx context.Context, gcs *backuppb.GCS, opts *ExternalStorageOptions) (*gcsStorage, error) {
- return newGCSStorage(ctx, gcs, opts)
+ return &GCSStorage{gcs: gcs, bucket: bucket}, nil
}
func hasSSTFiles(ctx context.Context, bucket *storage.BucketHandle, prefix string) bool {
@@ -332,7 +340,7 @@ func hasSSTFiles(ctx context.Context, bucket *storage.BucketHandle, prefix strin
// gcsObjectReader wrap storage.Reader and add the `Seek` method.
type gcsObjectReader struct {
- storage *gcsStorage
+ storage *GCSStorage
name string
objHandle *storage.ObjectHandle
reader io.ReadCloser
diff --git a/br/pkg/storage/gcs_test.go b/br/pkg/storage/gcs_test.go
index 5801adccf04b7..daefb2bd686d3 100644
--- a/br/pkg/storage/gcs_test.go
+++ b/br/pkg/storage/gcs_test.go
@@ -32,7 +32,7 @@ func TestGCS(t *testing.T) {
PredefinedAcl: "private",
CredentialsBlob: "Fake Credentials",
}
- stg, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+ stg, err := NewGCSStorage(ctx, gcs, &ExternalStorageOptions{
SendCredentials: false,
CheckPermissions: []Permission{AccessBuckets},
HTTPClient: server.HTTPClient(),
@@ -86,7 +86,7 @@ func TestGCS(t *testing.T) {
require.NoError(t, err)
require.False(t, exist)
- checkWalkDir := func(stg *gcsStorage, opt *WalkOption) {
+ checkWalkDir := func(stg *GCSStorage, opt *WalkOption) {
var totalSize int64 = 0
err = stg.WalkDir(ctx, opt, func(name string, size int64) error {
totalSize += size
@@ -112,7 +112,7 @@ func TestGCS(t *testing.T) {
PredefinedAcl: "private",
CredentialsBlob: "Fake Credentials",
}
- stg, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+ stg, err := NewGCSStorage(ctx, gcs, &ExternalStorageOptions{
SendCredentials: false,
CheckPermissions: []Permission{AccessBuckets},
HTTPClient: server.HTTPClient(),
@@ -130,7 +130,7 @@ func TestGCS(t *testing.T) {
PredefinedAcl: "private",
CredentialsBlob: "Fake Credentials",
}
- stg, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+ stg, err := NewGCSStorage(ctx, gcs, &ExternalStorageOptions{
SendCredentials: false,
CheckPermissions: []Permission{AccessBuckets},
HTTPClient: server.HTTPClient(),
@@ -147,7 +147,7 @@ func TestGCS(t *testing.T) {
PredefinedAcl: "private",
CredentialsBlob: "Fake Credentials",
}
- stg, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+ stg, err := NewGCSStorage(ctx, gcs, &ExternalStorageOptions{
SendCredentials: false,
CheckPermissions: []Permission{AccessBuckets},
HTTPClient: server.HTTPClient(),
@@ -254,7 +254,7 @@ func TestNewGCSStorage(t *testing.T) {
PredefinedAcl: "private",
CredentialsBlob: "FakeCredentials",
}
- _, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+ _, err := NewGCSStorage(ctx, gcs, &ExternalStorageOptions{
SendCredentials: true,
CheckPermissions: []Permission{AccessBuckets},
HTTPClient: server.HTTPClient(),
@@ -271,7 +271,7 @@ func TestNewGCSStorage(t *testing.T) {
PredefinedAcl: "private",
CredentialsBlob: "FakeCredentials",
}
- _, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+ _, err := NewGCSStorage(ctx, gcs, &ExternalStorageOptions{
SendCredentials: false,
CheckPermissions: []Permission{AccessBuckets},
HTTPClient: server.HTTPClient(),
@@ -302,7 +302,7 @@ func TestNewGCSStorage(t *testing.T) {
PredefinedAcl: "private",
CredentialsBlob: "",
}
- _, err = newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+ _, err = NewGCSStorage(ctx, gcs, &ExternalStorageOptions{
SendCredentials: true,
CheckPermissions: []Permission{AccessBuckets},
HTTPClient: server.HTTPClient(),
@@ -333,7 +333,7 @@ func TestNewGCSStorage(t *testing.T) {
PredefinedAcl: "private",
CredentialsBlob: "",
}
- s, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+ s, err := NewGCSStorage(ctx, gcs, &ExternalStorageOptions{
SendCredentials: false,
CheckPermissions: []Permission{AccessBuckets},
HTTPClient: server.HTTPClient(),
@@ -352,7 +352,7 @@ func TestNewGCSStorage(t *testing.T) {
PredefinedAcl: "private",
CredentialsBlob: "",
}
- _, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+ _, err := NewGCSStorage(ctx, gcs, &ExternalStorageOptions{
SendCredentials: true,
CheckPermissions: []Permission{AccessBuckets},
HTTPClient: server.HTTPClient(),
@@ -368,7 +368,7 @@ func TestNewGCSStorage(t *testing.T) {
PredefinedAcl: "private",
CredentialsBlob: "FakeCredentials",
}
- s, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+ s, err := NewGCSStorage(ctx, gcs, &ExternalStorageOptions{
SendCredentials: false,
CheckPermissions: []Permission{AccessBuckets},
HTTPClient: server.HTTPClient(),
diff --git a/br/pkg/storage/local.go b/br/pkg/storage/local.go
index 68dc760cc1c9a..0259e715c7968 100644
--- a/br/pkg/storage/local.go
+++ b/br/pkg/storage/local.go
@@ -9,7 +9,10 @@ import (
"path/filepath"
"strings"
+ "github.com/google/uuid"
"github.com/pingcap/errors"
+ "github.com/pingcap/log"
+ "go.uber.org/zap"
)
const (
@@ -36,9 +39,23 @@ func (l *LocalStorage) DeleteFile(_ context.Context, name string) error {
func (l *LocalStorage) WriteFile(_ context.Context, name string, data []byte) error {
// because `os.WriteFile` is not atomic, directly write into it may reset the file
// to an empty file if write is not finished.
- tmpPath := filepath.Join(l.base, name) + ".tmp"
+ tmpPath := filepath.Join(l.base, name) + ".tmp." + uuid.NewString()
if err := os.WriteFile(tmpPath, data, localFilePerm); err != nil {
- return errors.Trace(err)
+ path := filepath.Dir(tmpPath)
+ log.Info("failed to write file, try to mkdir the path", zap.String("path", path))
+ exists, existErr := pathExists(path)
+ if existErr != nil {
+ return errors.Annotatef(err, "after failed to write file, failed to check path exists : %v", existErr)
+ }
+ if exists {
+ return errors.Trace(err)
+ }
+ if mkdirErr := mkdirAll(path); mkdirErr != nil {
+ return errors.Annotatef(err, "after failed to write file, failed to mkdir : %v", mkdirErr)
+ }
+ if err := os.WriteFile(tmpPath, data, localFilePerm); err != nil {
+ return errors.Trace(err)
+ }
}
if err := os.Rename(tmpPath, filepath.Join(l.base, name)); err != nil {
return errors.Trace(err)
diff --git a/br/pkg/storage/local_test.go b/br/pkg/storage/local_test.go
index 82e7435ae29be..db1ba424b9d6b 100644
--- a/br/pkg/storage/local_test.go
+++ b/br/pkg/storage/local_test.go
@@ -9,6 +9,7 @@ import (
"runtime"
"testing"
+ "github.com/pingcap/errors"
"github.com/stretchr/testify/require"
)
@@ -99,4 +100,30 @@ func TestWalkDirWithSoftLinkFile(t *testing.T) {
})
require.NoError(t, err)
require.Equal(t, 1, i)
+
+ // test file not exists
+ exists, err := store.FileExists(context.TODO(), "/123/456")
+ require.NoError(t, err)
+ require.False(t, exists)
+
+ // test walk nonexistent directory
+ err = store.WalkDir(context.TODO(), &WalkOption{SubDir: "123/456"}, func(path string, size int64) error {
+ return errors.New("find file")
+ })
+ require.NoError(t, err)
+ // write file to a nonexistent directory
+ err = store.WriteFile(context.TODO(), "/123/456/789.txt", []byte(data))
+ require.NoError(t, err)
+ exists, err = store.FileExists(context.TODO(), "/123/456")
+ require.NoError(t, err)
+ require.True(t, exists)
+
+ // test walk existent directory
+ err = store.WalkDir(context.TODO(), &WalkOption{SubDir: "123/456"}, func(path string, size int64) error {
+ if path == "123/456/789.txt" {
+ return nil
+ }
+ return errors.Errorf("find other file: %s", path)
+ })
+ require.NoError(t, err)
}
diff --git a/br/pkg/storage/memstore_test.go b/br/pkg/storage/memstore_test.go
index a85a2ff467fa1..3ae9a08d168bc 100644
--- a/br/pkg/storage/memstore_test.go
+++ b/br/pkg/storage/memstore_test.go
@@ -17,7 +17,6 @@ import (
"bytes"
"context"
"io"
- "io/ioutil"
"sync"
"testing"
"time"
@@ -70,7 +69,7 @@ func TestMemStoreBasic(t *testing.T) {
require.Nil(t, err)
r2, err := store.Open(ctx, "/hello.txt")
require.Nil(t, err)
- fileContent, err = ioutil.ReadAll(r)
+ fileContent, err = io.ReadAll(r)
require.Nil(t, err)
require.True(t, bytes.Equal([]byte("hello world 3"), fileContent))
require.Nil(t, r.Close())
@@ -83,7 +82,7 @@ func TestMemStoreBasic(t *testing.T) {
_, err = r2.Seek(5, io.SeekStart)
require.Nil(t, err)
- fileContent, err = ioutil.ReadAll(r2)
+ fileContent, err = io.ReadAll(r2)
require.Nil(t, err)
require.True(t, bytes.Equal([]byte(" world 3"), fileContent))
diff --git a/br/pkg/storage/parse.go b/br/pkg/storage/parse.go
index 39aa8743f6d53..05c586fb5c322 100644
--- a/br/pkg/storage/parse.go
+++ b/br/pkg/storage/parse.go
@@ -35,6 +35,12 @@ func ParseRawURL(rawURL string) (*url.URL, error) {
return u, nil
}
+// ParseBackendFromURL constructs a structured backend description from the
+// *url.URL.
+func ParseBackendFromURL(u *url.URL, options *BackendOptions) (*backuppb.StorageBackend, error) {
+ return parseBackend(u, "", options)
+}
+
// ParseBackend constructs a structured backend description from the
// storage URL.
func ParseBackend(rawURL string, options *BackendOptions) (*backuppb.StorageBackend, error) {
@@ -45,6 +51,14 @@ func ParseBackend(rawURL string, options *BackendOptions) (*backuppb.StorageBack
if err != nil {
return nil, errors.Trace(err)
}
+ return parseBackend(u, rawURL, options)
+}
+
+func parseBackend(u *url.URL, rawURL string, options *BackendOptions) (*backuppb.StorageBackend, error) {
+ if rawURL == "" {
+ // try to handle hdfs for ParseBackendFromURL caller
+ rawURL = u.String()
+ }
switch u.Scheme {
case "":
absPath, err := filepath.Abs(rawURL)
diff --git a/br/pkg/storage/parse_test.go b/br/pkg/storage/parse_test.go
index b9a5d75d29322..2dbeaabcb4386 100644
--- a/br/pkg/storage/parse_test.go
+++ b/br/pkg/storage/parse_test.go
@@ -69,14 +69,15 @@ func TestCreateStorage(t *testing.T) {
require.Equal(t, "TestKey", s3.SseKmsKeyId)
// special character in access keys
- s, err = ParseBackend(`s3://bucket4/prefix/path?access-key=NXN7IPIOSAAKDEEOLMAF&secret-access-key=nREY/7Dt+PaIbYKrKlEEMMF/ExCiJEX=XMLPUANw`, nil)
+ s, err = ParseBackend(`s3://bucket4/prefix/path?access-key=******&secret-access-key=******+&session-token=******`, nil)
require.NoError(t, err)
s3 = s.GetS3()
require.NotNil(t, s3)
require.Equal(t, "bucket4", s3.Bucket)
require.Equal(t, "prefix/path", s3.Prefix)
- require.Equal(t, "NXN7IPIOSAAKDEEOLMAF", s3.AccessKey)
- require.Equal(t, "nREY/7Dt+PaIbYKrKlEEMMF/ExCiJEX=XMLPUANw", s3.SecretAccessKey)
+ require.Equal(t, "******", s3.AccessKey)
+ require.Equal(t, "******+", s3.SecretAccessKey)
+ require.Equal(t, "******", s3.SessionToken)
require.True(t, s3.ForcePathStyle)
// parse role ARN and external ID
diff --git a/br/pkg/storage/s3.go b/br/pkg/storage/s3.go
index ff2cbae25d030..aa052ebccc9a5 100644
--- a/br/pkg/storage/s3.go
+++ b/br/pkg/storage/s3.go
@@ -68,13 +68,24 @@ var permissionCheckFn = map[Permission]func(*s3.S3, *backuppb.S3) error{
GetObject: getObject,
}
-// S3Storage info for s3 storage.
+// S3Storage defines some standard operations for BR/Lightning on the S3 storage.
+// It implements the `ExternalStorage` interface.
type S3Storage struct {
session *session.Session
svc s3iface.S3API
options *backuppb.S3
}
+// GetS3APIHandle gets the handle to the S3 API.
+func (rs *S3Storage) GetS3APIHandle() s3iface.S3API {
+ return rs.svc
+}
+
+// GetOptions gets the external storage operations for the S3.
+func (rs *S3Storage) GetOptions() *backuppb.S3 {
+ return rs.options
+}
+
// S3Uploader does multi-part upload to s3.
type S3Uploader struct {
svc s3iface.S3API
@@ -129,6 +140,7 @@ type S3BackendOptions struct {
ACL string `json:"acl" toml:"acl"`
AccessKey string `json:"access-key" toml:"access-key"`
SecretAccessKey string `json:"secret-access-key" toml:"secret-access-key"`
+ SessionToken string `json:"session-token" toml:"session-token"`
Provider string `json:"provider" toml:"provider"`
ForcePathStyle bool `json:"force-path-style" toml:"force-path-style"`
UseAccelerateEndpoint bool `json:"use-accelerate-endpoint" toml:"use-accelerate-endpoint"`
@@ -173,6 +185,7 @@ func (options *S3BackendOptions) Apply(s3 *backuppb.S3) error {
s3.Acl = options.ACL
s3.AccessKey = options.AccessKey
s3.SecretAccessKey = options.SecretAccessKey
+ s3.SessionToken = options.SessionToken
s3.ForcePathStyle = options.ForcePathStyle
s3.RoleArn = options.RoleARN
s3.ExternalId = options.ExternalID
@@ -248,23 +261,10 @@ func NewS3StorageForTest(svc s3iface.S3API, options *backuppb.S3) *S3Storage {
}
}
-// NewS3Storage initialize a new s3 storage for metadata.
-//
-// Deprecated: Create the storage via `New()` instead of using this.
-func NewS3Storage( // revive:disable-line:flag-parameter
- backend *backuppb.S3,
- sendCredential bool,
-) (*S3Storage, error) {
- return newS3Storage(backend, &ExternalStorageOptions{
- SendCredentials: sendCredential,
- CheckPermissions: []Permission{AccessBuckets},
- })
-}
-
// auto access without ak / sk.
func autoNewCred(qs *backuppb.S3) (cred *credentials.Credentials, err error) {
if qs.AccessKey != "" && qs.SecretAccessKey != "" {
- return credentials.NewStaticCredentials(qs.AccessKey, qs.SecretAccessKey, ""), nil
+ return credentials.NewStaticCredentials(qs.AccessKey, qs.SecretAccessKey, qs.SessionToken), nil
}
endpoint := qs.Endpoint
// if endpoint is empty,return no error and run default(aws) follow.
@@ -288,7 +288,8 @@ func createOssRAMCred() (*credentials.Credentials, error) {
return credentials.NewStaticCredentials(ncred.AccessKeyId, ncred.AccessKeySecret, ncred.AccessKeyStsToken), nil
}
-func newS3Storage(backend *backuppb.S3, opts *ExternalStorageOptions) (obj *S3Storage, errRet error) {
+// NewS3Storage initialize a new s3 storage for metadata.
+func NewS3Storage(backend *backuppb.S3, opts *ExternalStorageOptions) (obj *S3Storage, errRet error) {
qs := *backend
awsConfig := aws.NewConfig().
WithS3ForcePathStyle(qs.ForcePathStyle).
@@ -331,6 +332,7 @@ func newS3Storage(backend *backuppb.S3, opts *ExternalStorageOptions) (obj *S3St
// Clear the credentials if exists so that they will not be sent to TiKV
backend.AccessKey = ""
backend.SecretAccessKey = ""
+ backend.SessionToken = ""
} else if ses.Config.Credentials != nil {
if qs.AccessKey == "" || qs.SecretAccessKey == "" {
v, cerr := ses.Config.Credentials.Get()
@@ -339,6 +341,7 @@ func newS3Storage(backend *backuppb.S3, opts *ExternalStorageOptions) (obj *S3St
}
backend.AccessKey = v.AccessKeyID
backend.SecretAccessKey = v.SecretAccessKey
+ backend.SessionToken = v.SessionToken
}
}
@@ -355,13 +358,18 @@ func newS3Storage(backend *backuppb.S3, opts *ExternalStorageOptions) (obj *S3St
)
}
c := s3.New(ses, s3CliConfigs...)
- // s3manager.GetBucketRegionWithClient will set credential anonymous, which works with s3.
- // we need reassign credential to be compatible with minio authentication.
confCred := ses.Config.Credentials
setCredOpt := func(req *request.Request) {
+ // s3manager.GetBucketRegionWithClient will set credential anonymous, which works with s3.
+ // we need reassign credential to be compatible with minio authentication.
if confCred != nil {
req.Config.Credentials = confCred
}
+ // s3manager.GetBucketRegionWithClient use path style addressing default.
+ // we need set S3ForcePathStyle by our config if we set endpoint.
+ if qs.Endpoint != "" {
+ req.Config.S3ForcePathStyle = ses.Config.S3ForcePathStyle
+ }
}
region, err := s3manager.GetBucketRegionWithClient(context.Background(), c, qs.Bucket, setCredOpt)
if err != nil {
@@ -400,7 +408,7 @@ func newS3Storage(backend *backuppb.S3, opts *ExternalStorageOptions) (obj *S3St
options: &qs,
}
if opts.CheckS3ObjectLockOptions {
- backend.ObjectLockEnabled = s3Storage.isObjectLockEnabled()
+ backend.ObjectLockEnabled = s3Storage.IsObjectLockEnabled()
}
return s3Storage, nil
}
@@ -447,7 +455,7 @@ func getObject(svc *s3.S3, qs *backuppb.S3) error {
return nil
}
-func (rs *S3Storage) isObjectLockEnabled() bool {
+func (rs *S3Storage) IsObjectLockEnabled() bool {
input := &s3.GetObjectLockConfigurationInput{
Bucket: aws.String(rs.options.Bucket),
}
@@ -456,8 +464,8 @@ func (rs *S3Storage) isObjectLockEnabled() bool {
log.Warn("failed to check object lock for bucket", zap.String("bucket", rs.options.Bucket), zap.Error(err))
return false
}
- if resp.ObjectLockConfiguration != nil {
- if s3.ObjectLockEnabledEnabled == *resp.ObjectLockConfiguration.ObjectLockEnabled {
+ if resp != nil && resp.ObjectLockConfiguration != nil {
+ if s3.ObjectLockEnabledEnabled == aws.StringValue(resp.ObjectLockConfiguration.ObjectLockEnabled) {
return true
}
}
diff --git a/br/pkg/storage/s3_test.go b/br/pkg/storage/s3_test.go
index 3990e5eb82bc1..817e8f46f7f7d 100644
--- a/br/pkg/storage/s3_test.go
+++ b/br/pkg/storage/s3_test.go
@@ -144,6 +144,7 @@ func TestApplyUpdate(t *testing.T) {
if test.setEnv {
require.NoError(t, os.Setenv("AWS_ACCESS_KEY_ID", "ab"))
require.NoError(t, os.Setenv("AWS_SECRET_ACCESS_KEY", "cd"))
+ require.NoError(t, os.Setenv("AWS_SESSION_TOKEN", "ef"))
}
u, err := ParseBackend("s3://bucket/prefix/", &BackendOptions{S3: test.options})
require.NoError(t, err)
@@ -260,11 +261,13 @@ func TestApplyUpdate(t *testing.T) {
Region: "us-west-2",
AccessKey: "ab",
SecretAccessKey: "cd",
+ SessionToken: "ef",
},
s3: &backuppb.S3{
Region: "us-west-2",
AccessKey: "ab",
SecretAccessKey: "cd",
+ SessionToken: "ef",
Bucket: "bucket",
Prefix: "prefix",
},
@@ -314,10 +317,11 @@ func TestS3Storage(t *testing.T) {
{
name: "no region",
s3: &backuppb.S3{
- Region: "",
- Endpoint: s.URL,
- Bucket: "bucket",
- Prefix: "prefix",
+ Region: "",
+ Endpoint: s.URL,
+ Bucket: "bucket",
+ Prefix: "prefix",
+ ForcePathStyle: true,
},
errReturn: false,
sendCredential: true,
@@ -325,10 +329,11 @@ func TestS3Storage(t *testing.T) {
{
name: "wrong region",
s3: &backuppb.S3{
- Region: "us-east-2",
- Endpoint: s.URL,
- Bucket: "bucket",
- Prefix: "prefix",
+ Region: "us-east-2",
+ Endpoint: s.URL,
+ Bucket: "bucket",
+ Prefix: "prefix",
+ ForcePathStyle: true,
},
errReturn: true,
sendCredential: true,
@@ -336,10 +341,11 @@ func TestS3Storage(t *testing.T) {
{
name: "right region",
s3: &backuppb.S3{
- Region: "us-west-2",
- Endpoint: s.URL,
- Bucket: "bucket",
- Prefix: "prefix",
+ Region: "us-west-2",
+ Endpoint: s.URL,
+ Bucket: "bucket",
+ Prefix: "prefix",
+ ForcePathStyle: true,
},
errReturn: false,
sendCredential: true,
@@ -351,8 +357,10 @@ func TestS3Storage(t *testing.T) {
Endpoint: s.URL,
AccessKey: "ab",
SecretAccessKey: "cd",
+ SessionToken: "ef",
Bucket: "bucket",
Prefix: "prefix",
+ ForcePathStyle: true,
},
errReturn: false,
sendCredential: true,
@@ -365,6 +373,7 @@ func TestS3Storage(t *testing.T) {
SecretAccessKey: "cd",
Bucket: "bucket",
Prefix: "prefix",
+ ForcePathStyle: true,
},
errReturn: false,
sendCredential: true,
@@ -372,11 +381,12 @@ func TestS3Storage(t *testing.T) {
{
name: "no secret access key",
s3: &backuppb.S3{
- Region: "us-west-2",
- Endpoint: s.URL,
- AccessKey: "ab",
- Bucket: "bucket",
- Prefix: "prefix",
+ Region: "us-west-2",
+ Endpoint: s.URL,
+ AccessKey: "ab",
+ Bucket: "bucket",
+ Prefix: "prefix",
+ ForcePathStyle: true,
},
errReturn: false,
sendCredential: true,
@@ -384,11 +394,12 @@ func TestS3Storage(t *testing.T) {
{
name: "no secret access key",
s3: &backuppb.S3{
- Region: "us-west-2",
- Endpoint: s.URL,
- AccessKey: "ab",
- Bucket: "bucket",
- Prefix: "prefix",
+ Region: "us-west-2",
+ Endpoint: s.URL,
+ AccessKey: "ab",
+ Bucket: "bucket",
+ Prefix: "prefix",
+ ForcePathStyle: true,
},
errReturn: false,
sendCredential: false,
@@ -1105,10 +1116,12 @@ func TestWalkDirWithEmptyPrefix(t *testing.T) {
func TestSendCreds(t *testing.T) {
accessKey := "ab"
secretAccessKey := "cd"
+ sessionToken := "ef"
backendOpt := BackendOptions{
S3: S3BackendOptions{
AccessKey: accessKey,
SecretAccessKey: secretAccessKey,
+ SessionToken: sessionToken,
},
}
backend, err := ParseBackend("s3://bucket/prefix/", &backendOpt)
@@ -1121,12 +1134,15 @@ func TestSendCreds(t *testing.T) {
sentAccessKey := backend.GetS3().AccessKey
require.Equal(t, accessKey, sentAccessKey)
sentSecretAccessKey := backend.GetS3().SecretAccessKey
- require.Equal(t, sentSecretAccessKey, sentSecretAccessKey)
+ require.Equal(t, secretAccessKey, sentSecretAccessKey)
+ sentSessionToken := backend.GetS3().SessionToken
+ require.Equal(t, sessionToken, sentSessionToken)
backendOpt = BackendOptions{
S3: S3BackendOptions{
AccessKey: accessKey,
SecretAccessKey: secretAccessKey,
+ SessionToken: sessionToken,
},
}
backend, err = ParseBackend("s3://bucket/prefix/", &backendOpt)
@@ -1140,4 +1156,51 @@ func TestSendCreds(t *testing.T) {
require.Equal(t, "", sentAccessKey)
sentSecretAccessKey = backend.GetS3().SecretAccessKey
require.Equal(t, "", sentSecretAccessKey)
+ sentSessionToken = backend.GetS3().SessionToken
+ require.Equal(t, "", sentSessionToken)
+}
+
+func TestObjectLock(t *testing.T) {
+ s := createS3Suite(t)
+ // resp is nil
+ s.s3.EXPECT().GetObjectLockConfiguration(gomock.Any()).Return(nil, nil)
+ require.Equal(t, false, s.storage.IsObjectLockEnabled())
+
+ // resp is not nil, but resp.ObjectLockConfiguration is nil
+ s.s3.EXPECT().GetObjectLockConfiguration(gomock.Any()).Return(
+ &s3.GetObjectLockConfigurationOutput{
+ ObjectLockConfiguration: nil,
+ }, nil,
+ )
+ require.Equal(t, false, s.storage.IsObjectLockEnabled())
+
+ // resp.ObjectLockConfiguration is not nil, but resp.ObjectLockConfiguration.ObjectLockEnabled is nil
+ s.s3.EXPECT().GetObjectLockConfiguration(gomock.Any()).Return(
+ &s3.GetObjectLockConfigurationOutput{
+ ObjectLockConfiguration: &s3.ObjectLockConfiguration{
+ ObjectLockEnabled: nil,
+ },
+ }, nil,
+ )
+ require.Equal(t, false, s.storage.IsObjectLockEnabled())
+
+ // resp.ObjectLockConfiguration.ObjectLockEnabled is illegal string
+ s.s3.EXPECT().GetObjectLockConfiguration(gomock.Any()).Return(
+ &s3.GetObjectLockConfigurationOutput{
+ ObjectLockConfiguration: &s3.ObjectLockConfiguration{
+ ObjectLockEnabled: aws.String("EnaBled"),
+ },
+ }, nil,
+ )
+ require.Equal(t, false, s.storage.IsObjectLockEnabled())
+
+ // resp.ObjectLockConfiguration.ObjectLockEnabled is enabled
+ s.s3.EXPECT().GetObjectLockConfiguration(gomock.Any()).Return(
+ &s3.GetObjectLockConfigurationOutput{
+ ObjectLockConfiguration: &s3.ObjectLockConfiguration{
+ ObjectLockEnabled: aws.String("Enabled"),
+ },
+ }, nil,
+ )
+ require.Equal(t, true, s.storage.IsObjectLockEnabled())
}
diff --git a/br/pkg/storage/storage.go b/br/pkg/storage/storage.go
index b73181e582158..1aa2df5f5e36a 100644
--- a/br/pkg/storage/storage.go
+++ b/br/pkg/storage/storage.go
@@ -158,6 +158,9 @@ func Create(ctx context.Context, backend *backuppb.StorageBackend, sendCreds boo
// New creates an ExternalStorage with options.
func New(ctx context.Context, backend *backuppb.StorageBackend, opts *ExternalStorageOptions) (ExternalStorage, error) {
+ if opts == nil {
+ opts = &ExternalStorageOptions{}
+ }
switch backend := backend.Backend.(type) {
case *backuppb.StorageBackend_Local:
if backend.Local == nil {
@@ -173,14 +176,14 @@ func New(ctx context.Context, backend *backuppb.StorageBackend, opts *ExternalSt
if backend.S3 == nil {
return nil, errors.Annotate(berrors.ErrStorageInvalidConfig, "s3 config not found")
}
- return newS3Storage(backend.S3, opts)
+ return NewS3Storage(backend.S3, opts)
case *backuppb.StorageBackend_Noop:
return newNoopStorage(), nil
case *backuppb.StorageBackend_Gcs:
if backend.Gcs == nil {
return nil, errors.Annotate(berrors.ErrStorageInvalidConfig, "GCS config not found")
}
- return newGCSStorage(ctx, backend.Gcs, opts)
+ return NewGCSStorage(ctx, backend.Gcs, opts)
case *backuppb.StorageBackend_AzureBlobStorage:
return newAzureBlobStorage(ctx, backend.AzureBlobStorage, opts)
default:
diff --git a/br/pkg/stream/BUILD.bazel b/br/pkg/stream/BUILD.bazel
index f75f9f37d81ea..e5fbc5c87b870 100644
--- a/br/pkg/stream/BUILD.bazel
+++ b/br/pkg/stream/BUILD.bazel
@@ -56,8 +56,11 @@ go_test(
"//br/pkg/storage",
"//br/pkg/streamhelper",
"//meta",
+ "//parser/ast",
"//parser/model",
+ "//parser/mysql",
"//tablecodec",
+ "//types",
"//util/codec",
"//util/table-filter",
"@com_github_pingcap_kvproto//pkg/brpb",
diff --git a/br/pkg/stream/meta_kv.go b/br/pkg/stream/meta_kv.go
index 9d054f0bef454..fb7c2f79f17d1 100644
--- a/br/pkg/stream/meta_kv.go
+++ b/br/pkg/stream/meta_kv.go
@@ -111,15 +111,34 @@ const (
flagShortValuePrefix = byte('v')
flagOverlappedRollback = byte('R')
flagGCFencePrefix = byte('F')
+ flagLastChangePrefix = byte('l')
+ flagTxnSourcePrefix = byte('S')
)
+// RawWriteCFValue represents the value in write columnFamily.
+// Detail see line: https://github.com/tikv/tikv/blob/release-6.5/components/txn_types/src/write.rs#L70
type RawWriteCFValue struct {
t WriteType
startTs uint64
shortValue []byte
hasOverlappedRollback bool
- hasGCFence bool
- gcFence uint64
+
+ // Records the next version after this version when overlapping rollback
+ // happens on an already existed commit record.
+ //
+ // See [`Write::gc_fence`] for more detail.
+ hasGCFence bool
+ gcFence uint64
+
+ // The number of versions that need skipping from this record
+ // to find the latest PUT/DELETE record.
+ // If versions_to_last_change > 0 but last_change_ts == 0, the key does not
+ // have a PUT/DELETE record before this write record.
+ lastChangeTs uint64
+ versionsToLastChange uint64
+
+ // The source of this txn.
+ txnSource uint64
}
// ParseFrom decodes the value to get the struct `RawWriteCFValue`.
@@ -146,6 +165,10 @@ l_for:
switch data[0] {
case flagShortValuePrefix:
vlen := data[1]
+ if len(data[2:]) < int(vlen) {
+ return errors.Annotatef(berrors.ErrInvalidArgument,
+ "the length of short value is invalid, vlen: %v", int(vlen))
+ }
v.shortValue = data[2 : vlen+2]
data = data[vlen+2:]
case flagOverlappedRollback:
@@ -157,6 +180,20 @@ l_for:
if err != nil {
return errors.Annotate(berrors.ErrInvalidArgument, "decode gc fence failed")
}
+ case flagLastChangePrefix:
+ data, v.lastChangeTs, err = codec.DecodeUint(data[1:])
+ if err != nil {
+ return errors.Annotate(berrors.ErrInvalidArgument, "decode last change ts failed")
+ }
+ data, v.versionsToLastChange, err = codec.DecodeUvarint(data)
+ if err != nil {
+ return errors.Annotate(berrors.ErrInvalidArgument, "decode versions to last change failed")
+ }
+ case flagTxnSourcePrefix:
+ data, v.txnSource, err = codec.DecodeUvarint(data[1:])
+ if err != nil {
+ return errors.Annotate(berrors.ErrInvalidArgument, "decode txn source failed")
+ }
default:
break l_for
}
@@ -164,6 +201,16 @@ l_for:
return nil
}
+// IsRollback checks whether the value in cf is a `rollback` record.
+func (v *RawWriteCFValue) IsRollback() bool {
+ return v.GetWriteType() == WriteTypeRollback
+}
+
+// IsRollback checks whether the value in cf is a `delete` record.
+func (v *RawWriteCFValue) IsDelete() bool {
+ return v.GetWriteType() == WriteTypeDelete
+}
+
// HasShortValue checks whether short value is stored in write cf.
func (v *RawWriteCFValue) HasShortValue() bool {
return len(v.shortValue) > 0
@@ -204,5 +251,14 @@ func (v *RawWriteCFValue) EncodeTo() []byte {
data = append(data, flagGCFencePrefix)
data = codec.EncodeUint(data, v.gcFence)
}
+ if v.lastChangeTs > 0 || v.versionsToLastChange > 0 {
+ data = append(data, flagLastChangePrefix)
+ data = codec.EncodeUint(data, v.lastChangeTs)
+ data = codec.EncodeUvarint(data, v.versionsToLastChange)
+ }
+ if v.txnSource > 0 {
+ data = append(data, flagTxnSourcePrefix)
+ data = codec.EncodeUvarint(data, v.txnSource)
+ }
return data
}
diff --git a/br/pkg/stream/meta_kv_test.go b/br/pkg/stream/meta_kv_test.go
index eaebf64526243..7a8c5e4fed8b6 100644
--- a/br/pkg/stream/meta_kv_test.go
+++ b/br/pkg/stream/meta_kv_test.go
@@ -68,29 +68,49 @@ func TestWriteType(t *testing.T) {
}
func TestWriteCFValueNoShortValue(t *testing.T) {
+ var (
+ ts uint64 = 400036290571534337
+ txnSource uint64 = 9527
+ )
+
buff := make([]byte, 0, 9)
- buff = append(buff, byte('P'))
- buff = codec.EncodeUvarint(buff, 400036290571534337)
+ buff = append(buff, WriteTypePut)
+ buff = codec.EncodeUvarint(buff, ts)
+ buff = append(buff, flagTxnSourcePrefix)
+ buff = codec.EncodeUvarint(buff, txnSource)
v := new(RawWriteCFValue)
err := v.ParseFrom(buff)
require.NoError(t, err)
+ require.False(t, v.IsDelete())
+ require.False(t, v.IsRollback())
require.False(t, v.HasShortValue())
+ require.False(t, v.hasGCFence)
+ require.Equal(t, v.lastChangeTs, uint64(0))
+ require.Equal(t, v.versionsToLastChange, uint64(0))
+ require.Equal(t, v.txnSource, txnSource)
encodedBuff := v.EncodeTo()
require.True(t, bytes.Equal(buff, encodedBuff))
}
func TestWriteCFValueWithShortValue(t *testing.T) {
- var ts uint64 = 400036290571534337
- shortValue := []byte("pingCAP")
+ var (
+ ts uint64 = 400036290571534337
+ shortValue = []byte("pingCAP")
+ lastChangeTs uint64 = 9527
+ versionsToLastChange uint64 = 95271
+ )
buff := make([]byte, 0, 9)
- buff = append(buff, byte('P'))
+ buff = append(buff, WriteTypePut)
buff = codec.EncodeUvarint(buff, ts)
buff = append(buff, flagShortValuePrefix)
buff = append(buff, byte(len(shortValue)))
buff = append(buff, shortValue...)
+ buff = append(buff, flagLastChangePrefix)
+ buff = codec.EncodeUint(buff, lastChangeTs)
+ buff = codec.EncodeUvarint(buff, versionsToLastChange)
v := new(RawWriteCFValue)
err := v.ParseFrom(buff)
@@ -99,7 +119,78 @@ func TestWriteCFValueWithShortValue(t *testing.T) {
require.True(t, bytes.Equal(v.GetShortValue(), shortValue))
require.False(t, v.hasGCFence)
require.False(t, v.hasOverlappedRollback)
+ require.Equal(t, v.lastChangeTs, lastChangeTs)
+ require.Equal(t, v.versionsToLastChange, versionsToLastChange)
+ require.Equal(t, v.txnSource, uint64(0))
data := v.EncodeTo()
require.True(t, bytes.Equal(data, buff))
}
+
+func TestWriteCFValueWithRollback(t *testing.T) {
+ var (
+ ts uint64 = 400036290571534337
+ protectedRollbackShortValue = []byte{'P'}
+ )
+
+ buff := make([]byte, 0, 9)
+ buff = append(buff, WriteTypeRollback)
+ buff = codec.EncodeUvarint(buff, ts)
+ buff = append(buff, flagShortValuePrefix, byte(len(protectedRollbackShortValue)))
+ buff = append(buff, protectedRollbackShortValue...)
+
+ v := new(RawWriteCFValue)
+ err := v.ParseFrom(buff)
+ require.NoError(t, err)
+ require.True(t, v.IsRollback())
+ require.True(t, v.HasShortValue())
+ require.Equal(t, v.GetShortValue(), protectedRollbackShortValue)
+ require.Equal(t, v.startTs, ts)
+ require.Equal(t, v.lastChangeTs, uint64(0))
+ require.Equal(t, v.versionsToLastChange, uint64(0))
+ require.Equal(t, v.txnSource, uint64(0))
+
+ data := v.EncodeTo()
+ require.Equal(t, data, buff)
+}
+
+func TestWriteCFValueWithDelete(t *testing.T) {
+ var ts uint64 = 400036290571534337
+ buff := make([]byte, 0, 9)
+ buff = append(buff, byte('D'))
+ buff = codec.EncodeUvarint(buff, ts)
+
+ v := new(RawWriteCFValue)
+ err := v.ParseFrom(buff)
+ require.NoError(t, err)
+ require.True(t, v.IsDelete())
+ require.False(t, v.HasShortValue())
+
+ data := v.EncodeTo()
+ require.Equal(t, data, buff)
+}
+
+func TestWriteCFValueWithGcFence(t *testing.T) {
+ var (
+ ts uint64 = 400036290571534337
+ gcFence uint64 = 9527
+ )
+
+ buff := make([]byte, 0, 9)
+ buff = append(buff, WriteTypePut)
+ buff = codec.EncodeUvarint(buff, ts)
+ buff = append(buff, flagOverlappedRollback)
+ buff = append(buff, flagGCFencePrefix)
+ buff = codec.EncodeUint(buff, gcFence)
+
+ v := new(RawWriteCFValue)
+ err := v.ParseFrom(buff)
+ require.NoError(t, err)
+ require.Equal(t, v.startTs, ts)
+ require.True(t, v.hasGCFence)
+ require.Equal(t, v.gcFence, gcFence)
+ require.True(t, v.hasOverlappedRollback)
+
+ data := v.EncodeTo()
+ require.Equal(t, data, buff)
+}
diff --git a/br/pkg/stream/rewrite_meta_rawkv.go b/br/pkg/stream/rewrite_meta_rawkv.go
index 40e76a6130358..3c559ec124ad8 100644
--- a/br/pkg/stream/rewrite_meta_rawkv.go
+++ b/br/pkg/stream/rewrite_meta_rawkv.go
@@ -336,6 +336,11 @@ func (sr *SchemasReplace) rewriteTableInfo(value []byte, dbID int64) ([]byte, bo
}
}
+ // Force to disable TTL_ENABLE when restore
+ if newTableInfo.TTLInfo != nil {
+ newTableInfo.TTLInfo.Enable = false
+ }
+
if sr.AfterTableRewritten != nil {
sr.AfterTableRewritten(false, newTableInfo)
}
@@ -451,13 +456,20 @@ func (sr *SchemasReplace) rewriteValueV2(value []byte, cf string, rewrite func([
return rewriteResult{}, errors.Trace(err)
}
- if rawWriteCFValue.t == WriteTypeDelete {
+ if rawWriteCFValue.IsDelete() {
return rewriteResult{
NewValue: value,
NeedRewrite: true,
Deleted: true,
}, nil
}
+ if rawWriteCFValue.IsRollback() {
+ return rewriteResult{
+ NewValue: value,
+ NeedRewrite: true,
+ Deleted: false,
+ }, nil
+ }
if !rawWriteCFValue.HasShortValue() {
return rewriteResult{
NewValue: value,
@@ -467,6 +479,9 @@ func (sr *SchemasReplace) rewriteValueV2(value []byte, cf string, rewrite func([
shortValue, needWrite, err := rewrite(rawWriteCFValue.GetShortValue())
if err != nil {
+ log.Info("failed to rewrite short value",
+ zap.ByteString("write-type", []byte{rawWriteCFValue.GetWriteType()}),
+ zap.Int("short-value-len", len(rawWriteCFValue.GetShortValue())))
return rewriteResult{}, errors.Trace(err)
}
if !needWrite {
diff --git a/br/pkg/stream/rewrite_meta_rawkv_test.go b/br/pkg/stream/rewrite_meta_rawkv_test.go
index d2cbe24e8295d..cd3cf00d46305 100644
--- a/br/pkg/stream/rewrite_meta_rawkv_test.go
+++ b/br/pkg/stream/rewrite_meta_rawkv_test.go
@@ -7,7 +7,10 @@ import (
"encoding/json"
"testing"
+ "github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/parser/mysql"
+ "github.com/pingcap/tidb/types"
filter "github.com/pingcap/tidb/util/table-filter"
"github.com/stretchr/testify/require"
)
@@ -312,6 +315,52 @@ func TestRewriteValueForExchangePartition(t *testing.T) {
require.Equal(t, tableInfo.ID, pt1ID+100)
}
+func TestRewriteValueForTTLTable(t *testing.T) {
+ var (
+ dbId int64 = 40
+ tableID int64 = 100
+ colID int64 = 1000
+ colName = "t"
+ tableName = "t1"
+ tableInfo model.TableInfo
+ )
+
+ tbl := model.TableInfo{
+ ID: tableID,
+ Name: model.NewCIStr(tableName),
+ Columns: []*model.ColumnInfo{
+ {
+ ID: colID,
+ Name: model.NewCIStr(colName),
+ FieldType: *types.NewFieldType(mysql.TypeTimestamp),
+ },
+ },
+ TTLInfo: &model.TTLInfo{
+ ColumnName: model.NewCIStr(colName),
+ IntervalExprStr: "1",
+ IntervalTimeUnit: int(ast.TimeUnitDay),
+ Enable: true,
+ },
+ }
+ value, err := json.Marshal(&tbl)
+ require.Nil(t, err)
+
+ sr := MockEmptySchemasReplace(nil)
+ newValue, needRewrite, err := sr.rewriteTableInfo(value, dbId)
+ require.Nil(t, err)
+ require.True(t, needRewrite)
+
+ err = json.Unmarshal(newValue, &tableInfo)
+ require.Nil(t, err)
+ require.Equal(t, tableInfo.Name.String(), tableName)
+ require.Equal(t, tableInfo.ID, sr.DbMap[dbId].TableMap[tableID].NewTableID)
+ require.NotNil(t, tableInfo.TTLInfo)
+ require.Equal(t, colName, tableInfo.TTLInfo.ColumnName.O)
+ require.Equal(t, "1", tableInfo.TTLInfo.IntervalExprStr)
+ require.Equal(t, int(ast.TimeUnitDay), tableInfo.TTLInfo.IntervalTimeUnit)
+ require.False(t, tableInfo.TTLInfo.Enable)
+}
+
// db:70->80 -
// | - t0:71->81 -
// | | - p0:72->82
diff --git a/br/pkg/stream/stream_mgr.go b/br/pkg/stream/stream_mgr.go
index 61c3e6772a431..5ee184ba04f03 100644
--- a/br/pkg/stream/stream_mgr.go
+++ b/br/pkg/stream/stream_mgr.go
@@ -312,7 +312,6 @@ func FastUnmarshalMetaData(
}
readPath := path
pool.ApplyOnErrorGroup(eg, func() error {
- log.Info("fast read meta file from storage", zap.String("path", readPath))
b, err := s.ReadFile(ectx, readPath)
if err != nil {
log.Error("failed to read file", zap.String("file", readPath))
diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel
index 93e13b1f8d543..0275b61592a1c 100644
--- a/br/pkg/streamhelper/BUILD.bazel
+++ b/br/pkg/streamhelper/BUILD.bazel
@@ -9,10 +9,10 @@ go_library(
"advancer_env.go",
"client.go",
"collector.go",
+ "flush_subscriber.go",
"models.go",
"prefix_scanner.go",
"regioniter.go",
- "tsheap.go",
],
importpath = "github.com/pingcap/tidb/br/pkg/streamhelper",
visibility = ["//visibility:public"],
@@ -21,17 +21,20 @@ go_library(
"//br/pkg/logutil",
"//br/pkg/redact",
"//br/pkg/streamhelper/config",
+ "//br/pkg/streamhelper/spans",
"//br/pkg/utils",
"//config",
"//kv",
"//metrics",
"//owner",
+ "//util/codec",
+ "//util/engine",
"//util/mathutil",
"@com_github_gogo_protobuf//proto",
"@com_github_golang_protobuf//proto",
- "@com_github_google_btree//:btree",
"@com_github_google_uuid//:uuid",
"@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/brpb",
"@com_github_pingcap_kvproto//pkg/logbackuppb",
"@com_github_pingcap_kvproto//pkg/metapb",
@@ -41,10 +44,12 @@ go_library(
"@com_github_tikv_pd_client//:client",
"@io_etcd_go_etcd_client_v3//:client",
"@org_golang_google_grpc//:grpc",
+ "@org_golang_google_grpc//codes",
"@org_golang_google_grpc//keepalive",
+ "@org_golang_google_grpc//status",
"@org_golang_x_sync//errgroup",
+ "@org_uber_go_multierr//:multierr",
"@org_uber_go_zap//:zap",
- "@org_uber_go_zap//zapcore",
],
)
@@ -56,7 +61,7 @@ go_test(
"basic_lib_for_test.go",
"integration_test.go",
"regioniter_test.go",
- "tsheap_test.go",
+ "subscription_test.go",
],
flaky = True,
race = "on",
@@ -68,10 +73,13 @@ go_test(
"//br/pkg/redact",
"//br/pkg/storage",
"//br/pkg/streamhelper/config",
+ "//br/pkg/streamhelper/spans",
"//br/pkg/utils",
"//kv",
"//tablecodec",
+ "//util/codec",
"@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/brpb",
"@com_github_pingcap_kvproto//pkg/errorpb",
"@com_github_pingcap_kvproto//pkg/logbackuppb",
@@ -84,6 +92,7 @@ go_test(
"@io_etcd_go_etcd_server_v3//mvcc",
"@org_golang_google_grpc//:grpc",
"@org_golang_google_grpc//codes",
+ "@org_golang_google_grpc//metadata",
"@org_golang_google_grpc//status",
"@org_uber_go_zap//:zap",
"@org_uber_go_zap//zapcore",
diff --git a/br/pkg/streamhelper/advancer.go b/br/pkg/streamhelper/advancer.go
index ac01c5167ffc7..33c0e0898b66f 100644
--- a/br/pkg/streamhelper/advancer.go
+++ b/br/pkg/streamhelper/advancer.go
@@ -3,11 +3,7 @@
package streamhelper
import (
- "bytes"
"context"
- "math"
- "reflect"
- "sort"
"strings"
"sync"
"time"
@@ -17,10 +13,12 @@ import (
"github.com/pingcap/log"
"github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/br/pkg/streamhelper/config"
+ "github.com/pingcap/tidb/br/pkg/streamhelper/spans"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/metrics"
"github.com/tikv/client-go/v2/oracle"
+ "go.uber.org/multierr"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
)
@@ -60,81 +58,31 @@ type CheckpointAdvancer struct {
// once tick begin, this should not be changed for now.
cfg config.Config
- // the cache of region checkpoints.
- // so we can advance only ranges with huge gap.
- cache CheckpointsCache
-
- // the internal state of advancer.
- state advancerState
// the cached last checkpoint.
// if no progress, this cache can help us don't to send useless requests.
lastCheckpoint uint64
-}
-// advancerState is the sealed type for the state of advancer.
-// the advancer has two stage: full scan and update small tree.
-type advancerState interface {
- // Note:
- // Go doesn't support sealed classes or ADTs currently.
- // (it can only be used at generic constraints...)
- // Leave it empty for now.
-
- // ~*fullScan | ~*updateSmallTree
-}
+ checkpoints *spans.ValueSortedFull
+ checkpointsMu sync.Mutex
-// fullScan is the initial state of advancer.
-// in this stage, we would "fill" the cache:
-// insert ranges that union of them become the full range of task.
-type fullScan struct {
- fullScanTick int
-}
-
-// updateSmallTree is the "incremental stage" of advancer.
-// we have build a "filled" cache, and we can pop a subrange of it,
-// try to advance the checkpoint of those ranges.
-type updateSmallTree struct {
- consistencyCheckTick int
+ subscriber *FlushSubscriber
+ subscriberMu sync.Mutex
}
// NewCheckpointAdvancer creates a checkpoint advancer with the env.
func NewCheckpointAdvancer(env Env) *CheckpointAdvancer {
return &CheckpointAdvancer{
- env: env,
- cfg: config.Default(),
- cache: NewCheckpoints(),
- state: &fullScan{},
+ env: env,
+ cfg: config.Default(),
}
}
-// disableCache removes the cache.
-// note this won't lock the checkpoint advancer at `fullScan` state forever,
-// you may need to change the config `AdvancingByCache`.
-func (c *CheckpointAdvancer) disableCache() {
- c.cache = NoOPCheckpointCache{}
- c.state = &fullScan{}
-}
-
-// enable the cache.
-// also check `AdvancingByCache` in the config.
-func (c *CheckpointAdvancer) enableCache() {
- c.cache = NewCheckpoints()
- c.state = &fullScan{}
-}
-
// UpdateConfig updates the config for the advancer.
// Note this should be called before starting the loop, because there isn't locks,
// TODO: support updating config when advancer starts working.
// (Maybe by applying changes at begin of ticking, and add locks.)
func (c *CheckpointAdvancer) UpdateConfig(newConf config.Config) {
- needRefreshCache := newConf.AdvancingByCache != c.cfg.AdvancingByCache
c.cfg = newConf
- if needRefreshCache {
- if c.cfg.AdvancingByCache {
- c.enableCache()
- } else {
- c.disableCache()
- }
- }
}
// UpdateConfigWith updates the config by modifying the current config.
@@ -161,7 +109,7 @@ func (c *CheckpointAdvancer) GetCheckpointInRange(ctx context.Context, start, en
}
log.Debug("scan region", zap.Int("len", len(rs)))
for _, r := range rs {
- err := collector.collectRegion(r)
+ err := collector.CollectRegion(r)
if err != nil {
log.Warn("meet error during getting checkpoint", logutil.ShortError(err))
return err
@@ -183,28 +131,24 @@ func (c *CheckpointAdvancer) recordTimeCost(message string, fields ...zap.Field)
}
// tryAdvance tries to advance the checkpoint ts of a set of ranges which shares the same checkpoint.
-func (c *CheckpointAdvancer) tryAdvance(ctx context.Context, rst RangesSharesTS) (err error) {
- defer c.recordTimeCost("try advance", zap.Uint64("checkpoint", rst.TS), zap.Int("len", len(rst.Ranges)))()
- defer func() {
- if err != nil {
- log.Warn("failed to advance", logutil.ShortError(err), zap.Object("target", rst.Zap()))
- c.cache.InsertRanges(rst)
- }
- }()
+func (c *CheckpointAdvancer) tryAdvance(ctx context.Context, length int, getRange func(int) kv.KeyRange) (err error) {
+ defer c.recordTimeCost("try advance", zap.Int("len", length))()
defer utils.PanicToErr(&err)
- ranges := CollapseRanges(len(rst.Ranges), func(i int) kv.KeyRange {
- return rst.Ranges[i]
- })
- workers := utils.NewWorkerPool(4, "sub ranges")
+ ranges := spans.Collapse(length, getRange)
+ workers := utils.NewWorkerPool(uint(config.DefaultMaxConcurrencyAdvance)*4, "sub ranges")
eg, cx := errgroup.WithContext(ctx)
collector := NewClusterCollector(ctx, c.env)
- collector.setOnSuccessHook(c.cache.InsertRange)
+ collector.SetOnSuccessHook(func(u uint64, kr kv.KeyRange) {
+ c.checkpointsMu.Lock()
+ defer c.checkpointsMu.Unlock()
+ c.checkpoints.Merge(spans.Valued{Key: kr, Value: u})
+ })
clampedRanges := utils.IntersectAll(ranges, utils.CloneSlice(c.taskRange))
for _, r := range clampedRanges {
r := r
workers.ApplyOnErrorGroup(eg, func() (e error) {
- defer c.recordTimeCost("get regions in range", zap.Uint64("checkpoint", rst.TS))()
+ defer c.recordTimeCost("get regions in range")()
defer utils.PanicToErr(&e)
return c.GetCheckpointInRange(cx, r.StartKey, r.EndKey, collector)
})
@@ -214,121 +158,47 @@ func (c *CheckpointAdvancer) tryAdvance(ctx context.Context, rst RangesSharesTS)
return err
}
- result, err := collector.Finish(ctx)
+ _, err = collector.Finish(ctx)
if err != nil {
return err
}
- fr := result.FailureSubRanges
- if len(fr) != 0 {
- log.Debug("failure regions collected", zap.Int("size", len(fr)))
- c.cache.InsertRanges(RangesSharesTS{
- TS: rst.TS,
- Ranges: fr,
- })
- }
return nil
}
-// CalculateGlobalCheckpointLight tries to advance the global checkpoint by the cache.
-func (c *CheckpointAdvancer) CalculateGlobalCheckpointLight(ctx context.Context) (uint64, error) {
- log.Info("[log backup advancer hint] advancer with cache: current tree", zap.Stringer("ct", c.cache))
- rsts := c.cache.PopRangesWithGapGT(config.DefaultTryAdvanceThreshold)
- if len(rsts) == 0 {
- return 0, nil
+func tsoBefore(n time.Duration) uint64 {
+ now := time.Now()
+ return oracle.ComposeTS(now.UnixMilli()-n.Milliseconds(), 0)
+}
+
+func (c *CheckpointAdvancer) CalculateGlobalCheckpointLight(ctx context.Context, threshold time.Duration) (uint64, error) {
+ var targets []spans.Valued
+ c.checkpoints.TraverseValuesLessThan(tsoBefore(threshold), func(v spans.Valued) bool {
+ targets = append(targets, v)
+ return true
+ })
+ if len(targets) == 0 {
+ c.checkpointsMu.Lock()
+ defer c.checkpointsMu.Unlock()
+ return c.checkpoints.MinValue(), nil
}
- samples := rsts
- if len(rsts) > 3 {
- samples = rsts[:3]
+ samples := targets
+ if len(targets) > 3 {
+ samples = targets[:3]
}
for _, sample := range samples {
- log.Info("[log backup advancer hint] sample range.", zap.Object("range", sample.Zap()), zap.Int("total-len", len(rsts)))
+ log.Info("[log backup advancer hint] sample range.", zap.Stringer("sample", sample), zap.Int("total-len", len(targets)))
}
- workers := utils.NewWorkerPool(uint(config.DefaultMaxConcurrencyAdvance), "regions")
- eg, cx := errgroup.WithContext(ctx)
- for _, rst := range rsts {
- rst := rst
- workers.ApplyOnErrorGroup(eg, func() (err error) {
- return c.tryAdvance(cx, *rst)
- })
- }
- err := eg.Wait()
+ err := c.tryAdvance(ctx, len(targets), func(i int) kv.KeyRange { return targets[i].Key })
if err != nil {
return 0, err
}
- ts := c.cache.CheckpointTS()
+ c.checkpointsMu.Lock()
+ ts := c.checkpoints.MinValue()
+ c.checkpointsMu.Unlock()
return ts, nil
}
-// CalculateGlobalCheckpoint calculates the global checkpoint, which won't use the cache.
-func (c *CheckpointAdvancer) CalculateGlobalCheckpoint(ctx context.Context) (uint64, error) {
- var (
- cp = uint64(math.MaxInt64)
- thisRun []kv.KeyRange = c.taskRange
- nextRun []kv.KeyRange
- )
- defer c.recordTimeCost("record all")
- for {
- coll := NewClusterCollector(ctx, c.env)
- coll.setOnSuccessHook(c.cache.InsertRange)
- for _, u := range thisRun {
- err := c.GetCheckpointInRange(ctx, u.StartKey, u.EndKey, coll)
- if err != nil {
- return 0, err
- }
- }
- result, err := coll.Finish(ctx)
- if err != nil {
- return 0, err
- }
- log.Debug("full: a run finished", zap.Any("checkpoint", result))
-
- nextRun = append(nextRun, result.FailureSubRanges...)
- if cp > result.Checkpoint {
- cp = result.Checkpoint
- }
- if len(nextRun) == 0 {
- return cp, nil
- }
- thisRun = nextRun
- nextRun = nil
- log.Debug("backoffing with subranges", zap.Int("subranges", len(thisRun)))
- time.Sleep(c.cfg.BackoffTime)
- }
-}
-
-// CollapseRanges collapse ranges overlapping or adjacent.
-// Example:
-// CollapseRanges({[1, 4], [2, 8], [3, 9]}) == {[1, 9]}
-// CollapseRanges({[1, 3], [4, 7], [2, 3]}) == {[1, 3], [4, 7]}
-func CollapseRanges(length int, getRange func(int) kv.KeyRange) []kv.KeyRange {
- frs := make([]kv.KeyRange, 0, length)
- for i := 0; i < length; i++ {
- frs = append(frs, getRange(i))
- }
-
- sort.Slice(frs, func(i, j int) bool {
- return bytes.Compare(frs[i].StartKey, frs[j].StartKey) < 0
- })
-
- result := make([]kv.KeyRange, 0, len(frs))
- i := 0
- for i < len(frs) {
- item := frs[i]
- for {
- i++
- if i >= len(frs) || (len(item.EndKey) != 0 && bytes.Compare(frs[i].StartKey, item.EndKey) > 0) {
- break
- }
- if len(item.EndKey) != 0 && bytes.Compare(item.EndKey, frs[i].EndKey) < 0 || len(frs[i].EndKey) == 0 {
- item.EndKey = frs[i].EndKey
- }
- }
- result = append(result, item)
- }
- return result
-}
-
func (c *CheckpointAdvancer) consumeAllTask(ctx context.Context, ch <-chan TaskEvent) error {
for {
select {
@@ -392,14 +262,16 @@ func (c *CheckpointAdvancer) StartTaskListener(ctx context.Context) {
return
case e, ok := <-ch:
if !ok {
+ log.Info("[log backup advancer] Task watcher exits due to stream ends.")
return
}
- log.Info("meet task event", zap.Stringer("event", &e))
+ log.Info("[log backup advancer] Meet task event", zap.Stringer("event", &e))
if err := c.onTaskEvent(ctx, e); err != nil {
if errors.Cause(e.Err) != context.Canceled {
log.Error("listen task meet error, would reopen.", logutil.ShortError(err))
time.AfterFunc(c.cfg.BackoffTime, func() { c.StartTaskListener(ctx) })
}
+ log.Info("[log backup advancer] Task watcher exits due to some error.", logutil.ShortError(err))
return
}
}
@@ -414,24 +286,42 @@ func (c *CheckpointAdvancer) onTaskEvent(ctx context.Context, e TaskEvent) error
case EventAdd:
utils.LogBackupTaskCountInc()
c.task = e.Info
- c.taskRange = CollapseRanges(len(e.Ranges), func(i int) kv.KeyRange { return e.Ranges[i] })
+ c.taskRange = spans.Collapse(len(e.Ranges), func(i int) kv.KeyRange { return e.Ranges[i] })
+ c.checkpoints = spans.Sorted(spans.NewFullWith(e.Ranges, 0))
+ c.lastCheckpoint = e.Info.StartTs
log.Info("added event", zap.Stringer("task", e.Info), zap.Stringer("ranges", logutil.StringifyKeys(c.taskRange)))
case EventDel:
utils.LogBackupTaskCountDec()
c.task = nil
c.taskRange = nil
- c.state = &fullScan{}
+ c.checkpoints = nil
+ // This would be synced by `taskMu`, perhaps we'd better rename that to `tickMu`.
+ // Do the null check because some of test cases won't equip the advancer with subscriber.
+ if c.subscriber != nil {
+ c.subscriber.Clear()
+ }
if err := c.env.ClearV3GlobalCheckpointForTask(ctx, e.Name); err != nil {
log.Warn("failed to clear global checkpoint", logutil.ShortError(err))
}
metrics.LastCheckpoint.DeleteLabelValues(e.Name)
- c.cache.Clear()
case EventErr:
return e.Err
}
return nil
}
+func (c *CheckpointAdvancer) setCheckpoint(cp uint64) bool {
+ if cp < c.lastCheckpoint {
+ log.Warn("failed to update global checkpoint: stale", zap.Uint64("old", c.lastCheckpoint), zap.Uint64("new", cp))
+ return false
+ }
+ if cp <= c.lastCheckpoint {
+ return false
+ }
+ c.lastCheckpoint = cp
+ return true
+}
+
// advanceCheckpointBy advances the checkpoint by a checkpoint getter function.
func (c *CheckpointAdvancer) advanceCheckpointBy(ctx context.Context, getCheckpoint func(context.Context) (uint64, error)) error {
start := time.Now()
@@ -439,79 +329,111 @@ func (c *CheckpointAdvancer) advanceCheckpointBy(ctx context.Context, getCheckpo
if err != nil {
return err
}
- log.Info("get checkpoint", zap.Uint64("old", c.lastCheckpoint), zap.Uint64("new", cp))
- if cp < c.lastCheckpoint {
- log.Warn("failed to update global checkpoint: stale", zap.Uint64("old", c.lastCheckpoint), zap.Uint64("new", cp))
+
+ if c.setCheckpoint(cp) {
+ log.Info("uploading checkpoint for task",
+ zap.Stringer("checkpoint", oracle.GetTimeFromTS(cp)),
+ zap.Uint64("checkpoint", cp),
+ zap.String("task", c.task.Name),
+ zap.Stringer("take", time.Since(start)))
+ metrics.LastCheckpoint.WithLabelValues(c.task.GetName()).Set(float64(c.lastCheckpoint))
}
- if cp <= c.lastCheckpoint {
+ return nil
+}
+
+func (c *CheckpointAdvancer) stopSubscriber() {
+ c.subscriberMu.Lock()
+ defer c.subscriberMu.Unlock()
+ c.subscriber.Drop()
+ c.subscriber = nil
+}
+
+func (c *CheckpointAdvancer) spawnSubscriptionHandler(ctx context.Context) {
+ c.subscriberMu.Lock()
+ defer c.subscriberMu.Unlock()
+ c.subscriber = NewSubscriber(c.env, c.env, WithMasterContext(ctx))
+ es := c.subscriber.Events()
+
+ go func() {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case event, ok := <-es:
+ if !ok {
+ return
+ }
+ c.checkpointsMu.Lock()
+ log.Debug("Accepting region flush event.",
+ zap.Stringer("range", logutil.StringifyRange(event.Key)),
+ zap.Uint64("checkpoint", event.Value))
+ c.checkpoints.Merge(event)
+ c.checkpointsMu.Unlock()
+ }
+ }
+ }()
+}
+
+func (c *CheckpointAdvancer) subscribeTick(ctx context.Context) error {
+ if c.subscriber == nil {
return nil
}
+ if err := c.subscriber.UpdateStoreTopology(ctx); err != nil {
+ log.Warn("[log backup advancer] Error when updating store topology.", logutil.ShortError(err))
+ }
+ c.subscriber.HandleErrors(ctx)
+ return c.subscriber.PendingErrors()
+}
- log.Info("uploading checkpoint for task",
- zap.Stringer("checkpoint", oracle.GetTimeFromTS(cp)),
- zap.Uint64("checkpoint", cp),
- zap.String("task", c.task.Name),
- zap.Stringer("take", time.Since(start)))
- if err := c.env.UploadV3GlobalCheckpointForTask(ctx, c.task.Name, cp); err != nil {
+func (c *CheckpointAdvancer) importantTick(ctx context.Context) error {
+ c.checkpointsMu.Lock()
+ c.setCheckpoint(c.checkpoints.MinValue())
+ c.checkpointsMu.Unlock()
+ if err := c.env.UploadV3GlobalCheckpointForTask(ctx, c.task.Name, c.lastCheckpoint); err != nil {
return errors.Annotate(err, "failed to upload global checkpoint")
}
- c.lastCheckpoint = cp
- metrics.LastCheckpoint.WithLabelValues(c.task.GetName()).Set(float64(c.lastCheckpoint))
return nil
}
-func (c *CheckpointAdvancer) onConsistencyCheckTick(s *updateSmallTree) error {
- if s.consistencyCheckTick > 0 {
- s.consistencyCheckTick--
- return nil
+func (c *CheckpointAdvancer) optionalTick(cx context.Context) error {
+ threshold := c.Config().GetDefaultStartPollThreshold()
+ if err := c.subscribeTick(cx); err != nil {
+ log.Warn("[log backup advancer] Subscriber meet error, would polling the checkpoint.", logutil.ShortError(err))
+ threshold = c.Config().GetSubscriberErrorStartPollThreshold()
}
- defer c.recordTimeCost("consistency check")()
- err := c.cache.ConsistencyCheck(c.taskRange)
+
+ err := c.advanceCheckpointBy(cx, func(cx context.Context) (uint64, error) {
+ return c.CalculateGlobalCheckpointLight(cx, threshold)
+ })
if err != nil {
- log.Error("consistency check failed! log backup may lose data! rolling back to full scan for saving.", logutil.ShortError(err))
- c.state = &fullScan{}
return err
}
- log.Debug("consistency check passed.")
- s.consistencyCheckTick = config.DefaultConsistencyCheckTick
return nil
}
func (c *CheckpointAdvancer) tick(ctx context.Context) error {
c.taskMu.Lock()
defer c.taskMu.Unlock()
+ if c.task == nil {
+ log.Debug("No tasks yet, skipping advancing.")
+ return nil
+ }
- switch s := c.state.(type) {
- case *fullScan:
- if s.fullScanTick > 0 {
- s.fullScanTick--
- break
- }
- if c.task == nil {
- log.Debug("No tasks yet, skipping advancing.")
- return nil
- }
- defer func() {
- s.fullScanTick = c.cfg.FullScanTick
- }()
- err := c.advanceCheckpointBy(ctx, c.CalculateGlobalCheckpoint)
- if err != nil {
- return err
- }
+ var errs error
- if c.cfg.AdvancingByCache {
- c.state = &updateSmallTree{}
- }
- case *updateSmallTree:
- if err := c.onConsistencyCheckTick(s); err != nil {
- return err
- }
- err := c.advanceCheckpointBy(ctx, c.CalculateGlobalCheckpointLight)
- if err != nil {
- return err
- }
- default:
- log.Error("Unknown state type, skipping tick", zap.Stringer("type", reflect.TypeOf(c.state)))
+ cx, cancel := context.WithTimeout(ctx, c.Config().TickTimeout())
+ defer cancel()
+ err := c.optionalTick(cx)
+ if err != nil {
+ log.Warn("[log backup advancer] option tick failed.", logutil.ShortError(err))
+ errs = multierr.Append(errs, err)
}
- return nil
+
+ err = c.importantTick(ctx)
+ if err != nil {
+ log.Warn("[log backup advancer] important tick failed.", logutil.ShortError(err))
+ errs = multierr.Append(errs, err)
+ }
+
+ return errs
}
diff --git a/br/pkg/streamhelper/advancer_cliext.go b/br/pkg/streamhelper/advancer_cliext.go
index 611ad3744dfa8..f3d7e1f279d5f 100644
--- a/br/pkg/streamhelper/advancer_cliext.go
+++ b/br/pkg/streamhelper/advancer_cliext.go
@@ -5,15 +5,22 @@ package streamhelper
import (
"bytes"
"context"
+ "encoding/binary"
"fmt"
+ "io"
"strings"
"github.com/golang/protobuf/proto"
"github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/log"
berrors "github.com/pingcap/tidb/br/pkg/errors"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/redact"
"github.com/pingcap/tidb/kv"
clientv3 "go.etcd.io/etcd/client/v3"
+ "go.uber.org/zap"
)
type EventType int
@@ -90,6 +97,9 @@ func (t AdvancerExt) toTaskEvent(ctx context.Context, event *clientv3.Event) (Ta
func (t AdvancerExt) eventFromWatch(ctx context.Context, resp clientv3.WatchResponse) ([]TaskEvent, error) {
result := make([]TaskEvent, 0, len(resp.Events))
+ if err := resp.Err(); err != nil {
+ return nil, err
+ }
for _, event := range resp.Events {
te, err := t.toTaskEvent(ctx, event)
if err != nil {
@@ -106,6 +116,7 @@ func (t AdvancerExt) startListen(ctx context.Context, rev int64, ch chan<- TaskE
handleResponse := func(resp clientv3.WatchResponse) bool {
events, err := t.eventFromWatch(ctx, resp)
if err != nil {
+ log.Warn("[log backup advancer] Meet error during receiving the task event.", logutil.ShortError(err))
ch <- errorEvent(err)
return false
}
@@ -114,33 +125,44 @@ func (t AdvancerExt) startListen(ctx context.Context, rev int64, ch chan<- TaskE
}
return true
}
+ collectRemaining := func() {
+ log.Info("[log backup advancer] Start collecting remaining events in the channel.", zap.Int("remained", len(c)))
+ defer log.Info("[log backup advancer] Finish collecting remaining events in the channel.")
+ for {
+ select {
+ case resp, ok := <-c:
+ if !ok {
+ return
+ }
+ if !handleResponse(resp) {
+ return
+ }
+ default:
+ return
+ }
+ }
+ }
go func() {
defer close(ch)
for {
select {
case resp, ok := <-c:
+ failpoint.Inject("advancer_close_channel", func() {
+ // We cannot really close the channel, just simulating it.
+ ok = false
+ })
if !ok {
+ ch <- errorEvent(io.EOF)
return
}
if !handleResponse(resp) {
return
}
case <-ctx.Done():
- // drain the remain event from channel.
- for {
- select {
- case resp, ok := <-c:
- if !ok {
- return
- }
- if !handleResponse(resp) {
- return
- }
- default:
- return
- }
- }
+ collectRemaining()
+ ch <- errorEvent(ctx.Err())
+ return
}
}
}()
@@ -181,11 +203,43 @@ func (t AdvancerExt) Begin(ctx context.Context, ch chan<- TaskEvent) error {
return nil
}
+func (t AdvancerExt) GetGlobalCheckpointForTask(ctx context.Context, taskName string) (uint64, error) {
+ key := GlobalCheckpointOf(taskName)
+ resp, err := t.KV.Get(ctx, key)
+ if err != nil {
+ return 0, err
+ }
+
+ if len(resp.Kvs) == 0 {
+ return 0, nil
+ }
+
+ firstKV := resp.Kvs[0]
+ value := firstKV.Value
+ if len(value) != 8 {
+ return 0, errors.Annotatef(berrors.ErrPiTRMalformedMetadata,
+ "the global checkpoint isn't 64bits (it is %d bytes, value = %s)",
+ len(value),
+ redact.Key(value))
+ }
+
+ return binary.BigEndian.Uint64(value), nil
+}
+
func (t AdvancerExt) UploadV3GlobalCheckpointForTask(ctx context.Context, taskName string, checkpoint uint64) error {
key := GlobalCheckpointOf(taskName)
value := string(encodeUint64(checkpoint))
- _, err := t.KV.Put(ctx, key, value)
+ oldValue, err := t.GetGlobalCheckpointForTask(ctx, taskName)
+ if err != nil {
+ return err
+ }
+
+ if checkpoint < oldValue {
+ log.Warn("[log backup advancer] skipping upload global checkpoint", zap.Uint64("old", oldValue), zap.Uint64("new", checkpoint))
+ return nil
+ }
+ _, err = t.KV.Put(ctx, key, value)
if err != nil {
return err
}
diff --git a/br/pkg/streamhelper/advancer_daemon.go b/br/pkg/streamhelper/advancer_daemon.go
index 263d3a761b518..10f43e105ccbe 100644
--- a/br/pkg/streamhelper/advancer_daemon.go
+++ b/br/pkg/streamhelper/advancer_daemon.go
@@ -30,6 +30,7 @@ func (c *CheckpointAdvancer) OnTick(ctx context.Context) (err error) {
func (c *CheckpointAdvancer) OnStart(ctx context.Context) {
metrics.AdvancerOwner.Set(1.0)
c.StartTaskListener(ctx)
+ c.spawnSubscriptionHandler(ctx)
go func() {
<-ctx.Done()
c.onStop()
@@ -43,6 +44,7 @@ func (c *CheckpointAdvancer) Name() string {
func (c *CheckpointAdvancer) onStop() {
metrics.AdvancerOwner.Set(0.0)
+ c.stopSubscriber()
}
func OwnerManagerForLogBackup(ctx context.Context, etcdCli *clientv3.Client) owner.Manager {
diff --git a/br/pkg/streamhelper/advancer_env.go b/br/pkg/streamhelper/advancer_env.go
index 181d8933449d4..cf27fda7d5c5b 100644
--- a/br/pkg/streamhelper/advancer_env.go
+++ b/br/pkg/streamhelper/advancer_env.go
@@ -9,6 +9,7 @@ import (
logbackup "github.com/pingcap/kvproto/pkg/logbackuppb"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/config"
+ "github.com/pingcap/tidb/util/engine"
pd "github.com/tikv/pd/client"
clientv3 "go.etcd.io/etcd/client/v3"
"google.golang.org/grpc"
@@ -18,7 +19,7 @@ import (
// Env is the interface required by the advancer.
type Env interface {
// The region scanner provides the region information.
- RegionScanner
+ TiKVClusterMeta
// LogBackupService connects to the TiKV, so we can collect the region checkpoints.
LogBackupService
// StreamMeta connects to the metadata service (normally PD).
@@ -48,6 +49,23 @@ func (c PDRegionScanner) RegionScan(ctx context.Context, key []byte, endKey []by
return rls, nil
}
+func (c PDRegionScanner) Stores(ctx context.Context) ([]Store, error) {
+ res, err := c.Client.GetAllStores(ctx, pd.WithExcludeTombstone())
+ if err != nil {
+ return nil, err
+ }
+ r := make([]Store, 0, len(res))
+ for _, re := range res {
+ if !engine.IsTiFlash(re) {
+ r = append(r, Store{
+ BootAt: uint64(re.StartTimestamp),
+ ID: re.GetId(),
+ })
+ }
+ }
+ return r, nil
+}
+
// clusterEnv is the environment for running in the real cluster.
type clusterEnv struct {
clis *utils.StoreManager
diff --git a/br/pkg/streamhelper/advancer_test.go b/br/pkg/streamhelper/advancer_test.go
index aeaadf820af7a..7dd4c71d35b9c 100644
--- a/br/pkg/streamhelper/advancer_test.go
+++ b/br/pkg/streamhelper/advancer_test.go
@@ -9,6 +9,8 @@ import (
"testing"
"time"
+ "github.com/pingcap/errors"
+ logbackup "github.com/pingcap/kvproto/pkg/logbackuppb"
"github.com/pingcap/log"
"github.com/pingcap/tidb/br/pkg/streamhelper"
"github.com/pingcap/tidb/br/pkg/streamhelper/config"
@@ -51,9 +53,6 @@ func TestTick(t *testing.T) {
env := &testEnv{fakeCluster: c, testCtx: t}
adv := streamhelper.NewCheckpointAdvancer(env)
adv.StartTaskListener(ctx)
- adv.UpdateConfigWith(func(cac *config.Config) {
- cac.FullScanTick = 0
- })
require.NoError(t, adv.OnTick(ctx))
for i := 0; i < 5; i++ {
cp := c.advanceCheckpoints()
@@ -76,9 +75,6 @@ func TestWithFailure(t *testing.T) {
env := &testEnv{fakeCluster: c, testCtx: t}
adv := streamhelper.NewCheckpointAdvancer(env)
adv.StartTaskListener(ctx)
- adv.UpdateConfigWith(func(cac *config.Config) {
- cac.FullScanTick = 0
- })
require.NoError(t, adv.OnTick(ctx))
cp := c.advanceCheckpoints()
@@ -226,3 +222,36 @@ func TestTaskRangesWithSplit(t *testing.T) {
shouldFinishInTime(t, 10*time.Second, "second advancing", func() { require.NoError(t, adv.OnTick(ctx)) })
require.Greater(t, env.getCheckpoint(), fstCheckpoint)
}
+
+func TestBlocked(t *testing.T) {
+ log.SetLevel(zapcore.DebugLevel)
+ c := createFakeCluster(t, 4, true)
+ ctx := context.Background()
+ req := require.New(t)
+ c.splitAndScatter("0012", "0034", "0048")
+ marked := false
+ for _, s := range c.stores {
+ s.clientMu.Lock()
+ s.onGetRegionCheckpoint = func(glftrr *logbackup.GetLastFlushTSOfRegionRequest) error {
+ // blocking the thread.
+ // this may happen when TiKV goes down or too busy.
+ <-(chan struct{})(nil)
+ return nil
+ }
+ s.clientMu.Unlock()
+ marked = true
+ }
+ req.True(marked, "failed to mark the cluster: ")
+ env := &testEnv{fakeCluster: c, testCtx: t}
+ adv := streamhelper.NewCheckpointAdvancer(env)
+ adv.StartTaskListener(ctx)
+ adv.UpdateConfigWith(func(c *config.Config) {
+ // ... So the tick timeout would be 100ms
+ c.TickDuration = 10 * time.Millisecond
+ })
+ var err error
+ shouldFinishInTime(t, time.Second, "ticking", func() {
+ err = adv.OnTick(ctx)
+ })
+ req.ErrorIs(errors.Cause(err), context.DeadlineExceeded)
+}
diff --git a/br/pkg/streamhelper/basic_lib_for_test.go b/br/pkg/streamhelper/basic_lib_for_test.go
index 9e438c32f0f1f..1dff77dd72864 100644
--- a/br/pkg/streamhelper/basic_lib_for_test.go
+++ b/br/pkg/streamhelper/basic_lib_for_test.go
@@ -7,6 +7,7 @@ import (
"context"
"encoding/hex"
"fmt"
+ "io"
"math"
"math/rand"
"sort"
@@ -21,10 +22,15 @@ import (
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/log"
"github.com/pingcap/tidb/br/pkg/streamhelper"
+ "github.com/pingcap/tidb/br/pkg/streamhelper/spans"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/util/codec"
"go.uber.org/zap"
"google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/metadata"
+ "google.golang.org/grpc/status"
)
type flushSimulator struct {
@@ -58,7 +64,7 @@ func (c *flushSimulator) fork() flushSimulator {
}
type region struct {
- rng kv.KeyRange
+ rng spans.Span
leader uint64
epoch uint64
id uint64
@@ -70,6 +76,12 @@ type region struct {
type fakeStore struct {
id uint64
regions map[uint64]*region
+
+ clientMu sync.Mutex
+ supportsSub bool
+ bootstrapAt uint64
+ fsub func(logbackup.SubscribeFlushEventResponse)
+ onGetRegionCheckpoint func(*logbackup.GetLastFlushTSOfRegionRequest) error
}
type fakeCluster struct {
@@ -82,16 +94,6 @@ type fakeCluster struct {
onGetClient func(uint64) error
}
-func overlaps(a, b kv.KeyRange) bool {
- if len(b.EndKey) == 0 {
- return len(a.EndKey) == 0 || bytes.Compare(a.EndKey, b.StartKey) > 0
- }
- if len(a.EndKey) == 0 {
- return len(b.EndKey) == 0 || bytes.Compare(b.EndKey, a.StartKey) > 0
- }
- return bytes.Compare(a.StartKey, b.EndKey) < 0 && bytes.Compare(b.StartKey, a.EndKey) < 0
-}
-
func (r *region) splitAt(newID uint64, k string) *region {
newRegion := ®ion{
rng: kv.KeyRange{StartKey: []byte(k), EndKey: r.rng.EndKey},
@@ -111,7 +113,84 @@ func (r *region) flush() {
r.fsim.flushedEpoch.Store(r.epoch)
}
+type trivialFlushStream struct {
+ c <-chan logbackup.SubscribeFlushEventResponse
+ cx context.Context
+}
+
+func (t trivialFlushStream) Recv() (*logbackup.SubscribeFlushEventResponse, error) {
+ select {
+ case item, ok := <-t.c:
+ if !ok {
+ return nil, io.EOF
+ }
+ return &item, nil
+ case <-t.cx.Done():
+ select {
+ case item, ok := <-t.c:
+ if !ok {
+ return nil, io.EOF
+ }
+ return &item, nil
+ default:
+ }
+ return nil, status.Error(codes.Canceled, t.cx.Err().Error())
+ }
+}
+
+func (t trivialFlushStream) Header() (metadata.MD, error) {
+ return make(metadata.MD), nil
+}
+
+func (t trivialFlushStream) Trailer() metadata.MD {
+ return make(metadata.MD)
+}
+
+func (t trivialFlushStream) CloseSend() error {
+ return nil
+}
+
+func (t trivialFlushStream) Context() context.Context {
+ return t.cx
+}
+
+func (t trivialFlushStream) SendMsg(m interface{}) error {
+ return nil
+}
+
+func (t trivialFlushStream) RecvMsg(m interface{}) error {
+ return nil
+}
+
+func (f *fakeStore) SubscribeFlushEvent(ctx context.Context, in *logbackup.SubscribeFlushEventRequest, opts ...grpc.CallOption) (logbackup.LogBackup_SubscribeFlushEventClient, error) {
+ f.clientMu.Lock()
+ defer f.clientMu.Unlock()
+ if !f.supportsSub {
+ return nil, status.Error(codes.Unimplemented, "meow?")
+ }
+
+ ch := make(chan logbackup.SubscribeFlushEventResponse, 1024)
+ f.fsub = func(glftrr logbackup.SubscribeFlushEventResponse) {
+ ch <- glftrr
+ }
+ return trivialFlushStream{c: ch, cx: ctx}, nil
+}
+
+func (f *fakeStore) SetSupportFlushSub(b bool) {
+ f.clientMu.Lock()
+ defer f.clientMu.Unlock()
+
+ f.bootstrapAt += 1
+ f.supportsSub = b
+}
+
func (f *fakeStore) GetLastFlushTSOfRegion(ctx context.Context, in *logbackup.GetLastFlushTSOfRegionRequest, opts ...grpc.CallOption) (*logbackup.GetLastFlushTSOfRegionResponse, error) {
+ if f.onGetRegionCheckpoint != nil {
+ err := f.onGetRegionCheckpoint(in)
+ if err != nil {
+ return nil, err
+ }
+ }
resp := &logbackup.GetLastFlushTSOfRegionResponse{
Checkpoints: []*logbackup.RegionCheckpoint{},
}
@@ -174,7 +253,7 @@ func (f *fakeCluster) RegionScan(ctx context.Context, key []byte, endKey []byte,
result := make([]streamhelper.RegionWithLeader, 0, limit)
for _, region := range f.regions {
- if overlaps(kv.KeyRange{StartKey: key, EndKey: endKey}, region.rng) && len(result) < limit {
+ if spans.Overlaps(kv.KeyRange{StartKey: key, EndKey: endKey}, region.rng) && len(result) < limit {
regionInfo := streamhelper.RegionWithLeader{
Region: &metapb.Region{
Id: region.id,
@@ -210,6 +289,15 @@ func (f *fakeCluster) GetLogBackupClient(ctx context.Context, storeID uint64) (l
return cli, nil
}
+// Stores returns the store metadata from the cluster.
+func (f *fakeCluster) Stores(ctx context.Context) ([]streamhelper.Store, error) {
+ r := make([]streamhelper.Store, 0, len(f.stores))
+ for id, s := range f.stores {
+ r = append(r, streamhelper.Store{ID: id, BootAt: s.bootstrapAt})
+ }
+ return r, nil
+}
+
func (f *fakeCluster) findRegionById(rid uint64) *region {
for _, r := range f.regions {
if r.id == rid {
@@ -304,6 +392,34 @@ func (f *fakeCluster) splitAndScatter(keys ...string) {
}
}
+// Remove a store.
+// Note: this won't add new peer for regions from the store.
+func (f *fakeCluster) removeStore(id uint64) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+
+ s := f.stores[id]
+ for _, r := range s.regions {
+ if r.leader == id {
+ f.updateRegion(r.id, func(r *region) {
+ ps := f.findPeers(r.id)
+ for _, p := range ps {
+ if p != r.leader {
+ log.Info("remove store: transforming leader",
+ zap.Uint64("region", r.id),
+ zap.Uint64("new-leader", p),
+ zap.Uint64("old-leader", r.leader))
+ r.leader = p
+ break
+ }
+ }
+ })
+ }
+ }
+
+ delete(f.stores, id)
+}
+
// a stub once in the future we want to make different stores hold different region instances.
func (f *fakeCluster) updateRegion(rid uint64, mut func(*region)) {
r := f.findRegionById(rid)
@@ -362,7 +478,7 @@ func createFakeCluster(t *testing.T, n int, simEnabled bool) *fakeCluster {
}
func (r *region) String() string {
- return fmt.Sprintf("%d(%d):[%s,%s);%dL%dF%d",
+ return fmt.Sprintf("%d(%d):[%s, %s);%dL%dF%d",
r.id,
r.epoch,
hex.EncodeToString(r.rng.StartKey),
@@ -382,14 +498,24 @@ func (f *fakeStore) String() string {
}
func (f *fakeCluster) flushAll() {
- for _, r := range f.regions {
+ for _, r := range f.stores {
r.flush()
}
}
func (f *fakeCluster) flushAllExcept(keys ...string) {
+ for _, s := range f.stores {
+ s.flushExcept(keys...)
+ }
+}
+
+func (f *fakeStore) flushExcept(keys ...string) {
+ resp := make([]*logbackup.FlushEvent, 0, len(f.regions))
outer:
for _, r := range f.regions {
+ if r.leader != f.id {
+ continue
+ }
// Note: can we make it faster?
for _, key := range keys {
if utils.CompareBytesExt(r.rng.StartKey, false, []byte(key), false) <= 0 &&
@@ -397,16 +523,25 @@ outer:
continue outer
}
}
- r.flush()
- }
-}
-
-func (f *fakeStore) flush() {
- for _, r := range f.regions {
if r.leader == f.id {
r.flush()
+ resp = append(resp, &logbackup.FlushEvent{
+ StartKey: codec.EncodeBytes(nil, r.rng.StartKey),
+ EndKey: codec.EncodeBytes(nil, r.rng.EndKey),
+ Checkpoint: r.checkpoint.Load(),
+ })
}
}
+
+ if f.fsub != nil {
+ f.fsub(logbackup.SubscribeFlushEventResponse{
+ Events: resp,
+ })
+ }
+}
+
+func (f *fakeStore) flush() {
+ f.flushExcept()
}
func (f *fakeCluster) String() string {
diff --git a/br/pkg/streamhelper/client.go b/br/pkg/streamhelper/client.go
index 4b52f590dce21..3a004fc80d3e1 100644
--- a/br/pkg/streamhelper/client.go
+++ b/br/pkg/streamhelper/client.go
@@ -163,6 +163,8 @@ func (c *MetaDataClient) DeleteTask(ctx context.Context, taskName string) error
clientv3.OpDelete(CheckPointsOf(taskName), clientv3.WithPrefix()),
clientv3.OpDelete(Pause(taskName)),
clientv3.OpDelete(LastErrorPrefixOf(taskName), clientv3.WithPrefix()),
+ clientv3.OpDelete(GlobalCheckpointOf(taskName)),
+ clientv3.OpDelete(StorageCheckpointOf(taskName), clientv3.WithPrefix()),
).
Commit()
if err != nil {
@@ -372,28 +374,6 @@ func (t *Task) GetStorageCheckpoint(ctx context.Context) (uint64, error) {
return storageCheckpoint, nil
}
-// MinNextBackupTS query the all next backup ts of a store, returning the minimal next backup ts of the store.
-func (t *Task) MinNextBackupTS(ctx context.Context, store uint64) (uint64, error) {
- key := CheckPointOf(t.Info.Name, store)
- resp, err := t.cli.KV.Get(ctx, key)
- if err != nil {
- return 0, errors.Annotatef(err, "failed to get checkpoints of %s", t.Info.Name)
- }
- if resp.Count != 1 {
- return 0, nil
- }
- kv := resp.Kvs[0]
- if len(kv.Value) != 8 {
- return 0, errors.Annotatef(berrors.ErrPiTRMalformedMetadata,
- "the next backup ts of store %d isn't 64bits (it is %d bytes, value = %s)",
- store,
- len(kv.Value),
- redact.Key(kv.Value))
- }
- nextBackupTS := binary.BigEndian.Uint64(kv.Value)
- return nextBackupTS, nil
-}
-
// GetGlobalCheckPointTS gets the global checkpoint timestamp according to log task.
func (t *Task) GetGlobalCheckPointTS(ctx context.Context) (uint64, error) {
checkPointMap, err := t.NextBackupTSList(ctx)
@@ -422,16 +402,6 @@ func (t *Task) GetGlobalCheckPointTS(ctx context.Context) (uint64, error) {
return mathutil.Max(checkpoint, ts), nil
}
-// Step forwards the progress (next_backup_ts) of some region.
-// The task should be done by TiKV. This function should only be used for test cases.
-func (t *Task) Step(ctx context.Context, store uint64, ts uint64) error {
- _, err := t.cli.KV.Put(ctx, CheckPointOf(t.Info.Name, store), string(encodeUint64(ts)))
- if err != nil {
- return errors.Annotatef(err, "failed forward the progress of %s to %d", t.Info.Name, ts)
- }
- return nil
-}
-
func (t *Task) UploadGlobalCheckpoint(ctx context.Context, ts uint64) error {
_, err := t.cli.KV.Put(ctx, GlobalCheckpointOf(t.Info.Name), string(encodeUint64(ts)))
if err != nil {
diff --git a/br/pkg/streamhelper/collector.go b/br/pkg/streamhelper/collector.go
index ad53acb03b577..bc9285e05e8a8 100644
--- a/br/pkg/streamhelper/collector.go
+++ b/br/pkg/streamhelper/collector.go
@@ -266,13 +266,13 @@ func NewClusterCollector(ctx context.Context, srv LogBackupService) *clusterColl
}
}
-// setOnSuccessHook sets the hook when getting checkpoint of some region.
-func (c *clusterCollector) setOnSuccessHook(hook onSuccessHook) {
+// SetOnSuccessHook sets the hook when getting checkpoint of some region.
+func (c *clusterCollector) SetOnSuccessHook(hook onSuccessHook) {
c.onSuccess = hook
}
-// collectRegion adds a region to the collector.
-func (c *clusterCollector) collectRegion(r RegionWithLeader) error {
+// CollectRegion adds a region to the collector.
+func (c *clusterCollector) CollectRegion(r RegionWithLeader) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.masterCtx.Err() != nil {
diff --git a/br/pkg/streamhelper/config/advancer_conf.go b/br/pkg/streamhelper/config/advancer_conf.go
index 548ea2472b172..1440a81f932f9 100644
--- a/br/pkg/streamhelper/config/advancer_conf.go
+++ b/br/pkg/streamhelper/config/advancer_conf.go
@@ -9,13 +9,14 @@ import (
)
const (
- flagBackoffTime = "backoff-time"
- flagTickInterval = "tick-interval"
- flagFullScanDiffTick = "full-scan-tick"
- flagAdvancingByCache = "advancing-by-cache"
+ flagBackoffTime = "backoff-time"
+ flagTickInterval = "tick-interval"
+ flagFullScanDiffTick = "full-scan-tick"
+ flagAdvancingByCache = "advancing-by-cache"
+ flagTryAdvanceThreshold = "try-advance-threshold"
DefaultConsistencyCheckTick = 5
- DefaultTryAdvanceThreshold = 108 * time.Second
+ DefaultTryAdvanceThreshold = 4 * time.Minute
DefaultBackOffTime = 5 * time.Second
DefaultTickInterval = 12 * time.Second
DefaultFullScanTick = 4
@@ -31,27 +32,21 @@ type Config struct {
BackoffTime time.Duration `toml:"backoff-time" json:"backoff-time"`
// The gap between calculating checkpoints.
TickDuration time.Duration `toml:"tick-interval" json:"tick-interval"`
- // The backoff time of full scan.
- FullScanTick int `toml:"full-scan-tick" json:"full-scan-tick"`
-
- // Whether enable the optimization -- use a cached heap to advancing the global checkpoint.
- // This may reduce the gap of checkpoint but may cost more CPU.
- AdvancingByCache bool `toml:"advancing-by-cache" json:"advancing-by-cache"`
+ // The threshold for polling TiKV for checkpoint of some range.
+ TryAdvanceThreshold time.Duration `toml:"try-advance-threshold" json:"try-advance-threshold"`
}
func DefineFlagsForCheckpointAdvancerConfig(f *pflag.FlagSet) {
f.Duration(flagBackoffTime, DefaultBackOffTime, "The gap between two retries.")
f.Duration(flagTickInterval, DefaultTickInterval, "From how long we trigger the tick (advancing the checkpoint).")
- f.Bool(flagAdvancingByCache, DefaultAdvanceByCache, "Whether enable the optimization -- use a cached heap to advancing the global checkpoint.")
- f.Int(flagFullScanDiffTick, DefaultFullScanTick, "The backoff of full scan.")
+ f.Duration(flagTryAdvanceThreshold, DefaultTryAdvanceThreshold, "If the checkpoint lag is greater than how long, we would try to poll TiKV for checkpoints.")
}
func Default() Config {
return Config{
- BackoffTime: DefaultBackOffTime,
- TickDuration: DefaultTickInterval,
- FullScanTick: DefaultFullScanTick,
- AdvancingByCache: DefaultAdvanceByCache,
+ BackoffTime: DefaultBackOffTime,
+ TickDuration: DefaultTickInterval,
+ TryAdvanceThreshold: DefaultTryAdvanceThreshold,
}
}
@@ -65,13 +60,34 @@ func (conf *Config) GetFromFlags(f *pflag.FlagSet) error {
if err != nil {
return err
}
- conf.FullScanTick, err = f.GetInt(flagFullScanDiffTick)
- if err != nil {
- return err
- }
- conf.AdvancingByCache, err = f.GetBool(flagAdvancingByCache)
+ conf.TryAdvanceThreshold, err = f.GetDuration(flagTryAdvanceThreshold)
if err != nil {
return err
}
return nil
}
+
+// GetDefaultStartPollThreshold returns the threshold of begin polling the checkpoint
+// in the normal condition (the subscribe manager is available.)
+func (conf Config) GetDefaultStartPollThreshold() time.Duration {
+ return conf.TryAdvanceThreshold
+}
+
+// GetSubscriberErrorStartPollThreshold returns the threshold of begin polling the checkpoint
+// when the subscriber meets error.
+func (conf Config) GetSubscriberErrorStartPollThreshold() time.Duration {
+ // 0.45x of the origin threshold.
+ // The origin threshold is 0.8x the target RPO,
+ // and the default flush interval is about 0.5x the target RPO.
+ // So the relationship between the RPO and the threshold is:
+ // When subscription is all available, it is 1.7x of the flush interval (which allow us to save in abnormal condition).
+ // When some of subscriptions are not available, it is 0.75x of the flush interval.
+ // NOTE: can we make subscription better and give up the poll model?
+ return conf.TryAdvanceThreshold * 9 / 20
+}
+
+// TickTimeout returns the max duration for each tick.
+func (conf Config) TickTimeout() time.Duration {
+ // If a tick blocks longer than the interval of ticking, we may need to break it and retry.
+ return conf.TickDuration
+}
diff --git a/br/pkg/streamhelper/daemon/BUILD.bazel b/br/pkg/streamhelper/daemon/BUILD.bazel
index e096487050d04..751ade885e324 100644
--- a/br/pkg/streamhelper/daemon/BUILD.bazel
+++ b/br/pkg/streamhelper/daemon/BUILD.bazel
@@ -18,6 +18,7 @@ go_library(
go_test(
name = "daemon_test",
+ timeout = "short",
srcs = ["owner_daemon_test.go"],
flaky = True,
deps = [
diff --git a/br/pkg/streamhelper/flush_subscriber.go b/br/pkg/streamhelper/flush_subscriber.go
new file mode 100644
index 0000000000000..70cd4d8e4501d
--- /dev/null
+++ b/br/pkg/streamhelper/flush_subscriber.go
@@ -0,0 +1,329 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+
+package streamhelper
+
+import (
+ "context"
+ "io"
+ "strconv"
+ "sync"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/pingcap/errors"
+ logbackup "github.com/pingcap/kvproto/pkg/logbackuppb"
+ "github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/streamhelper/spans"
+ "github.com/pingcap/tidb/metrics"
+ "github.com/pingcap/tidb/util/codec"
+ "go.uber.org/multierr"
+ "go.uber.org/zap"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+)
+
+// FlushSubscriber maintains the state of subscribing to the cluster.
+type FlushSubscriber struct {
+ dialer LogBackupService
+ cluster TiKVClusterMeta
+
+ // Current connections.
+ subscriptions map[uint64]*subscription
+ // The output channel.
+ eventsTunnel chan spans.Valued
+ // The background context for subscribes.
+ masterCtx context.Context
+}
+
+// SubscriberConfig is a config which cloud be applied into the subscriber.
+type SubscriberConfig func(*FlushSubscriber)
+
+// WithMasterContext sets the "master context" for the subscriber,
+// that context would be the "background" context for every subtasks created by the subscription manager.
+func WithMasterContext(ctx context.Context) SubscriberConfig {
+ return func(fs *FlushSubscriber) { fs.masterCtx = ctx }
+}
+
+// NewSubscriber creates a new subscriber via the environment and optional configs.
+func NewSubscriber(dialer LogBackupService, cluster TiKVClusterMeta, config ...SubscriberConfig) *FlushSubscriber {
+ subs := &FlushSubscriber{
+ dialer: dialer,
+ cluster: cluster,
+
+ subscriptions: map[uint64]*subscription{},
+ eventsTunnel: make(chan spans.Valued, 1024),
+ masterCtx: context.Background(),
+ }
+
+ for _, c := range config {
+ c(subs)
+ }
+
+ return subs
+}
+
+// UpdateStoreTopology fetches the current store topology and try to adapt the subscription state with it.
+func (f *FlushSubscriber) UpdateStoreTopology(ctx context.Context) error {
+ stores, err := f.cluster.Stores(ctx)
+ if err != nil {
+ return errors.Annotate(err, "failed to get store list")
+ }
+
+ storeSet := map[uint64]struct{}{}
+ for _, store := range stores {
+ sub, ok := f.subscriptions[store.ID]
+ if !ok {
+ f.addSubscription(ctx, store)
+ f.subscriptions[store.ID].connect(f.masterCtx, f.dialer)
+ } else if sub.storeBootAt != store.BootAt {
+ sub.storeBootAt = store.BootAt
+ sub.connect(f.masterCtx, f.dialer)
+ }
+ storeSet[store.ID] = struct{}{}
+ }
+
+ for id := range f.subscriptions {
+ _, ok := storeSet[id]
+ if !ok {
+ f.removeSubscription(id)
+ }
+ }
+ return nil
+}
+
+// Clear clears all the subscriptions.
+func (f *FlushSubscriber) Clear() {
+ log.Info("[log backup flush subscriber] Clearing.")
+ for id := range f.subscriptions {
+ f.removeSubscription(id)
+ }
+}
+
+// Drop terminates the lifetime of the subscriber.
+// This subscriber would be no more usable.
+func (f *FlushSubscriber) Drop() {
+ f.Clear()
+ close(f.eventsTunnel)
+}
+
+// HandleErrors execute the handlers over all pending errors.
+// Note that the handler may cannot handle the pending errors, at that time,
+// you can fetch the errors via `PendingErrors` call.
+func (f *FlushSubscriber) HandleErrors(ctx context.Context) {
+ for id, sub := range f.subscriptions {
+ err := sub.loadError()
+ if err != nil {
+ retry := f.canBeRetried(err)
+ log.Warn("[log backup flush subscriber] Meet error.", logutil.ShortError(err), zap.Bool("can-retry?", retry), zap.Uint64("store", id))
+ if retry {
+ sub.connect(f.masterCtx, f.dialer)
+ }
+ }
+ }
+}
+
+// Events returns the output channel of the events.
+func (f *FlushSubscriber) Events() <-chan spans.Valued {
+ return f.eventsTunnel
+}
+
+type eventStream = logbackup.LogBackup_SubscribeFlushEventClient
+
+type joinHandle <-chan struct{}
+
+func (jh joinHandle) WaitTimeOut(dur time.Duration) {
+ var t <-chan time.Time
+ if dur > 0 {
+ t = time.After(dur)
+ }
+ select {
+ case <-jh:
+ case <-t:
+ log.Warn("join handle timed out.")
+ }
+}
+
+func spawnJoinable(f func()) joinHandle {
+ c := make(chan struct{})
+ go func() {
+ defer close(c)
+ f()
+ }()
+ return c
+}
+
+// subscription is the state of subscription of one store.
+// initially, it is IDLE, where cancel == nil.
+// once `connect` called, it goto CONNECTED, where cancel != nil and err == nil.
+// once some error (both foreground or background) happens, it goto ERROR, where err != nil.
+type subscription struct {
+ // the handle to cancel the worker goroutine.
+ cancel context.CancelFunc
+ // the handle to wait until the worker goroutine exits.
+ background joinHandle
+ errMu sync.Mutex
+ err error
+
+ // Immutable state.
+ storeID uint64
+ // We record start bootstrap time and once a store restarts
+ // we need to try reconnect even there is a error cannot be retry.
+ storeBootAt uint64
+ output chan<- spans.Valued
+}
+
+func (s *subscription) emitError(err error) {
+ s.errMu.Lock()
+ defer s.errMu.Unlock()
+
+ s.err = err
+}
+
+func (s *subscription) loadError() error {
+ s.errMu.Lock()
+ defer s.errMu.Unlock()
+
+ return s.err
+}
+
+func (s *subscription) clearError() {
+ s.errMu.Lock()
+ defer s.errMu.Unlock()
+
+ s.err = nil
+}
+
+func newSubscription(toStore Store, output chan<- spans.Valued) *subscription {
+ return &subscription{
+ storeID: toStore.ID,
+ storeBootAt: toStore.BootAt,
+ output: output,
+ }
+}
+
+func (s *subscription) connect(ctx context.Context, dialer LogBackupService) {
+ err := s.doConnect(ctx, dialer)
+ if err != nil {
+ s.emitError(err)
+ }
+}
+
+func (s *subscription) doConnect(ctx context.Context, dialer LogBackupService) error {
+ log.Info("[log backup subscription manager] Adding subscription.", zap.Uint64("store", s.storeID), zap.Uint64("boot", s.storeBootAt))
+ // We should shutdown the background task firstly.
+ // Once it yields some error during shuting down, the error won't be brought to next run.
+ s.close()
+ s.clearError()
+
+ c, err := dialer.GetLogBackupClient(ctx, s.storeID)
+ if err != nil {
+ return errors.Annotate(err, "failed to get log backup client")
+ }
+ cx, cancel := context.WithCancel(ctx)
+ cli, err := c.SubscribeFlushEvent(cx, &logbackup.SubscribeFlushEventRequest{
+ ClientId: uuid.NewString(),
+ })
+ if err != nil {
+ cancel()
+ return errors.Annotate(err, "failed to subscribe events")
+ }
+ s.cancel = cancel
+ s.background = spawnJoinable(func() { s.listenOver(cli) })
+ return nil
+}
+
+func (s *subscription) close() {
+ if s.cancel != nil {
+ s.cancel()
+ s.background.WaitTimeOut(1 * time.Minute)
+ }
+ // HACK: don't close the internal channel here,
+ // because it is a ever-sharing channel.
+}
+
+func (s *subscription) listenOver(cli eventStream) {
+ storeID := s.storeID
+ log.Info("[log backup flush subscriber] Listen starting.", zap.Uint64("store", storeID))
+ for {
+ // Shall we use RecvMsg for better performance?
+ // Note that the spans.Full requires the input slice be immutable.
+ msg, err := cli.Recv()
+ if err != nil {
+ log.Info("[log backup flush subscriber] Listen stopped.", zap.Uint64("store", storeID), logutil.ShortError(err))
+ if err == io.EOF || err == context.Canceled || status.Code(err) == codes.Canceled {
+ return
+ }
+ s.emitError(errors.Annotatef(err, "while receiving from store id %d", storeID))
+ return
+ }
+
+ for _, m := range msg.Events {
+ start, err := decodeKey(m.StartKey)
+ if err != nil {
+ log.Warn("start key not encoded, skipping", logutil.Key("event", m.StartKey), logutil.ShortError(err))
+ continue
+ }
+ end, err := decodeKey(m.EndKey)
+ if err != nil {
+ log.Warn("end key not encoded, skipping", logutil.Key("event", m.EndKey), logutil.ShortError(err))
+ continue
+ }
+ s.output <- spans.Valued{
+ Key: spans.Span{
+ StartKey: start,
+ EndKey: end,
+ },
+ Value: m.Checkpoint,
+ }
+ }
+ metrics.RegionCheckpointSubscriptionEvent.WithLabelValues(strconv.Itoa(int(storeID))).Add(float64(len(msg.Events)))
+ }
+}
+
+func (f *FlushSubscriber) addSubscription(ctx context.Context, toStore Store) {
+ f.subscriptions[toStore.ID] = newSubscription(toStore, f.eventsTunnel)
+}
+
+func (f *FlushSubscriber) removeSubscription(toStore uint64) {
+ subs, ok := f.subscriptions[toStore]
+ if ok {
+ log.Info("[log backup subscription manager] Removing subscription.", zap.Uint64("store", toStore))
+ subs.close()
+ delete(f.subscriptions, toStore)
+ }
+}
+
+// decodeKey decodes the key from TiKV, because the region range is encoded in TiKV.
+func decodeKey(key []byte) ([]byte, error) {
+ if len(key) == 0 {
+ return key, nil
+ }
+ // Ignore the timestamp...
+ _, data, err := codec.DecodeBytes(key, nil)
+ if err != nil {
+ return key, err
+ }
+ return data, err
+}
+
+func (f *FlushSubscriber) canBeRetried(err error) bool {
+ for _, e := range multierr.Errors(errors.Cause(err)) {
+ s := status.Convert(e)
+ // Is there any other error cannot be retried?
+ if s.Code() == codes.Unimplemented {
+ return false
+ }
+ }
+ return true
+}
+
+func (f *FlushSubscriber) PendingErrors() error {
+ var allErr error
+ for _, s := range f.subscriptions {
+ if err := s.loadError(); err != nil {
+ allErr = multierr.Append(allErr, errors.Annotatef(err, "store %d has error", s.storeID))
+ }
+ }
+ return allErr
+}
diff --git a/br/pkg/streamhelper/integration_test.go b/br/pkg/streamhelper/integration_test.go
index b3baf433c43f6..4b989dc4ab2ba 100644
--- a/br/pkg/streamhelper/integration_test.go
+++ b/br/pkg/streamhelper/integration_test.go
@@ -7,13 +7,15 @@ import (
"context"
"encoding/binary"
"fmt"
+ "io"
"net"
"net/url"
"path"
"testing"
"github.com/pingcap/errors"
- backup "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/failpoint"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/log"
berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/pingcap/tidb/br/pkg/logutil"
@@ -138,11 +140,12 @@ func TestIntegration(t *testing.T) {
defer etcd.Server.Stop()
metaCli := streamhelper.MetaDataClient{Client: cli}
t.Run("TestBasic", func(t *testing.T) { testBasic(t, metaCli, etcd) })
- t.Run("TestForwardProgress", func(t *testing.T) { testForwardProgress(t, metaCli, etcd) })
t.Run("testGetStorageCheckpoint", func(t *testing.T) { testGetStorageCheckpoint(t, metaCli) })
t.Run("testGetGlobalCheckPointTS", func(t *testing.T) { testGetGlobalCheckPointTS(t, metaCli) })
t.Run("TestStreamListening", func(t *testing.T) { testStreamListening(t, streamhelper.AdvancerExt{MetaDataClient: metaCli}) })
t.Run("TestStreamCheckpoint", func(t *testing.T) { testStreamCheckpoint(t, streamhelper.AdvancerExt{MetaDataClient: metaCli}) })
+ t.Run("testStoptask", func(t *testing.T) { testStoptask(t, streamhelper.AdvancerExt{MetaDataClient: metaCli}) })
+ t.Run("TestStreamClose", func(t *testing.T) { testStreamClose(t, streamhelper.AdvancerExt{MetaDataClient: metaCli}) })
}
func TestChecking(t *testing.T) {
@@ -210,31 +213,6 @@ func testBasic(t *testing.T, metaCli streamhelper.MetaDataClient, etcd *embed.Et
rangeIsEmpty(t, []byte(streamhelper.RangesOf(taskName)), etcd)
}
-func testForwardProgress(t *testing.T, metaCli streamhelper.MetaDataClient, etcd *embed.Etcd) {
- ctx := context.Background()
- taskName := "many_tables"
- taskInfo := simpleTask(taskName, 65)
- defer func() {
- require.NoError(t, metaCli.DeleteTask(ctx, taskName))
- }()
-
- require.NoError(t, metaCli.PutTask(ctx, taskInfo))
- task, err := metaCli.GetTask(ctx, taskName)
- require.NoError(t, err)
- require.NoError(t, task.Step(ctx, 1, 41))
- require.NoError(t, task.Step(ctx, 1, 42))
- require.NoError(t, task.Step(ctx, 2, 40))
- rs, err := task.Ranges(ctx)
- require.NoError(t, err)
- require.Equal(t, simpleRanges(65), rs)
- store1Checkpoint, err := task.MinNextBackupTS(ctx, 1)
- require.NoError(t, err)
- require.Equal(t, store1Checkpoint, uint64(42))
- store2Checkpoint, err := task.MinNextBackupTS(ctx, 2)
- require.NoError(t, err)
- require.Equal(t, store2Checkpoint, uint64(40))
-}
-
func testGetStorageCheckpoint(t *testing.T, metaCli streamhelper.MetaDataClient) {
var (
taskName = "my_task"
@@ -298,7 +276,7 @@ func testGetGlobalCheckPointTS(t *testing.T, metaCli streamhelper.MetaDataClient
require.NoError(t, err)
}
- task := streamhelper.NewTask(&metaCli, backup.StreamBackupTaskInfo{Name: taskName})
+ task := streamhelper.NewTask(&metaCli, backuppb.StreamBackupTaskInfo{Name: taskName})
task.UploadGlobalCheckpoint(ctx, 1003)
globalTS, err := task.GetGlobalCheckPointTS(ctx)
@@ -320,6 +298,7 @@ func testStreamListening(t *testing.T, metaCli streamhelper.AdvancerExt) {
taskInfo2 := simpleTask(taskName2, 4)
require.NoError(t, metaCli.PutTask(ctx, taskInfo2))
require.NoError(t, metaCli.DeleteTask(ctx, taskName2))
+
first := <-ch
require.Equal(t, first.Type, streamhelper.EventAdd)
require.Equal(t, first.Name, taskName)
@@ -335,27 +314,130 @@ func testStreamListening(t *testing.T, metaCli streamhelper.AdvancerExt) {
require.Equal(t, forth.Type, streamhelper.EventDel)
require.Equal(t, forth.Name, taskName2)
cancel()
- _, ok := <-ch
- require.False(t, ok)
+ fifth, ok := <-ch
+ require.True(t, ok)
+ require.Equal(t, fifth.Type, streamhelper.EventErr)
+ require.Error(t, fifth.Err, context.Canceled)
+ item, ok := <-ch
+ require.False(t, ok, "%v", item)
+}
+
+func testStreamClose(t *testing.T, metaCli streamhelper.AdvancerExt) {
+ ctx := context.Background()
+ taskName := "close_simple"
+ taskInfo := simpleTask(taskName, 4)
+
+ require.NoError(t, metaCli.PutTask(ctx, taskInfo))
+ ch := make(chan streamhelper.TaskEvent, 1024)
+ require.NoError(t, metaCli.Begin(ctx, ch))
+ require.NoError(t, metaCli.DeleteTask(ctx, taskName))
+ first := <-ch
+ require.Equal(t, first.Type, streamhelper.EventAdd)
+ require.Equal(t, first.Name, taskName)
+ require.ElementsMatch(t, first.Ranges, simpleRanges(4))
+ second := <-ch
+ require.Equal(t, second.Type, streamhelper.EventDel, "%s", second)
+ require.Equal(t, second.Name, taskName, "%s", second)
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/streamhelper/advancer_close_channel", "return"))
+ defer failpoint.Disable("github.com/pingcap/tidb/br/pkg/streamhelper/advancer_close_channel")
+ // We need to make the channel file some events hence we can simulate the closed channel.
+ taskName2 := "close_simple2"
+ taskInfo2 := simpleTask(taskName2, 4)
+ require.NoError(t, metaCli.PutTask(ctx, taskInfo2))
+ require.NoError(t, metaCli.DeleteTask(ctx, taskName2))
+
+ third := <-ch
+ require.Equal(t, third.Type, streamhelper.EventErr)
+ require.Error(t, third.Err, io.EOF)
+ item, ok := <-ch
+ require.False(t, ok, "%#v", item)
}
func testStreamCheckpoint(t *testing.T, metaCli streamhelper.AdvancerExt) {
ctx := context.Background()
task := "simple"
req := require.New(t)
- getCheckpoint := func() uint64 {
- resp, err := metaCli.KV.Get(ctx, streamhelper.GlobalCheckpointOf(task))
- req.NoError(err)
- if len(resp.Kvs) == 0 {
- return 0
+
+ req.NoError(metaCli.UploadV3GlobalCheckpointForTask(ctx, task, 5))
+ ts, err := metaCli.GetGlobalCheckpointForTask(ctx, task)
+ req.NoError(err)
+ req.EqualValues(5, ts)
+ req.NoError(metaCli.UploadV3GlobalCheckpointForTask(ctx, task, 18))
+ ts, err = metaCli.GetGlobalCheckpointForTask(ctx, task)
+ req.NoError(err)
+ req.EqualValues(18, ts)
+ req.NoError(metaCli.UploadV3GlobalCheckpointForTask(ctx, task, 16))
+ ts, err = metaCli.GetGlobalCheckpointForTask(ctx, task)
+ req.NoError(err)
+ req.EqualValues(18, ts)
+ req.NoError(metaCli.ClearV3GlobalCheckpointForTask(ctx, task))
+ ts, err = metaCli.GetGlobalCheckpointForTask(ctx, task)
+ req.NoError(err)
+ req.EqualValues(0, ts)
+}
+
+func testStoptask(t *testing.T, metaCli streamhelper.AdvancerExt) {
+ var (
+ ctx = context.Background()
+ taskName = "stop_task"
+ req = require.New(t)
+ taskInfo = streamhelper.TaskInfo{
+ PBInfo: backuppb.StreamBackupTaskInfo{
+ Name: taskName,
+ StartTs: 0,
+ },
}
- req.Len(resp.Kvs, 1)
- return binary.BigEndian.Uint64(resp.Kvs[0].Value)
- }
- metaCli.UploadV3GlobalCheckpointForTask(ctx, task, 5)
- req.EqualValues(5, getCheckpoint())
- metaCli.UploadV3GlobalCheckpointForTask(ctx, task, 18)
- req.EqualValues(18, getCheckpoint())
- metaCli.ClearV3GlobalCheckpointForTask(ctx, task)
- req.EqualValues(0, getCheckpoint())
+ storeID = "5"
+ storageCheckpoint = make([]byte, 8)
+ )
+
+ // put task
+ req.NoError(metaCli.PutTask(ctx, taskInfo))
+ t2, err := metaCli.GetTask(ctx, taskName)
+ req.NoError(err)
+ req.EqualValues(taskInfo.PBInfo.Name, t2.Info.Name)
+
+ // upload global checkpoint
+ req.NoError(metaCli.UploadV3GlobalCheckpointForTask(ctx, taskName, 100))
+ ts, err := metaCli.GetGlobalCheckpointForTask(ctx, taskName)
+ req.NoError(err)
+ req.EqualValues(100, ts)
+
+ //upload storage checkpoint
+ key := path.Join(streamhelper.StorageCheckpointOf(taskName), storeID)
+ binary.BigEndian.PutUint64(storageCheckpoint, 90)
+ _, err = metaCli.Put(ctx, key, string(storageCheckpoint))
+ req.NoError(err)
+
+ task := streamhelper.NewTask(&metaCli.MetaDataClient, taskInfo.PBInfo)
+ ts, err = task.GetStorageCheckpoint(ctx)
+ req.NoError(err)
+ req.EqualValues(ts, 90)
+
+ // pause task
+ req.NoError(metaCli.PauseTask(ctx, taskName))
+ resp, err := metaCli.KV.Get(ctx, streamhelper.Pause(taskName))
+ req.NoError(err)
+ req.EqualValues(1, len(resp.Kvs))
+
+ // stop task
+ err = metaCli.DeleteTask(ctx, taskName)
+ req.NoError(err)
+
+ // check task and other meta infomations not existed
+ _, err = metaCli.GetTask(ctx, taskName)
+ req.Error(err)
+
+ ts, err = task.GetStorageCheckpoint(ctx)
+ req.NoError(err)
+ req.EqualValues(ts, 0)
+
+ ts, err = metaCli.GetGlobalCheckpointForTask(ctx, taskName)
+ req.NoError(err)
+ req.EqualValues(0, ts)
+
+ resp, err = metaCli.KV.Get(ctx, streamhelper.Pause(taskName))
+ req.NoError(err)
+ req.EqualValues(0, len(resp.Kvs))
}
diff --git a/br/pkg/streamhelper/models.go b/br/pkg/streamhelper/models.go
index 8aebfbaaf5aa9..7678e655d216a 100644
--- a/br/pkg/streamhelper/models.go
+++ b/br/pkg/streamhelper/models.go
@@ -61,12 +61,6 @@ func RangeKeyOf(name string, startKey []byte) string {
return RangesOf(name) + string(startKey)
}
-func writeUint64(buf *bytes.Buffer, num uint64) {
- items := [8]byte{}
- binary.BigEndian.PutUint64(items[:], num)
- buf.Write(items[:])
-}
-
func encodeUint64(num uint64) []byte {
items := [8]byte{}
binary.BigEndian.PutUint64(items[:], num)
@@ -83,25 +77,17 @@ func CheckPointsOf(task string) string {
}
// GlobalCheckpointOf returns the path to the "global" checkpoint of some task.
+// Normally it would be /checkpoint//central_globa.
func GlobalCheckpointOf(task string) string {
return path.Join(streamKeyPrefix, taskCheckpointPath, task, checkpointTypeGlobal)
}
// StorageCheckpointOf get the prefix path of the `storage checkpoint status` of a task.
+// Normally it would be /storage-checkpoint/.
func StorageCheckpointOf(task string) string {
return path.Join(streamKeyPrefix, storageCheckPoint, task)
}
-// CheckpointOf returns the checkpoint prefix of some store.
-// Normally it would be /checkpoint//.
-func CheckPointOf(task string, store uint64) string {
- buf := bytes.NewBuffer(nil)
- buf.WriteString(strings.TrimSuffix(path.Join(streamKeyPrefix, taskCheckpointPath, task), "/"))
- buf.WriteRune('/')
- writeUint64(buf, store)
- return buf.String()
-}
-
// Pause returns the path for pausing the task.
// Normally it would be /pause/.
func Pause(task string) string {
diff --git a/br/pkg/streamhelper/regioniter.go b/br/pkg/streamhelper/regioniter.go
index 9dc75e38553fc..239c710db1ba4 100644
--- a/br/pkg/streamhelper/regioniter.go
+++ b/br/pkg/streamhelper/regioniter.go
@@ -28,14 +28,22 @@ type RegionWithLeader struct {
Leader *metapb.Peer
}
-type RegionScanner interface {
+type TiKVClusterMeta interface {
// RegionScan gets a list of regions, starts from the region that contains key.
// Limit limits the maximum number of regions returned.
RegionScan(ctx context.Context, key, endKey []byte, limit int) ([]RegionWithLeader, error)
+
+ // Stores returns the store metadata from the cluster.
+ Stores(ctx context.Context) ([]Store, error)
+}
+
+type Store struct {
+ ID uint64
+ BootAt uint64
}
type RegionIter struct {
- cli RegionScanner
+ cli TiKVClusterMeta
startKey, endKey []byte
currentStartKey []byte
// When the endKey become "", we cannot check whether the scan is done by
@@ -57,7 +65,7 @@ func (r *RegionIter) String() string {
}
// IterateRegion creates an iterater over the region range.
-func IterateRegion(cli RegionScanner, startKey, endKey []byte) *RegionIter {
+func IterateRegion(cli TiKVClusterMeta, startKey, endKey []byte) *RegionIter {
return &RegionIter{
cli: cli,
startKey: startKey,
diff --git a/br/pkg/streamhelper/regioniter_test.go b/br/pkg/streamhelper/regioniter_test.go
index 04ccc04da8a66..1c0d6a28ab0fe 100644
--- a/br/pkg/streamhelper/regioniter_test.go
+++ b/br/pkg/streamhelper/regioniter_test.go
@@ -13,8 +13,11 @@ import (
"github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/br/pkg/redact"
"github.com/pingcap/tidb/br/pkg/streamhelper"
+ "github.com/pingcap/tidb/br/pkg/streamhelper/spans"
"github.com/pingcap/tidb/kv"
"github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
)
type constantRegions []streamhelper.RegionWithLeader
@@ -55,7 +58,7 @@ func (c constantRegions) String() string {
func (c constantRegions) RegionScan(ctx context.Context, key []byte, endKey []byte, limit int) ([]streamhelper.RegionWithLeader, error) {
result := make([]streamhelper.RegionWithLeader, 0, limit)
for _, region := range c {
- if overlaps(kv.KeyRange{StartKey: key, EndKey: endKey}, kv.KeyRange{StartKey: region.Region.StartKey, EndKey: region.Region.EndKey}) && len(result) < limit {
+ if spans.Overlaps(kv.KeyRange{StartKey: key, EndKey: endKey}, kv.KeyRange{StartKey: region.Region.StartKey, EndKey: region.Region.EndKey}) && len(result) < limit {
result = append(result, region)
} else if bytes.Compare(region.Region.StartKey, key) > 0 {
break
@@ -66,6 +69,11 @@ func (c constantRegions) RegionScan(ctx context.Context, key []byte, endKey []by
return result, nil
}
+// Stores returns the store metadata from the cluster.
+func (c constantRegions) Stores(ctx context.Context) ([]streamhelper.Store, error) {
+ return nil, status.Error(codes.Unimplemented, "Unsupported operation")
+}
+
func makeSubrangeRegions(keys ...string) constantRegions {
if len(keys) == 0 {
return nil
diff --git a/br/pkg/streamhelper/spans/BUILD.bazel b/br/pkg/streamhelper/spans/BUILD.bazel
new file mode 100644
index 0000000000000..67f9f14546a59
--- /dev/null
+++ b/br/pkg/streamhelper/spans/BUILD.bazel
@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "spans",
+ srcs = [
+ "sorted.go",
+ "utils.go",
+ "value_sorted.go",
+ ],
+ importpath = "github.com/pingcap/tidb/br/pkg/streamhelper/spans",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//br/pkg/logutil",
+ "//br/pkg/utils",
+ "//kv",
+ "@com_github_google_btree//:btree",
+ ],
+)
+
+go_test(
+ name = "spans_test",
+ timeout = "short",
+ srcs = [
+ "sorted_test.go",
+ "utils_test.go",
+ "value_sorted_test.go",
+ ],
+ flaky = True,
+ deps = [
+ ":spans",
+ "@com_github_stretchr_testify//require",
+ ],
+)
diff --git a/br/pkg/streamhelper/spans/sorted.go b/br/pkg/streamhelper/spans/sorted.go
new file mode 100644
index 0000000000000..a15138bf8124c
--- /dev/null
+++ b/br/pkg/streamhelper/spans/sorted.go
@@ -0,0 +1,186 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+
+package spans
+
+import (
+ "bytes"
+ "fmt"
+
+ "github.com/google/btree"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/kv"
+)
+
+// Value is the value type of stored in the span tree.
+type Value = uint64
+
+// join finds the upper bound of two values.
+func join(a, b Value) Value {
+ if a > b {
+ return a
+ }
+ return b
+}
+
+// Span is the type of an adjacent sub key space.
+type Span = kv.KeyRange
+
+// Valued is span binding to a value, which is the entry type of span tree.
+type Valued struct {
+ Key Span
+ Value Value
+}
+
+func (r Valued) String() string {
+ return fmt.Sprintf("(%s, %d)", logutil.StringifyRange(r.Key), r.Value)
+}
+
+func (r Valued) Less(other btree.Item) bool {
+ return bytes.Compare(r.Key.StartKey, other.(Valued).Key.StartKey) < 0
+}
+
+// ValuedFull represents a set of valued ranges, which doesn't overlap and union of them all is the full key space.
+type ValuedFull struct {
+ inner *btree.BTree
+}
+
+// NewFullWith creates a set of a subset of spans.
+func NewFullWith(initSpans []Span, init Value) *ValuedFull {
+ t := btree.New(16)
+ for _, r := range Collapse(len(initSpans), func(i int) Span { return initSpans[i] }) {
+ t.ReplaceOrInsert(Valued{Value: init, Key: r})
+ }
+ return &ValuedFull{inner: t}
+}
+
+// Merge merges a new interval into the span set. The value of overlapped
+// part with other spans would be "merged" by the `join` function.
+// An example:
+/*
+|___________________________________________________________________________|
+^-----------------^-----------------^-----------------^---------------------^
+| c = 42 | c = 43 | c = 45 | c = 41 |
+ ^--------------------------^
+ merge(| c = 44 |)
+Would Give:
+|___________________________________________________________________________|
+^-----------------^----^------------^-------------^---^---------------------^
+| c = 42 | 43 | c = 44 | c = 45 | c = 41 |
+ |-------------|
+ Unchanged, because 44 < 45.
+*/
+func (f *ValuedFull) Merge(val Valued) {
+ overlaps := make([]Valued, 0, 16)
+ f.overlapped(val.Key, &overlaps)
+ f.mergeWithOverlap(val, overlaps, nil)
+}
+
+// Traverse traverses all ranges by order.
+func (f *ValuedFull) Traverse(m func(Valued) bool) {
+ f.inner.Ascend(func(item btree.Item) bool {
+ return m(item.(Valued))
+ })
+}
+
+func (f *ValuedFull) mergeWithOverlap(val Valued, overlapped []Valued, newItems *[]Valued) {
+ // There isn't any range overlaps with the input range, perhaps the input range is empty.
+ // do nothing for this case.
+ if len(overlapped) == 0 {
+ return
+ }
+
+ for _, r := range overlapped {
+ f.inner.Delete(r)
+ // Assert All overlapped ranges are deleted.
+ }
+
+ var (
+ initialized = false
+ collected Valued
+ rightTrail *Valued
+ flushCollected = func() {
+ if initialized {
+ f.inner.ReplaceOrInsert(collected)
+ if newItems != nil {
+ *newItems = append(*newItems, collected)
+ }
+ }
+ }
+ emitToCollected = func(rng Valued, standalone bool) {
+ merged := rng.Value
+ if !standalone {
+ merged = join(val.Value, rng.Value)
+ }
+ if !initialized {
+ collected = rng
+ collected.Value = merged
+ initialized = true
+ return
+ }
+ if merged == collected.Value && utils.CompareBytesExt(collected.Key.EndKey, true, rng.Key.StartKey, false) == 0 {
+ collected.Key.EndKey = rng.Key.EndKey
+ } else {
+ flushCollected()
+ collected = Valued{
+ Key: rng.Key,
+ Value: merged,
+ }
+ }
+ }
+ )
+
+ leftmost := overlapped[0]
+ if bytes.Compare(leftmost.Key.StartKey, val.Key.StartKey) < 0 {
+ emitToCollected(Valued{
+ Key: Span{StartKey: leftmost.Key.StartKey, EndKey: val.Key.StartKey},
+ Value: leftmost.Value,
+ }, true)
+ overlapped[0].Key.StartKey = val.Key.StartKey
+ }
+
+ rightmost := overlapped[len(overlapped)-1]
+ if utils.CompareBytesExt(rightmost.Key.EndKey, true, val.Key.EndKey, true) > 0 {
+ rightTrail = &Valued{
+ Key: Span{StartKey: val.Key.EndKey, EndKey: rightmost.Key.EndKey},
+ Value: rightmost.Value,
+ }
+ overlapped[len(overlapped)-1].Key.EndKey = val.Key.EndKey
+ }
+
+ for _, rng := range overlapped {
+ emitToCollected(rng, false)
+ }
+
+ if rightTrail != nil {
+ emitToCollected(*rightTrail, true)
+ }
+
+ flushCollected()
+}
+
+// overlapped inserts the overlapped ranges of the span into the `result` slice.
+func (f *ValuedFull) overlapped(k Span, result *[]Valued) {
+ var (
+ first Span
+ hasFirst bool
+ )
+ // Firstly, let's find whether there is a overlapped region with less start key.
+ f.inner.DescendLessOrEqual(Valued{Key: k}, func(item btree.Item) bool {
+ first = item.(Valued).Key
+ hasFirst = true
+ return false
+ })
+ if !hasFirst || !Overlaps(first, k) {
+ first = k
+ }
+
+ f.inner.AscendGreaterOrEqual(Valued{Key: first}, func(item btree.Item) bool {
+ r := item.(Valued)
+ if !Overlaps(r.Key, k) {
+ return false
+ }
+ *result = append(*result, r)
+ return true
+ })
+}
diff --git a/br/pkg/streamhelper/spans/sorted_test.go b/br/pkg/streamhelper/spans/sorted_test.go
new file mode 100644
index 0000000000000..c56c2236a6690
--- /dev/null
+++ b/br/pkg/streamhelper/spans/sorted_test.go
@@ -0,0 +1,211 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+
+package spans_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/pingcap/tidb/br/pkg/streamhelper/spans"
+ "github.com/stretchr/testify/require"
+)
+
+func s(a, b string) spans.Span {
+ return spans.Span{
+ StartKey: []byte(a),
+ EndKey: []byte(b),
+ }
+}
+
+func kv(s spans.Span, v spans.Value) spans.Valued {
+ return spans.Valued{
+ Key: s,
+ Value: v,
+ }
+}
+
+func TestBasic(t *testing.T) {
+ type Case struct {
+ InputSequence []spans.Valued
+ Result []spans.Valued
+ }
+
+ run := func(t *testing.T, c Case) {
+ full := spans.NewFullWith(spans.Full(), 0)
+ fmt.Println(t.Name())
+ for _, i := range c.InputSequence {
+ full.Merge(i)
+ var result []spans.Valued
+ full.Traverse(func(v spans.Valued) bool {
+ result = append(result, v)
+ return true
+ })
+ fmt.Printf("%s -> %s\n", i, result)
+ }
+
+ var result []spans.Valued
+ full.Traverse(func(v spans.Valued) bool {
+ result = append(result, v)
+ return true
+ })
+
+ require.True(t, spans.ValuedSetEquals(result, c.Result), "%s\nvs\n%s", result, c.Result)
+ }
+
+ cases := []Case{
+ {
+ InputSequence: []spans.Valued{
+ kv(s("0001", "0002"), 1),
+ kv(s("0002", "0003"), 2),
+ },
+ Result: []spans.Valued{
+ kv(s("", "0001"), 0),
+ kv(s("0001", "0002"), 1),
+ kv(s("0002", "0003"), 2),
+ kv(s("0003", ""), 0),
+ },
+ },
+ {
+ InputSequence: []spans.Valued{
+ kv(s("0001", "0002"), 1),
+ kv(s("0002", "0003"), 2),
+ kv(s("0001", "0003"), 4),
+ },
+ Result: []spans.Valued{
+ kv(s("", "0001"), 0),
+ kv(s("0001", "0003"), 4),
+ kv(s("0003", ""), 0),
+ },
+ },
+ {
+ InputSequence: []spans.Valued{
+ kv(s("0001", "0004"), 3),
+ kv(s("0004", "0008"), 5),
+ kv(s("0001", "0007"), 4),
+ kv(s("", "0002"), 2),
+ },
+ Result: []spans.Valued{
+ kv(s("", "0001"), 2),
+ kv(s("0001", "0004"), 4),
+ kv(s("0004", "0008"), 5),
+ kv(s("0008", ""), 0),
+ },
+ },
+ {
+ InputSequence: []spans.Valued{
+ kv(s("0001", "0004"), 3),
+ kv(s("0004", "0008"), 5),
+ kv(s("0001", "0009"), 4),
+ },
+ Result: []spans.Valued{
+ kv(s("", "0001"), 0),
+ kv(s("0001", "0004"), 4),
+ kv(s("0004", "0008"), 5),
+ kv(s("0008", "0009"), 4),
+ kv(s("0009", ""), 0),
+ },
+ },
+ }
+
+ for i, c := range cases {
+ t.Run(fmt.Sprintf("#%d", i+1), func(t *testing.T) { run(t, c) })
+ }
+}
+
+func TestSubRange(t *testing.T) {
+ type Case struct {
+ Range []spans.Span
+ InputSequence []spans.Valued
+ Result []spans.Valued
+ }
+
+ run := func(t *testing.T, c Case) {
+ full := spans.NewFullWith(c.Range, 0)
+ fmt.Println(t.Name())
+ for _, i := range c.InputSequence {
+ full.Merge(i)
+ var result []spans.Valued
+ full.Traverse(func(v spans.Valued) bool {
+ result = append(result, v)
+ return true
+ })
+ fmt.Printf("%s -> %s\n", i, result)
+ }
+
+ var result []spans.Valued
+ full.Traverse(func(v spans.Valued) bool {
+ result = append(result, v)
+ return true
+ })
+
+ require.True(t, spans.ValuedSetEquals(result, c.Result), "%s\nvs\n%s", result, c.Result)
+ }
+
+ cases := []Case{
+ {
+ Range: []spans.Span{s("0001", "0004"), s("0008", "")},
+ InputSequence: []spans.Valued{
+ kv(s("0001", "0007"), 42),
+ kv(s("0000", "0009"), 41),
+ kv(s("0002", "0005"), 43),
+ },
+ Result: []spans.Valued{
+ kv(s("0001", "0002"), 42),
+ kv(s("0002", "0004"), 43),
+ kv(s("0008", "0009"), 41),
+ kv(s("0009", ""), 0),
+ },
+ },
+ {
+ Range: []spans.Span{
+ s("0001", "0004"),
+ s("0008", "")},
+ InputSequence: []spans.Valued{kv(s("", ""), 42)},
+ Result: []spans.Valued{
+ kv(s("0001", "0004"), 42),
+ kv(s("0008", ""), 42),
+ },
+ },
+ {
+ Range: []spans.Span{
+ s("0001", "0004"),
+ s("0005", "0008"),
+ },
+ InputSequence: []spans.Valued{
+ kv(s("0001", "0002"), 42),
+ kv(s("0002", "0008"), 43),
+ kv(s("0004", "0007"), 45),
+ kv(s("0000", "00015"), 48),
+ },
+ Result: []spans.Valued{
+ kv(s("0001", "00015"), 48),
+ kv(s("00015", "0002"), 42),
+ kv(s("0002", "0004"), 43),
+ kv(s("0005", "0007"), 45),
+ kv(s("0007", "0008"), 43),
+ },
+ },
+ {
+ Range: []spans.Span{
+ s("0001", "0004"),
+ s("0005", "0008"),
+ },
+ InputSequence: []spans.Valued{
+ kv(s("0004", "0008"), 32),
+ kv(s("00041", "0007"), 33),
+ kv(s("0004", "00041"), 99999),
+ kv(s("0005", "0006"), 34),
+ },
+ Result: []spans.Valued{
+ kv(s("0001", "0004"), 0),
+ kv(s("0005", "0006"), 34),
+ kv(s("0006", "0007"), 33),
+ kv(s("0007", "0008"), 32),
+ },
+ },
+ }
+
+ for i, c := range cases {
+ t.Run(fmt.Sprintf("#%d", i+1), func(t *testing.T) { run(t, c) })
+ }
+}
diff --git a/br/pkg/streamhelper/spans/utils.go b/br/pkg/streamhelper/spans/utils.go
new file mode 100644
index 0000000000000..621173983185d
--- /dev/null
+++ b/br/pkg/streamhelper/spans/utils.go
@@ -0,0 +1,150 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+
+package spans
+
+import (
+ "bytes"
+ "fmt"
+ "math"
+ "sort"
+
+ "github.com/pingcap/tidb/br/pkg/utils"
+)
+
+// Overlaps checks whether two spans have overlapped part.
+func Overlaps(a, b Span) bool {
+ if len(b.EndKey) == 0 {
+ return len(a.EndKey) == 0 || bytes.Compare(a.EndKey, b.StartKey) > 0
+ }
+ if len(a.EndKey) == 0 {
+ return len(b.EndKey) == 0 || bytes.Compare(b.EndKey, a.StartKey) > 0
+ }
+ return bytes.Compare(a.StartKey, b.EndKey) < 0 && bytes.Compare(b.StartKey, a.EndKey) < 0
+}
+
+func Debug(full *ValueSortedFull) {
+ var result []Valued
+ full.Traverse(func(v Valued) bool {
+ result = append(result, v)
+ return true
+ })
+ var idx []Valued
+ full.TraverseValuesLessThan(math.MaxUint64, func(v Valued) bool {
+ idx = append(idx, v)
+ return true
+ })
+ fmt.Printf("%s\n\tidx = %s\n", result, idx)
+}
+
+// Collapse collapse ranges overlapping or adjacent.
+// Example:
+// Collapse({[1, 4], [2, 8], [3, 9]}) == {[1, 9]}
+// Collapse({[1, 3], [4, 7], [2, 3]}) == {[1, 3], [4, 7]}
+func Collapse(length int, getRange func(int) Span) []Span {
+ frs := make([]Span, 0, length)
+ for i := 0; i < length; i++ {
+ frs = append(frs, getRange(i))
+ }
+
+ sort.Slice(frs, func(i, j int) bool {
+ start := bytes.Compare(frs[i].StartKey, frs[j].StartKey)
+ if start != 0 {
+ return start < 0
+ }
+ return utils.CompareBytesExt(frs[i].EndKey, true, frs[j].EndKey, true) < 0
+ })
+
+ result := make([]Span, 0, len(frs))
+ i := 0
+ for i < len(frs) {
+ item := frs[i]
+ for {
+ i++
+ if i >= len(frs) || (len(item.EndKey) != 0 && bytes.Compare(frs[i].StartKey, item.EndKey) > 0) {
+ break
+ }
+ if len(item.EndKey) != 0 && bytes.Compare(item.EndKey, frs[i].EndKey) < 0 || len(frs[i].EndKey) == 0 {
+ item.EndKey = frs[i].EndKey
+ }
+ }
+ result = append(result, item)
+ }
+ return result
+}
+
+// Full returns a full span crossing the key space.
+func Full() []Span {
+ return []Span{{}}
+}
+
+func (x Valued) Equals(y Valued) bool {
+ return x.Value == y.Value && bytes.Equal(x.Key.StartKey, y.Key.StartKey) && bytes.Equal(x.Key.EndKey, y.Key.EndKey)
+}
+
+func ValuedSetEquals(xs, ys []Valued) bool {
+ if len(xs) == 0 || len(ys) == 0 {
+ return len(ys) == len(xs)
+ }
+
+ sort.Slice(xs, func(i, j int) bool {
+ start := bytes.Compare(xs[i].Key.StartKey, xs[j].Key.StartKey)
+ if start != 0 {
+ return start < 0
+ }
+ return utils.CompareBytesExt(xs[i].Key.EndKey, true, xs[j].Key.EndKey, true) < 0
+ })
+ sort.Slice(ys, func(i, j int) bool {
+ start := bytes.Compare(ys[i].Key.StartKey, ys[j].Key.StartKey)
+ if start != 0 {
+ return start < 0
+ }
+ return utils.CompareBytesExt(ys[i].Key.EndKey, true, ys[j].Key.EndKey, true) < 0
+ })
+
+ xi := 0
+ yi := 0
+
+ for {
+ if xi >= len(xs) || yi >= len(ys) {
+ return (xi >= len(xs)) == (yi >= len(ys))
+ }
+ x := xs[xi]
+ y := ys[yi]
+
+ if !bytes.Equal(x.Key.StartKey, y.Key.StartKey) {
+ return false
+ }
+
+ for {
+ if xi >= len(xs) || yi >= len(ys) {
+ return (xi >= len(xs)) == (yi >= len(ys))
+ }
+ x := xs[xi]
+ y := ys[yi]
+
+ if x.Value != y.Value {
+ return false
+ }
+
+ c := utils.CompareBytesExt(x.Key.EndKey, true, y.Key.EndKey, true)
+ if c == 0 {
+ xi++
+ yi++
+ break
+ }
+ if c < 0 {
+ xi++
+ // If not adjacent key, return false directly.
+ if xi < len(xs) && utils.CompareBytesExt(x.Key.EndKey, true, xs[xi].Key.StartKey, false) != 0 {
+ return false
+ }
+ }
+ if c > 0 {
+ yi++
+ if yi < len(ys) && utils.CompareBytesExt(y.Key.EndKey, true, ys[yi].Key.StartKey, false) != 0 {
+ return false
+ }
+ }
+ }
+ }
+}
diff --git a/br/pkg/streamhelper/spans/utils_test.go b/br/pkg/streamhelper/spans/utils_test.go
new file mode 100644
index 0000000000000..48b8fc7f411a5
--- /dev/null
+++ b/br/pkg/streamhelper/spans/utils_test.go
@@ -0,0 +1,83 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+
+package spans_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/pingcap/tidb/br/pkg/streamhelper/spans"
+ "github.com/stretchr/testify/require"
+)
+
+func TestValuedEquals(t *testing.T) {
+ s := func(start, end string, val spans.Value) spans.Valued {
+ return spans.Valued{
+ Key: spans.Span{
+ StartKey: []byte(start),
+ EndKey: []byte(end),
+ },
+ Value: val,
+ }
+ }
+ type Case struct {
+ inputA []spans.Valued
+ inputB []spans.Valued
+ required bool
+ }
+ cases := []Case{
+ {
+ inputA: []spans.Valued{s("0001", "0002", 3)},
+ inputB: []spans.Valued{s("0001", "0003", 3)},
+ required: false,
+ },
+ {
+ inputA: []spans.Valued{s("0001", "0002", 3)},
+ inputB: []spans.Valued{s("0001", "0002", 3)},
+ required: true,
+ },
+ {
+ inputA: []spans.Valued{s("0001", "0003", 3)},
+ inputB: []spans.Valued{s("0001", "0002", 3), s("0002", "0003", 3)},
+ required: true,
+ },
+ {
+ inputA: []spans.Valued{s("0001", "0003", 4)},
+ inputB: []spans.Valued{s("0001", "0002", 3), s("0002", "0003", 3)},
+ required: false,
+ },
+ {
+ inputA: []spans.Valued{s("0001", "0003", 3)},
+ inputB: []spans.Valued{s("0001", "0002", 4), s("0002", "0003", 3)},
+ required: false,
+ },
+ {
+ inputA: []spans.Valued{s("0001", "0003", 3)},
+ inputB: []spans.Valued{s("0001", "0002", 3), s("0002", "0004", 3)},
+ required: false,
+ },
+ {
+ inputA: []spans.Valued{s("", "0003", 3)},
+ inputB: []spans.Valued{s("0001", "0002", 3), s("0002", "0003", 3)},
+ required: false,
+ },
+ {
+ inputA: []spans.Valued{s("0001", "", 1)},
+ inputB: []spans.Valued{s("0001", "0003", 1), s("0004", "", 1)},
+ required: false,
+ },
+ {
+ inputA: []spans.Valued{s("0001", "0004", 1), s("0001", "0002", 1)},
+ inputB: []spans.Valued{s("0001", "0002", 1), s("0001", "0004", 1)},
+ required: true,
+ },
+ }
+ run := func(t *testing.T, c Case) {
+ require.Equal(t, c.required, spans.ValuedSetEquals(c.inputA, c.inputB))
+ require.Equal(t, c.required, spans.ValuedSetEquals(c.inputB, c.inputA))
+ }
+
+ for i, c := range cases {
+ t.Run(fmt.Sprintf("#%d", i+1), func(t *testing.T) { run(t, c) })
+ }
+}
diff --git a/br/pkg/streamhelper/spans/value_sorted.go b/br/pkg/streamhelper/spans/value_sorted.go
new file mode 100644
index 0000000000000..2fc1ff2cdbbbc
--- /dev/null
+++ b/br/pkg/streamhelper/spans/value_sorted.go
@@ -0,0 +1,69 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+
+package spans
+
+import "github.com/google/btree"
+
+type sortedByValueThenStartKey Valued
+
+func (s sortedByValueThenStartKey) Less(o btree.Item) bool {
+ other := o.(sortedByValueThenStartKey)
+ if s.Value != other.Value {
+ return s.Value < other.Value
+ }
+ return Valued(s).Less(Valued(other))
+}
+
+// ValueSortedFull is almost the same as `Valued`, however it added an
+// extra index hence enabled query range by theirs value.
+type ValueSortedFull struct {
+ *ValuedFull
+ valueIdx *btree.BTree
+}
+
+// Sorted takes the ownership of a raw `ValuedFull` and then wrap it with `ValueSorted`.
+func Sorted(f *ValuedFull) *ValueSortedFull {
+ vf := &ValueSortedFull{
+ ValuedFull: f,
+ valueIdx: btree.New(16),
+ }
+ f.Traverse(func(v Valued) bool {
+ vf.valueIdx.ReplaceOrInsert(sortedByValueThenStartKey(v))
+ return true
+ })
+ return vf
+}
+
+func (v *ValueSortedFull) Merge(newItem Valued) {
+ v.MergeAll([]Valued{newItem})
+}
+
+func (v *ValueSortedFull) MergeAll(newItems []Valued) {
+ var overlapped []Valued
+ var inserted []Valued
+
+ for _, item := range newItems {
+ overlapped = overlapped[:0]
+ inserted = inserted[:0]
+
+ v.overlapped(item.Key, &overlapped)
+ v.mergeWithOverlap(item, overlapped, &inserted)
+
+ for _, o := range overlapped {
+ v.valueIdx.Delete(sortedByValueThenStartKey(o))
+ }
+ for _, i := range inserted {
+ v.valueIdx.ReplaceOrInsert(sortedByValueThenStartKey(i))
+ }
+ }
+}
+
+func (v *ValueSortedFull) TraverseValuesLessThan(n Value, action func(Valued) bool) {
+ v.valueIdx.AscendLessThan(sortedByValueThenStartKey{Value: n}, func(item btree.Item) bool {
+ return action(Valued(item.(sortedByValueThenStartKey)))
+ })
+}
+
+func (v *ValueSortedFull) MinValue() Value {
+ return v.valueIdx.Min().(sortedByValueThenStartKey).Value
+}
diff --git a/br/pkg/streamhelper/spans/value_sorted_test.go b/br/pkg/streamhelper/spans/value_sorted_test.go
new file mode 100644
index 0000000000000..ee1a5a8af6500
--- /dev/null
+++ b/br/pkg/streamhelper/spans/value_sorted_test.go
@@ -0,0 +1,98 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+
+package spans_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/pingcap/tidb/br/pkg/streamhelper/spans"
+ "github.com/stretchr/testify/require"
+)
+
+func TestSortedBasic(t *testing.T) {
+ type Case struct {
+ InputSequence []spans.Valued
+ RetainLessThan spans.Value
+ Result []spans.Valued
+ }
+
+ run := func(t *testing.T, c Case) {
+ full := spans.Sorted(spans.NewFullWith(spans.Full(), 0))
+ fmt.Println(t.Name())
+ for _, i := range c.InputSequence {
+ full.Merge(i)
+ spans.Debug(full)
+ }
+
+ var result []spans.Valued
+ full.TraverseValuesLessThan(c.RetainLessThan, func(v spans.Valued) bool {
+ result = append(result, v)
+ return true
+ })
+
+ require.True(t, spans.ValuedSetEquals(result, c.Result), "%s\nvs\n%s", result, c.Result)
+ }
+
+ cases := []Case{
+ {
+ InputSequence: []spans.Valued{
+ kv(s("0001", "0002"), 1),
+ kv(s("0002", "0003"), 2),
+ },
+ Result: []spans.Valued{
+ kv(s("", "0001"), 0),
+ kv(s("0001", "0002"), 1),
+ kv(s("0002", "0003"), 2),
+ kv(s("0003", ""), 0),
+ },
+ RetainLessThan: 10,
+ },
+ {
+ InputSequence: []spans.Valued{
+ kv(s("0001", "0002"), 1),
+ kv(s("0002", "0003"), 2),
+ kv(s("0001", "0003"), 4),
+ },
+ RetainLessThan: 1,
+ Result: []spans.Valued{
+ kv(s("", "0001"), 0),
+ kv(s("0003", ""), 0),
+ },
+ },
+ {
+ InputSequence: []spans.Valued{
+ kv(s("0001", "0004"), 3),
+ kv(s("0004", "0008"), 5),
+ kv(s("0001", "0007"), 4),
+ kv(s("", "0002"), 2),
+ },
+ RetainLessThan: 5,
+ Result: []spans.Valued{
+ kv(s("", "0001"), 2),
+ kv(s("0001", "0004"), 4),
+ kv(s("0008", ""), 0),
+ },
+ },
+ {
+ InputSequence: []spans.Valued{
+ kv(s("0001", "0004"), 3),
+ kv(s("0004", "0008"), 5),
+ kv(s("0001", "0007"), 4),
+ kv(s("", "0002"), 2),
+ kv(s("0001", "0004"), 5),
+ kv(s("0008", ""), 10),
+ kv(s("", "0001"), 20),
+ },
+ RetainLessThan: 11,
+ Result: []spans.Valued{
+ kv(s("0001", "0008"), 5),
+ kv(s("0008", ""), 10),
+ },
+ },
+ }
+
+ for i, c := range cases {
+ t.Run(fmt.Sprintf("#%d", i+1), func(t *testing.T) { run(t, c) })
+ }
+}
diff --git a/br/pkg/streamhelper/subscription_test.go b/br/pkg/streamhelper/subscription_test.go
new file mode 100644
index 0000000000000..2341cb05dc01e
--- /dev/null
+++ b/br/pkg/streamhelper/subscription_test.go
@@ -0,0 +1,226 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+
+package streamhelper_test
+
+import (
+ "context"
+ "fmt"
+ "sync"
+ "testing"
+
+ "github.com/pingcap/tidb/br/pkg/streamhelper"
+ "github.com/pingcap/tidb/br/pkg/streamhelper/spans"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+)
+
+func installSubscribeSupport(c *fakeCluster) {
+ for _, s := range c.stores {
+ s.SetSupportFlushSub(true)
+ }
+}
+
+func installSubscribeSupportForRandomN(c *fakeCluster, n int) {
+ i := 0
+ for _, s := range c.stores {
+ if i == n {
+ break
+ }
+ s.SetSupportFlushSub(true)
+ i++
+ }
+}
+
+func TestSubBasic(t *testing.T) {
+ req := require.New(t)
+ ctx := context.Background()
+ c := createFakeCluster(t, 4, true)
+ c.splitAndScatter("0001", "0002", "0003", "0008", "0009")
+ installSubscribeSupport(c)
+ sub := streamhelper.NewSubscriber(c, c)
+ req.NoError(sub.UpdateStoreTopology(ctx))
+ var cp uint64
+ for i := 0; i < 10; i++ {
+ cp = c.advanceCheckpoints()
+ c.flushAll()
+ }
+ sub.HandleErrors(ctx)
+ req.NoError(sub.PendingErrors())
+ sub.Drop()
+ s := spans.Sorted(spans.NewFullWith(spans.Full(), 1))
+ for k := range sub.Events() {
+ s.Merge(k)
+ }
+ defer func() {
+ if t.Failed() {
+ fmt.Println(c)
+ spans.Debug(s)
+ }
+ }()
+
+ req.Equal(cp, s.MinValue(), "%d vs %d", cp, s.MinValue())
+}
+
+func TestNormalError(t *testing.T) {
+ req := require.New(t)
+ ctx := context.Background()
+ c := createFakeCluster(t, 4, true)
+ c.splitAndScatter("0001", "0002", "0003", "0008", "0009")
+ installSubscribeSupport(c)
+
+ sub := streamhelper.NewSubscriber(c, c)
+ c.onGetClient = oneStoreFailure()
+ req.NoError(sub.UpdateStoreTopology(ctx))
+ c.onGetClient = nil
+ req.Error(sub.PendingErrors())
+ sub.HandleErrors(ctx)
+ req.NoError(sub.PendingErrors())
+ var cp uint64
+ for i := 0; i < 10; i++ {
+ cp = c.advanceCheckpoints()
+ c.flushAll()
+ }
+ sub.Drop()
+ s := spans.Sorted(spans.NewFullWith(spans.Full(), 1))
+ for k := range sub.Events() {
+ s.Merge(k)
+ }
+ req.Equal(cp, s.MinValue(), "%d vs %d", cp, s.MinValue())
+}
+
+func TestHasFailureStores(t *testing.T) {
+ req := require.New(t)
+ ctx := context.Background()
+ c := createFakeCluster(t, 4, true)
+ c.splitAndScatter("0001", "0002", "0003", "0008", "0009")
+
+ installSubscribeSupportForRandomN(c, 3)
+ sub := streamhelper.NewSubscriber(c, c)
+ req.NoError(sub.UpdateStoreTopology(ctx))
+ sub.HandleErrors(ctx)
+ req.Error(sub.PendingErrors())
+
+ installSubscribeSupport(c)
+ req.NoError(sub.UpdateStoreTopology(ctx))
+ sub.HandleErrors(ctx)
+ req.NoError(sub.PendingErrors())
+}
+
+func TestStoreOffline(t *testing.T) {
+ req := require.New(t)
+ ctx := context.Background()
+ c := createFakeCluster(t, 4, true)
+ c.splitAndScatter("0001", "0002", "0003", "0008", "0009")
+ installSubscribeSupport(c)
+
+ c.onGetClient = func(u uint64) error {
+ return status.Error(codes.DataLoss, "upon an eclipsed night, some of data (not all data) have fled from the dataset")
+ }
+ sub := streamhelper.NewSubscriber(c, c)
+ req.NoError(sub.UpdateStoreTopology(ctx))
+ req.Error(sub.PendingErrors())
+
+ c.onGetClient = nil
+ sub.HandleErrors(ctx)
+ req.NoError(sub.PendingErrors())
+}
+
+func TestStoreRemoved(t *testing.T) {
+ req := require.New(t)
+ ctx := context.Background()
+ c := createFakeCluster(t, 4, true)
+ c.splitAndScatter("0001", "0002", "0003", "0008", "0009", "0010", "0100", "0956", "1000")
+
+ installSubscribeSupport(c)
+ sub := streamhelper.NewSubscriber(c, c)
+ req.NoError(sub.UpdateStoreTopology(ctx))
+
+ var cp uint64
+ for i := 0; i < 10; i++ {
+ cp = c.advanceCheckpoints()
+ c.flushAll()
+ }
+ sub.HandleErrors(ctx)
+ req.NoError(sub.PendingErrors())
+ for _, s := range c.stores {
+ c.removeStore(s.id)
+ break
+ }
+ req.NoError(sub.UpdateStoreTopology(ctx))
+ for i := 0; i < 10; i++ {
+ cp = c.advanceCheckpoints()
+ c.flushAll()
+ }
+ sub.HandleErrors(ctx)
+ req.NoError(sub.PendingErrors())
+
+ sub.Drop()
+ s := spans.Sorted(spans.NewFullWith(spans.Full(), 1))
+ for k := range sub.Events() {
+ s.Merge(k)
+ }
+
+ defer func() {
+ if t.Failed() {
+ fmt.Println(c)
+ spans.Debug(s)
+ }
+ }()
+
+ req.Equal(cp, s.MinValue(), "cp = %d, s = %d", cp, s.MinValue())
+}
+
+func TestSomeOfStoreUnsupported(t *testing.T) {
+ req := require.New(t)
+ ctx := context.Background()
+ c := createFakeCluster(t, 4, true)
+ c.splitAndScatter("0001", "0002", "0003", "0008", "0009", "0010", "0100", "0956", "1000")
+
+ sub := streamhelper.NewSubscriber(c, c)
+ installSubscribeSupportForRandomN(c, 3)
+ req.NoError(sub.UpdateStoreTopology(ctx))
+
+ var cp uint64
+ for i := 0; i < 10; i++ {
+ cp = c.advanceCheckpoints()
+ c.flushAll()
+ }
+ s := spans.Sorted(spans.NewFullWith(spans.Full(), 1))
+ m := new(sync.Mutex)
+ sub.Drop()
+ for k := range sub.Events() {
+ s.Merge(k)
+ }
+
+ rngs := make([]spans.Span, 0)
+ s.TraverseValuesLessThan(cp, func(v spans.Valued) bool {
+ rngs = append(rngs, v.Key)
+ return true
+ })
+ coll := streamhelper.NewClusterCollector(ctx, c)
+ coll.SetOnSuccessHook(func(u uint64, kr spans.Span) {
+ m.Lock()
+ defer m.Unlock()
+ s.Merge(spans.Valued{Key: kr, Value: u})
+ })
+ ld := uint64(0)
+ for _, rng := range rngs {
+ iter := streamhelper.IterateRegion(c, rng.StartKey, rng.EndKey)
+ for !iter.Done() {
+ rs, err := iter.Next(ctx)
+ req.NoError(err)
+ for _, r := range rs {
+ if ld == 0 {
+ ld = r.Leader.StoreId
+ } else {
+ req.Equal(r.Leader.StoreId, ld, "the leader is from different store: some of events not pushed")
+ }
+ coll.CollectRegion(r)
+ }
+ }
+ }
+ _, err := coll.Finish(ctx)
+ req.NoError(err)
+ req.Equal(cp, s.MinValue())
+}
diff --git a/br/pkg/streamhelper/tsheap.go b/br/pkg/streamhelper/tsheap.go
deleted file mode 100644
index 6c2fb510776e7..0000000000000
--- a/br/pkg/streamhelper/tsheap.go
+++ /dev/null
@@ -1,326 +0,0 @@
-// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
-
-package streamhelper
-
-import (
- "encoding/hex"
- "fmt"
- "strings"
- "sync"
- "time"
-
- "github.com/google/btree"
- "github.com/pingcap/errors"
- berrors "github.com/pingcap/tidb/br/pkg/errors"
- "github.com/pingcap/tidb/br/pkg/logutil"
- "github.com/pingcap/tidb/br/pkg/redact"
- "github.com/pingcap/tidb/br/pkg/utils"
- "github.com/pingcap/tidb/kv"
- "github.com/tikv/client-go/v2/oracle"
- "go.uber.org/zap/zapcore"
-)
-
-// CheckpointsCache is the heap-like cache for checkpoints.
-//
-// "Checkpoint" is the "Resolved TS" of some range.
-// A resolved ts is a "watermark" for the system, which:
-// - implies there won't be any transactions (in some range) commit with `commit_ts` smaller than this TS.
-// - is monotonic increasing.
-// A "checkpoint" is a "safe" Resolved TS, which:
-// - is a TS *less than* the real resolved ts of now.
-// - is based on range (it only promises there won't be new committed txns in the range).
-// - the checkpoint of union of ranges is the minimal checkpoint of all ranges.
-// As an example:
-/*
- +----------------------------------+
- ^-----------^ (Checkpoint = 42)
- ^---------------^ (Checkpoint = 76)
- ^-----------------------^ (Checkpoint = min(42, 76) = 42)
-*/
-// For calculating the global checkpoint, we can make a heap-like structure:
-// Checkpoint Ranges
-// 42 -> {[0, 8], [16, 100]}
-// 1002 -> {[8, 16]}
-// 1082 -> {[100, inf]}
-// For now, the checkpoint of range [8, 16] and [100, inf] won't affect the global checkpoint
-// directly, so we can try to advance only the ranges of {[0, 8], [16, 100]} (which's checkpoint is steal).
-// Once them get advance, the global checkpoint would be advanced then,
-// and we don't need to update all ranges (because some new ranges don't need to be advanced so quickly.)
-type CheckpointsCache interface {
- fmt.Stringer
- // InsertRange inserts a range with specified TS to the cache.
- InsertRange(ts uint64, rng kv.KeyRange)
- // InsertRanges inserts a set of ranges that sharing checkpoint to the cache.
- InsertRanges(rst RangesSharesTS)
- // CheckpointTS returns the now global (union of all ranges) checkpoint of the cache.
- CheckpointTS() uint64
- // PopRangesWithGapGT pops the ranges which's checkpoint is
- PopRangesWithGapGT(d time.Duration) []*RangesSharesTS
- // Check whether the ranges in the cache is integrate.
- ConsistencyCheck(ranges []kv.KeyRange) error
- // Clear the cache.
- Clear()
-}
-
-// NoOPCheckpointCache is used when cache disabled.
-type NoOPCheckpointCache struct{}
-
-func (NoOPCheckpointCache) InsertRange(ts uint64, rng kv.KeyRange) {}
-
-func (NoOPCheckpointCache) InsertRanges(rst RangesSharesTS) {}
-
-func (NoOPCheckpointCache) Clear() {}
-
-func (NoOPCheckpointCache) String() string {
- return "NoOPCheckpointCache"
-}
-
-func (NoOPCheckpointCache) CheckpointTS() uint64 {
- panic("invalid state: NoOPCheckpointCache should never be used in advancing!")
-}
-
-func (NoOPCheckpointCache) PopRangesWithGapGT(d time.Duration) []*RangesSharesTS {
- panic("invalid state: NoOPCheckpointCache should never be used in advancing!")
-}
-
-func (NoOPCheckpointCache) ConsistencyCheck([]kv.KeyRange) error {
- return errors.Annotatef(berrors.ErrUnsupportedOperation, "invalid state: NoOPCheckpointCache should never be used in advancing!")
-}
-
-// RangesSharesTS is a set of ranges shares the same timestamp.
-type RangesSharesTS struct {
- TS uint64
- Ranges []kv.KeyRange
-}
-
-func (rst *RangesSharesTS) Zap() zapcore.ObjectMarshaler {
- return zapcore.ObjectMarshalerFunc(func(oe zapcore.ObjectEncoder) error {
- rngs := rst.Ranges
- if len(rst.Ranges) > 3 {
- rngs = rst.Ranges[:3]
- }
-
- oe.AddUint64("checkpoint", rst.TS)
- return oe.AddArray("items", zapcore.ArrayMarshalerFunc(func(ae zapcore.ArrayEncoder) error {
- return ae.AppendObject(zapcore.ObjectMarshalerFunc(func(oe1 zapcore.ObjectEncoder) error {
- for _, rng := range rngs {
- oe1.AddString("start-key", redact.String(hex.EncodeToString(rng.StartKey)))
- oe1.AddString("end-key", redact.String(hex.EncodeToString(rng.EndKey)))
- }
- return nil
- }))
- }))
- })
-}
-
-func (rst *RangesSharesTS) String() string {
- // Make a more friendly string.
- return fmt.Sprintf("@%sR%d", oracle.GetTimeFromTS(rst.TS).Format("0405"), len(rst.Ranges))
-}
-
-func (rst *RangesSharesTS) Less(other btree.Item) bool {
- return rst.TS < other.(*RangesSharesTS).TS
-}
-
-// Checkpoints is a heap that collects all checkpoints of
-// regions, it supports query the latest checkpoint fast.
-// This structure is thread safe.
-type Checkpoints struct {
- tree *btree.BTree
-
- mu sync.Mutex
-}
-
-func NewCheckpoints() *Checkpoints {
- return &Checkpoints{
- tree: btree.New(32),
- }
-}
-
-// String formats the slowest 5 ranges sharing TS to string.
-func (h *Checkpoints) String() string {
- h.mu.Lock()
- defer h.mu.Unlock()
-
- b := new(strings.Builder)
- count := 0
- total := h.tree.Len()
- h.tree.Ascend(func(i btree.Item) bool {
- rst := i.(*RangesSharesTS)
- b.WriteString(rst.String())
- b.WriteString(";")
- count++
- return count < 5
- })
- if total-count > 0 {
- fmt.Fprintf(b, "O%d", total-count)
- }
- return b.String()
-}
-
-// InsertRanges insert a RangesSharesTS directly to the tree.
-func (h *Checkpoints) InsertRanges(r RangesSharesTS) {
- h.mu.Lock()
- defer h.mu.Unlock()
- if items := h.tree.Get(&r); items != nil {
- i := items.(*RangesSharesTS)
- i.Ranges = append(i.Ranges, r.Ranges...)
- } else {
- h.tree.ReplaceOrInsert(&r)
- }
-}
-
-// InsertRange inserts the region and its TS into the region tree.
-func (h *Checkpoints) InsertRange(ts uint64, rng kv.KeyRange) {
- h.mu.Lock()
- defer h.mu.Unlock()
- r := h.tree.Get(&RangesSharesTS{TS: ts})
- if r == nil {
- r = &RangesSharesTS{TS: ts}
- h.tree.ReplaceOrInsert(r)
- }
- rr := r.(*RangesSharesTS)
- rr.Ranges = append(rr.Ranges, rng)
-}
-
-// Clear removes all records in the checkpoint cache.
-func (h *Checkpoints) Clear() {
- h.mu.Lock()
- defer h.mu.Unlock()
- h.tree.Clear(false)
-}
-
-// PopRangesWithGapGT pops ranges with gap greater than the specified duration.
-// NOTE: maybe make something like `DrainIterator` for better composing?
-func (h *Checkpoints) PopRangesWithGapGT(d time.Duration) []*RangesSharesTS {
- h.mu.Lock()
- defer h.mu.Unlock()
- result := []*RangesSharesTS{}
- for {
- item, ok := h.tree.Min().(*RangesSharesTS)
- if !ok {
- return result
- }
- if time.Since(oracle.GetTimeFromTS(item.TS)) >= d {
- result = append(result, item)
- h.tree.DeleteMin()
- } else {
- return result
- }
- }
-}
-
-// CheckpointTS returns the cached checkpoint TS by the current state of the cache.
-func (h *Checkpoints) CheckpointTS() uint64 {
- h.mu.Lock()
- defer h.mu.Unlock()
- item, ok := h.tree.Min().(*RangesSharesTS)
- if !ok {
- return 0
- }
- return item.TS
-}
-
-// ConsistencyCheck checks whether the tree contains the full range of key space.
-func (h *Checkpoints) ConsistencyCheck(rangesIn []kv.KeyRange) error {
- h.mu.Lock()
- rangesReal := make([]kv.KeyRange, 0, 1024)
- h.tree.Ascend(func(i btree.Item) bool {
- rangesReal = append(rangesReal, i.(*RangesSharesTS).Ranges...)
- return true
- })
- h.mu.Unlock()
-
- r := CollapseRanges(len(rangesReal), func(i int) kv.KeyRange { return rangesReal[i] })
- ri := CollapseRanges(len(rangesIn), func(i int) kv.KeyRange { return rangesIn[i] })
-
- return errors.Annotatef(checkIntervalIsSubset(r, ri), "ranges: (current) %s (not in) %s", logutil.StringifyKeys(r),
- logutil.StringifyKeys(ri))
-}
-
-// A simple algorithm to detect non-overlapped ranges.
-// It maintains the "current" probe, and let the ranges to check "consume" it.
-// For example:
-// toCheck: |_____________________| |_____________|
-// . ^checking
-// subsetOf: |_________| |_______| |__________|
-// . ^probing
-// probing is the subrange of checking, consume it and move forward the probe.
-// toCheck: |_____________________| |_____________|
-// . ^checking
-// subsetOf: |_________| |_______| |__________|
-// . ^probing
-// consume it, too.
-// toCheck: |_____________________| |_____________|
-// . ^checking
-// subsetOf: |_________| |_______| |__________|
-// . ^probing
-// checking is at the left of probing and no overlaps, moving it forward.
-// toCheck: |_____________________| |_____________|
-// . ^checking
-// subsetOf: |_________| |_______| |__________|
-// . ^probing
-// consume it. all subset ranges are consumed, check passed.
-func checkIntervalIsSubset(toCheck []kv.KeyRange, subsetOf []kv.KeyRange) error {
- i := 0
- si := 0
-
- for {
- // We have checked all ranges.
- if si >= len(subsetOf) {
- return nil
- }
- // There are some ranges doesn't reach the end.
- if i >= len(toCheck) {
- return errors.Annotatef(berrors.ErrPiTRMalformedMetadata,
- "there remains a range doesn't be fully consumed: %s",
- logutil.StringifyRange(subsetOf[si]))
- }
-
- checking := toCheck[i]
- probing := subsetOf[si]
- // checking: |___________|
- // probing: |_________|
- // A rare case: the "first" range is out of bound or not fully covers the probing range.
- if utils.CompareBytesExt(checking.StartKey, false, probing.StartKey, false) > 0 {
- holeEnd := checking.StartKey
- if utils.CompareBytesExt(holeEnd, false, probing.EndKey, true) > 0 {
- holeEnd = probing.EndKey
- }
- return errors.Annotatef(berrors.ErrPiTRMalformedMetadata, "probably a hole in key ranges: %s", logutil.StringifyRange{
- StartKey: probing.StartKey,
- EndKey: holeEnd,
- })
- }
-
- // checking: |_____|
- // probing: |_______|
- // Just move forward checking.
- if utils.CompareBytesExt(checking.EndKey, true, probing.StartKey, false) < 0 {
- i += 1
- continue
- }
-
- // checking: |_________|
- // probing: |__________________|
- // Given all of the ranges are "collapsed", the next checking range must
- // not be adjacent with the current checking range.
- // And hence there must be a "hole" in the probing key space.
- if utils.CompareBytesExt(checking.EndKey, true, probing.EndKey, true) < 0 {
- next := probing.EndKey
- if i+1 < len(toCheck) {
- next = toCheck[i+1].EndKey
- }
- return errors.Annotatef(berrors.ErrPiTRMalformedMetadata, "probably a hole in key ranges: %s", logutil.StringifyRange{
- StartKey: checking.EndKey,
- EndKey: next,
- })
- }
- // checking: |________________|
- // probing: |_____________|
- // The current checking range fills the current probing range,
- // or the current checking range is out of the current range.
- // let's move the probing forward.
- si += 1
- }
-}
diff --git a/br/pkg/streamhelper/tsheap_test.go b/br/pkg/streamhelper/tsheap_test.go
deleted file mode 100644
index 173bc2e0a0334..0000000000000
--- a/br/pkg/streamhelper/tsheap_test.go
+++ /dev/null
@@ -1,248 +0,0 @@
-// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
-package streamhelper_test
-
-import (
- "fmt"
- "math"
- "math/rand"
- "testing"
-
- "github.com/pingcap/tidb/br/pkg/streamhelper"
- "github.com/pingcap/tidb/kv"
- "github.com/stretchr/testify/require"
-)
-
-func TestInsert(t *testing.T) {
- cases := []func(func(ts uint64, a, b string)){
- func(insert func(ts uint64, a, b string)) {
- insert(1, "", "01")
- insert(1, "01", "02")
- insert(2, "02", "022")
- insert(4, "022", "")
- },
- func(insert func(ts uint64, a, b string)) {
- insert(1, "", "01")
- insert(2, "", "01")
- insert(2, "011", "02")
- insert(1, "", "")
- insert(65, "03", "04")
- },
- }
-
- for _, c := range cases {
- cps := streamhelper.NewCheckpoints()
- expected := map[uint64]*streamhelper.RangesSharesTS{}
- checkpoint := uint64(math.MaxUint64)
- insert := func(ts uint64, a, b string) {
- cps.InsertRange(ts, kv.KeyRange{
- StartKey: []byte(a),
- EndKey: []byte(b),
- })
- i, ok := expected[ts]
- if !ok {
- expected[ts] = &streamhelper.RangesSharesTS{TS: ts, Ranges: []kv.KeyRange{{StartKey: []byte(a), EndKey: []byte(b)}}}
- } else {
- i.Ranges = append(i.Ranges, kv.KeyRange{StartKey: []byte(a), EndKey: []byte(b)})
- }
- if ts < checkpoint {
- checkpoint = ts
- }
- }
- c(insert)
- require.Equal(t, checkpoint, cps.CheckpointTS())
- rngs := cps.PopRangesWithGapGT(0)
- for _, rng := range rngs {
- other := expected[rng.TS]
- require.Equal(t, other, rng)
- }
- }
-}
-
-func TestMergeRanges(t *testing.T) {
- r := func(a, b string) kv.KeyRange {
- return kv.KeyRange{StartKey: []byte(a), EndKey: []byte(b)}
- }
- type Case struct {
- expected []kv.KeyRange
- parameter []kv.KeyRange
- }
- cases := []Case{
- {
- parameter: []kv.KeyRange{r("01", "01111"), r("0111", "0112")},
- expected: []kv.KeyRange{r("01", "0112")},
- },
- {
- parameter: []kv.KeyRange{r("01", "03"), r("02", "04")},
- expected: []kv.KeyRange{r("01", "04")},
- },
- {
- parameter: []kv.KeyRange{r("04", "08"), r("09", "10")},
- expected: []kv.KeyRange{r("04", "08"), r("09", "10")},
- },
- {
- parameter: []kv.KeyRange{r("01", "03"), r("02", "04"), r("05", "07"), r("08", "09")},
- expected: []kv.KeyRange{r("01", "04"), r("05", "07"), r("08", "09")},
- },
- {
- parameter: []kv.KeyRange{r("01", "02"), r("012", "")},
- expected: []kv.KeyRange{r("01", "")},
- },
- {
- parameter: []kv.KeyRange{r("", "01"), r("02", "03"), r("021", "")},
- expected: []kv.KeyRange{r("", "01"), r("02", "")},
- },
- {
- parameter: []kv.KeyRange{r("", "01"), r("001", "")},
- expected: []kv.KeyRange{r("", "")},
- },
- {
- parameter: []kv.KeyRange{r("", "01"), r("", ""), r("", "02")},
- expected: []kv.KeyRange{r("", "")},
- },
- {
- parameter: []kv.KeyRange{r("", "01"), r("01", ""), r("", "02"), r("", "03"), r("01", "02")},
- expected: []kv.KeyRange{r("", "")},
- },
- {
- parameter: []kv.KeyRange{r("", ""), r("", "01"), r("01", ""), r("01", "02")},
- expected: []kv.KeyRange{r("", "")},
- },
- }
-
- for i, c := range cases {
- result := streamhelper.CollapseRanges(len(c.parameter), func(i int) kv.KeyRange {
- return c.parameter[i]
- })
- require.Equal(t, c.expected, result, "case = %d", i)
- }
-}
-
-func TestInsertRanges(t *testing.T) {
- r := func(a, b string) kv.KeyRange {
- return kv.KeyRange{StartKey: []byte(a), EndKey: []byte(b)}
- }
- rs := func(ts uint64, ranges ...kv.KeyRange) streamhelper.RangesSharesTS {
- return streamhelper.RangesSharesTS{TS: ts, Ranges: ranges}
- }
-
- type Case struct {
- Expected []streamhelper.RangesSharesTS
- Parameters []streamhelper.RangesSharesTS
- }
-
- cases := []Case{
- {
- Parameters: []streamhelper.RangesSharesTS{
- rs(1, r("0", "1"), r("1", "2")),
- rs(1, r("2", "3"), r("3", "4")),
- },
- Expected: []streamhelper.RangesSharesTS{
- rs(1, r("0", "1"), r("1", "2"), r("2", "3"), r("3", "4")),
- },
- },
- {
- Parameters: []streamhelper.RangesSharesTS{
- rs(1, r("0", "1")),
- rs(2, r("2", "3")),
- rs(1, r("4", "5"), r("6", "7")),
- },
- Expected: []streamhelper.RangesSharesTS{
- rs(1, r("0", "1"), r("4", "5"), r("6", "7")),
- rs(2, r("2", "3")),
- },
- },
- }
-
- for _, c := range cases {
- theTree := streamhelper.NewCheckpoints()
- for _, p := range c.Parameters {
- theTree.InsertRanges(p)
- }
- ranges := theTree.PopRangesWithGapGT(0)
- for i, rs := range ranges {
- require.ElementsMatch(t, c.Expected[i].Ranges, rs.Ranges, "case = %#v", c)
- }
- }
-}
-
-func TestConsistencyCheckOverRange(t *testing.T) {
- r := func(a, b string) kv.KeyRange {
- return kv.KeyRange{StartKey: []byte(a), EndKey: []byte(b)}
- }
- type Case struct {
- checking []kv.KeyRange
- probing []kv.KeyRange
- isSubset bool
- }
-
- cases := []Case{
- // basic: exactly match.
- {
- checking: []kv.KeyRange{r("0001", "0002"), r("0002", "0003"), r("0004", "0005")},
- probing: []kv.KeyRange{r("0001", "0003"), r("0004", "0005")},
- isSubset: true,
- },
- // not fully match, probing longer.
- {
- checking: []kv.KeyRange{r("0001", "0002"), r("0002", "0003"), r("0004", "0005")},
- probing: []kv.KeyRange{r("0000", "0003"), r("0004", "00051")},
- isSubset: false,
- },
- // with infinity end keys.
- {
- checking: []kv.KeyRange{r("0001", "0002"), r("0002", "0003"), r("0004", "")},
- probing: []kv.KeyRange{r("0001", "0003"), r("0004", "")},
- isSubset: true,
- },
- {
- checking: []kv.KeyRange{r("0001", "0002"), r("0002", "0003"), r("0004", "")},
- probing: []kv.KeyRange{r("0001", "0003"), r("0004", "0005")},
- isSubset: true,
- },
- {
- checking: []kv.KeyRange{r("0001", "0002"), r("0002", "0003"), r("0004", "0005")},
- probing: []kv.KeyRange{r("0001", "0003"), r("0004", "")},
- isSubset: false,
- },
- // overlapped probe.
- {
- checking: []kv.KeyRange{r("0001", "0002"), r("0002", "0003"), r("0004", "0007")},
- probing: []kv.KeyRange{r("0001", "0008")},
- isSubset: false,
- },
- {
- checking: []kv.KeyRange{r("0001", "0008")},
- probing: []kv.KeyRange{r("0001", "0002"), r("0002", "0003"), r("0004", "0007")},
- isSubset: true,
- },
- {
- checking: []kv.KeyRange{r("0100", "0120"), r("0130", "0141")},
- probing: []kv.KeyRange{r("0000", "0001")},
- isSubset: false,
- },
- {
- checking: []kv.KeyRange{r("0100", "0120")},
- probing: []kv.KeyRange{r("0090", "0110"), r("0115", "0120")},
- isSubset: false,
- },
- }
-
- run := func(t *testing.T, c Case) {
- tree := streamhelper.NewCheckpoints()
- for _, r := range c.checking {
- tree.InsertRange(rand.Uint64()%10, r)
- }
- err := tree.ConsistencyCheck(c.probing)
- if c.isSubset {
- require.NoError(t, err)
- } else {
- require.Error(t, err)
- }
- }
-
- for i, c := range cases {
- t.Run(fmt.Sprintf("#%d", i), func(tc *testing.T) {
- run(tc, c)
- })
- }
-}
diff --git a/br/pkg/summary/collector.go b/br/pkg/summary/collector.go
index 705c26df3e4ac..1a16fb6dc9cfc 100644
--- a/br/pkg/summary/collector.go
+++ b/br/pkg/summary/collector.go
@@ -46,6 +46,10 @@ type LogCollector interface {
SetSuccessStatus(success bool)
+ NowDureTime() time.Duration
+
+ AdjustStartTimeToEarlierTime(t time.Duration)
+
Summary(name string)
Log(msg string, fields ...zap.Field)
@@ -163,6 +167,18 @@ func logKeyFor(key string) string {
return strings.ReplaceAll(key, " ", "-")
}
+func (tc *logCollector) NowDureTime() time.Duration {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+ return time.Since(tc.startTime)
+}
+
+func (tc *logCollector) AdjustStartTimeToEarlierTime(t time.Duration) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+ tc.startTime = tc.startTime.Add(-t)
+}
+
func (tc *logCollector) Summary(name string) {
tc.mu.Lock()
defer func() {
diff --git a/br/pkg/summary/summary.go b/br/pkg/summary/summary.go
index 7ae488785760e..45c8fbbc55997 100644
--- a/br/pkg/summary/summary.go
+++ b/br/pkg/summary/summary.go
@@ -43,6 +43,15 @@ func SetSuccessStatus(success bool) {
collector.SetSuccessStatus(success)
}
+// NowDureTime returns the duration between start time and current time
+func NowDureTime() time.Duration {
+ return collector.NowDureTime()
+}
+
+func AdjustStartTimeToEarlierTime(t time.Duration) {
+ collector.AdjustStartTimeToEarlierTime(t)
+}
+
// Summary outputs summary log.
func Summary(name string) {
collector.Summary(name)
diff --git a/br/pkg/task/BUILD.bazel b/br/pkg/task/BUILD.bazel
index 979afd1ba9110..0ad8cde87137e 100644
--- a/br/pkg/task/BUILD.bazel
+++ b/br/pkg/task/BUILD.bazel
@@ -67,6 +67,7 @@ go_library(
"@com_github_spf13_pflag//:pflag",
"@com_github_tikv_client_go_v2//config",
"@com_github_tikv_client_go_v2//oracle",
+ "@com_github_tikv_client_go_v2//tikv",
"@com_github_tikv_pd_client//:client",
"@com_google_cloud_go_storage//:storage",
"@io_etcd_go_etcd_client_pkg_v3//transport",
diff --git a/br/pkg/task/backup.go b/br/pkg/task/backup.go
index 8b3a03c9a8719..5ddaf0e6ccd25 100644
--- a/br/pkg/task/backup.go
+++ b/br/pkg/task/backup.go
@@ -4,6 +4,8 @@ package task
import (
"context"
+ "crypto/sha256"
+ "encoding/json"
"fmt"
"os"
"strconv"
@@ -26,6 +28,8 @@ import (
"github.com/pingcap/tidb/br/pkg/storage"
"github.com/pingcap/tidb/br/pkg/summary"
"github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/version"
+ "github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/statistics/handle"
@@ -45,11 +49,15 @@ const (
flagRemoveSchedulers = "remove-schedulers"
flagIgnoreStats = "ignore-stats"
flagUseBackupMetaV2 = "use-backupmeta-v2"
+ flagUseCheckpoint = "use-checkpoint"
+ flagKeyspaceName = "keyspace-name"
+ flagReplicaReadLabel = "replica-read-label"
flagGCTTL = "gcttl"
defaultBackupConcurrency = 4
maxBackupConcurrency = 256
+ checkpointDefaultGCTTL = 72 * 60 // 72 minutes
)
const (
@@ -70,13 +78,15 @@ type CompressionConfig struct {
type BackupConfig struct {
Config
- TimeAgo time.Duration `json:"time-ago" toml:"time-ago"`
- BackupTS uint64 `json:"backup-ts" toml:"backup-ts"`
- LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"`
- GCTTL int64 `json:"gc-ttl" toml:"gc-ttl"`
- RemoveSchedulers bool `json:"remove-schedulers" toml:"remove-schedulers"`
- IgnoreStats bool `json:"ignore-stats" toml:"ignore-stats"`
- UseBackupMetaV2 bool `json:"use-backupmeta-v2"`
+ TimeAgo time.Duration `json:"time-ago" toml:"time-ago"`
+ BackupTS uint64 `json:"backup-ts" toml:"backup-ts"`
+ LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"`
+ GCTTL int64 `json:"gc-ttl" toml:"gc-ttl"`
+ RemoveSchedulers bool `json:"remove-schedulers" toml:"remove-schedulers"`
+ IgnoreStats bool `json:"ignore-stats" toml:"ignore-stats"`
+ UseBackupMetaV2 bool `json:"use-backupmeta-v2"`
+ UseCheckpoint bool `json:"use-checkpoint" toml:"use-checkpoint"`
+ ReplicaReadLabel map[string]string `json:"replica-read-label" toml:"replica-read-label"`
CompressionConfig
// for ebs-based backup
@@ -118,6 +128,8 @@ func DefineBackupFlags(flags *pflag.FlagSet) {
flags.Bool(flagUseBackupMetaV2, false,
"use backup meta v2 to store meta info")
+
+ flags.String(flagKeyspaceName, "", "keyspace name for backup")
// This flag will change the structure of backupmeta.
// we must make sure the old three version of br can parse the v2 meta to keep compatibility.
// so this flag should set to false for three version by default.
@@ -126,6 +138,11 @@ func DefineBackupFlags(flags *pflag.FlagSet) {
// but will generate v1 meta due to this flag is false. the behaviour is as same as v4.0.15, v4.0.16.
// finally v4.0.17 will set this flag to true, and generate v2 meta.
_ = flags.MarkHidden(flagUseBackupMetaV2)
+
+ flags.Bool(flagUseCheckpoint, true, "use checkpoint mode")
+ _ = flags.MarkHidden(flagUseCheckpoint)
+
+ flags.String(flagReplicaReadLabel, "", "specify the label of the stores to be used for backup, e.g. 'label_key:label_value'")
}
// ParseFromFlags parses the backup-related flags from the flag set.
@@ -150,10 +167,34 @@ func (cfg *BackupConfig) ParseFromFlags(flags *pflag.FlagSet) error {
if err != nil {
return errors.Trace(err)
}
+ cfg.UseBackupMetaV2, err = flags.GetBool(flagUseBackupMetaV2)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ cfg.UseCheckpoint, err = flags.GetBool(flagUseCheckpoint)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if cfg.LastBackupTS > 0 {
+ // TODO: compatible with incremental backup
+ cfg.UseCheckpoint = false
+ log.Info("since incremental backup is used, turn off checkpoint mode")
+ }
+ if cfg.UseBackupMetaV2 {
+ // TODO: compatible with backup meta v2, maybe just clean the meta files
+ cfg.UseCheckpoint = false
+ log.Info("since backup meta v2 is used, turn off checkpoint mode")
+ }
gcTTL, err := flags.GetInt64(flagGCTTL)
if err != nil {
return errors.Trace(err)
}
+ // if use checkpoint and gcTTL is the default value
+ // update gcttl to checkpoint's default gc ttl
+ if cfg.UseCheckpoint && gcTTL == utils.DefaultBRGCSafePointTTL {
+ gcTTL = checkpointDefaultGCTTL
+ log.Info("use checkpoint's default GC TTL", zap.Int64("GC TTL", gcTTL))
+ }
cfg.GCTTL = gcTTL
compressionCfg, err := parseCompressionFlags(flags)
@@ -173,7 +214,7 @@ func (cfg *BackupConfig) ParseFromFlags(flags *pflag.FlagSet) error {
if err != nil {
return errors.Trace(err)
}
- cfg.UseBackupMetaV2, err = flags.GetBool(flagUseBackupMetaV2)
+ cfg.KeyspaceName, err = flags.GetString(flagKeyspaceName)
if err != nil {
return errors.Trace(err)
}
@@ -206,6 +247,11 @@ func (cfg *BackupConfig) ParseFromFlags(flags *pflag.FlagSet) error {
}
}
+ cfg.ReplicaReadLabel, err = parseReplicaReadLabelFlag(flags)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
return nil
}
@@ -269,6 +315,23 @@ func (cfg *BackupConfig) Adjust() {
}
}
+// a rough hash for checkpoint checker
+func (cfg *BackupConfig) Hash() ([]byte, error) {
+ config := &BackupConfig{
+ LastBackupTS: cfg.LastBackupTS,
+ IgnoreStats: cfg.IgnoreStats,
+ UseCheckpoint: cfg.UseCheckpoint,
+ Config: cfg.Config,
+ }
+ data, err := json.Marshal(config)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ hash := sha256.Sum256(data)
+
+ return hash[:], nil
+}
+
func isFullBackup(cmdName string) bool {
return cmdName == FullBackupCmd
}
@@ -276,6 +339,9 @@ func isFullBackup(cmdName string) bool {
// RunBackup starts a backup task inside the current goroutine.
func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig) error {
cfg.Adjust()
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.KeyspaceName = cfg.KeyspaceName
+ })
defer summary.Summary(cmdName)
ctx, cancel := context.WithCancel(c)
@@ -301,6 +367,14 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
return errors.Trace(err)
}
defer mgr.Close()
+ // after version check, check the cluster whether support checkpoint mode
+ if cfg.UseCheckpoint {
+ err = version.CheckCheckpointSupport()
+ if err != nil {
+ log.Warn("unable to use checkpoint mode, fall back to normal mode", zap.Error(err))
+ cfg.UseCheckpoint = false
+ }
+ }
var statsHandle *handle.Handle
if !skipStats {
statsHandle = mgr.GetDomain().StatsHandle()
@@ -308,22 +382,23 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
var newCollationEnable string
err = g.UseOneShotSession(mgr.GetStorage(), !needDomain, func(se glue.Session) error {
- newCollationEnable, err = se.GetGlobalVariable(tidbNewCollationEnabled)
+ newCollationEnable, err = se.GetGlobalVariable(utils.GetTidbNewCollationEnabled())
if err != nil {
return errors.Trace(err)
}
log.Info("get new_collations_enabled_on_first_bootstrap config from system table",
- zap.String(tidbNewCollationEnabled, newCollationEnable))
+ zap.String(utils.GetTidbNewCollationEnabled(), newCollationEnable))
return nil
})
if err != nil {
return errors.Trace(err)
}
- client, err := backup.NewBackupClient(ctx, mgr)
- if err != nil {
- return errors.Trace(err)
- }
+ client := backup.NewBackupClient(ctx, mgr)
+
+ // set cipher only for checkpoint
+ client.SetCipher(&cfg.CipherInfo)
+
opts := storage.ExternalStorageOptions{
NoCredentials: cfg.NoCreds,
SendCredentials: cfg.SendCreds,
@@ -332,6 +407,16 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
if err = client.SetStorageAndCheckNotInUse(ctx, u, &opts); err != nil {
return errors.Trace(err)
}
+ // if checkpoint mode is unused at this time but there is checkpoint meta,
+ // CheckCheckpoint will stop backing up
+ cfgHash, err := cfg.Hash()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ err = client.CheckCheckpoint(cfgHash)
+ if err != nil {
+ return errors.Trace(err)
+ }
err = client.SetLockFile(ctx)
if err != nil {
return errors.Trace(err)
@@ -343,24 +428,45 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
return errors.Trace(err)
}
g.Record("BackupTS", backupTS)
+ safePointID := client.GetSafePointID()
sp := utils.BRServiceSafePoint{
BackupTS: backupTS,
TTL: client.GetGCTTL(),
- ID: utils.MakeSafePointID(),
+ ID: safePointID,
}
+
// use lastBackupTS as safePoint if exists
- if cfg.LastBackupTS > 0 {
+ isIncrementalBackup := cfg.LastBackupTS > 0
+ if isIncrementalBackup {
sp.BackupTS = cfg.LastBackupTS
}
log.Info("current backup safePoint job", zap.Object("safePoint", sp))
- err = utils.StartServiceSafePointKeeper(ctx, mgr.GetPDClient(), sp)
+ cctx, gcSafePointKeeperCancel := context.WithCancel(ctx)
+ gcSafePointKeeperRemovable := false
+ defer func() {
+ // don't reset the gc-safe-point if checkpoint mode is used and backup is not finished
+ if cfg.UseCheckpoint && !gcSafePointKeeperRemovable {
+ log.Info("skip removing gc-safepoint keeper for next retry", zap.String("gc-id", sp.ID))
+ return
+ }
+ log.Info("start to remove gc-safepoint keeper")
+ // close the gc safe point keeper at first
+ gcSafePointKeeperCancel()
+ // set the ttl to 0 to remove the gc-safe-point
+ sp.TTL = 0
+ if err := utils.UpdateServiceSafePoint(ctx, mgr.GetPDClient(), sp); err != nil {
+ log.Warn("failed to update service safe point, backup may fail if gc triggered",
+ zap.Error(err),
+ )
+ }
+ log.Info("finish removing gc-safepoint keeper")
+ }()
+ err = utils.StartServiceSafePointKeeper(cctx, mgr.GetPDClient(), sp)
if err != nil {
return errors.Trace(err)
}
- isIncrementalBackup := cfg.LastBackupTS > 0
-
if cfg.RemoveSchedulers {
log.Debug("removing some PD schedulers")
restore, e := mgr.RemoveSchedulers(ctx)
@@ -388,6 +494,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
CompressionType: cfg.CompressionType,
CompressionLevel: cfg.CompressionLevel,
CipherInfo: &cfg.CipherInfo,
+ ReplicaRead: len(cfg.ReplicaReadLabel) != 0,
}
brVersion := g.GetVersion()
clusterVersion, err := mgr.GetClusterVersion(ctx)
@@ -395,10 +502,15 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
return errors.Trace(err)
}
- ranges, schemas, policies, err := backup.BuildBackupRangeAndSchema(mgr.GetStorage(), cfg.TableFilter, backupTS, isFullBackup(cmdName))
+ ranges, schemas, policies, err := client.BuildBackupRangeAndSchema(mgr.GetStorage(), cfg.TableFilter, backupTS, isFullBackup(cmdName))
if err != nil {
return errors.Trace(err)
}
+ // Add keyspace prefix to BackupRequest
+ for i := range ranges {
+ start, end := ranges[i].StartKey, ranges[i].EndKey
+ ranges[i].StartKey, ranges[i].EndKey = mgr.GetStorage().GetCodec().EncodeRange(start, end)
+ }
// Metafile size should be less than 64MB.
metawriter := metautil.NewMetaWriter(client.GetStorage(),
@@ -412,6 +524,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
m.ClusterVersion = clusterVersion
m.BrVersion = brVersion
m.NewCollationsEnabled = newCollationEnable
+ m.ApiVersion = mgr.GetStorage().GetCodec().GetAPIVersion()
})
log.Info("get placement policies", zap.Int("count", len(policies)))
@@ -422,7 +535,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
}
// nothing to backup
- if ranges == nil || len(ranges) <= 0 {
+ if len(ranges) == 0 {
pdAddress := strings.Join(cfg.PD, ",")
log.Warn("Nothing to backup, maybe connected to cluster for restoring",
zap.String("PD address", pdAddress))
@@ -503,8 +616,20 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
})
}
}
+
+ if cfg.UseCheckpoint {
+ if err = client.StartCheckpointRunner(ctx, cfgHash, backupTS, ranges, safePointID, progressCallBack); err != nil {
+ return errors.Trace(err)
+ }
+ defer func() {
+ if !gcSafePointKeeperRemovable {
+ log.Info("wait for flush checkpoint...")
+ client.WaitForFinishCheckpoint(ctx)
+ }
+ }()
+ }
metawriter.StartWriteMetasAsync(ctx, metautil.AppendDataFile)
- err = client.BackupRanges(ctx, ranges, req, uint(cfg.Concurrency), metawriter, progressCallBack)
+ err = client.BackupRanges(ctx, ranges, req, uint(cfg.Concurrency), cfg.ReplicaReadLabel, metawriter, progressCallBack)
if err != nil {
return errors.Trace(err)
}
@@ -532,7 +657,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
schemasConcurrency := uint(mathutil.Min(backup.DefaultSchemaConcurrency, schemas.Len()))
err = schemas.BackupSchemas(
- ctx, metawriter, mgr.GetStorage(), statsHandle, backupTS, schemasConcurrency, cfg.ChecksumConcurrency, skipChecksum, updateCh)
+ ctx, metawriter, client.GetCheckpointRunner(), mgr.GetStorage(), statsHandle, backupTS, schemasConcurrency, cfg.ChecksumConcurrency, skipChecksum, updateCh)
if err != nil {
return errors.Trace(err)
}
@@ -541,6 +666,9 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
if err != nil {
return errors.Trace(err)
}
+ // Since backupmeta is flushed on the external storage,
+ // we can remove the gc safepoint keeper
+ gcSafePointKeeperRemovable = true
// Checksum has finished, close checksum progress.
updateCh.Close()
@@ -594,7 +722,7 @@ func ParseTSString(ts string, tzCheck bool) (uint64, error) {
return 0, errors.Errorf("must set timezone when using datetime format ts, e.g. '2018-05-11 01:42:23+0800'")
}
}
- t, err := types.ParseTime(sc, ts, mysql.TypeTimestamp, types.MaxFsp)
+ t, err := types.ParseTime(sc, ts, mysql.TypeTimestamp, types.MaxFsp, nil)
if err != nil {
return 0, errors.Trace(err)
}
@@ -619,3 +747,18 @@ func parseCompressionType(s string) (backuppb.CompressionType, error) {
}
return ct, nil
}
+
+func parseReplicaReadLabelFlag(flags *pflag.FlagSet) (map[string]string, error) {
+ replicaReadLabelStr, err := flags.GetString(flagReplicaReadLabel)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if replicaReadLabelStr == "" {
+ return nil, nil
+ }
+ kv := strings.Split(replicaReadLabelStr, ":")
+ if len(kv) != 2 {
+ return nil, errors.Annotatef(berrors.ErrInvalidArgument, "invalid replica read label '%s'", replicaReadLabelStr)
+ }
+ return map[string]string{kv[0]: kv[1]}, nil
+}
diff --git a/br/pkg/task/backup_ebs.go b/br/pkg/task/backup_ebs.go
index 5d9a262d7cce6..ff0fb6a01a461 100644
--- a/br/pkg/task/backup_ebs.go
+++ b/br/pkg/task/backup_ebs.go
@@ -111,10 +111,7 @@ func RunBackupEBS(c context.Context, g glue.Glue, cfg *BackupConfig) error {
return errors.Trace(err)
}
defer mgr.Close()
- client, err := backup.NewBackupClient(ctx, mgr)
- if err != nil {
- return errors.Trace(err)
- }
+ client := backup.NewBackupClient(ctx, mgr)
opts := storage.ExternalStorageOptions{
NoCredentials: cfg.NoCreds,
diff --git a/br/pkg/task/backup_raw.go b/br/pkg/task/backup_raw.go
index 9d46c151d23c7..ed7248fd21bf4 100644
--- a/br/pkg/task/backup_raw.go
+++ b/br/pkg/task/backup_raw.go
@@ -144,10 +144,7 @@ func RunBackupRaw(c context.Context, g glue.Glue, cmdName string, cfg *RawKvConf
}
defer mgr.Close()
- client, err := backup.NewBackupClient(ctx, mgr)
- if err != nil {
- return errors.Trace(err)
- }
+ client := backup.NewBackupClient(ctx, mgr)
opts := storage.ExternalStorageOptions{
NoCredentials: cfg.NoCreds,
SendCredentials: cfg.SendCreds,
@@ -216,9 +213,18 @@ func RunBackupRaw(c context.Context, g glue.Glue, cmdName string, cfg *RawKvConf
CompressionLevel: cfg.CompressionLevel,
CipherInfo: &cfg.CipherInfo,
}
+ rg := rtree.Range{
+ StartKey: backupRange.StartKey,
+ EndKey: backupRange.EndKey,
+ }
+ progressRange := &rtree.ProgressRange{
+ Res: rtree.NewRangeTree(),
+ Incomplete: []rtree.Range{rg},
+ Origin: rg,
+ }
metaWriter := metautil.NewMetaWriter(client.GetStorage(), metautil.MetaFileSize, false, metautil.MetaFile, &cfg.CipherInfo)
metaWriter.StartWriteMetasAsync(ctx, metautil.AppendDataFile)
- err = client.BackupRange(ctx, req, metaWriter, progressCallBack)
+ err = client.BackupRange(ctx, req, map[string]string{}, progressRange, metaWriter, progressCallBack)
if err != nil {
return errors.Trace(err)
}
diff --git a/br/pkg/task/common.go b/br/pkg/task/common.go
index 5d76a2db4f85b..ad223b7e34c67 100644
--- a/br/pkg/task/common.go
+++ b/br/pkg/task/common.go
@@ -96,8 +96,6 @@ const (
crypterAES192KeyLen = 24
crypterAES256KeyLen = 32
- tidbNewCollationEnabled = "new_collation_enabled"
-
flagFullBackupType = "type"
)
@@ -234,6 +232,9 @@ type Config struct {
// whether there's explicit filter
ExplicitFilter bool `json:"-" toml:"-"`
+
+ // KeyspaceName is the name of the keyspace of the task
+ KeyspaceName string `json:"keyspace-name" toml:"keyspace-name"`
}
// DefineCommonFlags defines the flags common to all BRIE commands.
diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go
index cf1ce4682a09f..2a5c58a9febf3 100644
--- a/br/pkg/task/restore.go
+++ b/br/pkg/task/restore.go
@@ -26,11 +26,11 @@ import (
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/br/pkg/version"
"github.com/pingcap/tidb/config"
- "github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/mathutil"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
+ "github.com/tikv/client-go/v2/tikv"
"go.uber.org/multierr"
"go.uber.org/zap"
)
@@ -52,20 +52,30 @@ const (
// FlagWithPlacementPolicy corresponds to tidb config with-tidb-placement-mode
// current only support STRICT or IGNORE, the default is STRICT according to tidb.
FlagWithPlacementPolicy = "with-tidb-placement-mode"
+ // FlagKeyspaceName corresponds to tidb config keyspace-name
+ FlagKeyspaceName = "keyspace-name"
// FlagStreamStartTS and FlagStreamRestoreTS is used for log restore timestamp range.
FlagStreamStartTS = "start-ts"
FlagStreamRestoreTS = "restored-ts"
// FlagStreamFullBackupStorage is used for log restore, represents the full backup storage.
FlagStreamFullBackupStorage = "full-backup-storage"
-
- defaultRestoreConcurrency = 128
- defaultRestoreStreamConcurrency = 16
- maxRestoreBatchSizeLimit = 10240
- defaultPDConcurrency = 1
- defaultBatchFlushInterval = 16 * time.Second
- defaultFlagDdlBatchSize = 128
- resetSpeedLimitRetryTimes = 3
+ // FlagPiTRBatchCount and FlagPiTRBatchSize are used for restore log with batch method.
+ FlagPiTRBatchCount = "pitr-batch-count"
+ FlagPiTRBatchSize = "pitr-batch-size"
+ FlagPiTRConcurrency = "pitr-concurrency"
+
+ FlagResetSysUsers = "reset-sys-users"
+
+ defaultPiTRBatchCount = 8
+ defaultPiTRBatchSize = 16 * 1024 * 1024
+ defaultRestoreConcurrency = 128
+ defaultPiTRConcurrency = 16
+ maxRestoreBatchSizeLimit = 10240
+ defaultPDConcurrency = 1
+ defaultBatchFlushInterval = 16 * time.Second
+ defaultFlagDdlBatchSize = 128
+ resetSpeedLimitRetryTimes = 3
)
const (
@@ -88,6 +98,8 @@ type RestoreCommonConfig struct {
// determines whether enable restore sys table on default, see fullClusterRestore in restore/client.go
WithSysTable bool `json:"with-sys-table" toml:"with-sys-table"`
+
+ ResetSysUsers []string `json:"reset-sys-users" toml:"reset-sys-users"`
}
// adjust adjusts the abnormal config value in the current config.
@@ -113,10 +125,12 @@ func DefineRestoreCommonFlags(flags *pflag.FlagSet) {
flags.Uint(FlagPDConcurrency, defaultPDConcurrency,
"concurrency pd-relative operations like split & scatter.")
flags.Duration(FlagBatchFlushInterval, defaultBatchFlushInterval,
- "after how long a restore batch would be auto sended.")
+ "after how long a restore batch would be auto sent.")
flags.Uint(FlagDdlBatchSize, defaultFlagDdlBatchSize,
- "batch size for ddl to create a batch of tabes once.")
+ "batch size for ddl to create a batch of tables once.")
flags.Bool(flagWithSysTable, false, "whether restore system privilege tables on default setting")
+ flags.StringArrayP(FlagResetSysUsers, "", []string{"cloud_admin", "root"}, "whether reset these users after restoration")
+ _ = flags.MarkHidden(FlagResetSysUsers)
_ = flags.MarkHidden(FlagMergeRegionSizeBytes)
_ = flags.MarkHidden(FlagMergeRegionKeyCount)
_ = flags.MarkHidden(FlagPDConcurrency)
@@ -145,6 +159,10 @@ func (cfg *RestoreCommonConfig) ParseFromFlags(flags *pflag.FlagSet) error {
return errors.Trace(err)
}
}
+ cfg.ResetSysUsers, err = flags.GetStringArray(FlagResetSysUsers)
+ if err != nil {
+ return errors.Trace(err)
+ }
return errors.Trace(err)
}
@@ -169,6 +187,9 @@ type RestoreConfig struct {
StartTS uint64 `json:"start-ts" toml:"start-ts"`
RestoreTS uint64 `json:"restore-ts" toml:"restore-ts"`
tiflashRecorder *tiflashrec.TiFlashRecorder `json:"-" toml:"-"`
+ PitrBatchCount uint32 `json:"pitr-batch-count" toml:"pitr-batch-count"`
+ PitrBatchSize uint32 `json:"pitr-batch-size" toml:"pitr-batch-size"`
+ PitrConcurrency uint32 `json:"-" toml:"-"`
// for ebs-based restore
FullBackupType FullBackupType `json:"full-backup-type" toml:"full-backup-type"`
@@ -188,6 +209,7 @@ func DefineRestoreFlags(flags *pflag.FlagSet) {
// Do not expose this flag
_ = flags.MarkHidden(flagNoSchema)
flags.String(FlagWithPlacementPolicy, "STRICT", "correspond to tidb global/session variable with-tidb-placement-mode")
+ flags.String(FlagKeyspaceName, "", "correspond to tidb config keyspace-name")
DefineRestoreCommonFlags(flags)
}
@@ -200,6 +222,9 @@ func DefineStreamRestoreFlags(command *cobra.Command) {
"support TSO or datetime, e.g. '400036290571534337' or '2018-05-11 01:42:23+0800'")
command.Flags().String(FlagStreamFullBackupStorage, "", "specify the backup full storage. "+
"fill it if want restore full backup before restore log.")
+ command.Flags().Uint32(FlagPiTRBatchCount, defaultPiTRBatchCount, "specify the batch count to restore log.")
+ command.Flags().Uint32(FlagPiTRBatchSize, defaultPiTRBatchSize, "specify the batch size to retore log.")
+ command.Flags().Uint32(FlagPiTRConcurrency, defaultPiTRConcurrency, "specify the concurrency to restore log.")
}
// ParseStreamRestoreFlags parses the `restore stream` flags from the flag set.
@@ -228,6 +253,15 @@ func (cfg *RestoreConfig) ParseStreamRestoreFlags(flags *pflag.FlagSet) error {
FlagStreamStartTS, FlagStreamFullBackupStorage)
}
+ if cfg.PitrBatchCount, err = flags.GetUint32(FlagPiTRBatchCount); err != nil {
+ return errors.Trace(err)
+ }
+ if cfg.PitrBatchSize, err = flags.GetUint32(FlagPiTRBatchSize); err != nil {
+ return errors.Trace(err)
+ }
+ if cfg.PitrConcurrency, err = flags.GetUint32(FlagPiTRConcurrency); err != nil {
+ return errors.Trace(err)
+ }
return nil
}
@@ -267,6 +301,10 @@ func (cfg *RestoreConfig) ParseFromFlags(flags *pflag.FlagSet) error {
if err != nil {
return errors.Annotatef(err, "failed to get flag %s", FlagWithPlacementPolicy)
}
+ cfg.KeyspaceName, err = flags.GetString(FlagKeyspaceName)
+ if err != nil {
+ return errors.Annotatef(err, "failed to get flag %s", FlagKeyspaceName)
+ }
if flags.Lookup(flagFullBackupType) != nil {
// for restore full only
@@ -354,10 +392,19 @@ func (cfg *RestoreConfig) Adjust() {
}
func (cfg *RestoreConfig) adjustRestoreConfigForStreamRestore() {
- if cfg.Config.Concurrency == 0 || cfg.Config.Concurrency > defaultRestoreStreamConcurrency {
- log.Info("set restore kv files concurrency", zap.Int("concurrency", defaultRestoreStreamConcurrency))
- cfg.Config.Concurrency = defaultRestoreStreamConcurrency
+ if cfg.PitrConcurrency == 0 {
+ cfg.PitrConcurrency = defaultPiTRConcurrency
+ }
+ if cfg.PitrBatchCount == 0 {
+ cfg.PitrBatchCount = defaultPiTRBatchCount
}
+ if cfg.PitrBatchSize == 0 {
+ cfg.PitrBatchSize = defaultPiTRBatchSize
+ }
+ // another goroutine is used to iterate the backup file
+ cfg.PitrConcurrency += 1
+ log.Info("set restore kv files concurrency", zap.Int("concurrency", int(cfg.PitrConcurrency)))
+ cfg.Config.Concurrency = cfg.PitrConcurrency
}
func configureRestoreClient(ctx context.Context, client *restore.Client, cfg *RestoreConfig) error {
@@ -375,7 +422,15 @@ func configureRestoreClient(ctx context.Context, client *restore.Client, cfg *Re
client.SetPlacementPolicyMode(cfg.WithPlacementPolicy)
client.SetWithSysTable(cfg.WithSysTable)
- err := client.LoadRestoreStores(ctx)
+ err := restore.CheckKeyspaceBREnable(ctx, client.GetPDClient())
+ if err != nil {
+ log.Warn("Keyspace BR is not supported in this cluster, fallback to legacy restore", zap.Error(err))
+ client.SetRewriteMode(restore.RewriteModeLegacy)
+ } else {
+ client.SetRewriteMode(restore.RewriteModeKeyspace)
+ }
+
+ err = client.LoadRestoreStores(ctx)
if err != nil {
return errors.Trace(err)
}
@@ -424,42 +479,6 @@ func CheckRestoreDBAndTable(client *restore.Client, cfg *RestoreConfig) error {
return nil
}
-func CheckNewCollationEnable(
- backupNewCollationEnable string,
- g glue.Glue,
- storage kv.Storage,
- CheckRequirements bool,
-) error {
- if backupNewCollationEnable == "" {
- if CheckRequirements {
- return errors.Annotatef(berrors.ErrUnknown,
- "the config 'new_collations_enabled_on_first_bootstrap' not found in backupmeta. "+
- "you can use \"show config WHERE name='new_collations_enabled_on_first_bootstrap';\" to manually check the config. "+
- "if you ensure the config 'new_collations_enabled_on_first_bootstrap' in backup cluster is as same as restore cluster, "+
- "use --check-requirements=false to skip this check")
- }
- log.Warn("the config 'new_collations_enabled_on_first_bootstrap' is not in backupmeta")
- return nil
- }
-
- se, err := g.CreateSession(storage)
- if err != nil {
- return errors.Trace(err)
- }
-
- newCollationEnable, err := se.GetGlobalVariable(tidbNewCollationEnabled)
- if err != nil {
- return errors.Trace(err)
- }
-
- if !strings.EqualFold(backupNewCollationEnable, newCollationEnable) {
- return errors.Annotatef(berrors.ErrUnknown,
- "the config 'new_collations_enabled_on_first_bootstrap' not match, upstream:%v, downstream: %v",
- backupNewCollationEnable, newCollationEnable)
- }
- return nil
-}
-
func isFullRestore(cmdName string) bool {
return cmdName == FullRestoreCmd
}
@@ -471,10 +490,20 @@ func IsStreamRestore(cmdName string) bool {
// RunRestore starts a restore task inside the current goroutine.
func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConfig) error {
+ if err := checkTaskExists(c, cfg); err != nil {
+ return errors.Annotate(err, "failed to check task exits")
+ }
+
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.KeyspaceName = cfg.KeyspaceName
+ })
if IsStreamRestore(cmdName) {
return RunStreamRestore(c, g, cmdName, cfg)
}
+ return runRestore(c, g, cmdName, cfg)
+}
+func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConfig) error {
cfg.Adjust()
defer summary.Summary(cmdName)
ctx, cancel := context.WithCancel(c)
@@ -494,6 +523,7 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
return errors.Trace(err)
}
defer mgr.Close()
+ codec := mgr.GetStorage().GetCodec()
mergeRegionSize := cfg.MergeSmallRegionSizeBytes
mergeRegionCount := cfg.MergeSmallRegionKeyCount
@@ -502,10 +532,7 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
// according to https://github.com/pingcap/tidb/issues/34167.
// we should get the real config from tikv to adapt the dynamic region.
httpCli := httputil.NewClient(mgr.GetTLSConfig())
- mergeRegionSize, mergeRegionCount, err = mgr.GetMergeRegionSizeAndCount(ctx, httpCli)
- if err != nil {
- return errors.Trace(err)
- }
+ mergeRegionSize, mergeRegionCount = mgr.GetMergeRegionSizeAndCount(ctx, httpCli)
}
keepaliveCfg.PermitWithoutStream = true
@@ -531,7 +558,7 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
return errors.Trace(versionErr)
}
}
- if err = CheckNewCollationEnable(backupMeta.GetNewCollationsEnabled(), g, mgr.GetStorage(), cfg.CheckRequirements); err != nil {
+ if err = restore.CheckNewCollationEnable(backupMeta.GetNewCollationsEnabled(), g, mgr.GetStorage(), cfg.CheckRequirements); err != nil {
return errors.Trace(err)
}
@@ -555,7 +582,7 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
g.Record(summary.RestoreDataSize, archiveSize)
//restore from tidb will fetch a general Size issue https://github.com/pingcap/tidb/issues/27247
g.Record("Size", archiveSize)
- restoreTS, err := client.GetTS(ctx)
+ restoreTS, err := client.GetTSWithRetry(ctx)
if err != nil {
return errors.Trace(err)
}
@@ -655,10 +682,36 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
errCh := make(chan error, 32)
tableStream := client.GoCreateTables(ctx, mgr.GetDomain(), tables, newTS, errCh)
+
if len(files) == 0 {
log.Info("no files, empty databases and tables are restored")
summary.SetSuccessStatus(true)
// don't return immediately, wait all pipeline done.
+ } else {
+ oldKeyspace, _, err := tikv.DecodeKey(files[0].GetStartKey(), backupMeta.ApiVersion)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ newKeyspace := codec.GetKeyspace()
+
+ // If the API V2 data occurs in the restore process, the cluster must
+ // support the keyspace rewrite mode.
+ if (len(oldKeyspace) > 0 || len(newKeyspace) > 0) && client.GetRewriteMode() == restore.RewriteModeLegacy {
+ return errors.Annotate(berrors.ErrRestoreModeMismatch, "cluster only supports legacy rewrite mode")
+ }
+
+ // Hijack the tableStream and rewrite the rewrite rules.
+ tableStream = util.ChanMap(tableStream, func(t restore.CreatedTable) restore.CreatedTable {
+ // Set the keyspace info for the checksum requests
+ t.RewriteRule.OldKeyspace = oldKeyspace
+ t.RewriteRule.NewKeyspace = newKeyspace
+
+ for _, rule := range t.RewriteRule.Data {
+ rule.OldKeyPrefix = append(append([]byte{}, oldKeyspace...), rule.OldKeyPrefix...)
+ rule.NewKeyPrefix = codec.EncodeKey(rule.NewKeyPrefix)
+ }
+ return t
+ })
}
if cfg.tiflashRecorder != nil {
diff --git a/br/pkg/task/restore_data.go b/br/pkg/task/restore_data.go
index f8e286dd0e72b..74663ea28d39d 100644
--- a/br/pkg/task/restore_data.go
+++ b/br/pkg/task/restore_data.go
@@ -19,6 +19,7 @@ import (
"github.com/pingcap/tidb/br/pkg/storage"
"github.com/pingcap/tidb/br/pkg/summary"
"github.com/pingcap/tidb/br/pkg/utils"
+ tidbconfig "github.com/pingcap/tidb/config"
"go.uber.org/zap"
)
@@ -71,6 +72,11 @@ func RunResolveKvData(c context.Context, g glue.Glue, cmdName string, cfg *Resto
defer mgr.Close()
keepaliveCfg.PermitWithoutStream = true
+ tc := tidbconfig.GetGlobalConfig()
+ tc.SkipRegisterToDashboard = true
+ tc.EnableGlobalKill = false
+ tidbconfig.StoreGlobalConfig(tc)
+
client := restore.NewRestoreClient(mgr.GetPDClient(), mgr.GetTLSConfig(), keepaliveCfg, false)
restoreTS, err := client.GetTS(ctx)
@@ -133,9 +139,9 @@ func RunResolveKvData(c context.Context, g glue.Glue, cmdName string, cfg *Resto
}
log.Debug("total tikv", zap.Int("total", numBackupStore), zap.String("progress file", cfg.ProgressFile))
- // progress = read meta + send recovery + iterate tikv + flashback.
- progress := g.StartProgress(ctx, cmdName, int64(numBackupStore*4), !cfg.LogProgress)
- go progressFileWriterRoutine(ctx, progress, int64(numBackupStore*4), cfg.ProgressFile)
+ // progress = read meta + send recovery + iterate tikv + (1 * prepareflashback + 1 * flashback)
+ progress := g.StartProgress(ctx, cmdName, int64(numBackupStore*3+2), !cfg.LogProgress)
+ go progressFileWriterRoutine(ctx, progress, int64(numBackupStore*3+2), cfg.ProgressFile)
// restore tikv data from a snapshot volume
var totalRegions int
@@ -153,7 +159,23 @@ func RunResolveKvData(c context.Context, g glue.Glue, cmdName string, cfg *Resto
//TODO: restore volume type into origin type
//ModifyVolume(*ec2.ModifyVolumeInput) (*ec2.ModifyVolumeOutput, error) by backupmeta
+ // this is used for cloud restoration
+ err = client.Init(g, mgr.GetStorage())
+ if err != nil {
+ return errors.Trace(err)
+ }
+ defer client.Close()
+ log.Info("start to clear system user for cloud")
+ err = client.ClearSystemUsers(ctx, cfg.ResetSysUsers)
+
+ if err != nil {
+ return errors.Trace(err)
+ }
+ // since we cannot reset tiflash automaticlly. so we should start it manually
+ if err = client.ResetTiFlashReplicas(ctx, g, mgr.GetStorage()); err != nil {
+ return errors.Trace(err)
+ }
progress.Close()
summary.CollectDuration("restore duration", time.Since(startAll))
summary.SetSuccessStatus(true)
diff --git a/br/pkg/task/restore_raw.go b/br/pkg/task/restore_raw.go
index 6c15cd9989512..7b80ac18b4d87 100644
--- a/br/pkg/task/restore_raw.go
+++ b/br/pkg/task/restore_raw.go
@@ -80,10 +80,7 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR
// according to https://github.com/pingcap/tidb/issues/34167.
// we should get the real config from tikv to adapt the dynamic region.
httpCli := httputil.NewClient(mgr.GetTLSConfig())
- mergeRegionSize, mergeRegionCount, err = mgr.GetMergeRegionSizeAndCount(ctx, httpCli)
- if err != nil {
- return errors.Trace(err)
- }
+ mergeRegionSize, mergeRegionCount = mgr.GetMergeRegionSizeAndCount(ctx, httpCli)
}
keepaliveCfg := GetKeepalive(&cfg.Config)
diff --git a/br/pkg/task/restore_test.go b/br/pkg/task/restore_test.go
index 94bbcb3c3692c..b13ecf0eccc08 100644
--- a/br/pkg/task/restore_test.go
+++ b/br/pkg/task/restore_test.go
@@ -63,6 +63,16 @@ func TestConfigureRestoreClient(t *testing.T) {
require.True(t, client.IsOnline())
}
+func TestAdjustRestoreConfigForStreamRestore(t *testing.T) {
+ restoreCfg := RestoreConfig{}
+
+ restoreCfg.adjustRestoreConfigForStreamRestore()
+ require.Equal(t, restoreCfg.PitrBatchCount, uint32(defaultPiTRBatchCount))
+ require.Equal(t, restoreCfg.PitrBatchSize, uint32(defaultPiTRBatchSize))
+ require.Equal(t, restoreCfg.PitrConcurrency, uint32(defaultPiTRConcurrency))
+ require.Equal(t, restoreCfg.Concurrency, restoreCfg.PitrConcurrency)
+}
+
func TestCheckRestoreDBAndTable(t *testing.T) {
cases := []struct {
cfgSchemas map[string]struct{}
diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go
index a259452b14b2d..40f851048da37 100644
--- a/br/pkg/task/stream.go
+++ b/br/pkg/task/stream.go
@@ -307,10 +307,7 @@ func NewStreamMgr(ctx context.Context, cfg *StreamConfig, g glue.Glue, isStreamS
mgr: mgr,
}
if isStreamStart {
- client, err := backup.NewBackupClient(ctx, mgr)
- if err != nil {
- return nil, errors.Trace(err)
- }
+ client := backup.NewBackupClient(ctx, mgr)
backend, err := storage.ParseBackend(cfg.Storage, &cfg.BackendOptions)
if err != nil {
@@ -427,7 +424,7 @@ func (s *streamMgr) backupFullSchemas(ctx context.Context, g glue.Glue) error {
}
schemasConcurrency := uint(mathutil.Min(backup.DefaultSchemaConcurrency, schemas.Len()))
- err = schemas.BackupSchemas(ctx, metaWriter, s.mgr.GetStorage(), nil,
+ err = schemas.BackupSchemas(ctx, metaWriter, nil, s.mgr.GetStorage(), nil,
s.cfg.StartTS, schemasConcurrency, 0, true, nil)
if err != nil {
return errors.Trace(err)
@@ -461,6 +458,39 @@ func (s *streamMgr) checkStreamStartEnable(g glue.Glue) error {
return nil
}
+type RestoreFunc func() error
+
+// KeepGcDisabled keeps GC disabled and return a function that used to gc enabled.
+// gc.ratio-threshold = "-1.0", which represents disable gc in TiKV.
+func KeepGcDisabled(g glue.Glue, store kv.Storage) (RestoreFunc, error) {
+ se, err := g.CreateSession(store)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ execCtx := se.GetSessionCtx().(sqlexec.RestrictedSQLExecutor)
+ oldRatio, err := utils.GetGcRatio(execCtx)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ newRatio := "-1.0"
+ err = utils.SetGcRatio(execCtx, newRatio)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ // If the oldRatio is negative, which is not normal status.
+ // It should set default value "1.1" after PiTR finished.
+ if strings.HasPrefix(oldRatio, "-") {
+ oldRatio = "1.1"
+ }
+
+ return func() error {
+ return utils.SetGcRatio(execCtx, oldRatio)
+ }, nil
+}
+
// RunStreamCommand run all kinds of `stream task`
func RunStreamCommand(
ctx context.Context,
@@ -838,8 +868,8 @@ func RunStreamAdvancer(c context.Context, g glue.Glue, cmdName string, cfg *Stre
return nil
}
-func checkConfigForStatus(cfg *StreamConfig) error {
- if len(cfg.PD) == 0 {
+func checkConfigForStatus(pd []string) error {
+ if len(pd) == 0 {
return errors.Annotatef(berrors.ErrInvalidArgument,
"the command needs access to PD, please specify `-u` or `--pd`")
}
@@ -889,7 +919,7 @@ func RunStreamStatus(
ctx = opentracing.ContextWithSpan(ctx, span1)
}
- if err := checkConfigForStatus(cfg); err != nil {
+ if err := checkConfigForStatus(cfg.PD); err != nil {
return err
}
ctl, err := makeStatusController(ctx, cfg, g)
@@ -1004,6 +1034,32 @@ func RunStreamTruncate(c context.Context, g glue.Glue, cmdName string, cfg *Stre
return nil
}
+// checkTaskExists checks whether there is a log backup task running.
+// If so, return an error.
+func checkTaskExists(ctx context.Context, cfg *RestoreConfig) error {
+ if err := checkConfigForStatus(cfg.PD); err != nil {
+ return err
+ }
+ etcdCLI, err := dialEtcdWithCfg(ctx, cfg.Config)
+ if err != nil {
+ return err
+ }
+ cli := streamhelper.NewMetaDataClient(etcdCLI)
+ defer func() {
+ if err := cli.Close(); err != nil {
+ log.Error("failed to close the etcd client", zap.Error(err))
+ }
+ }()
+ tasks, err := cli.GetAllTasks(ctx)
+ if err != nil {
+ return err
+ }
+ if len(tasks) > 0 {
+ return errors.Errorf("log backup task is running: %s, please stop the task before restore, and after PITR operation finished, create log-backup task again and create a full backup on this cluster", tasks[0].Info.Name)
+ }
+ return nil
+}
+
// RunStreamRestore restores stream log.
func RunStreamRestore(
c context.Context,
@@ -1065,7 +1121,7 @@ func RunStreamRestore(
logStorage := cfg.Config.Storage
cfg.Config.Storage = cfg.FullBackupStorage
// TiFlash replica is restored to down-stream on 'pitr' currently.
- if err = RunRestore(ctx, g, FullRestoreCmd, cfg); err != nil {
+ if err = runRestore(ctx, g, FullRestoreCmd, cfg); err != nil {
return errors.Trace(err)
}
cfg.Config.Storage = logStorage
@@ -1132,7 +1188,7 @@ func restoreStream(
}
defer client.Close()
- currentTS, err := client.GetTS(ctx)
+ currentTS, err := client.GetTSWithRetry(ctx)
if err != nil {
return errors.Trace(err)
}
@@ -1146,6 +1202,18 @@ func restoreStream(
// mode or emptied schedulers
defer restorePostWork(ctx, client, restoreSchedulers)
+ // It need disable GC in TiKV when PiTR.
+ // because the process of PITR is concurrent and kv events isn't sorted by tso.
+ restoreGc, err := KeepGcDisabled(g, mgr.GetStorage())
+ if err != nil {
+ return errors.Trace(err)
+ }
+ defer func() {
+ if err := restoreGc(); err != nil {
+ log.Error("failed to set gc enabled", zap.Error(err))
+ }
+ }()
+
err = client.InstallLogFileManager(ctx, cfg.StartTS, cfg.RestoreTS)
if err != nil {
return err
@@ -1200,10 +1268,17 @@ func restoreStream(
}
updateRewriteRules(rewriteRules, schemasReplace)
- dmlFiles, err := client.LoadDMLFiles(ctx)
+ logFilesIter, err := client.LoadDMLFiles(ctx)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ logFilesIterWithSplit, err := client.WrapLogFilesIterWithSplitHelper(logFilesIter, rewriteRules, g, mgr.GetStorage())
+ if err != nil {
+ return errors.Trace(err)
+ }
pd := g.StartProgress(ctx, "Restore KV Files", int64(dataFileCount), !cfg.LogProgress)
err = withProgress(pd, func(p glue.Progress) error {
- return client.RestoreKVFiles(ctx, rewriteRules, dmlFiles, updateStats, p.Inc)
+ return client.RestoreKVFiles(ctx, rewriteRules, logFilesIterWithSplit, cfg.PitrBatchCount, cfg.PitrBatchSize, updateStats, p.IncBy)
})
if err != nil {
return errors.Annotate(err, "failed to restore kv files")
@@ -1549,7 +1624,7 @@ func initRewriteRules(client *restore.Client, tables map[int64]*metautil.Table)
zap.Stringer("database", t.DB.Name),
zap.Int("old-id", int(t.Info.ID)),
zap.Array("rewrite-rules", zapcore.ArrayMarshalerFunc(func(ae zapcore.ArrayEncoder) error {
- for _, r := range rules {
+ for _, r := range tableRules {
for _, rule := range r.Data {
if err := ae.AppendObject(logutil.RewriteRuleObject(rule)); err != nil {
return err
diff --git a/br/pkg/utils/BUILD.bazel b/br/pkg/utils/BUILD.bazel
index 0ae948d18a779..498400fc53166 100644
--- a/br/pkg/utils/BUILD.bazel
+++ b/br/pkg/utils/BUILD.bazel
@@ -38,6 +38,7 @@ go_library(
"//util",
"//util/sqlexec",
"@com_github_cheggaaa_pb_v3//:pb",
+ "@com_github_docker_go_units//:go-units",
"@com_github_google_uuid//:uuid",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
@@ -50,6 +51,7 @@ go_library(
"@org_golang_google_grpc//backoff",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//credentials",
+ "@org_golang_google_grpc//credentials/insecure",
"@org_golang_google_grpc//keepalive",
"@org_golang_google_grpc//status",
"@org_golang_x_net//http/httpproxy",
@@ -79,6 +81,7 @@ go_test(
],
embed = [":utils"],
flaky = True,
+ shard_count = 30,
deps = [
"//br/pkg/errors",
"//br/pkg/metautil",
diff --git a/br/pkg/utils/backoff.go b/br/pkg/utils/backoff.go
index 5353c972d24ad..bff2490b56650 100644
--- a/br/pkg/utils/backoff.go
+++ b/br/pkg/utils/backoff.go
@@ -33,6 +33,11 @@ const (
resetTSRetryTimeExt = 600
resetTSWaitIntervalExt = 500 * time.Millisecond
resetTSMaxWaitIntervalExt = 300 * time.Second
+
+ // region heartbeat are 10 seconds by default, if some region has 2 heartbeat missing (15 seconds), it appear to be a network issue between PD and TiKV.
+ flashbackRetryTime = 3
+ flashbackWaitInterval = 3000 * time.Millisecond
+ flashbackMaxWaitInterval = 15 * time.Second
)
// RetryState is the mutable state needed for retrying.
@@ -204,3 +209,34 @@ func (bo *pdReqBackoffer) NextBackoff(err error) time.Duration {
func (bo *pdReqBackoffer) Attempt() int {
return bo.attempt
}
+
+type flashbackBackoffer struct {
+ attempt int
+ delayTime time.Duration
+ maxDelayTime time.Duration
+}
+
+// NewBackoffer creates a new controller regulating a truncated exponential backoff.
+func NewFlashBackBackoffer() Backoffer {
+ return &flashbackBackoffer{
+ attempt: flashbackRetryTime,
+ delayTime: flashbackWaitInterval,
+ maxDelayTime: flashbackMaxWaitInterval,
+ }
+}
+
+// retry 3 times when prepare flashback failure.
+func (bo *flashbackBackoffer) NextBackoff(err error) time.Duration {
+ bo.delayTime = 2 * bo.delayTime
+ bo.attempt--
+ log.Warn("region may not ready to serve, retry it...", zap.Error(err))
+
+ if bo.delayTime > bo.maxDelayTime {
+ return bo.maxDelayTime
+ }
+ return bo.delayTime
+}
+
+func (bo *flashbackBackoffer) Attempt() int {
+ return bo.attempt
+}
diff --git a/br/pkg/utils/db.go b/br/pkg/utils/db.go
index 23911fa6f0a93..060df603d16cb 100644
--- a/br/pkg/utils/db.go
+++ b/br/pkg/utils/db.go
@@ -5,16 +5,24 @@ package utils
import (
"context"
"database/sql"
+ "strconv"
"strings"
"sync"
+ "github.com/docker/go-units"
+ "github.com/pingcap/errors"
"github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/util/sqlexec"
"go.uber.org/zap"
)
+const (
+ tidbNewCollationEnabled = "new_collation_enabled"
+)
+
var (
// check sql.DB and sql.Conn implement QueryExecutor and DBExecutor
_ DBExecutor = &sql.DB{}
@@ -94,14 +102,113 @@ func IsLogBackupEnabled(ctx sqlexec.RestrictedSQLExecutor) (bool, error) {
return true, nil
}
-// CheckLogBackupTaskExist increases the count of log backup task.
+func GetRegionSplitInfo(ctx sqlexec.RestrictedSQLExecutor) (uint64, int64) {
+ return GetSplitSize(ctx), GetSplitKeys(ctx)
+}
+
+func GetSplitSize(ctx sqlexec.RestrictedSQLExecutor) uint64 {
+ const defaultSplitSize = 96 * 1024 * 1024
+ varStr := "show config where name = 'coprocessor.region-split-size' and type = 'tikv'"
+ rows, fields, err := ctx.ExecRestrictedSQL(
+ kv.WithInternalSourceType(context.Background(), kv.InternalTxnBR),
+ nil,
+ varStr,
+ )
+ if err != nil {
+ log.Warn("failed to get split size, use default value", logutil.ShortError(err))
+ return defaultSplitSize
+ }
+ if len(rows) == 0 {
+ // use the default value
+ return defaultSplitSize
+ }
+
+ d := rows[0].GetDatum(3, &fields[3].Column.FieldType)
+ splitSizeStr, err := d.ToString()
+ if err != nil {
+ log.Warn("failed to get split size, use default value", logutil.ShortError(err))
+ return defaultSplitSize
+ }
+ splitSize, err := units.FromHumanSize(splitSizeStr)
+ if err != nil {
+ log.Warn("failed to get split size, use default value", logutil.ShortError(err))
+ return defaultSplitSize
+ }
+ return uint64(splitSize)
+}
+
+func GetSplitKeys(ctx sqlexec.RestrictedSQLExecutor) int64 {
+ const defaultSplitKeys = 960000
+ varStr := "show config where name = 'coprocessor.region-split-keys' and type = 'tikv'"
+ rows, fields, err := ctx.ExecRestrictedSQL(
+ kv.WithInternalSourceType(context.Background(), kv.InternalTxnBR),
+ nil,
+ varStr,
+ )
+ if err != nil {
+ log.Warn("failed to get split keys, use default value", logutil.ShortError(err))
+ return defaultSplitKeys
+ }
+ if len(rows) == 0 {
+ // use the default value
+ return defaultSplitKeys
+ }
+
+ d := rows[0].GetDatum(3, &fields[3].Column.FieldType)
+ splitKeysStr, err := d.ToString()
+ if err != nil {
+ log.Warn("failed to get split keys, use default value", logutil.ShortError(err))
+ return defaultSplitKeys
+ }
+ splitKeys, err := strconv.ParseInt(splitKeysStr, 10, 64)
+ if err != nil {
+ log.Warn("failed to get split keys, use default value", logutil.ShortError(err))
+ return defaultSplitKeys
+ }
+ return splitKeys
+}
+
+func GetGcRatio(ctx sqlexec.RestrictedSQLExecutor) (string, error) {
+ valStr := "show config where name = 'gc.ratio-threshold' and type = 'tikv'"
+ rows, fields, errSQL := ctx.ExecRestrictedSQL(
+ kv.WithInternalSourceType(context.Background(), kv.InternalTxnBR),
+ nil,
+ valStr,
+ )
+ if errSQL != nil {
+ return "", errSQL
+ }
+ if len(rows) == 0 {
+ // no rows mean not support log backup.
+ return "", nil
+ }
+
+ d := rows[0].GetDatum(3, &fields[3].Column.FieldType)
+ return d.ToString()
+}
+
+func SetGcRatio(ctx sqlexec.RestrictedSQLExecutor, ratio string) error {
+ _, _, err := ctx.ExecRestrictedSQL(
+ kv.WithInternalSourceType(context.Background(), kv.InternalTxnBR),
+ nil,
+ "set config tikv `gc.ratio-threshold`=%?",
+ ratio,
+ )
+ if err != nil {
+ return errors.Annotatef(err, "failed to set config `gc.ratio-threshold`=%s", ratio)
+ }
+ log.Warn("set config tikv gc.ratio-threshold", zap.String("ratio", ratio))
+ return nil
+}
+
+// LogBackupTaskCountInc increases the count of log backup task.
func LogBackupTaskCountInc() {
LogBackupTaskMutex.Lock()
logBackupTaskCount++
LogBackupTaskMutex.Unlock()
}
-// CheckLogBackupTaskExist decreases the count of log backup task.
+// LogBackupTaskCountDec decreases the count of log backup task.
func LogBackupTaskCountDec() {
LogBackupTaskMutex.Lock()
logBackupTaskCount--
@@ -117,3 +224,8 @@ func CheckLogBackupTaskExist() bool {
func IsLogBackupInUse(ctx sessionctx.Context) bool {
return CheckLogBackupEnabled(ctx) && CheckLogBackupTaskExist()
}
+
+// GetTidbNewCollationEnabled returns the variable name of NewCollationEnabled.
+func GetTidbNewCollationEnabled() string {
+ return tidbNewCollationEnabled
+}
diff --git a/br/pkg/utils/db_test.go b/br/pkg/utils/db_test.go
index 08eac1e82594c..1004764b0d206 100644
--- a/br/pkg/utils/db_test.go
+++ b/br/pkg/utils/db_test.go
@@ -4,6 +4,7 @@ package utils_test
import (
"context"
+ "strings"
"testing"
"github.com/pingcap/errors"
@@ -35,7 +36,19 @@ func (m *mockRestrictedSQLExecutor) ExecRestrictedSQL(ctx context.Context, opts
if m.errHappen {
return nil, nil, errors.New("injected error")
}
- return m.rows, m.fields, nil
+
+ if strings.Contains(sql, "show config") {
+ return m.rows, m.fields, nil
+ } else if strings.Contains(sql, "set config") && strings.Contains(sql, "gc.ratio-threshold") {
+ value := args[0].(string)
+
+ for _, r := range m.rows {
+ d := types.Datum{}
+ d.SetString(value, "")
+ chunk.MutRow(r).SetDatum(3, d)
+ }
+ }
+ return nil, nil, nil
}
func TestIsLogBackupEnabled(t *testing.T) {
@@ -115,3 +128,84 @@ func TestCheckLogBackupTaskExist(t *testing.T) {
utils.LogBackupTaskCountDec()
require.False(t, utils.CheckLogBackupTaskExist())
}
+
+func TestGc(t *testing.T) {
+ // config format:
+ // MySQL [(none)]> show config where name = 'gc.ratio-threshold';
+ // +------+-------------------+--------------------+-------+
+ // | Type | Instance | Name | Value |
+ // +------+-------------------+--------------------+-------+
+ // | tikv | 172.16.6.46:3460 | gc.ratio-threshold | 1.1 |
+ // | tikv | 172.16.6.47:3460 | gc.ratio-threshold | 1.1 |
+ // +------+-------------------+--------------------+-------+
+ fields := make([]*ast.ResultField, 4)
+ tps := []*types.FieldType{
+ types.NewFieldType(mysql.TypeString),
+ types.NewFieldType(mysql.TypeString),
+ types.NewFieldType(mysql.TypeString),
+ types.NewFieldType(mysql.TypeString),
+ }
+ for i := 0; i < len(tps); i++ {
+ rf := new(ast.ResultField)
+ rf.Column = new(model.ColumnInfo)
+ rf.Column.FieldType = *tps[i]
+ fields[i] = rf
+ }
+ rows := make([]chunk.Row, 0, 2)
+ row := chunk.MutRowFromValues("tikv", " 127.0.0.1:20161", "log-backup.enable", "1.1").ToRow()
+ rows = append(rows, row)
+ row = chunk.MutRowFromValues("tikv", " 127.0.0.1:20162", "log-backup.enable", "1.1").ToRow()
+ rows = append(rows, row)
+
+ s := &mockRestrictedSQLExecutor{rows: rows, fields: fields}
+ ratio, err := utils.GetGcRatio(s)
+ require.Nil(t, err)
+ require.Equal(t, ratio, "1.1")
+
+ err = utils.SetGcRatio(s, "-1.0")
+ require.Nil(t, err)
+ ratio, err = utils.GetGcRatio(s)
+ require.Nil(t, err)
+ require.Equal(t, ratio, "-1.0")
+}
+
+func TestRegionSplitInfo(t *testing.T) {
+ // config format:
+ // MySQL [(none)]> show config where name = 'coprocessor.region-split-size';
+ // +------+-------------------+-------------------------------+-------+
+ // | Type | Instance | Name | Value |
+ // +------+-------------------+-------------------------------+-------+
+ // | tikv | 127.0.0.1:20161 | coprocessor.region-split-size | 10MB |
+ // +------+-------------------+-------------------------------+-------+
+ // MySQL [(none)]> show config where name = 'coprocessor.region-split-keys';
+ // +------+-------------------+-------------------------------+--------+
+ // | Type | Instance | Name | Value |
+ // +------+-------------------+-------------------------------+--------+
+ // | tikv | 127.0.0.1:20161 | coprocessor.region-split-keys | 100000 |
+ // +------+-------------------+-------------------------------+--------+
+
+ fields := make([]*ast.ResultField, 4)
+ tps := []*types.FieldType{
+ types.NewFieldType(mysql.TypeString),
+ types.NewFieldType(mysql.TypeString),
+ types.NewFieldType(mysql.TypeString),
+ types.NewFieldType(mysql.TypeString),
+ }
+ for i := 0; i < len(tps); i++ {
+ rf := new(ast.ResultField)
+ rf.Column = new(model.ColumnInfo)
+ rf.Column.FieldType = *tps[i]
+ fields[i] = rf
+ }
+ rows := make([]chunk.Row, 0, 1)
+ row := chunk.MutRowFromValues("tikv", "127.0.0.1:20161", "coprocessor.region-split-size", "10MB").ToRow()
+ rows = append(rows, row)
+ s := &mockRestrictedSQLExecutor{rows: rows, fields: fields}
+ require.Equal(t, utils.GetSplitSize(s), uint64(10000000))
+
+ rows = make([]chunk.Row, 0, 1)
+ row = chunk.MutRowFromValues("tikv", "127.0.0.1:20161", "coprocessor.region-split-keys", "100000").ToRow()
+ rows = append(rows, row)
+ s = &mockRestrictedSQLExecutor{rows: rows, fields: fields}
+ require.Equal(t, utils.GetSplitKeys(s), int64(100000))
+}
diff --git a/br/pkg/utils/iter/BUILD.bazel b/br/pkg/utils/iter/BUILD.bazel
index 0e4c55ed67d56..eecf8e82e1cd5 100644
--- a/br/pkg/utils/iter/BUILD.bazel
+++ b/br/pkg/utils/iter/BUILD.bazel
@@ -20,6 +20,7 @@ go_library(
go_test(
name = "iter_test",
+ timeout = "short",
srcs = ["combinator_test.go"],
flaky = True,
race = "on",
diff --git a/br/pkg/utils/json.go b/br/pkg/utils/json.go
index 736032b4beedc..a29725db6b1b3 100644
--- a/br/pkg/utils/json.go
+++ b/br/pkg/utils/json.go
@@ -114,6 +114,10 @@ func makeJSONSchema(schema *backuppb.Schema) (*jsonSchema, error) {
func fromJSONSchema(jSchema *jsonSchema) (*backuppb.Schema, error) {
schema := jSchema.Schema
+ if schema == nil {
+ schema = &backuppb.Schema{}
+ }
+
var err error
schema.Db, err = json.Marshal(jSchema.DB)
if err != nil {
diff --git a/br/pkg/utils/json_test.go b/br/pkg/utils/json_test.go
index 8d8eeb6457332..3f03f287d92f1 100644
--- a/br/pkg/utils/json_test.go
+++ b/br/pkg/utils/json_test.go
@@ -204,6 +204,44 @@ var testMetaJSONs = [][]byte{
"is_raw_kv": true,
"br_version": "BR\nRelease Version: v5.0.0-master\nGit Commit Hash: c0d60dae4998cf9ac40f02e5444731c15f0b2522\nGit Branch: HEAD\nGo Version: go1.13.4\nUTC Build Time: 2021-03-25 08:10:08\nRace Enabled: false"
}`),
+ []byte(`{
+ "files": [
+ {
+ "sha256": "3ae857ef9b379d498ae913434f1d47c3e90a55f3a4cd9074950bfbd163d5e5fc",
+ "start_key": "7480000000000000115f720000000000000000",
+ "end_key": "7480000000000000115f72ffffffffffffffff00",
+ "name": "1_20_9_36adb8cedcd7af34708edff520499e712e2cfdcb202f5707dc9305a031d55a98_1675066275424_write.sst",
+ "end_version": 439108573623222300,
+ "crc64xor": 16261462091570213000,
+ "total_kvs": 15,
+ "total_bytes": 1679,
+ "cf": "write",
+ "size": 2514,
+ "cipher_iv": "56MTbxA4CaNILpirKnBxUw=="
+ }
+ ],
+ "schemas": [
+ {
+ "db": {
+ "charset": "utf8mb4",
+ "collate": "utf8mb4_bin",
+ "db_name": {
+ "L": "test",
+ "O": "test"
+ },
+ "id": 1,
+ "policy_ref_info": null,
+ "state": 5
+ }
+ }
+ ],
+ "ddls": [],
+ "cluster_id": 7194351714070942000,
+ "cluster_version": "\"6.1.0\"\n",
+ "br_version": "BR\nRelease Version: v6.1.0\nGit Commit Hash: 1a89decdb192cbdce6a7b0020d71128bc964d30f\nGit Branch: heads/refs/tags/v6.1.0\nGo Version: go1.18.2\nUTC Build Time: 2022-06-05 05:09:12\nRace Enabled: false",
+ "end_version": 439108573623222300,
+ "new_collations_enabled": "True"
+ }`),
}
func TestEncodeAndDecode(t *testing.T) {
diff --git a/br/pkg/utils/key.go b/br/pkg/utils/key.go
index 062f4b5aac52d..62d194ca57a2e 100644
--- a/br/pkg/utils/key.go
+++ b/br/pkg/utils/key.go
@@ -163,7 +163,7 @@ func CloneSlice[T any](s []T) []T {
// toClampIn: |_____| |____| |________________|
// result: |_____| |_| |______________|
// we are assuming the arguments are sorted by the start key and no overlaps.
-// you can call CollapseRanges to get key ranges fits this requirements.
+// you can call spans.Collapse to get key ranges fits this requirements.
// Note: this algorithm is pretty like the `checkIntervalIsSubset`, can we get them together?
func IntersectAll(s1 []kv.KeyRange, s2 []kv.KeyRange) []kv.KeyRange {
currentClamping := 0
diff --git a/br/pkg/utils/misc.go b/br/pkg/utils/misc.go
index 8078802b2e055..ab62a7b7db534 100644
--- a/br/pkg/utils/misc.go
+++ b/br/pkg/utils/misc.go
@@ -23,6 +23,7 @@ import (
"github.com/pingcap/tidb/parser/types"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
)
// IsTypeCompatible checks whether type target is compatible with type src
@@ -84,7 +85,7 @@ func IsTypeCompatible(src types.FieldType, target types.FieldType) bool {
}
func GRPCConn(ctx context.Context, storeAddr string, tlsConf *tls.Config, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
- secureOpt := grpc.WithInsecure()
+ secureOpt := grpc.WithTransportCredentials(insecure.NewCredentials())
if tlsConf != nil {
secureOpt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConf))
}
diff --git a/br/pkg/utils/schema.go b/br/pkg/utils/schema.go
index 8ceba24e140ad..0987e446bfc49 100644
--- a/br/pkg/utils/schema.go
+++ b/br/pkg/utils/schema.go
@@ -16,6 +16,7 @@ import (
// temporaryDBNamePrefix is the prefix name of system db, e.g. mysql system db will be rename to __TiDB_BR_Temporary_mysql
const temporaryDBNamePrefix = "__TiDB_BR_Temporary_"
+const temporarySysDB = temporaryDBNamePrefix + "mysql"
// NeedAutoID checks whether the table needs backing up with an autoid.
func NeedAutoID(tblInfo *model.TableInfo) bool {
@@ -96,6 +97,11 @@ func EncloseDBAndTable(database, table string) string {
return fmt.Sprintf("%s.%s", EncloseName(database), EncloseName(table))
}
+// IsTemplateSysDB checks wheterh the dbname is temporary system database(__TiDB_BR_Temporary_mysql).
+func IsTemplateSysDB(dbname model.CIStr) bool {
+ return dbname.O == temporarySysDB
+}
+
// IsSysDB tests whether the database is system DB.
// Currently, the only system DB is mysql.
func IsSysDB(dbLowerName string) bool {
diff --git a/br/pkg/utils/store_manager.go b/br/pkg/utils/store_manager.go
index 8dd50a2a57c82..8a89e49022806 100644
--- a/br/pkg/utils/store_manager.go
+++ b/br/pkg/utils/store_manager.go
@@ -20,6 +20,7 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/backoff"
"google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/keepalive"
)
@@ -136,7 +137,7 @@ func (mgr *StoreManager) getGrpcConnLocked(ctx context.Context, storeID uint64)
if err != nil {
return nil, errors.Trace(err)
}
- opt := grpc.WithInsecure()
+ opt := grpc.WithTransportCredentials(insecure.NewCredentials())
if mgr.tlsConf != nil {
opt = grpc.WithTransportCredentials(credentials.NewTLS(mgr.tlsConf))
}
diff --git a/br/pkg/version/BUILD.bazel b/br/pkg/version/BUILD.bazel
index 8171081ae5df1..7a15014e378e6 100644
--- a/br/pkg/version/BUILD.bazel
+++ b/br/pkg/version/BUILD.bazel
@@ -10,7 +10,6 @@ go_library(
"//br/pkg/logutil",
"//br/pkg/utils",
"//br/pkg/version/build",
- "//sessionctx/variable",
"//util/engine",
"@com_github_coreos_go_semver//semver",
"@com_github_pingcap_errors//:errors",
@@ -29,7 +28,6 @@ go_test(
flaky = True,
deps = [
"//br/pkg/version/build",
- "//sessionctx/variable",
"@com_github_coreos_go_semver//semver",
"@com_github_data_dog_go_sqlmock//:go-sqlmock",
"@com_github_pingcap_kvproto//pkg/metapb",
diff --git a/br/pkg/version/version.go b/br/pkg/version/version.go
index ba3551f58b463..7bd629fb2c540 100644
--- a/br/pkg/version/version.go
+++ b/br/pkg/version/version.go
@@ -18,7 +18,6 @@ import (
"github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/br/pkg/version/build"
- "github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/util/engine"
pd "github.com/tikv/pd/client"
"go.uber.org/zap"
@@ -32,6 +31,10 @@ var (
compatibleTiFlashMajor4 = semver.New("4.0.0")
versionHash = regexp.MustCompile("-[0-9]+-g[0-9a-f]{7,}")
+
+ checkpointSupportError error = nil
+ // pitrSupportBatchKVFiles specifies whether TiKV-server supports batch PITR.
+ pitrSupportBatchKVFiles bool = false
)
// NextMajorVersion returns the next major version.
@@ -134,6 +137,12 @@ func CheckVersionForBRPiTR(s *metapb.Store, tikvVersion *semver.Version) error {
return errors.Annotatef(berrors.ErrVersionMismatch, "TiKV node %s version %s is too low when use PiTR, please update tikv's version to at least v6.1.0(v6.2.0+ recommanded)",
s.Address, tikvVersion)
}
+ // If tikv version < 6.5, PITR do not support restoring batch kv files.
+ if tikvVersion.Major < 6 || (tikvVersion.Major == 6 && tikvVersion.Minor < 5) {
+ pitrSupportBatchKVFiles = false
+ } else {
+ pitrSupportBatchKVFiles = true
+ }
// The versions of BR and TiKV should be the same when use BR 6.1.0
if BRVersion.Major == 6 && BRVersion.Minor == 1 {
@@ -148,7 +157,6 @@ func CheckVersionForBRPiTR(s *metapb.Store, tikvVersion *semver.Version) error {
s.Address, tikvVersion, build.ReleaseVersion)
}
}
-
return nil
}
@@ -157,9 +165,16 @@ func CheckVersionForDDL(s *metapb.Store, tikvVersion *semver.Version) error {
// use tikvVersion instead of tidbVersion since br doesn't have mysql client to connect tidb.
requireVersion := semver.New("6.2.0-alpha")
if tikvVersion.Compare(*requireVersion) < 0 {
- log.Info("detected the old version of tidb cluster. set enable concurrent ddl to false")
- variable.EnableConcurrentDDL.Store(false)
- return nil
+ return errors.Errorf("detected the old version of tidb cluster, require: >= 6.2.0, but got %s", tikvVersion.String())
+ }
+ return nil
+}
+
+// CheckVersionForKeyspaceBR checks whether the cluster is support Backup/Restore keyspace data.
+func CheckVersionForKeyspaceBR(_ *metapb.Store, tikvVersion *semver.Version) error {
+ requireVersion := semver.New("6.6.0-alpha")
+ if tikvVersion.Compare(*requireVersion) < 0 {
+ return errors.Errorf("detected the old version of tidb cluster, require: >= 6.6.0, but got %s", tikvVersion.String())
}
return nil
}
@@ -199,6 +214,14 @@ func CheckVersionForBR(s *metapb.Store, tikvVersion *semver.Version) error {
}
}
+ // reset the checkpoint support error
+ checkpointSupportError = nil
+ if tikvVersion.Major < 6 || (tikvVersion.Major == 6 && tikvVersion.Minor < 5) {
+ // checkpoint mode only support after v6.5.0
+ checkpointSupportError = errors.Annotatef(berrors.ErrVersionMismatch, "TiKV node %s version %s is too low when use checkpoint, please update tikv's version to at least v6.5.0",
+ s.Address, tikvVersion)
+ }
+
// don't warn if we are the master build, which always have the version v4.0.0-beta.2-*
if build.GitBranch != "master" && tikvVersion.Compare(*BRVersion) > 0 {
log.Warn(fmt.Sprintf("BR version is outdated, please consider use version %s of BR", tikvVersion))
@@ -306,6 +329,14 @@ func FetchVersion(ctx context.Context, db utils.QueryExecutor) (string, error) {
return versionInfo, nil
}
+func CheckCheckpointSupport() error {
+ return checkpointSupportError
+}
+
+func CheckPITRSupportBatchKVFiles() bool {
+ return pitrSupportBatchKVFiles
+}
+
type ServerType int
const (
diff --git a/br/pkg/version/version_test.go b/br/pkg/version/version_test.go
index f70a2074be0ec..927eeee119d5b 100644
--- a/br/pkg/version/version_test.go
+++ b/br/pkg/version/version_test.go
@@ -13,7 +13,6 @@ import (
"github.com/coreos/go-semver/semver"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/tidb/br/pkg/version/build"
- "github.com/pingcap/tidb/sessionctx/variable"
"github.com/stretchr/testify/require"
pd "github.com/tikv/pd/client"
)
@@ -76,6 +75,15 @@ func TestCheckClusterVersion(t *testing.T) {
require.Regexp(t, `^TiKV .* version mismatch when use PiTR v6.2.0\+, please `, err.Error())
}
+ {
+ // Default value of `pitrSupportBatchKVFiles` should be `false`.
+ build.ReleaseVersion = "v6.5.0"
+ mock.getAllStores = func() []*metapb.Store {
+ return []*metapb.Store{{Version: `v6.2.0`}}
+ }
+ require.Equal(t, CheckPITRSupportBatchKVFiles(), false)
+ }
+
{
build.ReleaseVersion = "v6.2.0"
mock.getAllStores = func() []*metapb.Store {
@@ -83,6 +91,29 @@ func TestCheckClusterVersion(t *testing.T) {
}
err := CheckClusterVersion(context.Background(), &mock, CheckVersionForBRPiTR)
require.NoError(t, err)
+ require.Equal(t, CheckPITRSupportBatchKVFiles(), false)
+ }
+
+ {
+ pitrSupportBatchKVFiles = true
+ build.ReleaseVersion = "v6.2.0"
+ mock.getAllStores = func() []*metapb.Store {
+ return []*metapb.Store{{Version: `v6.4.0`}}
+ }
+ err := CheckClusterVersion(context.Background(), &mock, CheckVersionForBRPiTR)
+ require.NoError(t, err)
+ require.Equal(t, CheckPITRSupportBatchKVFiles(), false)
+ }
+
+ {
+ pitrSupportBatchKVFiles = true
+ build.ReleaseVersion = "v6.2.0"
+ mock.getAllStores = func() []*metapb.Store {
+ return []*metapb.Store{{Version: `v6.5.0`}}
+ }
+ err := CheckClusterVersion(context.Background(), &mock, CheckVersionForBRPiTR)
+ require.NoError(t, err)
+ require.Equal(t, CheckPITRSupportBatchKVFiles(), true)
}
{
@@ -205,6 +236,29 @@ func TestCheckClusterVersion(t *testing.T) {
}
err := CheckClusterVersion(context.Background(), &mock, CheckVersionForBR)
require.NoError(t, err)
+ require.Error(t, CheckCheckpointSupport())
+ }
+
+ {
+ build.ReleaseVersion = "v6.0.0-rc.2"
+ mock.getAllStores = func() []*metapb.Store {
+ // TiKV v6.0.0-rc.1 with BR v6.0.0-rc.2 is ok
+ return []*metapb.Store{{Version: "v6.0.0-rc.1"}}
+ }
+ err := CheckClusterVersion(context.Background(), &mock, CheckVersionForBR)
+ require.NoError(t, err)
+ require.Error(t, CheckCheckpointSupport())
+ }
+
+ {
+ build.ReleaseVersion = "v6.5.0-rc.2"
+ mock.getAllStores = func() []*metapb.Store {
+ // TiKV v6.5.0-rc.1 with BR v6.5.0-rc.2 is ok
+ return []*metapb.Store{{Version: "v6.5.0-rc.1"}}
+ }
+ err := CheckClusterVersion(context.Background(), &mock, CheckVersionForBR)
+ require.NoError(t, err)
+ require.NoError(t, CheckCheckpointSupport())
}
{
@@ -266,50 +320,40 @@ func TestCheckClusterVersion(t *testing.T) {
mock.getAllStores = func() []*metapb.Store {
return []*metapb.Store{{Version: "v6.4.0"}}
}
- originVal := variable.EnableConcurrentDDL.Load()
err := CheckClusterVersion(context.Background(), &mock, CheckVersionForDDL)
require.NoError(t, err)
- require.Equal(t, originVal, variable.EnableConcurrentDDL.Load())
}
{
mock.getAllStores = func() []*metapb.Store {
return []*metapb.Store{{Version: "v6.2.0"}}
}
- originVal := variable.EnableConcurrentDDL.Load()
err := CheckClusterVersion(context.Background(), &mock, CheckVersionForDDL)
require.NoError(t, err)
- require.Equal(t, originVal, variable.EnableConcurrentDDL.Load())
}
{
mock.getAllStores = func() []*metapb.Store {
return []*metapb.Store{{Version: "v6.2.0-alpha"}}
}
- originVal := variable.EnableConcurrentDDL.Load()
err := CheckClusterVersion(context.Background(), &mock, CheckVersionForDDL)
require.NoError(t, err)
- require.Equal(t, originVal, variable.EnableConcurrentDDL.Load())
}
{
mock.getAllStores = func() []*metapb.Store {
return []*metapb.Store{{Version: "v6.1.0"}}
}
- variable.EnableConcurrentDDL.Store(true)
err := CheckClusterVersion(context.Background(), &mock, CheckVersionForDDL)
- require.NoError(t, err)
- require.False(t, variable.EnableConcurrentDDL.Load())
+ require.Error(t, err)
}
{
mock.getAllStores = func() []*metapb.Store {
return []*metapb.Store{{Version: "v5.4.0"}}
}
- variable.EnableConcurrentDDL.Store(true)
err := CheckClusterVersion(context.Background(), &mock, CheckVersionForDDL)
- require.NoError(t, err)
- require.False(t, variable.EnableConcurrentDDL.Load())
+ require.Error(t, err)
}
}
diff --git a/br/tests/br_backup_empty/run.sh b/br/tests/br_backup_empty/run.sh
index 41ca818139ba4..0289f1d401068 100644
--- a/br/tests/br_backup_empty/run.sh
+++ b/br/tests/br_backup_empty/run.sh
@@ -67,6 +67,7 @@ run_sql "CREATE TABLE ${DB}1.usertable1 ( \
PRIMARY KEY (YCSB_KEY) \
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
+# backup empty table
echo "backup empty table start..."
run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/empty_table"
@@ -75,6 +76,7 @@ while [ $i -le $DB_COUNT ]; do
i=$(($i+1))
done
+# restore empty table.
echo "restore empty table start..."
run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/empty_table"
@@ -85,3 +87,15 @@ while [ $i -le $DB_COUNT ]; do
run_sql "DROP DATABASE $DB$i;"
i=$(($i+1))
done
+
+
+# backup, skip temporary system database(__TiDB_BR_Temporary_mysql) when backup
+run_sql "CREATE DATABASE __TiDB_BR_Temporary_mysql";
+run_sql "CREATE TABLE __TiDB_BR_Temporary_mysql.tables_priv(id int);";
+echo "backup and skip __TiDB_BR_Temporary_mysql start..."
+run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/skip_temporary_mysql"
+
+# restore successfully without panic.
+run_sql "DROP DATABASE __TiDB_BR_Temporary_mysql";
+echo "restore the data start..."
+run_br restore full -s "local://$TEST_DIR/skip_temporary_mysql" --pd $PD_ADDR --ratelimit 1024
diff --git a/br/tests/br_foreign_key/run.sh b/br/tests/br_foreign_key/run.sh
new file mode 100644
index 0000000000000..eafd38a180d40
--- /dev/null
+++ b/br/tests/br_foreign_key/run.sh
@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+DB="$TEST_NAME"
+
+run_sql "set @@global.foreign_key_checks=1;"
+run_sql "set @@foreign_key_checks=1;"
+run_sql "create schema $DB;"
+run_sql "create table $DB.t1 (id int key);"
+run_sql "create table $DB.t2 (id int key, a int, b int, foreign key fk_1 (a) references t1(id) ON UPDATE SET NULL ON DELETE SET NULL, foreign key fk_2 (b) references t1(id) ON DELETE CASCADE ON UPDATE CASCADE);"
+run_sql "insert into $DB.t1 values (1), (2), (3);"
+run_sql "insert into $DB.t2 values (1, 1, 1), (2, 2, 2), (3, 3, 3);"
+run_sql "update $DB.t1 set id=id+10 where id in (1, 3);"
+run_sql "delete from $DB.t1 where id = 2;"
+
+echo "backup start..."
+run_br backup db --db "$DB" -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+
+run_sql "drop schema $DB;"
+
+echo "restore start..."
+run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+
+set -x
+
+run_sql "select count(*) from $DB.t1;"
+check_contains 'count(*): 2'
+
+run_sql "select count(*) from $DB.t2;"
+check_contains 'count(*): 2'
+
+run_sql "select id, a, b from $DB.t2;"
+check_contains 'id: 1'
+check_contains 'id: 3'
+check_contains 'a: NULL'
+check_contains 'b: 11'
+check_contains 'b: 13'
+
+run_sql "drop schema $DB"
diff --git a/br/tests/br_full_cluster_restore/run.sh b/br/tests/br_full_cluster_restore/run.sh
index e75b4d49fc914..14074e61f3825 100644
--- a/br/tests/br_full_cluster_restore/run.sh
+++ b/br/tests/br_full_cluster_restore/run.sh
@@ -55,7 +55,8 @@ restart_services
# mock incompatible manually
run_sql "alter table mysql.user add column xx int;"
run_br restore full --with-sys-table --log-file $br_log_file -s "local://$backup_dir" > $res_file 2>&1 || true
-check_contains "the target cluster is not compatible with the backup data"
+run_sql "select count(*) from mysql.user"
+check_contains "count(*): 6"
echo "--> incompatible system table: less column on target cluster"
restart_services
diff --git a/br/tests/br_replica_read/placement_rule_with_learner_template.json b/br/tests/br_replica_read/placement_rule_with_learner_template.json
new file mode 100644
index 0000000000000..4c0caa5820f27
--- /dev/null
+++ b/br/tests/br_replica_read/placement_rule_with_learner_template.json
@@ -0,0 +1,25 @@
+[
+ {
+ "group_id": "pd",
+ "group_index": 0,
+ "group_override": false,
+ "rules": [
+ {
+ "group_id": "pd",
+ "id": "default",
+ "start_key": "",
+ "end_key": "",
+ "role": "voter",
+ "count": 2
+ },
+ {
+ "group_id": "pd",
+ "id": "learner",
+ "start_key": "",
+ "end_key": "",
+ "role": "learner",
+ "count": 1
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git a/br/tests/br_replica_read/run.sh b/br/tests/br_replica_read/run.sh
new file mode 100755
index 0000000000000..c56f54b4fbef8
--- /dev/null
+++ b/br/tests/br_replica_read/run.sh
@@ -0,0 +1,89 @@
+#!/bin/sh
+#
+# Copyright 2023 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+DB="$TEST_NAME"
+
+VOTER_COUNT=$((TIKV_COUNT-1))
+if [ "$VOTER_COUNT" -lt "1" ];then
+ echo "Skip test because there is no enough tikv"
+ exit 0
+fi
+
+# set random store to read only
+random_store_id=$(run_pd_ctl -u https://$PD_ADDR store | jq 'first(.stores[]|select(.store.labels|(.!= null and any(.key == "engine" and .value=="tiflash"))| not)|.store.id)')
+echo "random store id: $random_store_id"
+run_pd_ctl -u https://$PD_ADDR store label $random_store_id '$mode' 'read_only'
+
+# set placement rule to add a learner replica for each region in the read only store
+run_pd_ctl -u https://$PD_ADDR config placement-rules rule-bundle load --out=$TEST_DIR/default_rules.json
+cat tests/br_replica_read/placement_rule_with_learner_template.json | jq ".[].rules[0].count = $VOTER_COUNT" > $TEST_DIR/placement_rule_with_learner.json
+run_pd_ctl -u https://$PD_ADDR config placement-rules rule-bundle save --in $TEST_DIR/placement_rule_with_learner.json
+sleep 3 # wait for PD to apply the placement rule
+
+run_sql "CREATE DATABASE $DB;"
+
+run_sql "CREATE TABLE $DB.usertable1 ( \
+ YCSB_KEY varchar(64) NOT NULL, \
+ FIELD0 varchar(1) DEFAULT NULL, \
+ PRIMARY KEY (YCSB_KEY) \
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
+
+run_sql "INSERT INTO $DB.usertable1 VALUES (\"a\", \"b\");"
+run_sql "INSERT INTO $DB.usertable1 VALUES (\"aa\", \"b\");"
+
+run_sql "CREATE TABLE $DB.usertable2 ( \
+ YCSB_KEY varchar(64) NOT NULL, \
+ FIELD0 varchar(1) DEFAULT NULL, \
+ PRIMARY KEY (YCSB_KEY) \
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
+
+run_sql "INSERT INTO $DB.usertable2 VALUES (\"c\", \"d\");"
+
+# backup db
+echo "backup start..."
+run_br -u https://$PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB" --replica-read-label '$mode:read_only'
+
+run_sql "DROP DATABASE $DB;"
+
+# restore db
+echo "restore start..."
+run_br restore db --db $DB -s "local://$TEST_DIR/$DB" -u https://$PD_ADDR
+
+run_sql "select count(*) from $DB.usertable1;"
+table1_count=$(read_result)
+echo "table1 count: $table1_count"
+if [ "$table1_count" -ne "2" ];then
+ echo "TEST: [$TEST_NAME] failed!"
+ exit 1
+fi
+
+run_sql "select count(*) from $DB.usertable2;"
+table2_count=$(read_result)
+echo "table2 count: $table2_count"
+if [ "$table2_count" -ne "1" ];then
+ echo "TEST: [$TEST_NAME] failed!"
+ exit 1
+fi
+
+# Test BR DDL query string
+echo "testing DDL query..."
+run_curl https://$TIDB_STATUS_ADDR/ddl/history | grep -E '/\*from\(br\)\*/CREATE TABLE'
+run_curl https://$TIDB_STATUS_ADDR/ddl/history | grep -E '/\*from\(br\)\*/CREATE DATABASE'
+
+run_sql "DROP DATABASE $DB;"
+run_pd_ctl -u https://$PD_ADDR store label $random_store_id '$mode' ''
+run_pd_ctl -u https://$PD_ADDR config placement-rules rule-bundle save --in $TEST_DIR/default_rules.json
\ No newline at end of file
diff --git a/br/tests/br_restore_log_task_enable/run.sh b/br/tests/br_restore_log_task_enable/run.sh
new file mode 100644
index 0000000000000..923f8fe7c2b33
--- /dev/null
+++ b/br/tests/br_restore_log_task_enable/run.sh
@@ -0,0 +1,56 @@
+#!/bin/sh
+#
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+DB="$TEST_NAME"
+TABLE="usertable"
+
+# start log task
+run_br log start --task-name 1234 -s "local://$TEST_DIR/$DB/log" --pd $PD_ADDR
+
+run_sql "CREATE DATABASE $DB;"
+run_sql "CREATE TABLE $DB.$TABLE (id int);"
+run_sql "INSERT INTO $DB.$TABLE VALUES (1), (2), (3);"
+
+# backup full
+run_br backup full -s "local://$TEST_DIR/$DB/full" --pd $PD_ADDR
+
+# clean db
+run_sql "DROP DATABASE $DB;"
+
+# restore full (should be failed)
+run_br restore full -s "local://$TEST_DIR/$DB/full" --pd $PD_ADDR && exit 1
+
+# restore point (should be failed)
+run_br restore point -s "local://$TEST_DIR/$DB/log" --pd $PD_ADDR && exit 1
+
+# pause log task
+run_br log pause --task-name 1234 --pd $PD_ADDR
+
+# restore full (should be failed)
+run_br restore full -s "local://$TEST_DIR/$DB/full" --pd $PD_ADDR && exit 1
+
+# restore point (should be failed)
+run_br restore point -s "local://$TEST_DIR/$DB/log" --pd $PD_ADDR && exit 1
+
+# stop log task
+run_br log stop --task-name 1234 --pd $PD_ADDR
+
+# restore full (should be success)
+run_br restore full -s "local://$TEST_DIR/$DB/full" --pd $PD_ADDR
+
+# clean db
+run_sql "DROP DATABASE $DB"
diff --git a/br/tests/br_ttl/run.sh b/br/tests/br_ttl/run.sh
new file mode 100644
index 0000000000000..cfb1a38c8281b
--- /dev/null
+++ b/br/tests/br_ttl/run.sh
@@ -0,0 +1,54 @@
+#!/bin/sh
+#
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+DB="$TEST_NAME"
+
+PROGRESS_FILE="$TEST_DIR/progress_file"
+BACKUPMETAV1_LOG="$TEST_DIR/backup.log"
+BACKUPMETAV2_LOG="$TEST_DIR/backupv2.log"
+RESTORE_LOG="$TEST_DIR/restore.log"
+rm -rf $PROGRESS_FILE
+
+run_sql "create schema $DB;"
+run_sql "create table $DB.ttl_test_tbl(id int primary key, t datetime) TTL=\`t\` + interval 1 day TTL_ENABLE='ON'"
+
+# backup db
+echo "full backup meta v2 start..."
+unset BR_LOG_TO_TERM
+rm -f $BACKUPMETAV2_LOG
+run_br backup full --log-file $BACKUPMETAV2_LOG -s "local://$TEST_DIR/${DB}v2" --pd $PD_ADDR --use-backupmeta-v2
+
+echo "full backup meta v1 start..."
+rm -f $BACKUPMETAV1_LOG
+run_br backup full --log-file $BACKUPMETAV1_LOG -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+
+TTL_MARK='![ttl]'
+CREATE_SQL_CONTAINS="/*T${TTL_MARK} TTL=\`t\` + INTERVAL 1 DAY */ /*T${TTL_MARK} TTL_ENABLE='OFF' */"
+
+# restore v2
+run_sql "DROP DATABASE $DB;"
+echo "restore ttl table start v2..."
+run_br restore db --db $DB -s "local://$TEST_DIR/${DB}v2" --pd $PD_ADDR
+run_sql "show create table $DB.ttl_test_tbl;"
+check_contains "$CREATE_SQL_CONTAINS"
+
+# restore v1
+run_sql "DROP DATABASE $DB;"
+echo "restore ttl table start v1..."
+run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+run_sql "show create table $DB.ttl_test_tbl;"
+check_contains "$CREATE_SQL_CONTAINS"
diff --git a/br/tests/lightning_bom_file/config.toml b/br/tests/lightning_bom_file/config.toml
new file mode 100644
index 0000000000000..291d1b166103a
--- /dev/null
+++ b/br/tests/lightning_bom_file/config.toml
@@ -0,0 +1,2 @@
+[mydumper.csv]
+header = true
diff --git a/br/tests/lightning_bom_file/data/mytest.testtbl-schema.sql b/br/tests/lightning_bom_file/data/mytest.testtbl-schema.sql
new file mode 100644
index 0000000000000..4232788898790
--- /dev/null
+++ b/br/tests/lightning_bom_file/data/mytest.testtbl-schema.sql
@@ -0,0 +1,5 @@
+CREATE TABLE testtbl (
+ id INTEGER,
+ val1 VARCHAR(40) NOT NULL,
+ INDEX `idx_val1` (`val1`)
+);
diff --git a/br/tests/lightning_bom_file/data/mytest.testtbl.csv b/br/tests/lightning_bom_file/data/mytest.testtbl.csv
new file mode 100644
index 0000000000000..e0931cce2a480
--- /dev/null
+++ b/br/tests/lightning_bom_file/data/mytest.testtbl.csv
@@ -0,0 +1,6 @@
+id,val1
+1,"aaa01"
+2,"aaa01"
+3,"aaa02"
+4,"aaa02"
+5,"aaa05"
diff --git a/br/tests/lightning_bom_file/original_data/mytest.testtbl-schema.sql b/br/tests/lightning_bom_file/original_data/mytest.testtbl-schema.sql
new file mode 100644
index 0000000000000..dc1e032a16618
--- /dev/null
+++ b/br/tests/lightning_bom_file/original_data/mytest.testtbl-schema.sql
@@ -0,0 +1,5 @@
+CREATE TABLE testtbl (
+ id INTEGER,
+ val1 VARCHAR(40) NOT NULL,
+ INDEX `idx_val1` (`val1`)
+);
diff --git a/br/tests/lightning_bom_file/original_data/mytest.testtbl.csv b/br/tests/lightning_bom_file/original_data/mytest.testtbl.csv
new file mode 100644
index 0000000000000..270c410cd79fd
--- /dev/null
+++ b/br/tests/lightning_bom_file/original_data/mytest.testtbl.csv
@@ -0,0 +1,6 @@
+id,val1
+1,"aaa01"
+2,"aaa01"
+3,"aaa02"
+4,"aaa02"
+5,"aaa05"
diff --git a/br/tests/lightning_bom_file/run.sh b/br/tests/lightning_bom_file/run.sh
new file mode 100755
index 0000000000000..88eada54c74a9
--- /dev/null
+++ b/br/tests/lightning_bom_file/run.sh
@@ -0,0 +1,56 @@
+#!/bin/sh
+#
+# Copyright 2023 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+mydir=$(dirname "${BASH_SOURCE[0]}")
+
+original_schema_file="${mydir}/original_data/mytest.testtbl-schema.sql"
+original_data_file="${mydir}/original_data/mytest.testtbl.csv"
+schema_file="${original_schema_file/original_data/data}"
+data_file="${original_data_file/original_data/data}"
+
+# add the BOM header
+printf '\xEF\xBB\xBF' | cat - <( sed '1s/^\xEF\xBB\xBF//' "${original_schema_file}" ) > "${schema_file}"
+printf '\xEF\xBB\xBF' | cat - <( sed '1s/^\xEF\xBB\xBF//' "${original_data_file}" ) > "${data_file}"
+
+# verify the BOM header
+if ! grep -q $'^\xEF\xBB\xBF' "${schema_file}"; then
+ echo "schema file doesn't contain the BOM header" >&2
+ exit 1
+fi
+
+if ! grep -q $'^\xEF\xBB\xBF' "${data_file}"; then
+ echo "data file doesn't contain the BOM header" >&2
+ exit 1
+fi
+
+row_count=$( sed '1d' "${data_file}" | wc -l | xargs echo )
+
+run_lightning --backend tidb
+
+# Check that everything is correctly imported
+run_sql 'SELECT count(*) FROM mytest.testtbl'
+check_contains "count(*): ${row_count}"
+
+check_cluster_version 4 0 0 'local backend' || exit 0
+run_sql "DROP TABLE mytest.testtbl"
+
+run_lightning --backend local
+
+# Check that everything is correctly imported
+run_sql 'SELECT count(*) FROM mytest.testtbl'
+check_contains "count(*): ${row_count}"
diff --git a/br/tests/lightning_check_partial_imported/config.toml b/br/tests/lightning_check_partial_imported/config.toml
new file mode 100644
index 0000000000000..30cb6fe6b4eb3
--- /dev/null
+++ b/br/tests/lightning_check_partial_imported/config.toml
@@ -0,0 +1,5 @@
+[tikv-importer]
+backend = "local"
+
+[mydumper.csv]
+header = true
diff --git a/br/tests/lightning_check_partial_imported/data/db01.tbl01-schema.sql b/br/tests/lightning_check_partial_imported/data/db01.tbl01-schema.sql
new file mode 100644
index 0000000000000..b6832e95d95e3
--- /dev/null
+++ b/br/tests/lightning_check_partial_imported/data/db01.tbl01-schema.sql
@@ -0,0 +1,12 @@
+CREATE TABLE tbl01 (
+ `id` INTEGER,
+ `val` VARCHAR(64),
+ `aaa` CHAR(66) DEFAULT NULL,
+ `bbb` CHAR(10) NOT NULL,
+ `ccc` CHAR(42) DEFAULT NULL,
+ `ddd` CHAR(42) DEFAULT NULL,
+ `eee` CHAR(66) DEFAULT NULL,
+ `fff` VARCHAR(128) DEFAULT NULL,
+ KEY `aaa` (`aaa`),
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
diff --git a/br/tests/lightning_check_partial_imported/data/db01.tbl01.csv b/br/tests/lightning_check_partial_imported/data/db01.tbl01.csv
new file mode 100644
index 0000000000000..108134af2ee72
--- /dev/null
+++ b/br/tests/lightning_check_partial_imported/data/db01.tbl01.csv
@@ -0,0 +1,6 @@
+id,val,aaa,bbb,ccc,ddd,eee,fff
+1,"v01","a01","b01","c01","d01","e01","f01"
+2,"v02","a02","b02","c02","d02","e02","f02"
+3,"v03","a03","b03","c03","d03","e03","f03"
+4,"v04","a04","b04","c04","d04","e04","f04"
+5,"v05","a05","b05","c05","d05","e05","f05"
diff --git a/br/tests/lightning_check_partial_imported/run.sh b/br/tests/lightning_check_partial_imported/run.sh
new file mode 100755
index 0000000000000..00ed78a5013d1
--- /dev/null
+++ b/br/tests/lightning_check_partial_imported/run.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+MYDIR=$(dirname "${BASH_SOURCE[0]}")
+set -eux
+
+check_cluster_version 4 0 0 'local backend' || exit 0
+
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/restore/FailBeforeStartImportingIndexEngine=return"
+set +e
+if run_lightning; then
+ echo "The first import doesn't fail as expected" >&2
+ exit 1
+fi
+set -e
+
+data_records=$(tail -n +2 "${MYDIR}/data/db01.tbl01.csv" | wc -l | xargs echo )
+run_sql "SELECT COUNT(*) FROM db01.tbl01 USE INDEX();"
+check_contains "${data_records}"
+
+export GO_FAILPOINTS=""
+set +e
+if run_lightning --check-requirements=1; then
+ echo "The pre-check doesn't find out the non-empty table problem"
+ exit 2
+fi
+set -e
+
+run_sql "TRUNCATE TABLE db01.tbl01;"
+run_lightning --check-requirements=1
+run_sql "SELECT COUNT(*) FROM db01.tbl01;"
+check_contains "${data_records}"
+run_sql "SELECT COUNT(*) FROM db01.tbl01 USE INDEX();"
+check_contains "${data_records}"
diff --git a/br/tests/lightning_checkpoint/run.sh b/br/tests/lightning_checkpoint/run.sh
index 86551fd6246eb..5263dd90f1acf 100755
--- a/br/tests/lightning_checkpoint/run.sh
+++ b/br/tests/lightning_checkpoint/run.sh
@@ -79,6 +79,9 @@ for i in $(seq "$TABLE_COUNT"); do
done
set -e
+# at the failure of last table, all data engines are imported so finished == total
+grep "print lightning status" "$TEST_DIR/lightning.log" | grep -q "equal=true"
+
export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS"
# After everything is done, there should be no longer new calls to ImportEngine
# (and thus `kill_lightning_after_one_import` will spare this final check)
diff --git a/br/tests/lightning_checkpoint_chunks/run.sh b/br/tests/lightning_checkpoint_chunks/run.sh
index 35cabe0aadfc5..48d24ca405070 100755
--- a/br/tests/lightning_checkpoint_chunks/run.sh
+++ b/br/tests/lightning_checkpoint_chunks/run.sh
@@ -48,6 +48,11 @@ for i in $(seq "$CHUNK_COUNT"); do
done
done
+PKG="github.com/pingcap/tidb/br/pkg/lightning"
+export GO_FAILPOINTS="$PKG/backend/local/orphanWriterGoRoutine=return();$PKG/restore/orphanWriterGoRoutine=return();$PKG/orphanWriterGoRoutine=return()"
+# test won't panic
+do_run_lightning config
+
# Set the failpoint to kill the lightning instance as soon as
# one file (after writing totally $ROW_COUNT rows) is imported.
# If checkpoint does work, this should kill exactly $CHUNK_COUNT instances of lightnings.
diff --git a/br/tests/lightning_checkpoint_columns/run.sh b/br/tests/lightning_checkpoint_columns/run.sh
index 401c75cfb9f64..5809d05a1b830 100755
--- a/br/tests/lightning_checkpoint_columns/run.sh
+++ b/br/tests/lightning_checkpoint_columns/run.sh
@@ -29,6 +29,8 @@ echo "INSERT INTO tbl (j, i) VALUES (3, 1),(4, 2);" > "$DBPATH/cp_tsr.tbl.sql"
# Set the failpoint to kill the lightning instance as soon as one row is written
PKG="github.com/pingcap/tidb/br/pkg/lightning/restore"
export GO_FAILPOINTS="$PKG/SlowDownWriteRows=sleep(1000);$PKG/FailAfterWriteRows=panic;$PKG/SetMinDeliverBytes=return(1)"
+# Check after 1 row is written in tidb backend, the finished progress is updated
+export GO_FAILPOINTS="${GO_FAILPOINTS};github.com/pingcap/tidb/br/pkg/lightning/PrintStatus=return()"
# Start importing the tables.
run_sql 'DROP DATABASE IF EXISTS cp_tsr'
@@ -40,11 +42,16 @@ set -e
run_sql 'SELECT count(*) FROM `cp_tsr`.tbl'
check_contains "count(*): 1"
+# After FailAfterWriteRows, the finished bytes is 36 as the first row size
+grep "PrintStatus Failpoint" "$TEST_DIR/lightning.log" | grep -q "finished=36"
+
# restart lightning from checkpoint, the second line should be written successfully
-export GO_FAILPOINTS=
+# also check after restart from checkpoint, final finished equals to total
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/PrintStatus=return()"
set +e
run_lightning -d "$DBPATH" --backend tidb --enable-checkpoint=1 2> /dev/null
set -e
run_sql 'SELECT j FROM `cp_tsr`.tbl WHERE i = 2;'
check_contains "j: 4"
+grep "PrintStatus Failpoint" "$TEST_DIR/lightning.log" | grep -q "equal=true"
diff --git a/br/tests/lightning_compress/config.toml b/br/tests/lightning_compress/config.toml
new file mode 100644
index 0000000000000..f4452fe7664a6
--- /dev/null
+++ b/br/tests/lightning_compress/config.toml
@@ -0,0 +1,14 @@
+[mydumper.csv]
+separator = ','
+delimiter = '"'
+header = true
+not-null = false
+null = '\N'
+backslash-escape = true
+trim-last-separator = false
+
+[checkpoint]
+enable = true
+schema = "tidb_lightning_checkpoint_test"
+driver = "mysql"
+keep-after-success = true
diff --git a/br/tests/lightning_compress/data.gzip/compress-schema-create.sql.gz b/br/tests/lightning_compress/data.gzip/compress-schema-create.sql.gz
new file mode 100644
index 0000000000000..6571d2a15b507
Binary files /dev/null and b/br/tests/lightning_compress/data.gzip/compress-schema-create.sql.gz differ
diff --git a/br/tests/lightning_compress/data.gzip/compress.empty_strings-schema.sql.gz b/br/tests/lightning_compress/data.gzip/compress.empty_strings-schema.sql.gz
new file mode 100644
index 0000000000000..542898561bab1
Binary files /dev/null and b/br/tests/lightning_compress/data.gzip/compress.empty_strings-schema.sql.gz differ
diff --git a/br/tests/lightning_compress/data.gzip/compress.empty_strings.000000000.csv.gz b/br/tests/lightning_compress/data.gzip/compress.empty_strings.000000000.csv.gz
new file mode 100644
index 0000000000000..bfa13ed67b006
Binary files /dev/null and b/br/tests/lightning_compress/data.gzip/compress.empty_strings.000000000.csv.gz differ
diff --git a/br/tests/lightning_compress/data.gzip/compress.escapes-schema.sql.gz b/br/tests/lightning_compress/data.gzip/compress.escapes-schema.sql.gz
new file mode 100644
index 0000000000000..bed4b7859ac92
Binary files /dev/null and b/br/tests/lightning_compress/data.gzip/compress.escapes-schema.sql.gz differ
diff --git a/br/tests/lightning_compress/data.gzip/compress.escapes.000000000.csv.gz b/br/tests/lightning_compress/data.gzip/compress.escapes.000000000.csv.gz
new file mode 100644
index 0000000000000..37028e36d9de8
Binary files /dev/null and b/br/tests/lightning_compress/data.gzip/compress.escapes.000000000.csv.gz differ
diff --git a/br/tests/lightning_compress/data.gzip/compress.multi_rows-schema.sql.gz b/br/tests/lightning_compress/data.gzip/compress.multi_rows-schema.sql.gz
new file mode 100644
index 0000000000000..328fed9cb3df8
Binary files /dev/null and b/br/tests/lightning_compress/data.gzip/compress.multi_rows-schema.sql.gz differ
diff --git a/br/tests/lightning_compress/data.gzip/compress.multi_rows.000000000.csv.gz b/br/tests/lightning_compress/data.gzip/compress.multi_rows.000000000.csv.gz
new file mode 100644
index 0000000000000..c732af263d576
Binary files /dev/null and b/br/tests/lightning_compress/data.gzip/compress.multi_rows.000000000.csv.gz differ
diff --git a/br/tests/lightning_compress/data.gzip/compress.threads-schema.sql.gz b/br/tests/lightning_compress/data.gzip/compress.threads-schema.sql.gz
new file mode 100644
index 0000000000000..1782675bfc7fe
Binary files /dev/null and b/br/tests/lightning_compress/data.gzip/compress.threads-schema.sql.gz differ
diff --git a/br/tests/lightning_compress/data.gzip/compress.threads.000000000.csv.gz b/br/tests/lightning_compress/data.gzip/compress.threads.000000000.csv.gz
new file mode 100644
index 0000000000000..683eade1cdb9f
Binary files /dev/null and b/br/tests/lightning_compress/data.gzip/compress.threads.000000000.csv.gz differ
diff --git a/br/tests/lightning_compress/data.snappy/compress-schema-create.sql.snappy b/br/tests/lightning_compress/data.snappy/compress-schema-create.sql.snappy
new file mode 100644
index 0000000000000..afa2211c77475
Binary files /dev/null and b/br/tests/lightning_compress/data.snappy/compress-schema-create.sql.snappy differ
diff --git a/br/tests/lightning_compress/data.snappy/compress.empty_strings-schema.sql.snappy b/br/tests/lightning_compress/data.snappy/compress.empty_strings-schema.sql.snappy
new file mode 100644
index 0000000000000..cab30d082385a
Binary files /dev/null and b/br/tests/lightning_compress/data.snappy/compress.empty_strings-schema.sql.snappy differ
diff --git a/br/tests/lightning_compress/data.snappy/compress.empty_strings.000000000.sql.snappy b/br/tests/lightning_compress/data.snappy/compress.empty_strings.000000000.sql.snappy
new file mode 100644
index 0000000000000..9c81e8f78f234
Binary files /dev/null and b/br/tests/lightning_compress/data.snappy/compress.empty_strings.000000000.sql.snappy differ
diff --git a/br/tests/lightning_compress/data.snappy/compress.escapes-schema.sql.snappy b/br/tests/lightning_compress/data.snappy/compress.escapes-schema.sql.snappy
new file mode 100644
index 0000000000000..9e27befa522a0
Binary files /dev/null and b/br/tests/lightning_compress/data.snappy/compress.escapes-schema.sql.snappy differ
diff --git a/br/tests/lightning_compress/data.snappy/compress.escapes.000000000.sql.snappy b/br/tests/lightning_compress/data.snappy/compress.escapes.000000000.sql.snappy
new file mode 100644
index 0000000000000..1380b47d9881e
Binary files /dev/null and b/br/tests/lightning_compress/data.snappy/compress.escapes.000000000.sql.snappy differ
diff --git a/br/tests/lightning_compress/data.snappy/compress.multi_rows-schema.sql.snappy b/br/tests/lightning_compress/data.snappy/compress.multi_rows-schema.sql.snappy
new file mode 100644
index 0000000000000..5cc0365d1c65d
Binary files /dev/null and b/br/tests/lightning_compress/data.snappy/compress.multi_rows-schema.sql.snappy differ
diff --git a/br/tests/lightning_compress/data.snappy/compress.multi_rows.000000000.sql.snappy b/br/tests/lightning_compress/data.snappy/compress.multi_rows.000000000.sql.snappy
new file mode 100644
index 0000000000000..7f5bf585e106c
Binary files /dev/null and b/br/tests/lightning_compress/data.snappy/compress.multi_rows.000000000.sql.snappy differ
diff --git a/br/tests/lightning_compress/data.snappy/compress.threads-schema.sql.snappy b/br/tests/lightning_compress/data.snappy/compress.threads-schema.sql.snappy
new file mode 100644
index 0000000000000..b1c8b89565bfb
Binary files /dev/null and b/br/tests/lightning_compress/data.snappy/compress.threads-schema.sql.snappy differ
diff --git a/br/tests/lightning_compress/data.snappy/compress.threads.000000000.sql.snappy b/br/tests/lightning_compress/data.snappy/compress.threads.000000000.sql.snappy
new file mode 100644
index 0000000000000..dc7c1ee8adc0b
Binary files /dev/null and b/br/tests/lightning_compress/data.snappy/compress.threads.000000000.sql.snappy differ
diff --git a/br/tests/lightning_compress/data.zstd/compress-schema-create.sql.zst b/br/tests/lightning_compress/data.zstd/compress-schema-create.sql.zst
new file mode 100644
index 0000000000000..12bdbd710973e
Binary files /dev/null and b/br/tests/lightning_compress/data.zstd/compress-schema-create.sql.zst differ
diff --git a/br/tests/lightning_compress/data.zstd/compress.empty_strings-schema.sql.zst b/br/tests/lightning_compress/data.zstd/compress.empty_strings-schema.sql.zst
new file mode 100644
index 0000000000000..f9b922954ff3d
Binary files /dev/null and b/br/tests/lightning_compress/data.zstd/compress.empty_strings-schema.sql.zst differ
diff --git a/br/tests/lightning_compress/data.zstd/compress.empty_strings.000000000.csv.zst b/br/tests/lightning_compress/data.zstd/compress.empty_strings.000000000.csv.zst
new file mode 100644
index 0000000000000..aa89918bb2cee
Binary files /dev/null and b/br/tests/lightning_compress/data.zstd/compress.empty_strings.000000000.csv.zst differ
diff --git a/br/tests/lightning_compress/data.zstd/compress.escapes-schema.sql.zst b/br/tests/lightning_compress/data.zstd/compress.escapes-schema.sql.zst
new file mode 100644
index 0000000000000..fa4b4e6b3497d
Binary files /dev/null and b/br/tests/lightning_compress/data.zstd/compress.escapes-schema.sql.zst differ
diff --git a/br/tests/lightning_compress/data.zstd/compress.escapes.000000000.csv.zst b/br/tests/lightning_compress/data.zstd/compress.escapes.000000000.csv.zst
new file mode 100644
index 0000000000000..40994e745bdf3
Binary files /dev/null and b/br/tests/lightning_compress/data.zstd/compress.escapes.000000000.csv.zst differ
diff --git a/br/tests/lightning_compress/data.zstd/compress.multi_rows-schema.sql.zst b/br/tests/lightning_compress/data.zstd/compress.multi_rows-schema.sql.zst
new file mode 100644
index 0000000000000..d64a9a4a879d3
Binary files /dev/null and b/br/tests/lightning_compress/data.zstd/compress.multi_rows-schema.sql.zst differ
diff --git a/br/tests/lightning_compress/data.zstd/compress.multi_rows.000000000.csv.zst b/br/tests/lightning_compress/data.zstd/compress.multi_rows.000000000.csv.zst
new file mode 100644
index 0000000000000..4db1bea4c69f9
Binary files /dev/null and b/br/tests/lightning_compress/data.zstd/compress.multi_rows.000000000.csv.zst differ
diff --git a/br/tests/lightning_compress/data.zstd/compress.threads-schema.sql.zst b/br/tests/lightning_compress/data.zstd/compress.threads-schema.sql.zst
new file mode 100644
index 0000000000000..3a41c8de4816c
Binary files /dev/null and b/br/tests/lightning_compress/data.zstd/compress.threads-schema.sql.zst differ
diff --git a/br/tests/lightning_compress/data.zstd/compress.threads.000000000.csv.zst b/br/tests/lightning_compress/data.zstd/compress.threads.000000000.csv.zst
new file mode 100644
index 0000000000000..13eef0ba83011
Binary files /dev/null and b/br/tests/lightning_compress/data.zstd/compress.threads.000000000.csv.zst differ
diff --git a/br/tests/lightning_compress/run.sh b/br/tests/lightning_compress/run.sh
new file mode 100755
index 0000000000000..bf48b09b2cccd
--- /dev/null
+++ b/br/tests/lightning_compress/run.sh
@@ -0,0 +1,61 @@
+#!/bin/sh
+
+set -eu
+
+for BACKEND in tidb local; do
+ for compress in gzip snappy zstd; do
+ if [ "$BACKEND" = 'local' ]; then
+ check_cluster_version 4 0 0 'local backend' || continue
+ fi
+
+ # Set minDeliverBytes to a small enough number to only write only 1 row each time
+ # Set the failpoint to kill the lightning instance as soon as one row is written
+ PKG="github.com/pingcap/tidb/br/pkg/lightning/restore"
+ export GO_FAILPOINTS="$PKG/SlowDownWriteRows=sleep(1000);$PKG/FailAfterWriteRows=panic;$PKG/SetMinDeliverBytes=return(1)"
+
+ # Start importing the tables.
+ run_sql 'DROP DATABASE IF EXISTS compress'
+ run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test'
+ set +e
+ run_lightning --backend $BACKEND -d "tests/$TEST_NAME/data.$compress" --enable-checkpoint=1 2> /dev/null
+ set -e
+
+ # restart lightning from checkpoint, the second line should be written successfully
+ export GO_FAILPOINTS=
+ set +e
+ run_lightning --backend $BACKEND -d "tests/$TEST_NAME/data.$compress" --enable-checkpoint=1 2> /dev/null
+ set -e
+
+ run_sql 'SELECT count(*), sum(PROCESSLIST_TIME), sum(THREAD_OS_ID), count(PROCESSLIST_STATE) FROM compress.threads'
+ check_contains 'count(*): 43'
+ check_contains 'sum(PROCESSLIST_TIME): 322253'
+ check_contains 'sum(THREAD_OS_ID): 303775702'
+ check_contains 'count(PROCESSLIST_STATE): 3'
+
+ run_sql 'SELECT count(*) FROM compress.threads WHERE PROCESSLIST_TIME IS NOT NULL'
+ check_contains 'count(*): 12'
+
+ run_sql 'SELECT count(*) FROM compress.multi_rows WHERE a="aaaaaaaaaa"'
+ check_contains 'count(*): 100000'
+
+ run_sql 'SELECT hex(t), j, hex(b) FROM compress.escapes WHERE i = 1'
+ check_contains 'hex(t): 5C'
+ check_contains 'j: {"?": []}'
+ check_contains 'hex(b): FFFFFFFF'
+
+ run_sql 'SELECT hex(t), j, hex(b) FROM compress.escapes WHERE i = 2'
+ check_contains 'hex(t): 22'
+ check_contains 'j: "\n\n\n"'
+ check_contains 'hex(b): 0D0A0D0A'
+
+ run_sql 'SELECT hex(t), j, hex(b) FROM compress.escapes WHERE i = 3'
+ check_contains 'hex(t): 0A'
+ check_contains 'j: [",,,"]'
+ check_contains 'hex(b): 5C2C5C2C'
+
+ run_sql 'SELECT id FROM compress.empty_strings WHERE a = """"'
+ check_contains 'id: 3'
+ run_sql 'SELECT id FROM compress.empty_strings WHERE b <> ""'
+ check_not_contains 'id:'
+ done
+done
diff --git a/br/tests/lightning_config_max_error/data/mytest.testtbl-schema.sql b/br/tests/lightning_config_max_error/data/mytest.testtbl-schema.sql
new file mode 100644
index 0000000000000..93582d5178139
--- /dev/null
+++ b/br/tests/lightning_config_max_error/data/mytest.testtbl-schema.sql
@@ -0,0 +1,5 @@
+CREATE TABLE testtbl (
+ id INTEGER PRIMARY KEY,
+ val1 VARCHAR(40) NOT NULL,
+ INDEX `idx_val1` (`val1`)
+);
diff --git a/br/tests/lightning_config_max_error/data/mytest.testtbl.csv b/br/tests/lightning_config_max_error/data/mytest.testtbl.csv
new file mode 100644
index 0000000000000..021f6bbf7be1c
--- /dev/null
+++ b/br/tests/lightning_config_max_error/data/mytest.testtbl.csv
@@ -0,0 +1,16 @@
+id,val1
+1,"aaa01"
+2,"aaa01"
+3,"aaa02"
+4,"aaa02"
+5,"aaa05"
+6,"aaa06"
+7,"aaa07"
+8,"aaa08"
+9,"aaa09"
+10,"aaa10"
+1,"bbb01"
+2,"bbb02"
+3,"bbb03"
+4,"bbb04"
+5,"bbb05"
diff --git a/br/tests/lightning_config_max_error/err_config.toml b/br/tests/lightning_config_max_error/err_config.toml
new file mode 100644
index 0000000000000..79447e685a8f5
--- /dev/null
+++ b/br/tests/lightning_config_max_error/err_config.toml
@@ -0,0 +1,8 @@
+[lightning.max-error]
+conflict = 4
+
+[mydumper.csv]
+header = true
+
+[tikv-importer]
+duplicate-resolution = 'remove'
diff --git a/br/tests/lightning_config_max_error/normal_config.toml b/br/tests/lightning_config_max_error/normal_config.toml
new file mode 100644
index 0000000000000..92e08739fe04a
--- /dev/null
+++ b/br/tests/lightning_config_max_error/normal_config.toml
@@ -0,0 +1,8 @@
+[lightning.max-error]
+conflict = 20
+
+[mydumper.csv]
+header = true
+
+[tikv-importer]
+duplicate-resolution = 'remove'
diff --git a/br/tests/lightning_config_max_error/normal_config_old_style.toml b/br/tests/lightning_config_max_error/normal_config_old_style.toml
new file mode 100644
index 0000000000000..fe402d071f5e0
--- /dev/null
+++ b/br/tests/lightning_config_max_error/normal_config_old_style.toml
@@ -0,0 +1,8 @@
+[lightning]
+max-error = 0 # this actually sets the type error
+
+[mydumper.csv]
+header = true
+
+[tikv-importer]
+duplicate-resolution = 'remove'
diff --git a/br/tests/lightning_config_max_error/run.sh b/br/tests/lightning_config_max_error/run.sh
new file mode 100755
index 0000000000000..1d850ae55f0d8
--- /dev/null
+++ b/br/tests/lightning_config_max_error/run.sh
@@ -0,0 +1,81 @@
+#!/bin/sh
+#
+# Copyright 2023 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+check_cluster_version 4 0 0 'local backend' || exit 0
+
+mydir=$(dirname "${BASH_SOURCE[0]}")
+
+data_file="${mydir}/data/mytest.testtbl.csv"
+
+total_row_count=$( sed '1d' "${data_file}" | wc -l | xargs echo )
+uniq_row_count=$( sed '1d' "${data_file}" | awk -F, '{print $1}' | sort | uniq -c | awk '{print $1}' | grep -c '1' | xargs echo )
+duplicated_row_count=$(( ${total_row_count} - ${uniq_row_count} ))
+
+run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
+run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v1'
+
+stderr_file="/tmp/${TEST_NAME}.stderr"
+
+set +e
+if run_lightning --backend local --config "${mydir}/err_config.toml" 2> "${stderr_file}"; then
+ echo "The lightning import doesn't fail as expected" >&2
+ exit 1
+fi
+set -e
+
+err_msg=$( cat << EOF
+tidb lightning encountered error: collect local duplicate rows failed: The number of conflict errors exceeds the threshold configured by \`max-error.conflict\`: '4'
+EOF
+)
+cat "${stderr_file}"
+grep -q "${err_msg}" "${stderr_file}"
+
+run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_error_v1'
+# Although conflict error number exceeds the max-error limit,
+# all the conflict errors are recorded,
+# because recording of conflict errors are executed batch by batch (batch size 1024),
+# this batch of conflict errors are all recorded
+check_contains "COUNT(*): ${duplicated_row_count}"
+
+# import a second time
+
+run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
+run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v1'
+
+run_lightning --backend local --config "${mydir}/normal_config.toml"
+
+run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_error_v1'
+check_contains "COUNT(*): ${duplicated_row_count}"
+
+# Check remaining records in the target table
+run_sql 'SELECT COUNT(*) FROM mytest.testtbl'
+check_contains "COUNT(*): ${uniq_row_count}"
+
+# import a third time
+
+run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
+run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v1'
+
+run_lightning --backend local --config "${mydir}/normal_config_old_style.toml"
+
+run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_error_v1'
+check_contains "COUNT(*): ${duplicated_row_count}"
+
+# Check remaining records in the target table
+run_sql 'SELECT COUNT(*) FROM mytest.testtbl'
+check_contains "COUNT(*): ${uniq_row_count}"
diff --git a/br/tests/lightning_config_skip_csv_header/data/mytest.testtbl-schema.sql b/br/tests/lightning_config_skip_csv_header/data/mytest.testtbl-schema.sql
new file mode 100644
index 0000000000000..93582d5178139
--- /dev/null
+++ b/br/tests/lightning_config_skip_csv_header/data/mytest.testtbl-schema.sql
@@ -0,0 +1,5 @@
+CREATE TABLE testtbl (
+ id INTEGER PRIMARY KEY,
+ val1 VARCHAR(40) NOT NULL,
+ INDEX `idx_val1` (`val1`)
+);
diff --git a/br/tests/lightning_config_skip_csv_header/data/mytest.testtbl.csv b/br/tests/lightning_config_skip_csv_header/data/mytest.testtbl.csv
new file mode 100644
index 0000000000000..5958ab0c80cb2
--- /dev/null
+++ b/br/tests/lightning_config_skip_csv_header/data/mytest.testtbl.csv
@@ -0,0 +1,6 @@
+aaa,bbb
+1,"aaa01"
+2,"aaa01"
+3,"aaa02"
+4,"aaa02"
+5,"aaa05"
diff --git a/br/tests/lightning_config_skip_csv_header/err_config.toml b/br/tests/lightning_config_skip_csv_header/err_config.toml
new file mode 100644
index 0000000000000..95493db0dff44
--- /dev/null
+++ b/br/tests/lightning_config_skip_csv_header/err_config.toml
@@ -0,0 +1,9 @@
+[lightning]
+check-requirements=true
+
+[mydumper.csv]
+header = true
+header-schema-match = true
+
+[tikv-importer]
+duplicate-resolution = 'remove'
diff --git a/br/tests/lightning_config_skip_csv_header/err_default_config.toml b/br/tests/lightning_config_skip_csv_header/err_default_config.toml
new file mode 100644
index 0000000000000..a7b17c7276d92
--- /dev/null
+++ b/br/tests/lightning_config_skip_csv_header/err_default_config.toml
@@ -0,0 +1,8 @@
+[lightning]
+check-requirements=true
+
+[mydumper.csv]
+header = true
+
+[tikv-importer]
+duplicate-resolution = 'remove'
diff --git a/br/tests/lightning_config_skip_csv_header/normal_config.toml b/br/tests/lightning_config_skip_csv_header/normal_config.toml
new file mode 100644
index 0000000000000..190e635cfc4e9
--- /dev/null
+++ b/br/tests/lightning_config_skip_csv_header/normal_config.toml
@@ -0,0 +1,9 @@
+[lightning]
+check-requirements=true
+
+[mydumper.csv]
+header = true
+header-schema-match = false
+
+[tikv-importer]
+duplicate-resolution = 'remove'
diff --git a/br/tests/lightning_config_skip_csv_header/run.sh b/br/tests/lightning_config_skip_csv_header/run.sh
new file mode 100755
index 0000000000000..80ad201e2d323
--- /dev/null
+++ b/br/tests/lightning_config_skip_csv_header/run.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Copyright 2023 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+mydir=$(dirname "${BASH_SOURCE[0]}")
+
+data_file="${mydir}/data/mytest.testtbl.csv"
+
+total_row_count=$( sed '1d' "${data_file}" | wc -l | xargs echo )
+
+run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
+
+console_output_file="/tmp/${TEST_NAME}.out"
+
+echo "Use config that causes errors"
+run_lightning --backend tidb --config "${mydir}/err_config.toml" 2>&1 | tee "${console_output_file}"
+if [[ ${PIPESTATUS[0]} -eq 0 ]]; then
+ echo "The lightning import doesn't fail as expected" >&2
+ exit 1
+fi
+
+grep -q "Lightning:Restore:ErrUnknownColumns" "${console_output_file}"
+
+# import a second time
+echo "Use default config that causes errors"
+run_lightning --backend tidb --config "${mydir}/err_default_config.toml" 2>&1 | tee "${console_output_file}"
+if [[ ${PIPESTATUS[0]} -eq 0 ]]; then
+ echo "The lightning import doesn't fail as expected" >&2
+ exit 1
+fi
+
+grep -q "Lightning:Restore:ErrUnknownColumns" "${console_output_file}"
+
+# import a thrid time
+
+run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
+
+echo "Use config that can sucessfully import the data"
+run_lightning --backend tidb --config "${mydir}/normal_config.toml"
+
+run_sql 'SELECT * FROM mytest.testtbl'
+run_sql 'SELECT COUNT(*) FROM mytest.testtbl'
+check_contains "COUNT(*): ${total_row_count}"
+run_sql 'SELECT COUNT(*) FROM mytest.testtbl WHERE id > 0'
+check_contains "COUNT(*): ${total_row_count}"
diff --git a/br/tests/lightning_csv/errData/db-schema-create.sql b/br/tests/lightning_csv/errData/db-schema-create.sql
new file mode 100755
index 0000000000000..6adfeca7f7dab
--- /dev/null
+++ b/br/tests/lightning_csv/errData/db-schema-create.sql
@@ -0,0 +1 @@
+create database if not exists db;
diff --git a/br/tests/lightning_csv/errData/db.test-schema.sql b/br/tests/lightning_csv/errData/db.test-schema.sql
new file mode 100755
index 0000000000000..955632c7761b2
--- /dev/null
+++ b/br/tests/lightning_csv/errData/db.test-schema.sql
@@ -0,0 +1 @@
+create table test(a int primary key, b int, c int, d int);
diff --git a/br/tests/lightning_csv/errData/db.test.1.csv b/br/tests/lightning_csv/errData/db.test.1.csv
new file mode 100755
index 0000000000000..2e8450c25786e
--- /dev/null
+++ b/br/tests/lightning_csv/errData/db.test.1.csv
@@ -0,0 +1,3 @@
+1,2,3,4
+2,10,4,5
+1111,",7,8
diff --git a/br/tests/lightning_csv/run.sh b/br/tests/lightning_csv/run.sh
index 83c4917b4b76c..682bc55b08e26 100755
--- a/br/tests/lightning_csv/run.sh
+++ b/br/tests/lightning_csv/run.sh
@@ -41,3 +41,11 @@ for BACKEND in tidb local; do
check_not_contains 'id:'
done
+
+set +e
+run_lightning --backend local -d "tests/$TEST_NAME/errData" --log-file "$TEST_DIR/lightning-err.log" 2>/dev/null
+set -e
+# err content presented
+grep ",7,8" "$TEST_DIR/lightning-err.log"
+# pos should not set to end
+grep "[\"syntax error\"] [pos=22]" "$TEST_DIR/lightning-err.log"
\ No newline at end of file
diff --git a/br/tests/lightning_disable_scheduler_by_key_range/run.sh b/br/tests/lightning_disable_scheduler_by_key_range/run.sh
index 9df6067c8baf8..2a88f0e0cac8e 100644
--- a/br/tests/lightning_disable_scheduler_by_key_range/run.sh
+++ b/br/tests/lightning_disable_scheduler_by_key_range/run.sh
@@ -50,16 +50,16 @@ ready_for_import_engine
run_curl "https://${PD_ADDR}/pd/api/v1/config/cluster-version"
-length=$(run_curl "https://${PD_ADDR}/pd/api/v1/config/region-label/rules" | jq 'select(.[].rule_type == "key-range") | length')
+length=$(run_curl "https://${PD_ADDR}/pd/api/v1/config/region-label/rules" | jq '[ .[] | select(.rule_type == "key-range" and .labels[0].key == "schedule") ] | length')
if [ "$length" != "1" ]; then
- echo "region-label key-range rules should be 1, but got $length" >&2
+ echo "region-label key-range schedule rules should be 1, but got $length" >&2
exit 1
fi
wait "$shpid"
-length=$(run_curl "https://${PD_ADDR}/pd/api/v1/config/region-label/rules" | jq 'select(.[].rule_type == "key-range") | length')
+length=$(run_curl "https://${PD_ADDR}/pd/api/v1/config/region-label/rules" | jq '[ .[] | select(.rule_type == "key-range" and .labels[0].key == "schedule") ] | length')
if [ -n "$length" ] && [ "$length" -ne 0 ]; then
- echo "region-label key-range rules should be 0, but got $length" >&2
+ echo "region-label key-range schedule rules should be 0, but got $length" >&2
exit 1
fi
diff --git a/br/tests/lightning_duplicate_resolution_incremental/config1.toml b/br/tests/lightning_duplicate_resolution_incremental/config1.toml
new file mode 100644
index 0000000000000..a72bc7a3718cb
--- /dev/null
+++ b/br/tests/lightning_duplicate_resolution_incremental/config1.toml
@@ -0,0 +1,41 @@
+[lightning]
+task-info-schema-name = 'lightning_task_info_dupe_resolve_incremental'
+index-concurrency = 10
+table-concurrency = 10
+
+[tikv-importer]
+backend = "local"
+on-duplicate = "replace"
+duplicate-resolution = "remove"
+incremental-import = true
+
+[checkpoint]
+enable = true
+schema = "tidb_lightning_checkpoint_dupe_resolve_incremental1"
+driver = "mysql"
+
+[[mydumper.files]]
+pattern = '(?i).*(-schema-trigger|-schema-post)\.sql$'
+type = 'ignore'
+
+[[mydumper.files]]
+pattern = '(?i)^(?:[^/]*/)*([^/.]+)-schema-create\.sql$'
+schema = '$1'
+type = 'schema-schema'
+
+[[mydumper.files]]
+pattern = '(?i)^(?:[^/]*/)*([^/.]+)\.(.*?)-schema\.sql$'
+schema = '$1'
+table = '$2'
+type = 'table-schema'
+
+[[mydumper.files]]
+pattern = '(?i)^(?:[^/]*/)*([^/.]+)\.(.*?)\.0\.sql$'
+schema = '$1'
+table = '$2'
+key = '0'
+type = 'sql'
+
+[post-restore]
+analyze = false
+checksum = "optional"
diff --git a/br/tests/lightning_duplicate_resolution_incremental/config2.toml b/br/tests/lightning_duplicate_resolution_incremental/config2.toml
new file mode 100644
index 0000000000000..bb29511a9b432
--- /dev/null
+++ b/br/tests/lightning_duplicate_resolution_incremental/config2.toml
@@ -0,0 +1,41 @@
+[lightning]
+task-info-schema-name = 'lightning_task_info_dupe_resolve_incremental'
+index-concurrency = 10
+table-concurrency = 10
+
+[tikv-importer]
+backend = "local"
+on-duplicate = "replace"
+duplicate-resolution = "remove"
+incremental-import = true
+
+[checkpoint]
+enable = true
+schema = "tidb_lightning_checkpoint_dupe_resolve_incremental2"
+driver = "mysql"
+
+[[mydumper.files]]
+pattern = '(?i).*(-schema-trigger|-schema-post)\.sql$'
+type = 'ignore'
+
+[[mydumper.files]]
+pattern = '(?i)^(?:[^/]*/)*([^/.]+)-schema-create\.sql$'
+schema = '$1'
+type = 'schema-schema'
+
+[[mydumper.files]]
+pattern = '(?i)^(?:[^/]*/)*([^/.]+)\.(.*?)-schema\.sql$'
+schema = '$1'
+table = '$2'
+type = 'table-schema'
+
+[[mydumper.files]]
+pattern = '(?i)^(?:[^/]*/)*([^/.]+)\.(.*?)\.1\.sql$'
+schema = '$1'
+table = '$2'
+key = '1'
+type = 'sql'
+
+[post-restore]
+analyze = false
+checksum = "optional"
diff --git a/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect-schema-create.sql b/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect-schema-create.sql
new file mode 100644
index 0000000000000..202de81067861
--- /dev/null
+++ b/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect-schema-create.sql
@@ -0,0 +1 @@
+create schema dup_resolve_detect;
diff --git a/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect.ta-schema.sql b/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect.ta-schema.sql
new file mode 100644
index 0000000000000..fb6cf2d5a7651
--- /dev/null
+++ b/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect.ta-schema.sql
@@ -0,0 +1,6 @@
+create table ta (
+ id varchar(11) not null primary key nonclustered, -- use varchar here to make sure _tidb_rowid will be generated
+ name varchar(20) not null,
+ size bigint not null,
+ unique key uni_name(name)
+);
diff --git a/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect.ta.0.sql b/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect.ta.0.sql
new file mode 100644
index 0000000000000..ee29f689e8792
--- /dev/null
+++ b/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect.ta.0.sql
@@ -0,0 +1,20 @@
+insert into ta values (3, '3c49f3bd', 6194643990092531757);
+insert into ta values (13, '1da87b44', 3724743701402246028);
+insert into ta values (6, '8b080186', 4840750639653607661);
+insert into ta values (1, 'c83c0e6a', 5057094372111243649);
+insert into ta values (12, 'dd73baf5', 2295098755414696158);
+insert into ta values (4, '1cf99fa1', 2520784525406914042);
+insert into ta values (11, 'b238a0e6', 3314555604794199537);
+insert into ta values (10, 'a489c47a', 7706822128523578708);
+insert into ta values (10, '9a54941e', 4969369552499069659);
+insert into ta values (2, 'e7c90179', 1305347797378229715);
+insert into ta values (9, '75e0344a', 500154046394880294);
+insert into ta values (9, 'c3e8fc36', 5880042654284780409);
+insert into ta values (6, 'd6835599', 2703142091339420770);
+insert into ta values (5, 'c4a9c3a3', 6725275961959702206);
+insert into ta values (14, 'eb1ab0dd', 5442878220607642694);
+insert into ta values (7, '78e166f4', 7062852002089313920);
+insert into ta values (8, '20986b65', 5485014514564267319);
+insert into ta values (8, '9bd4d7a9', 9085469020413045798);
+insert into ta values (15, 'd4aa9a8a', 546189610059969690);
+insert into ta values (7, 'a7870c06', 3615729521258364152);
diff --git a/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect.ta.1.sql b/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect.ta.1.sql
new file mode 100644
index 0000000000000..88b67b051fe6e
--- /dev/null
+++ b/br/tests/lightning_duplicate_resolution_incremental/data/dup_resolve_detect.ta.1.sql
@@ -0,0 +1,16 @@
+insert into ta values (111, 'bcf4e75f', 3304674741328415661);
+insert into ta values (112, 'c08078e9', 7464585077725645791);
+insert into ta values (113, 'ca05b4b2', 1280363363179468054);
+insert into ta values (114, '8a094c96', 107578474892900608);
+insert into ta values (115, 'f38efac2', 5273601814057696410);
+insert into ta values (116, '5bf0cb56', 7276272767003446282);
+insert into ta values (117, 'c8836b45', 653431702983792793);
+insert into ta values (118, '7470ba67', 5617407618564683998);
+insert into ta values (119, '466e1e95', 6827370124386922419);
+insert into ta values (120, '41df97f3', 2296443172527920942);
+insert into ta values (121, 'bd644f43', 6038622426427289955);
+insert into ta values (122, '96aeb918', 1496857236328804363);
+insert into ta values (123, '232448f7', 1199921720244646472);
+insert into ta values (124, 'd296d6e4', 5705035255191089143);
+insert into ta values (125, '194ec1d8', 6895413645725179445);
+insert into ta values (126, 'a53238ec', 1527836891202149330);
diff --git a/br/tests/lightning_duplicate_resolution_incremental/run.sh b/br/tests/lightning_duplicate_resolution_incremental/run.sh
new file mode 100644
index 0000000000000..b1bf1e3869d27
--- /dev/null
+++ b/br/tests/lightning_duplicate_resolution_incremental/run.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+#
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+check_cluster_version 5 2 0 'duplicate detection' || exit 0
+
+LOG_FILE1="$TEST_DIR/lightning-duplicate-resolution1.log"
+LOG_FILE2="$TEST_DIR/lightning-duplicate-resolution2.log"
+
+# let lightning run a bit slow to avoid some table in the first lightning finish too fast.
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/restore/SlowDownCheckDupe=return(10)"
+run_lightning --backend local --sorted-kv-dir "$TEST_DIR/lightning_duplicate_resolution_incremental.sorted1" \
+ --enable-checkpoint=1 --log-file "$LOG_FILE1" --config "tests/$TEST_NAME/config1.toml" &
+
+counter=0
+while [ $counter -lt 10 ]; do
+ if grep -Fq "start to sleep several seconds before checking other dupe" "$LOG_FILE1"; then
+ echo "lightning 1 already starts waiting for dupe"
+ break
+ fi
+ ((counter += 1))
+ echo "waiting for lightning 1 starts"
+ sleep 1
+done
+
+if [ $counter -ge 10 ]; then
+ echo "fail to wait for lightning 1 starts"
+ exit 1
+fi
+
+run_lightning --backend local --sorted-kv-dir "$TEST_DIR/lightning_duplicate_resolution_incremental.sorted2" \
+ --enable-checkpoint=1 --log-file "$LOG_FILE2" --config "tests/$TEST_NAME/config2.toml" &
+
+wait
+
+export GO_FAILPOINTS=""
+
+# Ensure table is consistent.
+run_sql 'admin check table dup_resolve_detect.ta'
+
+# Check data correctness
+run_sql 'select count(*), sum(id) from dup_resolve_detect.ta where id < 100'
+check_contains 'count(*): 10'
+check_contains 'sum(id): 80'
+
+run_sql 'select count(*), sum(id) from dup_resolve_detect.ta where id > 100'
+check_contains 'count(*): 16'
+check_contains 'sum(id): 1896'
diff --git a/br/tests/lightning_fail_fast/run.sh b/br/tests/lightning_fail_fast/run.sh
index 7e057e3ee999c..5f8efb8b44712 100755
--- a/br/tests/lightning_fail_fast/run.sh
+++ b/br/tests/lightning_fail_fast/run.sh
@@ -25,7 +25,7 @@ for CFG in chunk engine; do
! run_lightning --backend tidb --enable-checkpoint=0 --log-file "$TEST_DIR/lightning-tidb.log" --config "tests/$TEST_NAME/$CFG.toml"
[ $? -eq 0 ]
- tail -n 10 $TEST_DIR/lightning-tidb.log | grep "ERROR" | tail -n 1 | grep -Fq "Error 1062: Duplicate entry '1-1' for key 'tb.uq'"
+ tail -n 10 $TEST_DIR/lightning-tidb.log | grep "ERROR" | tail -n 1 | grep -Fq "Error 1062 (23000): Duplicate entry '1-1' for key 'tb.uq'"
! grep -Fq "restore file completed" $TEST_DIR/lightning-tidb.log
[ $? -eq 0 ]
diff --git a/br/tests/lightning_foreign_key/config.toml b/br/tests/lightning_foreign_key/config.toml
new file mode 100644
index 0000000000000..3c85a830bfa22
--- /dev/null
+++ b/br/tests/lightning_foreign_key/config.toml
@@ -0,0 +1,3 @@
+[tikv-importer]
+# Set on-duplicate=error to force using insert statement to write data.
+on-duplicate = "error"
diff --git a/br/tests/lightning_foreign_key/data/fk.child-schema.sql b/br/tests/lightning_foreign_key/data/fk.child-schema.sql
new file mode 100644
index 0000000000000..18c361bf4c2e0
--- /dev/null
+++ b/br/tests/lightning_foreign_key/data/fk.child-schema.sql
@@ -0,0 +1 @@
+create table child (id int key, pid int, constraint fk_1 foreign key (pid) references parent(id));
diff --git a/br/tests/lightning_foreign_key/data/fk.child.sql b/br/tests/lightning_foreign_key/data/fk.child.sql
new file mode 100644
index 0000000000000..12e531eb96a34
--- /dev/null
+++ b/br/tests/lightning_foreign_key/data/fk.child.sql
@@ -0,0 +1 @@
+insert into child values (1,1),(2,2),(3,3),(4,4);
diff --git a/br/tests/lightning_foreign_key/data/fk.parent-schema.sql b/br/tests/lightning_foreign_key/data/fk.parent-schema.sql
new file mode 100644
index 0000000000000..8ae8af2de6a2e
--- /dev/null
+++ b/br/tests/lightning_foreign_key/data/fk.parent-schema.sql
@@ -0,0 +1 @@
+create table parent(id int key, a int);
diff --git a/br/tests/lightning_foreign_key/data/fk.parent.sql b/br/tests/lightning_foreign_key/data/fk.parent.sql
new file mode 100644
index 0000000000000..7a31a9f18db0f
--- /dev/null
+++ b/br/tests/lightning_foreign_key/data/fk.parent.sql
@@ -0,0 +1 @@
+insert into parent values (1,1),(2,2),(3,3),(4,4);
diff --git a/br/tests/lightning_foreign_key/data/fk.t-schema.sql b/br/tests/lightning_foreign_key/data/fk.t-schema.sql
new file mode 100644
index 0000000000000..98f00b9cadca8
--- /dev/null
+++ b/br/tests/lightning_foreign_key/data/fk.t-schema.sql
@@ -0,0 +1,8 @@
+CREATE TABLE `t`
+(
+ `a` bigint(20) NOT NULL,
+ `b` bigint(20) DEFAULT NULL,
+ PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,
+ KEY `fk_1` (`b`),
+ CONSTRAINT `fk_1` FOREIGN KEY (`b`) REFERENCES `test`.`t2` (`a`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
diff --git a/br/tests/lightning_foreign_key/data/fk.t.csv b/br/tests/lightning_foreign_key/data/fk.t.csv
new file mode 100644
index 0000000000000..cd0368580a4c8
--- /dev/null
+++ b/br/tests/lightning_foreign_key/data/fk.t.csv
@@ -0,0 +1,6 @@
+a,b
+1,1
+2,2
+3,3
+4,4
+5,5
diff --git a/br/tests/lightning_foreign_key/run.sh b/br/tests/lightning_foreign_key/run.sh
new file mode 100755
index 0000000000000..1c045b61f43be
--- /dev/null
+++ b/br/tests/lightning_foreign_key/run.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+run_sql 'DROP DATABASE IF EXISTS fk;'
+run_sql 'CREATE DATABASE IF NOT EXISTS fk;'
+# Create existing tables that import data will reference.
+run_sql 'CREATE TABLE fk.t2 (a BIGINT PRIMARY KEY);'
+
+for BACKEND in tidb local; do
+ run_sql 'DROP TABLE IF EXISTS fk.t, fk.parent, fk.child;'
+
+ run_lightning --backend $BACKEND
+ run_sql 'SELECT GROUP_CONCAT(a) FROM fk.t ORDER BY a;'
+ check_contains '1,2,3,4,5'
+
+ run_sql 'SELECT count(1), sum(a) FROM fk.parent;'
+ check_contains 'count(1): 4'
+ check_contains 'sum(a): 10'
+
+ run_sql 'SELECT count(1), sum(pid) FROM fk.child;'
+ check_contains 'count(1): 4'
+ check_contains 'sum(pid): 10'
+done
diff --git a/br/tests/lightning_import_compress/config.toml b/br/tests/lightning_import_compress/config.toml
new file mode 100644
index 0000000000000..30df6c8e0c98e
--- /dev/null
+++ b/br/tests/lightning_import_compress/config.toml
@@ -0,0 +1,5 @@
+[tikv-importer]
+backend = 'local'
+
+[mydumper.csv]
+header = false
diff --git a/br/tests/lightning_import_compress/config_gz.toml b/br/tests/lightning_import_compress/config_gz.toml
new file mode 100644
index 0000000000000..d26e6ae237c18
--- /dev/null
+++ b/br/tests/lightning_import_compress/config_gz.toml
@@ -0,0 +1,6 @@
+[tikv-importer]
+backend = 'local'
+compress-kv-pairs = 'gz'
+
+[mydumper.csv]
+header = false
diff --git a/br/tests/lightning_import_compress/config_gzip.toml b/br/tests/lightning_import_compress/config_gzip.toml
new file mode 100644
index 0000000000000..24a873a27599b
--- /dev/null
+++ b/br/tests/lightning_import_compress/config_gzip.toml
@@ -0,0 +1,6 @@
+[tikv-importer]
+backend = 'local'
+compress-kv-pairs = 'gzip'
+
+[mydumper.csv]
+header = false
diff --git a/br/tests/lightning_import_compress/run.sh b/br/tests/lightning_import_compress/run.sh
new file mode 100644
index 0000000000000..e6414881e4e0e
--- /dev/null
+++ b/br/tests/lightning_import_compress/run.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Copyright 2023 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/backend/local/LoggingImportBytes=return"
+
+mkdir -p "$TEST_DIR/data"
+
+cat <"$TEST_DIR/data/test-schema-create.sql"
+CREATE DATABASE test;
+EOF
+cat <"$TEST_DIR/data/test.t-schema.sql"
+CREATE TABLE test.t (id int primary key, a int, b int, c int);
+EOF
+
+# Generate 200k rows. Total size is about 5MiB.
+for i in {1..200000}; do
+ echo "$i,$i,$i,$i" >>"$TEST_DIR/data/test.t.0.csv"
+done
+
+LOG_FILE1="$TEST_DIR/lightning-import-compress1.log"
+LOG_FILE2="$TEST_DIR/lightning-import-compress2.log"
+LOG_FILE3="$TEST_DIR/lightning-import-compress3.log"
+
+run_lightning --backend local -d "$TEST_DIR/data" --config "tests/$TEST_NAME/config.toml" --log-file "$LOG_FILE1" -L debug
+run_sql 'DROP DATABASE test;'
+run_lightning --backend local -d "$TEST_DIR/data" --config "tests/$TEST_NAME/config_gz.toml" --log-file "$LOG_FILE2" -L debug
+run_sql 'DROP DATABASE test;'
+run_lightning --backend local -d "$TEST_DIR/data" --config "tests/$TEST_NAME/config_gzip.toml" --log-file "$LOG_FILE3" -L debug
+
+uncompress=$(grep "import write" /tmp/backup_restore_test/lightning-import-compress1.log |
+ grep -Eo "bytes=[0-9]+" | sed 's/bytes=//g' | awk '{sum+=$1} END {print sum}')
+gzip=$(grep "import write" /tmp/backup_restore_test/lightning-import-compress2.log |
+ grep -Eo "bytes=[0-9]+" | sed 's/bytes=//g' | awk '{sum+=$1} END {print sum}')
+gz=$(grep "import write" /tmp/backup_restore_test/lightning-import-compress3.log |
+ grep -Eo "bytes=[0-9]+" | sed 's/bytes=//g' | awk '{sum+=$1} END {print sum}')
+
+echo "uncompress: ${uncompress}, gzip: ${gzip}, gz: ${gz}"
+if [ "$uncompress" -le "$gzip" ] || [ "$uncompress" -le "$gz" ]; then
+ echo "compress is not working"
+ exit 1
+fi
diff --git a/br/tests/lightning_issue_40657/config.toml b/br/tests/lightning_issue_40657/config.toml
new file mode 100644
index 0000000000000..74561bc05f026
--- /dev/null
+++ b/br/tests/lightning_issue_40657/config.toml
@@ -0,0 +1,6 @@
+[tikv-importer]
+backend = "local"
+duplicate-resolution = "remove"
+
+[mydumper.csv]
+header = true
diff --git a/br/tests/lightning_issue_40657/data1/test.t-schema.sql b/br/tests/lightning_issue_40657/data1/test.t-schema.sql
new file mode 100644
index 0000000000000..ef7136b531abc
--- /dev/null
+++ b/br/tests/lightning_issue_40657/data1/test.t-schema.sql
@@ -0,0 +1,6 @@
+CREATE TABLE `t` (
+ `id` int(11) NOT NULL,
+ `name` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,
+ UNIQUE KEY `uni_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
diff --git a/br/tests/lightning_issue_40657/data1/test.t.0.csv b/br/tests/lightning_issue_40657/data1/test.t.0.csv
new file mode 100644
index 0000000000000..2987cee08b206
--- /dev/null
+++ b/br/tests/lightning_issue_40657/data1/test.t.0.csv
@@ -0,0 +1,6 @@
+id,name
+1,"aaa01"
+2,"aaa02"
+3,"aaa03"
+4,"aaa04"
+5,"aaa04"
diff --git a/br/tests/lightning_issue_40657/data2/test.t-schema.sql b/br/tests/lightning_issue_40657/data2/test.t-schema.sql
new file mode 100644
index 0000000000000..ef7136b531abc
--- /dev/null
+++ b/br/tests/lightning_issue_40657/data2/test.t-schema.sql
@@ -0,0 +1,6 @@
+CREATE TABLE `t` (
+ `id` int(11) NOT NULL,
+ `name` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,
+ UNIQUE KEY `uni_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
diff --git a/br/tests/lightning_issue_40657/data2/test.t.0.csv b/br/tests/lightning_issue_40657/data2/test.t.0.csv
new file mode 100644
index 0000000000000..f64aebd0630d9
--- /dev/null
+++ b/br/tests/lightning_issue_40657/data2/test.t.0.csv
@@ -0,0 +1,6 @@
+id,name
+1,"aaa01"
+2,"aaa02"
+3,"aaa03"
+4,"aaa04"
+5,"aaa05"
diff --git a/br/tests/lightning_issue_40657/run.sh b/br/tests/lightning_issue_40657/run.sh
new file mode 100644
index 0000000000000..a20600b79d14b
--- /dev/null
+++ b/br/tests/lightning_issue_40657/run.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copyright 2023 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+check_cluster_version 5 2 0 'duplicate detection' || exit 0
+
+run_lightning -d "tests/$TEST_NAME/data1"
+run_sql 'admin check table test.t'
+run_sql 'select count(*) from test.t'
+check_contains 'count(*): 3'
+run_sql 'select count(*) from lightning_task_info.conflict_error_v1'
+check_contains 'count(*): 2'
+
+run_sql 'truncate table test.t'
+run_lightning -d "tests/$TEST_NAME/data2"
+run_sql 'admin check table test.t'
+run_sql 'select count(*) from test.t'
+check_contains 'count(*): 5'
diff --git a/br/tests/lightning_local_backend/data/cpeng.a-schema.sql b/br/tests/lightning_local_backend/data/cpeng.a-schema.sql
index fe3f493b6edfd..6c1f5ee154c58 100644
--- a/br/tests/lightning_local_backend/data/cpeng.a-schema.sql
+++ b/br/tests/lightning_local_backend/data/cpeng.a-schema.sql
@@ -1 +1 @@
-create table a (c int);
+create table a (c VARCHAR(20) PRIMARY KEY);
diff --git a/br/tests/lightning_local_backend/data/cpeng.a.1.sql b/br/tests/lightning_local_backend/data/cpeng.a.1.sql
index 58829b7d87492..a75039e1304e3 100644
--- a/br/tests/lightning_local_backend/data/cpeng.a.1.sql
+++ b/br/tests/lightning_local_backend/data/cpeng.a.1.sql
@@ -1 +1 @@
-insert into a values (1);
+insert into a values ('0000001');
diff --git a/br/tests/lightning_local_backend/data/cpeng.a.2.sql b/br/tests/lightning_local_backend/data/cpeng.a.2.sql
index ccbcb5801f72b..a1b15acdecb11 100644
--- a/br/tests/lightning_local_backend/data/cpeng.a.2.sql
+++ b/br/tests/lightning_local_backend/data/cpeng.a.2.sql
@@ -1 +1 @@
-insert into a values (2);
+insert into a values ('0000002');
diff --git a/br/tests/lightning_local_backend/data/cpeng.a.3.sql b/br/tests/lightning_local_backend/data/cpeng.a.3.sql
index effdc8f3eab3c..0a457febecf55 100644
--- a/br/tests/lightning_local_backend/data/cpeng.a.3.sql
+++ b/br/tests/lightning_local_backend/data/cpeng.a.3.sql
@@ -1 +1 @@
-insert into a values (3),(4);
+insert into a values ('0000003'),('0000004');
diff --git a/br/tests/lightning_local_backend/run.sh b/br/tests/lightning_local_backend/run.sh
index 5177f98d1ecba..60fca277d8cf6 100755
--- a/br/tests/lightning_local_backend/run.sh
+++ b/br/tests/lightning_local_backend/run.sh
@@ -36,10 +36,11 @@ grep -Fq 'table(s) [`cpeng`.`a`, `cpeng`.`b`] are not empty' $TEST_DIR/lightning
# First, verify that inject with not leader error is fine.
-export GO_FAILPOINTS='github.com/pingcap/tidb/br/pkg/lightning/backend/local/FailIngestMeta=1*return("notleader")'
+export GO_FAILPOINTS='github.com/pingcap/tidb/br/pkg/lightning/backend/local/FailIngestMeta=1*return("notleader");github.com/pingcap/tidb/br/pkg/lightning/backend/local/failToSplit=5*return("")'
rm -f "$TEST_DIR/lightning-local.log"
run_sql 'DROP DATABASE IF EXISTS cpeng;'
-run_lightning --backend local --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-local.log" --config "tests/$TEST_NAME/config.toml"
+run_lightning --backend local --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-local.log" --config "tests/$TEST_NAME/config.toml" -L debug
+grep -Eq "split regions.*retryable error" "$TEST_DIR/lightning-local.log"
# Check that everything is correctly imported
run_sql 'SELECT count(*), sum(c) FROM cpeng.a'
diff --git a/br/tests/lightning_record_network/config.toml b/br/tests/lightning_record_network/config.toml
new file mode 100644
index 0000000000000..2de41a1f43dab
--- /dev/null
+++ b/br/tests/lightning_record_network/config.toml
@@ -0,0 +1,2 @@
+[tikv-importer]
+backend = 'tidb'
diff --git a/br/tests/lightning_record_network/data/db-schema-create.sql b/br/tests/lightning_record_network/data/db-schema-create.sql
new file mode 100644
index 0000000000000..c88b0e3150e76
--- /dev/null
+++ b/br/tests/lightning_record_network/data/db-schema-create.sql
@@ -0,0 +1 @@
+create database db;
\ No newline at end of file
diff --git a/br/tests/lightning_record_network/data/db.test-schema.sql b/br/tests/lightning_record_network/data/db.test-schema.sql
new file mode 100644
index 0000000000000..7bee5f9ad639c
--- /dev/null
+++ b/br/tests/lightning_record_network/data/db.test-schema.sql
@@ -0,0 +1 @@
+create table test ( id int primary key, a int, b int );
\ No newline at end of file
diff --git a/br/tests/lightning_record_network/data/db.test.1.sql b/br/tests/lightning_record_network/data/db.test.1.sql
new file mode 100644
index 0000000000000..3748d5fa91e80
--- /dev/null
+++ b/br/tests/lightning_record_network/data/db.test.1.sql
@@ -0,0 +1,21 @@
+insert into db.test values
+(1,1,1),
+(2,1,1),
+(3,1,1),
+(4,1,1),
+(5,1,1),
+(6,1,1),
+(7,1,1),
+(8,1,1),
+(9,1,1),
+(10,1,1),
+(11,1,1),
+(12,1,1),
+(13,1,1),
+(14,1,1),
+(15,1,1),
+(16,1,1),
+(17,1,1),
+(18,1,1),
+(19,1,1),
+(20,1,1);
\ No newline at end of file
diff --git a/br/tests/lightning_record_network/run.sh b/br/tests/lightning_record_network/run.sh
new file mode 100644
index 0000000000000..e31f9d151d76a
--- /dev/null
+++ b/br/tests/lightning_record_network/run.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euE
+
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/SetIOTotalBytes=return(1)"
+run_lightning
+
+grep 'IOTotal' "$TEST_DIR/lightning.log" | grep -v 'IOTotalBytes=0'
diff --git a/br/tests/lightning_reload_cert/run.sh b/br/tests/lightning_reload_cert/run.sh
index e06ef8d7fbf51..be0c5ff40421e 100644
--- a/br/tests/lightning_reload_cert/run.sh
+++ b/br/tests/lightning_reload_cert/run.sh
@@ -29,7 +29,7 @@ shpid="$!"
sleep 15
ok=0
for _ in {0..60}; do
- if grep -Fq "connection closed before server preface received" "$TEST_DIR"/lightning.log; then
+ if grep -Fq "connection error" "$TEST_DIR"/lightning.log; then
ok=1
break
fi
diff --git a/br/tests/lightning_ttl/config.toml b/br/tests/lightning_ttl/config.toml
new file mode 100644
index 0000000000000..d2152b47c922a
--- /dev/null
+++ b/br/tests/lightning_ttl/config.toml
@@ -0,0 +1,2 @@
+[tikv-importer]
+backend = 'local'
diff --git a/br/tests/lightning_ttl/data/ttldb-schema-create.sql b/br/tests/lightning_ttl/data/ttldb-schema-create.sql
new file mode 100644
index 0000000000000..46609f11e6635
--- /dev/null
+++ b/br/tests/lightning_ttl/data/ttldb-schema-create.sql
@@ -0,0 +1 @@
+CREATE DATABASE `ttldb`;
diff --git a/br/tests/lightning_ttl/data/ttldb.t1-schema.sql b/br/tests/lightning_ttl/data/ttldb.t1-schema.sql
new file mode 100644
index 0000000000000..7531d7f18ae01
--- /dev/null
+++ b/br/tests/lightning_ttl/data/ttldb.t1-schema.sql
@@ -0,0 +1,4 @@
+CREATE TABLE `t1` (
+ `id` int(11) PRIMARY KEY,
+ `t` datetime
+) TTL = `t` + INTERVAL 1 DAY TTL_ENABLE = 'ON';
diff --git a/br/tests/lightning_ttl/run.sh b/br/tests/lightning_ttl/run.sh
new file mode 100644
index 0000000000000..4a1d9ffc04d57
--- /dev/null
+++ b/br/tests/lightning_ttl/run.sh
@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+run_sql 'drop database if exists ttldb;'
+run_lightning
+
+TTL_MARK='![ttl]'
+CREATE_SQL_CONTAINS="/*T${TTL_MARK} TTL=\`t\` + INTERVAL 1 DAY */ /*T${TTL_MARK} TTL_ENABLE='OFF' */"
+
+run_sql 'show create table ttldb.t1'
+check_contains "$CREATE_SQL_CONTAINS"
diff --git a/build/BUILD.bazel b/build/BUILD.bazel
index 23cf263d525e3..3c2a569dd80d8 100644
--- a/build/BUILD.bazel
+++ b/build/BUILD.bazel
@@ -55,6 +55,7 @@ STATICHECK_ANALYZERS = [
"S1039",
"S1040",
"SA1019",
+ "SA1029",
"SA2000",
"SA2001",
"SA2003",
diff --git a/build/image/.ci_bazel b/build/image/.ci_bazel
new file mode 100644
index 0000000000000..6114f8bf17db9
--- /dev/null
+++ b/build/image/.ci_bazel
@@ -0,0 +1 @@
+build:ci --remote_cache=http://bazel-cache.pingcap.net:8080/tidb --remote_timeout="15s"
diff --git a/build/image/README.md b/build/image/README.md
new file mode 100644
index 0000000000000..b109968354940
--- /dev/null
+++ b/build/image/README.md
@@ -0,0 +1,7 @@
+## CI Image
+
+Here is the Dockerfile for the CI image.
+
+- ```base``` is the base image with golang, development tools and so on.
+- ```centos7_jenkins``` is the production image with CI environment tool in tidb repo. it is based on ```base```.
+- ```.ci_bazel``` is the global default bazel config. it tell bazel where to get cache.
diff --git a/build/image/base b/build/image/base
new file mode 100644
index 0000000000000..85c3dbba6816f
--- /dev/null
+++ b/build/image/base
@@ -0,0 +1,44 @@
+# Copyright 2023 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM hub.pingcap.net/jenkins/centos7_jenkins
+
+USER root
+WORKDIR /root
+
+ENV GOLANG_VERSION 1.19.5
+ENV GOLANG_DOWNLOAD_URL https://dl.google.com/go/go$GOLANG_VERSION.linux-amd64.tar.gz
+ENV GOLANG_DOWNLOAD_SHA256 36519702ae2fd573c9869461990ae550c8c0d955cd28d2827a6b159fda81ff95
+ENV GOPATH /go
+ENV GOROOT /usr/local/go
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+ADD https://github.com/bazelbuild/bazel/releases/download/5.3.2/bazel-5.3.2-linux-x86_64 /usr/bin/bazel
+ADD https://uploader.codecov.io/latest/linux/codecov /usr/bin/codecov
+RUN curl https://setup.ius.io | sh || true && \
+ chmod u+x /usr/bin/bazel && yum update -y && \
+ yum install -y supervisor tree libcurl-devel gettext autoconf python-pip python3-pip patch git wget gcc python autoconf make curl-devel expat-devel gettext-devel openssl-devel zlib-devel perl-CPAN perl-devel && \
+ git clone --depth=1 --branch=v2.37.2 https://github.com/git/git && cd git && yum remove -y git && make configure && ./configure --prefix=/usr/local && make -j16 install && cd .. && rm -rf git && \
+ pip3 install s3cmd requests && pip3 install requests && \
+ curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
+ && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \
+ && tar -C /usr/local -xzf golang.tar.gz \
+ && rm golang.tar.gz && \
+ mkdir /go && chown jenkins:jenkins /go && \
+ curl -fsSL "http://pingcap-dev.hk.ufileos.com/jenkins/jenkins-slave-docker-sqllogictest.tar.gz" | tar xz -C "/git" \
+ && chown -R jenkins:jenkins /git && \
+ chown jenkins:jenkins /usr/bin/bazel && \
+ chown jenkins:jenkins /usr/bin/codecov && \
+ chmod +x /usr/bin/codecov
+USER jenkins
+WORKDIR /home/jenkins
diff --git a/build/image/centos7_jenkins b/build/image/centos7_jenkins
new file mode 100644
index 0000000000000..0f56c705ca114
--- /dev/null
+++ b/build/image/centos7_jenkins
@@ -0,0 +1,24 @@
+# Copyright 2023 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM hub.pingcap.net/wangweizhen/base_image:go11920230111
+USER root
+WORKDIR /root
+COPY .ci_bazel /data/bazel
+RUN mkdir -p /data/tikv1 /data/tikv2 /data/tikv3 /data/pd && \
+ chown -R jenkins:jenkins /data
+USER jenkins
+WORKDIR /home/jenkins
+RUN go install github.com/hawkingrei/bazel_collect@latest && \
+ go install github.com/bazelbuild/bazel-gazelle/cmd/gazelle@latest
diff --git a/build/linter/BUILD.bazel b/build/linter/BUILD.bazel
new file mode 100644
index 0000000000000..e5407284430fa
--- /dev/null
+++ b/build/linter/BUILD.bazel
@@ -0,0 +1,9 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "linter",
+ srcs = ["linter.go"],
+ importpath = "github.com/pingcap/tidb/build/linter",
+ visibility = ["//visibility:public"],
+ deps = ["@com_github_apache_skywalking_eyes//pkg/config"],
+)
diff --git a/build/linter/linter.go b/build/linter/linter.go
new file mode 100644
index 0000000000000..794a7c37039ef
--- /dev/null
+++ b/build/linter/linter.go
@@ -0,0 +1,20 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package linter
+
+import (
+ // it is necessary to make skywalking-eye into gomod.
+ _ "github.com/apache/skywalking-eyes/pkg/config"
+)
diff --git a/build/nogo_config.json b/build/nogo_config.json
index 731eed4c54f00..90a5b7cae853f 100644
--- a/build/nogo_config.json
+++ b/build/nogo_config.json
@@ -1,7 +1,7 @@
{
"all_revive": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"/rules_go_work-*": "ignore generated code",
".*_/testmain\\.go$": "ignore code"
@@ -9,32 +9,32 @@
},
"asciicheck": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"br/pkg/lightning/web/res_vfsdata.go": "ignore code"
}
},
"asmdecl": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"assign": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"atomic": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"atomicalign": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
@@ -46,31 +46,31 @@
},
"bools": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"buildtag": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"printf": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"unreachable": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"composites": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"br/pkg/glue/console_glue_test.go": "ignore code",
"br/pkg/restore/db_test.go": "ignore code",
@@ -79,45 +79,45 @@
},
"copylocks": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"/cgo/": "ignore cgo code"
}
},
"ctrlflow": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"deadcode": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"deepequalerrors": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"durationcheck": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
"/rules_go_work-*": "ignore generated code",
".*_generated\\.go$": "ignore generated code"
}
},
"errorsas": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"errcheck": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
".*_test\\.go$": "ignore generated code",
"util/logutil": "ignore util/logutil code",
@@ -131,20 +131,20 @@
},
"exportloopref": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"filepermission": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
".*_/testmain\\.go$": "ignore code"
}
},
"fieldalignment": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
".*_/testmain\\.go$": "ignore code",
".*_test\\.go$": "ignore test code"
@@ -188,13 +188,13 @@
},
"findcall": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"forcetypeassert": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
},
"only_files": {
@@ -219,7 +219,7 @@
},
"gofmt": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"/cgo/": "ignore cgo code",
"/rules_go_work-*": "ignore generated code",
@@ -229,7 +229,8 @@
},
"gci": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"/cgo/": "ignore cgo code",
".*\\.pb\\.go$": "generated code",
@@ -259,37 +260,37 @@
},
"httpresponse": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"ifaceassert": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"ineffassign": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"inspect": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"loopclosure": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"lostcancel": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
@@ -297,33 +298,33 @@
"exclude_files": {
"/cgo/": "ignore cgo code",
".*_test\\.go$": "ignore generated code",
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"misspell": {
"exclude_files": {
"/cgo/": "ignore cgo code",
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"nilfunc": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"nilness": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"/cgo/": "ignore cgo"
}
},
"noloopclosure": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
},
"only_files": {
@@ -338,13 +339,13 @@
},
"pkgfact": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"revive": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
"GOROOT/": "ignore code",
"/cgo/": "ignore cgo",
"tools/": "ignore tool code",
@@ -364,6 +365,8 @@
"ddl/backfilling.go": "ddl/backfilling.go",
"ddl/column.go": "ddl/column.go",
"ddl/index.go": "ddl/index.go",
+ "ddl/ttl.go": "ddl/ttl.go",
+ "ddl/ttl_test.go": "ddl/ttl_test.go",
"ddl/ingest/": "ddl/ingest/",
"expression/builtin_cast.go": "expression/builtin_cast code",
"server/conn.go": "server/conn.go",
@@ -401,54 +404,56 @@
"util/": "util code",
"parser/": "parser code",
"meta/": "parser code",
- "extension/": "extension code"
+ "extension/": "extension code",
+ "resourcemanager/": "resourcemanager code",
+ "keyspace": "keyspace code"
}
},
"shift": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"sortslice": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"stdmethods": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"stringintconv": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"structtag": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"testinggoroutine": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"tests": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"unconvert": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*\\.pb\\.go$": "generated code",
"parser/parser.go": "generated code",
"/cgo/": "no need to vet third party code for cgo",
@@ -460,27 +465,27 @@
},
"unmarshal": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"unsafeptr": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"parser/digester.go": "ignore code"
}
},
"unusedresult": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"parser/digester_test.go": "ignore code"
}
},
"rowserrcheck": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"server/tidb_test.go": "ignore test code",
"server/tidb_serial_test.go": "ignore test code",
@@ -492,250 +497,250 @@
},
"S1000": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1001": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1002": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1003": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1004": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1005": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1006": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1007": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1008": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1009": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1010": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1011": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1012": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1013": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1014": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1015": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1016": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1017": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1018": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1019": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"parser/parser.go": "ignore code"
}
},
"S1020": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1021": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"tools/check/ut.go": "ignore code"
}
},
"S1022": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1023": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"parser/parser.go": "ignore code"
}
},
"S1024": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1025": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1026": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1027": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1028": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1029": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1030": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1031": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1032": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1033": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1034": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1035": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1036": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1037": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1038": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1039": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"S1040": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"parser/parser.go": "ignore generated code"
}
@@ -743,155 +748,167 @@
"SA1019": {
"exclude_files": {
"/build/": "no need to linter code",
- "/external/": "no need to vet third party code",
- ".*_generated\\.go$": "ignore generated code"
+ "external/": "no need to vet third party code",
+ ".*_generated\\.go$": "ignore generated code",
+ ".*_test\\.go$": "ignore test code",
+ "br/pkg/restore/split/client.go": "github.com/golang/protobuf deprecated",
+ "br/pkg/streamhelper/advancer_cliext.go": "github.com/golang/protobuf deprecated",
+ "br/pkg/lightning/checkpoints/checkpoints.go": "cfg.TikvImporter.Addr is deprecated",
+ "br/pkg/lightning/checkpoints/glue_checkpoint.go": "cfg.TikvImporter.Addr is deprecated"
},
"only_files": {
"util/gctuner": "util/gctuner",
- "br/pkg/lightning/mydump/": "br/pkg/lightning/mydump/",
- "br/pkg/lightning/restore/opts": "br/pkg/lightning/restore/opts",
+ "util/cgroup": "util/cgroup code",
+ "util/watcher": "util/watcher",
+ "br/pkg/": "br/pkg",
"executor/aggregate.go": "executor/aggregate.go",
- "types/json_binary_functions.go": "types/json_binary_functions.go",
- "types/json_binary_test.go": "types/json_binary_test.go",
+ "types/": "types",
"ddl/": "enable to ddl",
- "util/cgroup": "util/cgroup code",
"expression/builtin_cast.go": "enable expression/builtin_cast.go",
"planner/core/plan.go": "planner/core/plan.go",
- "server/conn.go": "server/conn.go",
- "server/conn_stmt.go": "server/conn_stmt.go",
- "server/conn_test.go": "server/conn_test.go",
- "extension/": "extension code"
+ "extension/": "extension code",
+ "resourcemanager/": "resourcemanager code",
+ "keyspace/": "keyspace code",
+ "server/": "server code",
+ "meta": "meta code"
}
},
- "SA2000": {
+ "SA1029": {
"exclude_files": {
"/external/": "no need to vet third party code",
+ ".*_generated\\.go$": "ignore generated code",
+ ".*_test\\.go$": "ignore test code"
+ }
+ },
+ "SA2000": {
+ "exclude_files": {
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA2001": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA2003": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA3000": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA3001": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA4009": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5000": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5001": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5002": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5003": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5004": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5005": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5007": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5008": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5009": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5010": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5011": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA5012": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA6000": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA6001": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"SA6005": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
},
"prealloc": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"parser/yy_parser.go": "ignore generated code",
"/cgo/": "no need to vet third party code for cgo"
@@ -899,7 +916,7 @@
},
"predeclared": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"parser/yy_parser.go": "ignore generated code",
"parser/parser.go": "ignore generated code",
@@ -908,7 +925,7 @@
},
"U1000": {
"exclude_files": {
- "/external/": "no need to vet third party code",
+ "external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code"
}
}
diff --git a/build/patches/com_github_rivo_uniseg.patch b/build/patches/com_github_rivo_uniseg.patch
deleted file mode 100644
index 43c2c40933b11..0000000000000
--- a/build/patches/com_github_rivo_uniseg.patch
+++ /dev/null
@@ -1,542 +0,0 @@
-From 1492043a155839cb863210d4f564be3fa640c0d9 Mon Sep 17 00:00:00 2001
-From: Weizhen Wang
-Date: Sat, 8 Oct 2022 11:41:06 +0800
-Subject: [PATCH] update
-
-Signed-off-by: Weizhen Wang
----
- BUILD.bazel | 27 +++++
- WORKSPACE | 2 +
- gen_breaktest.go | 213 --------------------------------------
- gen_properties.go | 256 ----------------------------------------------
- 4 files changed, 29 insertions(+), 469 deletions(-)
- create mode 100644 BUILD.bazel
- create mode 100644 WORKSPACE
- delete mode 100644 gen_breaktest.go
- delete mode 100644 gen_properties.go
-
-diff --git a/BUILD.bazel b/BUILD.bazel
-new file mode 100644
-index 0000000..a1e5c89
---- /dev/null
-+++ b/BUILD.bazel
-@@ -0,0 +1,27 @@
-+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-+
-+go_library(
-+ name = "uniseg",
-+ srcs = [
-+ "doc.go",
-+ "eastasianwidth.go",
-+ "emojipresentation.go",
-+ "grapheme.go",
-+ "graphemeproperties.go",
-+ "graphemerules.go",
-+ "line.go",
-+ "lineproperties.go",
-+ "linerules.go",
-+ "properties.go",
-+ "sentence.go",
-+ "sentenceproperties.go",
-+ "sentencerules.go",
-+ "step.go",
-+ "width.go",
-+ "word.go",
-+ "wordproperties.go",
-+ "wordrules.go",
-+ ],
-+ importpath = "github.com/rivo/uniseg",
-+ visibility = ["//visibility:public"],
-+)
-diff --git a/WORKSPACE b/WORKSPACE
-new file mode 100644
-index 0000000..d596273
---- /dev/null
-+++ b/WORKSPACE
-@@ -0,0 +1,2 @@
-+# DO NOT EDIT: automatically generated WORKSPACE file for go_repository rule
-+workspace(name = "com_github_rivo_uniseg")
-diff --git a/gen_breaktest.go b/gen_breaktest.go
-deleted file mode 100644
-index e613c4c..0000000
---- a/gen_breaktest.go
-+++ /dev/null
-@@ -1,213 +0,0 @@
--//go:build generate
--
--// This program generates a Go containing a slice of test cases based on the
--// Unicode Character Database auxiliary data files. The command line arguments
--// are as follows:
--//
--// 1. The name of the Unicode data file (just the filename, without extension).
--// 2. The name of the locally generated Go file.
--// 3. The name of the slice containing the test cases.
--// 4. The name of the generator, for logging purposes.
--//
--//go:generate go run gen_breaktest.go GraphemeBreakTest graphemebreak_test.go graphemeBreakTestCases graphemes
--//go:generate go run gen_breaktest.go WordBreakTest wordbreak_test.go wordBreakTestCases words
--//go:generate go run gen_breaktest.go SentenceBreakTest sentencebreak_test.go sentenceBreakTestCases sentences
--//go:generate go run gen_breaktest.go LineBreakTest linebreak_test.go lineBreakTestCases lines
--
--package main
--
--import (
-- "bufio"
-- "bytes"
-- "errors"
-- "fmt"
-- "go/format"
-- "io/ioutil"
-- "log"
-- "net/http"
-- "os"
-- "time"
--)
--
--// We want to test against a specific version rather than the latest. When the
--// package is upgraded to a new version, change these to generate new tests.
--const (
-- testCaseURL = `https://www.unicode.org/Public/14.0.0/ucd/auxiliary/%s.txt`
--)
--
--func main() {
-- if len(os.Args) < 5 {
-- fmt.Println("Not enough arguments, see code for details")
-- os.Exit(1)
-- }
--
-- log.SetPrefix("gen_breaktest (" + os.Args[4] + "): ")
-- log.SetFlags(0)
--
-- // Read text of testcases and parse into Go source code.
-- src, err := parse(fmt.Sprintf(testCaseURL, os.Args[1]))
-- if err != nil {
-- log.Fatal(err)
-- }
--
-- // Format the Go code.
-- formatted, err := format.Source(src)
-- if err != nil {
-- log.Fatalln("gofmt:", err)
-- }
--
-- // Write it out.
-- log.Print("Writing to ", os.Args[2])
-- if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil {
-- log.Fatal(err)
-- }
--}
--
--// parse reads a break text file, either from a local file or from a URL. It
--// parses the file data into Go source code representing the test cases.
--func parse(url string) ([]byte, error) {
-- log.Printf("Parsing %s", url)
-- res, err := http.Get(url)
-- if err != nil {
-- return nil, err
-- }
-- body := res.Body
-- defer body.Close()
--
-- buf := new(bytes.Buffer)
-- buf.Grow(120 << 10)
-- buf.WriteString(`package uniseg
--
--// Code generated via go generate from gen_breaktest.go. DO NOT EDIT.
--
--// ` + os.Args[3] + ` are Grapheme testcases taken from
--// ` + url + `
--// on ` + time.Now().Format("January 2, 2006") + `. See
--// https://www.unicode.org/license.html for the Unicode license agreement.
--var ` + os.Args[3] + ` = []testCase {
--`)
--
-- sc := bufio.NewScanner(body)
-- num := 1
-- var line []byte
-- original := make([]byte, 0, 64)
-- expected := make([]byte, 0, 64)
-- for sc.Scan() {
-- num++
-- line = sc.Bytes()
-- if len(line) == 0 || line[0] == '#' {
-- continue
-- }
-- var comment []byte
-- if i := bytes.IndexByte(line, '#'); i >= 0 {
-- comment = bytes.TrimSpace(line[i+1:])
-- line = bytes.TrimSpace(line[:i])
-- }
-- original, expected, err := parseRuneSequence(line, original[:0], expected[:0])
-- if err != nil {
-- return nil, fmt.Errorf(`line %d: %v: %q`, num, err, line)
-- }
-- fmt.Fprintf(buf, "\t{original: \"%s\", expected: %s}, // %s\n", original, expected, comment)
-- }
-- if err := sc.Err(); err != nil {
-- return nil, err
-- }
--
-- // Check for final "# EOF", useful check if we're streaming via HTTP
-- if !bytes.Equal(line, []byte("# EOF")) {
-- return nil, fmt.Errorf(`line %d: exected "# EOF" as final line, got %q`, num, line)
-- }
-- buf.WriteString("}\n")
-- return buf.Bytes(), nil
--}
--
--// Used by parseRuneSequence to match input via bytes.HasPrefix.
--var (
-- prefixBreak = []byte("÷ ")
-- prefixDontBreak = []byte("× ")
-- breakOk = []byte("÷")
-- breakNo = []byte("×")
--)
--
--// parseRuneSequence parses a rune + breaking opportunity sequence from b
--// and appends the Go code for testcase.original to orig
--// and appends the Go code for testcase.expected to exp.
--// It retuns the new orig and exp slices.
--//
--// E.g. for the input b="÷ 0020 × 0308 ÷ 1F1E6 ÷"
--// it will append
--// "\u0020\u0308\U0001F1E6"
--// and "[][]rune{{0x0020,0x0308},{0x1F1E6},}"
--// to orig and exp respectively.
--//
--// The formatting of exp is expected to be cleaned up by gofmt or format.Source.
--// Note we explicitly require the sequence to start with ÷ and we implicitly
--// require it to end with ÷.
--func parseRuneSequence(b, orig, exp []byte) ([]byte, []byte, error) {
-- // Check for and remove first ÷ or ×.
-- if !bytes.HasPrefix(b, prefixBreak) && !bytes.HasPrefix(b, prefixDontBreak) {
-- return nil, nil, errors.New("expected ÷ or × as first character")
-- }
-- if bytes.HasPrefix(b, prefixBreak) {
-- b = b[len(prefixBreak):]
-- } else {
-- b = b[len(prefixDontBreak):]
-- }
--
-- boundary := true
-- exp = append(exp, "[][]rune{"...)
-- for len(b) > 0 {
-- if boundary {
-- exp = append(exp, '{')
-- }
-- exp = append(exp, "0x"...)
-- // Find end of hex digits.
-- var i int
-- for i = 0; i < len(b) && b[i] != ' '; i++ {
-- if d := b[i]; ('0' <= d || d <= '9') ||
-- ('A' <= d || d <= 'F') ||
-- ('a' <= d || d <= 'f') {
-- continue
-- }
-- return nil, nil, errors.New("bad hex digit")
-- }
-- switch i {
-- case 4:
-- orig = append(orig, "\\u"...)
-- case 5:
-- orig = append(orig, "\\U000"...)
-- default:
-- return nil, nil, errors.New("unsupport code point hex length")
-- }
-- orig = append(orig, b[:i]...)
-- exp = append(exp, b[:i]...)
-- b = b[i:]
--
-- // Check for space between hex and ÷ or ×.
-- if len(b) < 1 || b[0] != ' ' {
-- return nil, nil, errors.New("bad input")
-- }
-- b = b[1:]
--
-- // Check for next boundary.
-- switch {
-- case bytes.HasPrefix(b, breakOk):
-- boundary = true
-- b = b[len(breakOk):]
-- case bytes.HasPrefix(b, breakNo):
-- boundary = false
-- b = b[len(breakNo):]
-- default:
-- return nil, nil, errors.New("missing ÷ or ×")
-- }
-- if boundary {
-- exp = append(exp, '}')
-- }
-- exp = append(exp, ',')
-- if len(b) > 0 && b[0] == ' ' {
-- b = b[1:]
-- }
-- }
-- exp = append(exp, '}')
-- return orig, exp, nil
--}
-diff --git a/gen_properties.go b/gen_properties.go
-deleted file mode 100644
-index 999d5ef..0000000
---- a/gen_properties.go
-+++ /dev/null
-@@ -1,256 +0,0 @@
--//go:build generate
--
--// This program generates a property file in Go file from Unicode Character
--// Database auxiliary data files. The command line arguments are as follows:
--//
--// 1. The name of the Unicode data file (just the filename, without extension).
--// Can be "-" (to skip) if the emoji flag is included.
--// 2. The name of the locally generated Go file.
--// 3. The name of the slice mapping code points to properties.
--// 4. The name of the generator, for logging purposes.
--// 5. (Optional) Flags, comma-separated. The following flags are available:
--// - "emojis=": include the specified emoji properties (e.g.
--// "Extended_Pictographic").
--// - "gencat": include general category properties.
--//
--//go:generate go run gen_properties.go auxiliary/GraphemeBreakProperty graphemeproperties.go graphemeCodePoints graphemes emojis=Extended_Pictographic
--//go:generate go run gen_properties.go auxiliary/WordBreakProperty wordproperties.go workBreakCodePoints words emojis=Extended_Pictographic
--//go:generate go run gen_properties.go auxiliary/SentenceBreakProperty sentenceproperties.go sentenceBreakCodePoints sentences
--//go:generate go run gen_properties.go LineBreak lineproperties.go lineBreakCodePoints lines gencat
--//go:generate go run gen_properties.go EastAsianWidth eastasianwidth.go eastAsianWidth eastasianwidth
--//go:generate go run gen_properties.go - emojipresentation.go emojiPresentation emojipresentation emojis=Emoji_Presentation
--package main
--
--import (
-- "bufio"
-- "bytes"
-- "errors"
-- "fmt"
-- "go/format"
-- "io/ioutil"
-- "log"
-- "net/http"
-- "os"
-- "regexp"
-- "sort"
-- "strconv"
-- "strings"
-- "time"
--)
--
--// We want to test against a specific version rather than the latest. When the
--// package is upgraded to a new version, change these to generate new tests.
--const (
-- propertyURL = `https://www.unicode.org/Public/14.0.0/ucd/%s.txt`
-- emojiURL = `https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt`
--)
--
--// The regular expression for a line containing a code point range property.
--var propertyPattern = regexp.MustCompile(`^([0-9A-F]{4,6})(\.\.([0-9A-F]{4,6}))?\s*;\s*([A-Za-z0-9_]+)\s*#\s(.+)$`)
--
--func main() {
-- if len(os.Args) < 5 {
-- fmt.Println("Not enough arguments, see code for details")
-- os.Exit(1)
-- }
--
-- log.SetPrefix("gen_properties (" + os.Args[4] + "): ")
-- log.SetFlags(0)
--
-- // Parse flags.
-- flags := make(map[string]string)
-- if len(os.Args) >= 6 {
-- for _, flag := range strings.Split(os.Args[5], ",") {
-- flagFields := strings.Split(flag, "=")
-- if len(flagFields) == 1 {
-- flags[flagFields[0]] = "yes"
-- } else {
-- flags[flagFields[0]] = flagFields[1]
-- }
-- }
-- }
--
-- // Parse the text file and generate Go source code from it.
-- _, includeGeneralCategory := flags["gencat"]
-- var mainURL string
-- if os.Args[1] != "-" {
-- mainURL = fmt.Sprintf(propertyURL, os.Args[1])
-- }
-- src, err := parse(mainURL, flags["emojis"], includeGeneralCategory)
-- if err != nil {
-- log.Fatal(err)
-- }
--
-- // Format the Go code.
-- formatted, err := format.Source([]byte(src))
-- if err != nil {
-- log.Fatal("gofmt:", err)
-- }
--
-- // Save it to the (local) target file.
-- log.Print("Writing to ", os.Args[2])
-- if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil {
-- log.Fatal(err)
-- }
--}
--
--// parse parses the Unicode Properties text files located at the given URLs and
--// returns their equivalent Go source code to be used in the uniseg package. If
--// "emojiProperty" is not an empty string, emoji code points for that emoji
--// property (e.g. "Extended_Pictographic") will be included. In those cases, you
--// may pass an empty "propertyURL" to skip parsing the main properties file. If
--// "includeGeneralCategory" is true, the Unicode General Category property will
--// be extracted from the comments and included in the output.
--func parse(propertyURL, emojiProperty string, includeGeneralCategory bool) (string, error) {
-- if propertyURL == "" && emojiProperty == "" {
-- return "", errors.New("no properties to parse")
-- }
--
-- // Temporary buffer to hold properties.
-- var properties [][4]string
--
-- // Open the first URL.
-- if propertyURL != "" {
-- log.Printf("Parsing %s", propertyURL)
-- res, err := http.Get(propertyURL)
-- if err != nil {
-- return "", err
-- }
-- in1 := res.Body
-- defer in1.Close()
--
-- // Parse it.
-- scanner := bufio.NewScanner(in1)
-- num := 0
-- for scanner.Scan() {
-- num++
-- line := strings.TrimSpace(scanner.Text())
--
-- // Skip comments and empty lines.
-- if strings.HasPrefix(line, "#") || line == "" {
-- continue
-- }
--
-- // Everything else must be a code point range, a property and a comment.
-- from, to, property, comment, err := parseProperty(line)
-- if err != nil {
-- return "", fmt.Errorf("%s line %d: %v", os.Args[4], num, err)
-- }
-- properties = append(properties, [4]string{from, to, property, comment})
-- }
-- if err := scanner.Err(); err != nil {
-- return "", err
-- }
-- }
--
-- // Open the second URL.
-- if emojiProperty != "" {
-- log.Printf("Parsing %s", emojiURL)
-- res, err := http.Get(emojiURL)
-- if err != nil {
-- return "", err
-- }
-- in2 := res.Body
-- defer in2.Close()
--
-- // Parse it.
-- scanner := bufio.NewScanner(in2)
-- num := 0
-- for scanner.Scan() {
-- num++
-- line := scanner.Text()
--
-- // Skip comments, empty lines, and everything not containing
-- // "Extended_Pictographic".
-- if strings.HasPrefix(line, "#") || line == "" || !strings.Contains(line, emojiProperty) {
-- continue
-- }
--
-- // Everything else must be a code point range, a property and a comment.
-- from, to, property, comment, err := parseProperty(line)
-- if err != nil {
-- return "", fmt.Errorf("emojis line %d: %v", num, err)
-- }
-- properties = append(properties, [4]string{from, to, property, comment})
-- }
-- if err := scanner.Err(); err != nil {
-- return "", err
-- }
-- }
--
-- // Sort properties.
-- sort.Slice(properties, func(i, j int) bool {
-- left, _ := strconv.ParseUint(properties[i][0], 16, 64)
-- right, _ := strconv.ParseUint(properties[j][0], 16, 64)
-- return left < right
-- })
--
-- // Header.
-- var (
-- buf bytes.Buffer
-- emojiComment string
-- )
-- columns := 3
-- if includeGeneralCategory {
-- columns = 4
-- }
-- if emojiURL != "" {
-- emojiComment = `
--// and
--// ` + emojiURL + `
--// ("Extended_Pictographic" only)`
-- }
-- buf.WriteString(`package uniseg
--
--// Code generated via go generate from gen_properties.go. DO NOT EDIT.
--
--// ` + os.Args[3] + ` are taken from
--// ` + propertyURL + emojiComment + `
--// on ` + time.Now().Format("January 2, 2006") + `. See https://www.unicode.org/license.html for the Unicode
--// license agreement.
--var ` + os.Args[3] + ` = [][` + strconv.Itoa(columns) + `]int{
-- `)
--
-- // Properties.
-- for _, prop := range properties {
-- if includeGeneralCategory {
-- generalCategory := "gc" + prop[3][:2]
-- if generalCategory == "gcL&" {
-- generalCategory = "gcLC"
-- }
-- prop[3] = prop[3][3:]
-- fmt.Fprintf(&buf, "{0x%s,0x%s,%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), generalCategory, prop[3])
-- } else {
-- fmt.Fprintf(&buf, "{0x%s,0x%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), prop[3])
-- }
-- }
--
-- // Tail.
-- buf.WriteString("}")
--
-- return buf.String(), nil
--}
--
--// parseProperty parses a line of the Unicode properties text file containing a
--// property for a code point range and returns it along with its comment.
--func parseProperty(line string) (from, to, property, comment string, err error) {
-- fields := propertyPattern.FindStringSubmatch(line)
-- if fields == nil {
-- err = errors.New("no property found")
-- return
-- }
-- from = fields[1]
-- to = fields[3]
-- if to == "" {
-- to = from
-- }
-- property = fields[4]
-- comment = fields[5]
-- return
--}
--
--// translateProperty translates a property name as used in the Unicode data file
--// to a variable used in the Go code.
--func translateProperty(prefix, property string) string {
-- return prefix + strings.ReplaceAll(property, "_", "")
--}
---
-2.38.0
-
diff --git a/build/patches/io_etcd_go_etcd_raft_v3.patch b/build/patches/io_etcd_go_etcd_raft_v3.patch
deleted file mode 100644
index 11c777b38f4de..0000000000000
--- a/build/patches/io_etcd_go_etcd_raft_v3.patch
+++ /dev/null
@@ -1,22 +0,0 @@
-diff -urN a/raftpb/BUILD.bazel b/raftpb/BUILD.bazel
---- a/raftpb/BUILD.bazel 1969-12-31 19:00:00.000000000 -0500
-+++ b/raftpb/BUILD.bazel 2000-01-01 00:00:00.000000000 -0000
-@@ -1,4 +1,5 @@
- load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-+load("@rules_proto//proto:defs.bzl", "proto_library")
-
- go_library(
- name = "raftpb",
-@@ -28,3 +29,12 @@
- srcs = ["confstate_test.go"],
- embed = [":raftpb"],
- )
-+
-+# keep
-+proto_library(
-+ name = "raftpb_proto",
-+ srcs = ["raft.proto"],
-+ deps = ["@com_github_gogo_protobuf//gogoproto:gogo_proto"],
-+ import_prefix = "etcd/raft/v3/",
-+ visibility = ["//visibility:public"],
-+)
diff --git a/ci.md b/ci.md
index 8b5abfd8fefd3..f7ebabd7a1331 100644
--- a/ci.md
+++ b/ci.md
@@ -2,30 +2,24 @@
## Guide
-1. ci pipeline will be triggered when your comment on pull request matched command.
-2. "**Only triggered by command**". What does that mean?
- * Yes, this ci will be triggered only when your comment on pr matched command.
- * No, this ci will be triggered by every new commit on current pr, comment matched command also trigger ci pipeline.
+ci pipeline will be triggered when your comment on pull request matched command. But we have some task that will be triggered manually.
## Commands
-| ci pipeline | Commands | Only triggered by command |
-| ---------------------------------------- | ------------------------------------------------------------ | ------------------------- |
-| tidb_ghpr_build | /run-build
/run-all-tests
/merge | No |
-| tidb_ghpr_check | /run-check_dev
/run-all-tests
/merge | No |
-| tidb_ghpr_check_2 | /run-check_dev_2
/run-all-tests
/merge | No |
-| tidb_ghpr_coverage | /run-coverage | Yes |
-| tidb_ghpr_build_arm64 | /run-build-arm64 | Yes |
-| tidb_ghpr_common_test | /run-common-test
/run-integration-tests | Yes |
-| tidb_ghpr_integration_br_test | /run-integration-br-test
/run-integration-tests | Yes |
-| tidb_ghpr_integration_campatibility_test | /run-integration-compatibility-test
/run-integration-tests | Yes |
-| tidb_ghpr_integration_common_test | /run-integration-common-test
/run-integration-tests | Yes |
-| tidb_ghpr_integration_copr_test | /run-integration-copr-test
/run-integration-tests | Yes |
-| tidb_ghpr_integration_ddl_test | /run-integration-ddl-test
/run-integration-tests | Yes |
-| tidb_ghpr_monitor_test | /run-monitor-test | Yes |
-| tidb_ghpr_mybatis | /run-mybatis-test
/run-integration-tests | Yes |
-| tidb_ghpr_sqllogic_test_1 | /run-sqllogic-test
/run-integration-tests | Yes |
-| tidb_ghpr_sqllogic_test_2 | /run-sqllogic-test
/run-integration-tests | Yes |
-| tidb_ghpr_tics_test | /run-tics-test
/run-integration-tests | Yes |
-| tidb_ghpr_unit_test | /run-unit-test
/run-all-tests
/merge | Yes |
+| ci pipeline | Commands |
+| ---------------------------------------- |-----------------------------------------------------------------|
+| tidb_ghpr_coverage | /run-coverage |
+| tidb_ghpr_build_arm64 | /run-build-arm64 comment=true |
+| tidb_ghpr_common_test | /run-common-test
/run-integration-tests |
+| tidb_ghpr_integration_br_test | /run-integration-br-test
/run-integration-tests |
+| tidb_ghpr_integration_campatibility_test | /run-integration-compatibility-test
/run-integration-tests |
+| tidb_ghpr_integration_common_test | /run-integration-common-test
/run-integration-tests |
+| tidb_ghpr_integration_copr_test | /run-integration-copr-test
/run-integration-tests |
+| tidb_ghpr_integration_ddl_test | /run-integration-ddl-test
/run-integration-tests |
+| tidb_ghpr_monitor_test | /run-monitor-test |
+| tidb_ghpr_mybatis | /run-mybatis-test
/run-integration-tests |
+| tidb_ghpr_sqllogic_test_1 | /run-sqllogic-test
/run-integration-tests |
+| tidb_ghpr_sqllogic_test_2 | /run-sqllogic-test
/run-integration-tests |
+| tidb_ghpr_tics_test | /run-tics-test
/run-integration-tests |
+| tidb_ghpr_unit_test | /run-unit-test
/run-all-tests
/merge |
diff --git a/cmd/benchkv/main.go b/cmd/benchkv/main.go
index 73a388bebbfe5..2b7cf5c9e9abb 100644
--- a/cmd/benchkv/main.go
+++ b/cmd/benchkv/main.go
@@ -14,7 +14,6 @@
package main
-// #nosec G108
import (
"context"
"flag"
diff --git a/cmd/benchraw/main.go b/cmd/benchraw/main.go
index 80f1a1d2289bc..c042d59e9f180 100644
--- a/cmd/benchraw/main.go
+++ b/cmd/benchraw/main.go
@@ -14,7 +14,6 @@
package main
-// #nosec G108
import (
"context"
"flag"
diff --git a/cmd/ddltest/BUILD.bazel b/cmd/ddltest/BUILD.bazel
index 344d46dac3cad..39ac3c9bdb7b4 100644
--- a/cmd/ddltest/BUILD.bazel
+++ b/cmd/ddltest/BUILD.bazel
@@ -11,6 +11,7 @@ go_test(
"random_test.go",
],
flaky = True,
+ race = "on",
deps = [
"//config",
"//domain",
diff --git a/cmd/ddltest/index_test.go b/cmd/ddltest/index_test.go
index 7d3206197eba4..dbd96aa62aa39 100644
--- a/cmd/ddltest/index_test.go
+++ b/cmd/ddltest/index_test.go
@@ -18,11 +18,13 @@ import (
goctx "context"
"fmt"
"math"
+ "os"
"sync"
"sync/atomic"
"testing"
"time"
+ "github.com/pingcap/log"
"github.com/pingcap/tidb/store/gcworker"
"github.com/pingcap/tidb/table"
"github.com/stretchr/testify/require"
@@ -48,6 +50,11 @@ func (s *ddlSuite) checkDropIndex(t *testing.T, tableName string) {
// TestIndex operations on table test_index (c int, c1 bigint, c2 double, c3 varchar(256), primary key(c)).
func TestIndex(t *testing.T) {
+ err := os.Setenv("tidb_manager_ttl", fmt.Sprintf("%d", *lease+5))
+ if err != nil {
+ log.Fatal("set tidb_manager_ttl failed")
+ }
+
s := createDDLSuite(t)
defer s.teardown(t)
diff --git a/cmd/ddltest/main_test.go b/cmd/ddltest/main_test.go
index 7f6815d4f6340..d0ca25b750a14 100644
--- a/cmd/ddltest/main_test.go
+++ b/cmd/ddltest/main_test.go
@@ -41,6 +41,7 @@ func TestMain(m *testing.M) {
goleak.IgnoreTopFunction("net/http.(*persistConn).writeLoop"),
goleak.IgnoreTopFunction("github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1"),
goleak.IgnoreTopFunction("database/sql.(*DB).connectionOpener"),
+ goleak.IgnoreTopFunction("go.etcd.io/etcd/client/v3.waitRetryBackoff"),
}
goleak.VerifyTestMain(m, opts...)
}
diff --git a/cmd/explaintest/r/collation_agg_func_disabled.result b/cmd/explaintest/r/collation_agg_func_disabled.result
index 4b4a9d1074af8..b5075b3c82f83 100644
--- a/cmd/explaintest/r/collation_agg_func_disabled.result
+++ b/cmd/explaintest/r/collation_agg_func_disabled.result
@@ -133,13 +133,13 @@ approx_count_distinct(value collate utf8mb4_bin, value1)
create table tt(a char(10), b enum('a', 'B', 'c'), c set('a', 'B', 'c'), d json) collate utf8mb4_general_ci;
insert into tt values ("a", "a", "a", JSON_OBJECT("a", "a"));
insert into tt values ("A", "A", "A", JSON_OBJECT("A", "A"));
-Error 1265: Data truncated for column 'b' at row 1
+Error 1265 (01000): Data truncated for column 'b' at row 1
insert into tt values ("b", "b", "b", JSON_OBJECT("b", "b"));
-Error 1265: Data truncated for column 'b' at row 1
+Error 1265 (01000): Data truncated for column 'b' at row 1
insert into tt values ("B", "B", "B", JSON_OBJECT("B", "B"));
insert into tt values ("c", "c", "c", JSON_OBJECT("c", "c"));
insert into tt values ("C", "C", "C", JSON_OBJECT("C", "C"));
-Error 1265: Data truncated for column 'b' at row 1
+Error 1265 (01000): Data truncated for column 'b' at row 1
split table tt by (0), (1), (2), (3), (4), (5);
desc format='brief' select min(a) from tt;
id estRows task access object operator info
@@ -210,9 +210,9 @@ select min(b) from tt;
min(b)
B
desc format='brief' select min(b collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
select min(b collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
desc format='brief' select max(b) from tt;
id estRows task access object operator info
StreamAgg 1.00 root funcs:max(Column#8)->Column#6
@@ -223,9 +223,9 @@ select max(b) from tt;
max(b)
c
desc format='brief' select max(b collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
select max(b collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
desc format='brief' select min(c) from tt;
id estRows task access object operator info
HashAgg 1.00 root funcs:min(collation_agg_func.tt.c)->Column#6
@@ -235,9 +235,9 @@ select min(c) from tt;
min(c)
B
desc format='brief' select min(c collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
select min(c collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
desc format='brief' select max(c) from tt;
id estRows task access object operator info
HashAgg 1.00 root funcs:max(collation_agg_func.tt.c)->Column#6
@@ -247,9 +247,9 @@ select max(c) from tt;
max(c)
c
desc format='brief' select max(c collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
select max(c collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
desc format='brief' select min(d) from tt;
id estRows task access object operator info
StreamAgg 1.00 root funcs:min(collation_agg_func.tt.d)->Column#6
diff --git a/cmd/explaintest/r/collation_agg_func_enabled.result b/cmd/explaintest/r/collation_agg_func_enabled.result
index ea345c13ede58..5b587ff02d279 100644
--- a/cmd/explaintest/r/collation_agg_func_enabled.result
+++ b/cmd/explaintest/r/collation_agg_func_enabled.result
@@ -207,9 +207,9 @@ select min(b) from tt;
min(b)
a
desc format='brief' select min(b collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
select min(b collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
desc format='brief' select max(b) from tt;
id estRows task access object operator info
StreamAgg 1.00 root funcs:max(Column#8)->Column#6
@@ -220,9 +220,9 @@ select max(b) from tt;
max(b)
c
desc format='brief' select max(b collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
select max(b collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
desc format='brief' select min(c) from tt;
id estRows task access object operator info
HashAgg 1.00 root funcs:min(collation_agg_func.tt.c)->Column#6
@@ -232,9 +232,9 @@ select min(c) from tt;
min(c)
a
desc format='brief' select min(c collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
select min(c collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
desc format='brief' select max(c) from tt;
id estRows task access object operator info
HashAgg 1.00 root funcs:max(collation_agg_func.tt.c)->Column#6
@@ -244,9 +244,9 @@ select max(c) from tt;
max(c)
c
desc format='brief' select max(c collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
select max(c collate utf8mb4_bin) from tt;
-Error 1235: This version of TiDB doesn't yet support 'use collate clause for enum or set'
+Error 1235 (42000): This version of TiDB doesn't yet support 'use collate clause for enum or set'
desc format='brief' select min(d) from tt;
id estRows task access object operator info
StreamAgg 1.00 root funcs:min(collation_agg_func.tt.d)->Column#6
diff --git a/cmd/explaintest/r/collation_check_use_collation_disabled.result b/cmd/explaintest/r/collation_check_use_collation_disabled.result
index 493b26812bfbb..9e633133b1f4f 100644
--- a/cmd/explaintest/r/collation_check_use_collation_disabled.result
+++ b/cmd/explaintest/r/collation_check_use_collation_disabled.result
@@ -32,7 +32,7 @@ drop table if exists t;
create table t(a enum('a', 'b') charset utf8mb4 collate utf8mb4_general_ci, b varchar(20));
insert into t values ("b", "c");
insert into t values ("B", "b");
-Error 1265: Data truncated for column 'a' at row 1
+Error 1265 (01000): Data truncated for column 'a' at row 1
select * from t where 'B' collate utf8mb4_general_ci in (a);
a b
select * from t where 'B' collate utf8mb4_bin in (a);
@@ -82,7 +82,7 @@ drop table if exists t;
create table t(a set('a', 'b') charset utf8mb4 collate utf8mb4_general_ci, b varchar(20));
insert into t values ("b", "c");
insert into t values ("B", "b");
-Error 1265: Data truncated for column 'a' at row 1
+Error 1265 (01000): Data truncated for column 'a' at row 1
select * from t where 'B' collate utf8mb4_general_ci in (a);
a b
select * from t where 'B' collate utf8mb4_bin in (a);
@@ -154,4 +154,28 @@ insert into t1 values ('-1');
insert into t2 values (0x2d31, '');
select * from t1, t2 where t1.a in (t2.b, 3);
a b c
+drop table if exists t0;
+drop table if exists t1;
+CREATE TABLE t0(c0 BOOL, c1 INT);
+CREATE TABLE t1 LIKE t0;
+CREATE VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;
+INSERT INTO t0(c0, c1) VALUES (true, 0);
+INSERT INTO t1(c0, c1) VALUES (true, 2);
+SELECT v0.c0 FROM v0;
+c0
+0
+SELECT (v0.c0)NOT LIKE(BINARY v0.c0) FROM v0;
+(v0.c0)NOT LIKE(BINARY v0.c0)
+0
+SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+c0
+desc format='brief' SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+id estRows task access object operator info
+Projection 80000000.00 root is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20)))->Column#7
+└─HashJoin 80000000.00 root CARTESIAN inner join
+ ├─Selection(Build) 8000.00 root not(like(cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), var_string(20)), cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), binary(1)), 92))
+ │ └─TableReader 10000.00 root data:TableFullScan
+ │ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo
+ └─TableReader(Probe) 10000.00 root data:TableFullScan
+ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
use test
diff --git a/cmd/explaintest/r/collation_check_use_collation_enabled.result b/cmd/explaintest/r/collation_check_use_collation_enabled.result
index 838c6beba6535..3f1113fbcd868 100644
--- a/cmd/explaintest/r/collation_check_use_collation_enabled.result
+++ b/cmd/explaintest/r/collation_check_use_collation_enabled.result
@@ -173,4 +173,28 @@ insert into t1 values ('-1');
insert into t2 values (0x2d31, '');
select * from t1, t2 where t1.a in (t2.b, 3);
a b c
+drop table if exists t0;
+drop table if exists t1;
+CREATE TABLE t0(c0 BOOL, c1 INT);
+CREATE TABLE t1 LIKE t0;
+CREATE VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;
+INSERT INTO t0(c0, c1) VALUES (true, 0);
+INSERT INTO t1(c0, c1) VALUES (true, 2);
+SELECT v0.c0 FROM v0;
+c0
+0
+SELECT (v0.c0)NOT LIKE(BINARY v0.c0) FROM v0;
+(v0.c0)NOT LIKE(BINARY v0.c0)
+0
+SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+c0
+desc format='brief' SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+id estRows task access object operator info
+Projection 80000000.00 root is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20)))->Column#7
+└─HashJoin 80000000.00 root CARTESIAN inner join
+ ├─Selection(Build) 8000.00 root not(like(cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), var_string(20)), cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), binary(1)), 92))
+ │ └─TableReader 10000.00 root data:TableFullScan
+ │ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo
+ └─TableReader(Probe) 10000.00 root data:TableFullScan
+ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
use test
diff --git a/cmd/explaintest/r/collation_misc_disabled.result b/cmd/explaintest/r/collation_misc_disabled.result
index eff046aa33009..a66f63ead2db9 100644
--- a/cmd/explaintest/r/collation_misc_disabled.result
+++ b/cmd/explaintest/r/collation_misc_disabled.result
@@ -15,7 +15,7 @@ select * from t;
a
t_value
alter table t modify column a varchar(20) charset utf8;
-Error 8200: Unsupported modify charset from latin1 to utf8
+Error 8200 (HY000): Unsupported modify charset from latin1 to utf8
drop table t;
create table t(a varchar(20) charset latin1);
insert into t values ("t_value");
@@ -38,13 +38,13 @@ drop table t;
create table t(a varchar(20) charset latin1);
insert into t values ("t_value");
alter table t modify column a varchar(20) charset utf8 collate utf8_bin;
-Error 8200: Unsupported modify charset from latin1 to utf8
+Error 8200 (HY000): Unsupported modify charset from latin1 to utf8
alter table t modify column a varchar(20) charset utf8mb4 collate utf8bin;
[ddl:1273]Unknown collation: 'utf8bin'
alter table t collate LATIN1_GENERAL_CI charset utf8 collate utf8_bin;
-Error 1302: Conflicting declarations: 'CHARACTER SET latin1' and 'CHARACTER SET utf8'
+Error 1302 (HY000): Conflicting declarations: 'CHARACTER SET latin1' and 'CHARACTER SET utf8'
alter table t collate LATIN1_GENERAL_CI collate UTF8MB4_UNICODE_ci collate utf8_bin;
-Error 1253: COLLATION 'utf8mb4_unicode_ci' is not valid for CHARACTER SET 'latin1'
+Error 1253 (42000): COLLATION 'utf8mb4_unicode_ci' is not valid for CHARACTER SET 'latin1'
drop table t;
create table t(a varchar(20) charset latin1);
insert into t values ("t_value");
diff --git a/cmd/explaintest/r/collation_misc_enabled.result b/cmd/explaintest/r/collation_misc_enabled.result
index 235d5896380ee..a088ddb0b2c9d 100644
--- a/cmd/explaintest/r/collation_misc_enabled.result
+++ b/cmd/explaintest/r/collation_misc_enabled.result
@@ -15,7 +15,7 @@ select * from t;
a
t_value
alter table t modify column a varchar(20) charset utf8;
-Error 8200: Unsupported modify charset from latin1 to utf8
+Error 8200 (HY000): Unsupported modify charset from latin1 to utf8
drop table t;
create table t(a varchar(20) charset latin1);
insert into t values ("t_value");
@@ -38,13 +38,13 @@ drop table t;
create table t(a varchar(20) charset latin1);
insert into t values ("t_value");
alter table t modify column a varchar(20) charset utf8 collate utf8_bin;
-Error 8200: Unsupported modify charset from latin1 to utf8
+Error 8200 (HY000): Unsupported modify charset from latin1 to utf8
alter table t modify column a varchar(20) charset utf8mb4 collate utf8bin;
[ddl:1273]Unknown collation: 'utf8bin'
alter table t collate LATIN1_GENERAL_CI charset utf8 collate utf8_bin;
-Error 1273: Unsupported collation when new collation is enabled: 'latin1_general_ci'
+Error 1273 (HY000): Unsupported collation when new collation is enabled: 'latin1_general_ci'
alter table t collate LATIN1_GENERAL_CI collate UTF8MB4_UNICODE_ci collate utf8_bin;
-Error 1273: Unsupported collation when new collation is enabled: 'latin1_general_ci'
+Error 1273 (HY000): Unsupported collation when new collation is enabled: 'latin1_general_ci'
drop table t;
create table t(a varchar(20) charset latin1);
insert into t values ("t_value");
@@ -56,7 +56,7 @@ a
t_value
create database if not exists cd_test_utf8 CHARACTER SET utf8 COLLATE utf8_bin;
create database if not exists cd_test_latin1 CHARACTER SET latin1 COLLATE latin1_swedish_ci;
-Error 1273: Unsupported collation when new collation is enabled: 'latin1_swedish_ci'
+Error 1273 (HY000): Unsupported collation when new collation is enabled: 'latin1_swedish_ci'
use cd_test_utf8;
select @@character_set_database;
@@character_set_database
@@ -65,7 +65,7 @@ select @@collation_database;
@@collation_database
utf8_bin
use cd_test_latin1;
-Error 1049: Unknown database 'cd_test_latin1'
+Error 1049 (42000): Unknown database 'cd_test_latin1'
select @@character_set_database;
@@character_set_database
utf8
@@ -73,7 +73,7 @@ select @@collation_database;
@@collation_database
utf8_bin
create database if not exists test_db CHARACTER SET latin1 COLLATE latin1_swedish_ci;
-Error 1273: Unsupported collation when new collation is enabled: 'latin1_swedish_ci'
+Error 1273 (HY000): Unsupported collation when new collation is enabled: 'latin1_swedish_ci'
with cte as (select cast('2010-09-09' as date) a union select '2010-09-09 ') select count(*) from cte;
count(*)
1
diff --git a/cmd/explaintest/r/collation_pointget_disabled.result b/cmd/explaintest/r/collation_pointget_disabled.result
index a763c54c96344..db7b0a9ab630f 100644
--- a/cmd/explaintest/r/collation_pointget_disabled.result
+++ b/cmd/explaintest/r/collation_pointget_disabled.result
@@ -111,15 +111,15 @@ select *, a, b from t tmp where tmp.a = "aa";
a b a b
aa bb aa bb
select a from t where xxxxx.a = "aa";
-Error 1054: Unknown column 'xxxxx.a' in 'where clause'
+Error 1054 (42S22): Unknown column 'xxxxx.a' in 'where clause'
select xxxxx.a from t where a = "aa";
-Error 1054: Unknown column 'xxxxx.a' in 'field list'
+Error 1054 (42S22): Unknown column 'xxxxx.a' in 'field list'
select a from t tmp where t.a = "aa";
-Error 1054: Unknown column 't.a' in 'where clause'
+Error 1054 (42S22): Unknown column 't.a' in 'where clause'
select t.a from t tmp where a = "aa";
-Error 1054: Unknown column 't.a' in 'field list'
+Error 1054 (42S22): Unknown column 't.a' in 'field list'
select t.* from t tmp where a = "aa";
-Error 1051: Unknown table 't'
+Error 1051 (42S02): Unknown table 't'
drop table if exists t;
create table t(a char(4) primary key, b char(4));
insert into t values("aa", "bb");
diff --git a/cmd/explaintest/r/collation_pointget_enabled.result b/cmd/explaintest/r/collation_pointget_enabled.result
index 156e4d87f931c..7c404177ce587 100644
--- a/cmd/explaintest/r/collation_pointget_enabled.result
+++ b/cmd/explaintest/r/collation_pointget_enabled.result
@@ -124,15 +124,15 @@ select *, a, b from t tmp where tmp.a = "aa";
a b a b
aa bb aa bb
select a from t where xxxxx.a = "aa";
-Error 1054: Unknown column 'xxxxx.a' in 'where clause'
+Error 1054 (42S22): Unknown column 'xxxxx.a' in 'where clause'
select xxxxx.a from t where a = "aa";
-Error 1054: Unknown column 'xxxxx.a' in 'field list'
+Error 1054 (42S22): Unknown column 'xxxxx.a' in 'field list'
select a from t tmp where t.a = "aa";
-Error 1054: Unknown column 't.a' in 'where clause'
+Error 1054 (42S22): Unknown column 't.a' in 'where clause'
select t.a from t tmp where a = "aa";
-Error 1054: Unknown column 't.a' in 'field list'
+Error 1054 (42S22): Unknown column 't.a' in 'field list'
select t.* from t tmp where a = "aa";
-Error 1051: Unknown table 't'
+Error 1051 (42S02): Unknown table 't'
drop table if exists t;
create table t(a char(4) primary key, b char(4));
insert into t values("aa", "bb");
diff --git a/cmd/explaintest/r/cte.result b/cmd/explaintest/r/cte.result
index 7a8677c848294..6f277815bb566 100644
--- a/cmd/explaintest/r/cte.result
+++ b/cmd/explaintest/r/cte.result
@@ -91,7 +91,7 @@ c1 c2
1 1
1 2
with recursive tbl_0 (col_943,col_944,col_945,col_946,col_947) AS ( with recursive tbl_0 (col_948,col_949,col_950,col_951,col_952) AS ( select 1, 2,3,4,5 UNION ALL select col_948 + 1,col_949 + 1,col_950 + 1,col_951 + 1,col_952 + 1 from tbl_0 where col_948 < 5 ) select col_948,col_949,col_951,col_950,col_952 from tbl_0 UNION ALL select col_943 + 1,col_944 + 1,col_945 + 1,col_946 + 1,col_947 + 1 from tbl_0 where col_943 < 5 ) select * from tbl_0;
-Error 1054: Unknown column 'col_943' in 'where clause'
+Error 1054 (42S22): Unknown column 'col_943' in 'where clause'
with recursive cte1 (c1, c2) as (select 1, '1' union select concat(c1, 1), c2 + 1 from cte1 where c1 < 100) select * from cte1;
c1 c2
1 1
@@ -283,15 +283,15 @@ union all
select 3, 0 from qn
)
select * from qn;
-Error 1222: The used SELECT statements have a different number of columns
+Error 1222 (21000): The used SELECT statements have a different number of columns
with recursive cte1 as (select 1 union all (select 1 from cte1 limit 10)) select * from cte1;
-Error 1235: This version of TiDB doesn't yet support 'ORDER BY / LIMIT / SELECT DISTINCT in recursive query block of Common Table Expression'
+Error 1235 (42000): This version of TiDB doesn't yet support 'ORDER BY / LIMIT / SELECT DISTINCT in recursive query block of Common Table Expression'
with recursive qn as (select 123 as a union all select null from qn where a is not null) select * from qn;
a
123
NULL
with recursive q (b) as (select 1, 1 union all select 1, 1 from q) select b from q;
-Error 1353: In definition of view, derived table or common table expression, SELECT list and column names list have different column counts
+Error 1353 (HY000): In definition of view, derived table or common table expression, SELECT list and column names list have different column counts
drop table if exists t1;
create table t1(a int);
insert into t1 values(1);
@@ -356,7 +356,7 @@ drop table if exists t1;
create table t1(c1 bigint unsigned);
insert into t1 values(0);
with recursive cte1 as (select c1 - 1 c1 from t1 union all select c1 - 1 c1 from cte1 where c1 != 0) select * from cte1 dt1, cte1 dt2;
-Error 1690: BIGINT UNSIGNED value is out of range in '(test.t1.c1 - 1)'
+Error 1690 (22003): BIGINT UNSIGNED value is out of range in '(test.t1.c1 - 1)'
drop table if exists t;
create table t(a int, b int, key (b));
desc with cte as (select * from t) select * from cte;
diff --git a/cmd/explaintest/r/explain_complex.result b/cmd/explaintest/r/explain_complex.result
index 8d8b47237b453..d8e1f186a4028 100644
--- a/cmd/explaintest/r/explain_complex.result
+++ b/cmd/explaintest/r/explain_complex.result
@@ -243,6 +243,7 @@ created_on datetime DEFAULT NULL,
updated_on datetime DEFAULT NULL,
UNIQUE KEY org_employee_position_pk (hotel_id,user_id,position_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+set tidb_cost_model_version=2;
explain format = 'brief' SELECT d.id, d.ctx, d.name, d.left_value, d.right_value, d.depth, d.leader_id, d.status, d.created_on, d.updated_on FROM org_department AS d LEFT JOIN org_position AS p ON p.department_id = d.id AND p.status = 1000 LEFT JOIN org_employee_position AS ep ON ep.position_id = p.id AND ep.status = 1000 WHERE (d.ctx = 1 AND (ep.user_id = 62 OR d.id = 20 OR d.id = 20) AND d.status = 1000) GROUP BY d.id ORDER BY d.left_value;
id estRows task access object operator info
Sort 1.00 root test.org_department.left_value
@@ -262,6 +263,7 @@ Sort 1.00 root test.org_department.left_value
└─TableReader(Probe) 9.99 root data:Selection
└─Selection 9.99 cop[tikv] eq(test.org_employee_position.status, 1000), not(isnull(test.org_employee_position.position_id))
└─TableFullScan 10000.00 cop[tikv] table:ep keep order:false, stats:pseudo
+set tidb_cost_model_version=1;
create table test.Tab_A (id int primary key,bid int,cid int,name varchar(20),type varchar(20),num int,amt decimal(11,2));
create table test.Tab_B (id int primary key,name varchar(20));
create table test.Tab_C (id int primary key,name varchar(20),amt decimal(11,2));
diff --git a/cmd/explaintest/r/explain_foreign_key.result b/cmd/explaintest/r/explain_foreign_key.result
new file mode 100644
index 0000000000000..2e92440278a49
--- /dev/null
+++ b/cmd/explaintest/r/explain_foreign_key.result
@@ -0,0 +1,167 @@
+set @@foreign_key_checks=1;
+use test;
+drop table if exists t1,t2;
+create table t1 (id int key);
+create table t2 (id int key, foreign key fk(id) references t1(id) ON UPDATE CASCADE ON DELETE CASCADE);
+create table t3 (id int, unique index idx(id));
+create table t4 (id int, index idx_id(id),foreign key fk(id) references t3(id));
+create table t5 (id int key, id2 int, id3 int, unique index idx2(id2), index idx3(id3));
+create table t6 (id int, id2 int, id3 int, index idx_id(id), index idx_id2(id2), foreign key fk_1 (id) references t5(id) ON UPDATE CASCADE ON DELETE CASCADE, foreign key fk_2 (id2) references t5(id2) ON UPDATE CASCADE, foreign key fk_3 (id3) references t5(id3) ON DELETE CASCADE);
+explain format = 'brief' insert into t2 values (1);
+id estRows task access object operator info
+Insert N/A root N/A
+└─Foreign_Key_Check 0.00 root table:t1 foreign_key:fk, check_exist
+explain format = 'brief' update t2 set id=id+1 where id = 1;
+id estRows task access object operator info
+Update N/A root N/A
+├─Point_Get 1.00 root table:t2 handle:1, lock
+└─Foreign_Key_Check 0.00 root table:t1 foreign_key:fk, check_exist
+explain format = 'brief' delete from t1 where id > 1;
+id estRows task access object operator info
+Delete N/A root N/A
+├─SelectLock 3333.33 root for update 0
+│ └─TableReader 3333.33 root data:TableRangeScan
+│ └─TableRangeScan 3333.33 cop[tikv] table:t1 range:(1,+inf], keep order:false, stats:pseudo
+└─Foreign_Key_Cascade 0.00 root table:t2 foreign_key:fk, on_delete:CASCADE
+explain format = 'brief' update t1 set id=id+1 where id = 1;
+id estRows task access object operator info
+Update N/A root N/A
+├─Point_Get 1.00 root table:t1 handle:1, lock
+└─Foreign_Key_Cascade 0.00 root table:t2 foreign_key:fk, on_update:CASCADE
+explain format = 'brief' insert into t1 values (1);
+id estRows task access object operator info
+Insert N/A root N/A
+explain format = 'brief' insert into t1 values (1) on duplicate key update id = 100;
+id estRows task access object operator info
+Insert N/A root N/A
+└─Foreign_Key_Cascade 0.00 root table:t2 foreign_key:fk, on_update:CASCADE
+explain format = 'brief' insert into t4 values (1);
+id estRows task access object operator info
+Insert N/A root N/A
+└─Foreign_Key_Check 0.00 root table:t3, index:idx foreign_key:fk, check_exist
+explain format = 'brief' update t4 set id=id+1 where id = 1;
+id estRows task access object operator info
+Update N/A root N/A
+├─SelectLock 10.00 root for update 0
+│ └─IndexReader 10.00 root index:IndexRangeScan
+│ └─IndexRangeScan 10.00 cop[tikv] table:t4, index:idx_id(id) range:[1,1], keep order:false, stats:pseudo
+└─Foreign_Key_Check 0.00 root table:t3, index:idx foreign_key:fk, check_exist
+explain format = 'brief' delete from t3 where id > 1;
+id estRows task access object operator info
+Delete N/A root N/A
+├─SelectLock 3333.33 root for update 0
+│ └─IndexReader 3333.33 root index:IndexRangeScan
+│ └─IndexRangeScan 3333.33 cop[tikv] table:t3, index:idx(id) range:(1,+inf], keep order:false, stats:pseudo
+└─Foreign_Key_Check 0.00 root table:t4, index:idx_id foreign_key:fk, check_not_exist
+explain format = 'brief' update t3 set id=id+1 where id = 1;
+id estRows task access object operator info
+Update N/A root N/A
+├─Point_Get 1.00 root table:t3, index:idx(id) lock
+└─Foreign_Key_Check 0.00 root table:t4, index:idx_id foreign_key:fk, check_not_exist
+explain format = 'brief' insert into t3 values (1);
+id estRows task access object operator info
+Insert N/A root N/A
+explain format = 'brief' insert into t3 values (1) on duplicate key update id = 100;
+id estRows task access object operator info
+Insert N/A root N/A
+└─Foreign_Key_Check 0.00 root table:t4, index:idx_id foreign_key:fk, check_not_exist
+explain format = 'brief' insert into t6 values (1,1,1);
+id estRows task access object operator info
+Insert N/A root N/A
+├─Foreign_Key_Check 0.00 root table:t5 foreign_key:fk_1, check_exist
+├─Foreign_Key_Check 0.00 root table:t5, index:idx2 foreign_key:fk_2, check_exist
+└─Foreign_Key_Check 0.00 root table:t5, index:idx3 foreign_key:fk_3, check_exist
+explain format = 'brief' update t6 set id=id+1, id3=id2+1 where id = 1;
+id estRows task access object operator info
+Update N/A root N/A
+├─SelectLock 10.00 root for update 0
+│ └─IndexLookUp 10.00 root
+│ ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t6, index:idx_id(id) range:[1,1], keep order:false, stats:pseudo
+│ └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t6 keep order:false, stats:pseudo
+├─Foreign_Key_Check 0.00 root table:t5 foreign_key:fk_1, check_exist
+└─Foreign_Key_Check 0.00 root table:t5, index:idx3 foreign_key:fk_3, check_exist
+explain format = 'brief' delete from t5 where id > 1;
+id estRows task access object operator info
+Delete N/A root N/A
+├─SelectLock 3333.33 root for update 0
+│ └─TableReader 3333.33 root data:TableRangeScan
+│ └─TableRangeScan 3333.33 cop[tikv] table:t5 range:(1,+inf], keep order:false, stats:pseudo
+├─Foreign_Key_Check 0.00 root table:t6, index:idx_id2 foreign_key:fk_2, check_not_exist
+├─Foreign_Key_Cascade 0.00 root table:t6, index:idx_id foreign_key:fk_1, on_delete:CASCADE
+└─Foreign_Key_Cascade 0.00 root table:t6, index:fk_3 foreign_key:fk_3, on_delete:CASCADE
+explain format = 'brief' update t5 set id=id+1, id2=id2+1 where id = 1;
+id estRows task access object operator info
+Update N/A root N/A
+├─Point_Get 1.00 root table:t5 handle:1, lock
+├─Foreign_Key_Cascade 0.00 root table:t6, index:idx_id foreign_key:fk_1, on_update:CASCADE
+└─Foreign_Key_Cascade 0.00 root table:t6, index:idx_id2 foreign_key:fk_2, on_update:CASCADE
+explain format = 'brief' update t5 set id=id+1, id2=id2+1, id3=id3+1 where id = 1;
+id estRows task access object operator info
+Update N/A root N/A
+├─Point_Get 1.00 root table:t5 handle:1, lock
+├─Foreign_Key_Check 0.00 root table:t6, index:fk_3 foreign_key:fk_3, check_not_exist
+├─Foreign_Key_Cascade 0.00 root table:t6, index:idx_id foreign_key:fk_1, on_update:CASCADE
+└─Foreign_Key_Cascade 0.00 root table:t6, index:idx_id2 foreign_key:fk_2, on_update:CASCADE
+explain format = 'brief' insert into t5 values (1,1,1);
+id estRows task access object operator info
+Insert N/A root N/A
+explain format = 'brief' insert into t5 values (1,1,1) on duplicate key update id = 100, id3=100;
+id estRows task access object operator info
+Insert N/A root N/A
+├─Foreign_Key_Check 0.00 root table:t6, index:fk_3 foreign_key:fk_3, check_not_exist
+└─Foreign_Key_Cascade 0.00 root table:t6, index:idx_id foreign_key:fk_1, on_update:CASCADE
+explain format = 'brief' insert into t5 values (1,1,1) on duplicate key update id = 100, id2=100, id3=100;
+id estRows task access object operator info
+Insert N/A root N/A
+├─Foreign_Key_Check 0.00 root table:t6, index:fk_3 foreign_key:fk_3, check_not_exist
+├─Foreign_Key_Cascade 0.00 root table:t6, index:idx_id foreign_key:fk_1, on_update:CASCADE
+└─Foreign_Key_Cascade 0.00 root table:t6, index:idx_id2 foreign_key:fk_2, on_update:CASCADE
+drop table if exists t1,t2,t3,t4,t5,t6;
+drop table if exists t_1,t_2,t_3,t_4;
+create table t_1 (id int key);
+create table t_2 (id int key);
+create table t_3 (id int key, id2 int, foreign key fk_1(id) references t_1(id), foreign key fk_2(id2) references t_1(id), foreign key fk_3(id) references t_2(id) ON UPDATE CASCADE ON DELETE CASCADE);
+create table t_4 (id int key, id2 int, foreign key fk_1(id) references t_2(id), foreign key fk_2(id2) references t_1(id), foreign key fk_3(id) references t_1(id) ON UPDATE CASCADE ON DELETE CASCADE);
+explain format = 'brief' update t_1,t_2 set t_1.id=2,t_2.id=2 where t_1.id=t_2.id and t_1.id=1;
+id estRows task access object operator info
+Update N/A root N/A
+├─HashJoin 1.00 root CARTESIAN inner join
+│ ├─Point_Get(Build) 1.00 root table:t_2 handle:1
+│ └─Point_Get(Probe) 1.00 root table:t_1 handle:1
+├─Foreign_Key_Check 0.00 root table:t_3 foreign_key:fk_1, check_not_exist
+├─Foreign_Key_Check 0.00 root table:t_3, index:fk_2 foreign_key:fk_2, check_not_exist
+├─Foreign_Key_Check 0.00 root table:t_4, index:fk_2 foreign_key:fk_2, check_not_exist
+├─Foreign_Key_Check 0.00 root table:t_4 foreign_key:fk_1, check_not_exist
+├─Foreign_Key_Cascade 0.00 root table:t_4 foreign_key:fk_3, on_update:CASCADE
+└─Foreign_Key_Cascade 0.00 root table:t_3 foreign_key:fk_3, on_update:CASCADE
+explain format = 'brief' delete t_1,t_2 from t_1 join t_2 where t_1.id=t_2.id and t_1.id > 0;
+id estRows task access object operator info
+Delete N/A root N/A
+├─MergeJoin 4166.67 root inner join, left key:test.t_1.id, right key:test.t_2.id
+│ ├─TableReader(Build) 3333.33 root data:TableRangeScan
+│ │ └─TableRangeScan 3333.33 cop[tikv] table:t_2 range:(0,+inf], keep order:true, stats:pseudo
+│ └─TableReader(Probe) 3333.33 root data:TableRangeScan
+│ └─TableRangeScan 3333.33 cop[tikv] table:t_1 range:(0,+inf], keep order:true, stats:pseudo
+├─Foreign_Key_Check 0.00 root table:t_3 foreign_key:fk_1, check_not_exist
+├─Foreign_Key_Check 0.00 root table:t_3, index:fk_2 foreign_key:fk_2, check_not_exist
+├─Foreign_Key_Check 0.00 root table:t_4, index:fk_2 foreign_key:fk_2, check_not_exist
+├─Foreign_Key_Check 0.00 root table:t_4 foreign_key:fk_1, check_not_exist
+├─Foreign_Key_Cascade 0.00 root table:t_4 foreign_key:fk_3, on_delete:CASCADE
+└─Foreign_Key_Cascade 0.00 root table:t_3 foreign_key:fk_3, on_delete:CASCADE
+set @@foreign_key_checks=0;
+explain format = 'brief' update t_1,t_2 set t_1.id=2,t_2.id=2 where t_1.id=t_2.id and t_1.id=1;
+id estRows task access object operator info
+Update N/A root N/A
+└─HashJoin 1.00 root CARTESIAN inner join
+ ├─Point_Get(Build) 1.00 root table:t_2 handle:1
+ └─Point_Get(Probe) 1.00 root table:t_1 handle:1
+explain format = 'brief' delete t_1,t_2 from t_1 join t_2 where t_1.id=t_2.id and t_1.id > 0;
+id estRows task access object operator info
+Delete N/A root N/A
+└─MergeJoin 4166.67 root inner join, left key:test.t_1.id, right key:test.t_2.id
+ ├─TableReader(Build) 3333.33 root data:TableRangeScan
+ │ └─TableRangeScan 3333.33 cop[tikv] table:t_2 range:(0,+inf], keep order:true, stats:pseudo
+ └─TableReader(Probe) 3333.33 root data:TableRangeScan
+ └─TableRangeScan 3333.33 cop[tikv] table:t_1 range:(0,+inf], keep order:true, stats:pseudo
+drop table if exists t_1,t_2,t_3,t_4;
+set @@foreign_key_checks=0;
diff --git a/cmd/explaintest/r/explain_indexmerge.result b/cmd/explaintest/r/explain_indexmerge.result
index 07530b9596bb5..46ba855e6c2a8 100644
--- a/cmd/explaintest/r/explain_indexmerge.result
+++ b/cmd/explaintest/r/explain_indexmerge.result
@@ -7,33 +7,33 @@ create index td on t (d);
load stats 's/explain_indexmerge_stats_t.json';
explain format = 'brief' select * from t where a < 50 or b < 50;
id estRows task access object operator info
-IndexMerge 98.00 root
+IndexMerge 98.00 root type: union
├─TableRangeScan(Build) 49.00 cop[tikv] table:t range:[-inf,50), keep order:false
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tb(b) range:[-inf,50), keep order:false
└─TableRowIDScan(Probe) 98.00 cop[tikv] table:t keep order:false
explain format = 'brief' select * from t where (a < 50 or b < 50) and f > 100;
id estRows task access object operator info
-IndexMerge 98.00 root
+IndexMerge 98.00 root type: union
├─TableRangeScan(Build) 49.00 cop[tikv] table:t range:[-inf,50), keep order:false
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tb(b) range:[-inf,50), keep order:false
└─Selection(Probe) 98.00 cop[tikv] gt(test.t.f, 100)
└─TableRowIDScan 98.00 cop[tikv] table:t keep order:false
explain format = 'brief' select * from t where b < 50 or c < 50;
id estRows task access object operator info
-IndexMerge 98.00 root
+IndexMerge 98.00 root type: union
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tb(b) range:[-inf,50), keep order:false
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tc(c) range:[-inf,50), keep order:false
└─TableRowIDScan(Probe) 98.00 cop[tikv] table:t keep order:false
set session tidb_enable_index_merge = on;
explain format = 'brief' select * from t where a < 50 or b < 50;
id estRows task access object operator info
-IndexMerge 98.00 root
+IndexMerge 98.00 root type: union
├─TableRangeScan(Build) 49.00 cop[tikv] table:t range:[-inf,50), keep order:false
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tb(b) range:[-inf,50), keep order:false
└─TableRowIDScan(Probe) 98.00 cop[tikv] table:t keep order:false
explain format = 'brief' select * from t where (a < 50 or b < 50) and f > 100;
id estRows task access object operator info
-IndexMerge 98.00 root
+IndexMerge 98.00 root type: union
├─TableRangeScan(Build) 49.00 cop[tikv] table:t range:[-inf,50), keep order:false
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tb(b) range:[-inf,50), keep order:false
└─Selection(Probe) 98.00 cop[tikv] gt(test.t.f, 100)
@@ -45,7 +45,7 @@ TableReader 4999999.00 root data:Selection
└─TableFullScan 5000000.00 cop[tikv] table:t keep order:false
explain format = 'brief' select * from t where b < 50 or c < 50;
id estRows task access object operator info
-IndexMerge 98.00 root
+IndexMerge 98.00 root type: union
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tb(b) range:[-inf,50), keep order:false
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tc(c) range:[-inf,50), keep order:false
└─TableRowIDScan(Probe) 98.00 cop[tikv] table:t keep order:false
@@ -56,14 +56,14 @@ TableReader 4999999.00 root data:Selection
└─TableFullScan 5000000.00 cop[tikv] table:t keep order:false
explain format = 'brief' select * from t where a < 50 or b < 50 or c < 50;
id estRows task access object operator info
-IndexMerge 147.00 root
+IndexMerge 147.00 root type: union
├─TableRangeScan(Build) 49.00 cop[tikv] table:t range:[-inf,50), keep order:false
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tb(b) range:[-inf,50), keep order:false
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tc(c) range:[-inf,50), keep order:false
└─TableRowIDScan(Probe) 147.00 cop[tikv] table:t keep order:false
explain format = 'brief' select * from t where (b < 10000 or c < 10000) and (a < 10 or d < 10) and f < 10;
id estRows task access object operator info
-IndexMerge 0.00 root
+IndexMerge 0.00 root type: union
├─TableRangeScan(Build) 9.00 cop[tikv] table:t range:[-inf,10), keep order:false
├─IndexRangeScan(Build) 9.00 cop[tikv] table:t, index:td(d) range:[-inf,10), keep order:false
└─Selection(Probe) 0.00 cop[tikv] lt(test.t.f, 10), or(lt(test.t.b, 10000), lt(test.t.c, 10000))
@@ -104,20 +104,20 @@ label = "cop"
set session tidb_enable_index_merge = off;
explain format = 'brief' select /*+ use_index_merge(t, primary, tb, tc) */ * from t where a <= 500000 or b <= 1000000 or c <= 3000000;
id estRows task access object operator info
-IndexMerge 3560000.00 root
+IndexMerge 3560000.00 root type: union
├─TableRangeScan(Build) 500000.00 cop[tikv] table:t range:[-inf,500000], keep order:false
├─IndexRangeScan(Build) 1000000.00 cop[tikv] table:t, index:tb(b) range:[-inf,1000000], keep order:false
├─IndexRangeScan(Build) 3000000.00 cop[tikv] table:t, index:tc(c) range:[-inf,3000000], keep order:false
└─TableRowIDScan(Probe) 3560000.00 cop[tikv] table:t keep order:false
explain format = 'brief' select /*+ use_index_merge(t, tb, tc) */ * from t where b < 50 or c < 5000000;
id estRows task access object operator info
-IndexMerge 4999999.00 root
+IndexMerge 4999999.00 root type: union
├─IndexRangeScan(Build) 49.00 cop[tikv] table:t, index:tb(b) range:[-inf,50), keep order:false
├─IndexRangeScan(Build) 4999999.00 cop[tikv] table:t, index:tc(c) range:[-inf,5000000), keep order:false
└─TableRowIDScan(Probe) 4999999.00 cop[tikv] table:t keep order:false
explain format = 'brief' select /*+ use_index_merge(t, tb, tc) */ * from t where (b < 10000 or c < 10000) and (a < 10 or d < 10) and f < 10;
id estRows task access object operator info
-IndexMerge 0.00 root
+IndexMerge 0.00 root type: union
├─IndexRangeScan(Build) 9999.00 cop[tikv] table:t, index:tb(b) range:[-inf,10000), keep order:false
├─IndexRangeScan(Build) 9999.00 cop[tikv] table:t, index:tc(c) range:[-inf,10000), keep order:false
└─Selection(Probe) 0.00 cop[tikv] lt(test.t.f, 10), or(lt(test.t.a, 10), lt(test.t.d, 10))
@@ -134,7 +134,7 @@ TableReader 4999999.00 root data:Selection
└─TableFullScan 5000000.00 cop[tikv] table:t keep order:false
explain format = 'brief' select /*+ use_index_merge(t, primary, tb) */ * from t where a < 50 or b < 5000000;
id estRows task access object operator info
-IndexMerge 4999999.00 root
+IndexMerge 4999999.00 root type: union
├─TableRangeScan(Build) 49.00 cop[tikv] table:t range:[-inf,50), keep order:false
├─IndexRangeScan(Build) 4999999.00 cop[tikv] table:t, index:tb(b) range:[-inf,5000000), keep order:false
└─TableRowIDScan(Probe) 4999999.00 cop[tikv] table:t keep order:false
@@ -151,7 +151,7 @@ KEY `aid_c2` (`aid`,`c2`)
);
desc select /*+ USE_INDEX_MERGE(t, aid_c1, aid_c2) */ * from t where (aid = 1 and c1='aaa') or (aid = 2 and c2='bbb');
id estRows task access object operator info
-IndexMerge_8 8.08 root
+IndexMerge_8 8.08 root type: union
├─IndexRangeScan_5(Build) 0.10 cop[tikv] table:t, index:aid_c1(aid, c1) range:[1 "aaa",1 "aaa"], keep order:false, stats:pseudo
├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t, index:aid_c2(aid, c2) range:[2 "bbb",2 "bbb"], keep order:false, stats:pseudo
└─TableRowIDScan_7(Probe) 8.08 cop[tikv] table:t keep order:false, stats:pseudo
diff --git a/cmd/explaintest/r/index_merge.result b/cmd/explaintest/r/index_merge.result
index 1870b6bcd8d5d..6b44c6122987e 100644
--- a/cmd/explaintest/r/index_merge.result
+++ b/cmd/explaintest/r/index_merge.result
@@ -15,7 +15,7 @@ Sort_8 4433.77 root test.t1.c1
└─HashJoin_12 5542.21 root CARTESIAN left outer semi join, other cond:eq(test.t1.c3, test.t1.c3)
├─TableReader_18(Build) 10000.00 root data:TableFullScan_17
│ └─TableFullScan_17 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
- └─IndexMerge_16(Probe) 5542.21 root
+ └─IndexMerge_16(Probe) 5542.21 root type: union
├─IndexRangeScan_13(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_14(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_15(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -35,7 +35,7 @@ Sort_8 4433.77 root test.t1.c1
└─HashJoin_12 5542.21 root CARTESIAN anti left outer semi join, other cond:eq(test.t1.c3, test.t1.c3)
├─TableReader_18(Build) 10000.00 root data:TableFullScan_17
│ └─TableFullScan_17 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
- └─IndexMerge_16(Probe) 5542.21 root
+ └─IndexMerge_16(Probe) 5542.21 root type: union
├─IndexRangeScan_13(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_14(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_15(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -50,7 +50,7 @@ c1 c2 c3
explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 = (select max(c3) from t1) order by 1;
id estRows task access object operator info
Sort_33 3325.55 root test.t1.c1
-└─IndexMerge_40 1843.09 root
+└─IndexMerge_40 1843.09 root type: union
├─IndexRangeScan_36(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_37(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_39(Probe) 1843.09 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), eq(test.t1.c3, 5)))
@@ -71,7 +71,7 @@ Sort_9 4433.77 root test.t1.c1
└─HashJoin_22 5542.21 root left outer semi join, equal:[eq(test.t1.c1, test.t2.c1)]
├─IndexReader_30(Build) 10000.00 root index:IndexFullScan_29
│ └─IndexFullScan_29 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo
- └─IndexMerge_26(Probe) 5542.21 root
+ └─IndexMerge_26(Probe) 5542.21 root type: union
├─IndexRangeScan_23(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_24(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_25(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -91,7 +91,7 @@ Sort_9 4433.77 root test.t1.c1
└─HashJoin_22 5542.21 root anti left outer semi join, equal:[eq(test.t1.c1, test.t2.c1)]
├─IndexReader_30(Build) 10000.00 root index:IndexFullScan_29
│ └─IndexFullScan_29 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo
- └─IndexMerge_26(Probe) 5542.21 root
+ └─IndexMerge_26(Probe) 5542.21 root type: union
├─IndexRangeScan_23(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_24(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_25(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -106,7 +106,7 @@ c1 c2 c3
explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 = (select count(1) from t2) order by 1;
id estRows task access object operator info
Sort_38 3325.55 root test.t1.c1
-└─IndexMerge_45 1843.09 root
+└─IndexMerge_45 1843.09 root type: union
├─IndexRangeScan_41(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_42(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_44(Probe) 1843.09 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), eq(test.t1.c3, 5)))
@@ -128,7 +128,7 @@ Sort_11 5098.44 root test.t1.c1
│ └─IndexReader_44 1.00 root index:StreamAgg_27
│ └─StreamAgg_27 1.00 cop[tikv] funcs:count(1)->Column#25
│ └─IndexFullScan_41 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo
- └─IndexMerge_21(Probe) 2825.66 root
+ └─IndexMerge_21(Probe) 2825.66 root type: union
├─IndexRangeScan_17(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_18(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_20(Probe) 2825.66 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), if(isnull(test.t1.c3), NULL, 1)))
@@ -150,7 +150,7 @@ Sort_11 5098.44 root test.t1.c1
│ └─IndexReader_44 1.00 root index:StreamAgg_27
│ └─StreamAgg_27 1.00 cop[tikv] funcs:count(1)->Column#25
│ └─IndexFullScan_41 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo
- └─IndexMerge_21(Probe) 2825.66 root
+ └─IndexMerge_21(Probe) 2825.66 root type: union
├─IndexRangeScan_17(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_18(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_20(Probe) 2825.66 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), if(isnull(test.t1.c3), NULL, 1)))
@@ -172,7 +172,7 @@ Sort_11 5542.21 root test.t1.c1
│ └─IndexReader_43 1.00 root index:StreamAgg_26
│ └─StreamAgg_26 1.00 cop[tikv] funcs:count(1)->Column#25
│ └─IndexFullScan_40 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo
- └─IndexMerge_20(Probe) 5542.21 root
+ └─IndexMerge_20(Probe) 5542.21 root type: union
├─IndexRangeScan_17(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_18(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_19(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -194,7 +194,7 @@ Sort_39 5542.21 root test.t1.c1
│ └─IndexReader_71 1.00 root index:StreamAgg_54
│ └─StreamAgg_54 1.00 cop[tikv] funcs:count(1)->Column#38
│ └─IndexFullScan_68 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo
- └─IndexMerge_48(Probe) 5542.21 root
+ └─IndexMerge_48(Probe) 5542.21 root type: union
├─IndexRangeScan_45(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_46(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_47(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -220,7 +220,7 @@ Sort_14 4433.77 root test.t1.c1
│ └─TableReader_51(Probe) 9990.00 root data:Selection_50
│ └─Selection_50 9990.00 cop[tikv] not(isnull(test.t2.c2))
│ └─TableFullScan_49 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
- └─IndexMerge_22(Probe) 5542.21 root
+ └─IndexMerge_22(Probe) 5542.21 root type: union
├─IndexRangeScan_19(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_20(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_21(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -239,7 +239,7 @@ explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and
id estRows task access object operator info
Sort_5 4060.74 root test.t1.c1
└─Selection_12 2250.55 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
- └─IndexMerge_11 5542.21 root
+ └─IndexMerge_11 5542.21 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -254,7 +254,7 @@ explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and
id estRows task access object operator info
Sort_5 5098.44 root test.t1.c1
└─Selection_12 2825.66 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), eq(test.t1.c3, plus(test.t1.c1, test.t1.c2))))
- └─IndexMerge_11 5542.21 root
+ └─IndexMerge_11 5542.21 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -269,7 +269,7 @@ explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and
id estRows task access object operator info
Sort_5 5098.44 root test.t1.c1
└─Selection_12 2825.66 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), istrue_with_null(cast(substring(cast(test.t1.c3, var_string(20)), test.t1.c2), double BINARY))))
- └─IndexMerge_11 5542.21 root
+ └─IndexMerge_11 5542.21 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -284,7 +284,7 @@ explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and
id estRows task access object operator info
Sort_5 4800.37 root test.t1.c1
└─Selection_12 2660.47 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), test.t1.c3))
- └─IndexMerge_11 5542.21 root
+ └─IndexMerge_11 5542.21 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -304,7 +304,7 @@ explain select * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1;
id estRows task access object operator info
Sort_5 4060.74 root test.t1.c1
└─Selection_12 2250.55 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
- └─IndexMerge_11 5542.21 root
+ └─IndexMerge_11 5542.21 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -338,7 +338,7 @@ insert into t1 values(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4), (5, 5, 5);
explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1;
id estRows task access object operator info
Sort_5 4060.74 root test.t1.c1
-└─IndexMerge_12 2250.55 root
+└─IndexMerge_12 2250.55 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
@@ -368,7 +368,7 @@ alter table t1 add index c1(c1);
explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1;
id estRows task access object operator info
Sort_5 4060.74 root test.t1.c1
-└─IndexMerge_12 2250.55 root
+└─IndexMerge_12 2250.55 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
@@ -391,7 +391,7 @@ Delete_11 N/A root N/A
└─SelectLock_17 4056.68 root for update 0
└─HashJoin_33 4056.68 root inner join, equal:[eq(test.t1.c1, test.t1.c1)]
├─HashAgg_36(Build) 3245.34 root group by:test.t1.c1, funcs:firstrow(test.t1.c1)->test.t1.c1
- │ └─IndexMerge_41 2248.30 root
+ │ └─IndexMerge_41 2248.30 root type: union
│ ├─IndexRangeScan_37(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
│ ├─IndexRangeScan_38(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
│ └─Selection_40(Probe) 2248.30 cop[tikv] not(isnull(test.t1.c1)), or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
@@ -409,7 +409,7 @@ Update_10 N/A root N/A
└─SelectLock_14 4056.68 root for update 0
└─HashJoin_30 4056.68 root inner join, equal:[eq(test.t1.c1, test.t1.c1)]
├─HashAgg_33(Build) 3245.34 root group by:test.t1.c1, funcs:firstrow(test.t1.c1)->test.t1.c1
- │ └─IndexMerge_38 2248.30 root
+ │ └─IndexMerge_38 2248.30 root type: union
│ ├─IndexRangeScan_34(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
│ ├─IndexRangeScan_35(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
│ └─Selection_37(Probe) 2248.30 cop[tikv] not(isnull(test.t1.c1)), or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
@@ -426,7 +426,7 @@ id estRows task access object operator info
Sort_6 4060.74 root test.t1.c1
└─Projection_8 4060.74 root test.t1.c1, test.t1.c2, test.t1.c3
└─SelectLock_9 4060.74 root for update 0
- └─IndexMerge_14 2250.55 root
+ └─IndexMerge_14 2250.55 root type: union
├─IndexRangeScan_10(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_11(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_13(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
@@ -455,9 +455,9 @@ c1 c2 c3
///// MEMORY Table
explain select count(c1) from (select /*+ use_index_merge(t_alias), stream_agg() */ count(1) c1 from information_schema.statements_summary where sum_latency >= 0 or max_latency >= 0 order by 1) dt;
id estRows task access object operator info
-StreamAgg_10 1.00 root funcs:count(Column#93)->Column#94
-└─Sort_11 1.00 root Column#93
- └─StreamAgg_14 1.00 root funcs:count(1)->Column#93
+StreamAgg_10 1.00 root funcs:count(Column#96)->Column#97
+└─Sort_11 1.00 root Column#96
+ └─StreamAgg_14 1.00 root funcs:count(1)->Column#96
└─MemTableScan_18 10000.00 root table:STATEMENTS_SUMMARY
show warnings;
Level Code Message
@@ -471,7 +471,7 @@ insert into t1 values(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4), (5, 5, 5);
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and c3 < 10 order by 1 limit 1 offset 2;
id estRows task access object operator info
TopN_10 1.00 root test.t1.c1, offset:2, count:1
-└─IndexMerge_19 1841.86 root
+└─IndexMerge_19 1841.86 root type: union
├─IndexRangeScan_15(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_16(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_18(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)
@@ -485,7 +485,7 @@ id estRows task access object operator info
Sort_6 1473.49 root Column#5
└─HashAgg_11 1473.49 root group by:Column#10, funcs:sum(Column#9)->Column#5
└─Projection_18 1841.86 root cast(test.t1.c1, decimal(10,0) BINARY)->Column#9, test.t1.c1
- └─IndexMerge_16 1841.86 root
+ └─IndexMerge_16 1841.86 root type: union
├─IndexRangeScan_12(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_13(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_15(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)
@@ -507,7 +507,7 @@ Sort_12 1841.86 root test.t1.c1
└─Projection_14 1841.86 root test.t1.c1, test.t1.c2, test.t1.c3
└─Apply_16 1841.86 root inner join, equal:[eq(Column#10, Column#9)]
├─Projection_17(Build) 1841.86 root test.t1.c1, test.t1.c2, test.t1.c3, cast(test.t1.c1, decimal(10,0) BINARY)->Column#10
- │ └─IndexMerge_22 1841.86 root
+ │ └─IndexMerge_22 1841.86 root type: union
│ ├─IndexRangeScan_18(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
│ ├─IndexRangeScan_19(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,-1), keep order:false, stats:pseudo
│ └─Selection_21(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)
@@ -530,7 +530,7 @@ Sort_16 1841.86 root test.t1.c1
└─Projection_18 1841.86 root test.t1.c1, test.t1.c2, test.t1.c3
└─Apply_20 1841.86 root inner join, equal:[eq(Column#11, Column#9)]
├─Projection_21(Build) 1841.86 root test.t1.c1, test.t1.c2, test.t1.c3, cast(test.t1.c1, decimal(10,0) BINARY)->Column#11
- │ └─IndexMerge_26 1841.86 root
+ │ └─IndexMerge_26 1841.86 root type: union
│ ├─IndexRangeScan_22(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
│ ├─IndexRangeScan_23(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,-1), keep order:false, stats:pseudo
│ └─Selection_25(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)
@@ -538,7 +538,7 @@ Sort_16 1841.86 root test.t1.c1
└─TopN_29(Probe) 1841.86 root test.t2.c1, offset:2, count:1
└─HashAgg_36 4900166.23 root group by:Column#21, funcs:avg(Column#19)->Column#9, funcs:firstrow(Column#20)->test.t2.c1
└─Projection_48 6125207.79 root cast(test.t2.c1, decimal(10,0) BINARY)->Column#19, test.t2.c1, test.t2.c1
- └─IndexMerge_41 6125207.79 root
+ └─IndexMerge_41 6125207.79 root type: union
├─Selection_38(Build) 6121.12 cop[tikv] eq(test.t1.c1, test.t2.c1)
│ └─IndexRangeScan_37 6121120.92 cop[tikv] table:t2, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_39(Build) 6121120.92 cop[tikv] table:t2, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
@@ -553,7 +553,7 @@ insert into t1 values(1, 1, 1, 1, 1), (2, 2, 2, 2, 2), (3, 3, 3, 3, 3), (4, 4, 4
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and (c3 < 10 or c4 < 10) order by 1;
id estRows task access object operator info
Sort_5 3071.61 root test.t1.c1
-└─IndexMerge_12 3071.61 root
+└─IndexMerge_12 3071.61 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 3071.61 cop[tikv] or(lt(test.t1.c3, 10), lt(test.t1.c4, 10))
@@ -568,7 +568,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 and c2 < 10) or (c3 < 10 and c4 < 10) order by 1;
id estRows task access object operator info
Sort_5 2086.93 root test.t1.c1
-└─IndexMerge_12 1156.62 root
+└─IndexMerge_12 1156.62 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c3(c3) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 1156.62 cop[tikv] or(and(lt(test.t1.c1, 10), lt(test.t1.c2, 10)), and(lt(test.t1.c3, 10), lt(test.t1.c4, 10)))
@@ -583,7 +583,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 and c2 < 10) or (c3 < 10 and c4 < 10) and c5 < 10 order by 1;
id estRows task access object operator info
Sort_5 1430.96 root test.t1.c1
-└─IndexMerge_12 793.07 root
+└─IndexMerge_12 793.07 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c3(c3) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 793.07 cop[tikv] or(and(lt(test.t1.c1, 10), lt(test.t1.c2, 10)), and(lt(test.t1.c3, 10), and(lt(test.t1.c4, 10), lt(test.t1.c5, 10))))
@@ -598,7 +598,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where ((c1 < 10 and c4 < 10) or c2 < 10) and (c3 < 10 or c5 < 10) order by 1;
id estRows task access object operator info
Sort_5 2250.55 root test.t1.c1
-└─IndexMerge_12 1247.30 root
+└─IndexMerge_12 1247.30 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 1247.30 cop[tikv] or(and(lt(test.t1.c1, 10), lt(test.t1.c4, 10)), lt(test.t1.c2, 10)), or(lt(test.t1.c3, 10), lt(test.t1.c5, 10))
@@ -628,7 +628,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (((c1 < 10 or c3 < 10) and c1 < 10) or c2 < 10) and (c3 < 10 or c5 < 10) order by 1;
id estRows task access object operator info
Sort_5 2523.42 root test.t1.c1
-└─IndexMerge_12 1398.53 root
+└─IndexMerge_12 1398.53 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 1398.53 cop[tikv] or(and(or(lt(test.t1.c1, 10), lt(test.t1.c3, 10)), lt(test.t1.c1, 10)), lt(test.t1.c2, 10)), or(lt(test.t1.c3, 10), lt(test.t1.c5, 10))
@@ -645,7 +645,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and coalesce(c1, c2, c4) = 1 order by 1;
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
-└─IndexMerge_12 4433.77 root
+└─IndexMerge_12 4433.77 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 4433.77 cop[tikv] eq(coalesce(test.t1.c1, test.t1.c2, test.t1.c4), 1)
@@ -657,7 +657,7 @@ explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) a
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
└─Selection_12 4433.77 root eq(greatest(test.t1.c1, test.t1.c2, test.t1.c4), 1)
- └─IndexMerge_11 5542.21 root
+ └─IndexMerge_11 5542.21 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -668,7 +668,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and abs(c1) = 1 order by 1;
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
-└─IndexMerge_12 4433.77 root
+└─IndexMerge_12 4433.77 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 4433.77 cop[tikv] eq(abs(test.t1.c1), 1)
@@ -679,7 +679,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and pi() order by 1;
id estRows task access object operator info
Sort_5 5542.21 root test.t1.c1
-└─IndexMerge_11 5542.21 root
+└─IndexMerge_11 5542.21 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -693,7 +693,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and ceil(c1) order by 1;
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
-└─IndexMerge_12 4433.77 root
+└─IndexMerge_12 4433.77 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 4433.77 cop[tikv] ceil(test.t1.c1)
@@ -709,7 +709,7 @@ explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) a
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
└─Selection_8 4433.77 root eq(truncate(test.t1.c1, 1), 1)
- └─IndexMerge_12 5542.21 root
+ └─IndexMerge_12 5542.21 root type: union
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_10(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_11(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -725,7 +725,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1;
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
-└─IndexMerge_12 4433.77 root
+└─IndexMerge_12 4433.77 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 4433.77 cop[tikv] eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1")
@@ -737,7 +737,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and ifnull(c1, c2) order by 1;
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
-└─IndexMerge_12 4433.77 root
+└─IndexMerge_12 4433.77 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 4433.77 cop[tikv] ifnull(test.t1.c1, test.t1.c2)
@@ -752,7 +752,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and if(c1, c2, c3) order by 1;
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
-└─IndexMerge_12 4433.77 root
+└─IndexMerge_12 4433.77 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 4433.77 cop[tikv] if(test.t1.c1, test.t1.c2, test.t1.c3)
@@ -767,7 +767,7 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and (c1 between 1 and 2) order by 1;
id estRows task access object operator info
Sort_5 138.56 root test.t1.c1
-└─IndexMerge_12 138.56 root
+└─IndexMerge_12 138.56 root type: union
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 138.56 cop[tikv] ge(test.t1.c1, 1), le(test.t1.c1, 2)
@@ -782,7 +782,7 @@ explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) a
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
└─Selection_8 4433.77 root eq(length(substring(cast(sqrt(cast(test.t1.c3, double BINARY)), var_string(5)), getvar("a"), 1)), char_length(cast(if(test.t1.c1, test.t1.c2, test.t1.c3), var_string(20))))
- └─IndexMerge_12 5542.21 root
+ └─IndexMerge_12 5542.21 root type: union
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_10(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_11(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
@@ -800,7 +800,7 @@ insert into t1 values(1, 1, 1, 1, 1), (2, 2, 2, 2, 2), (3, 3, 3, 3, 3), (4, 4, 4
explain with cte1 as (select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10) select * from cte1 order by 1;
id estRows task access object operator info
Sort_10 4060.74 root test.t1.c1
-└─IndexMerge_17 2250.55 root
+└─IndexMerge_17 2250.55 root type: union
├─IndexRangeScan_13(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_14(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_16(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
@@ -818,7 +818,7 @@ Sort_23 7309.33 root test.t1.c1
└─CTEFullScan_26 7309.33 root CTE:cte1 data:CTE_0
CTE_0 7309.33 root Recursive CTE
├─Projection_14(Seed Part) 4060.74 root test.t1.c1
-│ └─IndexMerge_19 2250.55 root
+│ └─IndexMerge_19 2250.55 root type: union
│ ├─IndexRangeScan_15(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
│ ├─IndexRangeScan_16(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
│ └─Selection_18(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
diff --git a/cmd/explaintest/r/new_character_set_builtin.result b/cmd/explaintest/r/new_character_set_builtin.result
index f044ecca1019c..77c9400e3128a 100644
--- a/cmd/explaintest/r/new_character_set_builtin.result
+++ b/cmd/explaintest/r/new_character_set_builtin.result
@@ -399,17 +399,17 @@ a like 0xe4b880 b like 0xd2bb
1 1
1 1
select a = 0xb6fe from t;
-Error 3854: Cannot convert string '\xB6\xFE' from binary to utf8mb4
+Error 3854 (HY000): Cannot convert string '\xB6\xFE' from binary to utf8mb4
select b = 0xe4ba8c from t;
-Error 3854: Cannot convert string '\xE4\xBA\x8C' from binary to gbk
+Error 3854 (HY000): Cannot convert string '\xE4\xBA\x8C' from binary to gbk
select concat(a, 0xb6fe) from t;
-Error 3854: Cannot convert string '\xB6\xFE' from binary to utf8mb4
+Error 3854 (HY000): Cannot convert string '\xB6\xFE' from binary to utf8mb4
select concat(b, 0xe4ba8c) from t;
-Error 3854: Cannot convert string '\xE4\xBA\x8C' from binary to gbk
+Error 3854 (HY000): Cannot convert string '\xE4\xBA\x8C' from binary to gbk
select concat(convert('a' using gbk), 0x3fff) from t;
-Error 3854: Cannot convert string '?\xFF' from binary to gbk
+Error 3854 (HY000): Cannot convert string '?\xFF' from binary to gbk
select concat(convert('a' using gbk), 0x3fffffffffffffff) from t;
-Error 3854: Cannot convert string '?\xFF\xFF\xFF\xFF\xFF...' from binary to gbk
+Error 3854 (HY000): Cannot convert string '?\xFF\xFF\xFF\xFF\xFF...' from binary to gbk
set @@tidb_enable_vectorized_expression = false;
select hex(concat(a, c)), hex(concat(b, c)) from t;
hex(concat(a, c)) hex(concat(b, c))
@@ -502,13 +502,13 @@ a like 0xe4b880 b like 0xd2bb
1 1
1 1
select a = 0xb6fe from t;
-Error 3854: Cannot convert string '\xB6\xFE' from binary to utf8mb4
+Error 3854 (HY000): Cannot convert string '\xB6\xFE' from binary to utf8mb4
select b = 0xe4ba8c from t;
-Error 3854: Cannot convert string '\xE4\xBA\x8C' from binary to gbk
+Error 3854 (HY000): Cannot convert string '\xE4\xBA\x8C' from binary to gbk
select concat(a, 0xb6fe) from t;
-Error 3854: Cannot convert string '\xB6\xFE' from binary to utf8mb4
+Error 3854 (HY000): Cannot convert string '\xB6\xFE' from binary to utf8mb4
select concat(b, 0xe4ba8c) from t;
-Error 3854: Cannot convert string '\xE4\xBA\x8C' from binary to gbk
+Error 3854 (HY000): Cannot convert string '\xE4\xBA\x8C' from binary to gbk
drop table if exists t;
create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
insert into t values ('一二三', '一二三', '一二三');
diff --git a/cmd/explaintest/r/new_character_set_invalid.result b/cmd/explaintest/r/new_character_set_invalid.result
index 7eea6348e2bc7..e0c749d81ab15 100644
--- a/cmd/explaintest/r/new_character_set_invalid.result
+++ b/cmd/explaintest/r/new_character_set_invalid.result
@@ -4,11 +4,11 @@ drop table if exists t;
create table t (a varchar(255) charset gbk, b varchar(255) charset ascii, c varchar(255) charset utf8);
insert into t values ('中文', 'asdf', '字符集');
insert into t values ('À', 'ø', '😂');
-Error 1366: Incorrect string value '\xC3\x80' for column 'a'
+Error 1366 (HY000): Incorrect string value '\xC3\x80' for column 'a'
insert into t values ('中文À中文', 'asdføfdsa', '字符集😂字符集');
-Error 1366: Incorrect string value '\xC3\x80' for column 'a'
+Error 1366 (HY000): Incorrect string value '\xC3\x80' for column 'a'
insert into t values (0x4040ffff, 0x4040ffff, 0x4040ffff);
-Error 1366: Incorrect string value '\xFF\xFF' for column 'a'
+Error 1366 (HY000): Incorrect string value '\xFF\xFF' for column 'a'
select * from t;
a b c
中文 asdf 字符集
diff --git a/cmd/explaintest/r/select.result b/cmd/explaintest/r/select.result
index 02f2486542f45..70101f0218ca8 100644
--- a/cmd/explaintest/r/select.result
+++ b/cmd/explaintest/r/select.result
@@ -96,9 +96,9 @@ SELECT * from t a left join t2 b on a.c1 = b.c1;
c1 c2 c3 c1 c2
1 2 3 1 2
SELECT * from (SELECT 1, 1) as a;
-Error 1060: Duplicate column name '1'
+Error 1060 (42S21): Duplicate column name '1'
SELECT * from (SELECT * FROM t, t2) as a;
-Error 1060: Duplicate column name 'c1'
+Error 1060 (42S21): Duplicate column name 'c1'
DROP TABLE IF EXISTS t;
CREATE TABLE t (c1 INT, c2 INT);
INSERT INTO t VALUES (1, 2), (1, 1), (1, 3);
@@ -656,5 +656,5 @@ drop table if exists t3;
create table t3(a char(10), primary key (a));
insert into t3 values ('a');
select * from t3 where a > 0x80;
-Error 1105: Cannot convert string '\x80' from binary to utf8mb4
+Error 1105 (HY000): Cannot convert string '\x80' from binary to utf8mb4
set @@tidb_enable_outer_join_reorder=false;
diff --git a/cmd/explaintest/r/subquery.result b/cmd/explaintest/r/subquery.result
index 0cf9302f425c0..ea5a17f2ff3e3 100644
--- a/cmd/explaintest/r/subquery.result
+++ b/cmd/explaintest/r/subquery.result
@@ -56,13 +56,11 @@ insert into exam values(1, 'math', 100);
set names utf8 collate utf8_general_ci;
explain format = 'brief' select * from stu where stu.name not in (select 'guo' from exam where exam.stu_id = stu.id);
id estRows task access object operator info
-Apply 10000.00 root CARTESIAN anti semi join, other cond:eq(test.stu.name, Column#8)
+HashJoin 8000.00 root anti semi join, equal:[eq(test.stu.id, test.exam.stu_id)], other cond:eq(test.stu.name, "guo")
├─TableReader(Build) 10000.00 root data:TableFullScan
-│ └─TableFullScan 10000.00 cop[tikv] table:stu keep order:false, stats:pseudo
-└─Projection(Probe) 100000.00 root guo->Column#8
- └─TableReader 100000.00 root data:Selection
- └─Selection 100000.00 cop[tikv] eq(test.exam.stu_id, test.stu.id)
- └─TableFullScan 100000000.00 cop[tikv] table:exam keep order:false, stats:pseudo
+│ └─TableFullScan 10000.00 cop[tikv] table:exam keep order:false, stats:pseudo
+└─TableReader(Probe) 10000.00 root data:TableFullScan
+ └─TableFullScan 10000.00 cop[tikv] table:stu keep order:false, stats:pseudo
select * from stu where stu.name not in (select 'guo' from exam where exam.stu_id = stu.id);
id name
set names utf8mb4;
diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result
index 28328e4c63241..a2fc266a653d7 100644
--- a/cmd/explaintest/r/tpch.result
+++ b/cmd/explaintest/r/tpch.result
@@ -27,16 +27,13 @@ S_NATIONKEY INTEGER NOT NULL,
S_PHONE CHAR(15) NOT NULL,
S_ACCTBAL DECIMAL(15,2) NOT NULL,
S_COMMENT VARCHAR(101) NOT NULL,
-PRIMARY KEY (S_SUPPKEY),
-CONSTRAINT FOREIGN KEY SUPPLIER_FK1 (S_NATIONKEY) references nation(N_NATIONKEY));
+PRIMARY KEY (S_SUPPKEY));
CREATE TABLE IF NOT EXISTS partsupp ( PS_PARTKEY INTEGER NOT NULL,
PS_SUPPKEY INTEGER NOT NULL,
PS_AVAILQTY INTEGER NOT NULL,
PS_SUPPLYCOST DECIMAL(15,2) NOT NULL,
PS_COMMENT VARCHAR(199) NOT NULL,
-PRIMARY KEY (PS_PARTKEY,PS_SUPPKEY),
-CONSTRAINT FOREIGN KEY PARTSUPP_FK1 (PS_SUPPKEY) references supplier(S_SUPPKEY),
-CONSTRAINT FOREIGN KEY PARTSUPP_FK2 (PS_PARTKEY) references part(P_PARTKEY));
+PRIMARY KEY (PS_PARTKEY,PS_SUPPKEY));
CREATE TABLE IF NOT EXISTS customer ( C_CUSTKEY INTEGER NOT NULL,
C_NAME VARCHAR(25) NOT NULL,
C_ADDRESS VARCHAR(40) NOT NULL,
@@ -45,8 +42,7 @@ C_PHONE CHAR(15) NOT NULL,
C_ACCTBAL DECIMAL(15,2) NOT NULL,
C_MKTSEGMENT CHAR(10) NOT NULL,
C_COMMENT VARCHAR(117) NOT NULL,
-PRIMARY KEY (C_CUSTKEY),
-CONSTRAINT FOREIGN KEY CUSTOMER_FK1 (C_NATIONKEY) references nation(N_NATIONKEY));
+PRIMARY KEY (C_CUSTKEY));
CREATE TABLE IF NOT EXISTS orders ( O_ORDERKEY INTEGER NOT NULL,
O_CUSTKEY INTEGER NOT NULL,
O_ORDERSTATUS CHAR(1) NOT NULL,
@@ -56,8 +52,7 @@ O_ORDERPRIORITY CHAR(15) NOT NULL,
O_CLERK CHAR(15) NOT NULL,
O_SHIPPRIORITY INTEGER NOT NULL,
O_COMMENT VARCHAR(79) NOT NULL,
-PRIMARY KEY (O_ORDERKEY),
-CONSTRAINT FOREIGN KEY ORDERS_FK1 (O_CUSTKEY) references customer(C_CUSTKEY));
+PRIMARY KEY (O_ORDERKEY));
CREATE TABLE IF NOT EXISTS lineitem ( L_ORDERKEY INTEGER NOT NULL,
L_PARTKEY INTEGER NOT NULL,
L_SUPPKEY INTEGER NOT NULL,
@@ -74,9 +69,7 @@ L_RECEIPTDATE DATE NOT NULL,
L_SHIPINSTRUCT CHAR(25) NOT NULL,
L_SHIPMODE CHAR(10) NOT NULL,
L_COMMENT VARCHAR(44) NOT NULL,
-PRIMARY KEY (L_ORDERKEY,L_LINENUMBER),
-CONSTRAINT FOREIGN KEY LINEITEM_FK1 (L_ORDERKEY) references orders(O_ORDERKEY),
-CONSTRAINT FOREIGN KEY LINEITEM_FK2 (L_PARTKEY,L_SUPPKEY) references partsupp(PS_PARTKEY, PS_SUPPKEY));
+PRIMARY KEY (L_ORDERKEY,L_LINENUMBER));
load stats 's/tpch_stats/nation.json';
load stats 's/tpch_stats/region.json';
load stats 's/tpch_stats/part.json';
diff --git a/cmd/explaintest/t/collation_check_use_collation.test b/cmd/explaintest/t/collation_check_use_collation.test
index 62fbea05ae628..adcd8695b38c0 100644
--- a/cmd/explaintest/t/collation_check_use_collation.test
+++ b/cmd/explaintest/t/collation_check_use_collation.test
@@ -110,5 +110,19 @@ insert into t1 values ('-1');
insert into t2 values (0x2d31, '');
select * from t1, t2 where t1.a in (t2.b, 3);
+# issue 38736
+drop table if exists t0;
+drop table if exists t1;
+CREATE TABLE t0(c0 BOOL, c1 INT);
+CREATE TABLE t1 LIKE t0;
+CREATE VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;
+INSERT INTO t0(c0, c1) VALUES (true, 0);
+INSERT INTO t1(c0, c1) VALUES (true, 2);
+
+SELECT v0.c0 FROM v0;
+SELECT (v0.c0)NOT LIKE(BINARY v0.c0) FROM v0;
+SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+desc format='brief' SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+
# cleanup environment
use test
diff --git a/cmd/explaintest/t/explain_complex.test b/cmd/explaintest/t/explain_complex.test
index d9d1b9285f727..187ec571857d8 100644
--- a/cmd/explaintest/t/explain_complex.test
+++ b/cmd/explaintest/t/explain_complex.test
@@ -174,7 +174,9 @@ CREATE TABLE org_position (
UNIQUE KEY org_employee_position_pk (hotel_id,user_id,position_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+set tidb_cost_model_version=2;
explain format = 'brief' SELECT d.id, d.ctx, d.name, d.left_value, d.right_value, d.depth, d.leader_id, d.status, d.created_on, d.updated_on FROM org_department AS d LEFT JOIN org_position AS p ON p.department_id = d.id AND p.status = 1000 LEFT JOIN org_employee_position AS ep ON ep.position_id = p.id AND ep.status = 1000 WHERE (d.ctx = 1 AND (ep.user_id = 62 OR d.id = 20 OR d.id = 20) AND d.status = 1000) GROUP BY d.id ORDER BY d.left_value;
+set tidb_cost_model_version=1;
create table test.Tab_A (id int primary key,bid int,cid int,name varchar(20),type varchar(20),num int,amt decimal(11,2));
create table test.Tab_B (id int primary key,name varchar(20));
diff --git a/cmd/explaintest/t/explain_foreign_key.test b/cmd/explaintest/t/explain_foreign_key.test
new file mode 100644
index 0000000000000..ec222d020a5ab
--- /dev/null
+++ b/cmd/explaintest/t/explain_foreign_key.test
@@ -0,0 +1,45 @@
+set @@foreign_key_checks=1;
+use test;
+drop table if exists t1,t2;
+create table t1 (id int key);
+create table t2 (id int key, foreign key fk(id) references t1(id) ON UPDATE CASCADE ON DELETE CASCADE);
+create table t3 (id int, unique index idx(id));
+create table t4 (id int, index idx_id(id),foreign key fk(id) references t3(id));
+create table t5 (id int key, id2 int, id3 int, unique index idx2(id2), index idx3(id3));
+create table t6 (id int, id2 int, id3 int, index idx_id(id), index idx_id2(id2), foreign key fk_1 (id) references t5(id) ON UPDATE CASCADE ON DELETE CASCADE, foreign key fk_2 (id2) references t5(id2) ON UPDATE CASCADE, foreign key fk_3 (id3) references t5(id3) ON DELETE CASCADE);
+
+explain format = 'brief' insert into t2 values (1);
+explain format = 'brief' update t2 set id=id+1 where id = 1;
+explain format = 'brief' delete from t1 where id > 1;
+explain format = 'brief' update t1 set id=id+1 where id = 1;
+explain format = 'brief' insert into t1 values (1);
+explain format = 'brief' insert into t1 values (1) on duplicate key update id = 100;
+explain format = 'brief' insert into t4 values (1);
+explain format = 'brief' update t4 set id=id+1 where id = 1;
+explain format = 'brief' delete from t3 where id > 1;
+explain format = 'brief' update t3 set id=id+1 where id = 1;
+explain format = 'brief' insert into t3 values (1);
+explain format = 'brief' insert into t3 values (1) on duplicate key update id = 100;
+explain format = 'brief' insert into t6 values (1,1,1);
+explain format = 'brief' update t6 set id=id+1, id3=id2+1 where id = 1;
+explain format = 'brief' delete from t5 where id > 1;
+explain format = 'brief' update t5 set id=id+1, id2=id2+1 where id = 1;
+explain format = 'brief' update t5 set id=id+1, id2=id2+1, id3=id3+1 where id = 1;
+explain format = 'brief' insert into t5 values (1,1,1);
+explain format = 'brief' insert into t5 values (1,1,1) on duplicate key update id = 100, id3=100;
+explain format = 'brief' insert into t5 values (1,1,1) on duplicate key update id = 100, id2=100, id3=100;
+drop table if exists t1,t2,t3,t4,t5,t6;
+
+drop table if exists t_1,t_2,t_3,t_4;
+create table t_1 (id int key);
+create table t_2 (id int key);
+create table t_3 (id int key, id2 int, foreign key fk_1(id) references t_1(id), foreign key fk_2(id2) references t_1(id), foreign key fk_3(id) references t_2(id) ON UPDATE CASCADE ON DELETE CASCADE);
+create table t_4 (id int key, id2 int, foreign key fk_1(id) references t_2(id), foreign key fk_2(id2) references t_1(id), foreign key fk_3(id) references t_1(id) ON UPDATE CASCADE ON DELETE CASCADE);
+
+explain format = 'brief' update t_1,t_2 set t_1.id=2,t_2.id=2 where t_1.id=t_2.id and t_1.id=1;
+explain format = 'brief' delete t_1,t_2 from t_1 join t_2 where t_1.id=t_2.id and t_1.id > 0;
+set @@foreign_key_checks=0;
+explain format = 'brief' update t_1,t_2 set t_1.id=2,t_2.id=2 where t_1.id=t_2.id and t_1.id=1;
+explain format = 'brief' delete t_1,t_2 from t_1 join t_2 where t_1.id=t_2.id and t_1.id > 0;
+drop table if exists t_1,t_2,t_3,t_4;
+set @@foreign_key_checks=0;
diff --git a/cmd/explaintest/t/tpch.test b/cmd/explaintest/t/tpch.test
index 33e4256b59e99..fbf20e25637f4 100644
--- a/cmd/explaintest/t/tpch.test
+++ b/cmd/explaintest/t/tpch.test
@@ -31,17 +31,14 @@ CREATE TABLE IF NOT EXISTS supplier ( S_SUPPKEY INTEGER NOT NULL,
S_PHONE CHAR(15) NOT NULL,
S_ACCTBAL DECIMAL(15,2) NOT NULL,
S_COMMENT VARCHAR(101) NOT NULL,
- PRIMARY KEY (S_SUPPKEY),
- CONSTRAINT FOREIGN KEY SUPPLIER_FK1 (S_NATIONKEY) references nation(N_NATIONKEY));
+ PRIMARY KEY (S_SUPPKEY));
CREATE TABLE IF NOT EXISTS partsupp ( PS_PARTKEY INTEGER NOT NULL,
PS_SUPPKEY INTEGER NOT NULL,
PS_AVAILQTY INTEGER NOT NULL,
PS_SUPPLYCOST DECIMAL(15,2) NOT NULL,
PS_COMMENT VARCHAR(199) NOT NULL,
- PRIMARY KEY (PS_PARTKEY,PS_SUPPKEY),
- CONSTRAINT FOREIGN KEY PARTSUPP_FK1 (PS_SUPPKEY) references supplier(S_SUPPKEY),
- CONSTRAINT FOREIGN KEY PARTSUPP_FK2 (PS_PARTKEY) references part(P_PARTKEY));
+ PRIMARY KEY (PS_PARTKEY,PS_SUPPKEY));
CREATE TABLE IF NOT EXISTS customer ( C_CUSTKEY INTEGER NOT NULL,
C_NAME VARCHAR(25) NOT NULL,
@@ -51,8 +48,7 @@ CREATE TABLE IF NOT EXISTS customer ( C_CUSTKEY INTEGER NOT NULL,
C_ACCTBAL DECIMAL(15,2) NOT NULL,
C_MKTSEGMENT CHAR(10) NOT NULL,
C_COMMENT VARCHAR(117) NOT NULL,
- PRIMARY KEY (C_CUSTKEY),
- CONSTRAINT FOREIGN KEY CUSTOMER_FK1 (C_NATIONKEY) references nation(N_NATIONKEY));
+ PRIMARY KEY (C_CUSTKEY));
CREATE TABLE IF NOT EXISTS orders ( O_ORDERKEY INTEGER NOT NULL,
O_CUSTKEY INTEGER NOT NULL,
@@ -63,8 +59,7 @@ CREATE TABLE IF NOT EXISTS orders ( O_ORDERKEY INTEGER NOT NULL,
O_CLERK CHAR(15) NOT NULL,
O_SHIPPRIORITY INTEGER NOT NULL,
O_COMMENT VARCHAR(79) NOT NULL,
- PRIMARY KEY (O_ORDERKEY),
- CONSTRAINT FOREIGN KEY ORDERS_FK1 (O_CUSTKEY) references customer(C_CUSTKEY));
+ PRIMARY KEY (O_ORDERKEY));
CREATE TABLE IF NOT EXISTS lineitem ( L_ORDERKEY INTEGER NOT NULL,
L_PARTKEY INTEGER NOT NULL,
@@ -82,9 +77,7 @@ CREATE TABLE IF NOT EXISTS lineitem ( L_ORDERKEY INTEGER NOT NULL,
L_SHIPINSTRUCT CHAR(25) NOT NULL,
L_SHIPMODE CHAR(10) NOT NULL,
L_COMMENT VARCHAR(44) NOT NULL,
- PRIMARY KEY (L_ORDERKEY,L_LINENUMBER),
- CONSTRAINT FOREIGN KEY LINEITEM_FK1 (L_ORDERKEY) references orders(O_ORDERKEY),
- CONSTRAINT FOREIGN KEY LINEITEM_FK2 (L_PARTKEY,L_SUPPKEY) references partsupp(PS_PARTKEY, PS_SUPPKEY));
+ PRIMARY KEY (L_ORDERKEY,L_LINENUMBER));
-- load stats.
load stats 's/tpch_stats/nation.json';
load stats 's/tpch_stats/region.json';
diff --git a/config/BUILD.bazel b/config/BUILD.bazel
index e06843bec3df2..e3034c60caff6 100644
--- a/config/BUILD.bazel
+++ b/config/BUILD.bazel
@@ -13,6 +13,7 @@ go_library(
"//br/pkg/streamhelper/config",
"//parser/terror",
"//util/logutil",
+ "//util/tiflashcompute",
"//util/tikvutil",
"//util/versioninfo",
"@com_github_burntsushi_toml//:toml",
@@ -36,6 +37,7 @@ go_test(
data = glob(["**"]),
embed = [":config"],
flaky = True,
+ shard_count = 2,
deps = [
"//testkit/testsetup",
"//util/logutil",
diff --git a/config/config.go b/config/config.go
index 1ba2c7fbd1595..b868e5ac8bdb3 100644
--- a/config/config.go
+++ b/config/config.go
@@ -35,6 +35,7 @@ import (
logbackupconf "github.com/pingcap/tidb/br/pkg/streamhelper/config"
"github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/tiflashcompute"
"github.com/pingcap/tidb/util/tikvutil"
"github.com/pingcap/tidb/util/versioninfo"
tikvcfg "github.com/tikv/client-go/v2/config"
@@ -46,10 +47,13 @@ import (
// Config number limitations
const (
MaxLogFileSize = 4096 // MB
+ // MaxTxnEntrySize is the max value of TxnEntrySizeLimit.
+ MaxTxnEntrySizeLimit = 120 * 1024 * 1024 // 120MB
// DefTxnEntrySizeLimit is the default value of TxnEntrySizeLimit.
DefTxnEntrySizeLimit = 6 * 1024 * 1024
// DefTxnTotalSizeLimit is the default value of TxnTxnTotalSizeLimit.
- DefTxnTotalSizeLimit = 100 * 1024 * 1024
+ DefTxnTotalSizeLimit = 100 * 1024 * 1024
+ SuperLargeTxnSize uint64 = 100 * 1024 * 1024 * 1024 * 1024 // 100T, we expect a txn can never be this large
// DefMaxIndexLength is the maximum index length(in bytes). This value is consistent with MySQL.
DefMaxIndexLength = 3072
// DefMaxOfMaxIndexLength is the maximum index length(in bytes) for TiDB v3.0.7 and previous version.
@@ -88,6 +92,8 @@ const (
DefTempDir = "/tmp/tidb"
// DefAuthTokenRefreshInterval is the default time interval to refresh tidb auth token.
DefAuthTokenRefreshInterval = time.Hour
+ // EnvVarKeyspaceName is the system env name for keyspace name.
+ EnvVarKeyspaceName = "KEYSPACE_NAME"
)
// Valid config maps
@@ -182,6 +188,7 @@ type Config struct {
VersionComment string `toml:"version-comment" json:"version-comment"`
TiDBEdition string `toml:"tidb-edition" json:"tidb-edition"`
TiDBReleaseVersion string `toml:"tidb-release-version" json:"tidb-release-version"`
+ KeyspaceName string `toml:"keyspace-name" json:"keyspace-name"`
Log Log `toml:"log" json:"log"`
Instance Instance `toml:"instance" json:"instance"`
Security Security `toml:"security" json:"security"`
@@ -260,6 +267,9 @@ type Config struct {
// EnableGlobalKill indicates whether to enable global kill.
TrxSummary TrxSummary `toml:"transaction-summary" json:"transaction-summary"`
EnableGlobalKill bool `toml:"enable-global-kill" json:"enable-global-kill"`
+ // InitializeSQLFile is a file that will be executed after first bootstrap only.
+ // It can be used to set GLOBAL system variable values
+ InitializeSQLFile string `toml:"initialize-sql-file" json:"initialize-sql-file"`
// The following items are deprecated. We need to keep them here temporarily
// to support the upgrade process. They can be removed in future.
@@ -278,10 +288,22 @@ type Config struct {
Plugin Plugin `toml:"plugin" json:"plugin"`
MaxServerConnections uint32 `toml:"max-server-connections" json:"max-server-connections"`
RunDDL bool `toml:"run-ddl" json:"run-ddl"`
+
+ // These configs are related to disaggregated-tiflash mode.
+ DisaggregatedTiFlash bool `toml:"disaggregated-tiflash" json:"disaggregated-tiflash"`
+ TiFlashComputeAutoScalerType string `toml:"autoscaler-type" json:"autoscaler-type"`
+ TiFlashComputeAutoScalerAddr string `toml:"autoscaler-addr" json:"autoscaler-addr"`
+ IsTiFlashComputeFixedPool bool `toml:"is-tiflashcompute-fixed-pool" json:"is-tiflashcompute-fixed-pool"`
+ AutoScalerClusterID string `toml:"autoscaler-cluster-id" json:"autoscaler-cluster-id"`
+ // todo: remove this after AutoScaler is stable.
+ UseAutoScaler bool `toml:"use-autoscaler" json:"use-autoscaler"`
+
// TiDBMaxReuseChunk indicates max cached chunk num
TiDBMaxReuseChunk uint32 `toml:"tidb-max-reuse-chunk" json:"tidb-max-reuse-chunk"`
// TiDBMaxReuseColumn indicates max cached column num
TiDBMaxReuseColumn uint32 `toml:"tidb-max-reuse-column" json:"tidb-max-reuse-column"`
+ // TiDBEnableExitCheck indicates whether exit-checking in domain for background process
+ TiDBEnableExitCheck bool `toml:"tidb-enable-exit-check" json:"tidb-enable-exit-check"`
}
// UpdateTempStoragePath is to update the `TempStoragePath` if port/statusPort was changed
@@ -480,6 +502,20 @@ type Instance struct {
DDLSlowOprThreshold uint32 `toml:"ddl_slow_threshold" json:"ddl_slow_threshold"`
// ExpensiveQueryTimeThreshold indicates the time threshold of expensive query.
ExpensiveQueryTimeThreshold uint64 `toml:"tidb_expensive_query_time_threshold" json:"tidb_expensive_query_time_threshold"`
+ // StmtSummaryEnablePersistent indicates whether to enable file persistence for stmtsummary.
+ StmtSummaryEnablePersistent bool `toml:"tidb_stmt_summary_enable_persistent" json:"tidb_stmt_summary_enable_persistent"`
+ // StmtSummaryFilename indicates the file name written by stmtsummary
+ // when StmtSummaryEnablePersistent is true.
+ StmtSummaryFilename string `toml:"tidb_stmt_summary_filename" json:"tidb_stmt_summary_filename"`
+ // StmtSummaryFileMaxDays indicates how many days the files written by
+ // stmtsummary will be kept when StmtSummaryEnablePersistent is true.
+ StmtSummaryFileMaxDays int `toml:"tidb_stmt_summary_file_max_days" json:"tidb_stmt_summary_file_max_days"`
+ // StmtSummaryFileMaxSize indicates the maximum size (in mb) of a single file
+ // written by stmtsummary when StmtSummaryEnablePersistent is true.
+ StmtSummaryFileMaxSize int `toml:"tidb_stmt_summary_file_max_size" json:"tidb_stmt_summary_file_max_size"`
+ // StmtSummaryFileMaxBackups indicates the maximum number of files written
+ // by stmtsummary when StmtSummaryEnablePersistent is true.
+ StmtSummaryFileMaxBackups int `toml:"tidb_stmt_summary_file_max_backups" json:"tidb_stmt_summary_file_max_backups"`
// These variables exist in both 'instance' section and another place.
// The configuration in 'instance' section takes precedence.
@@ -556,6 +592,8 @@ type Security struct {
AuthTokenJWKS string `toml:"auth-token-jwks" json:"auth-token-jwks"`
// The refresh time interval of JWKS
AuthTokenRefreshInterval string `toml:"auth-token-refresh-interval" json:"auth-token-refresh-interval"`
+ // Disconnect directly when the password is expired
+ DisconnectOnExpiredPassword bool `toml:"disconnect-on-expired-password" json:"disconnect-on-expired-password"`
}
// The ErrConfigValidationFailed error is used so that external callers can do a type assertion
@@ -655,14 +693,15 @@ type Performance struct {
// Deprecated
MemProfileInterval string `toml:"-" json:"-"`
- IndexUsageSyncLease string `toml:"index-usage-sync-lease" json:"index-usage-sync-lease"`
- PlanReplayerGCLease string `toml:"plan-replayer-gc-lease" json:"plan-replayer-gc-lease"`
- GOGC int `toml:"gogc" json:"gogc"`
- EnforceMPP bool `toml:"enforce-mpp" json:"enforce-mpp"`
- StatsLoadConcurrency uint `toml:"stats-load-concurrency" json:"stats-load-concurrency"`
- StatsLoadQueueSize uint `toml:"stats-load-queue-size" json:"stats-load-queue-size"`
- AnalyzePartitionConcurrencyQuota uint `toml:"analyze-partition-concurrency-quota" json:"analyze-partition-concurrency-quota"`
- EnableStatsCacheMemQuota bool `toml:"enable-stats-cache-mem-quota" json:"enable-stats-cache-mem-quota"`
+ IndexUsageSyncLease string `toml:"index-usage-sync-lease" json:"index-usage-sync-lease"`
+ PlanReplayerGCLease string `toml:"plan-replayer-gc-lease" json:"plan-replayer-gc-lease"`
+ GOGC int `toml:"gogc" json:"gogc"`
+ EnforceMPP bool `toml:"enforce-mpp" json:"enforce-mpp"`
+ StatsLoadConcurrency uint `toml:"stats-load-concurrency" json:"stats-load-concurrency"`
+ StatsLoadQueueSize uint `toml:"stats-load-queue-size" json:"stats-load-queue-size"`
+ AnalyzePartitionConcurrencyQuota uint `toml:"analyze-partition-concurrency-quota" json:"analyze-partition-concurrency-quota"`
+ PlanReplayerDumpWorkerConcurrency uint `toml:"plan-replayer-dump-worker-concurrency" json:"plan-replayer-dump-worker-concurrency"`
+ EnableStatsCacheMemQuota bool `toml:"enable-stats-cache-mem-quota" json:"enable-stats-cache-mem-quota"`
// The following items are deprecated. We need to keep them here temporarily
// to support the upgrade process. They can be removed in future.
@@ -876,6 +915,11 @@ var defaultConf = Config{
EnablePProfSQLCPU: false,
DDLSlowOprThreshold: DefDDLSlowOprThreshold,
ExpensiveQueryTimeThreshold: DefExpensiveQueryTimeThreshold,
+ StmtSummaryEnablePersistent: false,
+ StmtSummaryFilename: "tidb-statements.log",
+ StmtSummaryFileMaxDays: 3,
+ StmtSummaryFileMaxSize: 64,
+ StmtSummaryFileMaxBackups: 0,
EnableSlowLog: *NewAtomicBool(logutil.DefaultTiDBEnableSlowLog),
SlowThreshold: logutil.DefaultSlowThreshold,
RecordPlanInSlowLog: logutil.DefaultRecordPlanInSlowLog,
@@ -920,16 +964,17 @@ var defaultConf = Config{
CommitterConcurrency: defTiKVCfg.CommitterConcurrency,
MaxTxnTTL: defTiKVCfg.MaxTxnTTL, // 1hour
// TODO: set indexUsageSyncLease to 60s.
- IndexUsageSyncLease: "0s",
- GOGC: 100,
- EnforceMPP: false,
- PlanReplayerGCLease: "10m",
- StatsLoadConcurrency: 5,
- StatsLoadQueueSize: 1000,
- AnalyzePartitionConcurrencyQuota: 16,
- EnableStatsCacheMemQuota: false,
- RunAutoAnalyze: true,
- EnableLoadFMSketch: false,
+ IndexUsageSyncLease: "0s",
+ GOGC: 100,
+ EnforceMPP: false,
+ PlanReplayerGCLease: "10m",
+ StatsLoadConcurrency: 5,
+ StatsLoadQueueSize: 1000,
+ AnalyzePartitionConcurrencyQuota: 16,
+ PlanReplayerDumpWorkerConcurrency: 1,
+ EnableStatsCacheMemQuota: false,
+ RunAutoAnalyze: true,
+ EnableLoadFMSketch: false,
},
ProxyProtocol: ProxyProtocol{
Networks: "",
@@ -974,6 +1019,7 @@ var defaultConf = Config{
RSAKeySize: 4096,
AuthTokenJWKS: "",
AuthTokenRefreshInterval: DefAuthTokenRefreshInterval.String(),
+ DisconnectOnExpiredPassword: true,
},
DeprecateIntegerDisplayWidth: false,
EnableEnumLengthLimit: true,
@@ -982,8 +1028,15 @@ var defaultConf = Config{
NewCollationsEnabledOnFirstBootstrap: true,
EnableGlobalKill: true,
TrxSummary: DefaultTrxSummary(),
+ DisaggregatedTiFlash: false,
+ TiFlashComputeAutoScalerType: tiflashcompute.DefASStr,
+ TiFlashComputeAutoScalerAddr: tiflashcompute.DefAWSAutoScalerAddr,
+ IsTiFlashComputeFixedPool: false,
+ AutoScalerClusterID: "",
+ UseAutoScaler: true,
TiDBMaxReuseChunk: 64,
TiDBMaxReuseColumn: 256,
+ TiDBEnableExitCheck: false,
}
var (
@@ -1012,6 +1065,26 @@ func StoreGlobalConfig(config *Config) {
tikvcfg.StoreGlobalConfig(&cfg)
}
+// GetAutoScalerClusterID returns KeyspaceName or AutoScalerClusterID.
+func GetAutoScalerClusterID() (string, error) {
+ c := GetGlobalConfig()
+ keyspaceName := c.KeyspaceName
+ clusterID := c.AutoScalerClusterID
+
+ if keyspaceName != "" && clusterID != "" {
+ return "", errors.Errorf("config.KeyspaceName(%s) and config.AutoScalerClusterID(%s) are not empty both", keyspaceName, clusterID)
+ }
+ if keyspaceName == "" && clusterID == "" {
+ return "", errors.Errorf("config.KeyspaceName and config.AutoScalerClusterID are both empty")
+ }
+
+ res := keyspaceName
+ if res == "" {
+ res = clusterID
+ }
+ return res, nil
+}
+
// removedConfig contains items that are no longer supported.
// they might still be in the config struct to support import,
// but are not actively used.
@@ -1296,6 +1369,17 @@ func (c *Config) Valid() error {
return fmt.Errorf("stats-load-queue-size should be [%d, %d]", DefStatsLoadQueueSizeLimit, DefMaxOfStatsLoadQueueSizeLimit)
}
+ // Check tiflash_compute topo fetch is valid.
+ if c.DisaggregatedTiFlash && c.UseAutoScaler {
+ if !tiflashcompute.IsValidAutoScalerConfig(c.TiFlashComputeAutoScalerType) {
+ return fmt.Errorf("invalid AutoScaler type, expect %s, %s or %s, got %s",
+ tiflashcompute.MockASStr, tiflashcompute.AWSASStr, tiflashcompute.GCPASStr, c.TiFlashComputeAutoScalerType)
+ }
+ if c.TiFlashComputeAutoScalerAddr == "" {
+ return fmt.Errorf("autoscaler-addr cannot be empty when disaggregated-tiflash mode is true")
+ }
+ }
+
// test log level
l := zap.NewAtomicLevel()
return l.UnmarshalText([]byte(c.Log.Level))
@@ -1446,3 +1530,9 @@ func ContainHiddenConfig(s string) bool {
}
return false
}
+
+// GetGlobalKeyspaceName is used to get global keyspace name
+// from config file or command line.
+func GetGlobalKeyspaceName() string {
+ return GetGlobalConfig().KeyspaceName
+}
diff --git a/config/config.toml.example b/config/config.toml.example
index 588379f204602..74cbc9413a496 100644
--- a/config/config.toml.example
+++ b/config/config.toml.example
@@ -114,6 +114,25 @@ enable-enum-length-limit = true
# command can be forwarded to the right TiDB instance to execute.
enable-global-kill = true
+# disaggregated-tiflash indicates whether TiDB is in disaggregated tiflash mode, if true, MPP will runs on tiflash_compute nodes.
+disaggregated-tiflash = false
+
+# autoscaler-type indicates which type of AutoScaler will be used. Possible values are: mock, aws, gcp.
+# Only meaningful when disaggregated-tiflash is true.
+autoscaler-type = "aws"
+
+# autoscaler-addr is the host of AutoScaler, Only meaningful when disaggregated-tiflash is true.
+# Only meaningful when disaggregated-tiflash is true.
+autoscaler-addr = "tiflash-autoscale-lb.tiflash-autoscale.svc.cluster.local:8081"
+
+# autoscaler-cluster-id is the unique id for each TiDB cluster, which will used by AutoScaler.
+# Only meaningful when disaggregated-tiflash is true.
+autoscaler-cluster-id = ""
+
+# use-autoscaler indicates whether use AutoScaler or PD for tiflash_compute nodes, only meaningful when disaggregated-tiflash is true.
+# Will remove this after AutoScaler is stable.
+use-autoscaler = true
+
[log]
# Log level: debug, info, warn, error, fatal.
level = "info"
diff --git a/config/config_test.go b/config/config_test.go
index 9a6d12a284817..4bd0911661e11 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -1288,3 +1288,18 @@ func TestStatsLoadLimit(t *testing.T) {
checkQueueSizeValid(DefMaxOfStatsLoadQueueSizeLimit, true)
checkQueueSizeValid(DefMaxOfStatsLoadQueueSizeLimit+1, false)
}
+
+func TestGetGlobalKeyspaceName(t *testing.T) {
+ conf := NewConfig()
+ require.Empty(t, conf.KeyspaceName)
+
+ UpdateGlobal(func(conf *Config) {
+ conf.KeyspaceName = "test"
+ })
+
+ require.Equal(t, "test", GetGlobalKeyspaceName())
+
+ UpdateGlobal(func(conf *Config) {
+ conf.KeyspaceName = ""
+ })
+}
diff --git a/config/main_test.go b/config/main_test.go
index 3458dc358ad86..9c9d4ac738f72 100644
--- a/config/main_test.go
+++ b/config/main_test.go
@@ -28,6 +28,7 @@ func TestMain(m *testing.M) {
goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
+ goleak.IgnoreTopFunction("syscall.syscall"),
}
goleak.VerifyTestMain(m, opts...)
}
diff --git a/ddl/BUILD.bazel b/ddl/BUILD.bazel
index b54f52a6d192c..2fbe94b2fcd16 100644
--- a/ddl/BUILD.bazel
+++ b/ddl/BUILD.bazel
@@ -24,9 +24,12 @@ go_library(
"ddl_workerpool.go",
"delete_range.go",
"delete_range_util.go",
+ "dist_backfilling.go",
+ "dist_owner.go",
"foreign_key.go",
"generated_column.go",
"index.go",
+ "index_cop.go",
"index_merge_tmp.go",
"job_table.go",
"mock.go",
@@ -35,6 +38,7 @@ go_library(
"partition.go",
"placement_policy.go",
"reorg.go",
+ "resource_group.go",
"rollingback.go",
"sanity_check.go",
"schema.go",
@@ -44,17 +48,20 @@ go_library(
"stat.go",
"table.go",
"table_lock.go",
+ "ttl.go",
],
importpath = "github.com/pingcap/tidb/ddl",
visibility = [
":ddl_friend",
],
deps = [
+ "//br/pkg/lightning/common",
"//br/pkg/utils",
"//config",
"//ddl/ingest",
"//ddl/label",
"//ddl/placement",
+ "//ddl/resourcegroup",
"//ddl/syncer",
"//ddl/util",
"//distsql",
@@ -75,6 +82,9 @@ go_library(
"//parser/opcode",
"//parser/terror",
"//parser/types",
+ "//privilege",
+ "//resourcemanager/pooltask",
+ "//resourcemanager/util",
"//sessionctx",
"//sessionctx/binloginfo",
"//sessionctx/stmtctx",
@@ -99,7 +109,11 @@ go_library(
"//util/domainutil",
"//util/filter",
"//util/gcutil",
+ "//util/generic",
+ "//util/gpool",
+ "//util/gpool/spmc",
"//util/hack",
+ "//util/intest",
"//util/logutil",
"//util/mathutil",
"//util/mock",
@@ -110,6 +124,7 @@ go_library(
"//util/slice",
"//util/sqlexec",
"//util/stringutil",
+ "//util/syncutil",
"//util/timeutil",
"//util/topsql",
"//util/topsql/state",
@@ -117,6 +132,7 @@ go_library(
"@com_github_ngaut_pools//:pools",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_pingcap_kvproto//pkg/errorpb",
"@com_github_pingcap_kvproto//pkg/kvrpcpb",
"@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_pingcap_log//:log",
@@ -141,7 +157,6 @@ go_test(
srcs = [
"attributes_sql_test.go",
"backfilling_test.go",
- "callback_test.go",
"cancel_test.go",
"cluster_test.go",
"column_change_test.go",
@@ -161,27 +176,29 @@ go_test(
"ddl_api_test.go",
"ddl_error_test.go",
"ddl_test.go",
- "ddl_tiflash_test.go",
"ddl_worker_test.go",
"ddl_workerpool_test.go",
"export_test.go",
"fail_test.go",
"foreign_key_test.go",
"index_change_test.go",
- "index_merge_tmp_test.go",
+ "index_cop_test.go",
"index_modify_test.go",
"integration_test.go",
"job_table_test.go",
"main_test.go",
"modify_column_test.go",
"multi_schema_change_test.go",
+ "mv_index_test.go",
"options_test.go",
"partition_test.go",
"placement_policy_ddl_test.go",
"placement_policy_test.go",
"placement_sql_test.go",
"primary_key_handle_test.go",
+ "reorg_partition_test.go",
"repair_table_test.go",
+ "resource_group_test.go",
"restart_test.go",
"rollingback_test.go",
"schema_test.go",
@@ -192,13 +209,15 @@ go_test(
"table_split_test.go",
"table_test.go",
"tiflash_replica_test.go",
+ "ttl_test.go",
],
embed = [":ddl"],
flaky = True,
shard_count = 50,
deps = [
+ "//autoid_service",
"//config",
- "//ddl/ingest",
+ "//ddl/internal/callback",
"//ddl/placement",
"//ddl/schematracker",
"//ddl/testutil",
@@ -208,6 +227,7 @@ go_test(
"//errno",
"//executor",
"//infoschema",
+ "//keyspace",
"//kv",
"//meta",
"//meta/autoid",
@@ -229,7 +249,6 @@ go_test(
"//store/gcworker",
"//store/helper",
"//store/mockstore",
- "//store/mockstore/unistore",
"//table",
"//table/tables",
"//tablecodec",
@@ -245,13 +264,13 @@ go_test(
"//util/domainutil",
"//util/gcutil",
"//util/logutil",
+ "//util/mathutil",
"//util/mock",
"//util/sem",
"//util/sqlexec",
"@com_github_ngaut_pools//:pools",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
- "@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@com_github_tikv_client_go_v2//oracle",
diff --git a/ddl/attributes_sql_test.go b/ddl/attributes_sql_test.go
index 95f881e6fb3fe..4d5300d600470 100644
--- a/ddl/attributes_sql_test.go
+++ b/ddl/attributes_sql_test.go
@@ -24,6 +24,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl/util"
"github.com/pingcap/tidb/domain/infosync"
+ "github.com/pingcap/tidb/keyspace"
"github.com/pingcap/tidb/store/gcworker"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/util/gcutil"
@@ -73,6 +74,8 @@ func TestAlterTableAttributes(t *testing.T) {
// without equal
tk.MustExec(`alter table alter_t attributes " merge_option=allow ";`)
tk.MustExec(`alter table alter_t attributes " merge_option=allow , key=value ";`)
+
+ tk.MustExec("drop table alter_t")
}
func TestAlterTablePartitionAttributes(t *testing.T) {
@@ -134,6 +137,8 @@ PARTITION BY RANGE (c) (
require.Len(t, rows4, 1)
require.NotEqual(t, rows3[0][3], rows4[0][3])
require.NotEqual(t, rows[0][3], rows4[0][3])
+
+ tk.MustExec("drop table alter_p")
}
func TestTruncateTable(t *testing.T) {
@@ -269,7 +274,7 @@ PARTITION BY RANGE (c) (
func TestFlashbackTable(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
- _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), true)
+ _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), dom.GetEtcdClient(), dom.GetPDClient(), keyspace.CodecV1, true)
require.NoError(t, err)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -327,7 +332,7 @@ PARTITION BY RANGE (c) (
func TestDropTable(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
- _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), true)
+ _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), dom.GetEtcdClient(), dom.GetPDClient(), keyspace.CodecV1, true)
require.NoError(t, err)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -380,7 +385,7 @@ PARTITION BY RANGE (c) (
func TestCreateWithSameName(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
- _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), true)
+ _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), dom.GetEtcdClient(), dom.GetPDClient(), keyspace.CodecV1, true)
require.NoError(t, err)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -444,7 +449,7 @@ PARTITION BY RANGE (c) (
func TestPartition(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
- _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), true)
+ _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), dom.GetEtcdClient(), dom.GetPDClient(), keyspace.CodecV1, true)
require.NoError(t, err)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -504,7 +509,7 @@ PARTITION BY RANGE (c) (
func TestDropSchema(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
- _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), true)
+ _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), dom.GetEtcdClient(), dom.GetPDClient(), keyspace.CodecV1, true)
require.NoError(t, err)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -530,7 +535,7 @@ PARTITION BY RANGE (c) (
func TestDefaultKeyword(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
- _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), true)
+ _, err := infosync.GlobalInfoSyncerInit(context.Background(), dom.DDL().GetID(), dom.ServerID, dom.GetEtcdClient(), dom.GetEtcdClient(), dom.GetPDClient(), keyspace.CodecV1, true)
require.NoError(t, err)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
diff --git a/ddl/backfilling.go b/ddl/backfilling.go
index dfd213652ab2e..c6bb38297402d 100644
--- a/ddl/backfilling.go
+++ b/ddl/backfilling.go
@@ -15,7 +15,6 @@
package ddl
import (
- "bytes"
"context"
"encoding/hex"
"fmt"
@@ -32,13 +31,16 @@ import (
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/store/copr"
"github.com/pingcap/tidb/store/driver/backoff"
"github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/tablecodec"
+ "github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/logutil"
@@ -46,29 +48,108 @@ import (
decoder "github.com/pingcap/tidb/util/rowDecoder"
"github.com/pingcap/tidb/util/timeutil"
"github.com/pingcap/tidb/util/topsql"
+ "github.com/tikv/client-go/v2/oracle"
"github.com/tikv/client-go/v2/tikv"
"go.uber.org/zap"
)
-type backfillWorkerType byte
+type backfillerType byte
const (
- typeAddIndexWorker backfillWorkerType = 0
- typeUpdateColumnWorker backfillWorkerType = 1
- typeCleanUpIndexWorker backfillWorkerType = 2
- typeAddIndexMergeTmpWorker backfillWorkerType = 3
+ typeAddIndexWorker backfillerType = 0
+ typeUpdateColumnWorker backfillerType = 1
+ typeCleanUpIndexWorker backfillerType = 2
+ typeAddIndexMergeTmpWorker backfillerType = 3
+ typeReorgPartitionWorker backfillerType = 4
+
+ // InstanceLease is the instance lease.
+ InstanceLease = 1 * time.Minute
+ updateInstanceLease = 25 * time.Second
+ genTaskBatch = 4096
+ genPhysicalTableTaskBatch = 256
+ minGenTaskBatch = 1024
+ minGenPhysicalTableTaskBatch = 64
+ minDistTaskCnt = 64
+ retrySQLTimes = 10
)
+// RetrySQLInterval is export for test.
+var RetrySQLInterval = 300 * time.Millisecond
+
+func (bT backfillerType) String() string {
+ switch bT {
+ case typeAddIndexWorker:
+ return "add index"
+ case typeUpdateColumnWorker:
+ return "update column"
+ case typeCleanUpIndexWorker:
+ return "clean up index"
+ case typeAddIndexMergeTmpWorker:
+ return "merge temporary index"
+ case typeReorgPartitionWorker:
+ return "reorganize partition"
+ default:
+ return "unknown"
+ }
+}
+
+// BackfillJob is for a tidb_background_subtask table's record.
+type BackfillJob struct {
+ ID int64
+ JobID int64
+ EleID int64
+ EleKey []byte
+ PhysicalTableID int64
+ Tp backfillerType
+ State model.JobState
+ InstanceID string
+ InstanceLease types.Time
+ StartTS uint64
+ StateUpdateTS uint64
+ Meta *model.BackfillMeta
+}
+
+// AbbrStr returns the BackfillJob's info without the Meta info.
+func (bj *BackfillJob) AbbrStr() string {
+ return fmt.Sprintf("ID:%d, JobID:%d, EleID:%d, Type:%s, State:%s, InstanceID:%s, InstanceLease:%s",
+ bj.ID, bj.JobID, bj.EleID, bj.Tp, bj.State, bj.InstanceID, bj.InstanceLease)
+}
+
+// GetOracleTimeWithStartTS returns the current time with txn's startTS.
+func GetOracleTimeWithStartTS(se *session) (time.Time, error) {
+ txn, err := se.Txn(true)
+ if err != nil {
+ return time.Time{}, err
+ }
+ return oracle.GetTimeFromTS(txn.StartTS()).UTC(), nil
+}
+
+// GetOracleTime returns the current time from TS without txn.
+func GetOracleTime(store kv.Storage) (time.Time, error) {
+ currentVer, err := store.CurrentVersion(kv.GlobalTxnScope)
+ if err != nil {
+ return time.Time{}, errors.Trace(err)
+ }
+ return oracle.GetTimeFromTS(currentVer.Ver).UTC(), nil
+}
+
+// GetLeaseGoTime returns a types.Time by adding a lease.
+func GetLeaseGoTime(currTime time.Time, lease time.Duration) types.Time {
+ leaseTime := currTime.Add(lease)
+ return types.NewTime(types.FromGoTime(leaseTime.In(time.UTC)), mysql.TypeTimestamp, types.MaxFsp)
+}
+
// By now the DDL jobs that need backfilling include:
// 1: add-index
// 2: modify-column-type
// 3: clean-up global index
+// 4: reorganize partition
//
// They all have a write reorganization state to back fill data into the rows existed.
// Backfilling is time consuming, to accelerate this process, TiDB has built some sub
// workers to do this in the DDL owner node.
//
-// DDL owner thread
+// DDL owner thread (also see comments before runReorgJob func)
// ^
// | (reorgCtx.doneCh)
// |
@@ -111,24 +192,52 @@ const (
// Instead, it is divided into batches, each time a kv transaction completes the backfilling
// of a partial batch.
-func (bWT backfillWorkerType) String() string {
- switch bWT {
- case typeAddIndexWorker:
- return "add index"
- case typeUpdateColumnWorker:
- return "update column"
- case typeCleanUpIndexWorker:
- return "clean up index"
- case typeAddIndexMergeTmpWorker:
- return "merge temporary index"
- default:
- return "unknown"
+// backfillTaskContext is the context of the batch adding indices or updating column values.
+// After finishing the batch adding indices or updating column values, result in backfillTaskContext will be merged into backfillResult.
+type backfillTaskContext struct {
+ nextKey kv.Key
+ done bool
+ addedCount int
+ scanCount int
+ warnings map[errors.ErrorID]*terror.Error
+ warningsCount map[errors.ErrorID]int64
+ finishTS uint64
+}
+
+type backfillCtx struct {
+ id int
+ *ddlCtx
+ reorgTp model.ReorgType
+ sessCtx sessionctx.Context
+ schemaName string
+ table table.Table
+ batchCnt int
+}
+
+func newBackfillCtx(ctx *ddlCtx, id int, sessCtx sessionctx.Context, reorgTp model.ReorgType,
+ schemaName string, tbl table.Table) *backfillCtx {
+ if id == 0 {
+ id = int(backfillContextID.Add(1))
+ }
+ return &backfillCtx{
+ id: id,
+ ddlCtx: ctx,
+ sessCtx: sessCtx,
+ reorgTp: reorgTp,
+ schemaName: schemaName,
+ table: tbl,
+ batchCnt: int(variable.GetDDLReorgBatchSize()),
}
}
type backfiller interface {
BackfillDataInTxn(handleRange reorgBackfillTask) (taskCtx backfillTaskContext, errInTxn error)
AddMetricInfo(float64)
+ GetTasks() ([]*BackfillJob, error)
+ UpdateTask(bfJob *BackfillJob) error
+ FinishTask(bfJob *BackfillJob) error
+ GetCtx() *backfillCtx
+ String() string
}
type backfillResult struct {
@@ -139,30 +248,40 @@ type backfillResult struct {
err error
}
-// backfillTaskContext is the context of the batch adding indices or updating column values.
-// After finishing the batch adding indices or updating column values, result in backfillTaskContext will be merged into backfillResult.
-type backfillTaskContext struct {
- nextKey kv.Key
- done bool
- addedCount int
- scanCount int
- warnings map[errors.ErrorID]*terror.Error
- warningsCount map[errors.ErrorID]int64
+type reorgBackfillTask struct {
+ bfJob *BackfillJob
+ physicalTable table.PhysicalTable
+
+ // TODO: Remove the following fields after remove the function of run.
+ id int
+ startKey kv.Key
+ endKey kv.Key
+ endInclude bool
+ jobID int64
+ sqlQuery string
+ priority int
}
-type reorgBackfillTask struct {
- id int
- physicalTableID int64
- startKey kv.Key
- endKey kv.Key
- endInclude bool
+func (r *reorgBackfillTask) getJobID() int64 {
+ jobID := r.jobID
+ if r.bfJob != nil {
+ jobID = r.bfJob.JobID
+ }
+ return jobID
+}
+
+func (r *reorgBackfillTask) excludedEndKey() kv.Key {
+ if r.endInclude {
+ return r.endKey.Next()
+ }
+ return r.endKey
}
func (r *reorgBackfillTask) String() string {
- physicalID := strconv.FormatInt(r.physicalTableID, 10)
- startKey := tryDecodeToHandleString(r.startKey)
- endKey := tryDecodeToHandleString(r.endKey)
- rangeStr := "physicalTableID_" + physicalID + "_" + "[" + startKey + "," + endKey
+ physicalID := strconv.FormatInt(r.physicalTable.GetPhysicalID(), 10)
+ startKey := hex.EncodeToString(r.startKey)
+ endKey := hex.EncodeToString(r.endKey)
+ rangeStr := "taskID_" + strconv.Itoa(r.id) + "_physicalTableID_" + physicalID + "_" + "[" + startKey + "," + endKey
if r.endInclude {
return rangeStr + "]"
}
@@ -177,35 +296,43 @@ func mergeBackfillCtxToResult(taskCtx *backfillTaskContext, result *backfillResu
}
type backfillWorker struct {
- id int
- reorgInfo *reorgInfo
- batchCnt int
- sessCtx sessionctx.Context
- taskCh chan *reorgBackfillTask
- resultCh chan *backfillResult
- table table.Table
- priority int
- tp backfillWorkerType
- ctx context.Context
- cancel func()
-}
-
-func newBackfillWorker(ctx context.Context, sessCtx sessionctx.Context, id int, t table.PhysicalTable,
- reorgInfo *reorgInfo, tp backfillWorkerType) *backfillWorker {
+ backfiller
+ taskCh chan *reorgBackfillTask
+ resultCh chan *backfillResult
+ ctx context.Context
+ cancel func()
+}
+
+func newBackfillWorker(ctx context.Context, bf backfiller) *backfillWorker {
bfCtx, cancel := context.WithCancel(ctx)
return &backfillWorker{
- id: id,
- table: t,
- reorgInfo: reorgInfo,
- batchCnt: int(variable.GetDDLReorgBatchSize()),
- sessCtx: sessCtx,
- priority: reorgInfo.Job.Priority,
- tp: tp,
- ctx: bfCtx,
- cancel: cancel,
+ backfiller: bf,
+ taskCh: make(chan *reorgBackfillTask, 1),
+ resultCh: make(chan *backfillResult, 1),
+ ctx: bfCtx,
+ cancel: cancel,
}
}
+func (w *backfillWorker) updateLease(execID string, bfJob *BackfillJob, nextKey kv.Key) error {
+ leaseTime, err := GetOracleTime(w.GetCtx().store)
+ if err != nil {
+ return err
+ }
+ bfJob.Meta.CurrKey = nextKey
+ bfJob.InstanceID = execID
+ bfJob.InstanceLease = GetLeaseGoTime(leaseTime, InstanceLease)
+ return w.backfiller.UpdateTask(bfJob)
+}
+
+func (w *backfillWorker) finishJob(bfJob *BackfillJob) error {
+ return w.backfiller.FinishTask(bfJob)
+}
+
+func (w *backfillWorker) String() string {
+ return fmt.Sprintf("backfill-worker %d, tp %s", w.GetCtx().id, w.backfiller.String())
+}
+
func (w *backfillWorker) Close() {
if w.cancel != nil {
w.cancel()
@@ -231,17 +358,23 @@ func (w *backfillWorker) handleBackfillTask(d *ddlCtx, task *reorgBackfillTask,
addedCount: 0,
nextKey: handleRange.startKey,
}
+ batchStartTime := time.Now()
lastLogCount := 0
lastLogTime := time.Now()
startTime := lastLogTime
- rc := d.getReorgCtx(w.reorgInfo.Job)
+ jobID := task.getJobID()
+ rc := d.getReorgCtx(jobID)
+ isDistReorg := task.bfJob != nil
+ if isDistReorg {
+ w.initPartitionIndexInfo(task)
+ }
for {
// Give job chance to be canceled, if we not check it here,
// if there is panic in bf.BackfillDataInTxn we will never cancel the job.
// Because reorgRecordTask may run a long time,
// we should check whether this ddl job is still runnable.
- err := d.isReorgRunnable(w.reorgInfo.Job)
+ err := d.isReorgRunnable(jobID, isDistReorg)
if err != nil {
result.err = err
return result
@@ -267,13 +400,11 @@ func (w *backfillWorker) handleBackfillTask(d *ddlCtx, task *reorgBackfillTask,
rc.increaseRowCount(int64(taskCtx.addedCount))
rc.mergeWarnings(taskCtx.warnings, taskCtx.warningsCount)
- if num := result.scanCount - lastLogCount; num >= 30000 {
+ if num := result.scanCount - lastLogCount; num >= 90000 {
lastLogCount = result.scanCount
- logutil.BgLogger().Info("[ddl] backfill worker back fill index",
- zap.Int("workerID", w.id),
- zap.Int("addedCount", result.addedCount),
- zap.Int("scanCount", result.scanCount),
- zap.String("nextHandle", tryDecodeToHandleString(taskCtx.nextKey)),
+ logutil.BgLogger().Info("[ddl] backfill worker back fill index", zap.Stringer("worker", w),
+ zap.Int("addedCount", result.addedCount), zap.Int("scanCount", result.scanCount),
+ zap.String("next key", hex.EncodeToString(taskCtx.nextKey)),
zap.Float64("speed(rows/s)", float64(num)/time.Since(lastLogTime).Seconds()))
lastLogTime = time.Now()
}
@@ -282,47 +413,110 @@ func (w *backfillWorker) handleBackfillTask(d *ddlCtx, task *reorgBackfillTask,
if taskCtx.done {
break
}
+
+ if isDistReorg {
+ // TODO: Adjust the updating lease frequency by batch processing time carefully.
+ if time.Since(batchStartTime) < updateInstanceLease {
+ continue
+ }
+ batchStartTime = time.Now()
+ if err := w.updateLease(w.GetCtx().uuid, task.bfJob, result.nextKey); err != nil {
+ logutil.BgLogger().Info("[ddl] backfill worker handle task, update lease failed", zap.Stringer("worker", w),
+ zap.Stringer("task", task), zap.String("backfill job", task.bfJob.AbbrStr()), zap.Error(err))
+ result.err = err
+ return result
+ }
+ }
}
logutil.BgLogger().Info("[ddl] backfill worker finish task",
- zap.Stringer("type", w.tp),
- zap.Int("workerID", w.id),
- zap.String("task", task.String()),
- zap.Int("addedCount", result.addedCount),
- zap.Int("scanCount", result.scanCount),
- zap.String("nextHandle", tryDecodeToHandleString(result.nextKey)),
- zap.String("takeTime", time.Since(startTime).String()))
+ zap.Stringer("worker", w), zap.Stringer("task", task),
+ zap.Int("added count", result.addedCount),
+ zap.Int("scan count", result.scanCount),
+ zap.String("next key", hex.EncodeToString(result.nextKey)),
+ zap.Stringer("take time", time.Since(startTime)))
if ResultCounterForTest != nil && result.err == nil {
ResultCounterForTest.Add(1)
}
return result
}
+func (w *backfillWorker) initPartitionIndexInfo(task *reorgBackfillTask) {
+ if pt, ok := w.GetCtx().table.(table.PartitionedTable); ok {
+ if addIdxWorker, ok := w.backfiller.(*addIndexWorker); ok {
+ indexInfo := model.FindIndexInfoByID(pt.Meta().Indices, task.bfJob.EleID)
+ addIdxWorker.index = tables.NewIndex(task.bfJob.PhysicalTableID, pt.Meta(), indexInfo)
+ }
+ }
+}
+
+func (w *backfillWorker) runTask(task *reorgBackfillTask) (result *backfillResult) {
+ logutil.BgLogger().Info("[ddl] backfill worker start", zap.Stringer("worker", w), zap.String("task", task.String()))
+ defer util.Recover(metrics.LabelDDL, "backfillWorker.runTask", func() {
+ result = &backfillResult{taskID: task.id, err: dbterror.ErrReorgPanic}
+ }, false)
+ defer w.GetCtx().setDDLLabelForTopSQL(task.jobID, task.sqlQuery)
+
+ failpoint.Inject("mockBackfillRunErr", func() {
+ if w.GetCtx().id == 0 {
+ result := &backfillResult{taskID: task.id, addedCount: 0, nextKey: nil, err: errors.Errorf("mock backfill error")}
+ failpoint.Return(result)
+ }
+ })
+ failpoint.Inject("mockHighLoadForAddIndex", func() {
+ sqlPrefixes := []string{"alter"}
+ topsql.MockHighCPULoad(task.sqlQuery, sqlPrefixes, 5)
+ })
+ failpoint.Inject("mockBackfillSlow", func() {
+ time.Sleep(100 * time.Millisecond)
+ })
+
+ // Change the batch size dynamically.
+ w.GetCtx().batchCnt = int(variable.GetDDLReorgBatchSize())
+ result = w.handleBackfillTask(w.GetCtx().ddlCtx, task, w.backfiller)
+ task.bfJob.Meta.RowCount = int64(result.addedCount)
+ if result.err != nil {
+ logutil.BgLogger().Warn("[ddl] backfill worker runTask failed",
+ zap.Stringer("worker", w), zap.String("backfillJob", task.bfJob.AbbrStr()), zap.Error(result.err))
+ if dbterror.ErrDDLJobNotFound.Equal(result.err) {
+ result.err = nil
+ return result
+ }
+ task.bfJob.State = model.JobStateCancelled
+ task.bfJob.Meta.Error = toTError(result.err)
+ if err := w.finishJob(task.bfJob); err != nil {
+ logutil.BgLogger().Info("[ddl] backfill worker runTask, finishJob failed",
+ zap.Stringer("worker", w), zap.String("backfillJob", task.bfJob.AbbrStr()), zap.Error(err))
+ result.err = err
+ }
+ } else {
+ task.bfJob.State = model.JobStateDone
+ result.err = w.finishJob(task.bfJob)
+ }
+ return result
+}
+
func (w *backfillWorker) run(d *ddlCtx, bf backfiller, job *model.Job) {
- logutil.BgLogger().Info("[ddl] backfill worker start",
- zap.Stringer("type", w.tp),
- zap.Int("workerID", w.id))
+ logutil.BgLogger().Info("[ddl] backfill worker start", zap.Stringer("worker", w))
var curTaskID int
defer util.Recover(metrics.LabelDDL, "backfillWorker.run", func() {
w.resultCh <- &backfillResult{taskID: curTaskID, err: dbterror.ErrReorgPanic}
}, false)
for {
if util.HasCancelled(w.ctx) {
- logutil.BgLogger().Info("[ddl] backfill worker exit on context done",
- zap.Stringer("type", w.tp), zap.Int("workerID", w.id))
+ logutil.BgLogger().Info("[ddl] backfill worker exit on context done", zap.Stringer("worker", w))
return
}
task, more := <-w.taskCh
if !more {
- logutil.BgLogger().Info("[ddl] backfill worker exit",
- zap.Stringer("type", w.tp), zap.Int("workerID", w.id))
+ logutil.BgLogger().Info("[ddl] backfill worker exit", zap.Stringer("worker", w))
return
}
curTaskID = task.id
- d.setDDLLabelForTopSQL(job)
+ d.setDDLLabelForTopSQL(job.ID, job.Query)
- logutil.BgLogger().Debug("[ddl] backfill worker got task", zap.Int("workerID", w.id), zap.String("task", task.String()))
+ logutil.BgLogger().Debug("[ddl] backfill worker got task", zap.Int("workerID", w.GetCtx().id), zap.String("task", task.String()))
failpoint.Inject("mockBackfillRunErr", func() {
- if w.id == 0 {
+ if w.GetCtx().id == 0 {
result := &backfillResult{taskID: task.id, addedCount: 0, nextKey: nil, err: errors.Errorf("mock backfill error")}
w.resultCh <- result
failpoint.Continue()
@@ -339,20 +533,25 @@ func (w *backfillWorker) run(d *ddlCtx, bf backfiller, job *model.Job) {
})
// Change the batch size dynamically.
- w.batchCnt = int(variable.GetDDLReorgBatchSize())
+ w.GetCtx().batchCnt = int(variable.GetDDLReorgBatchSize())
result := w.handleBackfillTask(d, task, bf)
w.resultCh <- result
+ if result.err != nil {
+ logutil.BgLogger().Info("[ddl] backfill worker exit on error",
+ zap.Stringer("worker", w), zap.Error(result.err))
+ return
+ }
}
}
// splitTableRanges uses PD region's key ranges to split the backfilling table key range space,
// to speed up backfilling data in table with disperse handle.
// The `t` should be a non-partitioned table or a partition.
-func splitTableRanges(t table.PhysicalTable, store kv.Storage, startKey, endKey kv.Key) ([]kv.KeyRange, error) {
+func splitTableRanges(t table.PhysicalTable, store kv.Storage, startKey, endKey kv.Key, limit int) ([]kv.KeyRange, error) {
logutil.BgLogger().Info("[ddl] split table range from PD",
zap.Int64("physicalTableID", t.GetPhysicalID()),
- zap.String("startHandle", tryDecodeToHandleString(startKey)),
- zap.String("endHandle", tryDecodeToHandleString(endKey)))
+ zap.String("start key", hex.EncodeToString(startKey)),
+ zap.String("end key", hex.EncodeToString(endKey)))
kvRange := kv.KeyRange{StartKey: startKey, EndKey: endKey}
s, ok := store.(tikv.Storage)
if !ok {
@@ -363,7 +562,7 @@ func splitTableRanges(t table.PhysicalTable, store kv.Storage, startKey, endKey
maxSleep := 10000 // ms
bo := backoff.NewBackofferWithVars(context.Background(), maxSleep, nil)
rc := copr.NewRegionCache(s.GetRegionCache())
- ranges, err := rc.SplitRegionRanges(bo, []kv.KeyRange{kvRange})
+ ranges, err := rc.SplitRegionRanges(bo, []kv.KeyRange{kvRange}, limit)
if err != nil {
return nil, errors.Trace(err)
}
@@ -428,6 +627,9 @@ func (dc *ddlCtx) sendTasksAndWait(scheduler *backfillScheduler, totalAddedCount
batchTasks []*reorgBackfillTask) error {
reorgInfo := scheduler.reorgInfo
for _, task := range batchTasks {
+ if scheduler.copReqSenderPool != nil {
+ scheduler.copReqSenderPool.sendTask(task)
+ }
scheduler.taskCh <- task
}
@@ -436,77 +638,48 @@ func (dc *ddlCtx) sendTasksAndWait(scheduler *backfillScheduler, totalAddedCount
nextKey, taskAddedCount, err := waitTaskResults(scheduler, batchTasks, totalAddedCount)
elapsedTime := time.Since(startTime)
if err == nil {
- err = dc.isReorgRunnable(reorgInfo.Job)
+ err = dc.isReorgRunnable(reorgInfo.Job.ID, false)
}
+ // Update the reorg handle that has been processed.
+ err1 := reorgInfo.UpdateReorgMeta(nextKey, scheduler.sessPool)
+
if err != nil {
- // Update the reorg handle that has been processed.
- err1 := reorgInfo.UpdateReorgMeta(nextKey, scheduler.sessPool)
metrics.BatchAddIdxHistogram.WithLabelValues(metrics.LblError).Observe(elapsedTime.Seconds())
logutil.BgLogger().Warn("[ddl] backfill worker handle batch tasks failed",
- zap.ByteString("elementType", reorgInfo.currElement.TypeKey),
- zap.Int64("elementID", reorgInfo.currElement.ID),
- zap.Int64("totalAddedCount", *totalAddedCount),
- zap.String("startHandle", tryDecodeToHandleString(startKey)),
- zap.String("nextHandle", tryDecodeToHandleString(nextKey)),
- zap.Int64("batchAddedCount", taskAddedCount),
- zap.String("taskFailedError", err.Error()),
- zap.String("takeTime", elapsedTime.String()),
+
+ zap.Int64("total added count", *totalAddedCount),
+ zap.String("start key", hex.EncodeToString(startKey)),
+ zap.String("next key", hex.EncodeToString(nextKey)),
+ zap.Int64("batch added count", taskAddedCount),
+ zap.String("task failed error", err.Error()),
+ zap.String("take time", elapsedTime.String()),
zap.NamedError("updateHandleError", err1))
+ failpoint.Inject("MockGetIndexRecordErr", func() {
+ // Make sure this job didn't failed because by the "Write conflict" error.
+ if dbterror.ErrNotOwner.Equal(err) {
+ time.Sleep(50 * time.Millisecond)
+ }
+ })
return errors.Trace(err)
}
- // nextHandle will be updated periodically in runReorgJob, so no need to update it here.
- dc.getReorgCtx(reorgInfo.Job).setNextKey(nextKey)
+ dc.getReorgCtx(reorgInfo.Job.ID).setNextKey(nextKey)
metrics.BatchAddIdxHistogram.WithLabelValues(metrics.LblOK).Observe(elapsedTime.Seconds())
logutil.BgLogger().Info("[ddl] backfill workers successfully processed batch",
- zap.ByteString("elementType", reorgInfo.currElement.TypeKey),
- zap.Int64("elementID", reorgInfo.currElement.ID),
- zap.Int64("totalAddedCount", *totalAddedCount),
- zap.String("startHandle", tryDecodeToHandleString(startKey)),
- zap.String("nextHandle", tryDecodeToHandleString(nextKey)),
- zap.Int64("batchAddedCount", taskAddedCount),
- zap.String("takeTime", elapsedTime.String()))
+ zap.Stringer("element", reorgInfo.currElement),
+ zap.Int64("total added count", *totalAddedCount),
+ zap.String("start key", hex.EncodeToString(startKey)),
+ zap.String("next key", hex.EncodeToString(nextKey)),
+ zap.Int64("batch added count", taskAddedCount),
+ zap.String("take time", elapsedTime.String()),
+ zap.NamedError("updateHandleError", err1))
return nil
}
-func tryDecodeToHandleString(key kv.Key) string {
- defer func() {
- if r := recover(); r != nil {
- logutil.BgLogger().Warn("tryDecodeToHandleString panic",
- zap.Any("recover()", r),
- zap.Binary("key", key))
- }
- }()
- handle, err := tablecodec.DecodeRowKey(key)
- if err != nil {
- recordPrefixIdx := bytes.Index(key, []byte("_r"))
- if recordPrefixIdx == -1 {
- return fmt.Sprintf("key: %x", key)
- }
- handleBytes := key[recordPrefixIdx+2:]
- terminatedWithZero := len(handleBytes) > 0 && handleBytes[len(handleBytes)-1] == 0
- if terminatedWithZero {
- handle, err := tablecodec.DecodeRowKey(key[:len(key)-1])
- if err == nil {
- return handle.String() + ".next"
- }
- }
- return fmt.Sprintf("%x", handleBytes)
- }
- return handle.String()
-}
-
-// handleRangeTasks sends tasks to workers, and returns remaining kvRanges that is not handled.
-func (dc *ddlCtx) handleRangeTasks(scheduler *backfillScheduler, t table.Table,
- totalAddedCount *int64, kvRanges []kv.KeyRange) ([]kv.KeyRange, error) {
- batchTasks := make([]*reorgBackfillTask, 0, backfillTaskChanSize)
- reorgInfo := scheduler.reorgInfo
- physicalTableID := reorgInfo.PhysicalTableID
+func getBatchTasks(t table.Table, reorgInfo *reorgInfo, kvRanges []kv.KeyRange, batch int) []*reorgBackfillTask {
+ batchTasks := make([]*reorgBackfillTask, 0, batch)
var prefix kv.Key
- if tbl, ok := t.(table.PartitionedTable); ok {
- t = tbl.GetPartition(physicalTableID)
- }
if reorgInfo.mergingTmpIdx {
prefix = t.IndexPrefix()
} else {
@@ -514,31 +687,49 @@ func (dc *ddlCtx) handleRangeTasks(scheduler *backfillScheduler, t table.Table,
}
// Build reorg tasks.
job := reorgInfo.Job
+ //nolint:forcetypeassert
+ phyTbl := t.(table.PhysicalTable)
+ jobCtx := reorgInfo.d.jobContext(job.ID)
for i, keyRange := range kvRanges {
+ startKey := keyRange.StartKey
endKey := keyRange.EndKey
- endK, err := getRangeEndKey(scheduler.jobCtx, dc.store, job.Priority, prefix, keyRange.StartKey, endKey)
+ endK, err := getRangeEndKey(jobCtx, reorgInfo.d.store, job.Priority, prefix, keyRange.StartKey, endKey)
if err != nil {
- logutil.BgLogger().Info("[ddl] send range task to workers, get reverse key failed", zap.Error(err))
+ logutil.BgLogger().Info("[ddl] get backfill range task, get reverse key failed", zap.Error(err))
} else {
- logutil.BgLogger().Info("[ddl] send range task to workers, change end key",
+ logutil.BgLogger().Info("[ddl] get backfill range task, change end key", zap.Int64("pTbl", phyTbl.GetPhysicalID()),
zap.String("end key", hex.EncodeToString(endKey)), zap.String("current end key", hex.EncodeToString(endK)))
endKey = endK
}
+ if len(startKey) == 0 {
+ startKey = prefix
+ }
+ if len(endKey) == 0 {
+ endKey = prefix.PrefixNext()
+ }
task := &reorgBackfillTask{
- id: i,
- physicalTableID: physicalTableID,
- startKey: keyRange.StartKey,
- endKey: endKey,
+ id: i,
+ jobID: reorgInfo.Job.ID,
+ physicalTable: phyTbl,
+ priority: reorgInfo.Priority,
+ startKey: startKey,
+ endKey: endKey,
// If the boundaries overlap, we should ignore the preceding endKey.
endInclude: endK.Cmp(keyRange.EndKey) != 0 || i == len(kvRanges)-1}
batchTasks = append(batchTasks, task)
- if len(batchTasks) >= backfillTaskChanSize {
+ if len(batchTasks) >= batch {
break
}
}
+ return batchTasks
+}
+// handleRangeTasks sends tasks to workers, and returns remaining kvRanges that is not handled.
+func (dc *ddlCtx) handleRangeTasks(scheduler *backfillScheduler, t table.PhysicalTable,
+ totalAddedCount *int64, kvRanges []kv.KeyRange) ([]kv.KeyRange, error) {
+ batchTasks := getBatchTasks(t, scheduler.reorgInfo, kvRanges, backfillTaskChanSize)
if len(batchTasks) == 0 {
return nil, nil
}
@@ -577,8 +768,7 @@ func loadDDLReorgVars(ctx context.Context, sessPool *sessionPool) error {
return ddlutil.LoadDDLReorgVars(ctx, sCtx)
}
-func makeupDecodeColMap(sessCtx sessionctx.Context, t table.Table) (map[int64]decoder.Column, error) {
- dbName := model.NewCIStr(sessCtx.GetSessionVars().CurrentDB)
+func makeupDecodeColMap(sessCtx sessionctx.Context, dbName model.CIStr, t table.Table) (map[int64]decoder.Column, error) {
writableColInfos := make([]*model.ColumnInfo, 0, len(t.WritableCols()))
for _, col := range t.WritableCols() {
writableColInfos = append(writableColInfos, col.ColumnInfo)
@@ -594,11 +784,16 @@ func makeupDecodeColMap(sessCtx sessionctx.Context, t table.Table) (map[int64]de
return decodeColMap, nil
}
-func setSessCtxLocation(sctx sessionctx.Context, info *reorgInfo) error {
+func setSessCtxLocation(sctx sessionctx.Context, tzLocation *model.TimeZoneLocation) error {
// It is set to SystemLocation to be compatible with nil LocationInfo.
- *sctx.GetSessionVars().TimeZone = *timeutil.SystemLocation()
- if info.ReorgMeta.Location != nil {
- loc, err := info.ReorgMeta.Location.GetLocation()
+ tz := *timeutil.SystemLocation()
+ if sctx.GetSessionVars().TimeZone == nil {
+ sctx.GetSessionVars().TimeZone = &tz
+ } else {
+ *sctx.GetSessionVars().TimeZone = tz
+ }
+ if tzLocation != nil {
+ loc, err := tzLocation.GetLocation()
if err != nil {
return errors.Trace(err)
}
@@ -611,7 +806,7 @@ type backfillScheduler struct {
ctx context.Context
reorgInfo *reorgInfo
sessPool *sessionPool
- tp backfillWorkerType
+ tp backfillerType
tbl table.PhysicalTable
decodeColMap map[int64]decoder.Column
jobCtx *JobContext
@@ -621,12 +816,14 @@ type backfillScheduler struct {
taskCh chan *reorgBackfillTask
resultCh chan *backfillResult
+
+ copReqSenderPool *copReqSenderPool // for add index in ingest way.
}
const backfillTaskChanSize = 1024
func newBackfillScheduler(ctx context.Context, info *reorgInfo, sessPool *sessionPool,
- tp backfillWorkerType, tbl table.PhysicalTable, decColMap map[int64]decoder.Column,
+ tp backfillerType, tbl table.PhysicalTable, decColMap map[int64]decoder.Column,
jobCtx *JobContext) *backfillScheduler {
return &backfillScheduler{
ctx: ctx,
@@ -645,15 +842,26 @@ func newBackfillScheduler(ctx context.Context, info *reorgInfo, sessPool *sessio
func (b *backfillScheduler) newSessCtx() (sessionctx.Context, error) {
reorgInfo := b.reorgInfo
sessCtx := newContext(reorgInfo.d.store)
+ if err := initSessCtx(sessCtx, reorgInfo.ReorgMeta.SQLMode, reorgInfo.ReorgMeta.Location); err != nil {
+ return nil, errors.Trace(err)
+ }
+ return sessCtx, nil
+}
+
+func initSessCtx(sessCtx sessionctx.Context, sqlMode mysql.SQLMode, tzLocation *model.TimeZoneLocation) error {
+ // Unify the TimeZone settings in newContext.
+ if sessCtx.GetSessionVars().StmtCtx.TimeZone == nil {
+ tz := *time.UTC
+ sessCtx.GetSessionVars().StmtCtx.TimeZone = &tz
+ }
sessCtx.GetSessionVars().StmtCtx.IsDDLJobInQueue = true
// Set the row encode format version.
rowFormat := variable.GetDDLReorgRowFormat()
sessCtx.GetSessionVars().RowEncoder.Enable = rowFormat != variable.DefTiDBRowFormatV1
// Simulate the sql mode environment in the worker sessionCtx.
- sqlMode := reorgInfo.ReorgMeta.SQLMode
sessCtx.GetSessionVars().SQLMode = sqlMode
- if err := setSessCtxLocation(sessCtx, reorgInfo); err != nil {
- return nil, errors.Trace(err)
+ if err := setSessCtxLocation(sessCtx, tzLocation); err != nil {
+ return errors.Trace(err)
}
sessCtx.GetSessionVars().StmtCtx.BadNullAsWarning = !sqlMode.HasStrictMode()
sessCtx.GetSessionVars().StmtCtx.TruncateAsWarning = !sqlMode.HasStrictMode()
@@ -662,7 +870,10 @@ func (b *backfillScheduler) newSessCtx() (sessionctx.Context, error) {
sessCtx.GetSessionVars().StmtCtx.DividedByZeroAsWarning = !sqlMode.HasStrictMode()
sessCtx.GetSessionVars().StmtCtx.IgnoreZeroInDate = !sqlMode.HasStrictMode() || sqlMode.HasAllowInvalidDatesMode()
sessCtx.GetSessionVars().StmtCtx.NoZeroDate = sqlMode.HasStrictMode()
- return sessCtx, nil
+ // Prevent initializing the mock context in the workers concurrently.
+ // For details, see https://github.com/pingcap/tidb/issues/40879.
+ _ = sessCtx.GetDomainInfoSchema()
+ return nil
}
func (b *backfillScheduler) setMaxWorkerSize(maxSize int) {
@@ -674,6 +885,7 @@ func (b *backfillScheduler) workerSize() int {
}
func (b *backfillScheduler) adjustWorkerSize() error {
+ b.initCopReqSenderPool()
reorgInfo := b.reorgInfo
job := reorgInfo.Job
jc := b.jobCtx
@@ -681,7 +893,11 @@ func (b *backfillScheduler) adjustWorkerSize() error {
logutil.BgLogger().Error("[ddl] load DDL reorganization variable failed", zap.Error(err))
}
workerCnt := int(variable.GetDDLReorgWorkerCounter())
- workerCnt = mathutil.Min(workerCnt, b.maxSize)
+ if b.copReqSenderPool != nil {
+ workerCnt = mathutil.Min(workerCnt/2+1, b.maxSize)
+ } else {
+ workerCnt = mathutil.Min(workerCnt, b.maxSize)
+ }
// Increase the worker.
for i := len(b.workers); i < workerCnt; i++ {
sessCtx, err := b.newSessCtx()
@@ -694,22 +910,40 @@ func (b *backfillScheduler) adjustWorkerSize() error {
)
switch b.tp {
case typeAddIndexWorker:
- idxWorker, err := newAddIndexWorker(sessCtx, i, b.tbl, b.decodeColMap, reorgInfo, jc, job)
+ backfillCtx := newBackfillCtx(reorgInfo.d, i, sessCtx, reorgInfo.ReorgMeta.ReorgTp, job.SchemaName, b.tbl)
+ idxWorker, err := newAddIndexWorker(b.decodeColMap, b.tbl, backfillCtx,
+ jc, job.ID, reorgInfo.currElement.ID, reorgInfo.currElement.TypeKey)
if err != nil {
- return errors.Trace(err)
+ if canSkipError(b.reorgInfo.ID, len(b.workers), err) {
+ continue
+ }
+ return err
}
- worker, runner = idxWorker, idxWorker.backfillWorker
+ idxWorker.copReqSenderPool = b.copReqSenderPool
+ runner = newBackfillWorker(jc.ddlJobCtx, idxWorker)
+ worker = idxWorker
case typeAddIndexMergeTmpWorker:
- tmpIdxWorker := newMergeTempIndexWorker(sessCtx, i, b.tbl, reorgInfo, jc)
- worker, runner = tmpIdxWorker, tmpIdxWorker.backfillWorker
+ backfillCtx := newBackfillCtx(reorgInfo.d, i, sessCtx, reorgInfo.ReorgMeta.ReorgTp, job.SchemaName, b.tbl)
+ tmpIdxWorker := newMergeTempIndexWorker(backfillCtx, i, b.tbl, reorgInfo.currElement.ID, jc)
+ runner = newBackfillWorker(jc.ddlJobCtx, tmpIdxWorker)
+ worker = tmpIdxWorker
case typeUpdateColumnWorker:
// Setting InCreateOrAlterStmt tells the difference between SELECT casting and ALTER COLUMN casting.
sessCtx.GetSessionVars().StmtCtx.InCreateOrAlterStmt = true
updateWorker := newUpdateColumnWorker(sessCtx, i, b.tbl, b.decodeColMap, reorgInfo, jc)
- worker, runner = updateWorker, updateWorker.backfillWorker
+ runner = newBackfillWorker(jc.ddlJobCtx, updateWorker)
+ worker = updateWorker
case typeCleanUpIndexWorker:
idxWorker := newCleanUpIndexWorker(sessCtx, i, b.tbl, b.decodeColMap, reorgInfo, jc)
- worker, runner = idxWorker, idxWorker.backfillWorker
+ runner = newBackfillWorker(jc.ddlJobCtx, idxWorker)
+ worker = idxWorker
+ case typeReorgPartitionWorker:
+ partWorker, err := newReorgPartitionWorker(sessCtx, i, b.tbl, b.decodeColMap, reorgInfo, jc)
+ if err != nil {
+ return err
+ }
+ runner = newBackfillWorker(jc.ddlJobCtx, partWorker)
+ worker = partWorker
default:
return errors.New("unknown backfill type")
}
@@ -724,10 +958,51 @@ func (b *backfillScheduler) adjustWorkerSize() error {
b.workers = b.workers[:workerCnt]
closeBackfillWorkers(workers)
}
- return injectCheckBackfillWorkerNum(len(b.workers))
+ if b.copReqSenderPool != nil {
+ b.copReqSenderPool.adjustSize(len(b.workers))
+ }
+ return injectCheckBackfillWorkerNum(len(b.workers), b.tp == typeAddIndexMergeTmpWorker)
+}
+
+func (b *backfillScheduler) initCopReqSenderPool() {
+ if b.tp != typeAddIndexWorker || b.reorgInfo.Job.ReorgMeta.ReorgTp != model.ReorgTypeLitMerge ||
+ b.copReqSenderPool != nil || len(b.workers) > 0 {
+ return
+ }
+ indexInfo := model.FindIndexInfoByID(b.tbl.Meta().Indices, b.reorgInfo.currElement.ID)
+ if indexInfo == nil {
+ logutil.BgLogger().Warn("[ddl-ingest] cannot init cop request sender",
+ zap.Int64("table ID", b.tbl.Meta().ID), zap.Int64("index ID", b.reorgInfo.currElement.ID))
+ return
+ }
+ sessCtx, err := b.newSessCtx()
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl-ingest] cannot init cop request sender", zap.Error(err))
+ return
+ }
+ copCtx, err := newCopContext(b.tbl.Meta(), indexInfo, sessCtx)
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl-ingest] cannot init cop request sender", zap.Error(err))
+ return
+ }
+ b.copReqSenderPool = newCopReqSenderPool(b.ctx, copCtx, sessCtx.GetStore())
+}
+
+func canSkipError(jobID int64, workerCnt int, err error) bool {
+ if workerCnt > 0 {
+ // The error can be skipped because the rest workers can handle the tasks.
+ return true
+ }
+ logutil.BgLogger().Warn("[ddl] create add index backfill worker failed",
+ zap.Int("current worker count", workerCnt),
+ zap.Int64("job ID", jobID), zap.Error(err))
+ return false
}
func (b *backfillScheduler) Close() {
+ if b.copReqSenderPool != nil {
+ b.copReqSenderPool.close()
+ }
closeBackfillWorkers(b.workers)
close(b.taskCh)
close(b.resultCh)
@@ -748,18 +1023,18 @@ func (b *backfillScheduler) Close() {
//
// The above operations are completed in a transaction.
// Finally, update the concurrent processing of the total number of rows, and store the completed handle value.
-func (dc *ddlCtx) writePhysicalTableRecord(sessPool *sessionPool, t table.PhysicalTable, bfWorkerType backfillWorkerType, reorgInfo *reorgInfo) error {
+func (dc *ddlCtx) writePhysicalTableRecord(sessPool *sessionPool, t table.PhysicalTable, bfWorkerType backfillerType, reorgInfo *reorgInfo) error {
job := reorgInfo.Job
totalAddedCount := job.GetRowCount()
startKey, endKey := reorgInfo.StartKey, reorgInfo.EndKey
sessCtx := newContext(reorgInfo.d.store)
- decodeColMap, err := makeupDecodeColMap(sessCtx, t)
+ decodeColMap, err := makeupDecodeColMap(sessCtx, reorgInfo.dbInfo.Name, t)
if err != nil {
return errors.Trace(err)
}
- if err := dc.isReorgRunnable(reorgInfo.Job); err != nil {
+ if err := dc.isReorgRunnable(reorgInfo.Job.ID, false); err != nil {
return errors.Trace(err)
}
if startKey == nil && endKey == nil {
@@ -773,12 +1048,21 @@ func (dc *ddlCtx) writePhysicalTableRecord(sessPool *sessionPool, t table.Physic
}
})
- jc := dc.jobContext(job)
+ jc := dc.jobContext(job.ID)
scheduler := newBackfillScheduler(dc.ctx, reorgInfo, sessPool, bfWorkerType, t, decodeColMap, jc)
defer scheduler.Close()
+ var ingestBeCtx *ingest.BackendContext
+ if bfWorkerType == typeAddIndexWorker && job.ReorgMeta.ReorgTp == model.ReorgTypeLitMerge {
+ if bc, ok := ingest.LitBackCtxMgr.Load(job.ID); ok {
+ ingestBeCtx = bc
+ } else {
+ return errors.New(ingest.LitErrGetBackendFail)
+ }
+ }
+
for {
- kvRanges, err := splitTableRanges(t, reorgInfo.d.store, startKey, endKey)
+ kvRanges, err := splitTableRanges(t, reorgInfo.d.store, startKey, endKey, backfillTaskChanSize)
if err != nil {
return errors.Trace(err)
}
@@ -795,14 +1079,11 @@ func (dc *ddlCtx) writePhysicalTableRecord(sessPool *sessionPool, t table.Physic
zap.Int("regionCnt", len(kvRanges)),
zap.String("startKey", hex.EncodeToString(startKey)),
zap.String("endKey", hex.EncodeToString(endKey)))
- if bfWorkerType == typeAddIndexWorker && job.ReorgMeta.ReorgTp == model.ReorgTypeLitMerge {
- if bc, ok := ingest.LitBackCtxMgr.Load(job.ID); ok {
- err := bc.Flush(reorgInfo.currElement.ID)
- if err != nil {
- return errors.Trace(err)
- }
- } else {
- return errors.New(ingest.LitErrGetBackendFail)
+
+ if ingestBeCtx != nil {
+ err := ingestBeCtx.Flush(reorgInfo.currElement.ID)
+ if err != nil {
+ return errors.Trace(err)
}
}
remains, err := dc.handleRangeTasks(scheduler, t, &totalAddedCount, kvRanges)
@@ -811,6 +1092,9 @@ func (dc *ddlCtx) writePhysicalTableRecord(sessPool *sessionPool, t table.Physic
}
if len(remains) == 0 {
+ if ingestBeCtx != nil {
+ ingestBeCtx.EngMgr.ResetWorkers(ingestBeCtx, job.ID, reorgInfo.currElement.ID)
+ }
break
}
startKey = remains[0].StartKey
@@ -818,7 +1102,10 @@ func (dc *ddlCtx) writePhysicalTableRecord(sessPool *sessionPool, t table.Physic
return nil
}
-func injectCheckBackfillWorkerNum(curWorkerSize int) error {
+func injectCheckBackfillWorkerNum(curWorkerSize int, isMergeWorker bool) error {
+ if isMergeWorker {
+ return nil
+ }
failpoint.Inject("checkBackfillWorkerNum", func(val failpoint.Value) {
//nolint:forcetypeassert
if val.(bool) {
@@ -837,6 +1124,100 @@ func injectCheckBackfillWorkerNum(curWorkerSize int) error {
return nil
}
+func addBatchBackfillJobs(sess *session, reorgInfo *reorgInfo, sJobCtx *splitJobContext, phyTblID int64, notDistTask bool,
+ batchTasks []*reorgBackfillTask, bJobs []*BackfillJob) error {
+ bJobs = bJobs[:0]
+ instanceID := ""
+ if notDistTask {
+ instanceID = reorgInfo.d.uuid
+ }
+
+ // TODO: Adjust the number of ranges(region) for each task.
+ for _, task := range batchTasks {
+ bm := &model.BackfillMeta{
+ IsUnique: sJobCtx.isUnique,
+ EndInclude: task.endInclude,
+ ReorgTp: reorgInfo.Job.ReorgMeta.ReorgTp,
+ SQLMode: reorgInfo.ReorgMeta.SQLMode,
+ Location: reorgInfo.ReorgMeta.Location,
+ JobMeta: &model.JobMeta{
+ SchemaID: reorgInfo.Job.SchemaID,
+ TableID: reorgInfo.Job.TableID,
+ Type: reorgInfo.Job.Type,
+ Query: reorgInfo.Job.Query,
+ },
+ StartKey: task.startKey,
+ EndKey: task.endKey,
+ }
+ bj := &BackfillJob{
+ ID: sJobCtx.currBackfillJobID.Add(1),
+ JobID: reorgInfo.Job.ID,
+ EleID: reorgInfo.currElement.ID,
+ EleKey: reorgInfo.currElement.TypeKey,
+ PhysicalTableID: phyTblID,
+ Tp: sJobCtx.bfWorkerType,
+ State: model.JobStateNone,
+ InstanceID: instanceID,
+ Meta: bm,
+ }
+ bj.Meta.CurrKey = task.startKey
+ bJobs = append(bJobs, bj)
+ }
+ if err := AddBackfillJobs(sess, bJobs); err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+func (dc *ddlCtx) splitTableToBackfillJobs(sess *session, reorgInfo *reorgInfo, sJobCtx *splitJobContext, pTblMeta *BackfillJobRangeMeta) error {
+ isFirstOps := !sJobCtx.isMultiPhyTbl
+ batchSize := sJobCtx.batchSize
+ startKey, endKey := pTblMeta.StartKey, pTblMeta.EndKey
+ bJobs := make([]*BackfillJob, 0, batchSize)
+ for {
+ kvRanges, err := splitTableRanges(pTblMeta.PhyTbl, reorgInfo.d.store, startKey, endKey, batchSize)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ batchTasks := getBatchTasks(pTblMeta.PhyTbl, reorgInfo, kvRanges, batchSize)
+ if len(batchTasks) == 0 {
+ break
+ }
+ notNeedDistProcess := isFirstOps && (len(kvRanges) < minDistTaskCnt)
+ if err = addBatchBackfillJobs(sess, reorgInfo, sJobCtx, pTblMeta.PhyTblID, notNeedDistProcess, batchTasks, bJobs); err != nil {
+ return errors.Trace(err)
+ }
+ isFirstOps = false
+
+ remains := kvRanges[len(batchTasks):]
+ dc.asyncNotifyWorker(dc.backfillJobCh, addingBackfillJob, reorgInfo.Job.ID, "backfill_job")
+ logutil.BgLogger().Info("[ddl] split backfill jobs to the backfill table",
+ zap.Int64("physicalID", pTblMeta.PhyTblID),
+ zap.Int("batchTasksCnt", len(batchTasks)),
+ zap.Int("totalRegionCnt", len(kvRanges)),
+ zap.Int("remainRegionCnt", len(remains)),
+ zap.String("startHandle", hex.EncodeToString(startKey)),
+ zap.String("endHandle", hex.EncodeToString(endKey)))
+
+ if len(remains) == 0 {
+ break
+ }
+
+ for {
+ bJobCnt, err := checkBackfillJobCount(sess, reorgInfo.Job.ID, reorgInfo.currElement.ID, reorgInfo.currElement.TypeKey, pTblMeta.PhyTblID)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if bJobCnt < sJobCtx.minBatchSize {
+ break
+ }
+ time.Sleep(RetrySQLInterval)
+ }
+ startKey = remains[0].StartKey
+ }
+ return nil
+}
+
// recordIterFunc is used for low-level record iteration.
type recordIterFunc func(h kv.Handle, rowKey kv.Key, rawRecord []byte) (more bool, err error)
diff --git a/ddl/cancel_test.go b/ddl/cancel_test.go
index cd446f3441baf..3f02029ffced7 100644
--- a/ddl/cancel_test.go
+++ b/ddl/cancel_test.go
@@ -22,6 +22,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/testkit"
@@ -149,8 +150,8 @@ var allTestCase = []testCancelJob{
{"alter table t modify column c11 char(10)", true, model.StateWriteReorganization, true, true, nil},
{"alter table t modify column c11 char(10)", false, model.StatePublic, false, true, nil},
// Add foreign key.
- {"alter table t add constraint fk foreign key a(c1) references t_ref(c1)", true, model.StateNone, true, false, []string{"create table t_ref (c1 int, c2 int, c3 int, c11 tinyint);"}},
- {"alter table t add constraint fk foreign key a(c1) references t_ref(c1)", false, model.StatePublic, false, true, nil},
+ {"alter table t add constraint fk foreign key a(c1) references t_ref(c1)", true, model.StateNone, true, false, []string{"create table t_ref (c1 int key, c2 int, c3 int, c11 tinyint);"}},
+ {"alter table t add constraint fk foreign key a(c1) references t_ref(c1)", false, model.StatePublic, false, true, []string{"insert into t_ref (c1) select c1 from t;"}},
// Drop foreign key.
{"alter table t drop foreign key fk", true, model.StatePublic, true, false, nil},
{"alter table t drop foreign key fk", false, model.StateNone, false, true, nil},
@@ -264,7 +265,7 @@ func TestCancel(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockBackfillSlow"))
}()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
i := atomicutil.NewInt64(0)
cancel := atomicutil.NewBool(false)
cancelResult := atomicutil.NewBool(false)
@@ -282,12 +283,12 @@ func TestCancel(t *testing.T) {
}
dom.DDL().SetHook(hook.Clone())
- restHook := func(h *ddl.TestDDLCallback) {
+ restHook := func(h *callback.TestDDLCallback) {
h.OnJobRunBeforeExported = nil
h.OnJobUpdatedExported.Store(nil)
dom.DDL().SetHook(h.Clone())
}
- registHook := func(h *ddl.TestDDLCallback, onJobRunBefore bool) {
+ registHook := func(h *callback.TestDDLCallback, onJobRunBefore bool) {
if onJobRunBefore {
h.OnJobRunBeforeExported = hookFunc
} else {
diff --git a/ddl/cluster.go b/ddl/cluster.go
index ebb833156cec2..ea0567dfb9c73 100644
--- a/ddl/cluster.go
+++ b/ddl/cluster.go
@@ -24,10 +24,10 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
+ "github.com/pingcap/kvproto/pkg/errorpb"
"github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/tidb/ddl/util"
"github.com/pingcap/tidb/domain/infosync"
- "github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
@@ -50,11 +50,7 @@ import (
)
var pdScheduleKey = []string{
- "hot-region-schedule-limit",
- "leader-schedule-limit",
"merge-schedule-limit",
- "region-schedule-limit",
- "replica-schedule-limit",
}
const (
@@ -68,7 +64,10 @@ const (
autoAnalyzeOffset
readOnlyOffset
totalLockedRegionsOffset
+ startTSOffset
commitTSOffset
+ ttlJobEnableOffSet
+ keyRangesOffset
)
func closePDSchedule() error {
@@ -99,6 +98,16 @@ func recoverPDSchedule(pdScheduleParam map[string]interface{}) error {
return infosync.SetPDScheduleConfig(context.Background(), pdScheduleParam)
}
+func getStoreGlobalMinSafeTS(s kv.Storage) time.Time {
+ minSafeTS := s.GetMinSafeTS(kv.GlobalTxnScope)
+ // Inject mocked SafeTS for test.
+ failpoint.Inject("injectSafeTS", func(val failpoint.Value) {
+ injectTS := val.(int)
+ minSafeTS = uint64(injectTS)
+ })
+ return oracle.GetTimeFromTS(minSafeTS)
+}
+
// ValidateFlashbackTS validates that flashBackTS in range [gcSafePoint, currentTS).
func ValidateFlashbackTS(ctx context.Context, sctx sessionctx.Context, flashBackTS uint64) error {
currentTS, err := sctx.GetStore().GetOracle().GetStaleTimestamp(ctx, oracle.GlobalTxnScope, 0)
@@ -111,12 +120,34 @@ func ValidateFlashbackTS(ctx context.Context, sctx sessionctx.Context, flashBack
}
currentTS = currentVer.Ver
}
- if oracle.GetTimeFromTS(flashBackTS).After(oracle.GetTimeFromTS(currentTS)) {
+ oracleFlashbackTS := oracle.GetTimeFromTS(flashBackTS)
+ if oracleFlashbackTS.After(oracle.GetTimeFromTS(currentTS)) {
return errors.Errorf("cannot set flashback timestamp to future time")
}
- if oracle.GetTimeFromTS(flashBackTS).After(expression.GetMinSafeTime(sctx)) {
- return errors.Errorf("cannot set flashback timestamp to too close to present time")
+
+ flashbackGetMinSafeTimeTimeout := time.Minute
+ failpoint.Inject("changeFlashbackGetMinSafeTimeTimeout", func(val failpoint.Value) {
+ t := val.(int)
+ flashbackGetMinSafeTimeTimeout = time.Duration(t)
+ })
+
+ start := time.Now()
+ minSafeTime := getStoreGlobalMinSafeTS(sctx.GetStore())
+ ticker := time.NewTicker(time.Second)
+ defer ticker.Stop()
+ for oracleFlashbackTS.After(minSafeTime) {
+ if time.Since(start) >= flashbackGetMinSafeTimeTimeout {
+ return errors.Errorf("cannot set flashback timestamp after min-resolved-ts(%s)", minSafeTime)
+ }
+ select {
+ case <-ticker.C:
+ minSafeTime = getStoreGlobalMinSafeTS(sctx.GetStore())
+ break
+ case <-ctx.Done():
+ return ctx.Err()
+ }
}
+
gcSafePoint, err := gcutil.GetGCSafePoint(sctx)
if err != nil {
return err
@@ -125,6 +156,18 @@ func ValidateFlashbackTS(ctx context.Context, sctx sessionctx.Context, flashBack
return gcutil.ValidateSnapshotWithGCSafePoint(flashBackTS, gcSafePoint)
}
+func getTiDBTTLJobEnable(sess sessionctx.Context) (string, error) {
+ val, err := sess.GetSessionVars().GlobalVarsAccessor.GetGlobalSysVar(variable.TiDBTTLJobEnable)
+ if err != nil {
+ return "", errors.Trace(err)
+ }
+ return val, nil
+}
+
+func setTiDBTTLJobEnable(ctx context.Context, sess sessionctx.Context, value string) error {
+ return sess.GetSessionVars().GlobalVarsAccessor.SetGlobalSysVar(ctx, variable.TiDBTTLJobEnable, value)
+}
+
func setTiDBEnableAutoAnalyze(ctx context.Context, sess sessionctx.Context, value string) error {
return sess.GetSessionVars().GlobalVarsAccessor.SetGlobalSysVar(ctx, variable.TiDBEnableAutoAnalyze, value)
}
@@ -149,6 +192,32 @@ func getTiDBSuperReadOnly(sess sessionctx.Context) (string, error) {
return val, nil
}
+func isFlashbackSupportedDDLAction(action model.ActionType) bool {
+ switch action {
+ case model.ActionSetTiFlashReplica, model.ActionUpdateTiFlashReplicaStatus, model.ActionAlterPlacementPolicy,
+ model.ActionAlterTablePlacement, model.ActionAlterTablePartitionPlacement, model.ActionCreatePlacementPolicy,
+ model.ActionDropPlacementPolicy, model.ActionModifySchemaDefaultPlacement,
+ model.ActionAlterTableAttributes, model.ActionAlterTablePartitionAttributes:
+ return false
+ default:
+ return true
+ }
+}
+
+func checkSystemSchemaID(t *meta.Meta, schemaID int64, flashbackTSString string) error {
+ if schemaID <= 0 {
+ return nil
+ }
+ DBInfo, err := t.GetDatabase(schemaID)
+ if err != nil || DBInfo == nil {
+ return errors.Trace(err)
+ }
+ if filter.IsSystemSchema(DBInfo.Name.L) {
+ return errors.Errorf("Detected modified system table during [%s, now), can't do flashback", flashbackTSString)
+ }
+ return nil
+}
+
func checkAndSetFlashbackClusterInfo(sess sessionctx.Context, d *ddlCtx, t *meta.Meta, job *model.Job, flashbackTS uint64) (err error) {
if err = ValidateFlashbackTS(d.ctx, sess, flashbackTS); err != nil {
return err
@@ -166,25 +235,63 @@ func checkAndSetFlashbackClusterInfo(sess sessionctx.Context, d *ddlCtx, t *meta
if err = setTiDBSuperReadOnly(d.ctx, sess, variable.On); err != nil {
return err
}
+ if err = setTiDBTTLJobEnable(d.ctx, sess, variable.Off); err != nil {
+ return err
+ }
nowSchemaVersion, err := t.GetSchemaVersion()
if err != nil {
return errors.Trace(err)
}
- flashbackSchemaVersion, err := meta.NewSnapshotMeta(d.store.GetSnapshot(kv.NewVersion(flashbackTS))).GetSchemaVersion()
+ flashbackSnapshotMeta := meta.NewSnapshotMeta(d.store.GetSnapshot(kv.NewVersion(flashbackTS)))
+ flashbackSchemaVersion, err := flashbackSnapshotMeta.GetSchemaVersion()
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ flashbackTSString := oracle.GetTimeFromTS(flashbackTS).String()
+
+ // Check if there is an upgrade during [flashbackTS, now)
+ sql := fmt.Sprintf("select VARIABLE_VALUE from mysql.tidb as of timestamp '%s' where VARIABLE_NAME='tidb_server_version'", flashbackTSString)
+ rows, err := newSession(sess).execute(d.ctx, sql, "check_tidb_server_version")
+ if err != nil || len(rows) == 0 {
+ return errors.Errorf("Get history `tidb_server_version` failed, can't do flashback")
+ }
+ sql = fmt.Sprintf("select 1 from mysql.tidb where VARIABLE_NAME='tidb_server_version' and VARIABLE_VALUE=%s", rows[0].GetString(0))
+ rows, err = newSession(sess).execute(d.ctx, sql, "check_tidb_server_version")
if err != nil {
return errors.Trace(err)
}
+ if len(rows) == 0 {
+ return errors.Errorf("Detected TiDB upgrade during [%s, now), can't do flashback", flashbackTSString)
+ }
+
+ // Check is there a DDL task at flashbackTS.
+ sql = fmt.Sprintf("select count(*) from mysql.%s as of timestamp '%s'", JobTable, flashbackTSString)
+ rows, err = newSession(sess).execute(d.ctx, sql, "check_history_job")
+ if err != nil || len(rows) == 0 {
+ return errors.Errorf("Get history ddl jobs failed, can't do flashback")
+ }
+ if rows[0].GetInt64(0) != 0 {
+ return errors.Errorf("Detected another DDL job at %s, can't do flashback", flashbackTSString)
+ }
- // If flashbackSchemaVersion not same as nowSchemaVersion, we've done ddl during [flashbackTs, now).
+ // If flashbackSchemaVersion not same as nowSchemaVersion, we should check all schema diffs during [flashbackTs, now).
for i := flashbackSchemaVersion + 1; i <= nowSchemaVersion; i++ {
diff, err := t.GetSchemaDiff(i)
if err != nil {
return errors.Trace(err)
}
- if diff != nil && diff.Type != model.ActionFlashbackCluster {
- return errors.Errorf("Detected schema change due to another DDL job during [%s, now), can't do flashback", oracle.GetTimeFromTS(flashbackTS))
+ if diff == nil {
+ continue
+ }
+ if !isFlashbackSupportedDDLAction(diff.Type) {
+ return errors.Errorf("Detected unsupported DDL job type(%s) during [%s, now), can't do flashback", diff.Type.String(), flashbackTSString)
+ }
+ err = checkSystemSchemaID(flashbackSnapshotMeta, diff.SchemaID, flashbackTSString)
+ if err != nil {
+ return errors.Trace(err)
}
}
@@ -206,73 +313,100 @@ func checkAndSetFlashbackClusterInfo(sess sessionctx.Context, d *ddlCtx, t *meta
return nil
}
-type flashbackID struct {
- id int64
- excluded bool
+func addToSlice(schema string, tableName string, tableID int64, flashbackIDs []int64) []int64 {
+ if filter.IsSystemSchema(schema) && !strings.HasPrefix(tableName, "stats_") && tableName != "gc_delete_range" {
+ flashbackIDs = append(flashbackIDs, tableID)
+ }
+ return flashbackIDs
}
-func addToSlice(schema string, tableName string, tableID int64, flashbackIDs []flashbackID) []flashbackID {
- var excluded bool
- if filter.IsSystemSchema(schema) && !strings.HasPrefix(tableName, "stats_") {
- excluded = true
+// GetTableDataKeyRanges get keyRanges by `flashbackIDs`.
+// This func will return all flashback table data key ranges.
+func GetTableDataKeyRanges(nonFlashbackTableIDs []int64) []kv.KeyRange {
+ var keyRanges []kv.KeyRange
+
+ nonFlashbackTableIDs = append(nonFlashbackTableIDs, -1)
+
+ slices.SortFunc(nonFlashbackTableIDs, func(a, b int64) bool {
+ return a < b
+ })
+
+ for i := 1; i < len(nonFlashbackTableIDs); i++ {
+ keyRanges = append(keyRanges, kv.KeyRange{
+ StartKey: tablecodec.EncodeTablePrefix(nonFlashbackTableIDs[i-1] + 1),
+ EndKey: tablecodec.EncodeTablePrefix(nonFlashbackTableIDs[i]),
+ })
}
- flashbackIDs = append(flashbackIDs, flashbackID{
- id: tableID,
- excluded: excluded,
+
+ // Add all other key ranges.
+ keyRanges = append(keyRanges, kv.KeyRange{
+ StartKey: tablecodec.EncodeTablePrefix(nonFlashbackTableIDs[len(nonFlashbackTableIDs)-1] + 1),
+ EndKey: tablecodec.EncodeTablePrefix(meta.MaxGlobalID),
})
- return flashbackIDs
+
+ return keyRanges
}
-// GetFlashbackKeyRanges make keyRanges efficiently for flashback cluster when many tables in cluster,
+// GetFlashbackKeyRanges get keyRanges for flashback cluster.
+// It contains all non system table key ranges and meta data key ranges.
// The time complexity is O(nlogn).
-func GetFlashbackKeyRanges(sess sessionctx.Context) ([]kv.KeyRange, error) {
+func GetFlashbackKeyRanges(sess sessionctx.Context, flashbackTS uint64) ([]kv.KeyRange, error) {
schemas := sess.GetDomainInfoSchema().(infoschema.InfoSchema).AllSchemas()
// The semantic of keyRanges(output).
- var keyRanges []kv.KeyRange
+ keyRanges := make([]kv.KeyRange, 0)
+
+ // get snapshot schema IDs.
+ flashbackSnapshotMeta := meta.NewSnapshotMeta(sess.GetStore().GetSnapshot(kv.NewVersion(flashbackTS)))
+ snapshotSchemas, err := flashbackSnapshotMeta.ListDatabases()
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ schemaIDs := make(map[int64]struct{})
+ for _, schema := range schemas {
+ if !filter.IsSystemSchema(schema.Name.L) {
+ schemaIDs[schema.ID] = struct{}{}
+ }
+ }
+ for _, schema := range snapshotSchemas {
+ if !filter.IsSystemSchema(schema.Name.L) {
+ schemaIDs[schema.ID] = struct{}{}
+ }
+ }
+
+ // The meta data key ranges.
+ for schemaID := range schemaIDs {
+ metaStartKey := tablecodec.EncodeMetaKeyPrefix(meta.DBkey(schemaID))
+ metaEndKey := tablecodec.EncodeMetaKeyPrefix(meta.DBkey(schemaID + 1))
+ keyRanges = append(keyRanges, kv.KeyRange{
+ StartKey: metaStartKey,
+ EndKey: metaEndKey,
+ })
+ }
+
+ startKey := tablecodec.EncodeMetaKeyPrefix([]byte("DBs"))
+ keyRanges = append(keyRanges, kv.KeyRange{
+ StartKey: startKey,
+ EndKey: startKey.PrefixNext(),
+ })
- var flashbackIDs []flashbackID
+ var nonFlashbackTableIDs []int64
for _, db := range schemas {
for _, table := range db.Tables {
if !table.IsBaseTable() || table.ID > meta.MaxGlobalID {
continue
}
- flashbackIDs = addToSlice(db.Name.L, table.Name.L, table.ID, flashbackIDs)
+ nonFlashbackTableIDs = addToSlice(db.Name.L, table.Name.L, table.ID, nonFlashbackTableIDs)
if table.Partition != nil {
for _, partition := range table.Partition.Definitions {
- flashbackIDs = addToSlice(db.Name.L, table.Name.L, partition.ID, flashbackIDs)
+ nonFlashbackTableIDs = addToSlice(db.Name.L, table.Name.L, partition.ID, nonFlashbackTableIDs)
}
}
}
}
- slices.SortFunc(flashbackIDs, func(a, b flashbackID) bool {
- return a.id < b.id
- })
-
- lastExcludeIdx := -1
- for i, id := range flashbackIDs {
- if id.excluded {
- // Found a range [lastExcludeIdx, i) needs to be added.
- if i > lastExcludeIdx+1 {
- keyRanges = append(keyRanges, kv.KeyRange{
- StartKey: tablecodec.EncodeTablePrefix(flashbackIDs[lastExcludeIdx+1].id),
- EndKey: tablecodec.EncodeTablePrefix(flashbackIDs[i-1].id + 1),
- })
- }
- lastExcludeIdx = i
- }
- }
-
- // The last part needs to be added.
- if lastExcludeIdx < len(flashbackIDs)-1 {
- keyRanges = append(keyRanges, kv.KeyRange{
- StartKey: tablecodec.EncodeTablePrefix(flashbackIDs[lastExcludeIdx+1].id),
- EndKey: tablecodec.EncodeTablePrefix(flashbackIDs[len(flashbackIDs)-1].id + 1),
- })
- }
-
- return keyRanges, nil
+ return append(keyRanges, GetTableDataKeyRanges(nonFlashbackTableIDs)...), nil
}
// SendPrepareFlashbackToVersionRPC prepares regions for flashback, the purpose is to put region into flashback state which region stop write
@@ -280,6 +414,7 @@ func GetFlashbackKeyRanges(sess sessionctx.Context) ([]kv.KeyRange, error) {
func SendPrepareFlashbackToVersionRPC(
ctx context.Context,
s tikv.Storage,
+ flashbackTS, startTS uint64,
r tikvstore.KeyRange,
) (rangetask.TaskStat, error) {
startKey, rangeEndKey := r.StartKey, r.EndKey
@@ -314,6 +449,8 @@ func SendPrepareFlashbackToVersionRPC(
req := tikvrpc.NewRequest(tikvrpc.CmdPrepareFlashbackToVersion, &kvrpcpb.PrepareFlashbackToVersionRequest{
StartKey: startKey,
EndKey: endKey,
+ StartTs: startTS,
+ Version: flashbackTS,
})
resp, err := s.SendReq(bo, req, loc.Region, flashbackTimeout)
@@ -324,15 +461,36 @@ func SendPrepareFlashbackToVersionRPC(
if err != nil {
return taskStat, err
}
+ failpoint.Inject("mockPrepareMeetsEpochNotMatch", func(val failpoint.Value) {
+ if val.(bool) && bo.ErrorsNum() == 0 {
+ regionErr = &errorpb.Error{
+ Message: "stale epoch",
+ EpochNotMatch: &errorpb.EpochNotMatch{},
+ }
+ }
+ })
if regionErr != nil {
- return taskStat, errors.Errorf(regionErr.String())
+ err = bo.Backoff(tikv.BoRegionMiss(), errors.New(regionErr.String()))
+ if err != nil {
+ return taskStat, err
+ }
+ continue
}
if resp.Resp == nil {
- return taskStat, errors.Errorf("prepare flashback missing resp body")
+ logutil.BgLogger().Warn("prepare flashback miss resp body", zap.Uint64("region_id", loc.Region.GetID()))
+ err = bo.Backoff(tikv.BoTiKVRPC(), errors.New("prepare flashback rpc miss resp body"))
+ if err != nil {
+ return taskStat, err
+ }
+ continue
}
prepareFlashbackToVersionResp := resp.Resp.(*kvrpcpb.PrepareFlashbackToVersionResponse)
if err := prepareFlashbackToVersionResp.GetError(); err != "" {
- return taskStat, errors.Errorf(err)
+ boErr := bo.Backoff(tikv.BoTiKVRPC(), errors.New(err))
+ if boErr != nil {
+ return taskStat, boErr
+ }
+ continue
}
taskStat.CompletedRegions++
if isLast {
@@ -465,8 +623,8 @@ func splitRegionsByKeyRanges(d *ddlCtx, keyRanges []kv.KeyRange) {
// A Flashback has 4 different stages.
// 1. before lock flashbackClusterJobID, check clusterJobID and lock it.
-// 2. before flashback start, check timestamp, disable GC and close PD schedule.
-// 3. phase 1, get key ranges, lock all regions.
+// 2. before flashback start, check timestamp, disable GC and close PD schedule, get flashback key ranges.
+// 3. phase 1, lock flashback key ranges.
// 4. phase 2, send flashback RPC, do flashback jobs.
func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
inFlashbackTest := false
@@ -481,11 +639,12 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return ver, errors.Errorf("Not support flashback cluster in non-TiKV env")
}
- var flashbackTS, lockedRegions, commitTS uint64
+ var flashbackTS, lockedRegions, startTS, commitTS uint64
var pdScheduleValue map[string]interface{}
- var autoAnalyzeValue, readOnlyValue string
+ var autoAnalyzeValue, readOnlyValue, ttlJobEnableValue string
var gcEnabledValue bool
- if err := job.DecodeArgs(&flashbackTS, &pdScheduleValue, &gcEnabledValue, &autoAnalyzeValue, &readOnlyValue, &lockedRegions, &commitTS); err != nil {
+ var keyRanges []kv.KeyRange
+ if err := job.DecodeArgs(&flashbackTS, &pdScheduleValue, &gcEnabledValue, &autoAnalyzeValue, &readOnlyValue, &lockedRegions, &startTS, &commitTS, &ttlJobEnableValue, &keyRanges); err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
@@ -525,34 +684,48 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return ver, errors.Trace(err)
}
job.Args[readOnlyOffset] = &readOnlyValue
+ ttlJobEnableValue, err = getTiDBTTLJobEnable(sess)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ job.Args[ttlJobEnableOffSet] = &ttlJobEnableValue
job.SchemaState = model.StateDeleteOnly
return ver, nil
- // Stage 2, check flashbackTS, close GC and PD schedule.
+ // Stage 2, check flashbackTS, close GC and PD schedule, get flashback key ranges.
case model.StateDeleteOnly:
if err = checkAndSetFlashbackClusterInfo(sess, d, t, job, flashbackTS); err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
+ // We should get startTS here to avoid lost startTS when TiDB crashed during send prepare flashback RPC.
+ startTS, err = d.store.GetOracle().GetTimestamp(d.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ job.Args[startTSOffset] = startTS
+ keyRanges, err = GetFlashbackKeyRanges(sess, flashbackTS)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ job.Args[keyRangesOffset] = keyRanges
job.SchemaState = model.StateWriteOnly
- return ver, nil
- // Stage 3, get key ranges and get locks.
+ return updateSchemaVersion(d, t, job)
+ // Stage 3, lock related key ranges.
case model.StateWriteOnly:
// TODO: Support flashback in unistore.
if inFlashbackTest {
job.SchemaState = model.StateWriteReorganization
return updateSchemaVersion(d, t, job)
}
- keyRanges, err := GetFlashbackKeyRanges(sess)
- if err != nil {
- return ver, errors.Trace(err)
- }
// Split region by keyRanges, make sure no unrelated key ranges be locked.
splitRegionsByKeyRanges(d, keyRanges)
totalRegions.Store(0)
for _, r := range keyRanges {
if err = flashbackToVersion(d.ctx, d,
func(ctx context.Context, r tikvstore.KeyRange) (rangetask.TaskStat, error) {
- stats, err := SendPrepareFlashbackToVersionRPC(ctx, d.store.(tikv.Storage), r)
+ stats, err := SendPrepareFlashbackToVersionRPC(ctx, d.store.(tikv.Storage), flashbackTS, startTS, r)
totalRegions.Add(uint64(stats.CompletedRegions))
return stats, err
}, r.StartKey, r.EndKey); err != nil {
@@ -569,7 +742,7 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
}
job.Args[commitTSOffset] = commitTS
job.SchemaState = model.StateWriteReorganization
- return updateSchemaVersion(d, t, job)
+ return ver, nil
// Stage 4, get key ranges and send flashback RPC.
case model.StateWriteReorganization:
// TODO: Support flashback in unistore.
@@ -579,16 +752,12 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
job.SchemaState = model.StatePublic
return ver, nil
}
- keyRanges, err := GetFlashbackKeyRanges(sess)
- if err != nil {
- return ver, errors.Trace(err)
- }
for _, r := range keyRanges {
if err = flashbackToVersion(d.ctx, d,
func(ctx context.Context, r tikvstore.KeyRange) (rangetask.TaskStat, error) {
- // Use commitTS - 1 as startTS, make sure it less than commitTS.
- stats, err := SendFlashbackToVersionRPC(ctx, d.store.(tikv.Storage), flashbackTS, commitTS-1, commitTS, r)
+ // Use same startTS as prepare phase to simulate 1PC txn.
+ stats, err := SendFlashbackToVersionRPC(ctx, d.store.(tikv.Storage), flashbackTS, startTS, commitTS, r)
completedRegions.Add(uint64(stats.CompletedRegions))
logutil.BgLogger().Info("[ddl] flashback cluster stats",
zap.Uint64("complete regions", completedRegions.Load()),
@@ -604,7 +773,7 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
asyncNotifyEvent(d, &util.Event{Tp: model.ActionFlashbackCluster})
job.State = model.JobStateDone
job.SchemaState = model.StatePublic
- return ver, nil
+ return updateSchemaVersion(d, t, job)
}
return ver, nil
}
@@ -615,12 +784,12 @@ func finishFlashbackCluster(w *worker, job *model.Job) error {
return nil
}
- var flashbackTS, lockedRegions, commitTS uint64
+ var flashbackTS, lockedRegions, startTS, commitTS uint64
var pdScheduleValue map[string]interface{}
- var autoAnalyzeValue, readOnlyValue string
+ var autoAnalyzeValue, readOnlyValue, ttlJobEnableValue string
var gcEnabled bool
- if err := job.DecodeArgs(&flashbackTS, &pdScheduleValue, &gcEnabled, &autoAnalyzeValue, &readOnlyValue, &lockedRegions, &commitTS); err != nil {
+ if err := job.DecodeArgs(&flashbackTS, &pdScheduleValue, &gcEnabled, &autoAnalyzeValue, &readOnlyValue, &lockedRegions, &startTS, &commitTS, &ttlJobEnableValue); err != nil {
return errors.Trace(err)
}
sess, err := w.sessPool.get()
@@ -641,6 +810,14 @@ func finishFlashbackCluster(w *worker, job *model.Job) error {
if err = setTiDBSuperReadOnly(w.ctx, sess, readOnlyValue); err != nil {
return err
}
+
+ if job.IsCancelled() {
+ // only restore `tidb_ttl_job_enable` when flashback failed
+ if err = setTiDBTTLJobEnable(w.ctx, sess, ttlJobEnableValue); err != nil {
+ return err
+ }
+ }
+
return setTiDBEnableAutoAnalyze(w.ctx, sess, autoAnalyzeValue)
})
if err != nil {
diff --git a/ddl/cluster_test.go b/ddl/cluster_test.go
index 2cbf5ee45336a..29c237270c5f9 100644
--- a/ddl/cluster_test.go
+++ b/ddl/cluster_test.go
@@ -22,11 +22,13 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/domain/infosync"
"github.com/pingcap/tidb/errno"
+ "github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/parser/model"
- "github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/util/dbterror"
"github.com/stretchr/testify/assert"
@@ -34,55 +36,32 @@ import (
"github.com/tikv/client-go/v2/oracle"
)
-func TestGetFlashbackKeyRanges(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
- se, err := session.CreateSession4Test(store)
- require.NoError(t, err)
-
- kvRanges, err := ddl.GetFlashbackKeyRanges(se)
- require.NoError(t, err)
- // The results are 6 key ranges
- // 0: (stats_meta,stats_histograms,stats_buckets)
- // 1: (stats_feedback)
- // 2: (stats_top_n)
- // 3: (stats_extended)
- // 4: (stats_fm_sketch)
- // 5: (stats_history, stats_meta_history)
- // 6: (stats_table_locked)
- require.Len(t, kvRanges, 7)
-
- tk.MustExec("use test")
- tk.MustExec("CREATE TABLE employees (" +
- " id INT NOT NULL," +
- " store_id INT NOT NULL" +
- ") PARTITION BY RANGE (store_id) (" +
- " PARTITION p0 VALUES LESS THAN (6)," +
- " PARTITION p1 VALUES LESS THAN (11)," +
- " PARTITION p2 VALUES LESS THAN (16)," +
- " PARTITION p3 VALUES LESS THAN (21)" +
- ");")
- tk.MustExec("truncate table mysql.analyze_jobs")
-
- // truncate all `stats_` tables, make table ID consecutive.
- tk.MustExec("truncate table mysql.stats_meta")
- tk.MustExec("truncate table mysql.stats_histograms")
- tk.MustExec("truncate table mysql.stats_buckets")
- tk.MustExec("truncate table mysql.stats_feedback")
- tk.MustExec("truncate table mysql.stats_top_n")
- tk.MustExec("truncate table mysql.stats_extended")
- tk.MustExec("truncate table mysql.stats_fm_sketch")
- tk.MustExec("truncate table mysql.stats_history")
- tk.MustExec("truncate table mysql.stats_meta_history")
- tk.MustExec("truncate table mysql.stats_table_locked")
- kvRanges, err = ddl.GetFlashbackKeyRanges(se)
- require.NoError(t, err)
- require.Len(t, kvRanges, 2)
-
- tk.MustExec("truncate table test.employees")
- kvRanges, err = ddl.GetFlashbackKeyRanges(se)
- require.NoError(t, err)
- require.Len(t, kvRanges, 1)
+func TestGetTableDataKeyRanges(t *testing.T) {
+ // case 1, empty flashbackIDs
+ keyRanges := ddl.GetTableDataKeyRanges([]int64{})
+ require.Len(t, keyRanges, 1)
+ require.Equal(t, keyRanges[0].StartKey, tablecodec.EncodeTablePrefix(0))
+ require.Equal(t, keyRanges[0].EndKey, tablecodec.EncodeTablePrefix(meta.MaxGlobalID))
+
+ // case 2, insert a execluded table ID
+ keyRanges = ddl.GetTableDataKeyRanges([]int64{3})
+ require.Len(t, keyRanges, 2)
+ require.Equal(t, keyRanges[0].StartKey, tablecodec.EncodeTablePrefix(0))
+ require.Equal(t, keyRanges[0].EndKey, tablecodec.EncodeTablePrefix(3))
+ require.Equal(t, keyRanges[1].StartKey, tablecodec.EncodeTablePrefix(4))
+ require.Equal(t, keyRanges[1].EndKey, tablecodec.EncodeTablePrefix(meta.MaxGlobalID))
+
+ // case 3, insert some execluded table ID
+ keyRanges = ddl.GetTableDataKeyRanges([]int64{3, 5, 9})
+ require.Len(t, keyRanges, 4)
+ require.Equal(t, keyRanges[0].StartKey, tablecodec.EncodeTablePrefix(0))
+ require.Equal(t, keyRanges[0].EndKey, tablecodec.EncodeTablePrefix(3))
+ require.Equal(t, keyRanges[1].StartKey, tablecodec.EncodeTablePrefix(4))
+ require.Equal(t, keyRanges[1].EndKey, tablecodec.EncodeTablePrefix(5))
+ require.Equal(t, keyRanges[2].StartKey, tablecodec.EncodeTablePrefix(6))
+ require.Equal(t, keyRanges[2].EndKey, tablecodec.EncodeTablePrefix(9))
+ require.Equal(t, keyRanges[3].StartKey, tablecodec.EncodeTablePrefix(10))
+ require.Equal(t, keyRanges[3].EndKey, tablecodec.EncodeTablePrefix(meta.MaxGlobalID))
}
func TestFlashbackCloseAndResetPDSchedule(t *testing.T) {
@@ -92,13 +71,11 @@ func TestFlashbackCloseAndResetPDSchedule(t *testing.T) {
injectSafeTS := oracle.GoTimeToTS(time.Now().Add(10 * time.Second))
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockFlashbackTest", `return(true)`))
- require.NoError(t, failpoint.Enable("tikvclient/injectSafeTS",
- fmt.Sprintf("return(%v)", injectSafeTS)))
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS",
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/injectSafeTS",
fmt.Sprintf("return(%v)", injectSafeTS)))
oldValue := map[string]interface{}{
- "hot-region-schedule-limit": 1,
+ "merge-schedule-limit": 1,
}
require.NoError(t, infosync.SetPDScheduleConfig(context.Background(), oldValue))
@@ -106,13 +83,13 @@ func TestFlashbackCloseAndResetPDSchedule(t *testing.T) {
defer resetGC()
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
hook.OnJobRunBeforeExported = func(job *model.Job) {
assert.Equal(t, model.ActionFlashbackCluster, job.Type)
if job.SchemaState == model.StateWriteReorganization {
closeValue, err := infosync.GetPDScheduleConfig(context.Background())
assert.NoError(t, err)
- assert.Equal(t, closeValue["hot-region-schedule-limit"], 0)
+ assert.Equal(t, closeValue["merge-schedule-limit"], 0)
// cancel flashback job
job.State = model.JobStateCancelled
job.Error = dbterror.ErrCancelledDDLJob
@@ -120,6 +97,7 @@ func TestFlashbackCloseAndResetPDSchedule(t *testing.T) {
}
dom.DDL().SetHook(hook)
+ time.Sleep(10 * time.Millisecond)
ts, err := tk.Session().GetStore().GetOracle().GetTimestamp(context.Background(), &oracle.Option{})
require.NoError(t, err)
@@ -128,11 +106,10 @@ func TestFlashbackCloseAndResetPDSchedule(t *testing.T) {
finishValue, err := infosync.GetPDScheduleConfig(context.Background())
require.NoError(t, err)
- require.EqualValues(t, finishValue["hot-region-schedule-limit"], 1)
+ require.EqualValues(t, finishValue["merge-schedule-limit"], 1)
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockFlashbackTest"))
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS"))
- require.NoError(t, failpoint.Disable("tikvclient/injectSafeTS"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/injectSafeTS"))
}
func TestAddDDLDuringFlashback(t *testing.T) {
@@ -142,25 +119,25 @@ func TestAddDDLDuringFlashback(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("create table t(a int)")
+ time.Sleep(10 * time.Millisecond)
ts, err := tk.Session().GetStore().GetOracle().GetTimestamp(context.Background(), &oracle.Option{})
require.NoError(t, err)
injectSafeTS := oracle.GoTimeToTS(oracle.GetTimeFromTS(ts).Add(10 * time.Second))
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockFlashbackTest", `return(true)`))
- require.NoError(t, failpoint.Enable("tikvclient/injectSafeTS",
- fmt.Sprintf("return(%v)", injectSafeTS)))
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS",
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/injectSafeTS",
fmt.Sprintf("return(%v)", injectSafeTS)))
timeBeforeDrop, _, safePointSQL, resetGC := MockGC(tk)
defer resetGC()
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
hook.OnJobRunBeforeExported = func(job *model.Job) {
assert.Equal(t, model.ActionFlashbackCluster, job.Type)
if job.SchemaState == model.StateWriteOnly {
- _, err := tk.Exec("alter table t add column b int")
+ tk1 := testkit.NewTestKit(t, store)
+ _, err := tk1.Exec("alter table test.t add column b int")
assert.ErrorContains(t, err, "Can't add ddl job, have flashback cluster job")
}
}
@@ -169,8 +146,7 @@ func TestAddDDLDuringFlashback(t *testing.T) {
dom.DDL().SetHook(originHook)
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockFlashbackTest"))
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS"))
- require.NoError(t, failpoint.Disable("tikvclient/injectSafeTS"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/injectSafeTS"))
}
func TestGlobalVariablesOnFlashback(t *testing.T) {
@@ -180,21 +156,20 @@ func TestGlobalVariablesOnFlashback(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("create table t(a int)")
+ time.Sleep(10 * time.Millisecond)
ts, err := tk.Session().GetStore().GetOracle().GetTimestamp(context.Background(), &oracle.Option{})
require.NoError(t, err)
injectSafeTS := oracle.GoTimeToTS(oracle.GetTimeFromTS(ts).Add(10 * time.Second))
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockFlashbackTest", `return(true)`))
- require.NoError(t, failpoint.Enable("tikvclient/injectSafeTS",
- fmt.Sprintf("return(%v)", injectSafeTS)))
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS",
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/injectSafeTS",
fmt.Sprintf("return(%v)", injectSafeTS)))
timeBeforeDrop, _, safePointSQL, resetGC := MockGC(tk)
defer resetGC()
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
hook.OnJobRunBeforeExported = func(job *model.Job) {
assert.Equal(t, model.ActionFlashbackCluster, job.Type)
if job.SchemaState == model.StateWriteReorganization {
@@ -207,12 +182,16 @@ func TestGlobalVariablesOnFlashback(t *testing.T) {
rs, err = tk.Exec("show variables like 'tidb_super_read_only'")
assert.NoError(t, err)
assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.On)
+ rs, err = tk.Exec("show variables like 'tidb_ttl_job_enable'")
+ assert.NoError(t, err)
+ assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.Off)
}
}
dom.DDL().SetHook(hook)
- // first try with `tidb_gc_enable` = on and `tidb_super_read_only` = off
+ // first try with `tidb_gc_enable` = on and `tidb_super_read_only` = off and `tidb_ttl_job_enable` = on
tk.MustExec("set global tidb_gc_enable = on")
tk.MustExec("set global tidb_super_read_only = off")
+ tk.MustExec("set global tidb_ttl_job_enable = on")
tk.MustExec(fmt.Sprintf("flashback cluster to timestamp '%s'", oracle.GetTimeFromTS(ts)))
@@ -222,10 +201,14 @@ func TestGlobalVariablesOnFlashback(t *testing.T) {
rs, err = tk.Exec("show variables like 'tidb_gc_enable'")
require.NoError(t, err)
require.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.On)
+ rs, err = tk.Exec("show variables like 'tidb_ttl_job_enable'")
+ require.NoError(t, err)
+ require.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.Off)
- // second try with `tidb_gc_enable` = off and `tidb_super_read_only` = on
+ // second try with `tidb_gc_enable` = off and `tidb_super_read_only` = on and `tidb_ttl_job_enable` = off
tk.MustExec("set global tidb_gc_enable = off")
tk.MustExec("set global tidb_super_read_only = on")
+ tk.MustExec("set global tidb_ttl_job_enable = off")
ts, err = tk.Session().GetStore().GetOracle().GetTimestamp(context.Background(), &oracle.Option{})
require.NoError(t, err)
@@ -236,25 +219,27 @@ func TestGlobalVariablesOnFlashback(t *testing.T) {
rs, err = tk.Exec("show variables like 'tidb_gc_enable'")
require.NoError(t, err)
require.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.Off)
+ rs, err = tk.Exec("show variables like 'tidb_ttl_job_enable'")
+ assert.NoError(t, err)
+ assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.Off)
dom.DDL().SetHook(originHook)
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockFlashbackTest"))
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS"))
- require.NoError(t, failpoint.Disable("tikvclient/injectSafeTS"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/injectSafeTS"))
}
func TestCancelFlashbackCluster(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
originHook := dom.DDL().GetHook()
tk := testkit.NewTestKit(t, store)
+
+ time.Sleep(10 * time.Millisecond)
ts, err := tk.Session().GetStore().GetOracle().GetTimestamp(context.Background(), &oracle.Option{})
require.NoError(t, err)
injectSafeTS := oracle.GoTimeToTS(oracle.GetTimeFromTS(ts).Add(10 * time.Second))
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockFlashbackTest", `return(true)`))
- require.NoError(t, failpoint.Enable("tikvclient/injectSafeTS",
- fmt.Sprintf("return(%v)", injectSafeTS)))
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS",
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/injectSafeTS",
fmt.Sprintf("return(%v)", injectSafeTS)))
timeBeforeDrop, _, safePointSQL, resetGC := MockGC(tk)
@@ -266,9 +251,14 @@ func TestCancelFlashbackCluster(t *testing.T) {
return job.SchemaState == model.StateDeleteOnly
})
dom.DDL().SetHook(hook)
+ tk.MustExec("set global tidb_ttl_job_enable = on")
tk.MustGetErrCode(fmt.Sprintf("flashback cluster to timestamp '%s'", oracle.GetTimeFromTS(ts)), errno.ErrCancelledDDLJob)
hook.MustCancelDone(t)
+ rs, err := tk.Exec("show variables like 'tidb_ttl_job_enable'")
+ assert.NoError(t, err)
+ assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.On)
+
// Try canceled on StateWriteReorganization, cancel failed
hook = newCancelJobHook(t, store, dom, func(job *model.Job) bool {
return job.SchemaState == model.StateWriteReorganization
@@ -277,9 +267,12 @@ func TestCancelFlashbackCluster(t *testing.T) {
tk.MustExec(fmt.Sprintf("flashback cluster to timestamp '%s'", oracle.GetTimeFromTS(ts)))
hook.MustCancelFailed(t)
+ rs, err = tk.Exec("show variables like 'tidb_ttl_job_enable'")
+ assert.NoError(t, err)
+ assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.Off)
+
dom.DDL().SetHook(originHook)
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockFlashbackTest"))
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS"))
- require.NoError(t, failpoint.Disable("tikvclient/injectSafeTS"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/injectSafeTS"))
}
diff --git a/ddl/column.go b/ddl/column.go
index 06f6e2897f3ae..4148537ac1d88 100644
--- a/ddl/column.go
+++ b/ddl/column.go
@@ -17,6 +17,7 @@ package ddl
import (
"bytes"
"context"
+ "encoding/hex"
"fmt"
"math/bits"
"strings"
@@ -341,6 +342,9 @@ func checkDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (*model.TableInfo,
if err = checkDropColumnWithForeignKeyConstraintInOwner(d, t, job, tblInfo, colName.L); err != nil {
return nil, nil, nil, false, errors.Trace(err)
}
+ if err = checkDropColumnWithTTLConfig(tblInfo, colName.L); err != nil {
+ return nil, nil, nil, false, errors.Trace(err)
+ }
idxInfos := listIndicesWithColumn(colName.L, tblInfo.Indices)
return tblInfo, colInfo, idxInfos, false, nil
}
@@ -548,6 +552,10 @@ func (w *worker) onModifyColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
+ if tblInfo.Partition != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("table is partition table"))
+ }
changingCol := modifyInfo.changingCol
if changingCol == nil {
@@ -802,8 +810,18 @@ func doReorgWorkForModifyColumnMultiSchema(w *worker, d *ddlCtx, t *meta.Meta, j
func doReorgWorkForModifyColumn(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, tbl table.Table,
oldCol, changingCol *model.ColumnInfo, changingIdxs []*model.IndexInfo) (done bool, ver int64, err error) {
job.ReorgMeta.ReorgTp = model.ReorgTypeTxn
- rh := newReorgHandler(t, w.sess, w.concurrentDDL)
- reorgInfo, err := getReorgInfo(d.jobContext(job), d, rh, job, tbl, BuildElements(changingCol, changingIdxs), false)
+ sctx, err1 := w.sessPool.get()
+ if err1 != nil {
+ err = errors.Trace(err1)
+ return
+ }
+ defer w.sessPool.put(sctx)
+ rh := newReorgHandler(newSession(sctx))
+ dbInfo, err := t.GetDatabase(job.SchemaID)
+ if err != nil {
+ return false, ver, errors.Trace(err)
+ }
+ reorgInfo, err := getReorgInfo(d.jobContext(job.ID), d, rh, job, dbInfo, tbl, BuildElements(changingCol, changingIdxs), false)
if err != nil || reorgInfo.first {
// If we run reorg firstly, we should update the job snapshot version
// and then run the reorg next time.
@@ -830,7 +848,7 @@ func doReorgWorkForModifyColumn(w *worker, d *ddlCtx, t *meta.Meta, job *model.J
// If timeout, we should return, check for the owner and re-wait job done.
return false, ver, nil
}
- if kv.IsTxnRetryableError(err) {
+ if kv.IsTxnRetryableError(err) || dbterror.ErrNotOwner.Equal(err) {
return false, ver, errors.Trace(err)
}
if err1 := rh.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil {
@@ -857,6 +875,9 @@ func adjustTableInfoAfterModifyColumnWithData(tblInfo *model.TableInfo, pos *ast
indexesToRemove := filterIndexesToRemove(changingIdxs, newName, tblInfo)
replaceOldIndexes(tblInfo, indexesToRemove)
}
+ if tblInfo.TTLInfo != nil {
+ updateTTLInfoWhenModifyColumn(tblInfo, oldCol.Name, changingCol.Name)
+ }
// Move the new column to a correct offset.
destOffset, err := LocateOffsetToMove(changingCol.Offset, pos, tblInfo)
if err != nil {
@@ -931,6 +952,17 @@ func updateFKInfoWhenModifyColumn(tblInfo *model.TableInfo, oldCol, newCol model
}
}
+func updateTTLInfoWhenModifyColumn(tblInfo *model.TableInfo, oldCol, newCol model.CIStr) {
+ if oldCol.L == newCol.L {
+ return
+ }
+ if tblInfo.TTLInfo != nil {
+ if tblInfo.TTLInfo.ColumnName.L == oldCol.L {
+ tblInfo.TTLInfo.ColumnName = newCol
+ }
+ }
+}
+
// filterIndexesToRemove filters out the indexes that can be removed.
func filterIndexesToRemove(changingIdxs []*model.IndexInfo, colName model.CIStr, tblInfo *model.TableInfo) []*model.IndexInfo {
indexesToRemove := make([]*model.IndexInfo, 0, len(changingIdxs))
@@ -1029,11 +1061,18 @@ func (w *worker) updatePhysicalTableRow(t table.Table, reorgInfo *reorgInfo) err
if p == nil {
return dbterror.ErrCancelledDDLJob.GenWithStack("Can not find partition id %d for table %d", reorgInfo.PhysicalTableID, t.Meta().ID)
}
- err := w.writePhysicalTableRecord(w.sessPool, p, typeUpdateColumnWorker, reorgInfo)
+ workType := typeReorgPartitionWorker
+ if reorgInfo.Job.Type != model.ActionReorganizePartition {
+ // workType = typeUpdateColumnWorker
+ // TODO: Support Modify Column on partitioned table
+ // https://github.com/pingcap/tidb/issues/38297
+ return dbterror.ErrCancelledDDLJob.GenWithStack("Modify Column on partitioned table / typeUpdateColumnWorker not yet supported.")
+ }
+ err := w.writePhysicalTableRecord(w.sessPool, p, workType, reorgInfo)
if err != nil {
return err
}
- done, err = w.updateReorgInfo(tbl, reorgInfo)
+ done, err = updateReorgInfo(w.sessPool, tbl, reorgInfo)
if err != nil {
return errors.Trace(err)
}
@@ -1058,32 +1097,34 @@ func (w *worker) updateCurrentElement(t table.Table, reorgInfo *reorgInfo) error
TestReorgGoroutineRunning <- a
for {
time.Sleep(30 * time.Millisecond)
- if w.getReorgCtx(reorgInfo.Job).isReorgCanceled() {
+ if w.getReorgCtx(reorgInfo.Job.ID).isReorgCanceled() {
// Job is cancelled. So it can't be done.
failpoint.Return(dbterror.ErrCancelledDDLJob)
}
}
}
})
+ // TODO: Support partition tables.
if bytes.Equal(reorgInfo.currElement.TypeKey, meta.ColumnElementKey) {
- err := w.updatePhysicalTableRow(t, reorgInfo)
+ //nolint:forcetypeassert
+ err := w.updatePhysicalTableRow(t.(table.PhysicalTable), reorgInfo)
if err != nil {
return errors.Trace(err)
}
}
- var physTbl table.PhysicalTable
- if tbl, ok := t.(table.PartitionedTable); ok {
- physTbl = tbl.GetPartition(reorgInfo.PhysicalTableID)
- } else if tbl, ok := t.(table.PhysicalTable); ok {
- physTbl = tbl
+ if _, ok := t.(table.PartitionedTable); ok {
+ // TODO: remove when modify column of partitioned table is supported
+ // https://github.com/pingcap/tidb/issues/38297
+ return dbterror.ErrCancelledDDLJob.GenWithStack("Modify Column on partitioned table / typeUpdateColumnWorker not yet supported.")
}
// Get the original start handle and end handle.
currentVer, err := getValidCurrentVersion(reorgInfo.d.store)
if err != nil {
return errors.Trace(err)
}
- originalStartHandle, originalEndHandle, err := getTableRange(reorgInfo.d.jobContext(reorgInfo.Job), reorgInfo.d, physTbl, currentVer.Ver, reorgInfo.Job.Priority)
+ //nolint:forcetypeassert
+ originalStartHandle, originalEndHandle, err := getTableRange(reorgInfo.d.jobContext(reorgInfo.Job.ID), reorgInfo.d, t.(table.PhysicalTable), currentVer.Ver, reorgInfo.Job.Priority)
if err != nil {
return errors.Trace(err)
}
@@ -1106,21 +1147,21 @@ func (w *worker) updateCurrentElement(t table.Table, reorgInfo *reorgInfo) error
// Then the handle range of the rest elements' is [originalStartHandle, originalEndHandle].
if i == startElementOffsetToResetHandle+1 {
reorgInfo.StartKey, reorgInfo.EndKey = originalStartHandle, originalEndHandle
+ w.getReorgCtx(reorgInfo.Job.ID).setNextKey(reorgInfo.StartKey)
}
// Update the element in the reorgCtx to keep the atomic access for daemon-worker.
- w.getReorgCtx(reorgInfo.Job).setCurrentElement(reorgInfo.elements[i+1])
+ w.getReorgCtx(reorgInfo.Job.ID).setCurrentElement(reorgInfo.elements[i+1])
// Update the element in the reorgInfo for updating the reorg meta below.
reorgInfo.currElement = reorgInfo.elements[i+1]
// Write the reorg info to store so the whole reorganize process can recover from panic.
err := reorgInfo.UpdateReorgMeta(reorgInfo.StartKey, w.sessPool)
logutil.BgLogger().Info("[ddl] update column and indexes",
- zap.Int64("jobID", reorgInfo.Job.ID),
- zap.ByteString("elementType", reorgInfo.currElement.TypeKey),
- zap.Int64("elementID", reorgInfo.currElement.ID),
- zap.String("startHandle", tryDecodeToHandleString(reorgInfo.StartKey)),
- zap.String("endHandle", tryDecodeToHandleString(reorgInfo.EndKey)))
+ zap.Int64("job ID", reorgInfo.Job.ID),
+ zap.Stringer("element", reorgInfo.currElement),
+ zap.String("start key", hex.EncodeToString(reorgInfo.StartKey)),
+ zap.String("end key", hex.EncodeToString(reorgInfo.EndKey)))
if err != nil {
return errors.Trace(err)
}
@@ -1133,7 +1174,7 @@ func (w *worker) updateCurrentElement(t table.Table, reorgInfo *reorgInfo) error
}
type updateColumnWorker struct {
- *backfillWorker
+ *backfillCtx
oldColInfo *model.ColumnInfo
newColInfo *model.ColumnInfo
metricCounter prometheus.Counter
@@ -1145,7 +1186,6 @@ type updateColumnWorker struct {
rowMap map[int64]types.Datum
// For SQL Mode and warnings.
- sqlMode mysql.SQLMode
jobContext *JobContext
}
@@ -1165,14 +1205,13 @@ func newUpdateColumnWorker(sessCtx sessionctx.Context, id int, t table.PhysicalT
}
rowDecoder := decoder.NewRowDecoder(t, t.WritableCols(), decodeColMap)
return &updateColumnWorker{
- backfillWorker: newBackfillWorker(jc.ddlJobCtx, sessCtx, id, t, reorgInfo, typeUpdateColumnWorker),
- oldColInfo: oldCol,
- newColInfo: newCol,
- metricCounter: metrics.BackfillTotalCounter.WithLabelValues(metrics.GenerateReorgLabel("update_col_rate", reorgInfo.SchemaName, t.Meta().Name.String())),
- rowDecoder: rowDecoder,
- rowMap: make(map[int64]types.Datum, len(decodeColMap)),
- sqlMode: reorgInfo.ReorgMeta.SQLMode,
- jobContext: jc,
+ backfillCtx: newBackfillCtx(reorgInfo.d, id, sessCtx, reorgInfo.ReorgMeta.ReorgTp, reorgInfo.SchemaName, t),
+ oldColInfo: oldCol,
+ newColInfo: newCol,
+ metricCounter: metrics.BackfillTotalCounter.WithLabelValues(metrics.GenerateReorgLabel("update_col_rate", reorgInfo.SchemaName, t.Meta().Name.String())),
+ rowDecoder: rowDecoder,
+ rowMap: make(map[int64]types.Datum, len(decodeColMap)),
+ jobContext: jc,
}
}
@@ -1180,14 +1219,34 @@ func (w *updateColumnWorker) AddMetricInfo(cnt float64) {
w.metricCounter.Add(cnt)
}
+func (*updateColumnWorker) String() string {
+ return typeUpdateColumnWorker.String()
+}
+
+func (*updateColumnWorker) GetTasks() ([]*BackfillJob, error) {
+ panic("[ddl] update column worker GetTask function doesn't implement")
+}
+
+func (*updateColumnWorker) UpdateTask(*BackfillJob) error {
+ panic("[ddl] update column worker UpdateTask function doesn't implement")
+}
+
+func (*updateColumnWorker) FinishTask(*BackfillJob) error {
+ panic("[ddl] update column worker FinishTask function doesn't implement")
+}
+
+func (w *updateColumnWorker) GetCtx() *backfillCtx {
+ return w.backfillCtx
+}
+
type rowRecord struct {
key []byte // It's used to lock a record. Record it to reduce the encoding time.
vals []byte // It's the record.
warning *terror.Error // It's used to record the cast warning of a record.
}
-// getNextKey gets next handle of entry that we are going to process.
-func (*updateColumnWorker) getNextKey(taskRange reorgBackfillTask,
+// getNextHandleKey gets next handle of entry that we are going to process.
+func getNextHandleKey(taskRange reorgBackfillTask,
taskDone bool, lastAccessedHandle kv.Key) (nextHandle kv.Key) {
if !taskDone {
// The task is not done. So we need to pick the last processed entry's handle and add one.
@@ -1205,8 +1264,8 @@ func (w *updateColumnWorker) fetchRowColVals(txn kv.Transaction, taskRange reorg
taskDone := false
var lastAccessedHandle kv.Key
oprStartTime := startTime
- err := iterateSnapshotKeys(w.reorgInfo.d.jobContext(w.reorgInfo.Job), w.sessCtx.GetStore(), w.priority, w.table.RecordPrefix(), txn.StartTS(), taskRange.startKey, taskRange.endKey,
- func(handle kv.Handle, recordKey kv.Key, rawRow []byte) (bool, error) {
+ err := iterateSnapshotKeys(w.GetCtx().jobContext(taskRange.getJobID()), w.sessCtx.GetStore(), taskRange.priority, taskRange.physicalTable.RecordPrefix(),
+ txn.StartTS(), taskRange.startKey, taskRange.endKey, func(handle kv.Handle, recordKey kv.Key, rawRow []byte) (bool, error) {
oprEndTime := time.Now()
logSlowOperations(oprEndTime.Sub(oprStartTime), "iterateSnapshotKeys in updateColumnWorker fetchRowColVals", 0)
oprStartTime = oprEndTime
@@ -1237,7 +1296,7 @@ func (w *updateColumnWorker) fetchRowColVals(txn kv.Transaction, taskRange reorg
}
logutil.BgLogger().Debug("[ddl] txn fetches handle info", zap.Uint64("txnStartTS", txn.StartTS()), zap.String("taskRange", taskRange.String()), zap.Duration("takeTime", time.Since(startTime)))
- return w.rowRecords, w.getNextKey(taskRange, taskDone, lastAccessedHandle), taskDone, errors.Trace(err)
+ return w.rowRecords, getNextHandleKey(taskRange, taskDone, lastAccessedHandle), taskDone, errors.Trace(err)
}
func (w *updateColumnWorker) getRowRecord(handle kv.Handle, recordKey []byte, rawRow []byte) error {
@@ -1274,8 +1333,8 @@ func (w *updateColumnWorker) getRowRecord(handle kv.Handle, recordKey []byte, ra
if err != nil {
return w.reformatErrors(err)
}
- if w.sessCtx.GetSessionVars().StmtCtx.GetWarnings() != nil && len(w.sessCtx.GetSessionVars().StmtCtx.GetWarnings()) != 0 {
- warn := w.sessCtx.GetSessionVars().StmtCtx.GetWarnings()
+ warn := w.sessCtx.GetSessionVars().StmtCtx.GetWarnings()
+ if len(warn) != 0 {
//nolint:forcetypeassert
recordWarning = errors.Cause(w.reformatErrors(warn[0].Err)).(*terror.Error)
}
@@ -1347,8 +1406,8 @@ func (w *updateColumnWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (t
errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(ctx context.Context, txn kv.Transaction) error {
taskCtx.addedCount = 0
taskCtx.scanCount = 0
- txn.SetOption(kv.Priority, w.priority)
- if tagger := w.reorgInfo.d.getResourceGroupTaggerForTopSQL(w.reorgInfo.Job); tagger != nil {
+ txn.SetOption(kv.Priority, handleRange.priority)
+ if tagger := w.GetCtx().getResourceGroupTaggerForTopSQL(handleRange.getJobID()); tagger != nil {
txn.SetOption(kv.ResourceGroupTagger, tagger)
}
@@ -1359,8 +1418,9 @@ func (w *updateColumnWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (t
taskCtx.nextKey = nextKey
taskCtx.done = taskDone
- warningsMap := make(map[errors.ErrorID]*terror.Error, len(rowRecords))
- warningsCountMap := make(map[errors.ErrorID]int64, len(rowRecords))
+ // Optimize for few warnings!
+ warningsMap := make(map[errors.ErrorID]*terror.Error, 2)
+ warningsCountMap := make(map[errors.ErrorID]int64, 2)
for _, rowRecord := range rowRecords {
taskCtx.scanCount++
@@ -1473,6 +1533,7 @@ func adjustTableInfoAfterModifyColumn(
tblInfo.MoveColumnInfo(oldCol.Offset, destOffset)
updateNewIdxColsNameOffset(tblInfo.Indices, oldCol.Name, newCol)
updateFKInfoWhenModifyColumn(tblInfo, oldCol.Name, newCol.Name)
+ updateTTLInfoWhenModifyColumn(tblInfo, oldCol.Name, newCol.Name)
return nil
}
@@ -1524,7 +1585,7 @@ func checkAndApplyAutoRandomBits(d *ddlCtx, m *meta.Meta, dbInfo *model.DBInfo,
return nil
}
idAcc := m.GetAutoIDAccessors(dbInfo.ID, tblInfo.ID)
- err := checkNewAutoRandomBits(idAcc, oldCol, newCol, newAutoRandBits, tblInfo.AutoRandomRangeBits, tblInfo.Version)
+ err := checkNewAutoRandomBits(idAcc, oldCol, newCol, newAutoRandBits, tblInfo.AutoRandomRangeBits, tblInfo.SepAutoInc())
if err != nil {
return err
}
@@ -1533,13 +1594,17 @@ func checkAndApplyAutoRandomBits(d *ddlCtx, m *meta.Meta, dbInfo *model.DBInfo,
// checkNewAutoRandomBits checks whether the new auto_random bits number can cause overflow.
func checkNewAutoRandomBits(idAccessors meta.AutoIDAccessors, oldCol *model.ColumnInfo,
- newCol *model.ColumnInfo, newShardBits, newRangeBits uint64, tblVer uint16) error {
+ newCol *model.ColumnInfo, newShardBits, newRangeBits uint64, sepAutoInc bool) error {
shardFmt := autoid.NewShardIDFormat(&newCol.FieldType, newShardBits, newRangeBits)
idAcc := idAccessors.RandomID()
convertedFromAutoInc := mysql.HasAutoIncrementFlag(oldCol.GetFlag())
if convertedFromAutoInc {
- idAcc = idAccessors.IncrementID(tblVer)
+ if sepAutoInc {
+ idAcc = idAccessors.IncrementID(model.TableInfoVersion5)
+ } else {
+ idAcc = idAccessors.RowID()
+ }
}
// Generate a new auto ID first to prevent concurrent update in DML.
_, err := idAcc.Inc(1)
@@ -1650,6 +1715,15 @@ func updateColumnDefaultValue(d *ddlCtx, t *meta.Meta, job *model.Job, newCol *m
job.State = model.JobStateCancelled
return ver, infoschema.ErrColumnNotExists.GenWithStackByArgs(newCol.Name, tblInfo.Name)
}
+
+ if hasDefaultValue, _, err := checkColumnDefaultValue(newContext(d.store), table.ToColumn(oldCol.Clone()), newCol.DefaultValue); err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ } else if !hasDefaultValue {
+ job.State = model.JobStateCancelled
+ return ver, dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(newCol.Name)
+ }
+
// The newCol's offset may be the value of the old schema version, so we can't use newCol directly.
oldCol.DefaultValue = newCol.DefaultValue
oldCol.DefaultValueBit = newCol.DefaultValueBit
diff --git a/ddl/column_change_test.go b/ddl/column_change_test.go
index 4528564d2f231..76a3b377a5abe 100644
--- a/ddl/column_change_test.go
+++ b/ddl/column_change_test.go
@@ -23,6 +23,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/parser/model"
@@ -48,7 +49,7 @@ func TestColumnAdd(t *testing.T) {
tk.MustExec("insert t values (1, 2);")
d := dom.DDL()
- tc := &ddl.TestDDLCallback{Do: dom}
+ tc := &callback.TestDDLCallback{Do: dom}
ct := testNewContext(store)
// set up hook
@@ -149,7 +150,7 @@ func TestModifyAutoRandColumnWithMetaKeyChanged(t *testing.T) {
tk.MustExec("create table t (a bigint primary key clustered AUTO_RANDOM(5));")
d := dom.DDL()
- tc := &ddl.TestDDLCallback{Do: dom}
+ tc := &callback.TestDDLCallback{Do: dom}
var errCount int32 = 3
var genAutoRandErr error
@@ -437,3 +438,36 @@ func testNewContext(store kv.Storage) sessionctx.Context {
ctx.Store = store
return ctx
}
+
+func TestIssue40150(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk.MustExec("CREATE TABLE t40150 (a int) PARTITION BY HASH (a) PARTITIONS 2")
+ tk.MustContainErrMsg(`alter table t40150 rename column a to c`, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed")
+}
+
+func TestIssue40135(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ tk.MustExec("CREATE TABLE t40135 ( a tinyint DEFAULT NULL, b varchar(32) DEFAULT 'md') PARTITION BY HASH (a) PARTITIONS 2")
+ one := true
+ hook := &callback.TestDDLCallback{Do: dom}
+ var checkErr error
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if one {
+ one = false
+ _, checkErr = tk1.Exec("alter table t40135 change column a aNew SMALLINT NULL DEFAULT '-14996'")
+ }
+ }
+ dom.DDL().SetHook(hook)
+ tk.MustExec("alter table t40135 modify column a MEDIUMINT NULL DEFAULT '6243108' FIRST")
+
+ require.ErrorContains(t, checkErr, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed")
+}
diff --git a/ddl/column_modify_test.go b/ddl/column_modify_test.go
index f933182737d05..574de0ee8f08a 100644
--- a/ddl/column_modify_test.go
+++ b/ddl/column_modify_test.go
@@ -24,7 +24,7 @@ import (
"time"
"github.com/pingcap/errors"
- "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
testddlutil "github.com/pingcap/tidb/ddl/testutil"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/errno"
@@ -289,8 +289,7 @@ func TestDropColumn(t *testing.T) {
tk.MustExec("drop table if exists t1")
tk.MustExec("create table t1 (a int,b int) partition by hash(a) partitions 4;")
err := tk.ExecToErr("alter table t1 drop column a")
- // TODO: refine the error message to compatible with MySQL
- require.EqualError(t, err, "[planner:1054]Unknown column 'a' in 'expression'")
+ require.EqualError(t, err, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed")
}
func TestChangeColumn(t *testing.T) {
@@ -665,7 +664,7 @@ func TestTransactionWithWriteOnlyColumn(t *testing.T) {
},
}
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
hook.OnJobRunBeforeExported = func(job *model.Job) {
if checkErr != nil {
@@ -689,7 +688,7 @@ func TestTransactionWithWriteOnlyColumn(t *testing.T) {
dom.DDL().SetHook(hook)
done := make(chan error, 1)
// test transaction on add column.
- go backgroundExec(store, "alter table t1 add column c int not null", done)
+ go backgroundExec(store, "test", "alter table t1 add column c int not null", done)
err := <-done
require.NoError(t, err)
require.NoError(t, checkErr)
@@ -697,7 +696,7 @@ func TestTransactionWithWriteOnlyColumn(t *testing.T) {
tk.MustExec("delete from t1")
// test transaction on drop column.
- go backgroundExec(store, "alter table t1 drop column c", done)
+ go backgroundExec(store, "test", "alter table t1 drop column c", done)
err = <-done
require.NoError(t, err)
require.NoError(t, checkErr)
@@ -873,7 +872,7 @@ func TestAddGeneratedColumnAndInsert(t *testing.T) {
tk1.MustExec("use test")
d := dom.DDL()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
ctx := mock.NewContext()
ctx.Store = store
times := 0
@@ -917,7 +916,7 @@ func TestColumnTypeChangeGenUniqueChangingName(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
- hook := &ddl.TestDDLCallback{}
+ hook := &callback.TestDDLCallback{}
var checkErr error
assertChangingColName := "_col$_c2_0"
assertChangingIdxName := "_idx$_idx_0"
@@ -1030,77 +1029,3 @@ func TestColumnTypeChangeGenUniqueChangingName(t *testing.T) {
tk.MustExec("drop table if exists t")
}
-
-func TestWriteReorgForColumnTypeChangeOnAmendTxn(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, columnModifyLease)
-
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set global tidb_enable_metadata_lock=0")
- tk.MustExec("set global tidb_enable_amend_pessimistic_txn = ON")
- defer tk.MustExec("set global tidb_enable_amend_pessimistic_txn = OFF")
-
- d := dom.DDL()
- testInsertOnModifyColumn := func(sql string, startColState, commitColState model.SchemaState, retStrs []string, retErr error) {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk.MustExec("drop table if exists t1")
- tk.MustExec("create table t1 (c1 int, c2 int, c3 int, unique key(c1))")
- tk.MustExec("insert into t1 values (20, 20, 20);")
-
- var checkErr error
- tk1 := testkit.NewTestKit(t, store)
- defer func() {
- if tk1.Session() != nil {
- tk1.Session().Close()
- }
- }()
- hook := &ddl.TestDDLCallback{Do: dom}
- times := 0
- hook.OnJobRunBeforeExported = func(job *model.Job) {
- if job.Type != model.ActionModifyColumn || checkErr != nil || job.SchemaState != startColState {
- return
- }
-
- tk1.MustExec("use test")
- tk1.MustExec("begin pessimistic;")
- tk1.MustExec("insert into t1 values(101, 102, 103)")
- }
- onJobUpdatedExportedFunc := func(job *model.Job) {
- if job.Type != model.ActionModifyColumn || checkErr != nil || job.SchemaState != commitColState {
- return
- }
- if times == 0 {
- _, checkErr = tk1.Exec("commit;")
- }
- times++
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(hook)
-
- tk.MustExec(sql)
- if retErr == nil {
- require.NoError(t, checkErr)
- } else {
- require.Error(t, checkErr)
- require.Contains(t, checkErr.Error(), retErr.Error())
- }
- tk.MustQuery("select * from t1").Check(testkit.Rows(retStrs...))
- tk.MustExec("admin check table t1")
- }
-
- // Testing it needs reorg data.
- ddlStatement := "alter table t1 change column c2 cc smallint;"
- testInsertOnModifyColumn(ddlStatement, model.StateNone, model.StateWriteReorganization, []string{"20 20 20"}, domain.ErrInfoSchemaChanged)
- testInsertOnModifyColumn(ddlStatement, model.StateDeleteOnly, model.StateWriteReorganization, []string{"20 20 20"}, domain.ErrInfoSchemaChanged)
- testInsertOnModifyColumn(ddlStatement, model.StateWriteOnly, model.StateWriteReorganization, []string{"20 20 20"}, domain.ErrInfoSchemaChanged)
- testInsertOnModifyColumn(ddlStatement, model.StateNone, model.StatePublic, []string{"20 20 20"}, domain.ErrInfoSchemaChanged)
- testInsertOnModifyColumn(ddlStatement, model.StateDeleteOnly, model.StatePublic, []string{"20 20 20"}, domain.ErrInfoSchemaChanged)
- testInsertOnModifyColumn(ddlStatement, model.StateWriteOnly, model.StatePublic, []string{"20 20 20"}, domain.ErrInfoSchemaChanged)
-
- // Testing it needs not reorg data. This case only have two states: none, public.
- ddlStatement = "alter table t1 change column c2 cc bigint;"
- testInsertOnModifyColumn(ddlStatement, model.StateNone, model.StateWriteReorganization, []string{"20 20 20"}, nil)
- testInsertOnModifyColumn(ddlStatement, model.StateWriteOnly, model.StateWriteReorganization, []string{"20 20 20"}, nil)
- testInsertOnModifyColumn(ddlStatement, model.StateNone, model.StatePublic, []string{"20 20 20", "101 102 103"}, nil)
- testInsertOnModifyColumn(ddlStatement, model.StateWriteOnly, model.StatePublic, []string{"20 20 20"}, nil)
-}
diff --git a/ddl/column_test.go b/ddl/column_test.go
index cae9a27318dec..d378c03e297b5 100644
--- a/ddl/column_test.go
+++ b/ddl/column_test.go
@@ -23,7 +23,7 @@ import (
"testing"
"github.com/pingcap/errors"
- "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/model"
@@ -672,7 +672,7 @@ func TestAddColumn(t *testing.T) {
checkOK := false
- tc := &ddl.TestDDLCallback{Do: dom}
+ tc := &callback.TestDDLCallback{Do: dom}
onJobUpdatedExportedFunc := func(job *model.Job) {
if checkOK {
return
@@ -740,7 +740,7 @@ func TestAddColumns(t *testing.T) {
err = txn.Commit(context.Background())
require.NoError(t, err)
- tc := &ddl.TestDDLCallback{Do: dom}
+ tc := &callback.TestDDLCallback{Do: dom}
onJobUpdatedExportedFunc := func(job *model.Job) {
mu.Lock()
defer mu.Unlock()
@@ -810,7 +810,7 @@ func TestDropColumnInColumnTest(t *testing.T) {
var mu sync.Mutex
d := dom.DDL()
- tc := &ddl.TestDDLCallback{Do: dom}
+ tc := &callback.TestDDLCallback{Do: dom}
onJobUpdatedExportedFunc := func(job *model.Job) {
mu.Lock()
defer mu.Unlock()
@@ -872,7 +872,7 @@ func TestDropColumns(t *testing.T) {
var mu sync.Mutex
d := dom.DDL()
- tc := &ddl.TestDDLCallback{Do: dom}
+ tc := &callback.TestDDLCallback{Do: dom}
onJobUpdatedExportedFunc := func(job *model.Job) {
mu.Lock()
defer mu.Unlock()
@@ -959,3 +959,65 @@ func TestGetDefaultValueOfColumn(t *testing.T) {
tk.MustQuery("select * from t1").Check(testkit.RowsWithSep("|", ""+
"1962-03-03 1962-03-03 00:00:00 12:23:23 2020-10-13 2020-03-27"))
}
+
+func TestIssue39080(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("CREATE TABLE t1(id INTEGER PRIMARY KEY, authorId INTEGER AUTO_INCREMENT UNIQUE)")
+
+ tk.MustQuery("show create table t1").Check(testkit.RowsWithSep("|", ""+
+ "t1 CREATE TABLE `t1` (\n"+
+ " `id` int(11) NOT NULL,\n"+
+ " `authorId` int(11) NOT NULL AUTO_INCREMENT,\n"+
+ " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n"+
+ " UNIQUE KEY `authorId` (`authorId`)\n"+
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+ //Do not affect the specified name
+ tk.MustExec("CREATE TABLE `t2`( `id` INTEGER PRIMARY KEY, `authorId` int(11) AUTO_INCREMENT, UNIQUE KEY `authorIdx` (`authorId`))")
+
+ tk.MustQuery("show create table t2").Check(testkit.RowsWithSep("|", ""+
+ "t2 CREATE TABLE `t2` (\n"+
+ " `id` int(11) NOT NULL,\n"+
+ " `authorId` int(11) NOT NULL AUTO_INCREMENT,\n"+
+ " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n"+
+ " UNIQUE KEY `authorIdx` (`authorId`)\n"+
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+}
+
+func TestWriteDataWriteOnlyMode(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
+
+ tk := testkit.NewTestKit(t, store)
+ tk2 := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk2.MustExec("use test")
+ tk.MustExec("CREATE TABLE t (`col1` bigint(20) DEFAULT 1,`col2` float,UNIQUE KEY `key1` (`col1`))")
+
+ originalCallback := dom.DDL().GetHook()
+ defer dom.DDL().SetHook(originalCallback)
+
+ hook := &callback.TestDDLCallback{Do: dom}
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.SchemaState != model.StateWriteOnly {
+ return
+ }
+ tk2.MustExec("insert ignore into t values (1, 2)")
+ tk2.MustExec("insert ignore into t values (2, 2)")
+ }
+ dom.DDL().SetHook(hook)
+ tk.MustExec("alter table t change column `col1` `col1` varchar(20)")
+
+ hook = &callback.TestDDLCallback{Do: dom}
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.SchemaState != model.StateWriteOnly {
+ return
+ }
+ tk2.MustExec("insert ignore into t values (1)")
+ tk2.MustExec("insert ignore into t values (2)")
+ }
+ dom.DDL().SetHook(hook)
+ tk.MustExec("alter table t drop column `col1`")
+ dom.DDL().SetHook(originalCallback)
+}
diff --git a/ddl/column_type_change_test.go b/ddl/column_type_change_test.go
index 818d0714080f1..0aa303c984398 100644
--- a/ddl/column_type_change_test.go
+++ b/ddl/column_type_change_test.go
@@ -23,6 +23,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/model"
@@ -121,7 +122,7 @@ func TestColumnTypeChangeStateBetweenInteger(t *testing.T) {
require.Equal(t, 2, len(tbl.Cols()))
require.NotNil(t, external.GetModifyColumn(t, tk, "test", "t", "c2", false))
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
hook.OnJobRunBeforeExported = func(job *model.Job) {
if checkErr != nil {
@@ -185,7 +186,7 @@ func TestRollbackColumnTypeChangeBetweenInteger(t *testing.T) {
require.Equal(t, 2, len(tbl.Cols()))
require.NotNil(t, external.GetModifyColumn(t, tk, "test", "t", "c2", false))
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
// Mock roll back at model.StateNone.
customizeHookRollbackAtState(hook, tbl, model.StateNone)
dom.DDL().SetHook(hook)
@@ -217,7 +218,7 @@ func TestRollbackColumnTypeChangeBetweenInteger(t *testing.T) {
assertRollBackedColUnchanged(t, tk)
}
-func customizeHookRollbackAtState(hook *ddl.TestDDLCallback, tbl table.Table, state model.SchemaState) {
+func customizeHookRollbackAtState(hook *callback.TestDDLCallback, tbl table.Table, state model.SchemaState) {
hook.OnJobRunBeforeExported = func(job *model.Job) {
if tbl.Meta().ID != job.TableID {
return
@@ -934,7 +935,7 @@ func TestColumnTypeChangeIgnoreDisplayLength(t *testing.T) {
assertHasAlterWriteReorg := func(tbl table.Table) {
// Restore assertResult to false.
assertResult = false
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
hook.OnJobRunBeforeExported = func(job *model.Job) {
if tbl.Meta().ID != job.TableID {
return
@@ -1600,7 +1601,7 @@ func TestChangingColOriginDefaultValue(t *testing.T) {
tbl := external.GetTableByName(t, tk, "test", "t")
originalHook := dom.DDL().GetHook()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var (
once bool
checkErr error
@@ -1679,7 +1680,7 @@ func TestChangingColOriginDefaultValueAfterAddColAndCastSucc(t *testing.T) {
tbl := external.GetTableByName(t, tk, "test", "t")
originalHook := dom.DDL().GetHook()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var (
once bool
checkErr error
@@ -1764,7 +1765,7 @@ func TestChangingColOriginDefaultValueAfterAddColAndCastFail(t *testing.T) {
tbl := external.GetTableByName(t, tk, "test", "t")
originalHook := dom.DDL().GetHook()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
hook.OnJobRunBeforeExported = func(job *model.Job) {
if checkErr != nil {
@@ -1812,8 +1813,7 @@ func TestChangingAttributeOfColumnWithFK(t *testing.T) {
tk.MustExec("use test")
prepare := func() {
- tk.MustExec("drop table if exists users")
- tk.MustExec("drop table if exists orders")
+ tk.MustExec("drop table if exists users, orders")
tk.MustExec("CREATE TABLE users (id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, doc JSON);")
tk.MustExec("CREATE TABLE orders (id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, user_id INT NOT NULL, doc JSON, FOREIGN KEY fk_user_id (user_id) REFERENCES users(id));")
}
@@ -1894,7 +1894,7 @@ func TestDDLExitWhenCancelMeetPanic(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockExceedErrorLimit"))
}()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var jobID int64
hook.OnJobRunBeforeExported = func(job *model.Job) {
if jobID != 0 {
@@ -1969,7 +1969,7 @@ func TestCancelCTCInReorgStateWillCauseGoroutineLeak(t *testing.T) {
tk.MustExec("insert into ctc_goroutine_leak values(1),(2),(3)")
tbl := external.GetTableByName(t, tk, "test", "ctc_goroutine_leak")
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var jobID int64
hook.OnJobRunBeforeExported = func(job *model.Job) {
if jobID != 0 {
@@ -2211,7 +2211,7 @@ func TestCastDateToTimestampInReorgAttribute(t *testing.T) {
var checkErr1 error
var checkErr2 error
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
hook.OnJobRunBeforeExported = func(job *model.Job) {
if checkErr1 != nil || checkErr2 != nil || tbl.Meta().ID != job.TableID {
return
@@ -2422,3 +2422,18 @@ func TestColumnTypeChangeTimestampToInt(t *testing.T) {
tk.MustExec("alter table t add index idx1(id, c1);")
tk.MustExec("admin check table t")
}
+
+func TestFixDDLTxnWillConflictWithReorgTxn(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk.MustExec("create table t (a int)")
+ tk.MustExec("set global tidb_ddl_enable_fast_reorg = OFF")
+ tk.MustExec("alter table t add index(a)")
+ tk.MustExec("set @@sql_mode=''")
+ tk.MustExec("insert into t values(128),(129)")
+ tk.MustExec("alter table t modify column a tinyint")
+
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1690 2 warnings with this error code, first warning: constant 128 overflows tinyint"))
+}
diff --git a/ddl/concurrentddltest/switch_test.go b/ddl/concurrentddltest/switch_test.go
deleted file mode 100644
index d487859260eb2..0000000000000
--- a/ddl/concurrentddltest/switch_test.go
+++ /dev/null
@@ -1,136 +0,0 @@
-// Copyright 2022 PingCAP, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package concurrentddltest
-
-import (
- "context"
- "fmt"
- "math/rand"
- "testing"
- "time"
-
- "github.com/pingcap/tidb/kv"
- "github.com/pingcap/tidb/meta"
- "github.com/pingcap/tidb/testkit"
- "github.com/pingcap/tidb/util"
- "github.com/stretchr/testify/require"
- "go.uber.org/atomic"
-)
-
-func TestConcurrentDDLSwitch(t *testing.T) {
- store := testkit.CreateMockStore(t)
-
- type table struct {
- columnIdx int
- indexIdx int
- }
-
- var tables []*table
- tblCount := 20
- for i := 0; i < tblCount; i++ {
- tables = append(tables, &table{1, 0})
- }
-
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk.MustExec("set global tidb_enable_metadata_lock=0")
- tk.MustExec("set @@global.tidb_ddl_reorg_worker_cnt=1")
- tk.MustExec("set @@global.tidb_ddl_reorg_batch_size=32")
-
- for i := range tables {
- tk.MustExec(fmt.Sprintf("create table t%d (col0 int)", i))
- for j := 0; j < 1000; j++ {
- tk.MustExec(fmt.Sprintf("insert into t%d values (%d)", i, j))
- }
- }
-
- ddls := make([]string, 0, tblCount)
- ddlCount := 100
- for i := 0; i < ddlCount; i++ {
- tblIdx := rand.Intn(tblCount)
- if rand.Intn(2) == 0 {
- ddls = append(ddls, fmt.Sprintf("alter table t%d add index idx%d (col0)", tblIdx, tables[tblIdx].indexIdx))
- tables[tblIdx].indexIdx++
- } else {
- ddls = append(ddls, fmt.Sprintf("alter table t%d add column col%d int", tblIdx, tables[tblIdx].columnIdx))
- tables[tblIdx].columnIdx++
- }
- }
-
- c := atomic.NewInt32(0)
- ch := make(chan struct{})
- go func() {
- var wg util.WaitGroupWrapper
- for i := range ddls {
- wg.Add(1)
- go func(idx int) {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk.MustExec(ddls[idx])
- c.Add(1)
- wg.Done()
- }(i)
- }
- wg.Wait()
- ch <- struct{}{}
- }()
-
- // sleep 2s to make sure the ddl jobs is into table.
- time.Sleep(2 * time.Second)
- ticker := time.NewTicker(time.Second)
- count := 0
- done := false
- for !done {
- select {
- case <-ch:
- done = true
- case <-ticker.C:
- var b bool
- var err error
- err = kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), store, false, func(ctx context.Context, txn kv.Transaction) error {
- b, err = meta.NewMeta(txn).IsConcurrentDDL()
- return err
- })
- require.NoError(t, err)
- rs, err := testkit.NewTestKit(t, store).Exec(fmt.Sprintf("set @@global.tidb_enable_concurrent_ddl=%t", !b))
- if rs != nil {
- require.NoError(t, rs.Close())
- }
- if err == nil {
- count++
- if b {
- tk := testkit.NewTestKit(t, store)
- tk.Session().GetSessionVars().MemQuotaQuery = -1
- tk.MustQuery("select count(*) from mysql.tidb_ddl_job").Check(testkit.Rows("0"))
- tk.MustQuery("select count(*) from mysql.tidb_ddl_reorg").Check(testkit.Rows("0"))
- }
- }
- }
- }
-
- require.Equal(t, int32(ddlCount), c.Load())
- require.Greater(t, count, 0)
-
- tk = testkit.NewTestKit(t, store)
- tk.Session().GetSessionVars().MemQuotaQuery = -1
- tk.MustExec("use test")
- for i, tbl := range tables {
- tk.MustQuery(fmt.Sprintf("select count(*) from information_schema.columns where TABLE_SCHEMA = 'test' and TABLE_NAME = 't%d'", i)).Check(testkit.Rows(fmt.Sprintf("%d", tbl.columnIdx)))
- tk.MustExec(fmt.Sprintf("admin check table t%d", i))
- for j := 0; j < tbl.indexIdx; j++ {
- tk.MustExec(fmt.Sprintf("admin check index t%d idx%d", i, j))
- }
- }
-}
diff --git a/ddl/constant.go b/ddl/constant.go
index bf4d69fb8fd33..99d6498c0cea1 100644
--- a/ddl/constant.go
+++ b/ddl/constant.go
@@ -25,6 +25,10 @@ const (
ReorgTable = "tidb_ddl_reorg"
// HistoryTable stores the history DDL jobs.
HistoryTable = "tidb_ddl_history"
+ // BackgroundSubtaskTable stores the information of backfill jobs.
+ BackgroundSubtaskTable = "tidb_background_subtask"
+ // BackgroundSubtaskHistoryTable stores the information of history backfill jobs.
+ BackgroundSubtaskHistoryTable = "tidb_background_subtask_history"
// JobTableID is the table ID of `tidb_ddl_job`.
JobTableID = meta.MaxInt48 - 1
@@ -34,6 +38,10 @@ const (
HistoryTableID = meta.MaxInt48 - 3
// MDLTableID is the table ID of `tidb_mdl_info`.
MDLTableID = meta.MaxInt48 - 4
+ // BackgroundSubtaskTableID is the table ID of `tidb_background_subtask`.
+ BackgroundSubtaskTableID = meta.MaxInt48 - 5
+ // BackgroundSubtaskHistoryTableID is the table ID of `tidb_background_subtask_history`.
+ BackgroundSubtaskHistoryTableID = meta.MaxInt48 - 6
// JobTableSQL is the CREATE TABLE SQL of `tidb_ddl_job`.
JobTableSQL = "create table " + JobTable + "(job_id bigint not null, reorg int, schema_ids text(65535), table_ids text(65535), job_meta longblob, type int, processing int, primary key(job_id))"
@@ -41,4 +49,34 @@ const (
ReorgTableSQL = "create table " + ReorgTable + "(job_id bigint not null, ele_id bigint, ele_type blob, start_key blob, end_key blob, physical_id bigint, reorg_meta longblob, unique key(job_id, ele_id, ele_type(20)))"
// HistoryTableSQL is the CREATE TABLE SQL of `tidb_ddl_history`.
HistoryTableSQL = "create table " + HistoryTable + "(job_id bigint not null, job_meta longblob, db_name char(64), table_name char(64), schema_ids text(65535), table_ids text(65535), create_time datetime, primary key(job_id))"
+ // BackgroundSubtaskTableSQL is the CREATE TABLE SQL of `tidb_background_subtask`.
+ BackgroundSubtaskTableSQL = "create table " + BackgroundSubtaskTable + `(
+ id bigint not null auto_increment primary key,
+ namespace varchar(256),
+ task_key varchar(256),
+ ddl_physical_tid bigint(20),
+ type int,
+ exec_id varchar(256),
+ exec_expired timestamp,
+ state varchar(64) not null,
+ checkpoint longblob not null,
+ start_time bigint,
+ state_update_time bigint,
+ meta longblob,
+ unique key(namespace, task_key))`
+ // BackgroundSubtaskHistoryTableSQL is the CREATE TABLE SQL of `tidb_background_subtask_history`.
+ BackgroundSubtaskHistoryTableSQL = "create table " + BackgroundSubtaskHistoryTable + `(
+ id bigint not null auto_increment primary key,
+ namespace varchar(256),
+ task_key varchar(256),
+ ddl_physical_tid bigint(20),
+ type int,
+ exec_id varchar(256),
+ exec_expired timestamp,
+ state varchar(64) not null,
+ checkpoint longblob not null,
+ start_time bigint,
+ state_update_time bigint,
+ meta longblob,
+ unique key(namespace, task_key))`
)
diff --git a/ddl/db_change_test.go b/ddl/db_change_test.go
index d865c970e7f42..2dc12c6cda9e8 100644
--- a/ddl/db_change_test.go
+++ b/ddl/db_change_test.go
@@ -24,6 +24,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/executor"
@@ -74,7 +75,7 @@ func TestShowCreateTable(t *testing.T) {
"CREATE TABLE `t2` (\n `a` int(11) DEFAULT NULL,\n `b` varchar(10) COLLATE utf8mb4_general_ci DEFAULT NULL,\n `c` varchar(1) COLLATE utf8mb4_general_ci DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci"},
}
prevState := model.StateNone
- callback := &ddl.TestDDLCallback{}
+ callback := &callback.TestDDLCallback{}
currTestCaseOffset := 0
onJobUpdatedExportedFunc := func(job *model.Job) {
if job.SchemaState == prevState || checkErr != nil {
@@ -143,7 +144,7 @@ func TestDropNotNullColumn(t *testing.T) {
var checkErr error
d := dom.DDL()
originalCallback := d.GetHook()
- callback := &ddl.TestDDLCallback{}
+ callback := &callback.TestDDLCallback{}
sqlNum := 0
onJobUpdatedExportedFunc := func(job *model.Job) {
if checkErr != nil {
@@ -222,7 +223,7 @@ func TestTwoStates(t *testing.T) {
key(c1, c2))`)
tk.MustExec("insert into t values(1, 'a', 'N', '2017-07-01')")
- callback := &ddl.TestDDLCallback{}
+ callback := &callback.TestDDLCallback{}
prevState := model.StateNone
require.NoError(t, testInfo.parseSQLs(parser.New()))
@@ -809,7 +810,7 @@ func runTestInSchemaState(
// Make sure these SQLs use the plan of index scan.
tk.MustExec("drop stats t")
- callback := &ddl.TestDDLCallback{Do: dom}
+ callback := &callback.TestDDLCallback{Do: dom}
prevState := model.StateNone
var checkErr error
se, err := session.CreateSession(store)
@@ -872,7 +873,7 @@ func TestShowIndex(t *testing.T) {
tk.MustExec("use test_db_state")
tk.MustExec(`create table t(c1 int primary key nonclustered, c2 int)`)
- callback := &ddl.TestDDLCallback{}
+ callback := &callback.TestDDLCallback{}
prevState := model.StateNone
showIndexSQL := `show index from t`
var checkErr error
@@ -1325,7 +1326,7 @@ func TestParallelAlterAndDropSchema(t *testing.T) {
}
func prepareTestControlParallelExecSQL(t *testing.T, store kv.Storage, dom *domain.Domain) (*testkit.TestKit, *testkit.TestKit, chan struct{}, ddl.Callback) {
- callback := &ddl.TestDDLCallback{}
+ callback := &callback.TestDDLCallback{}
times := 0
callback.OnJobRunBeforeExported = func(job *model.Job) {
if times != 0 {
@@ -1433,7 +1434,7 @@ func dbChangeTestParallelExecSQL(t *testing.T, store kv.Storage, dom *domain.Dom
var err2, err3 error
var wg util.WaitGroupWrapper
- callback := &ddl.TestDDLCallback{}
+ callback := &callback.TestDDLCallback{}
once := sync.Once{}
onJobUpdatedExportedFunc := func(job *model.Job) {
// sleep a while, let other job enqueue.
@@ -1531,7 +1532,7 @@ func TestParallelDDLBeforeRunDDLJob(t *testing.T) {
tk2 := testkit.NewTestKit(t, store)
tk2.MustExec("use test_db_state")
- intercept := &ddl.TestInterceptor{}
+ intercept := &callback.TestInterceptor{}
var sessionToStart sync.WaitGroup // sessionToStart is a waitgroup to wait for two session to get the same information schema
sessionToStart.Add(2)
@@ -1574,7 +1575,7 @@ func TestParallelDDLBeforeRunDDLJob(t *testing.T) {
wg.Wait()
- intercept = &ddl.TestInterceptor{}
+ intercept = &callback.TestInterceptor{}
d.(ddl.DDLForTest).SetInterceptor(intercept)
}
@@ -1666,7 +1667,7 @@ func TestCreateExpressionIndex(t *testing.T) {
d := dom.DDL()
originalCallback := d.GetHook()
defer d.SetHook(originalCallback)
- callback := &ddl.TestDDLCallback{}
+ callback := &callback.TestDDLCallback{}
onJobUpdatedExportedFunc := func(job *model.Job) {
if checkErr != nil {
return
@@ -1712,6 +1713,14 @@ func TestCreateExpressionIndex(t *testing.T) {
require.NoError(t, checkErr)
tk.MustExec("admin check table t")
tk.MustQuery("select * from t order by a, b").Check(testkit.Rows("0 9", "0 11", "0 11", "1 7", "2 7", "5 7", "8 8", "10 10", "10 10"))
+
+ // https://github.com/pingcap/tidb/issues/39784
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(name varchar(20))")
+ tk.MustExec("insert into t values ('Abc'), ('Bcd'), ('abc')")
+ tk.MustExec("create index idx on test.t((lower(test.t.name)))")
+ tk.MustExec("admin check table t")
}
func TestCreateUniqueExpressionIndex(t *testing.T) {
@@ -1733,13 +1742,11 @@ func TestCreateUniqueExpressionIndex(t *testing.T) {
d := dom.DDL()
originalCallback := d.GetHook()
defer d.SetHook(originalCallback)
- callback := &ddl.TestDDLCallback{}
+ callback := &callback.TestDDLCallback{}
onJobUpdatedExportedFunc := func(job *model.Job) {
if checkErr != nil {
return
}
- err := originalCallback.OnChanged(nil)
- require.NoError(t, err)
switch job.SchemaState {
case model.StateDeleteOnly:
for _, sql := range stateDeleteOnlySQLs {
@@ -1844,7 +1851,7 @@ func TestDropExpressionIndex(t *testing.T) {
d := dom.DDL()
originalCallback := d.GetHook()
defer d.SetHook(originalCallback)
- callback := &ddl.TestDDLCallback{}
+ callback := &callback.TestDDLCallback{}
onJobUpdatedExportedFunc := func(job *model.Job) {
if checkErr != nil {
return
@@ -1916,7 +1923,7 @@ func TestParallelRenameTable(t *testing.T) {
d2 := dom.DDL()
originalCallback := d2.GetHook()
defer d2.SetHook(originalCallback)
- callback := &ddl.TestDDLCallback{Do: dom}
+ callback := &callback.TestDDLCallback{Do: dom}
callback.OnJobRunBeforeExported = func(job *model.Job) {
switch job.SchemaState {
case model.StateNone:
@@ -2031,7 +2038,7 @@ func TestConcurrentSetDefaultValue(t *testing.T) {
d := dom.DDL()
originalCallback := d.GetHook()
defer d.SetHook(originalCallback)
- callback := &ddl.TestDDLCallback{Do: dom}
+ callback := &callback.TestDDLCallback{Do: dom}
skip := false
callback.OnJobRunBeforeExported = func(job *model.Job) {
switch job.SchemaState {
diff --git a/ddl/db_foreign_key_test.go b/ddl/db_foreign_key_test.go
index dc60b53112291..f45db3934d89a 100644
--- a/ddl/db_foreign_key_test.go
+++ b/ddl/db_foreign_key_test.go
@@ -26,8 +26,7 @@ func TestDuplicateForeignKey(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
- tk.MustExec("drop table if exists t")
- tk.MustExec("drop table if exists t1")
+ tk.MustExec("drop table if exists t, t1")
// Foreign table.
tk.MustExec("create table t(id int key)")
// Create target table with duplicate fk.
@@ -38,8 +37,7 @@ func TestDuplicateForeignKey(t *testing.T) {
// Alter target table with duplicate fk.
tk.MustGetErrCode("alter table t1 add CONSTRAINT `fk_aaa` FOREIGN KEY (`id_fk`) REFERENCES `t` (`id`)", mysql.ErrFkDupName)
tk.MustGetErrCode("alter table t1 add CONSTRAINT `fk_aAa` FOREIGN KEY (`id_fk`) REFERENCES `t` (`id`)", mysql.ErrFkDupName)
- tk.MustExec("drop table if exists t")
- tk.MustExec("drop table if exists t1")
+ tk.MustExec("drop table if exists t, t1")
}
func TestTemporaryTableForeignKey(t *testing.T) {
diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go
index b7cb35a9f8727..ad2d0066c56dc 100644
--- a/ddl/db_integration_test.go
+++ b/ddl/db_integration_test.go
@@ -26,8 +26,9 @@ import (
"time"
"github.com/pingcap/errors"
+ _ "github.com/pingcap/tidb/autoid_service"
"github.com/pingcap/tidb/config"
- "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/ddl/schematracker"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/errno"
@@ -686,7 +687,7 @@ func TestUpdateMultipleTable(t *testing.T) {
tk2.MustExec("use test")
d := dom.DDL()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
onJobUpdatedExportedFunc := func(job *model.Job) {
if job.SchemaState == model.StateWriteOnly {
tk2.MustExec("update t1, t2 set t1.c1 = 8, t2.c2 = 10 where t1.c2 = t2.c1")
@@ -1214,14 +1215,14 @@ func TestBitDefaultValue(t *testing.T) {
);`)
}
-func backgroundExec(s kv.Storage, sql string, done chan error) {
+func backgroundExec(s kv.Storage, schema, sql string, done chan error) {
se, err := session.CreateSession4Test(s)
if err != nil {
done <- errors.Trace(err)
return
}
defer se.Close()
- _, err = se.Execute(context.Background(), "use test")
+ _, err = se.Execute(context.Background(), "use "+schema)
if err != nil {
done <- errors.Trace(err)
return
@@ -2373,11 +2374,49 @@ func TestSqlFunctionsInGeneratedColumns(t *testing.T) {
tk.MustExec("create table t (a int, b int as ((a)))")
}
+func TestSchemaNameAndTableNameInGeneratedExpr(t *testing.T) {
+ store := testkit.CreateMockStore(t, mockstore.WithDDLChecker())
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("create database if not exists test")
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+
+ tk.MustExec("create table t(a int, b int as (lower(test.t.a)))")
+ tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n" +
+ " `a` int(11) DEFAULT NULL,\n" +
+ " `b` int(11) GENERATED ALWAYS AS (lower(`a`)) VIRTUAL\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+ tk.MustExec("drop table t")
+ tk.MustExec("create table t(a int)")
+ tk.MustExec("alter table t add column b int as (lower(test.t.a))")
+ tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n" +
+ " `a` int(11) DEFAULT NULL,\n" +
+ " `b` int(11) GENERATED ALWAYS AS (lower(`a`)) VIRTUAL\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+ tk.MustGetErrCode("alter table t add index idx((lower(test.t1.a)))", errno.ErrBadField)
+
+ tk.MustExec("drop table t")
+ tk.MustGetErrCode("create table t(a int, b int as (lower(test1.t.a)))", errno.ErrWrongDBName)
+
+ tk.MustExec("create table t(a int)")
+ tk.MustGetErrCode("alter table t add column b int as (lower(test.t1.a))", errno.ErrWrongTableName)
+
+ tk.MustExec("alter table t add column c int")
+ tk.MustGetErrCode("alter table t modify column c int as (test.t1.a + 1) stored", errno.ErrWrongTableName)
+
+ tk.MustExec("alter table t add column d int as (lower(test.T.a))")
+ tk.MustExec("alter table t add column e int as (lower(Test.t.a))")
+}
+
func TestParserIssue284(t *testing.T) {
store := testkit.CreateMockStore(t, mockstore.WithDDLChecker())
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
+ tk.MustExec("set @@foreign_key_checks=0")
tk.MustExec("create table test.t_parser_issue_284(c1 int not null primary key)")
_, err := tk.Exec("create table test.t_parser_issue_284_2(id int not null primary key, c1 int not null, constraint foreign key (c1) references t_parser_issue_284(c1))")
require.NoError(t, err)
@@ -2894,17 +2933,23 @@ func TestAutoIncrementTableOption(t *testing.T) {
tk.MustExec("create database test_auto_inc_table_opt;")
tk.MustExec("use test_auto_inc_table_opt;")
- // Empty auto_inc allocator should not cause error.
- tk.MustExec("create table t (a bigint primary key clustered) auto_increment = 10;")
- tk.MustExec("alter table t auto_increment = 10;")
- tk.MustExec("alter table t auto_increment = 12345678901234567890;")
-
- // Rebase the auto_inc allocator to a large integer should work.
- tk.MustExec("drop table t;")
- tk.MustExec("create table t (a bigint unsigned auto_increment, unique key idx(a));")
- tk.MustExec("alter table t auto_increment = 12345678901234567890;")
- tk.MustExec("insert into t values ();")
- tk.MustQuery("select * from t;").Check(testkit.Rows("12345678901234567890"))
+ for _, str := range []string{"", " AUTO_ID_CACHE 1"} {
+ // Empty auto_inc allocator should not cause error.
+ tk.MustExec("create table t (a bigint primary key clustered) auto_increment = 10" + str)
+ tk.MustExec("alter table t auto_increment = 10;")
+ tk.MustExec("alter table t auto_increment = 12345678901234567890;")
+ tk.MustExec("drop table t;")
+
+ // Rebase the auto_inc allocator to a large integer should work.
+ tk.MustExec("create table t (a bigint unsigned auto_increment, unique key idx(a))" + str)
+ // Set auto_inc to negative is not supported
+ err := tk.ExecToErr("alter table t auto_increment = -1;")
+ require.Error(t, err)
+ tk.MustExec("alter table t auto_increment = 12345678901234567890;")
+ tk.MustExec("insert into t values ();")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("12345678901234567890"))
+ tk.MustExec("drop table t;")
+ }
}
func TestAutoIncrementForce(t *testing.T) {
@@ -2919,8 +2964,9 @@ func TestAutoIncrementForce(t *testing.T) {
require.NoError(t, err)
return gid
}
+
// Rebase _tidb_row_id.
- tk.MustExec("create table t (a int);")
+ tk.MustExec("create table t (a int)")
tk.MustExec("alter table t force auto_increment = 2;")
tk.MustExec("insert into t values (1),(2);")
tk.MustQuery("select a, _tidb_rowid from t;").Check(testkit.Rows("1 2", "2 3"))
@@ -2932,10 +2978,10 @@ func TestAutoIncrementForce(t *testing.T) {
tk.MustExec("insert into t values (3);")
require.Equal(t, "[kv:1062]Duplicate entry '2' for key 't.PRIMARY'", tk.ExecToErr("insert into t values (3);").Error())
tk.MustQuery("select a, _tidb_rowid from t;").Check(testkit.Rows("3 1", "1 2", "2 3"))
+ tk.MustExec("drop table if exists t;")
// Rebase auto_increment.
- tk.MustExec("drop table if exists t;")
- tk.MustExec("create table t (a int primary key auto_increment, b int);")
+ tk.MustExec("create table t (a int primary key auto_increment, b int)")
tk.MustExec("insert into t values (1, 1);")
tk.MustExec("insert into t values (100000000, 1);")
tk.MustExec("delete from t where a = 100000000;")
@@ -2946,10 +2992,10 @@ func TestAutoIncrementForce(t *testing.T) {
require.Equal(t, uint64(2), getNextGlobalID())
tk.MustExec("insert into t(b) values (2);")
tk.MustQuery("select a, b from t;").Check(testkit.Rows("1 1", "2 2"))
+ tk.MustExec("drop table if exists t;")
// Rebase auto_random.
- tk.MustExec("drop table if exists t;")
- tk.MustExec("create table t (a bigint primary key auto_random(5));")
+ tk.MustExec("create table t (a bigint primary key auto_random(5))")
tk.MustExec("insert into t values ();")
tk.MustExec("set @@allow_auto_random_explicit_insert = true")
tk.MustExec("insert into t values (100000000);")
@@ -2961,14 +3007,15 @@ func TestAutoIncrementForce(t *testing.T) {
require.Equal(t, uint64(2), getNextGlobalID())
tk.MustExec("insert into t values ();")
tk.MustQuery("select (a & 3) from t order by 1;").Check(testkit.Rows("1", "2"))
+ tk.MustExec("drop table if exists t;")
// Change next global ID.
- tk.MustExec("drop table if exists t;")
- tk.MustExec("create table t (a bigint primary key auto_increment);")
+ tk.MustExec("create table t (a bigint primary key auto_increment)")
tk.MustExec("insert into t values (1);")
bases := []uint64{1, 65535, 10, math.MaxUint64, math.MaxInt64 + 1, 1, math.MaxUint64, math.MaxInt64, 2}
lastBase := fmt.Sprintf("%d", bases[len(bases)-1])
for _, b := range bases {
+ fmt.Println("execute alter table force increment to ==", b)
tk.MustExec(fmt.Sprintf("alter table t force auto_increment = %d;", b))
require.Equal(t, b, getNextGlobalID())
}
@@ -2976,7 +3023,7 @@ func TestAutoIncrementForce(t *testing.T) {
tk.MustQuery("select a from t;").Check(testkit.Rows("1", lastBase))
// Force alter unsigned int auto_increment column.
tk.MustExec("drop table if exists t;")
- tk.MustExec("create table t (a bigint unsigned primary key auto_increment);")
+ tk.MustExec("create table t (a bigint unsigned primary key auto_increment)")
for _, b := range bases {
tk.MustExec(fmt.Sprintf("alter table t force auto_increment = %d;", b))
require.Equal(t, b, getNextGlobalID())
@@ -2984,10 +3031,10 @@ func TestAutoIncrementForce(t *testing.T) {
tk.MustQuery("select a from t;").Check(testkit.Rows(fmt.Sprintf("%d", b)))
tk.MustExec("delete from t;")
}
+ tk.MustExec("drop table if exists t;")
// Force alter with @@auto_increment_increment and @@auto_increment_offset.
- tk.MustExec("drop table if exists t;")
- tk.MustExec("create table t(a int key auto_increment);")
+ tk.MustExec("create table t(a int key auto_increment)")
tk.MustExec("set @@auto_increment_offset=2;")
tk.MustExec("set @@auto_increment_increment = 11;")
tk.MustExec("insert into t values (500);")
@@ -3015,6 +3062,135 @@ func TestAutoIncrementForce(t *testing.T) {
tk.MustExec("drop table t")
}
+func TestAutoIncrementForceAutoIDCache(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("drop database if exists auto_inc_force;")
+ tk.MustExec("create database auto_inc_force;")
+ tk.MustExec("use auto_inc_force;")
+ getNextGlobalID := func() uint64 {
+ gidStr := tk.MustQuery("show table t next_row_id").Rows()[0][3]
+ gid, err := strconv.ParseUint(gidStr.(string), 10, 64)
+ require.NoError(t, err)
+ return gid
+ }
+
+ // When AUTO_ID_CACHE is 1, row id and auto increment id use separate allocator, so the behaviour differs.
+ // "Alter table t force auto_increment" has no effect on row id.
+ tk.MustExec("create table t (a int) AUTO_ID_CACHE 1")
+ tk.MustExec("alter table t force auto_increment = 2;")
+ tk.MustExec("insert into t values (1),(2);")
+ tk.MustQuery("select a, _tidb_rowid from t;").Check(testkit.Rows("1 1", "2 2"))
+ // Cannot set next global ID to 0.
+ tk.MustExec("alter table t force auto_increment = 0;")
+ tk.MustExec("alter table t force auto_increment = 1;")
+ tk.MustQuery("show table t next_row_id").Check(testkit.Rows(
+ "auto_inc_force t _tidb_rowid 5001 _TIDB_ROWID",
+ ))
+
+ // inserting new rows can overwrite the existing data.
+ tk.MustExec("insert into t values (3);")
+ tk.MustExec("insert into t values (3);")
+ tk.MustQuery("select a, _tidb_rowid from t;").Check(testkit.Rows("1 1", "2 2", "3 5001", "3 5002"))
+ tk.MustExec("drop table if exists t;")
+
+ // Rebase auto_increment.
+ tk.MustExec("create table t (a int primary key auto_increment, b int) AUTO_ID_CACHE 1")
+ tk.MustExec("insert into t values (1, 1);")
+ tk.MustExec("insert into t values (100000000, 1);")
+ tk.MustExec("delete from t where a = 100000000;")
+ tk.MustQuery("show table t next_row_id").Check(testkit.Rows(
+ "auto_inc_force t a 1 _TIDB_ROWID",
+ "auto_inc_force t a 100000001 AUTO_INCREMENT",
+ ))
+ // Cannot set next global ID to 0.
+ tk.MustGetErrCode("alter table t /*T![force_inc] force */ auto_increment = 0;", errno.ErrAutoincReadFailed)
+ tk.MustExec("alter table t /*T![force_inc] force */ auto_increment = 2;")
+ tk.MustQuery("show table t next_row_id").Check(testkit.Rows(
+ "auto_inc_force t a 1 _TIDB_ROWID",
+ "auto_inc_force t a 2 AUTO_INCREMENT",
+ ))
+
+ tk.MustExec("insert into t(b) values (2);")
+ tk.MustQuery("select a, b from t;").Check(testkit.Rows("1 1", "2 2"))
+ tk.MustExec("drop table if exists t;")
+
+ // Rebase auto_random.
+ tk.MustExec("create table t (a bigint primary key auto_random(5)) AUTO_ID_CACHE 1")
+ tk.MustExec("insert into t values ();")
+ tk.MustExec("set @@allow_auto_random_explicit_insert = true")
+ tk.MustExec("insert into t values (100000000);")
+ tk.MustExec("delete from t where a = 100000000;")
+ require.Greater(t, getNextGlobalID(), uint64(100000000))
+ // Cannot set next global ID to 0.
+ tk.MustGetErrCode("alter table t force auto_random_base = 0;", errno.ErrAutoincReadFailed)
+ tk.MustExec("alter table t force auto_random_base = 2;")
+ require.Equal(t, uint64(2), getNextGlobalID())
+ tk.MustExec("insert into t values ();")
+ tk.MustQuery("select (a & 3) from t order by 1;").Check(testkit.Rows("1", "2"))
+ tk.MustExec("drop table if exists t;")
+
+ // Change next global ID.
+ tk.MustExec("create table t (a bigint primary key auto_increment) AUTO_ID_CACHE 1")
+ tk.MustExec("insert into t values (1);")
+ bases := []uint64{1, 65535, 10, math.MaxUint64, math.MaxInt64 + 1, 1, math.MaxUint64, math.MaxInt64, 2}
+ lastBase := fmt.Sprintf("%d", bases[len(bases)-1])
+ for _, b := range bases {
+ fmt.Println("execute alter table force increment to ==", b)
+ tk.MustExec(fmt.Sprintf("alter table t force auto_increment = %d;", b))
+ tk.MustQuery("show table t next_row_id").Check(testkit.Rows(
+ "auto_inc_force t a 1 _TIDB_ROWID",
+ fmt.Sprintf("auto_inc_force t a %d AUTO_INCREMENT", b),
+ ))
+ }
+ tk.MustExec("insert into t values ();")
+ tk.MustQuery("select a from t;").Check(testkit.Rows("1", lastBase))
+ // Force alter unsigned int auto_increment column.
+ tk.MustExec("drop table if exists t;")
+ tk.MustExec("create table t (a bigint unsigned primary key auto_increment) AUTO_ID_CACHE 1")
+ for _, b := range bases {
+ tk.MustExec(fmt.Sprintf("alter table t force auto_increment = %d;", b))
+ tk.MustQuery("show table t next_row_id").Check(testkit.Rows(
+ "auto_inc_force t a 1 _TIDB_ROWID",
+ fmt.Sprintf("auto_inc_force t a %d AUTO_INCREMENT", b),
+ ))
+ tk.MustExec("insert into t values ();")
+ tk.MustQuery("select a from t;").Check(testkit.Rows(fmt.Sprintf("%d", b)))
+ tk.MustExec("delete from t;")
+ }
+ tk.MustExec("drop table if exists t;")
+
+ // Force alter with @@auto_increment_increment and @@auto_increment_offset.
+ tk.MustExec("create table t(a int key auto_increment) AUTO_ID_CACHE 1")
+ tk.MustExec("set @@auto_increment_offset=2;")
+ tk.MustExec("set @@auto_increment_increment = 11;")
+ tk.MustExec("insert into t values (500);")
+ tk.MustExec("alter table t force auto_increment=100;")
+ tk.MustExec("insert into t values (), ();")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("101", "112", "500"))
+ tk.MustQuery("select * from t order by a;").Check(testkit.Rows("101", "112", "500"))
+ tk.MustExec("drop table if exists t;")
+
+ // Check for warning in case we can't set the auto_increment to the desired value
+ tk.MustExec("create table t(a int primary key auto_increment) AUTO_ID_CACHE 1")
+ tk.MustExec("insert into t values (200)")
+ tk.MustQuery("show create table t").Check(testkit.Rows(
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(11) NOT NULL AUTO_INCREMENT,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![auto_id_cache] AUTO_ID_CACHE=1 */"))
+ tk.MustExec("alter table t auto_increment=100;")
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Can't reset AUTO_INCREMENT to 100 without FORCE option, using 201 instead"))
+ tk.MustExec("insert into t values ()")
+ tk.MustQuery("select * from t").Check(testkit.Rows("200", "211"))
+ tk.MustQuery("show create table t").Check(testkit.Rows(
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(11) NOT NULL AUTO_INCREMENT,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![auto_id_cache] AUTO_ID_CACHE=1 */"))
+ tk.MustExec("drop table t")
+}
+
func TestIssue20490(t *testing.T) {
store := testkit.CreateMockStore(t, mockstore.WithDDLChecker())
@@ -4135,3 +4311,78 @@ func TestRegexpFunctionsGeneratedColumn(t *testing.T) {
tk.MustExec("drop table if exists reg_like")
}
+
+func TestReorgPartitionRangeFailure(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec(`create schema reorgfail`)
+ tk.MustExec("use reorgfail")
+
+ tk.MustExec("CREATE TABLE t (id int, d varchar(255)) partition by range (id) (partition p0 values less than (1000000), partition p1 values less than (2000000), partition p2 values less than (3000000))")
+ tk.MustContainErrMsg(`ALTER TABLE t REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (1000000))`, "[ddl:8200]Unsupported REORGANIZE PARTITION of RANGE; not adjacent partitions")
+ tk.MustContainErrMsg(`ALTER TABLE t REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (4000000))`, "[ddl:8200]Unsupported REORGANIZE PARTITION of RANGE; not adjacent partitions")
+}
+
+func TestReorgPartitionDocs(t *testing.T) {
+ // To test what is added as partition management in the docs
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec(`create schema reorgdocs`)
+ tk.MustExec("use reorgdocs")
+ tk.MustExec(`CREATE TABLE members (
+ id int,
+ fname varchar(255),
+ lname varchar(255),
+ dob date,
+ data json
+)
+PARTITION BY RANGE (YEAR(dob)) (
+ PARTITION pBefore1950 VALUES LESS THAN (1950),
+ PARTITION p1950 VALUES LESS THAN (1960),
+ PARTITION p1960 VALUES LESS THAN (1970),
+ PARTITION p1970 VALUES LESS THAN (1980),
+ PARTITION p1980 VALUES LESS THAN (1990),
+ PARTITION p1990 VALUES LESS THAN (2000))`)
+ tk.MustExec(`CREATE TABLE member_level (
+ id int,
+ level int,
+ achievements json
+)
+PARTITION BY LIST (level) (
+ PARTITION l1 VALUES IN (1),
+ PARTITION l2 VALUES IN (2),
+ PARTITION l3 VALUES IN (3),
+ PARTITION l4 VALUES IN (4),
+ PARTITION l5 VALUES IN (5));`)
+ tk.MustExec(`ALTER TABLE members DROP PARTITION p1990`)
+ tk.MustExec(`ALTER TABLE member_level DROP PARTITION l5`)
+ tk.MustExec(`ALTER TABLE members TRUNCATE PARTITION p1980`)
+ tk.MustExec(`ALTER TABLE member_level TRUNCATE PARTITION l4`)
+ tk.MustExec("ALTER TABLE members ADD PARTITION (PARTITION `p1990to2010` VALUES LESS THAN (2010))")
+ tk.MustExec(`ALTER TABLE member_level ADD PARTITION (PARTITION l5_6 VALUES IN (5,6))`)
+ tk.MustContainErrMsg(`ALTER TABLE members ADD PARTITION (PARTITION p1990 VALUES LESS THAN (2000))`, "[ddl:1493]VALUES LESS THAN value must be strictly increasing for each partition")
+ tk.MustExec(`ALTER TABLE members REORGANIZE PARTITION p1990to2010 INTO
+(PARTITION p1990 VALUES LESS THAN (2000),
+ PARTITION p2000 VALUES LESS THAN (2010),
+ PARTITION p2010 VALUES LESS THAN (2020),
+ PARTITION p2020 VALUES LESS THAN (2030),
+ PARTITION pMax VALUES LESS THAN (MAXVALUE))`)
+ tk.MustExec(`ALTER TABLE member_level REORGANIZE PARTITION l5_6 INTO
+(PARTITION l5 VALUES IN (5),
+ PARTITION l6 VALUES IN (6))`)
+ tk.MustExec(`ALTER TABLE members REORGANIZE PARTITION pBefore1950,p1950 INTO (PARTITION pBefore1960 VALUES LESS THAN (1960))`)
+ tk.MustExec(`ALTER TABLE member_level REORGANIZE PARTITION l1,l2 INTO (PARTITION l1_2 VALUES IN (1,2))`)
+ tk.MustExec(`ALTER TABLE members REORGANIZE PARTITION pBefore1960,p1960,p1970,p1980,p1990,p2000,p2010,p2020,pMax INTO
+(PARTITION p1800 VALUES LESS THAN (1900),
+ PARTITION p1900 VALUES LESS THAN (2000),
+ PARTITION p2000 VALUES LESS THAN (2100))`)
+ tk.MustExec(`ALTER TABLE member_level REORGANIZE PARTITION l1_2,l3,l4,l5,l6 INTO
+(PARTITION lOdd VALUES IN (1,3,5),
+ PARTITION lEven VALUES IN (2,4,6))`)
+ tk.MustContainErrMsg(`ALTER TABLE members REORGANIZE PARTITION p1800,p2000 INTO (PARTITION p2000 VALUES LESS THAN (2100))`, "[ddl:8200]Unsupported REORGANIZE PARTITION of RANGE; not adjacent partitions")
+ tk.MustExec(`INSERT INTO members VALUES (313, "John", "Doe", "2022-11-22", NULL)`)
+ tk.MustExec(`ALTER TABLE members REORGANIZE PARTITION p2000 INTO (PARTITION p2000 VALUES LESS THAN (2050))`)
+ tk.MustContainErrMsg(`ALTER TABLE members REORGANIZE PARTITION p2000 INTO (PARTITION p2000 VALUES LESS THAN (2020))`, "[table:1526]Table has no partition for value 2022")
+ tk.MustExec(`INSERT INTO member_level (id, level) values (313, 6)`)
+ tk.MustContainErrMsg(`ALTER TABLE member_level REORGANIZE PARTITION lEven INTO (PARTITION lEven VALUES IN (2,4))`, "[table:1526]Table has no partition for value 6")
+}
diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go
index e5ad2aa2bbfec..d9d609858bb06 100644
--- a/ddl/db_partition_test.go
+++ b/ddl/db_partition_test.go
@@ -20,6 +20,7 @@ import (
"fmt"
"math/rand"
"strings"
+ "sync"
"sync/atomic"
"testing"
"time"
@@ -27,6 +28,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/ddl/testutil"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/errno"
@@ -1409,7 +1411,7 @@ func TestAlterTableDropPartitionByList(t *testing.T) {
);`)
tk.MustExec(`insert into t values (1),(3),(5),(null)`)
tk.MustExec(`alter table t drop partition p1`)
- tk.MustQuery("select * from t").Sort().Check(testkit.Rows("1", "5", ""))
+ tk.MustQuery("select * from t order by id").Check(testkit.Rows("", "1", "5"))
ctx := tk.Session()
is := domain.GetDomain(ctx).InfoSchema()
tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
@@ -2246,14 +2248,6 @@ func TestExchangePartitionTableCompatiable(t *testing.T) {
"alter table pt8 exchange partition p0 with table nt8;",
dbterror.ErrTablesDifferentMetadata,
},
- {
- // foreign key test
- // Partition table doesn't support to add foreign keys in mysql
- "create table pt9 (id int not null primary key auto_increment,t_id int not null) partition by hash(id) partitions 1;",
- "create table nt9 (id int not null primary key auto_increment, t_id int not null,foreign key fk_id (t_id) references pt5(id));",
- "alter table pt9 exchange partition p0 with table nt9;",
- dbterror.ErrPartitionExchangeForeignKey,
- },
{
// Generated column (virtual)
"create table pt10 (id int not null, lname varchar(30), fname varchar(100) generated always as (concat(lname,' ')) virtual) partition by hash(id) partitions 1;",
@@ -2454,7 +2448,7 @@ func TestExchangePartitionHook(t *testing.T) {
tk.MustExec(`insert into pt values (0), (4), (7)`)
tk.MustExec("insert into nt values (1)")
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
dom.DDL().SetHook(hook)
hookFunc := func(job *model.Job) {
@@ -3343,9 +3337,6 @@ func TestPartitionErrorCode(t *testing.T) {
);`)
tk.MustGetDBError("alter table t_part coalesce partition 4;", dbterror.ErrCoalesceOnlyOnHashPartition)
- tk.MustGetErrCode(`alter table t_part reorganize partition p0, p1 into (
- partition p0 values less than (1980));`, errno.ErrUnsupportedDDLOperation)
-
tk.MustGetErrCode("alter table t_part check partition p0, p1;", errno.ErrUnsupportedDDLOperation)
tk.MustGetErrCode("alter table t_part optimize partition p0,p1;", errno.ErrUnsupportedDDLOperation)
tk.MustGetErrCode("alter table t_part rebuild partition p0,p1;", errno.ErrUnsupportedDDLOperation)
@@ -3740,7 +3731,7 @@ func TestTruncatePartitionMultipleTimes(t *testing.T) {
dom := domain.GetDomain(tk.Session())
originHook := dom.DDL().GetHook()
defer dom.DDL().SetHook(originHook)
- hook := &ddl.TestDDLCallback{}
+ hook := &callback.TestDDLCallback{}
dom.DDL().SetHook(hook)
injected := false
hook.OnJobRunBeforeExported = func(job *model.Job) {
@@ -3757,9 +3748,9 @@ func TestTruncatePartitionMultipleTimes(t *testing.T) {
}
hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
done1 := make(chan error, 1)
- go backgroundExec(store, "alter table test.t truncate partition p0;", done1)
+ go backgroundExec(store, "test", "alter table test.t truncate partition p0;", done1)
done2 := make(chan error, 1)
- go backgroundExec(store, "alter table test.t truncate partition p0;", done2)
+ go backgroundExec(store, "test", "alter table test.t truncate partition p0;", done2)
<-done1
<-done2
require.LessOrEqual(t, errCount, int32(1))
@@ -4536,134 +4527,61 @@ func TestPartitionTableWithAnsiQuotes(t *testing.T) {
` PARTITION "pMax" VALUES LESS THAN (MAXVALUE,MAXVALUE))`))
}
-func TestAlterModifyColumnOnPartitionedTable(t *testing.T) {
+func TestIssue40135Ver2(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ tk3 := testkit.NewTestKit(t, store)
+ tk3.MustExec("use test")
+
+ tk.MustExec("CREATE TABLE t40135 ( a int DEFAULT NULL, b varchar(32) DEFAULT 'md', index(a)) PARTITION BY HASH (a) PARTITIONS 6")
+ tk.MustExec("insert into t40135 values (1, 'md'), (2, 'ma'), (3, 'md'), (4, 'ma'), (5, 'md'), (6, 'ma')")
+ one := true
+ hook := &callback.TestDDLCallback{Do: dom}
+ var checkErr error
+ var wg sync.WaitGroup
+ wg.Add(1)
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.SchemaState == model.StateDeleteOnly {
+ tk3.MustExec("delete from t40135 where a = 1")
+ }
+ if one {
+ one = false
+ go func() {
+ _, checkErr = tk1.Exec("alter table t40135 modify column a int NULL")
+ wg.Done()
+ }()
+ }
+ }
+ dom.DDL().SetHook(hook)
+ tk.MustExec("alter table t40135 modify column a bigint NULL DEFAULT '6243108' FIRST")
+ wg.Wait()
+ require.ErrorContains(t, checkErr, "[ddl:8200]Unsupported modify column: table is partition table")
+ tk.MustExec("admin check table t40135")
+}
+
+func TestAlterModifyPartitionColTruncateWarning(t *testing.T) {
+ t.Skip("waiting for supporting Modify Partition Column again")
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
- tk.MustExec("create database AlterPartTable")
- tk.MustExec("use AlterPartTable")
- tk.MustExec(`create table t (a int unsigned PRIMARY KEY, b varchar(255), key (b))`)
- tk.MustExec(`insert into t values (7, "07"), (8, "08"),(23,"23"),(34,"34💥"),(46,"46"),(57,"57")`)
- tk.MustQuery(`show create table t`).Check(testkit.Rows(
- "t CREATE TABLE `t` (\n" +
- " `a` int(10) unsigned NOT NULL,\n" +
- " `b` varchar(255) DEFAULT NULL,\n" +
- " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
- " KEY `b` (`b`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
- // TODO: Why does it allow 💥 as a latin1 character?
- tk.MustQuery(`select hex(b) from t where a = 34`).Check(testkit.Rows("3334F09F92A5"))
- tk.MustExec(`alter table t modify b varchar(200) charset latin1`)
- tk.MustQuery(`show create table t`).Check(testkit.Rows(
- "t CREATE TABLE `t` (\n" +
- " `a` int(10) unsigned NOT NULL,\n" +
- " `b` varchar(200) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,\n" +
- " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
- " KEY `b` (`b`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
- tk.MustQuery(`select hex(b) from t where a = 34`).Check(testkit.Rows("3334F09F92A5"))
- tk.MustQuery(`select * from t`).Sort().Check(testkit.Rows(""+
- "23 23",
- "34 34💥",
- "46 46",
- "57 57",
- "7 07",
- "8 08"))
- tk.MustQuery(`select * from t order by b`).Check(testkit.Rows(""+
- "7 07",
- "8 08",
- "23 23",
- "34 34💥",
- "46 46",
- "57 57"))
- tk.MustExec(`alter table t change b c varchar(200) charset utf8mb4`)
- tk.MustExec(`drop table t`)
- tk.MustExec(`create table t (a int unsigned PRIMARY KEY, b varchar(255), key (b)) partition by range (a) ` +
- `(partition p0 values less than (10),` +
- ` partition p1 values less than (20),` +
- ` partition p2 values less than (30),` +
- ` partition pMax values less than (MAXVALUE))`)
- tk.MustExec(`insert into t values (7, "07"), (8, "08"),(23,"23"),(34,"34💥"),(46,"46"),(57,"57")`)
- tk.MustQuery(`select * from t`).Sort().Check(testkit.Rows(""+
- "23 23",
- "34 34💥",
- "46 46",
- "57 57",
- "7 07",
- "8 08"))
- tk.MustQuery(`select * from t order by b`).Check(testkit.Rows(""+
- "7 07",
- "8 08",
- "23 23",
- "34 34💥",
- "46 46",
- "57 57"))
- tk.MustQuery(`show create table t`).Check(testkit.Rows(
- "t CREATE TABLE `t` (\n" +
- " `a` int(10) unsigned NOT NULL,\n" +
- " `b` varchar(255) DEFAULT NULL,\n" +
- " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
- " KEY `b` (`b`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
- "PARTITION BY RANGE (`a`)\n" +
- "(PARTITION `p0` VALUES LESS THAN (10),\n" +
- " PARTITION `p1` VALUES LESS THAN (20),\n" +
- " PARTITION `p2` VALUES LESS THAN (30),\n" +
- " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
- tk.MustExec(`alter table t modify b varchar(200) charset latin1`)
- tk.MustQuery(`show create table t`).Check(testkit.Rows(
- "t CREATE TABLE `t` (\n" +
- " `a` int(10) unsigned NOT NULL,\n" +
- " `b` varchar(200) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,\n" +
- " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
- " KEY `b` (`b`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
- "PARTITION BY RANGE (`a`)\n" +
- "(PARTITION `p0` VALUES LESS THAN (10),\n" +
- " PARTITION `p1` VALUES LESS THAN (20),\n" +
- " PARTITION `p2` VALUES LESS THAN (30),\n" +
- " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
- tk.MustQuery(`select * from t`).Sort().Check(testkit.Rows(""+
- "23 23",
- "34 34💥",
- "46 46",
- "57 57",
- "7 07",
- "8 08"))
- tk.MustQuery(`select * from t order by b`).Check(testkit.Rows(""+
- "7 07",
- "8 08",
- "23 23",
- "34 34💥",
- "46 46",
- "57 57"))
- tk.MustExec(`alter table t change b c varchar(150) charset utf8mb4`)
- tk.MustQuery(`show create table t`).Check(testkit.Rows(
- "t CREATE TABLE `t` (\n" +
- " `a` int(10) unsigned NOT NULL,\n" +
- " `c` varchar(150) DEFAULT NULL,\n" +
- " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
- " KEY `b` (`c`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
- "PARTITION BY RANGE (`a`)\n" +
- "(PARTITION `p0` VALUES LESS THAN (10),\n" +
- " PARTITION `p1` VALUES LESS THAN (20),\n" +
- " PARTITION `p2` VALUES LESS THAN (30),\n" +
- " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
- tk.MustQuery(`select * from t`).Sort().Check(testkit.Rows(""+
- "23 23",
- "34 34💥",
- "46 46",
- "57 57",
- "7 07",
- "8 08"))
- tk.MustQuery(`select * from t order by c`).Check(testkit.Rows(""+
- "7 07",
- "8 08",
- "23 23",
- "34 34💥",
- "46 46",
- "57 57"))
- tk.MustGetErrCode(`alter table t modify a varchar(20)`, errno.ErrUnsupportedDDLOperation)
+ schemaName := "truncWarn"
+ tk.MustExec("create database " + schemaName)
+ tk.MustExec("use " + schemaName)
+ tk.MustExec(`set sql_mode = default`)
+ tk.MustExec(`create table t (a varchar(255)) partition by range columns (a) (partition p1 values less than ("0"), partition p2 values less than ("zzzz"))`)
+ tk.MustExec(`insert into t values ("123456"),(" 654321")`)
+ tk.MustContainErrMsg(`alter table t modify a varchar(5)`, "[types:1265]Data truncated for column 'a', value is '")
+ tk.MustExec(`set sql_mode = ''`)
+ tk.MustExec(`alter table t modify a varchar(5)`)
+ // Fix the duplicate warning, see https://github.com/pingcap/tidb/issues/38699
+ tk.MustQuery(`show warnings`).Check(testkit.Rows(""+
+ "Warning 1265 Data truncated for column 'a', value is ' 654321'",
+ "Warning 1265 Data truncated for column 'a', value is ' 654321'"))
+ tk.MustExec(`admin check table t`)
}
func TestAlterModifyColumnOnPartitionedTableRename(t *testing.T) {
@@ -4673,94 +4591,49 @@ func TestAlterModifyColumnOnPartitionedTableRename(t *testing.T) {
tk.MustExec("create database " + schemaName)
tk.MustExec("use " + schemaName)
tk.MustExec(`create table t (a int, b char) partition by range (a) (partition p0 values less than (10))`)
- tk.MustContainErrMsg(`alter table t change a c int`, "[planner:1054]Unknown column 'a' in 'expression'")
+ tk.MustContainErrMsg(`alter table t change a c int`, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed")
tk.MustExec(`drop table t`)
tk.MustExec(`create table t (a char, b char) partition by range columns (a) (partition p0 values less than ('z'))`)
- tk.MustContainErrMsg(`alter table t change a c char`, "[ddl:8200]New column does not match partition definitions: [ddl:1567]partition column name cannot be found")
+ tk.MustContainErrMsg(`alter table t change a c char`, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed")
tk.MustExec(`drop table t`)
tk.MustExec(`create table t (a int, b char) partition by list (a) (partition p0 values in (10))`)
- tk.MustContainErrMsg(`alter table t change a c int`, "[planner:1054]Unknown column 'a' in 'expression'")
+ tk.MustContainErrMsg(`alter table t change a c int`, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed")
tk.MustExec(`drop table t`)
tk.MustExec(`create table t (a char, b char) partition by list columns (a) (partition p0 values in ('z'))`)
- tk.MustContainErrMsg(`alter table t change a c char`, "[ddl:8200]New column does not match partition definitions: [ddl:1567]partition column name cannot be found")
+ tk.MustContainErrMsg(`alter table t change a c char`, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed")
tk.MustExec(`drop table t`)
tk.MustExec(`create table t (a int, b char) partition by hash (a) partitions 3`)
- tk.MustContainErrMsg(`alter table t change a c int`, "[planner:1054]Unknown column 'a' in 'expression'")
+ tk.MustContainErrMsg(`alter table t change a c int`, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed")
}
-func TestAlterModifyColumnOnPartitionedTableFail(t *testing.T) {
+func TestDropPartitionKeyColumn(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
- schemaName := "modColPartFail"
- tk.MustExec("create database " + schemaName)
- tk.MustExec("use " + schemaName)
- tk.MustExec(`create table t (a int unsigned, b varchar(255), key (b)) partition by range (a) (partition p0 values less than (10), partition p1 values less than (20), partition pMax values less than (MAXVALUE))`)
- tk.MustExec(`insert into t values (7, "07"), (8, "08"),(23,"23"),(34,"34💥"),(46,"46"),(57,"57")`)
- tk.MustGetErrCode(`alter table t modify a varchar(255)`, errno.ErrUnsupportedDDLOperation)
- tk.MustGetErrCode(`alter table t modify a float`, mysql.ErrFieldTypeNotAllowedAsPartitionField)
- tk.MustExec(`drop table t`)
- tk.MustExec(`create table t (b int unsigned, a varchar(255), key (b)) partition by range columns (a) (partition p0 values less than (""), partition p1 values less than ("11111"), partition pMax values less than (MAXVALUE))`)
- tk.MustExec(`insert into t values (7, "07"), (8, "08"),(23,"23"),(34,"34 💥💥Longer than 11111"),(46,"46"),(57,"57")`)
- tk.MustExec(`alter table t modify a varchar(50)`)
- tk.MustGetErrCode(`alter table t modify a float`, mysql.ErrFieldTypeNotAllowedAsPartitionField)
- tk.MustGetErrCode(`alter table t modify a int`, errno.ErrUnsupportedDDLOperation)
- tk.MustContainErrMsg(`alter table t modify a varchar(4)`, "[ddl:8200]New column does not match partition definitions: [ddl:1654]Partition column values of incorrect type")
- tk.MustGetErrCode(`alter table t modify a varchar(5)`, errno.WarnDataTruncated)
- tk.MustExec(`SET SQL_MODE = ''`)
- tk.MustExec(`alter table t modify a varchar(5)`)
- // fix https://github.com/pingcap/tidb/issues/38669 and update this
- tk.MustQuery(`show warnings`).Check(testkit.Rows())
- tk.MustExec(`SET SQL_MODE = DEFAULT`)
- tk.MustQuery(`select * from t`).Sort().Check(testkit.Rows(""+
- "23 23",
- "34 34 💥💥",
- "46 46",
- "57 57",
- "7 07",
- "8 08"))
- tStr := "" +
- "CREATE TABLE `t` (\n" +
- " `b` int(10) unsigned DEFAULT NULL,\n" +
- " `a` varchar(5) DEFAULT NULL,\n" +
- " KEY `b` (`b`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
- "PARTITION BY RANGE COLUMNS(`a`)\n" +
- "(PARTITION `p0` VALUES LESS THAN (''),\n" +
- " PARTITION `p1` VALUES LESS THAN ('11111'),\n" +
- " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"
- tk.MustQuery(`show create table t`).Check(testkit.Rows("t " + tStr))
- tk.MustExec(`drop table t`)
- tk.MustExec(tStr)
- tk.MustExec(`drop table t`)
- tk.MustExec("create table t (a int, b varchar(255), key (b)) partition by range (a) (partition `p-300` values less than (-300), partition p0 values less than (0), partition p300 values less than (300))")
- tk.MustExec(`insert into t values (-400, "-400"), (-100, "-100"), (0, "0"), (100, "100"), (290, "290")`)
- tk.MustContainErrMsg(`alter table t modify a int unsigned`, "[ddl:8200]Unsupported modify column, decreasing length of int may result in truncation and change of partition")
- tk.MustContainErrMsg(`alter table t modify a tinyint`, "[ddl:8200]Unsupported modify column, decreasing length of int may result in truncation and change of partition")
- tk.MustExec(`set sql_mode = ''`)
- tk.MustContainErrMsg(`alter table t modify a tinyint`, "[ddl:8200]Unsupported modify column, decreasing length of int may result in truncation and change of partition")
- tk.MustQuery("select * from t partition (`p-300`)").Sort().Check(testkit.Rows("-400 -400"))
- tk.MustExec(`set sql_mode = default`)
- tk.MustContainErrMsg(`alter table t modify a smallint`, "[ddl:8200]Unsupported modify column, decreasing length of int may result in truncation and change of partition")
- tk.MustExec(`alter table t modify a bigint`)
- tk.MustExec(`drop table t`)
- tk.MustExec("create table t (a int, b varchar(255), key (b)) partition by range columns (a) (partition `p-300` values less than (-300), partition p0 values less than (0), partition p300 values less than (300))")
- tk.MustExec(`insert into t values (-400, "-400"), (-100, "-100"), (0, "0"), (100, "100"), (290, "290")`)
- tk.MustContainErrMsg(`alter table t modify a int unsigned`, "[ddl:8200]Unsupported modify column: can't change the partitioning column, since it would require reorganize all partitions")
- tk.MustContainErrMsg(`alter table t modify a tinyint`, "[ddl:8200]New column does not match partition definitions: [ddl:1654]Partition column values of incorrect type")
- tk.MustExec(`set sql_mode = ''`)
- tk.MustContainErrMsg(`alter table t modify a tinyint`, "[ddl:8200]New column does not match partition definitions: [ddl:1654]Partition column values of incorrect type")
- tk.MustQuery("select * from t partition (`p-300`)").Sort().Check(testkit.Rows("-400 -400"))
- tk.MustExec(`set sql_mode = default`)
- // OK to decrease, since with RANGE COLUMNS, it will check the partition definition values against the new type
- tk.MustExec(`alter table t modify a smallint`)
- tk.MustExec(`alter table t modify a bigint`)
+ tk.MustExec("create database DropPartitionKeyColumn")
+ defer tk.MustExec("drop database DropPartitionKeyColumn")
+ tk.MustExec("use DropPartitionKeyColumn")
- tk.MustExec(`drop table t`)
+ tk.MustExec("create table t1 (a tinyint, b char) partition by range (a) ( partition p0 values less than (10) )")
+ err := tk.ExecToErr("alter table t1 drop column a")
+ require.Error(t, err)
+ require.Equal(t, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed", err.Error())
+ tk.MustExec("alter table t1 drop column b")
- tk.MustExec(`create table t (a int, b varchar(255), key (b)) partition by list columns (b) (partition p1 values in ("1", "ab", "12345"), partition p2 values in ("2", "abc", "999999"))`)
- tk.MustExec(`insert into t values (1, "1"), (2, "2"), (999999, "999999")`)
- tk.MustContainErrMsg(`alter table t modify column b varchar(5)`, "[ddl:8200]New column does not match partition definitions: [ddl:1654]Partition column values of incorrect type")
- tk.MustExec(`set sql_mode = ''`)
- tk.MustContainErrMsg(`alter table t modify column b varchar(5)`, "[ddl:8200]New column does not match partition definitions: [ddl:1654]Partition column values of incorrect type")
- tk.MustExec(`set sql_mode = default`)
+ tk.MustExec("create table t2 (a tinyint, b char) partition by range (a-1) ( partition p0 values less than (10) )")
+ err = tk.ExecToErr("alter table t2 drop column a")
+ require.Error(t, err)
+ require.Equal(t, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed", err.Error())
+ tk.MustExec("alter table t2 drop column b")
+
+ tk.MustExec("create table t3 (a tinyint, b char) partition by hash(a) partitions 4;")
+ err = tk.ExecToErr("alter table t3 drop column a")
+ require.Error(t, err)
+ require.Equal(t, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed", err.Error())
+ tk.MustExec("alter table t3 drop column b")
+
+ tk.MustExec("create table t4 (a char, b char) partition by list columns (a) ( partition p0 values in ('0'), partition p1 values in ('a'), partition p2 values in ('b'));")
+ err = tk.ExecToErr("alter table t4 drop column a")
+ require.Error(t, err)
+ require.Equal(t, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed", err.Error())
+ tk.MustExec("alter table t4 drop column b")
}
diff --git a/ddl/db_table_test.go b/ddl/db_table_test.go
index 2771a06ecc291..96a0e3524ae3c 100644
--- a/ddl/db_table_test.go
+++ b/ddl/db_table_test.go
@@ -26,6 +26,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
testddlutil "github.com/pingcap/tidb/ddl/testutil"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/errno"
@@ -51,7 +52,7 @@ func TestTableForeignKey(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
- tk.MustExec("create table t1 (a int, b int);")
+ tk.MustExec("create table t1 (a int, b int, index(a), index(b));")
// test create table with foreign key.
failSQL := "create table t2 (c int, foreign key (a) references t1(a));"
tk.MustGetErrCode(failSQL, errno.ErrKeyColumnDoesNotExits)
@@ -185,7 +186,7 @@ func TestTransactionOnAddDropColumn(t *testing.T) {
originHook := dom.DDL().GetHook()
defer dom.DDL().SetHook(originHook)
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
hook.OnJobRunBeforeExported = func(job *model.Job) {
if checkErr != nil {
@@ -209,7 +210,7 @@ func TestTransactionOnAddDropColumn(t *testing.T) {
dom.DDL().SetHook(hook)
done := make(chan error, 1)
// test transaction on add column.
- go backgroundExec(store, "alter table t1 add column c int not null after a", done)
+ go backgroundExec(store, "test", "alter table t1 add column c int not null after a", done)
err := <-done
require.NoError(t, err)
require.Nil(t, checkErr)
@@ -217,7 +218,7 @@ func TestTransactionOnAddDropColumn(t *testing.T) {
tk.MustExec("delete from t1")
// test transaction on drop column.
- go backgroundExec(store, "alter table t1 drop column c", done)
+ go backgroundExec(store, "test", "alter table t1 drop column c", done)
err = <-done
require.NoError(t, err)
require.Nil(t, checkErr)
@@ -871,8 +872,7 @@ func TestDDLWithInvalidTableInfo(t *testing.T) {
tk.MustExec("create table t (a bigint, b int, c int generated always as (b+1)) partition by hash(a) partitions 4;")
// Test drop partition column.
- // TODO: refine the error message to compatible with MySQL
- tk.MustGetErrMsg("alter table t drop column a;", "[planner:1054]Unknown column 'a' in 'expression'")
+ tk.MustGetErrMsg("alter table t drop column a;", "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed")
// Test modify column with invalid expression.
tk.MustGetErrMsg("alter table t modify column c int GENERATED ALWAYS AS ((case when (a = 0) then 0when (a > 0) then (b / a) end));", "[parser:1064]You have an error in your SQL syntax; check the manual that corresponds to your TiDB version for the right syntax to use line 1 column 97 near \"then (b / a) end));\" ")
// Test add column with invalid expression.
@@ -889,7 +889,7 @@ func TestAddColumn2(t *testing.T) {
originHook := dom.DDL().GetHook()
defer dom.DDL().SetHook(originHook)
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var writeOnlyTable table.Table
hook.OnJobRunBeforeExported = func(job *model.Job) {
if job.SchemaState == model.StateWriteOnly {
@@ -899,7 +899,7 @@ func TestAddColumn2(t *testing.T) {
dom.DDL().SetHook(hook)
done := make(chan error, 1)
// test transaction on add column.
- go backgroundExec(store, "alter table t1 add column c int not null", done)
+ go backgroundExec(store, "test", "alter table t1 add column c int not null", done)
err := <-done
require.NoError(t, err)
@@ -918,7 +918,7 @@ func TestAddColumn2(t *testing.T) {
require.NoError(t, err)
_, err = writeOnlyTable.AddRecord(tk.Session(), types.MakeDatums(oldRow[0].GetInt64(), 2, oldRow[2].GetInt64()), table.IsUpdate)
require.NoError(t, err)
- tk.Session().StmtCommit()
+ tk.Session().StmtCommit(ctx)
err = tk.Session().CommitTxn(ctx)
require.NoError(t, err)
@@ -940,7 +940,7 @@ func TestAddColumn2(t *testing.T) {
}
dom.DDL().SetHook(hook)
- go backgroundExec(store, "alter table t2 add column b int not null default 3", done)
+ go backgroundExec(store, "test", "alter table t2 add column b int not null default 3", done)
err = <-done
require.NoError(t, err)
re.Check(testkit.Rows("1 2"))
diff --git a/ddl/db_test.go b/ddl/db_test.go
index 207aad4e0f116..ca442f15c9ba5 100644
--- a/ddl/db_test.go
+++ b/ddl/db_test.go
@@ -20,6 +20,7 @@ import (
"math"
"strconv"
"strings"
+ "sync"
"testing"
"time"
@@ -27,6 +28,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
ddlutil "github.com/pingcap/tidb/ddl/util"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/errno"
@@ -40,7 +42,6 @@ import (
"github.com/pingcap/tidb/parser/terror"
parsertypes "github.com/pingcap/tidb/parser/types"
"github.com/pingcap/tidb/planner/core"
- "github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/sessiontxn"
"github.com/pingcap/tidb/store/mockstore"
@@ -54,7 +55,6 @@ import (
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/oracle"
"github.com/tikv/client-go/v2/tikv"
- "golang.org/x/exp/slices"
)
const (
@@ -270,7 +270,7 @@ func TestIssue22307(t *testing.T) {
tk.MustExec("create table t (a int, b int)")
tk.MustExec("insert into t values(1, 1);")
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var checkErr1, checkErr2 error
hook.OnJobRunBeforeExported = func(job *model.Job) {
if job.SchemaState != model.StateWriteOnly {
@@ -282,7 +282,7 @@ func TestIssue22307(t *testing.T) {
dom.DDL().SetHook(hook)
done := make(chan error, 1)
// test transaction on add column.
- go backgroundExec(store, "alter table t drop column b;", done)
+ go backgroundExec(store, "test", "alter table t drop column b;", done)
err := <-done
require.NoError(t, err)
require.EqualError(t, checkErr1, "[planner:1054]Unknown column 'b' in 'where clause'")
@@ -571,7 +571,7 @@ func TestAddExpressionIndexRollback(t *testing.T) {
tk1.MustExec("use test")
d := dom.DDL()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var currJob *model.Job
ctx := mock.NewContext()
ctx.Store = store
@@ -619,10 +619,7 @@ func TestAddExpressionIndexRollback(t *testing.T) {
// Check whether the reorg information is cleaned up.
err := sessiontxn.NewTxn(context.Background(), ctx)
require.NoError(t, err)
- txn, err := ctx.Txn(true)
- require.NoError(t, err)
- m := meta.NewMeta(txn)
- element, start, end, physicalID, err := ddl.NewReorgHandlerForTest(m, testkit.NewTestKit(t, store).Session()).GetDDLReorgHandle(currJob)
+ element, start, end, physicalID, err := ddl.NewReorgHandlerForTest(testkit.NewTestKit(t, store).Session()).GetDDLReorgHandle(currJob)
require.True(t, meta.ErrDDLReorgElementNotExist.Equal(err))
require.Nil(t, element)
require.Nil(t, start)
@@ -898,11 +895,11 @@ func TestAutoIncrementIDOnTemporaryTable(t *testing.T) {
tk.MustExec("drop table if exists global_temp_auto_id")
tk.MustExec("create global temporary table global_temp_auto_id(id int primary key auto_increment) on commit delete rows")
tk.MustExec("begin")
- tk.MustQuery("show table global_temp_auto_id next_row_id").Check(testkit.Rows("test global_temp_auto_id id 1 AUTO_INCREMENT"))
+ tk.MustQuery("show table global_temp_auto_id next_row_id").Check(testkit.Rows("test global_temp_auto_id id 1 _TIDB_ROWID"))
tk.MustExec("insert into global_temp_auto_id value(null)")
tk.MustQuery("select @@last_insert_id").Check(testkit.Rows("1"))
tk.MustQuery("select id from global_temp_auto_id").Check(testkit.Rows("1"))
- tk.MustQuery("show table global_temp_auto_id next_row_id").Check(testkit.Rows("test global_temp_auto_id id 2 AUTO_INCREMENT"))
+ tk.MustQuery("show table global_temp_auto_id next_row_id").Check(testkit.Rows("test global_temp_auto_id id 2 _TIDB_ROWID"))
tk.MustExec("commit")
tk.MustExec("drop table global_temp_auto_id")
@@ -914,12 +911,12 @@ func TestAutoIncrementIDOnTemporaryTable(t *testing.T) {
" `id` int(11) NOT NULL AUTO_INCREMENT,\n" +
" PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=100 ON COMMIT DELETE ROWS"))
- tk.MustQuery("show table global_temp_auto_id next_row_id").Check(testkit.Rows("test global_temp_auto_id id 100 AUTO_INCREMENT"))
+ tk.MustQuery("show table global_temp_auto_id next_row_id").Check(testkit.Rows("test global_temp_auto_id id 100 _TIDB_ROWID"))
tk.MustExec("begin")
tk.MustExec("insert into global_temp_auto_id value(null)")
tk.MustQuery("select @@last_insert_id").Check(testkit.Rows("100"))
tk.MustQuery("select id from global_temp_auto_id").Check(testkit.Rows("100"))
- tk.MustQuery("show table global_temp_auto_id next_row_id").Check(testkit.Rows("test global_temp_auto_id id 101 AUTO_INCREMENT"))
+ tk.MustQuery("show table global_temp_auto_id next_row_id").Check(testkit.Rows("test global_temp_auto_id id 101 _TIDB_ROWID"))
tk.MustExec("commit")
}
tk.MustExec("drop table global_temp_auto_id")
@@ -962,7 +959,7 @@ func TestDDLJobErrorCount(t *testing.T) {
}()
var jobID int64
- hook := &ddl.TestDDLCallback{}
+ hook := &callback.TestDDLCallback{}
onJobUpdatedExportedFunc := func(job *model.Job) {
jobID = job.ID
}
@@ -980,201 +977,6 @@ func TestDDLJobErrorCount(t *testing.T) {
require.True(t, kv.ErrEntryTooLarge.Equal(historyJob.Error))
}
-func TestCommitTxnWithIndexChange(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
- // Prepare work.
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set global tidb_enable_metadata_lock=0")
- tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (c1 int primary key, c2 int, c3 int, index ok2(c2))")
- tk.MustExec("insert t1 values (1, 10, 100), (2, 20, 200)")
- tk.MustExec("alter table t1 add index k2(c2)")
- tk.MustExec("alter table t1 drop index k2")
- tk.MustExec("alter table t1 add index k2(c2)")
- tk.MustExec("alter table t1 drop index k2")
- tk2 := testkit.NewTestKit(t, store)
- tk2.MustExec("use test")
-
- // tkSQLs are the sql statements for the pessimistic transaction.
- // tk2DDL are the ddl statements executed before the pessimistic transaction.
- // idxDDL is the DDL statement executed between pessimistic transaction begin and commit.
- // failCommit means the pessimistic transaction commit should fail not.
- type caseUnit struct {
- tkSQLs []string
- tk2DDL []string
- idxDDL string
- checkSQLs []string
- rowsExps [][]string
- failCommit bool
- stateEnd model.SchemaState
- }
-
- cases := []caseUnit{
- // Test secondary index
- {[]string{"insert into t1 values(3, 30, 300)",
- "insert into t2 values(11, 11, 11)"},
- []string{"alter table t1 add index k2(c2)",
- "alter table t1 drop index k2",
- "alter table t1 add index kk2(c2, c1)",
- "alter table t1 add index k2(c2)",
- "alter table t1 drop index k2"},
- "alter table t1 add index k2(c2)",
- []string{"select c3, c2 from t1 use index(k2) where c2 = 20",
- "select c3, c2 from t1 use index(k2) where c2 = 10",
- "select * from t1",
- "select * from t2 where c1 = 11"},
- [][]string{{"200 20"},
- {"100 10"},
- {"1 10 100", "2 20 200", "3 30 300"},
- {"11 11 11"}},
- false,
- model.StateNone},
- // Test secondary index
- {[]string{"insert into t2 values(5, 50, 500)",
- "insert into t2 values(11, 11, 11)",
- "delete from t2 where c2 = 11",
- "update t2 set c2 = 110 where c1 = 11"},
- // "update t2 set c1 = 10 where c3 = 100"},
- []string{"alter table t1 add index k2(c2)",
- "alter table t1 drop index k2",
- "alter table t1 add index kk2(c2, c1)",
- "alter table t1 add index k2(c2)",
- "alter table t1 drop index k2"},
- "alter table t1 add index k2(c2)",
- []string{"select c3, c2 from t1 use index(k2) where c2 = 20",
- "select c3, c2 from t1 use index(k2) where c2 = 10",
- "select * from t1",
- "select * from t2 where c1 = 11",
- "select * from t2 where c3 = 100"},
- [][]string{{"200 20"},
- {"100 10"},
- {"1 10 100", "2 20 200"},
- {},
- {"1 10 100"}},
- false,
- model.StateNone},
- // Test unique index
- {[]string{"insert into t1 values(3, 30, 300)",
- "insert into t1 values(4, 40, 400)",
- "insert into t2 values(11, 11, 11)",
- "insert into t2 values(12, 12, 11)"},
- []string{"alter table t1 add unique index uk3(c3)",
- "alter table t1 drop index uk3",
- "alter table t2 add unique index ukc1c3(c1, c3)",
- "alter table t2 add unique index ukc3(c3)",
- "alter table t2 drop index ukc1c3",
- "alter table t2 drop index ukc3",
- "alter table t2 add index kc3(c3)"},
- "alter table t1 add unique index uk3(c3)",
- []string{"select c3, c2 from t1 use index(uk3) where c3 = 200",
- "select c3, c2 from t1 use index(uk3) where c3 = 300",
- "select c3, c2 from t1 use index(uk3) where c3 = 400",
- "select * from t1",
- "select * from t2"},
- [][]string{{"200 20"},
- {"300 30"},
- {"400 40"},
- {"1 10 100", "2 20 200", "3 30 300", "4 40 400"},
- {"1 10 100", "2 20 200", "11 11 11", "12 12 11"}},
- false, model.StateNone},
- // Test unique index fail to commit, this case needs the new index could be inserted
- {[]string{"insert into t1 values(3, 30, 300)",
- "insert into t1 values(4, 40, 300)",
- "insert into t2 values(11, 11, 11)",
- "insert into t2 values(12, 11, 12)"},
- //[]string{"alter table t1 add unique index uk3(c3)", "alter table t1 drop index uk3"},
- []string{},
- "alter table t1 add unique index uk3(c3)",
- []string{"select c3, c2 from t1 use index(uk3) where c3 = 200",
- "select c3, c2 from t1 use index(uk3) where c3 = 300",
- "select c3, c2 from t1 where c1 = 4",
- "select * from t1",
- "select * from t2"},
- [][]string{{"200 20"},
- {},
- {},
- {"1 10 100", "2 20 200"},
- {"1 10 100", "2 20 200"}},
- true,
- model.StateWriteOnly},
- }
- tk.MustQuery("select * from t1;").Check(testkit.Rows("1 10 100", "2 20 200"))
-
- // Test add index state change
- do := dom.DDL()
- startStates := []model.SchemaState{model.StateNone, model.StateDeleteOnly}
- for _, startState := range startStates {
- endStatMap := session.ConstOpAddIndex[startState]
- var endStates []model.SchemaState
- for st := range endStatMap {
- endStates = append(endStates, st)
- }
- slices.Sort(endStates)
- for _, endState := range endStates {
- for _, curCase := range cases {
- if endState < curCase.stateEnd {
- break
- }
- tk2.MustExec("drop table if exists t1")
- tk2.MustExec("drop table if exists t2")
- tk2.MustExec("create table t1 (c1 int primary key, c2 int, c3 int, index ok2(c2))")
- tk2.MustExec("create table t2 (c1 int primary key, c2 int, c3 int, index ok2(c2))")
- tk2.MustExec("insert t1 values (1, 10, 100), (2, 20, 200)")
- tk2.MustExec("insert t2 values (1, 10, 100), (2, 20, 200)")
- tk2.MustQuery("select * from t1;").Check(testkit.Rows("1 10 100", "2 20 200"))
- tk.MustQuery("select * from t1;").Check(testkit.Rows("1 10 100", "2 20 200"))
- tk.MustQuery("select * from t2;").Check(testkit.Rows("1 10 100", "2 20 200"))
-
- for _, DDLSQL := range curCase.tk2DDL {
- tk2.MustExec(DDLSQL)
- }
- hook := &ddl.TestDDLCallback{Do: dom}
- prepared := false
- committed := false
- hook.OnJobRunBeforeExported = func(job *model.Job) {
- if job.SchemaState == startState {
- if !prepared {
- tk.MustExec("begin pessimistic")
- for _, tkSQL := range curCase.tkSQLs {
- tk.MustExec(tkSQL)
- }
- prepared = true
- }
- }
- }
- onJobUpdatedExportedFunc := func(job *model.Job) {
- if job.SchemaState == endState {
- if !committed {
- if curCase.failCommit {
- err := tk.ExecToErr("commit")
- require.Error(t, err)
- } else {
- tk.MustExec("commit")
- }
- }
- committed = true
- }
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- originalCallback := do.GetHook()
- do.SetHook(hook)
- tk2.MustExec(curCase.idxDDL)
- do.SetHook(originalCallback)
- tk2.MustExec("admin check table t1")
- for i, checkSQL := range curCase.checkSQLs {
- if len(curCase.rowsExps[i]) > 0 {
- tk2.MustQuery(checkSQL).Check(testkit.Rows(curCase.rowsExps[i]...))
- } else {
- tk2.MustQuery(checkSQL).Check(nil)
- }
- }
- }
- }
- }
- tk.MustExec("admin check table t1")
-}
-
// TestAddIndexFailOnCaseWhenCanExit is used to close #19325.
func TestAddIndexFailOnCaseWhenCanExit(t *testing.T) {
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/MockCaseWhenParseFailure", `return(true)`))
@@ -1189,7 +991,11 @@ func TestAddIndexFailOnCaseWhenCanExit(t *testing.T) {
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int, b int)")
tk.MustExec("insert into t values(1, 1)")
- tk.MustGetErrMsg("alter table t add index idx(b)", "[ddl:-1]DDL job rollback, error msg: job.ErrCount:1, mock unknown type: ast.whenClause.")
+ if variable.DDLEnableDistributeReorg.Load() {
+ tk.MustGetErrMsg("alter table t add index idx(b)", "[ddl:-1]job.ErrCount:0, mock unknown type: ast.whenClause.")
+ } else {
+ tk.MustGetErrMsg("alter table t add index idx(b)", "[ddl:-1]DDL job rollback, error msg: job.ErrCount:1, mock unknown type: ast.whenClause.")
+ }
tk.MustExec("drop table if exists t")
}
@@ -1289,7 +1095,7 @@ func TestCancelJobWriteConflict(t *testing.T) {
var cancelErr error
var rs []sqlexec.RecordSet
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
d := dom.DDL()
originalHook := d.GetHook()
d.SetHook(hook)
@@ -1378,54 +1184,6 @@ func TestTxnSavepointWithDDL(t *testing.T) {
tk.MustExec("admin check table t1, t2")
}
-func TestAmendTxnSavepointWithDDL(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
- tk := testkit.NewTestKit(t, store)
- tk2 := testkit.NewTestKit(t, store)
- tk.MustExec("use test;")
- tk.MustExec("set global tidb_enable_metadata_lock=0")
- tk2.MustExec("use test;")
- tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
-
- prepareFn := func() {
- tk.MustExec("drop table if exists t1, t2")
- tk.MustExec("create table t1 (c1 int primary key, c2 int)")
- tk.MustExec("create table t2 (c1 int primary key, c2 int)")
- }
-
- prepareFn()
- tk.MustExec("truncate table t1")
- tk.MustExec("begin pessimistic")
- tk.MustExec("savepoint s1")
- tk.MustExec("insert t1 values (1, 11)")
- tk.MustExec("savepoint s2")
- tk.MustExec("insert t2 values (1, 11)")
- tk.MustExec("rollback to s2")
- tk2.MustExec("alter table t1 add index idx2(c2)")
- tk2.MustExec("alter table t2 add index idx2(c2)")
- tk.MustExec("commit")
- tk.MustQuery("select * from t1").Check(testkit.Rows("1 11"))
- tk.MustQuery("select * from t2").Check(testkit.Rows())
- tk.MustExec("admin check table t1, t2")
-
- prepareFn()
- tk.MustExec("truncate table t1")
- tk.MustExec("begin pessimistic")
- tk.MustExec("savepoint s1")
- tk.MustExec("insert t1 values (1, 11)")
- tk.MustExec("savepoint s2")
- tk.MustExec("insert t2 values (1, 11)")
- tk.MustExec("savepoint s3")
- tk.MustExec("insert t2 values (2, 22)")
- tk.MustExec("rollback to s3")
- tk2.MustExec("alter table t1 add index idx2(c2)")
- tk2.MustExec("alter table t2 add index idx2(c2)")
- tk.MustExec("commit")
- tk.MustQuery("select * from t1").Check(testkit.Rows("1 11"))
- tk.MustQuery("select * from t2").Check(testkit.Rows("1 11"))
- tk.MustExec("admin check table t1, t2")
-}
-
func TestSnapshotVersion(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
@@ -1582,13 +1340,11 @@ func TestLogAndShowSlowLog(t *testing.T) {
}
func TestReportingMinStartTimestamp(t *testing.T) {
- _, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
+ store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
+ tk := testkit.NewTestKit(t, store)
+ se := tk.Session()
infoSyncer := dom.InfoSyncer()
- sm := &testkit.MockSessionManager{
- PS: make([]*util.ProcessInfo, 0),
- }
- infoSyncer.SetSessionManager(sm)
beforeTS := oracle.GoTimeToTS(time.Now())
infoSyncer.ReportMinStartTS(dom.Store())
afterTS := oracle.GoTimeToTS(time.Now())
@@ -1597,13 +1353,21 @@ func TestReportingMinStartTimestamp(t *testing.T) {
now := time.Now()
validTS := oracle.GoTimeToLowerLimitStartTS(now.Add(time.Minute), tikv.MaxTxnTimeUse)
lowerLimit := oracle.GoTimeToLowerLimitStartTS(now, tikv.MaxTxnTimeUse)
+ sm := se.GetSessionManager().(*testkit.MockSessionManager)
sm.PS = []*util.ProcessInfo{
- {CurTxnStartTS: 0},
- {CurTxnStartTS: math.MaxUint64},
- {CurTxnStartTS: lowerLimit},
- {CurTxnStartTS: validTS},
+ {CurTxnStartTS: 0, ProtectedTSList: &se.GetSessionVars().ProtectedTSList},
+ {CurTxnStartTS: math.MaxUint64, ProtectedTSList: &se.GetSessionVars().ProtectedTSList},
+ {CurTxnStartTS: lowerLimit, ProtectedTSList: &se.GetSessionVars().ProtectedTSList},
+ {CurTxnStartTS: validTS, ProtectedTSList: &se.GetSessionVars().ProtectedTSList},
}
- infoSyncer.SetSessionManager(sm)
+ infoSyncer.ReportMinStartTS(dom.Store())
+ require.Equal(t, validTS, infoSyncer.GetMinStartTS())
+
+ unhold := se.GetSessionVars().ProtectedTSList.HoldTS(validTS - 1)
+ infoSyncer.ReportMinStartTS(dom.Store())
+ require.Equal(t, validTS-1, infoSyncer.GetMinStartTS())
+
+ unhold()
infoSyncer.ReportMinStartTS(dom.Store())
require.Equal(t, validTS, infoSyncer.GetMinStartTS())
}
@@ -1736,3 +1500,131 @@ func TestTiDBDownBeforeUpdateGlobalVersion(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockDownBeforeUpdateGlobalVersion"))
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/checkDownBeforeUpdateGlobalVersion"))
}
+
+func TestDDLBlockedCreateView(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int)")
+
+ hook := &callback.TestDDLCallback{Do: dom}
+ first := true
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.SchemaState != model.StateWriteOnly {
+ return
+ }
+ if !first {
+ return
+ }
+ first = false
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk2.MustExec("create view v as select * from t")
+ }
+ dom.DDL().SetHook(hook)
+ tk.MustExec("alter table t modify column a char(10)")
+}
+
+func TestHashPartitionAddColumn(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int, b int) partition by hash(a) partitions 4")
+
+ hook := &callback.TestDDLCallback{Do: dom}
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.SchemaState != model.StateWriteOnly {
+ return
+ }
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk2.MustExec("delete from t")
+ }
+ dom.DDL().SetHook(hook)
+ tk.MustExec("alter table t add column c int")
+}
+
+func TestSetInvalidDefaultValueAfterModifyColumn(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int, b int)")
+
+ var wg sync.WaitGroup
+ var checkErr error
+ one := false
+ hook := &callback.TestDDLCallback{Do: dom}
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.SchemaState != model.StateDeleteOnly {
+ return
+ }
+ if !one {
+ one = true
+ } else {
+ return
+ }
+ wg.Add(1)
+ go func() {
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ _, checkErr = tk2.Exec("alter table t alter column a set default 1")
+ wg.Done()
+ }()
+ }
+ dom.DDL().SetHook(hook)
+ tk.MustExec("alter table t modify column a text(100)")
+ wg.Wait()
+ require.EqualError(t, checkErr, "[ddl:1101]BLOB/TEXT/JSON column 'a' can't have a default value")
+}
+
+func TestMDLTruncateTable(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk2 := testkit.NewTestKit(t, store)
+ tk3 := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int);")
+ tk.MustExec("begin")
+ tk.MustExec("select * from t for update")
+
+ var wg sync.WaitGroup
+
+ hook := &callback.TestDDLCallback{Do: dom}
+ wg.Add(2)
+ var timetk2 time.Time
+ var timetk3 time.Time
+
+ one := false
+ f := func(job *model.Job) {
+ if !one {
+ one = true
+ } else {
+ return
+ }
+ go func() {
+ tk3.MustExec("truncate table test.t")
+ timetk3 = time.Now()
+ wg.Done()
+ }()
+ }
+
+ hook.OnJobUpdatedExported.Store(&f)
+ dom.DDL().SetHook(hook)
+
+ go func() {
+ tk2.MustExec("truncate table test.t")
+ timetk2 = time.Now()
+ wg.Done()
+ }()
+
+ time.Sleep(2 * time.Second)
+ timeMain := time.Now()
+ tk.MustExec("commit")
+ wg.Wait()
+ require.True(t, timetk2.After(timeMain))
+ require.True(t, timetk3.After(timeMain))
+}
diff --git a/ddl/ddl.go b/ddl/ddl.go
index 7eeaa8c1f357e..e658547b471d3 100644
--- a/ddl/ddl.go
+++ b/ddl/ddl.go
@@ -21,7 +21,6 @@ package ddl
import (
"context"
"encoding/json"
- "flag"
"fmt"
"runtime"
"strconv"
@@ -49,6 +48,7 @@ import (
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/terror"
+ rmutil "github.com/pingcap/tidb/resourcemanager/util"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/binloginfo"
"github.com/pingcap/tidb/sessionctx/variable"
@@ -60,9 +60,11 @@ import (
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/gcutil"
+ "github.com/pingcap/tidb/util/gpool/spmc"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/mathutil"
"github.com/pingcap/tidb/util/sqlexec"
+ "github.com/pingcap/tidb/util/syncutil"
"github.com/tikv/client-go/v2/tikvrpc"
clientv3 "go.etcd.io/etcd/client/v3"
atomicutil "go.uber.org/atomic"
@@ -83,8 +85,9 @@ const (
batchAddingJobs = 10
- reorgWorkerCnt = 10
- generalWorkerCnt = 1
+ reorgWorkerCnt = 10
+ generalWorkerCnt = 1
+ backfillWorkerCnt = 32
// checkFlagIndexInJobArgs is the recoverCheckFlag index used in RecoverTable/RecoverSchema job arg list.
checkFlagIndexInJobArgs = 1
@@ -188,6 +191,9 @@ type DDL interface {
CreatePlacementPolicy(ctx sessionctx.Context, stmt *ast.CreatePlacementPolicyStmt) error
DropPlacementPolicy(ctx sessionctx.Context, stmt *ast.DropPlacementPolicyStmt) error
AlterPlacementPolicy(ctx sessionctx.Context, stmt *ast.AlterPlacementPolicyStmt) error
+ AddResourceGroup(ctx sessionctx.Context, stmt *ast.CreateResourceGroupStmt) error
+ AlterResourceGroup(ctx sessionctx.Context, stmt *ast.AlterResourceGroupStmt) error
+ DropResourceGroup(ctx sessionctx.Context, stmt *ast.DropResourceGroupStmt) error
FlashbackCluster(ctx sessionctx.Context, flashbackTS uint64) error
// CreateSchemaWithInfo creates a database (schema) given its database info.
@@ -252,10 +258,6 @@ type DDL interface {
GetInfoSchemaWithInterceptor(ctx sessionctx.Context) infoschema.InfoSchema
// DoDDLJob does the DDL job, it's exported for test.
DoDDLJob(ctx sessionctx.Context, job *model.Job) error
- // MoveJobFromQueue2Table move existing DDLs from queue to table.
- MoveJobFromQueue2Table(bool) error
- // MoveJobFromTable2Queue move existing DDLs from table to queue.
- MoveJobFromTable2Queue() error
}
type limitJobTask struct {
@@ -270,13 +272,14 @@ type ddl struct {
limitJobCh chan *limitJobTask
*ddlCtx
- workers map[workerType]*worker
sessPool *sessionPool
delRangeMgr delRangeManager
enableTiFlashPoll *atomicutil.Bool
// used in the concurrency ddl.
reorgWorkerPool *workerPool
generalDDLWorkerPool *workerPool
+ backfillCtxPool *backfillCtxPool
+ backfillWorkerPool *spmc.Pool[*reorgBackfillTask, *backfillResult, int, *backfillWorker, *backfillWorkerContext]
// get notification if any DDL coming.
ddlJobCh chan struct{}
}
@@ -332,6 +335,8 @@ type ddlCtx struct {
statsHandle *handle.Handle
tableLockCkr util.DeadTableLockChecker
etcdCli *clientv3.Client
+ // backfillJobCh gets notification if any backfill jobs coming.
+ backfillJobCh chan struct{}
*waitSchemaSyncedController
*schemaVersionManager
@@ -343,10 +348,11 @@ type ddlCtx struct {
// It holds the running DDL jobs ID.
runningJobIDs []string
// reorgCtx is used for reorganization.
- reorgCtx struct {
- sync.RWMutex
- // reorgCtxMap maps job ID to reorg context.
- reorgCtxMap map[int64]*reorgCtx
+ reorgCtx reorgContexts
+ // backfillCtx is used for backfill workers.
+ backfillCtx struct {
+ syncutil.RWMutex
+ jobCtxMap map[int64]*JobContext
}
jobCtx struct {
@@ -422,32 +428,32 @@ func (dc *ddlCtx) isOwner() bool {
return isOwner
}
-func (dc *ddlCtx) setDDLLabelForTopSQL(job *model.Job) {
+func (dc *ddlCtx) setDDLLabelForTopSQL(jobID int64, jobQuery string) {
dc.jobCtx.Lock()
defer dc.jobCtx.Unlock()
- ctx, exists := dc.jobCtx.jobCtxMap[job.ID]
+ ctx, exists := dc.jobCtx.jobCtxMap[jobID]
if !exists {
ctx = NewJobContext()
- dc.jobCtx.jobCtxMap[job.ID] = ctx
+ dc.jobCtx.jobCtxMap[jobID] = ctx
}
- ctx.setDDLLabelForTopSQL(job)
+ ctx.setDDLLabelForTopSQL(jobQuery)
}
-func (dc *ddlCtx) setDDLSourceForDiagnosis(job *model.Job) {
+func (dc *ddlCtx) setDDLSourceForDiagnosis(jobID int64, jobType model.ActionType) {
dc.jobCtx.Lock()
defer dc.jobCtx.Unlock()
- ctx, exists := dc.jobCtx.jobCtxMap[job.ID]
+ ctx, exists := dc.jobCtx.jobCtxMap[jobID]
if !exists {
ctx = NewJobContext()
- ctx.setDDLLabelForDiagnosis(job)
- dc.jobCtx.jobCtxMap[job.ID] = ctx
+ dc.jobCtx.jobCtxMap[jobID] = ctx
}
+ ctx.setDDLLabelForDiagnosis(jobType)
}
-func (dc *ddlCtx) getResourceGroupTaggerForTopSQL(job *model.Job) tikvrpc.ResourceGroupTagger {
+func (dc *ddlCtx) getResourceGroupTaggerForTopSQL(jobID int64) tikvrpc.ResourceGroupTagger {
dc.jobCtx.Lock()
defer dc.jobCtx.Unlock()
- ctx, exists := dc.jobCtx.jobCtxMap[job.ID]
+ ctx, exists := dc.jobCtx.jobCtxMap[jobID]
if !exists {
return nil
}
@@ -460,44 +466,105 @@ func (dc *ddlCtx) removeJobCtx(job *model.Job) {
delete(dc.jobCtx.jobCtxMap, job.ID)
}
-func (dc *ddlCtx) jobContext(job *model.Job) *JobContext {
+func (dc *ddlCtx) jobContext(jobID int64) *JobContext {
dc.jobCtx.RLock()
defer dc.jobCtx.RUnlock()
- if jobContext, exists := dc.jobCtx.jobCtxMap[job.ID]; exists {
+ if jobContext, exists := dc.jobCtx.jobCtxMap[jobID]; exists {
return jobContext
}
return NewJobContext()
}
-func (dc *ddlCtx) getReorgCtx(job *model.Job) *reorgCtx {
+func (dc *ddlCtx) removeBackfillCtxJobCtx(jobID int64) {
+ dc.backfillCtx.Lock()
+ delete(dc.backfillCtx.jobCtxMap, jobID)
+ dc.backfillCtx.Unlock()
+}
+
+func (dc *ddlCtx) backfillCtxJobIDs() []int64 {
+ dc.backfillCtx.Lock()
+ defer dc.backfillCtx.Unlock()
+
+ runningJobIDs := make([]int64, 0, len(dc.backfillCtx.jobCtxMap))
+ for id := range dc.backfillCtx.jobCtxMap {
+ runningJobIDs = append(runningJobIDs, id)
+ }
+ return runningJobIDs
+}
+
+func (dc *ddlCtx) setBackfillCtxJobContext(jobID int64, jobQuery string, jobType model.ActionType) (*JobContext, bool) {
+ dc.backfillCtx.Lock()
+ defer dc.backfillCtx.Unlock()
+
+ jobCtx, existent := dc.backfillCtx.jobCtxMap[jobID]
+ if !existent {
+ dc.setDDLLabelForTopSQL(jobID, jobQuery)
+ dc.setDDLSourceForDiagnosis(jobID, jobType)
+ jobCtx = dc.jobContext(jobID)
+ dc.backfillCtx.jobCtxMap[jobID] = jobCtx
+ }
+ return jobCtx, existent
+}
+
+type reorgContexts struct {
+ sync.RWMutex
+ // reorgCtxMap maps job ID to reorg context.
+ reorgCtxMap map[int64]*reorgCtx
+}
+
+func getReorgCtx(reorgCtxs *reorgContexts, jobID int64) *reorgCtx {
+ reorgCtxs.RLock()
+ defer reorgCtxs.RUnlock()
+ return reorgCtxs.reorgCtxMap[jobID]
+}
+
+// TODO: Using getReorgCtx instead of dc.getReorgCtx.
+func (dc *ddlCtx) getReorgCtx(jobID int64) *reorgCtx {
dc.reorgCtx.RLock()
defer dc.reorgCtx.RUnlock()
- return dc.reorgCtx.reorgCtxMap[job.ID]
+ return dc.reorgCtx.reorgCtxMap[jobID]
}
-func (dc *ddlCtx) newReorgCtx(r *reorgInfo) *reorgCtx {
+func (dc *ddlCtx) newReorgCtx(jobID int64, startKey []byte, currElement *meta.Element, rowCount int64) *reorgCtx {
rc := &reorgCtx{}
rc.doneCh = make(chan error, 1)
// initial reorgCtx
- rc.setRowCount(r.Job.GetRowCount())
- rc.setNextKey(r.StartKey)
- rc.setCurrentElement(r.currElement)
+ rc.setRowCount(rowCount)
+ rc.setNextKey(startKey)
+ rc.setCurrentElement(currElement)
rc.mu.warnings = make(map[errors.ErrorID]*terror.Error)
rc.mu.warningsCount = make(map[errors.ErrorID]int64)
+ rc.references.Add(1)
dc.reorgCtx.Lock()
defer dc.reorgCtx.Unlock()
- dc.reorgCtx.reorgCtxMap[r.Job.ID] = rc
+ dc.reorgCtx.reorgCtxMap[jobID] = rc
return rc
}
-func (dc *ddlCtx) removeReorgCtx(job *model.Job) {
+func (dc *ddlCtx) setReorgCtxForBackfill(bfJob *BackfillJob) {
+ rc := dc.getReorgCtx(bfJob.JobID)
+ if rc == nil {
+ ele := &meta.Element{ID: bfJob.EleID, TypeKey: bfJob.EleKey}
+ dc.newReorgCtx(bfJob.JobID, bfJob.Meta.StartKey, ele, bfJob.Meta.RowCount)
+ } else {
+ rc.references.Add(1)
+ }
+}
+
+func (dc *ddlCtx) removeReorgCtx(jobID int64) {
dc.reorgCtx.Lock()
defer dc.reorgCtx.Unlock()
- delete(dc.reorgCtx.reorgCtxMap, job.ID)
+ ctx, ok := dc.reorgCtx.reorgCtxMap[jobID]
+ if ok {
+ ctx.references.Sub(1)
+ if ctx.references.Load() == 0 {
+ delete(dc.reorgCtx.reorgCtxMap, jobID)
+ }
+ }
}
func (dc *ddlCtx) notifyReorgCancel(job *model.Job) {
- rc := dc.getReorgCtx(job)
+ rc := dc.getReorgCtx(job.ID)
if rc == nil {
return
}
@@ -624,7 +691,6 @@ func newDDL(ctx context.Context, options ...Option) *ddl {
// Register functions for enable/disable ddl when changing system variable `tidb_enable_ddl`.
variable.EnableDDL = d.EnableDDL
variable.DisableDDL = d.DisableDDL
- variable.SwitchConcurrentDDL = d.SwitchConcurrentDDL
variable.SwitchMDL = d.SwitchMDL
return d
@@ -656,7 +722,7 @@ func (d *ddl) newDeleteRangeManager(mock bool) delRangeManager {
func (d *ddl) prepareWorkers4ConcurrencyDDL() {
workerFactory := func(tp workerType) func() (pools.Resource, error) {
return func() (pools.Resource, error) {
- wk := newWorker(d.ctx, tp, d.sessPool, d.delRangeMgr, d.ddlCtx, true)
+ wk := newWorker(d.ctx, tp, d.sessPool, d.delRangeMgr, d.ddlCtx)
sessForJob, err := d.sessPool.get()
if err != nil {
return nil, err
@@ -679,21 +745,22 @@ func (d *ddl) prepareWorkers4ConcurrencyDDL() {
d.wg.Run(d.startDispatchLoop)
}
-func (d *ddl) prepareWorkers4legacyDDL() {
- d.workers = make(map[workerType]*worker, 2)
- d.workers[generalWorker] = newWorker(d.ctx, generalWorker, d.sessPool, d.delRangeMgr, d.ddlCtx, false)
- d.workers[addIdxWorker] = newWorker(d.ctx, addIdxWorker, d.sessPool, d.delRangeMgr, d.ddlCtx, false)
- for _, worker := range d.workers {
- worker.wg.Add(1)
- w := worker
- go w.start(d.ddlCtx)
-
- metrics.DDLCounter.WithLabelValues(fmt.Sprintf("%s_%s", metrics.CreateDDL, worker.String())).Inc()
-
- // When the start function is called, we will send a fake job to let worker
- // checks owner firstly and try to find whether a job exists and run.
- asyncNotify(worker.ddlJobCh)
+func (d *ddl) prepareBackfillWorkers() error {
+ workerFactory := func() (pools.Resource, error) {
+ bk := newBackfillWorker(context.Background(), nil)
+ metrics.DDLCounter.WithLabelValues(fmt.Sprintf("%s_backfill_worker", metrics.CreateDDL)).Inc()
+ return bk, nil
+ }
+ d.backfillCtxPool = newBackfillContextPool(pools.NewResourcePool(workerFactory, backfillWorkerCnt, backfillWorkerCnt, 0))
+ var err error
+ d.backfillWorkerPool, err = spmc.NewSPMCPool[*reorgBackfillTask, *backfillResult, int, *backfillWorker,
+ *backfillWorkerContext]("backfill", int32(backfillWorkerCnt), rmutil.DDL)
+ if err != nil {
+ return err
}
+ d.backfillJobCh = make(chan struct{}, 1)
+ d.wg.Run(d.startDispatchBackfillJobsLoop)
+ return nil
}
// Start implements DDL.Start interface.
@@ -713,19 +780,23 @@ func (d *ddl) Start(ctxPool *pools.ResourcePool) error {
d.delRangeMgr = d.newDeleteRangeManager(ctxPool == nil)
d.prepareWorkers4ConcurrencyDDL()
- d.prepareWorkers4legacyDDL()
if config.TableLockEnabled() {
d.wg.Add(1)
go d.startCleanDeadTableLock()
}
- // If tidb_enable_ddl is true, we need campaign owner and do DDL job.
+ // If tidb_enable_ddl is true, we need campaign owner and do DDL jobs. Besides, we also can do backfill jobs.
// Otherwise, we needn't do that.
if config.GetGlobalConfig().Instance.TiDBEnableDDL.Load() {
if err := d.EnableDDL(); err != nil {
return err
}
+
+ // TODO: Currently, it is only processed during initialization and is expected to be added to EnableDDL later.
+ if err := d.prepareBackfillWorkers(); err != nil {
+ return err
+ }
}
variable.RegisterStatistics(d)
@@ -798,10 +869,13 @@ func (d *ddl) close() {
if d.generalDDLWorkerPool != nil {
d.generalDDLWorkerPool.close()
}
-
- for _, worker := range d.workers {
- worker.Close()
+ if d.backfillCtxPool != nil {
+ d.backfillCtxPool.close()
+ }
+ if d.backfillWorkerPool != nil {
+ d.backfillWorkerPool.ReleaseAndWait()
}
+
// d.delRangeMgr using sessions from d.sessPool.
// Put it before d.sessPool.close to reduce the time spent by d.sessPool.close.
if d.delRangeMgr != nil {
@@ -907,7 +981,8 @@ func getIntervalFromPolicy(policy []time.Duration, i int) (time.Duration, bool)
func getJobCheckInterval(job *model.Job, i int) (time.Duration, bool) {
switch job.Type {
- case model.ActionAddIndex, model.ActionAddPrimaryKey, model.ActionModifyColumn:
+ case model.ActionAddIndex, model.ActionAddPrimaryKey, model.ActionModifyColumn,
+ model.ActionReorganizePartition:
return getIntervalFromPolicy(slowDDLIntervalPolicy, i)
case model.ActionCreateTable, model.ActionCreateSchema:
return getIntervalFromPolicy(fastDDLIntervalPolicy, i)
@@ -916,29 +991,16 @@ func getJobCheckInterval(job *model.Job, i int) (time.Duration, bool) {
}
}
-func (d *ddl) asyncNotifyWorker(job *model.Job) {
+func (dc *ddlCtx) asyncNotifyWorker(ch chan struct{}, etcdPath string, jobID int64, jobType string) {
// If the workers don't run, we needn't notify workers.
+ // TODO: It does not affect informing the backfill worker.
if !config.GetGlobalConfig().Instance.TiDBEnableDDL.Load() {
return
}
- if variable.EnableConcurrentDDL.Load() {
- if d.isOwner() {
- asyncNotify(d.ddlJobCh)
- } else {
- d.asyncNotifyByEtcd(addingDDLJobConcurrent, job)
- }
+ if dc.isOwner() {
+ asyncNotify(ch)
} else {
- var worker *worker
- if job.MayNeedReorg() {
- worker = d.workers[addIdxWorker]
- } else {
- worker = d.workers[generalWorker]
- }
- if d.ownerManager.IsOwner() {
- asyncNotify(worker.ddlJobCh)
- } else {
- d.asyncNotifyByEtcd(worker.addingDDLJobKey, job)
- }
+ dc.asyncNotifyByEtcd(etcdPath, jobID, jobType)
}
}
@@ -979,7 +1041,6 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
// Instead, we merge all the jobs into one pending job.
return appendToSubJobs(mci, job)
}
-
// Get a global job ID and put the DDL job in the queue.
setDDLJobQuery(ctx, job)
task := &limitJobTask{job, make(chan error)}
@@ -1008,7 +1069,7 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
sessVars.StmtCtx.IsDDLJobInQueue = true
// Notice worker that we push a new job and wait the job done.
- d.asyncNotifyWorker(job)
+ d.asyncNotifyWorker(d.ddlJobCh, addingDDLJobConcurrent, job.ID, job.Type.String())
logutil.BgLogger().Info("[ddl] start DDL job", zap.String("job", job.String()), zap.String("query", job.Query))
var historyJob *model.Job
@@ -1056,7 +1117,7 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
continue
}
sessVars.StmtCtx.DDLJobID = 0 // Avoid repeat.
- errs, err := CancelJobs(se, d.store, []int64{jobID})
+ errs, err := CancelJobs(se, []int64{jobID})
d.sessPool.put(se)
if len(errs) > 0 {
logutil.BgLogger().Warn("error canceling DDL job", zap.Error(errs[0]))
@@ -1094,8 +1155,13 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
logutil.BgLogger().Info("[ddl] DDL warnings doesn't match the warnings count", zap.Int64("jobID", jobID))
} else {
for key, warning := range historyJob.ReorgMeta.Warnings {
- for j := int64(0); j < historyJob.ReorgMeta.WarningsCount[key]; j++ {
+ keyCount := historyJob.ReorgMeta.WarningsCount[key]
+ if keyCount == 1 {
ctx.GetSessionVars().StmtCtx.AppendWarning(warning)
+ } else {
+ newMsg := fmt.Sprintf("%d warnings with this error code, first warning: "+warning.GetMsg(), keyCount)
+ newWarning := dbterror.ClassTypes.Synthesize(terror.ErrCode(warning.Code()), newMsg)
+ ctx.GetSessionVars().StmtCtx.AppendWarning(newWarning)
}
}
}
@@ -1178,49 +1244,12 @@ func (d *ddl) startCleanDeadTableLock() {
}
}
-// SwitchConcurrentDDL changes the DDL to concurrent DDL if toConcurrentDDL is true, otherwise, queue based DDL.
-func (d *ddl) SwitchConcurrentDDL(toConcurrentDDL bool) error {
- if !d.isOwner() {
- return kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), d.store, true, func(ctx context.Context, txn kv.Transaction) error {
- isConcurrentDDL, err := meta.NewMeta(txn).IsConcurrentDDL()
- if err != nil {
- return err
- }
- if isConcurrentDDL != toConcurrentDDL {
- return errors.New("please set it on the DDL owner node")
- }
- return nil
- })
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
- defer cancel()
- d.waiting.Store(true)
- defer d.waiting.Store(false)
- if err := d.wait4Switch(ctx); err != nil {
- return err
- }
-
- var err error
- if toConcurrentDDL {
- err = d.MoveJobFromQueue2Table(false)
- } else {
- err = d.MoveJobFromTable2Queue()
- }
- if err == nil {
- variable.EnableConcurrentDDL.Store(toConcurrentDDL)
- }
- logutil.BgLogger().Info("[ddl] SwitchConcurrentDDL", zap.Bool("toConcurrentDDL", toConcurrentDDL), zap.Error(err))
- return err
-}
-
// SwitchMDL enables MDL or disable DDL.
func (d *ddl) SwitchMDL(enable bool) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
- // Disable MDL for test.
- if enable && !variable.DefTiDBEnableConcurrentDDL {
+ if enable {
sql := fmt.Sprintf("UPDATE HIGH_PRIORITY %[1]s.%[2]s SET VARIABLE_VALUE = %[4]d WHERE VARIABLE_NAME = '%[3]s'",
mysql.SystemDB, mysql.GlobalVariablesTable, variable.TiDBEnableMDL, 0)
sess, err := d.sessPool.get()
@@ -1259,25 +1288,23 @@ func (d *ddl) SwitchMDL(enable bool) error {
}
variable.EnableMDL.Store(enable)
- logutil.BgLogger().Info("[ddl] switch metadata lock feature", zap.Bool("enable", enable), zap.Error(err))
- return nil
-}
-
-func (d *ddl) wait4Switch(ctx context.Context) error {
- for {
- select {
- case <-ctx.Done():
- return ctx.Err()
- default:
+ err = kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), d.store, true, func(ctx context.Context, txn kv.Transaction) error {
+ m := meta.NewMeta(txn)
+ oldEnable, _, err := m.GetMetadataLock()
+ if err != nil {
+ return err
}
- d.runningJobs.RLock()
- if len(d.runningJobs.ids) == 0 {
- d.runningJobs.RUnlock()
- return nil
+ if oldEnable != enable {
+ err = m.SetMetadataLock(enable)
}
- d.runningJobs.RUnlock()
- time.Sleep(time.Second * 1)
+ return err
+ })
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] switch metadata lock feature", zap.Bool("enable", enable), zap.Error(err))
+ return err
}
+ logutil.BgLogger().Info("[ddl] switch metadata lock feature", zap.Bool("enable", enable))
+ return nil
}
// RecoverInfo contains information needed by DDL.RecoverTable.
@@ -1304,6 +1331,12 @@ type RecoverSchemaInfo struct {
// It should be called before any DDL that could break data consistency.
// This provides a safe window for async commit and 1PC to commit with an old schema.
func delayForAsyncCommit() {
+ if variable.EnableMDL.Load() {
+ // If metadata lock is enabled. The transaction of DDL must begin after prewrite of the async commit transaction,
+ // then the commit ts of DDL must be greater than the async commit transaction. In this case, the corresponding schema of the async commit transaction
+ // is correct. But if metadata lock is disabled, we can't ensure that the corresponding schema of the async commit transaction isn't change.
+ return
+ }
cfg := config.GetGlobalConfig().TiKVClient.AsyncCommit
duration := cfg.SafeWindow + cfg.AllowedClockDrift
logutil.BgLogger().Info("sleep before DDL finishes to make async commit and 1PC safe",
@@ -1316,12 +1349,6 @@ var (
RunInGoTest bool
)
-func init() {
- if flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil {
- RunInGoTest = true
- }
-}
-
// GetDropOrTruncateTableInfoFromJobsByStore implements GetDropOrTruncateTableInfoFromJobs
func GetDropOrTruncateTableInfoFromJobsByStore(jobs []*model.Job, gcSafePoint uint64, getTable func(uint64, int64, int64) (*model.TableInfo, error), fn func(*model.Job, *model.TableInfo) (bool, error)) (bool, error) {
for _, job := range jobs {
@@ -1388,13 +1415,8 @@ func GetDDLInfo(s sessionctx.Context) (*Info, error) {
}
t := meta.NewMeta(txn)
info.Jobs = make([]*model.Job, 0, 2)
- enable := variable.EnableConcurrentDDL.Load()
var generalJob, reorgJob *model.Job
- if enable {
- generalJob, reorgJob, err = get2JobsFromTable(sess)
- } else {
- generalJob, reorgJob, err = get2JobsFromQueue(t)
- }
+ generalJob, reorgJob, err = get2JobsFromTable(sess)
if err != nil {
return nil, errors.Trace(err)
}
@@ -1415,7 +1437,7 @@ func GetDDLInfo(s sessionctx.Context) (*Info, error) {
return info, nil
}
- _, info.ReorgHandle, _, _, err = newReorgHandler(t, sess, enable).GetDDLReorgHandle(reorgJob)
+ _, info.ReorgHandle, _, _, err = newReorgHandler(sess).GetDDLReorgHandle(reorgJob)
if err != nil {
if meta.ErrDDLReorgElementNotExist.Equal(err) {
return info, nil
@@ -1426,19 +1448,6 @@ func GetDDLInfo(s sessionctx.Context) (*Info, error) {
return info, nil
}
-func get2JobsFromQueue(t *meta.Meta) (*model.Job, *model.Job, error) {
- generalJob, err := t.GetDDLJobByIdx(0)
- if err != nil {
- return nil, nil, errors.Trace(err)
- }
- reorgJob, err := t.GetDDLJobByIdx(0, meta.AddIndexJobListKey)
- if err != nil {
- return nil, nil, errors.Trace(err)
- }
-
- return generalJob, reorgJob, nil
-}
-
func get2JobsFromTable(sess *session) (*model.Job, *model.Job, error) {
var generalJob, reorgJob *model.Job
jobs, err := getJobsBySQL(sess, JobTable, "not reorg order by job_id limit 1")
@@ -1460,82 +1469,8 @@ func get2JobsFromTable(sess *session) (*model.Job, *model.Job, error) {
}
// CancelJobs cancels the DDL jobs.
-func CancelJobs(se sessionctx.Context, store kv.Storage, ids []int64) (errs []error, err error) {
- if variable.EnableConcurrentDDL.Load() {
- return cancelConcurrencyJobs(se, ids)
- }
-
- err = kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), store, true, func(ctx context.Context, txn kv.Transaction) error {
- errs, err = cancelLegacyJobs(txn, ids)
- return err
- })
- return
-}
-
-func cancelLegacyJobs(txn kv.Transaction, ids []int64) ([]error, error) {
- if len(ids) == 0 {
- return nil, nil
- }
-
- errs := make([]error, len(ids))
- t := meta.NewMeta(txn)
- generalJobs, err := getDDLJobsInQueue(t, meta.DefaultJobListKey)
- if err != nil {
- return nil, errors.Trace(err)
- }
- addIdxJobs, err := getDDLJobsInQueue(t, meta.AddIndexJobListKey)
- if err != nil {
- return nil, errors.Trace(err)
- }
- jobs := append(generalJobs, addIdxJobs...)
- jobsMap := make(map[int64]int)
- for i, id := range ids {
- jobsMap[id] = i
- }
- for j, job := range jobs {
- i, ok := jobsMap[job.ID]
- if !ok {
- logutil.BgLogger().Debug("the job that needs to be canceled isn't equal to current job",
- zap.Int64("need to canceled job ID", job.ID),
- zap.Int64("current job ID", job.ID))
- continue
- }
- delete(jobsMap, job.ID)
- // These states can't be cancelled.
- if job.IsDone() || job.IsSynced() {
- errs[i] = dbterror.ErrCancelFinishedDDLJob.GenWithStackByArgs(job.ID)
- continue
- }
- // If the state is rolling back, it means the work is cleaning the data after cancelling the job.
- if job.IsCancelled() || job.IsRollingback() || job.IsRollbackDone() {
- continue
- }
- if !job.IsRollbackable() {
- errs[i] = dbterror.ErrCannotCancelDDLJob.GenWithStackByArgs(job.ID)
- continue
- }
-
- job.State = model.JobStateCancelling
- // Make sure RawArgs isn't overwritten.
- err := json.Unmarshal(job.RawArgs, &job.Args)
- if err != nil {
- errs[i] = errors.Trace(err)
- continue
- }
- if j >= len(generalJobs) {
- offset := int64(j - len(generalJobs))
- err = t.UpdateDDLJob(offset, job, true, meta.AddIndexJobListKey)
- } else {
- err = t.UpdateDDLJob(int64(j), job, true)
- }
- if err != nil {
- errs[i] = errors.Trace(err)
- }
- }
- for id, i := range jobsMap {
- errs[i] = dbterror.ErrDDLJobNotFound.GenWithStackByArgs(id)
- }
- return errs, nil
+func CancelJobs(se sessionctx.Context, ids []int64) (errs []error, err error) {
+ return cancelConcurrencyJobs(se, ids)
}
// cancelConcurrencyJobs cancels the DDL jobs that are in the concurrent state.
@@ -1614,45 +1549,9 @@ func cancelConcurrencyJobs(se sessionctx.Context, ids []int64) ([]error, error)
return errs, nil
}
-func getDDLJobsInQueue(t *meta.Meta, jobListKey meta.JobListKeyType) ([]*model.Job, error) {
- cnt, err := t.DDLJobQueueLen(jobListKey)
- if err != nil {
- return nil, errors.Trace(err)
- }
- jobs := make([]*model.Job, cnt)
- for i := range jobs {
- jobs[i], err = t.GetDDLJobByIdx(int64(i), jobListKey)
- if err != nil {
- return nil, errors.Trace(err)
- }
- }
- return jobs, nil
-}
-
// GetAllDDLJobs get all DDL jobs and sorts jobs by job.ID.
func GetAllDDLJobs(sess sessionctx.Context, t *meta.Meta) ([]*model.Job, error) {
- if variable.EnableConcurrentDDL.Load() {
- return getJobsBySQL(newSession(sess), JobTable, "1 order by job_id")
- }
-
- return getDDLJobs(t)
-}
-
-// getDDLJobs get all DDL jobs and sorts jobs by job.ID.
-func getDDLJobs(t *meta.Meta) ([]*model.Job, error) {
- generalJobs, err := getDDLJobsInQueue(t, meta.DefaultJobListKey)
- if err != nil {
- return nil, errors.Trace(err)
- }
- addIdxJobs, err := getDDLJobsInQueue(t, meta.AddIndexJobListKey)
- if err != nil {
- return nil, errors.Trace(err)
- }
- jobs := append(generalJobs, addIdxJobs...)
- slices.SortFunc(jobs, func(i, j *model.Job) bool {
- return i.ID < j.ID
- })
- return jobs, nil
+ return getJobsBySQL(newSession(sess), JobTable, "1 order by job_id")
}
// MaxHistoryJobs is exported for testing.
@@ -1732,7 +1631,7 @@ func (s *session) begin() error {
}
func (s *session) commit() error {
- s.StmtCommit()
+ s.StmtCommit(context.Background())
return s.CommitTxn(context.Background())
}
@@ -1741,12 +1640,12 @@ func (s *session) txn() (kv.Transaction, error) {
}
func (s *session) rollback() {
- s.StmtRollback()
+ s.StmtRollback(context.Background(), false)
s.RollbackTxn(context.Background())
}
func (s *session) reset() {
- s.StmtRollback()
+ s.StmtRollback(context.Background(), false)
}
func (s *session) execute(ctx context.Context, query string, label string) ([]chunk.Row, error) {
@@ -1755,7 +1654,11 @@ func (s *session) execute(ctx context.Context, query string, label string) ([]ch
defer func() {
metrics.DDLJobTableDuration.WithLabelValues(label + "-" + metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
}()
- rs, err := s.Context.(sqlexec.SQLExecutor).ExecuteInternal(kv.WithInternalSourceType(ctx, kv.InternalTxnDDL), query)
+
+ if ctx.Value(kv.RequestSourceKey) == nil {
+ ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnDDL)
+ }
+ rs, err := s.Context.(sqlexec.SQLExecutor).ExecuteInternal(ctx, query)
if err != nil {
return nil, errors.Trace(err)
}
@@ -1775,6 +1678,31 @@ func (s *session) session() sessionctx.Context {
return s.Context
}
+func (s *session) runInTxn(f func(*session) error) (err error) {
+ err = s.begin()
+ if err != nil {
+ return err
+ }
+ failpoint.Inject("NotifyBeginTxnCh", func(val failpoint.Value) {
+ //nolint:forcetypeassert
+ v := val.(int)
+ if v == 1 {
+ mockDDLErrOnce = 1
+ TestNotifyBeginTxnCh <- struct{}{}
+ } else if v == 2 && mockDDLErrOnce == 1 {
+ <-TestNotifyBeginTxnCh
+ mockDDLErrOnce = 0
+ }
+ })
+
+ err = f(s)
+ if err != nil {
+ s.rollback()
+ return
+ }
+ return errors.Trace(s.commit())
+}
+
// GetAllHistoryDDLJobs get all the done DDL jobs.
func GetAllHistoryDDLJobs(m *meta.Meta) ([]*model.Job, error) {
iterator, err := GetLastHistoryDDLJobsIterator(m)
@@ -1838,19 +1766,11 @@ func GetHistoryJobByID(sess sessionctx.Context, id int64) (*model.Job, error) {
return job, errors.Trace(err)
}
-// AddHistoryDDLJobForTest used for test.
-func AddHistoryDDLJobForTest(sess sessionctx.Context, t *meta.Meta, job *model.Job, updateRawArgs bool) error {
- return AddHistoryDDLJob(newSession(sess), t, job, updateRawArgs, variable.EnableConcurrentDDL.Load())
-}
-
// AddHistoryDDLJob record the history job.
-func AddHistoryDDLJob(sess *session, t *meta.Meta, job *model.Job, updateRawArgs bool, concurrentDDL bool) error {
- if concurrentDDL {
- // only add history job into table if it is concurrent DDL.
- err := addHistoryDDLJob2Table(sess, job, updateRawArgs)
- if err != nil {
- logutil.BgLogger().Info("[ddl] failed to add DDL job to history table", zap.Error(err))
- }
+func AddHistoryDDLJob(sess *session, t *meta.Meta, job *model.Job, updateRawArgs bool) error {
+ err := addHistoryDDLJob2Table(sess, job, updateRawArgs)
+ if err != nil {
+ logutil.BgLogger().Info("[ddl] failed to add DDL job to history table", zap.Error(err))
}
// we always add history DDL job to job list at this moment.
return t.AddHistoryDDLJob(job, updateRawArgs)
diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go
index bbf3d4fc59856..cc2a0f8d2f5fc 100644
--- a/ddl/ddl_api.go
+++ b/ddl/ddl_api.go
@@ -33,6 +33,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/ddl/label"
+ "github.com/pingcap/tidb/ddl/resourcegroup"
ddlutil "github.com/pingcap/tidb/ddl/util"
"github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/infoschema"
@@ -46,6 +47,7 @@ import (
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/terror"
field_types "github.com/pingcap/tidb/parser/types"
+ "github.com/pingcap/tidb/privilege"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/sessiontxn"
@@ -946,7 +948,7 @@ func checkColumnDefaultValue(ctx sessionctx.Context, col *table.Column, value in
if value != nil && ctx.GetSessionVars().SQLMode.HasNoZeroDateMode() &&
ctx.GetSessionVars().SQLMode.HasStrictMode() && types.IsTypeTime(col.GetType()) {
if vv, ok := value.(string); ok {
- timeValue, err := expression.GetTimeValue(ctx, vv, col.GetType(), col.GetDecimal())
+ timeValue, err := expression.GetTimeValue(ctx, vv, col.GetType(), col.GetDecimal(), nil)
if err != nil {
return hasDefaultValue, value, errors.Trace(err)
}
@@ -971,7 +973,7 @@ func convertTimestampDefaultValToUTC(ctx sessionctx.Context, defaultVal interfac
}
if vv, ok := defaultVal.(string); ok {
if vv != types.ZeroDatetimeStr && !strings.EqualFold(vv, ast.CurrentTimestamp) {
- t, err := types.ParseTime(ctx.GetSessionVars().StmtCtx, vv, col.GetType(), col.GetDecimal())
+ t, err := types.ParseTime(ctx.GetSessionVars().StmtCtx, vv, col.GetType(), col.GetDecimal(), nil)
if err != nil {
return defaultVal, errors.Trace(err)
}
@@ -1082,7 +1084,7 @@ func columnDefToCol(ctx sessionctx.Context, offset int, colDef *ast.ColumnDef, o
var sb strings.Builder
restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
- format.RestoreSpacesAroundBinaryOperation
+ format.RestoreSpacesAroundBinaryOperation | format.RestoreWithoutSchemaName | format.RestoreWithoutTableName
restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
for _, v := range colDef.Options {
@@ -1142,7 +1144,10 @@ func columnDefToCol(ctx sessionctx.Context, offset int, colDef *ast.ColumnDef, o
}
col.GeneratedExprString = sb.String()
col.GeneratedStored = v.Stored
- _, dependColNames := findDependedColumnNames(colDef)
+ _, dependColNames, err := findDependedColumnNames(model.NewCIStr(""), model.NewCIStr(""), colDef)
+ if err != nil {
+ return nil, nil, errors.Trace(err)
+ }
col.Dependences = dependColNames
case ast.ColumnOptionCollate:
if field_types.HasCharset(colDef.Tp) {
@@ -1165,7 +1170,7 @@ func columnDefToCol(ctx sessionctx.Context, offset int, colDef *ast.ColumnDef, o
// getFuncCallDefaultValue gets the default column value of function-call expression.
func getFuncCallDefaultValue(col *table.Column, option *ast.ColumnOption, expr *ast.FuncCallExpr) (interface{}, bool, error) {
switch expr.FnName.L {
- case ast.CurrentTimestamp:
+ case ast.CurrentTimestamp, ast.CurrentDate:
tp, fsp := col.FieldType.GetType(), col.FieldType.GetDecimal()
if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime {
defaultFsp := 0
@@ -1218,8 +1223,8 @@ func getDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.Colu
// If the function call is ast.CurrentTimestamp, it needs to be continuously processed.
}
- if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime {
- vd, err := expression.GetTimeValue(ctx, option.Expr, tp, fsp)
+ if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime || tp == mysql.TypeDate {
+ vd, err := expression.GetTimeValue(ctx, option.Expr, tp, fsp, nil)
value := vd.GetValue()
if err != nil {
return nil, false, dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
@@ -1561,7 +1566,7 @@ func IsAutoRandomColumnID(tblInfo *model.TableInfo, colID int64) bool {
return false
}
-func checkGeneratedColumn(ctx sessionctx.Context, colDefs []*ast.ColumnDef) error {
+func checkGeneratedColumn(ctx sessionctx.Context, schemaName model.CIStr, tableName model.CIStr, colDefs []*ast.ColumnDef) error {
var colName2Generation = make(map[string]columnGenerationInDDL, len(colDefs))
var exists bool
var autoIncrementColumn string
@@ -1576,7 +1581,10 @@ func checkGeneratedColumn(ctx sessionctx.Context, colDefs []*ast.ColumnDef) erro
if containsColumnOption(colDef, ast.ColumnOptionAutoIncrement) {
exists, autoIncrementColumn = true, colDef.Name.Name.L
}
- generated, depCols := findDependedColumnNames(colDef)
+ generated, depCols, err := findDependedColumnNames(schemaName, tableName, colDef)
+ if err != nil {
+ return errors.Trace(err)
+ }
if !generated {
colName2Generation[colDef.Name.Name.L] = columnGenerationInDDL{
position: i,
@@ -1693,7 +1701,7 @@ func setEmptyConstraintName(namesMap map[string]bool, constr *ast.Constraint) {
}
}
if colName == "" {
- colName = constr.Keys[0].Column.Name.L
+ colName = constr.Keys[0].Column.Name.O
}
constrName := colName
i := 2
@@ -1980,7 +1988,7 @@ func addIndexForForeignKey(ctx sessionctx.Context, tbInfo *model.TableInfo) erro
if handleCol != nil && len(fk.Cols) == 1 && handleCol.Name.L == fk.Cols[0].L {
continue
}
- if model.FindIndexByColumns(tbInfo, fk.Cols...) != nil {
+ if model.FindIndexByColumns(tbInfo, tbInfo.Indices, fk.Cols...) != nil {
continue
}
idxName := fk.Name
@@ -2094,7 +2102,7 @@ func CheckTableInfoValidWithStmt(ctx sessionctx.Context, tbInfo *model.TableInfo
func checkTableInfoValidWithStmt(ctx sessionctx.Context, tbInfo *model.TableInfo, s *ast.CreateTableStmt) (err error) {
// All of these rely on the AST structure of expressions, which were
// lost in the model (got serialized into strings).
- if err := checkGeneratedColumn(ctx, s.Cols); err != nil {
+ if err := checkGeneratedColumn(ctx, s.Table.Schema, tbInfo.Name, s.Cols); err != nil {
return errors.Trace(err)
}
@@ -2115,6 +2123,11 @@ func checkTableInfoValidWithStmt(ctx sessionctx.Context, tbInfo *model.TableInfo
}
}
}
+ if tbInfo.TTLInfo != nil {
+ if err := checkTTLInfoValid(ctx, s.Table.Schema, tbInfo); err != nil {
+ return errors.Trace(err)
+ }
+ }
return nil
}
@@ -2147,7 +2160,7 @@ func checkPartitionDefinitionConstraints(ctx sessionctx.Context, tbInfo *model.T
// checkTableInfoValid uses to check table info valid. This is used to validate table info.
func checkTableInfoValid(tblInfo *model.TableInfo) error {
- _, err := tables.TableFromMeta(nil, tblInfo)
+ _, err := tables.TableFromMeta(autoid.NewAllocators(false), tblInfo)
if err != nil {
return err
}
@@ -2193,6 +2206,10 @@ func BuildTableInfoWithLike(ctx sessionctx.Context, ident ast.Ident, referTblInf
copy(pi.Definitions, referTblInfo.Partition.Definitions)
tblInfo.Partition = &pi
}
+
+ if referTblInfo.TTLInfo != nil {
+ tblInfo.TTLInfo = referTblInfo.TTLInfo.Clone()
+ }
return &tblInfo, nil
}
@@ -2484,7 +2501,13 @@ func (d *ddl) createTableWithInfoPost(
// Default tableAutoIncID base is 0.
// If the first ID is expected to greater than 1, we need to do rebase.
newEnd := tbInfo.AutoIncID - 1
- if err = d.handleAutoIncID(tbInfo, schemaID, newEnd, autoid.RowIDAllocType); err != nil {
+ var allocType autoid.AllocatorType
+ if tbInfo.SepAutoInc() {
+ allocType = autoid.AutoIncrementType
+ } else {
+ allocType = autoid.RowIDAllocType
+ }
+ if err = d.handleAutoIncID(tbInfo, schemaID, newEnd, allocType); err != nil {
return errors.Trace(err)
}
}
@@ -2709,6 +2732,14 @@ func (d *ddl) preSplitAndScatter(ctx sessionctx.Context, tbInfo *model.TableInfo
func (d *ddl) FlashbackCluster(ctx sessionctx.Context, flashbackTS uint64) error {
logutil.BgLogger().Info("[ddl] get flashback cluster job", zap.String("flashbackTS", oracle.GetTimeFromTS(flashbackTS).String()))
+ nowTS, err := ctx.GetStore().GetOracle().GetTimestamp(d.ctx, &oracle.Option{})
+ if err != nil {
+ return errors.Trace(err)
+ }
+ gap := time.Until(oracle.GetTimeFromTS(nowTS)).Abs()
+ if gap > 1*time.Second {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("Gap between local time and PD TSO is %s, please check PD/system time", gap))
+ }
job := &model.Job{
Type: model.ActionFlashbackCluster,
BinlogInfo: &model.HistoryInfo{},
@@ -2720,9 +2751,12 @@ func (d *ddl) FlashbackCluster(ctx sessionctx.Context, flashbackTS uint64) error
variable.On, /* tidb_enable_auto_analyze */
variable.Off, /* tidb_super_read_only */
0, /* totalRegions */
- 0 /* newCommitTS */},
+ 0, /* startTS */
+ 0, /* commitTS */
+ variable.On, /* tidb_ttl_job_enable */
+ []kv.KeyRange{} /* flashback key_ranges */},
}
- err := d.DoDDLJob(ctx, job)
+ err = d.DoDDLJob(ctx, job)
err = d.callHookOnChanged(job, err)
return errors.Trace(err)
}
@@ -2992,14 +3026,45 @@ func SetDirectPlacementOpt(placementSettings *model.PlacementSettings, placement
placementSettings.FollowerConstraints = stringVal
case ast.PlacementOptionVoterConstraints:
placementSettings.VoterConstraints = stringVal
+ case ast.PlacementOptionSurvivalPreferences:
+ placementSettings.SurvivalPreferences = stringVal
default:
return errors.Trace(errors.New("unknown placement policy option"))
}
return nil
}
+// SetDirectResourceGroupUnit tries to set the ResourceGroupSettings.
+func SetDirectResourceGroupUnit(resourceGroupSettings *model.ResourceGroupSettings, typ ast.ResourceUnitType, stringVal string, uintVal uint64, boolValue bool) error {
+ switch typ {
+ case ast.ResourceRURate:
+ resourceGroupSettings.RURate = uintVal
+ case ast.ResourceUnitCPU:
+ resourceGroupSettings.CPULimiter = stringVal
+ case ast.ResourceUnitIOReadBandwidth:
+ resourceGroupSettings.IOReadBandwidth = stringVal
+ case ast.ResourceUnitIOWriteBandwidth:
+ resourceGroupSettings.IOWriteBandwidth = stringVal
+ case ast.ResourceBurstableOpiton:
+ // Some about BurstLimit(b):
+ // - If b == 0, that means the limiter is unlimited capacity. default use in resource controller (burst with a rate within a unlimited capacity).
+ // - If b < 0, that means the limiter is unlimited capacity and fillrate(r) is ignored, can be seen as r == Inf (burst with a inf rate within a unlimited capacity).
+ // - If b > 0, that means the limiter is limited capacity. (current not used).
+ limit := int64(0)
+ if boolValue {
+ limit = -1
+ }
+ resourceGroupSettings.BurstLimit = limit
+ default:
+ return errors.Trace(errors.New("unknown resource unit type"))
+ }
+ return nil
+}
+
// handleTableOptions updates tableInfo according to table options.
func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) error {
+ var ttlOptionsHandled bool
+
for _, op := range options {
switch op.Tp {
case ast.TableOptionAutoIncrement:
@@ -3036,6 +3101,28 @@ func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) err
tbInfo.PlacementPolicyRef = &model.PolicyRefInfo{
Name: model.NewCIStr(op.StrValue),
}
+ case ast.TableOptionTTL, ast.TableOptionTTLEnable, ast.TableOptionTTLJobInterval:
+ if ttlOptionsHandled {
+ continue
+ }
+
+ ttlInfo, ttlEnable, ttlJobInterval, err := getTTLInfoInOptions(options)
+ if err != nil {
+ return err
+ }
+ // It's impossible that `ttlInfo` and `ttlEnable` are all nil, because we have met this option.
+ // After exclude the situation `ttlInfo == nil && ttlEnable != nil`, we could say `ttlInfo != nil`
+ if ttlInfo == nil {
+ if ttlEnable != nil {
+ return errors.Trace(dbterror.ErrSetTTLOptionForNonTTLTable.FastGenByArgs("TTL_ENABLE"))
+ }
+ if ttlJobInterval != nil {
+ return errors.Trace(dbterror.ErrSetTTLOptionForNonTTLTable.FastGenByArgs("TTL_JOB_INTERVAL"))
+ }
+ }
+
+ tbInfo.TTLInfo = ttlInfo
+ ttlOptionsHandled = true
}
}
shardingBits := shardingBits(tbInfo)
@@ -3221,12 +3308,21 @@ func (d *ddl) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast
return dbterror.ErrOptOnCacheTable.GenWithStackByArgs("Alter Table")
}
}
+ // set name for anonymous foreign key.
+ maxForeignKeyID := tb.Meta().MaxForeignKeyID
+ for _, spec := range validSpecs {
+ if spec.Tp == ast.AlterTableAddConstraint && spec.Constraint.Tp == ast.ConstraintForeignKey && spec.Constraint.Name == "" {
+ maxForeignKeyID++
+ spec.Constraint.Name = fmt.Sprintf("fk_%d", maxForeignKeyID)
+ }
+ }
if len(validSpecs) > 1 {
sctx.GetSessionVars().StmtCtx.MultiSchemaInfo = model.NewMultiSchemaInfo()
}
for _, spec := range validSpecs {
var handledCharsetOrCollate bool
+ var ttlOptionsHandled bool
switch spec.Tp {
case ast.AlterTableAddColumns:
err = d.AddColumn(sctx, ident, spec)
@@ -3235,7 +3331,7 @@ func (d *ddl) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast
case ast.AlterTableCoalescePartitions:
err = d.CoalescePartitions(sctx, ident, spec)
case ast.AlterTableReorganizePartition:
- err = errors.Trace(dbterror.ErrUnsupportedReorganizePartition)
+ err = d.ReorganizePartitions(sctx, ident, spec)
case ast.AlterTableReorganizeFirstPartition:
err = dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("MERGE FIRST PARTITION")
case ast.AlterTableReorganizeLastPartition:
@@ -3332,7 +3428,7 @@ func (d *ddl) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast
}
err = d.ShardRowID(sctx, ident, opt.UintValue)
case ast.TableOptionAutoIncrement:
- err = d.RebaseAutoID(sctx, ident, int64(opt.UintValue), autoid.RowIDAllocType, opt.BoolValue)
+ err = d.RebaseAutoID(sctx, ident, int64(opt.UintValue), autoid.AutoIncrementType, opt.BoolValue)
case ast.TableOptionAutoIdCache:
if opt.UintValue > uint64(math.MaxInt64) {
// TODO: Refine this error.
@@ -3363,6 +3459,21 @@ func (d *ddl) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast
Name: model.NewCIStr(opt.StrValue),
}
case ast.TableOptionEngine:
+ case ast.TableOptionTTL, ast.TableOptionTTLEnable, ast.TableOptionTTLJobInterval:
+ var ttlInfo *model.TTLInfo
+ var ttlEnable *bool
+ var ttlJobInterval *string
+
+ if ttlOptionsHandled {
+ continue
+ }
+ ttlInfo, ttlEnable, ttlJobInterval, err = getTTLInfoInOptions(spec.Options)
+ if err != nil {
+ return err
+ }
+ err = d.AlterTableTTLInfoOrEnable(sctx, ident, ttlInfo, ttlEnable, ttlJobInterval)
+
+ ttlOptionsHandled = true
default:
err = dbterror.ErrUnsupportedAlterTableOption
}
@@ -3406,6 +3517,9 @@ func (d *ddl) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast
case ast.AlterTableDisableKeys, ast.AlterTableEnableKeys:
// Nothing to do now, see https://github.com/pingcap/tidb/issues/1051
// MyISAM specific
+ case ast.AlterTableRemoveTTL:
+ // the parser makes sure we have only one `ast.AlterTableRemoveTTL` in an alter statement
+ err = d.AlterTableRemoveTTL(sctx, ident)
default:
err = errors.Trace(dbterror.ErrUnsupportedAlterTableSpec)
}
@@ -3446,6 +3560,10 @@ func (d *ddl) RebaseAutoID(ctx sessionctx.Context, ident ast.Ident, newBase int6
actionType = model.ActionRebaseAutoRandomBase
case autoid.RowIDAllocType:
actionType = model.ActionRebaseAutoID
+ case autoid.AutoIncrementType:
+ actionType = model.ActionRebaseAutoID
+ default:
+ panic(fmt.Sprintf("unimplemented rebase autoid type %s", tp))
}
if !force {
@@ -3599,7 +3717,10 @@ func CreateNewColumn(ctx sessionctx.Context, ti ast.Ident, schema *model.DBInfo,
return nil, dbterror.ErrUnsupportedOnGeneratedColumn.GenWithStackByArgs("Adding generated stored column through ALTER TABLE")
}
- _, dependColNames := findDependedColumnNames(specNewColumn)
+ _, dependColNames, err := findDependedColumnNames(schema.Name, t.Meta().Name, specNewColumn)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
if !ctx.GetSessionVars().EnableAutoIncrementInGenerated {
if err := checkAutoIncrementRef(specNewColumn.Name.Name.L, dependColNames, t.Meta()); err != nil {
return nil, errors.Trace(err)
@@ -3786,6 +3907,181 @@ func (d *ddl) AddTablePartitions(ctx sessionctx.Context, ident ast.Ident, spec *
return errors.Trace(err)
}
+// getReorganizedDefinitions return the definitions as they would look like after the REORGANIZE PARTITION is done.
+func getReorganizedDefinitions(pi *model.PartitionInfo, firstPartIdx, lastPartIdx int, idMap map[int]struct{}) []model.PartitionDefinition {
+ tmpDefs := make([]model.PartitionDefinition, 0, len(pi.Definitions)+len(pi.AddingDefinitions)-len(idMap))
+ if pi.Type == model.PartitionTypeList {
+ replaced := false
+ for i := range pi.Definitions {
+ if _, ok := idMap[i]; ok {
+ if !replaced {
+ tmpDefs = append(tmpDefs, pi.AddingDefinitions...)
+ replaced = true
+ }
+ continue
+ }
+ tmpDefs = append(tmpDefs, pi.Definitions[i])
+ }
+ if !replaced {
+ // For safety, for future non-partitioned table -> partitioned
+ tmpDefs = append(tmpDefs, pi.AddingDefinitions...)
+ }
+ return tmpDefs
+ }
+ // Range
+ tmpDefs = append(tmpDefs, pi.Definitions[:firstPartIdx]...)
+ tmpDefs = append(tmpDefs, pi.AddingDefinitions...)
+ if len(pi.Definitions) > (lastPartIdx + 1) {
+ tmpDefs = append(tmpDefs, pi.Definitions[lastPartIdx+1:]...)
+ }
+ return tmpDefs
+}
+
+func getReplacedPartitionIDs(names []model.CIStr, pi *model.PartitionInfo) (int, int, map[int]struct{}, error) {
+ idMap := make(map[int]struct{})
+ var firstPartIdx, lastPartIdx = -1, -1
+ for _, name := range names {
+ partIdx := pi.FindPartitionDefinitionByName(name.L)
+ if partIdx == -1 {
+ return 0, 0, nil, errors.Trace(dbterror.ErrWrongPartitionName)
+ }
+ if _, ok := idMap[partIdx]; ok {
+ return 0, 0, nil, errors.Trace(dbterror.ErrSameNamePartition)
+ }
+ idMap[partIdx] = struct{}{}
+ if firstPartIdx == -1 {
+ firstPartIdx = partIdx
+ } else {
+ firstPartIdx = mathutil.Min[int](firstPartIdx, partIdx)
+ }
+ if lastPartIdx == -1 {
+ lastPartIdx = partIdx
+ } else {
+ lastPartIdx = mathutil.Max[int](lastPartIdx, partIdx)
+ }
+ }
+ if pi.Type == model.PartitionTypeRange {
+ if len(idMap) != (lastPartIdx - firstPartIdx + 1) {
+ return 0, 0, nil, errors.Trace(dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
+ "REORGANIZE PARTITION of RANGE; not adjacent partitions"))
+ }
+ }
+
+ return firstPartIdx, lastPartIdx, idMap, nil
+}
+
+// ReorganizePartitions reorganize one set of partitions to a new set of partitions.
+func (d *ddl) ReorganizePartitions(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+ schema, t, err := d.getSchemaAndTableByIdent(ctx, ident)
+ if err != nil {
+ return errors.Trace(infoschema.ErrTableNotExists.FastGenByArgs(ident.Schema, ident.Name))
+ }
+
+ meta := t.Meta()
+ pi := meta.GetPartitionInfo()
+ if pi == nil {
+ return dbterror.ErrPartitionMgmtOnNonpartitioned
+ }
+ switch pi.Type {
+ case model.PartitionTypeRange, model.PartitionTypeList:
+ default:
+ return errors.Trace(dbterror.ErrUnsupportedReorganizePartition)
+ }
+ firstPartIdx, lastPartIdx, idMap, err := getReplacedPartitionIDs(spec.PartitionNames, pi)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ partInfo, err := BuildAddedPartitionInfo(ctx, meta, spec)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if err = d.assignPartitionIDs(partInfo.Definitions); err != nil {
+ return errors.Trace(err)
+ }
+ if err = checkReorgPartitionDefs(ctx, meta, partInfo, firstPartIdx, lastPartIdx, idMap); err != nil {
+ return errors.Trace(err)
+ }
+ if err = handlePartitionPlacement(ctx, partInfo); err != nil {
+ return errors.Trace(err)
+ }
+
+ tzName, tzOffset := ddlutil.GetTimeZone(ctx)
+ job := &model.Job{
+ SchemaID: schema.ID,
+ TableID: meta.ID,
+ SchemaName: schema.Name.L,
+ TableName: t.Meta().Name.L,
+ Type: model.ActionReorganizePartition,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{spec.PartitionNames, partInfo},
+ ReorgMeta: &model.DDLReorgMeta{
+ SQLMode: ctx.GetSessionVars().SQLMode,
+ Warnings: make(map[errors.ErrorID]*terror.Error),
+ WarningsCount: make(map[errors.ErrorID]int64),
+ Location: &model.TimeZoneLocation{Name: tzName, Offset: tzOffset},
+ },
+ }
+
+ // No preSplitAndScatter here, it will be done by the worker in onReorganizePartition instead.
+ err = d.DoDDLJob(ctx, job)
+ err = d.callHookOnChanged(job, err)
+ return errors.Trace(err)
+}
+
+func checkReorgPartitionDefs(ctx sessionctx.Context, tblInfo *model.TableInfo, partInfo *model.PartitionInfo, firstPartIdx, lastPartIdx int, idMap map[int]struct{}) error {
+ // partInfo contains only the new added partition, we have to combine it with the
+ // old partitions to check all partitions is strictly increasing.
+ pi := tblInfo.Partition
+ clonedMeta := tblInfo.Clone()
+ clonedMeta.Partition.AddingDefinitions = partInfo.Definitions
+ clonedMeta.Partition.Definitions = getReorganizedDefinitions(clonedMeta.Partition, firstPartIdx, lastPartIdx, idMap)
+ if err := checkPartitionDefinitionConstraints(ctx, clonedMeta); err != nil {
+ return errors.Trace(err)
+ }
+ if pi.Type == model.PartitionTypeRange {
+ if lastPartIdx == len(pi.Definitions)-1 {
+ // Last partition dropped, OK to change the end range
+ // Also includes MAXVALUE
+ return nil
+ }
+ // Check if the replaced end range is the same as before
+ lastAddingPartition := partInfo.Definitions[len(partInfo.Definitions)-1]
+ lastOldPartition := pi.Definitions[lastPartIdx]
+ if len(pi.Columns) > 0 {
+ newGtOld, err := checkTwoRangeColumns(ctx, &lastAddingPartition, &lastOldPartition, pi, tblInfo)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if newGtOld {
+ return errors.Trace(dbterror.ErrRangeNotIncreasing)
+ }
+ oldGtNew, err := checkTwoRangeColumns(ctx, &lastOldPartition, &lastAddingPartition, pi, tblInfo)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if oldGtNew {
+ return errors.Trace(dbterror.ErrRangeNotIncreasing)
+ }
+ return nil
+ }
+
+ isUnsigned := isPartExprUnsigned(tblInfo)
+ currentRangeValue, _, err := getRangeValue(ctx, pi.Definitions[lastPartIdx].LessThan[0], isUnsigned)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ newRangeValue, _, err := getRangeValue(ctx, partInfo.Definitions[len(partInfo.Definitions)-1].LessThan[0], isUnsigned)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ if currentRangeValue != newRangeValue {
+ return errors.Trace(dbterror.ErrRangeNotIncreasing)
+ }
+ }
+ return nil
+}
+
// CoalescePartitions coalesce partitions can be used with a table that is partitioned by hash or key to reduce the number of partitions by number.
func (d *ddl) CoalescePartitions(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
is := d.infoCache.GetLatest()
@@ -4233,11 +4529,19 @@ func checkIsDroppableColumn(ctx sessionctx.Context, is infoschema.InfoSchema, sc
if err = isDroppableColumn(tblInfo, colName); err != nil {
return false, errors.Trace(err)
}
+ if err = checkDropColumnWithPartitionConstraint(t, colName); err != nil {
+ return false, errors.Trace(err)
+ }
// Check the column with foreign key.
err = checkDropColumnWithForeignKeyConstraint(is, schema.Name.L, tblInfo, colName.L)
if err != nil {
return false, errors.Trace(err)
}
+ // Check the column with TTL config
+ err = checkDropColumnWithTTLConfig(tblInfo, colName.L)
+ if err != nil {
+ return false, errors.Trace(err)
+ }
// We don't support dropping column with PK handle covered now.
if col.IsPKHandleColumn(tblInfo) {
return false, dbterror.ErrUnsupportedPKHandle
@@ -4248,6 +4552,24 @@ func checkIsDroppableColumn(ctx sessionctx.Context, is infoschema.InfoSchema, sc
return true, nil
}
+// checkDropColumnWithPartitionConstraint is used to check the partition constraint of the drop column.
+func checkDropColumnWithPartitionConstraint(t table.Table, colName model.CIStr) error {
+ if t.Meta().Partition == nil {
+ return nil
+ }
+ pt, ok := t.(table.PartitionedTable)
+ if !ok {
+ // Should never happen!
+ return errors.Trace(dbterror.ErrDependentByPartitionFunctional.GenWithStackByArgs(colName.L))
+ }
+ for _, name := range pt.GetPartitionColumnNames() {
+ if strings.EqualFold(name.L, colName.L) {
+ return errors.Trace(dbterror.ErrDependentByPartitionFunctional.GenWithStackByArgs(colName.L))
+ }
+ }
+ return nil
+}
+
func checkVisibleColumnCnt(t table.Table, addCnt, dropCnt int) error {
tblInfo := t.Meta()
visibleColumCnt := 0
@@ -4386,7 +4708,7 @@ func setColumnComment(ctx sessionctx.Context, col *table.Column, option *ast.Col
func processColumnOptions(ctx sessionctx.Context, col *table.Column, options []*ast.ColumnOption) error {
var sb strings.Builder
restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
- format.RestoreSpacesAroundBinaryOperation
+ format.RestoreSpacesAroundBinaryOperation | format.RestoreWithoutSchemaName | format.RestoreWithoutSchemaName
restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
var hasDefaultValue, setOnUpdateNow bool
@@ -4611,6 +4933,9 @@ func GetModifiableColumnJob(
if err = isGeneratedRelatedColumn(t.Meta(), newCol.ColumnInfo, col.ColumnInfo); err != nil {
return nil, errors.Trace(err)
}
+ if t.Meta().Partition != nil {
+ return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("table is partition table")
+ }
}
// Check that the column change does not affect the partitioning column
@@ -4625,9 +4950,15 @@ func GetModifiableColumnJob(
for _, name := range pt.GetPartitionColumnNames() {
if strings.EqualFold(name.L, col.Name.L) {
isPartitioningColumn = true
+ break
}
}
if isPartitioningColumn {
+ // TODO: update the partitioning columns with new names if column is renamed
+ // Would be an extension from MySQL which does not support it.
+ if col.Name.L != newCol.Name.L {
+ return nil, dbterror.ErrDependentByPartitionFunctional.GenWithStackByArgs(col.Name.L)
+ }
if !isColTypeAllowedAsPartitioningCol(newCol.FieldType) {
return nil, dbterror.ErrNotAllowedTypeInPartition.GenWithStackByArgs(newCol.Name.O)
}
@@ -4671,7 +5002,6 @@ func GetModifiableColumnJob(
newTblInfo.Columns = newCols
var buf bytes.Buffer
- // TODO: update the partitioning columns with new names if column is renamed
AppendPartitionInfo(tblInfo.GetPartitionInfo(), &buf, mysql.ModeNone)
// The parser supports ALTER TABLE ... PARTITION BY ... even if the ddl code does not yet :)
// Ignoring warnings
@@ -4720,10 +5050,17 @@ func GetModifiableColumnJob(
}
// As same with MySQL, we don't support modifying the stored status for generated columns.
- if err = checkModifyGeneratedColumn(sctx, t, col, newCol, specNewColumn, spec.Position); err != nil {
+ if err = checkModifyGeneratedColumn(sctx, schema.Name, t, col, newCol, specNewColumn, spec.Position); err != nil {
return nil, errors.Trace(err)
}
+ if t.Meta().TTLInfo != nil {
+ // the column referenced by TTL should be a time type
+ if t.Meta().TTLInfo.ColumnName.L == originalColName.L && !types.IsTypeTime(newCol.ColumnInfo.FieldType.GetType()) {
+ return nil, errors.Trace(dbterror.ErrUnsupportedColumnInTTLConfig.GenWithStackByArgs(newCol.ColumnInfo.Name.O))
+ }
+ }
+
var newAutoRandBits uint64
if newAutoRandBits, err = checkAutoRandom(t.Meta(), col, specNewColumn); err != nil {
return nil, errors.Trace(err)
@@ -4996,6 +5333,11 @@ func (d *ddl) RenameColumn(ctx sessionctx.Context, ident ast.Ident, spec *ast.Al
}
}
+ err = checkDropColumnWithPartitionConstraint(tbl, oldColName)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
tzName, tzOffset := ddlutil.GetTimeZone(ctx)
newCol := oldCol.Clone()
@@ -5145,6 +5487,11 @@ func (d *ddl) AlterTableAutoIDCache(ctx sessionctx.Context, ident ast.Ident, new
if err != nil {
return errors.Trace(err)
}
+ tbInfo := tb.Meta()
+ if (newCache == 1 && tbInfo.AutoIdCache != 1) ||
+ (newCache != 1 && tbInfo.AutoIdCache == 1) {
+ return fmt.Errorf("Can't Alter AUTO_ID_CACHE between 1 and non-1, the underlying implementation is different")
+ }
job := &model.Job{
SchemaID: schema.ID,
@@ -5262,6 +5609,96 @@ func (d *ddl) AlterTableSetTiFlashReplica(ctx sessionctx.Context, ident ast.Iden
return errors.Trace(err)
}
+// AlterTableTTLInfoOrEnable submit ddl job to change table info according to the ttlInfo, or ttlEnable
+// at least one of the `ttlInfo`, `ttlEnable` or `ttlCronJobSchedule` should be not nil.
+// When `ttlInfo` is nil, and `ttlEnable` is not, it will use the original `.TTLInfo` in the table info and modify the
+// `.Enable`. If the `.TTLInfo` in the table info is empty, this function will return an error.
+// When `ttlInfo` is nil, and `ttlCronJobSchedule` is not, it will use the original `.TTLInfo` in the table info and modify the
+// `.JobInterval`. If the `.TTLInfo` in the table info is empty, this function will return an error.
+// When `ttlInfo` is not nil, it simply submits the job with the `ttlInfo` and ignore the `ttlEnable`.
+func (d *ddl) AlterTableTTLInfoOrEnable(ctx sessionctx.Context, ident ast.Ident, ttlInfo *model.TTLInfo, ttlEnable *bool, ttlCronJobSchedule *string) error {
+ is := d.infoCache.GetLatest()
+ schema, ok := is.SchemaByName(ident.Schema)
+ if !ok {
+ return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(ident.Schema)
+ }
+
+ tb, err := is.TableByName(ident.Schema, ident.Name)
+ if err != nil {
+ return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
+ }
+
+ tblInfo := tb.Meta().Clone()
+ tableID := tblInfo.ID
+ tableName := tblInfo.Name.L
+
+ var job *model.Job
+ if ttlInfo != nil {
+ tblInfo.TTLInfo = ttlInfo
+ err = checkTTLInfoValid(ctx, ident.Schema, tblInfo)
+ if err != nil {
+ return err
+ }
+ } else {
+ if tblInfo.TTLInfo == nil {
+ if ttlEnable != nil {
+ return errors.Trace(dbterror.ErrSetTTLOptionForNonTTLTable.FastGenByArgs("TTL_ENABLE"))
+ }
+ if ttlCronJobSchedule != nil {
+ return errors.Trace(dbterror.ErrSetTTLOptionForNonTTLTable.FastGenByArgs("TTL_JOB_INTERVAL"))
+ }
+ }
+ }
+
+ job = &model.Job{
+ SchemaID: schema.ID,
+ TableID: tableID,
+ SchemaName: schema.Name.L,
+ TableName: tableName,
+ Type: model.ActionAlterTTLInfo,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{ttlInfo, ttlEnable, ttlCronJobSchedule},
+ }
+
+ err = d.DoDDLJob(ctx, job)
+ err = d.callHookOnChanged(job, err)
+ return errors.Trace(err)
+}
+
+func (d *ddl) AlterTableRemoveTTL(ctx sessionctx.Context, ident ast.Ident) error {
+ is := d.infoCache.GetLatest()
+
+ schema, ok := is.SchemaByName(ident.Schema)
+ if !ok {
+ return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(ident.Schema)
+ }
+
+ tb, err := is.TableByName(ident.Schema, ident.Name)
+ if err != nil {
+ return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
+ }
+
+ tblInfo := tb.Meta().Clone()
+ tableID := tblInfo.ID
+ tableName := tblInfo.Name.L
+
+ if tblInfo.TTLInfo != nil {
+ job := &model.Job{
+ SchemaID: schema.ID,
+ TableID: tableID,
+ SchemaName: schema.Name.L,
+ TableName: tableName,
+ Type: model.ActionAlterTTLRemove,
+ BinlogInfo: &model.HistoryInfo{},
+ }
+ err = d.DoDDLJob(ctx, job)
+ err = d.callHookOnChanged(job, err)
+ return errors.Trace(err)
+ }
+
+ return nil
+}
+
func isTableTiFlashSupported(schema *model.DBInfo, tb table.Table) error {
// Memory tables and system tables are not supported by TiFlash
if util.IsMemOrSysDB(schema.Name.L) {
@@ -5987,7 +6424,7 @@ func (d *ddl) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexName m
// After DDL job is put to the queue, and if the check fail, TiDB will run the DDL cancel logic.
// The recover step causes DDL wait a few seconds, makes the unit test painfully slow.
// For same reason, decide whether index is global here.
- indexColumns, err := buildIndexColumns(ctx, tblInfo.Columns, indexPartSpecifications)
+ indexColumns, _, err := buildIndexColumns(ctx, tblInfo.Columns, indexPartSpecifications)
if err != nil {
return errors.Trace(err)
}
@@ -6090,14 +6527,14 @@ func BuildHiddenColumnInfo(ctx sessionctx.Context, indexPartSpecifications []*as
var sb strings.Builder
restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
- format.RestoreSpacesAroundBinaryOperation
+ format.RestoreSpacesAroundBinaryOperation | format.RestoreWithoutSchemaName | format.RestoreWithoutTableName
restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
sb.Reset()
err := idxPart.Expr.Restore(restoreCtx)
if err != nil {
return nil, errors.Trace(err)
}
- expr, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, idxPart.Expr)
+ expr, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, idxPart.Expr, true)
if err != nil {
// TODO: refine the error message.
return nil, err
@@ -6212,7 +6649,7 @@ func (d *ddl) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde
// After DDL job is put to the queue, and if the check fail, TiDB will run the DDL cancel logic.
// The recover step causes DDL wait a few seconds, makes the unit test painfully slow.
// For same reason, decide whether index is global here.
- indexColumns, err := buildIndexColumns(ctx, finalColumns, indexPartSpecifications)
+ indexColumns, _, err := buildIndexColumns(ctx, finalColumns, indexPartSpecifications)
if err != nil {
return errors.Trace(err)
}
@@ -6393,7 +6830,7 @@ func (d *ddl) CreateForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName mode
if err != nil {
return err
}
- if model.FindIndexByColumns(t.Meta(), fkInfo.Cols...) == nil {
+ if model.FindIndexByColumns(t.Meta(), t.Meta().Indices, fkInfo.Cols...) == nil {
// Need to auto create index for fk cols
if ctx.GetSessionVars().StmtCtx.MultiSchemaInfo == nil {
ctx.GetSessionVars().StmtCtx.MultiSchemaInfo = model.NewMultiSchemaInfo()
@@ -7353,6 +7790,142 @@ func checkIgnorePlacementDDL(ctx sessionctx.Context) bool {
return false
}
+// AddResourceGroup implements the DDL interface, creates a resource group.
+func (d *ddl) AddResourceGroup(ctx sessionctx.Context, stmt *ast.CreateResourceGroupStmt) (err error) {
+ groupName := stmt.ResourceGroupName
+ groupInfo := &model.ResourceGroupInfo{Name: groupName, ResourceGroupSettings: &model.ResourceGroupSettings{}}
+ groupInfo, err = buildResourceGroup(groupInfo, stmt.ResourceGroupOptionList)
+ if err != nil {
+ return err
+ }
+
+ if _, ok := d.GetInfoSchemaWithInterceptor(ctx).ResourceGroupByName(groupName); ok {
+ if stmt.IfNotExists {
+ err = infoschema.ErrResourceGroupExists.GenWithStackByArgs(groupName)
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ return nil
+ }
+ return infoschema.ErrResourceGroupExists.GenWithStackByArgs(groupName)
+ }
+
+ if groupName.L == variable.DefaultResourceGroupName {
+ return errors.Trace(infoschema.ErrReservedSyntax.GenWithStackByArgs(groupName))
+ }
+
+ if err := d.checkResourceGroupValidation(groupInfo); err != nil {
+ return err
+ }
+
+ logutil.BgLogger().Debug("create resource group", zap.String("name", groupName.O), zap.Stringer("resource group settings", groupInfo.ResourceGroupSettings))
+ groupIDs, err := d.genGlobalIDs(1)
+ if err != nil {
+ return err
+ }
+ groupInfo.ID = groupIDs[0]
+
+ job := &model.Job{
+ SchemaName: groupName.L,
+ Type: model.ActionCreateResourceGroup,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{groupInfo, false},
+ }
+ err = d.DoDDLJob(ctx, job)
+ err = d.callHookOnChanged(job, err)
+ return err
+}
+
+func (d *ddl) checkResourceGroupValidation(groupInfo *model.ResourceGroupInfo) error {
+ _, err := resourcegroup.NewGroupFromOptions(groupInfo.Name.L, groupInfo.ResourceGroupSettings)
+ return err
+}
+
+// DropResourceGroup implements the DDL interface.
+func (d *ddl) DropResourceGroup(ctx sessionctx.Context, stmt *ast.DropResourceGroupStmt) (err error) {
+ groupName := stmt.ResourceGroupName
+ is := d.GetInfoSchemaWithInterceptor(ctx)
+ // Check group existence.
+ group, ok := is.ResourceGroupByName(groupName)
+ if !ok {
+ err = infoschema.ErrResourceGroupNotExists.GenWithStackByArgs(groupName)
+ if stmt.IfExists {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ return nil
+ }
+ return err
+ }
+
+ // check to see if some user has dependency on the group
+ checker := privilege.GetPrivilegeManager(ctx)
+ if checker == nil {
+ return errors.New("miss privilege checker")
+ }
+ user, matched := checker.MatchUserResourceGroupName(groupName.L)
+ if matched {
+ err = errors.Errorf("user [%s] depends on the resource group to drop", user)
+ return err
+ }
+
+ job := &model.Job{
+ SchemaID: group.ID,
+ SchemaName: group.Name.L,
+ Type: model.ActionDropResourceGroup,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{groupName},
+ }
+ err = d.DoDDLJob(ctx, job)
+ err = d.callHookOnChanged(job, err)
+ return err
+}
+
+func buildResourceGroup(oldGroup *model.ResourceGroupInfo, options []*ast.ResourceGroupOption) (*model.ResourceGroupInfo, error) {
+ groupInfo := &model.ResourceGroupInfo{Name: oldGroup.Name, ID: oldGroup.ID, ResourceGroupSettings: &model.ResourceGroupSettings{}}
+ for _, opt := range options {
+ err := SetDirectResourceGroupUnit(groupInfo.ResourceGroupSettings, opt.Tp, opt.StrValue, opt.UintValue, opt.BoolValue)
+ if err != nil {
+ return nil, err
+ }
+ }
+ groupInfo.ResourceGroupSettings.Adjust()
+ return groupInfo, nil
+}
+
+// AlterResourceGroup implements the DDL interface.
+func (d *ddl) AlterResourceGroup(ctx sessionctx.Context, stmt *ast.AlterResourceGroupStmt) (err error) {
+ groupName := stmt.ResourceGroupName
+ is := d.GetInfoSchemaWithInterceptor(ctx)
+ // Check group existence.
+ group, ok := is.ResourceGroupByName(groupName)
+ if !ok {
+ err := infoschema.ErrResourceGroupNotExists.GenWithStackByArgs(groupName)
+ if stmt.IfExists {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ return nil
+ }
+ return err
+ }
+ newGroupInfo, err := buildResourceGroup(group, stmt.ResourceGroupOptionList)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ if err := d.checkResourceGroupValidation(newGroupInfo); err != nil {
+ return err
+ }
+
+ logutil.BgLogger().Debug("alter resource group", zap.String("name", groupName.L), zap.Stringer("new resource group settings", newGroupInfo.ResourceGroupSettings))
+
+ job := &model.Job{
+ SchemaID: newGroupInfo.ID,
+ SchemaName: newGroupInfo.Name.L,
+ Type: model.ActionAlterResourceGroup,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{newGroupInfo},
+ }
+ err = d.DoDDLJob(ctx, job)
+ err = d.callHookOnChanged(job, err)
+ return err
+}
+
func (d *ddl) CreatePlacementPolicy(ctx sessionctx.Context, stmt *ast.CreatePlacementPolicyStmt) (err error) {
if checkIgnorePlacementDDL(ctx) {
return nil
diff --git a/ddl/ddl_api_test.go b/ddl/ddl_api_test.go
index f4010015f5456..9f36cc95f806c 100644
--- a/ddl/ddl_api_test.go
+++ b/ddl/ddl_api_test.go
@@ -115,73 +115,6 @@ func TestGetDDLJobsIsSort(t *testing.T) {
require.NoError(t, err)
}
-func TestGetHistoryDDLJobs(t *testing.T) {
- store := testkit.CreateMockStore(t)
-
- // delete the internal DDL record.
- err := kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), store, false, func(ctx context.Context, txn kv.Transaction) error {
- return meta.NewMeta(txn).ClearAllHistoryJob()
- })
- require.NoError(t, err)
- testkit.NewTestKit(t, store).MustExec("delete from mysql.tidb_ddl_history")
-
- tk := testkit.NewTestKit(t, store)
- sess := tk.Session()
- tk.MustExec("begin")
-
- txn, err := sess.Txn(true)
- require.NoError(t, err)
-
- m := meta.NewMeta(txn)
- cnt := 11
- jobs := make([]*model.Job, cnt)
- for i := 0; i < cnt; i++ {
- jobs[i] = &model.Job{
- ID: int64(i),
- SchemaID: 1,
- Type: model.ActionCreateTable,
- }
- err = ddl.AddHistoryDDLJobForTest(sess, m, jobs[i], true)
- require.NoError(t, err)
-
- historyJobs, err := ddl.GetLastNHistoryDDLJobs(m, ddl.DefNumHistoryJobs)
- require.NoError(t, err)
-
- if i+1 > ddl.MaxHistoryJobs {
- require.Len(t, historyJobs, ddl.MaxHistoryJobs)
- } else {
- require.Len(t, historyJobs, i+1)
- }
- }
-
- delta := cnt - ddl.MaxHistoryJobs
- historyJobs, err := ddl.GetLastNHistoryDDLJobs(m, ddl.DefNumHistoryJobs)
- require.NoError(t, err)
- require.Len(t, historyJobs, ddl.MaxHistoryJobs)
-
- l := len(historyJobs) - 1
- for i, job := range historyJobs {
- require.Equal(t, jobs[delta+l-i].ID, job.ID)
- require.Equal(t, int64(1), job.SchemaID)
- require.Equal(t, model.ActionCreateTable, job.Type)
- }
-
- var historyJobs2 []*model.Job
- err = ddl.IterHistoryDDLJobs(txn, func(jobs []*model.Job) (b bool, e error) {
- for _, job := range jobs {
- historyJobs2 = append(historyJobs2, job)
- if len(historyJobs2) == ddl.DefNumHistoryJobs {
- return true, nil
- }
- }
- return false, nil
- })
- require.NoError(t, err)
- require.Equal(t, historyJobs, historyJobs2)
-
- tk.MustExec("rollback")
-}
-
func TestIsJobRollbackable(t *testing.T) {
cases := []struct {
tp model.ActionType
diff --git a/ddl/ddl_test.go b/ddl/ddl_test.go
index 9760608674c6f..99e01a8e740c4 100644
--- a/ddl/ddl_test.go
+++ b/ddl/ddl_test.go
@@ -19,7 +19,6 @@ import (
"testing"
"time"
- "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
@@ -40,9 +39,14 @@ import (
const testLease = 5 * time.Millisecond
+// DDLForTest exports for testing.
type DDLForTest interface {
// SetInterceptor sets the interceptor.
SetInterceptor(h Interceptor)
+ NewReorgCtx(jobID int64, startKey []byte, currElement *meta.Element, rowCount int64) *reorgCtx
+ SetReorgCtxForBackfill(bfJob *BackfillJob)
+ GetReorgCtx(jobID int64) *reorgCtx
+ RemoveReorgCtx(id int64)
}
// SetInterceptor implements DDL.SetInterceptor interface.
@@ -53,9 +57,45 @@ func (d *ddl) SetInterceptor(i Interceptor) {
d.mu.interceptor = i
}
+// IsReorgCanceled exports for testing.
+func (rc *reorgCtx) IsReorgCanceled() bool {
+ return rc.isReorgCanceled()
+}
+
+// NewReorgCtx exports for testing.
+func (d *ddl) NewReorgCtx(jobID int64, startKey []byte, currElement *meta.Element, rowCount int64) *reorgCtx {
+ return d.newReorgCtx(jobID, startKey, currElement, rowCount)
+}
+
+// SetReorgCtxForBackfill exports for testing.
+func (d *ddl) SetReorgCtxForBackfill(bfJob *BackfillJob) {
+ d.setReorgCtxForBackfill(bfJob)
+}
+
+// GetReorgCtx exports for testing.
+func (d *ddl) GetReorgCtx(jobID int64) *reorgCtx {
+ return d.getReorgCtx(jobID)
+}
+
+// RemoveReorgCtx exports for testing.
+func (d *ddl) RemoveReorgCtx(id int64) {
+ d.removeReorgCtx(id)
+}
+
// JobNeedGCForTest is only used for test.
var JobNeedGCForTest = jobNeedGC
+// NewSession is only used for test.
+var NewSession = newSession
+
+// GetJobWithoutPartition is only used for test.
+const GetJobWithoutPartition = getJobWithoutPartition
+
+// GetDDLCtx returns ddlCtx for test.
+func GetDDLCtx(d DDL) *ddlCtx {
+ return d.(*ddl).ddlCtx
+}
+
// GetMaxRowID is used for test.
func GetMaxRowID(store kv.Storage, priority int, t table.Table, startHandle, endHandle kv.Key) (kv.Key, error) {
return getRangeEndKey(NewJobContext(), store, priority, t.RecordPrefix(), startHandle, endHandle)
@@ -268,109 +308,6 @@ func TestBuildJobDependence(t *testing.T) {
require.NoError(t, err)
}
-func TestNotifyDDLJob(t *testing.T) {
- store := createMockStore(t)
- defer func() {
- require.NoError(t, store.Close())
- }()
-
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/NoDDLDispatchLoop", `return(true)`))
- defer require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/NoDDLDispatchLoop"))
-
- getFirstNotificationAfterStartDDL := func(d *ddl) {
- select {
- case <-d.workers[addIdxWorker].ddlJobCh:
- default:
- // The notification may be received by the worker.
- }
- select {
- case <-d.workers[generalWorker].ddlJobCh:
- default:
- // The notification may be received by the worker.
- }
-
- select {
- case <-d.ddlJobCh:
- default:
- }
- }
-
- d, err := testNewDDLAndStart(
- context.Background(),
- WithStore(store),
- WithLease(testLease),
- )
- require.NoError(t, err)
- defer func() {
- require.NoError(t, d.Stop())
- }()
- getFirstNotificationAfterStartDDL(d)
- // Ensure that the notification is not handled in workers `start` function.
- d.cancel()
- for _, worker := range d.workers {
- worker.Close()
- }
-
- job := &model.Job{
- SchemaID: 1,
- TableID: 2,
- Type: model.ActionCreateTable,
- BinlogInfo: &model.HistoryInfo{},
- Args: []interface{}{},
- }
- // Test the notification mechanism of the owner and the server receiving the DDL request on the same TiDB.
- // This DDL request is a general DDL job.
- d.asyncNotifyWorker(job)
- select {
- case <-d.workers[generalWorker].ddlJobCh:
- case <-d.ddlJobCh:
- default:
- require.FailNow(t, "do not get the general job notification")
- }
- // Test the notification mechanism of the owner and the server receiving the DDL request on the same TiDB.
- // This DDL request is a add index DDL job.
- job.Type = model.ActionAddIndex
- d.asyncNotifyWorker(job)
- select {
- case <-d.workers[addIdxWorker].ddlJobCh:
- case <-d.ddlJobCh:
- default:
- require.FailNow(t, "do not get the add index job notification")
- }
-
- // Test the notification mechanism that the owner and the server receiving the DDL request are not on the same TiDB.
- // And the etcd client is nil.
- d1, err := testNewDDLAndStart(
- context.Background(),
- WithStore(store),
- WithLease(testLease),
- )
- require.NoError(t, err)
- defer func() {
- require.NoError(t, d1.Stop())
- }()
- getFirstNotificationAfterStartDDL(d1)
- // Ensure that the notification is not handled by worker's "start".
- d1.cancel()
- for _, worker := range d1.workers {
- worker.Close()
- }
- d1.ownerManager.RetireOwner()
- d1.asyncNotifyWorker(job)
- job.Type = model.ActionCreateTable
- d1.asyncNotifyWorker(job)
- require.False(t, d1.OwnerManager().IsOwner())
- select {
- case <-d1.workers[addIdxWorker].ddlJobCh:
- require.FailNow(t, "should not get the add index job notification")
- case <-d1.workers[generalWorker].ddlJobCh:
- require.FailNow(t, "should not get the general job notification")
- case <-d1.ddlJobCh:
- require.FailNow(t, "should not get the job notification")
- default:
- }
-}
-
func TestError(t *testing.T) {
kvErrs := []*terror.Error{
dbterror.ErrDDLJobNotFound,
diff --git a/ddl/ddl_tiflash_api.go b/ddl/ddl_tiflash_api.go
index 7e456815acd8c..c601093ca5963 100644
--- a/ddl/ddl_tiflash_api.go
+++ b/ddl/ddl_tiflash_api.go
@@ -29,20 +29,15 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
- "github.com/pingcap/tidb/ddl/placement"
ddlutil "github.com/pingcap/tidb/ddl/util"
"github.com/pingcap/tidb/domain/infosync"
"github.com/pingcap/tidb/infoschema"
- "github.com/pingcap/tidb/kv"
- "github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/parser/model"
- "github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/sessionctx"
- "github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/store/helper"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/util"
- "github.com/pingcap/tidb/util/gcutil"
+ "github.com/pingcap/tidb/util/intest"
"github.com/pingcap/tidb/util/logutil"
atomicutil "go.uber.org/atomic"
"go.uber.org/zap"
@@ -50,12 +45,13 @@ import (
// TiFlashReplicaStatus records status for each TiFlash replica.
type TiFlashReplicaStatus struct {
- ID int64
- Count uint64
- LocationLabels []string
- Available bool
- HighPriority bool
- IsPartition bool
+ ID int64
+ Count uint64
+ LocationLabels []string
+ Available bool
+ LogicalTableAvailable bool
+ HighPriority bool
+ IsPartition bool
}
// TiFlashTick is type for backoff threshold.
@@ -283,16 +279,16 @@ func LoadTiFlashReplicaInfo(tblInfo *model.TableInfo, tableList *[]TiFlashReplic
for _, p := range pi.Definitions {
logutil.BgLogger().Debug(fmt.Sprintf("Table %v has partition %v\n", tblInfo.ID, p.ID))
*tableList = append(*tableList, TiFlashReplicaStatus{p.ID,
- tblInfo.TiFlashReplica.Count, tblInfo.TiFlashReplica.LocationLabels, tblInfo.TiFlashReplica.IsPartitionAvailable(p.ID), false, true})
+ tblInfo.TiFlashReplica.Count, tblInfo.TiFlashReplica.LocationLabels, tblInfo.TiFlashReplica.IsPartitionAvailable(p.ID), tblInfo.TiFlashReplica.Available, false, true})
}
// partitions that in adding mid-state
for _, p := range pi.AddingDefinitions {
logutil.BgLogger().Debug(fmt.Sprintf("Table %v has partition adding %v\n", tblInfo.ID, p.ID))
- *tableList = append(*tableList, TiFlashReplicaStatus{p.ID, tblInfo.TiFlashReplica.Count, tblInfo.TiFlashReplica.LocationLabels, tblInfo.TiFlashReplica.IsPartitionAvailable(p.ID), true, true})
+ *tableList = append(*tableList, TiFlashReplicaStatus{p.ID, tblInfo.TiFlashReplica.Count, tblInfo.TiFlashReplica.LocationLabels, tblInfo.TiFlashReplica.IsPartitionAvailable(p.ID), tblInfo.TiFlashReplica.Available, true, true})
}
} else {
logutil.BgLogger().Debug(fmt.Sprintf("Table %v has no partition\n", tblInfo.ID))
- *tableList = append(*tableList, TiFlashReplicaStatus{tblInfo.ID, tblInfo.TiFlashReplica.Count, tblInfo.TiFlashReplica.LocationLabels, tblInfo.TiFlashReplica.Available, false, false})
+ *tableList = append(*tableList, TiFlashReplicaStatus{tblInfo.ID, tblInfo.TiFlashReplica.Count, tblInfo.TiFlashReplica.LocationLabels, tblInfo.TiFlashReplica.Available, tblInfo.TiFlashReplica.Available, false, false})
}
}
@@ -355,22 +351,6 @@ func updateTiFlashStores(pollTiFlashContext *TiFlashManagementContext) error {
return nil
}
-func getTiFlashPeerWithoutLagCount(pollTiFlashContext *TiFlashManagementContext, tableID int64) (int, error) {
- // storeIDs -> regionID, PD will not create two peer on the same store
- var flashPeerCount int
- for _, store := range pollTiFlashContext.TiFlashStores {
- regionReplica := make(map[int64]int)
- err := helper.CollectTiFlashStatus(store.Store.StatusAddress, tableID, ®ionReplica)
- if err != nil {
- logutil.BgLogger().Error("Fail to get peer status from TiFlash.",
- zap.Int64("tableID", tableID))
- return 0, err
- }
- flashPeerCount += len(regionReplica)
- }
- return flashPeerCount, nil
-}
-
func pollAvailableTableProgress(schemas infoschema.InfoSchema, ctx sessionctx.Context, pollTiFlashContext *TiFlashManagementContext) {
pollMaxCount := RefreshProgressMaxTableCount
failpoint.Inject("PollAvailableTableProgressMaxCount", func(val failpoint.Value) {
@@ -419,6 +399,13 @@ func pollAvailableTableProgress(schemas infoschema.InfoSchema, ctx sessionctx.Co
zap.Int64("tableID", availableTableID.ID),
zap.Bool("IsPartition", availableTableID.IsPartition),
)
+ if intest.InTest {
+ // In the test, the server cannot start up because the port is occupied.
+ // Although the port is random. so we need to quickly return when to
+ // fail to get tiflash sync.
+ // https://github.com/pingcap/tidb/issues/39949
+ panic(err)
+ }
continue
}
err = infosync.UpdateTiFlashProgressCache(availableTableID.ID, progress)
@@ -445,6 +432,14 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T
return err
}
}
+
+ failpoint.Inject("OneTiFlashStoreDown", func() {
+ for storeID, store := range pollTiFlashContext.TiFlashStores {
+ store.Store.StateName = "Down"
+ pollTiFlashContext.TiFlashStores[storeID] = store
+ break
+ }
+ })
pollTiFlashContext.PollCounter++
// Start to process every table.
@@ -466,6 +461,21 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T
}
}
+ failpoint.Inject("waitForAddPartition", func(val failpoint.Value) {
+ for _, phyTable := range tableList {
+ is := d.infoCache.GetLatest()
+ _, ok := is.TableByID(phyTable.ID)
+ if !ok {
+ tb, _, _ := is.FindTableByPartitionID(phyTable.ID)
+ if tb == nil {
+ logutil.BgLogger().Info("[ddl] waitForAddPartition")
+ sleepSecond := val.(int)
+ time.Sleep(time.Duration(sleepSecond) * time.Second)
+ }
+ }
+ }
+ })
+
needPushPending := false
if pollTiFlashContext.UpdatingProgressTables.Len() == 0 {
needPushPending = true
@@ -479,7 +489,7 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T
available = val.(bool)
})
// We only check unavailable tables here, so doesn't include blocked add partition case.
- if !available {
+ if !available && !tb.LogicalTableAvailable {
enabled, inqueue, _ := pollTiFlashContext.Backoff.Tick(tb.ID)
if inqueue && !enabled {
logutil.BgLogger().Info("Escape checking available status due to backoff", zap.Int64("tableId", tb.ID))
@@ -540,110 +550,6 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T
return nil
}
-func getDropOrTruncateTableTiflash(ctx sessionctx.Context, currentSchema infoschema.InfoSchema, tikvHelper *helper.Helper, replicaInfos *[]TiFlashReplicaStatus) error {
- store := tikvHelper.Store.(kv.Storage)
-
- txn, err := store.Begin()
- if err != nil {
- return errors.Trace(err)
- }
- gcSafePoint, err := gcutil.GetGCSafePoint(ctx)
- if err != nil {
- return err
- }
- uniqueIDMap := make(map[int64]struct{})
- handleJobAndTableInfo := func(job *model.Job, tblInfo *model.TableInfo) (bool, error) {
- // Avoid duplicate table ID info.
- if _, ok := currentSchema.TableByID(tblInfo.ID); ok {
- return false, nil
- }
- if _, ok := uniqueIDMap[tblInfo.ID]; ok {
- return false, nil
- }
- uniqueIDMap[tblInfo.ID] = struct{}{}
- LoadTiFlashReplicaInfo(tblInfo, replicaInfos)
- return false, nil
- }
- fn := func(jobs []*model.Job) (bool, error) {
- getTable := func(StartTS uint64, SchemaID int64, TableID int64) (*model.TableInfo, error) {
- snapMeta := meta.NewSnapshotMeta(store.GetSnapshot(kv.NewVersion(StartTS)))
- if err != nil {
- return nil, err
- }
- tbl, err := snapMeta.GetTable(SchemaID, TableID)
- return tbl, err
- }
- return GetDropOrTruncateTableInfoFromJobsByStore(jobs, gcSafePoint, getTable, handleJobAndTableInfo)
- }
-
- err = IterAllDDLJobs(ctx, txn, fn)
- if err != nil {
- if terror.ErrorEqual(variable.ErrSnapshotTooOld, err) {
- // The err indicate that current ddl job and remain DDL jobs was been deleted by GC,
- // just ignore the error and return directly.
- return nil
- }
- return err
- }
- return nil
-}
-
-// HandlePlacementRuleRoutine fetch all rules from pd, remove all obsolete rules.
-// It handles rare situation, when we fail to alter pd rules.
-func HandlePlacementRuleRoutine(ctx sessionctx.Context, d *ddl, tableList []TiFlashReplicaStatus) error {
- c := context.Background()
- tikvStore, ok := ctx.GetStore().(helper.Storage)
- if !ok {
- return errors.New("Can not get Helper")
- }
- tikvHelper := &helper.Helper{
- Store: tikvStore,
- RegionCache: tikvStore.GetRegionCache(),
- }
-
- allRulesArr, err := infosync.GetTiFlashGroupRules(c, "tiflash")
- if err != nil {
- return errors.Trace(err)
- }
- allRules := make(map[string]placement.TiFlashRule)
- for _, r := range allRulesArr {
- allRules[r.ID] = r
- }
-
- start := time.Now()
- originLen := len(tableList)
- currentSchema := d.GetInfoSchemaWithInterceptor(ctx)
- if err := getDropOrTruncateTableTiflash(ctx, currentSchema, tikvHelper, &tableList); err != nil {
- // may fail when no `tikv_gc_safe_point` available, should return in order to remove valid pd rules.
- logutil.BgLogger().Error("getDropOrTruncateTableTiflash returns error", zap.Error(err))
- return errors.Trace(err)
- }
- elapsed := time.Since(start)
- logutil.BgLogger().Info("getDropOrTruncateTableTiflash cost", zap.Duration("time", elapsed), zap.Int("updated", len(tableList)-originLen))
- for _, tb := range tableList {
- // For every region in each table, if it has one replica, we reckon it ready.
- ruleID := fmt.Sprintf("table-%v-r", tb.ID)
- if _, ok := allRules[ruleID]; !ok {
- // Mostly because of a previous failure of setting pd rule.
- logutil.BgLogger().Warn(fmt.Sprintf("Table %v exists, but there are no rule for it", tb.ID))
- newRule := infosync.MakeNewRule(tb.ID, tb.Count, tb.LocationLabels)
- _ = infosync.SetTiFlashPlacementRule(context.Background(), *newRule)
- }
- // For every existing table, we do not remove their rules.
- delete(allRules, ruleID)
- }
-
- // Remove rules of non-existing table
- for _, v := range allRules {
- logutil.BgLogger().Info("Remove TiFlash rule", zap.String("id", v.ID))
- if err := infosync.DeleteTiFlashPlacementRule(c, "tiflash", v.ID); err != nil {
- logutil.BgLogger().Warn("delete TiFlash pd rule failed", zap.Error(err), zap.String("ruleID", v.ID))
- }
- }
-
- return nil
-}
-
func (d *ddl) PollTiFlashRoutine() {
pollTiflashContext, err := NewTiFlashManagementContext()
if err != nil {
diff --git a/ddl/ddl_worker.go b/ddl/ddl_worker.go
index 956f4c805347f..48faea6c6e02c 100644
--- a/ddl/ddl_worker.go
+++ b/ddl/ddl_worker.go
@@ -40,6 +40,7 @@ import (
tidbutil "github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/mathutil"
"github.com/pingcap/tidb/util/resourcegrouptag"
"github.com/pingcap/tidb/util/topsql"
topsqlstate "github.com/pingcap/tidb/util/topsql/state"
@@ -52,10 +53,14 @@ import (
var (
// ddlWorkerID is used for generating the next DDL worker ID.
ddlWorkerID = atomicutil.NewInt32(0)
+ // backfillContextID is used for generating the next backfill context ID.
+ backfillContextID = atomicutil.NewInt32(0)
// WaitTimeWhenErrorOccurred is waiting interval when processing DDL jobs encounter errors.
WaitTimeWhenErrorOccurred = int64(1 * time.Second)
mockDDLErrOnce = int64(0)
+ // TestNotifyBeginTxnCh is used for if the txn is beginning in runInTxn.
+ TestNotifyBeginTxnCh = make(chan struct{})
)
// GetWaitTimeWhenErrorOccurred return waiting interval when processing DDL jobs encounter errors.
@@ -97,8 +102,6 @@ type worker struct {
logCtx context.Context
lockSeqNum bool
- concurrentDDL bool
-
*ddlCtx
}
@@ -119,11 +122,11 @@ func NewJobContext() *JobContext {
cacheSQL: "",
cacheNormalizedSQL: "",
cacheDigest: nil,
- tp: "unknown",
+ tp: "",
}
}
-func newWorker(ctx context.Context, tp workerType, sessPool *sessionPool, delRangeMgr delRangeManager, dCtx *ddlCtx, concurrentDDL bool) *worker {
+func newWorker(ctx context.Context, tp workerType, sessPool *sessionPool, delRangeMgr delRangeManager, dCtx *ddlCtx) *worker {
worker := &worker{
id: ddlWorkerID.Add(1),
tp: tp,
@@ -132,7 +135,6 @@ func newWorker(ctx context.Context, tp workerType, sessPool *sessionPool, delRan
ddlCtx: dCtx,
sessPool: sessPool,
delRangeManager: delRangeMgr,
- concurrentDDL: concurrentDDL,
}
worker.addingDDLJobKey = addingDDLJobPrefix + worker.typeStr()
worker.logCtx = logutil.WithKeyValue(context.Background(), "worker", worker.String())
@@ -165,71 +167,19 @@ func (w *worker) Close() {
logutil.Logger(w.logCtx).Info("[ddl] DDL worker closed", zap.Duration("take time", time.Since(startTime)))
}
-// start is used for async online schema changing, it will try to become the owner firstly,
-// then wait or pull the job queue to handle a schema change job.
-func (w *worker) start(d *ddlCtx) {
- logutil.Logger(w.logCtx).Info("[ddl] start DDL worker")
- defer w.wg.Done()
- defer tidbutil.Recover(
- metrics.LabelDDLWorker,
- fmt.Sprintf("DDL ID %s, %s start", d.uuid, w),
- nil, true,
- )
-
- // We use 4 * lease time to check owner's timeout, so here, we will update owner's status
- // every 2 * lease time. If lease is 0, we will use default 1s.
- // But we use etcd to speed up, normally it takes less than 1s now, so we use 1s as the max value.
- checkTime := chooseLeaseTime(2*d.lease, 1*time.Second)
-
- ticker := time.NewTicker(checkTime)
- defer ticker.Stop()
- var notifyDDLJobByEtcdCh clientv3.WatchChan
- if d.etcdCli != nil {
- notifyDDLJobByEtcdCh = d.etcdCli.Watch(context.Background(), w.addingDDLJobKey)
- }
-
- rewatchCnt := 0
- for {
- ok := true
- select {
- case <-ticker.C:
- logutil.Logger(w.logCtx).Debug("[ddl] wait to check DDL status again", zap.Duration("interval", checkTime))
- case <-w.ddlJobCh:
- case _, ok = <-notifyDDLJobByEtcdCh:
- case <-w.ctx.Done():
- return
- }
-
- if !ok {
- logutil.Logger(w.logCtx).Warn("[ddl] start worker watch channel closed", zap.String("watch key", w.addingDDLJobKey))
- notifyDDLJobByEtcdCh = d.etcdCli.Watch(context.Background(), w.addingDDLJobKey)
- rewatchCnt++
- if rewatchCnt > 10 {
- time.Sleep(time.Duration(rewatchCnt) * time.Second)
- }
- continue
- }
-
- rewatchCnt = 0
- err := w.handleDDLJobQueue(d)
- if err != nil {
- logutil.Logger(w.logCtx).Warn("[ddl] handle DDL job failed", zap.Error(err))
- }
- }
-}
-
-func (d *ddl) asyncNotifyByEtcd(addingDDLJobKey string, job *model.Job) {
+func (d *ddlCtx) asyncNotifyByEtcd(etcdPath string, jobID int64, jobType string) {
if d.etcdCli == nil {
return
}
- jobID := strconv.FormatInt(job.ID, 10)
+ jobIDStr := strconv.FormatInt(jobID, 10)
timeStart := time.Now()
- err := util.PutKVToEtcd(d.ctx, d.etcdCli, 1, addingDDLJobKey, jobID)
+ err := util.PutKVToEtcd(d.ctx, d.etcdCli, 1, etcdPath, jobIDStr)
if err != nil {
- logutil.BgLogger().Info("[ddl] notify handling DDL job failed", zap.String("jobID", jobID), zap.Error(err))
+ logutil.BgLogger().Info("[ddl] notify handling DDL job failed",
+ zap.String("etcdPath", etcdPath), zap.Int64("jobID", jobID), zap.String("type", jobType), zap.Error(err))
}
- metrics.DDLWorkerHistogram.WithLabelValues(metrics.WorkerNotifyDDLJob, job.Type.String(), metrics.RetLabel(err)).Observe(time.Since(timeStart).Seconds())
+ metrics.DDLWorkerHistogram.WithLabelValues(metrics.WorkerNotifyDDLJob, jobType, metrics.RetLabel(err)).Observe(time.Since(timeStart).Seconds())
}
func asyncNotify(ch chan struct{}) {
@@ -239,37 +189,6 @@ func asyncNotify(ch chan struct{}) {
}
}
-// buildJobDependence sets the curjob's dependency-ID.
-// The dependency-job's ID must less than the current job's ID, and we need the largest one in the list.
-func buildJobDependence(t *meta.Meta, curJob *model.Job) error {
- // Jobs in the same queue are ordered. If we want to find a job's dependency-job, we need to look for
- // it from the other queue. So if the job is "ActionAddIndex" job, we need find its dependency-job from DefaultJobList.
- jobListKey := meta.DefaultJobListKey
- if !curJob.MayNeedReorg() {
- jobListKey = meta.AddIndexJobListKey
- }
- jobs, err := t.GetAllDDLJobsInQueue(jobListKey)
- if err != nil {
- return errors.Trace(err)
- }
-
- for _, job := range jobs {
- if curJob.ID < job.ID {
- continue
- }
- isDependent, err := curJob.IsDependentOn(job)
- if err != nil {
- return errors.Trace(err)
- }
- if isDependent {
- logutil.BgLogger().Info("[ddl] current DDL job depends on other job", zap.String("currentJob", curJob.String()), zap.String("dependentJob", job.String()))
- curJob.DependencyID = job.ID
- break
- }
- }
- return nil
-}
-
func (d *ddl) limitDDLJobs() {
defer tidbutil.Recover(metrics.LabelDDL, "limitDDLJobs", nil, true)
@@ -295,7 +214,7 @@ func (d *ddl) addBatchDDLJobs(tasks []*limitJobTask) {
startTime := time.Now()
var err error
// DDLForce2Queue is a flag to tell DDL worker to always push the job to the DDL queue.
- toTable := variable.EnableConcurrentDDL.Load() && !variable.DDLForce2Queue.Load()
+ toTable := !variable.DDLForce2Queue.Load()
if toTable {
err = d.addBatchDDLJobs2Table(tasks)
} else {
@@ -315,6 +234,37 @@ func (d *ddl) addBatchDDLJobs(tasks []*limitJobTask) {
}
}
+// buildJobDependence sets the curjob's dependency-ID.
+// The dependency-job's ID must less than the current job's ID, and we need the largest one in the list.
+func buildJobDependence(t *meta.Meta, curJob *model.Job) error {
+ // Jobs in the same queue are ordered. If we want to find a job's dependency-job, we need to look for
+ // it from the other queue. So if the job is "ActionAddIndex" job, we need find its dependency-job from DefaultJobList.
+ jobListKey := meta.DefaultJobListKey
+ if !curJob.MayNeedReorg() {
+ jobListKey = meta.AddIndexJobListKey
+ }
+ jobs, err := t.GetAllDDLJobsInQueue(jobListKey)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ for _, job := range jobs {
+ if curJob.ID < job.ID {
+ continue
+ }
+ isDependent, err := curJob.IsDependentOn(job)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if isDependent {
+ logutil.BgLogger().Info("[ddl] current DDL job depends on other job", zap.String("currentJob", curJob.String()), zap.String("dependentJob", job.String()))
+ curJob.DependencyID = job.ID
+ break
+ }
+ }
+ return nil
+}
+
func (d *ddl) addBatchDDLJobs2Queue(tasks []*limitJobTask) error {
ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
return kv.RunInNewTxn(ctx, d.store, true, func(ctx context.Context, txn kv.Transaction) error {
@@ -425,6 +375,7 @@ func (d *ddl) addBatchDDLJobs2Table(tasks []*limitJobTask) error {
jobTasks[i] = job
injectModifyJobArgFailPoint(job)
}
+
sess.SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
err = insertDDLJobs2Table(newSession(sess), true, jobTasks...)
}
@@ -443,13 +394,6 @@ func injectFailPointForGetJob(job *model.Job) {
})
}
-// getFirstDDLJob gets the first DDL job form DDL queue.
-func (w *worker) getFirstDDLJob(t *meta.Meta) (*model.Job, error) {
- job, err := t.GetDDLJobByIdx(0)
- injectFailPointForGetJob(job)
- return job, errors.Trace(err)
-}
-
// handleUpdateJobError handles the too large DDL job.
func (w *worker) handleUpdateJobError(t *meta.Meta, job *model.Job, err error) error {
if err == nil {
@@ -470,7 +414,7 @@ func (w *worker) handleUpdateJobError(t *meta.Meta, job *model.Job, err error) e
// updateDDLJob updates the DDL job information.
// Every time we enter another state except final state, we must call this function.
-func (w *worker) updateDDLJob(t *meta.Meta, job *model.Job, meetErr bool) error {
+func (w *worker) updateDDLJob(job *model.Job, meetErr bool) error {
failpoint.Inject("mockErrEntrySizeTooLarge", func(val failpoint.Value) {
if val.(bool) {
failpoint.Return(kv.ErrEntryTooLarge)
@@ -481,13 +425,7 @@ func (w *worker) updateDDLJob(t *meta.Meta, job *model.Job, meetErr bool) error
logutil.Logger(w.logCtx).Info("[ddl] meet something wrong before update DDL job, shouldn't update raw args",
zap.String("job", job.String()))
}
- var err error
- if w.concurrentDDL {
- err = updateDDLJob2Table(w.sess, job, updateRawArgs)
- } else {
- err = t.UpdateDDLJob(0, job, updateRawArgs)
- }
- return errors.Trace(err)
+ return errors.Trace(updateDDLJob2Table(w.sess, job, updateRawArgs))
}
// registerMDLInfo registers metadata lock info.
@@ -512,7 +450,7 @@ func (w *worker) registerMDLInfo(job *model.Job, ver int64) error {
}
// cleanMDLInfo cleans metadata lock info.
-func cleanMDLInfo(pool *sessionPool, jobID int64) {
+func cleanMDLInfo(pool *sessionPool, jobID int64, ec *clientv3.Client) {
if !variable.EnableMDL.Load() {
return
}
@@ -525,19 +463,30 @@ func cleanMDLInfo(pool *sessionPool, jobID int64) {
if err != nil {
logutil.BgLogger().Warn("unexpected error when clean mdl info", zap.Error(err))
}
+ if ec != nil {
+ path := fmt.Sprintf("%s/%d/", util.DDLAllSchemaVersionsByJob, jobID)
+ _, err = ec.Delete(context.Background(), path, clientv3.WithPrefix())
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] delete versions failed", zap.Any("job id", jobID), zap.Error(err))
+ }
+ }
}
// checkMDLInfo checks if metadata lock info exists. It means the schema is locked by some TiDBs if exists.
-func checkMDLInfo(jobID int64, pool *sessionPool) (bool, error) {
- sql := fmt.Sprintf("select * from mysql.tidb_mdl_info where job_id = %d", jobID)
+func checkMDLInfo(jobID int64, pool *sessionPool) (bool, int64, error) {
+ sql := fmt.Sprintf("select version from mysql.tidb_mdl_info where job_id = %d", jobID)
sctx, _ := pool.get()
defer pool.put(sctx)
sess := newSession(sctx)
rows, err := sess.execute(context.Background(), sql, "check-mdl-info")
if err != nil {
- return false, err
+ return false, 0, err
+ }
+ if len(rows) == 0 {
+ return false, 0, nil
}
- return len(rows) > 0, nil
+ ver := rows[0].GetInt64(0)
+ return true, ver, nil
}
func needUpdateRawArgs(job *model.Job, meetErr bool) bool {
@@ -571,7 +520,8 @@ func jobNeedGC(job *model.Job) bool {
switch job.Type {
case model.ActionDropSchema, model.ActionDropTable, model.ActionTruncateTable, model.ActionDropIndex, model.ActionDropPrimaryKey,
model.ActionDropTablePartition, model.ActionTruncateTablePartition, model.ActionDropColumn, model.ActionModifyColumn,
- model.ActionAddIndex, model.ActionAddPrimaryKey:
+ model.ActionAddIndex, model.ActionAddPrimaryKey,
+ model.ActionReorganizePartition:
return true
case model.ActionMultiSchemaChange:
for _, sub := range job.MultiSchemaInfo.SubJobs {
@@ -619,11 +569,7 @@ func (w *worker) finishDDLJob(t *meta.Meta, job *model.Job) (err error) {
if err != nil {
return errors.Trace(err)
}
- if w.concurrentDDL {
- err = w.deleteDDLJob(job)
- } else {
- _, err = t.DeQueueDDLJob()
- }
+ err = w.deleteDDLJob(job)
if err != nil {
return errors.Trace(err)
}
@@ -638,7 +584,7 @@ func (w *worker) finishDDLJob(t *meta.Meta, job *model.Job) (err error) {
}
w.writeDDLSeqNum(job)
w.removeJobCtx(job)
- err = AddHistoryDDLJob(w.sess, t, job, updateRawArgs, w.concurrentDDL)
+ err = AddHistoryDDLJob(w.sess, t, job, updateRawArgs)
return errors.Trace(err)
}
@@ -702,21 +648,14 @@ func isDependencyJobDone(t *meta.Meta, job *model.Job) (bool, error) {
return true, nil
}
-func newMetaWithQueueTp(txn kv.Transaction, tp workerType) *meta.Meta {
- if tp == addIdxWorker {
- return meta.NewMeta(txn, meta.AddIndexJobListKey)
- }
- return meta.NewMeta(txn)
-}
-
-func (w *JobContext) setDDLLabelForTopSQL(job *model.Job) {
- if !topsqlstate.TopSQLEnabled() || job == nil {
+func (w *JobContext) setDDLLabelForTopSQL(jobQuery string) {
+ if !topsqlstate.TopSQLEnabled() || jobQuery == "" {
return
}
- if job.Query != w.cacheSQL || w.cacheDigest == nil {
- w.cacheNormalizedSQL, w.cacheDigest = parser.NormalizeDigest(job.Query)
- w.cacheSQL = job.Query
+ if jobQuery != w.cacheSQL || w.cacheDigest == nil {
+ w.cacheNormalizedSQL, w.cacheDigest = parser.NormalizeDigest(jobQuery)
+ w.cacheSQL = jobQuery
w.ddlJobCtx = topsql.AttachAndRegisterSQLInfo(context.Background(), w.cacheNormalizedSQL, w.cacheDigest, false)
} else {
topsql.AttachAndRegisterSQLInfo(w.ddlJobCtx, w.cacheNormalizedSQL, w.cacheDigest, false)
@@ -736,20 +675,24 @@ func (w *worker) unlockSeqNum(err error) {
// DDLBackfillers contains the DDL need backfill step.
var DDLBackfillers = map[model.ActionType]string{
- model.ActionAddIndex: "add_index",
- model.ActionModifyColumn: "modify_column",
- model.ActionDropIndex: "drop_index",
+ model.ActionAddIndex: "add_index",
+ model.ActionModifyColumn: "modify_column",
+ model.ActionDropIndex: "drop_index",
+ model.ActionReorganizePartition: "reorganize_partition",
}
-func getDDLRequestSource(job *model.Job) string {
- if tp, ok := DDLBackfillers[job.Type]; ok {
+func getDDLRequestSource(jobType model.ActionType) string {
+ if tp, ok := DDLBackfillers[jobType]; ok {
return kv.InternalTxnBackfillDDLPrefix + tp
}
return kv.InternalTxnDDL
}
-func (w *JobContext) setDDLLabelForDiagnosis(job *model.Job) {
- w.tp = getDDLRequestSource(job)
+func (w *JobContext) setDDLLabelForDiagnosis(jobType model.ActionType) {
+ if w.tp != "" {
+ return
+ }
+ w.tp = getDDLRequestSource(jobType)
w.ddlJobCtx = kv.WithInternalSourceType(w.ddlJobCtx, w.ddlJobSourceType())
}
@@ -764,6 +707,7 @@ func (w *worker) HandleJobDone(d *ddlCtx, job *model.Job, t *meta.Meta) error {
if err != nil {
return err
}
+ CleanupDDLReorgHandles(job, w.sess)
asyncNotify(d.ddlJobDoneCh)
return nil
}
@@ -782,7 +726,7 @@ func (w *worker) HandleDDLJobTable(d *ddlCtx, job *model.Job) (int64, error) {
if err != nil {
return 0, err
}
- if !variable.EnableConcurrentDDL.Load() || d.waiting.Load() {
+ if d.waiting.Load() {
w.sess.rollback()
return 0, nil
}
@@ -800,10 +744,10 @@ func (w *worker) HandleDDLJobTable(d *ddlCtx, job *model.Job) (int64, error) {
if w.tp == addIdxWorker && job.IsRunning() {
txn.SetDiskFullOpt(kvrpcpb.DiskFullOpt_NotAllowedOnFull)
}
- w.setDDLLabelForTopSQL(job)
- w.setDDLSourceForDiagnosis(job)
- jobContext := w.jobContext(job)
- if tagger := w.getResourceGroupTaggerForTopSQL(job); tagger != nil {
+ w.setDDLLabelForTopSQL(job.ID, job.Query)
+ w.setDDLSourceForDiagnosis(job.ID, job.Type)
+ jobContext := w.jobContext(job.ID)
+ if tagger := w.getResourceGroupTaggerForTopSQL(job.ID); tagger != nil {
txn.SetOption(kv.ResourceGroupTagger, tagger)
}
t := meta.NewMeta(txn)
@@ -852,7 +796,7 @@ func (w *worker) HandleDDLJobTable(d *ddlCtx, job *model.Job) (int64, error) {
d.unlockSchemaVersion(job.ID)
return 0, err
}
- err = w.updateDDLJob(t, job, runJobErr != nil)
+ err = w.updateDDLJob(job, runJobErr != nil)
if err = w.handleUpdateJobError(t, job, err); err != nil {
w.sess.rollback()
d.unlockSchemaVersion(job.ID)
@@ -896,152 +840,6 @@ func (w *JobContext) ddlJobSourceType() string {
return w.tp
}
-// handleDDLJobQueue handles DDL jobs in DDL Job queue.
-func (w *worker) handleDDLJobQueue(d *ddlCtx) error {
- once := true
- waitDependencyJobCnt := 0
- for {
- if isChanClosed(w.ctx.Done()) {
- return nil
- }
-
- var (
- job *model.Job
- schemaVer int64
- runJobErr error
- )
- waitTime := 2 * d.lease
- ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
- err := kv.RunInNewTxn(ctx, d.store, false, func(ctx context.Context, txn kv.Transaction) error {
- d.runningJobs.Lock()
- // We are not owner, return and retry checking later.
- if !d.isOwner() || variable.EnableConcurrentDDL.Load() || d.waiting.Load() {
- d.runningJobs.Unlock()
- return nil
- }
-
- var err error
- t := newMetaWithQueueTp(txn, w.tp)
-
- // We become the owner. Get the first job and run it.
- job, err = w.getFirstDDLJob(t)
- if job == nil || err != nil {
- d.runningJobs.Unlock()
- return errors.Trace(err)
- }
- d.runningJobs.ids[job.ID] = struct{}{}
- d.runningJobs.Unlock()
-
- defer d.deleteRunningDDLJobMap(job.ID)
-
- // only general ddls allowed to be executed when TiKV is disk full.
- if w.tp == addIdxWorker && job.IsRunning() {
- txn.SetDiskFullOpt(kvrpcpb.DiskFullOpt_NotAllowedOnFull)
- }
-
- w.setDDLLabelForTopSQL(job)
- w.setDDLSourceForDiagnosis(job)
- jobContext := w.jobContext(job)
- if tagger := w.getResourceGroupTaggerForTopSQL(job); tagger != nil {
- txn.SetOption(kv.ResourceGroupTagger, tagger)
- }
- if isDone, err1 := isDependencyJobDone(t, job); err1 != nil || !isDone {
- return errors.Trace(err1)
- }
-
- if once {
- err = waitSchemaSynced(d, job, waitTime)
- if err == nil {
- once = false
- }
- return err
- }
-
- if job.IsDone() || job.IsRollbackDone() {
- if !job.IsRollbackDone() {
- job.State = model.JobStateSynced
- }
- err = w.finishDDLJob(t, job)
- return errors.Trace(err)
- }
-
- d.mu.RLock()
- d.mu.hook.OnJobRunBefore(job)
- d.mu.RUnlock()
-
- // set request source type to DDL type
- txn.SetOption(kv.RequestSourceType, jobContext.ddlJobSourceType())
- // If running job meets error, we will save this error in job Error
- // and retry later if the job is not cancelled.
- schemaVer, runJobErr = w.runDDLJob(d, t, job)
- if job.IsCancelled() {
- txn.Reset()
- err = w.finishDDLJob(t, job)
- return errors.Trace(err)
- }
- if runJobErr != nil && !job.IsRollingback() && !job.IsRollbackDone() {
- // If the running job meets an error
- // and the job state is rolling back, it means that we have already handled this error.
- // Some DDL jobs (such as adding indexes) may need to update the table info and the schema version,
- // then shouldn't discard the KV modification.
- // And the job state is rollback done, it means the job was already finished, also shouldn't discard too.
- // Otherwise, we should discard the KV modification when running job.
- txn.Reset()
- // If error happens after updateSchemaVersion(), then the schemaVer is updated.
- // Result in the retry duration is up to 2 * lease.
- schemaVer = 0
- }
- err = w.updateDDLJob(t, job, runJobErr != nil)
- if err = w.handleUpdateJobError(t, job, err); err != nil {
- return errors.Trace(err)
- }
- writeBinlog(d.binlogCli, txn, job)
- return nil
- })
-
- if runJobErr != nil {
- // wait a while to retry again. If we don't wait here, DDL will retry this job immediately,
- // which may act like a deadlock.
- logutil.Logger(w.logCtx).Info("[ddl] run DDL job failed, sleeps a while then retries it.",
- zap.Duration("waitTime", GetWaitTimeWhenErrorOccurred()), zap.Error(runJobErr))
- time.Sleep(GetWaitTimeWhenErrorOccurred())
- }
- if job != nil {
- d.unlockSchemaVersion(job.ID)
- }
-
- if err != nil {
- w.unlockSeqNum(err)
- return errors.Trace(err)
- } else if job == nil {
- // No job now, return and retry getting later.
- return nil
- }
- w.unlockSeqNum(err)
- w.waitDependencyJobFinished(job, &waitDependencyJobCnt)
-
- // Here means the job enters another state (delete only, write only, public, etc...) or is cancelled.
- // If the job is done or still running or rolling back, we will wait 2 * lease time to guarantee other servers to update
- // the newest schema.
- waitSchemaChanged(context.Background(), d, waitTime, schemaVer, job)
-
- if RunInGoTest {
- // d.mu.hook is initialed from domain / test callback, which will force the owner host update schema diff synchronously.
- d.mu.RLock()
- d.mu.hook.OnSchemaStateChanged(schemaVer)
- d.mu.RUnlock()
- }
-
- d.mu.RLock()
- d.mu.hook.OnJobUpdated(job)
- d.mu.RUnlock()
-
- if job.IsSynced() || job.IsCancelled() || job.IsRollbackDone() {
- asyncNotify(d.ddlJobDoneCh)
- }
- }
-}
-
func skipWriteBinlog(job *model.Job) bool {
switch job.Type {
// ActionUpdateTiFlashReplicaStatus is a TiDB internal DDL,
@@ -1232,7 +1030,7 @@ func (w *worker) runDDLJob(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
case model.ActionTruncateTable:
ver, err = onTruncateTable(d, t, job)
case model.ActionRebaseAutoID:
- ver, err = onRebaseRowIDType(d, t, job)
+ ver, err = onRebaseAutoIncrementIDType(d, t, job)
case model.ActionRebaseAutoRandomBase:
ver, err = onRebaseAutoRandomType(d, t, job)
case model.ActionRenameTable:
@@ -1279,6 +1077,12 @@ func (w *worker) runDDLJob(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
ver, err = onAlterTablePartitionPlacement(d, t, job)
case model.ActionAlterTablePlacement:
ver, err = onAlterTablePlacement(d, t, job)
+ case model.ActionCreateResourceGroup:
+ ver, err = onCreateResourceGroup(d, t, job)
+ case model.ActionAlterResourceGroup:
+ ver, err = onAlterResourceGroup(d, t, job)
+ case model.ActionDropResourceGroup:
+ ver, err = onDropResourceGroup(d, t, job)
case model.ActionAlterCacheTable:
ver, err = onAlterCacheTable(d, t, job)
case model.ActionAlterNoCacheTable:
@@ -1287,6 +1091,12 @@ func (w *worker) runDDLJob(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
ver, err = w.onFlashbackCluster(d, t, job)
case model.ActionMultiSchemaChange:
ver, err = onMultiSchemaChange(w, d, t, job)
+ case model.ActionReorganizePartition:
+ ver, err = w.onReorganizePartition(d, t, job)
+ case model.ActionAlterTTLInfo:
+ ver, err = onTTLInfoChange(d, t, job)
+ case model.ActionAlterTTLRemove:
+ ver, err = onTTLInfoRemove(d, t, job)
default:
// Invalid job, cancel it.
job.State = model.JobStateCancelled
@@ -1365,6 +1175,32 @@ func waitSchemaChanged(ctx context.Context, d *ddlCtx, waitTime time.Duration, l
zap.String("job", job.String()))
}
+// waitSchemaSyncedForMDL likes waitSchemaSynced, but it waits for getting the metadata lock of the latest version of this DDL.
+func waitSchemaSyncedForMDL(d *ddlCtx, job *model.Job, latestSchemaVersion int64) error {
+ failpoint.Inject("checkDownBeforeUpdateGlobalVersion", func(val failpoint.Value) {
+ if val.(bool) {
+ if mockDDLErrOnce > 0 && mockDDLErrOnce != latestSchemaVersion {
+ panic("check down before update global version failed")
+ } else {
+ mockDDLErrOnce = -1
+ }
+ }
+ })
+
+ timeStart := time.Now()
+ // OwnerCheckAllVersions returns only when all TiDB schemas are synced(exclude the isolated TiDB).
+ err := d.schemaSyncer.OwnerCheckAllVersions(context.Background(), job.ID, latestSchemaVersion)
+ if err != nil {
+ logutil.Logger(d.ctx).Info("[ddl] wait latest schema version encounter error", zap.Int64("ver", latestSchemaVersion), zap.Error(err))
+ return err
+ }
+ logutil.Logger(d.ctx).Info("[ddl] wait latest schema version changed(get the metadata lock if tidb_enable_metadata_lock is true)",
+ zap.Int64("ver", latestSchemaVersion),
+ zap.Duration("take time", time.Since(timeStart)),
+ zap.String("job", job.String()))
+ return nil
+}
+
// waitSchemaSynced handles the following situation:
// If the job enters a new state, and the worker crashs when it's in the process of waiting for 2 * lease time,
// Then the worker restarts quickly, we may run the job immediately again,
@@ -1530,6 +1366,22 @@ func updateSchemaVersion(d *ddlCtx, t *meta.Meta, job *model.Job, multiInfos ...
diff.AffectedOpts = buildPlacementAffects(oldIDs, oldIDs)
}
}
+ case model.ActionReorganizePartition:
+ diff.TableID = job.TableID
+ if len(job.CtxVars) > 0 {
+ if droppedIDs, ok := job.CtxVars[0].([]int64); ok {
+ if addedIDs, ok := job.CtxVars[1].([]int64); ok {
+ // to use AffectedOpts we need both new and old to have the same length
+ maxParts := mathutil.Max[int](len(droppedIDs), len(addedIDs))
+ // Also initialize them to 0!
+ oldIDs := make([]int64, maxParts)
+ copy(oldIDs, droppedIDs)
+ newIDs := make([]int64, maxParts)
+ copy(newIDs, addedIDs)
+ diff.AffectedOpts = buildPlacementAffects(oldIDs, newIDs)
+ }
+ }
+ }
case model.ActionCreateTable:
diff.TableID = job.TableID
if len(job.Args) > 0 {
@@ -1564,6 +1416,11 @@ func updateSchemaVersion(d *ddlCtx, t *meta.Meta, job *model.Job, multiInfos ...
OldTableID: recoverTabsInfo[i].TableInfo.ID,
}
}
+ case model.ActionFlashbackCluster:
+ diff.TableID = -1
+ if job.SchemaState == model.StatePublic {
+ diff.RegenerateSchemaMap = true
+ }
default:
diff.TableID = job.TableID
}
diff --git a/ddl/ddl_worker_test.go b/ddl/ddl_worker_test.go
index e07d1661f7d99..b946490e18e25 100644
--- a/ddl/ddl_worker_test.go
+++ b/ddl/ddl_worker_test.go
@@ -23,6 +23,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/sessionctx"
@@ -97,7 +98,7 @@ func TestParallelDDL(t *testing.T) {
// set hook to execute jobs after all jobs are in queue.
jobCnt := 11
- tc := &ddl.TestDDLCallback{Do: dom}
+ tc := &callback.TestDDLCallback{Do: dom}
once := sync.Once{}
var checkErr error
tc.OnJobRunBeforeExported = func(job *model.Job) {
@@ -308,3 +309,31 @@ func TestJobNeedGC(t *testing.T) {
}}}
require.True(t, ddl.JobNeedGCForTest(job))
}
+
+func TestUsingReorgCtx(t *testing.T) {
+ _, domain := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
+ d := domain.DDL()
+
+ wg := util.WaitGroupWrapper{}
+ wg.Run(func() {
+ jobID := int64(1)
+ m := &model.BackfillMeta{StartKey: []byte("skey"), RowCount: 1}
+ bfJob := &ddl.BackfillJob{JobID: jobID, EleID: 1, EleKey: nil, Meta: m}
+ for i := 0; i < 100; i++ {
+ d.(ddl.DDLForTest).SetReorgCtxForBackfill(bfJob)
+ d.(ddl.DDLForTest).GetReorgCtx(jobID).IsReorgCanceled()
+ d.(ddl.DDLForTest).RemoveReorgCtx(jobID)
+ }
+ })
+ wg.Run(func() {
+ jobID := int64(1)
+ startKey := []byte("skey")
+ ele := &meta.Element{ID: 1, TypeKey: nil}
+ for i := 0; i < 100; i++ {
+ d.(ddl.DDLForTest).NewReorgCtx(jobID, startKey, ele, 0)
+ d.(ddl.DDLForTest).GetReorgCtx(jobID).IsReorgCanceled()
+ d.(ddl.DDLForTest).RemoveReorgCtx(jobID)
+ }
+ })
+ wg.Wait()
+}
diff --git a/ddl/ddl_workerpool.go b/ddl/ddl_workerpool.go
index de709b6faeb3b..3ed4d8f499fa9 100644
--- a/ddl/ddl_workerpool.go
+++ b/ddl/ddl_workerpool.go
@@ -88,38 +88,38 @@ func (wp *workerPool) tp() jobType {
return wp.t
}
-// backfilWorkerPool is used to new backfill worker.
-type backfilWorkerPool struct {
+// backfillCtxPool is used to new backfill context.
+type backfillCtxPool struct {
exit atomic.Bool
resPool *pools.ResourcePool
}
-func newBackfillWorkerPool(resPool *pools.ResourcePool) *backfilWorkerPool {
- return &backfilWorkerPool{
+func newBackfillContextPool(resPool *pools.ResourcePool) *backfillCtxPool {
+ return &backfillCtxPool{
exit: *atomic.NewBool(false),
resPool: resPool,
}
}
// setCapacity changes the capacity of the pool.
-// A setCapacity of 0 is equivalent to closing the backfilWorkerPool.
-func (bwp *backfilWorkerPool) setCapacity(capacity int) error {
- return bwp.resPool.SetCapacity(capacity)
+// A setCapacity of 0 is equivalent to closing the backfillCtxPool.
+func (bcp *backfillCtxPool) setCapacity(capacity int) error {
+ return bcp.resPool.SetCapacity(capacity)
}
-// get gets backfilWorkerPool from context resource pool.
-// Please remember to call put after you finished using backfilWorkerPool.
-func (bwp *backfilWorkerPool) get() (*backfillWorker, error) {
- if bwp.resPool == nil {
+// get gets backfillCtxPool from context resource pool.
+// Please remember to call put after you finished using backfillCtxPool.
+func (bcp *backfillCtxPool) get() (*backfillWorker, error) {
+ if bcp.resPool == nil {
return nil, nil
}
- if bwp.exit.Load() {
+ if bcp.exit.Load() {
return nil, errors.Errorf("backfill worker pool is closed")
}
- // no need to protect bwp.resPool
- resource, err := bwp.resPool.TryGet()
+ // no need to protect bcp.resPool
+ resource, err := bcp.resPool.TryGet()
if err != nil {
return nil, errors.Trace(err)
}
@@ -131,24 +131,55 @@ func (bwp *backfilWorkerPool) get() (*backfillWorker, error) {
return worker, nil
}
+// batchGet gets a batch backfillWorkers from context resource pool.
+// Please remember to call batchPut after you finished using backfillWorkerPool.
+func (bcp *backfillCtxPool) batchGet(cnt int) ([]*backfillWorker, error) {
+ if bcp.resPool == nil {
+ return nil, nil
+ }
+
+ if bcp.exit.Load() {
+ return nil, errors.Errorf("backfill worker pool is closed")
+ }
+
+ workers := make([]*backfillWorker, 0, cnt)
+ for i := 0; i < cnt; i++ {
+ // no need to protect bcp.resPool
+ res, err := bcp.resPool.TryGet()
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if res == nil {
+ if len(workers) == 0 {
+ return nil, nil
+ }
+ return workers, nil
+ }
+ worker := res.(*backfillWorker)
+ workers = append(workers, worker)
+ }
+
+ return workers, nil
+}
+
// put returns workerPool to context resource pool.
-func (bwp *backfilWorkerPool) put(wk *backfillWorker) {
- if bwp.resPool == nil || bwp.exit.Load() {
+func (bcp *backfillCtxPool) put(wk *backfillWorker) {
+ if bcp.resPool == nil || bcp.exit.Load() {
return
}
- // No need to protect bwp.resPool, even the bwp.resPool is closed, the ctx still need to
+ // No need to protect bcp.resPool, even the bcp.resPool is closed, the ctx still need to
// put into resPool, because when resPool is closing, it will wait all the ctx returns, then resPool finish closing.
- bwp.resPool.Put(wk)
+ bcp.resPool.Put(wk)
}
-// close clean up the backfilWorkerPool.
-func (bwp *backfilWorkerPool) close() {
+// close clean up the backfillCtxPool.
+func (bcp *backfillCtxPool) close() {
// Prevent closing resPool twice.
- if bwp.resPool == nil || bwp.exit.Load() {
+ if bcp.resPool == nil || bcp.exit.Load() {
return
}
- bwp.exit.Store(true)
+ bcp.exit.Store(true)
logutil.BgLogger().Info("[ddl] closing workerPool")
- bwp.resPool.Close()
+ bcp.resPool.Close()
}
diff --git a/ddl/ddl_workerpool_test.go b/ddl/ddl_workerpool_test.go
index e9f324ce9dff8..8f86e816507d5 100644
--- a/ddl/ddl_workerpool_test.go
+++ b/ddl/ddl_workerpool_test.go
@@ -19,14 +19,13 @@ import (
"testing"
"github.com/ngaut/pools"
- "github.com/pingcap/tidb/parser/model"
"github.com/stretchr/testify/require"
)
func TestDDLWorkerPool(t *testing.T) {
f := func() func() (pools.Resource, error) {
return func() (pools.Resource, error) {
- wk := newWorker(nil, addIdxWorker, nil, nil, nil, true)
+ wk := newWorker(nil, addIdxWorker, nil, nil, nil)
return wk, nil
}
}
@@ -36,39 +35,56 @@ func TestDDLWorkerPool(t *testing.T) {
}
func TestBackfillWorkerPool(t *testing.T) {
- reorgInfo := &reorgInfo{Job: &model.Job{ID: 1}}
f := func() func() (pools.Resource, error) {
return func() (pools.Resource, error) {
- wk := newBackfillWorker(context.Background(), nil, 1, nil, reorgInfo, typeAddIndexWorker)
+ wk := newBackfillWorker(context.Background(), nil)
return wk, nil
}
}
- pool := newBackfillWorkerPool(pools.NewResourcePool(f(), 1, 2, 0))
- bwp, err := pool.get()
+ pool := newBackfillContextPool(pools.NewResourcePool(f(), 3, 4, 0))
+ bc, err := pool.get()
require.NoError(t, err)
- require.Equal(t, 1, bwp.id)
+ require.NotNil(t, bc)
+ require.Nil(t, bc.backfiller)
+ bcs, err := pool.batchGet(3)
+ require.NoError(t, err)
+ require.Len(t, bcs, 2)
// test it to reach the capacity
- bwp1, err := pool.get()
+ bc1, err := pool.get()
+ require.NoError(t, err)
+ require.Nil(t, bc1)
+ bcs1, err := pool.batchGet(1)
require.NoError(t, err)
- require.Nil(t, bwp1)
+ require.Nil(t, bcs1)
// test setCapacity
- err = pool.setCapacity(2)
+ err = pool.setCapacity(5)
+ require.Equal(t, "capacity 5 is out of range", err.Error())
+ err = pool.setCapacity(4)
require.NoError(t, err)
- bwp1, err = pool.get()
+ bc1, err = pool.get()
require.NoError(t, err)
- require.Equal(t, 1, bwp1.id)
- pool.put(bwp)
- pool.put(bwp1)
+ require.NotNil(t, bc)
+ require.Nil(t, bc.backfiller)
+ pool.put(bc)
+ pool.put(bc1)
+ for _, bc := range bcs {
+ pool.put(bc)
+ }
// test close
pool.close()
pool.close()
require.Equal(t, true, pool.exit.Load())
- pool.put(bwp1)
+ pool.put(bc1)
+
+ bc, err = pool.get()
+ require.Error(t, err)
+ require.Equal(t, "backfill worker pool is closed", err.Error())
+ require.Nil(t, bc)
- bwp, err = pool.get()
+ bcs, err = pool.batchGet(1)
require.Error(t, err)
require.Equal(t, "backfill worker pool is closed", err.Error())
- require.Nil(t, bwp)
+ require.Nil(t, bcs)
}
diff --git a/ddl/delete_range.go b/ddl/delete_range.go
index 669ff286ea9dd..8734d2c482968 100644
--- a/ddl/delete_range.go
+++ b/ddl/delete_range.go
@@ -266,7 +266,7 @@ func insertJobIntoDeleteRangeTable(ctx context.Context, sctx sessionctx.Context,
return errors.Trace(err)
}
- ctx = kv.WithInternalSourceType(ctx, getDDLRequestSource(job))
+ ctx = kv.WithInternalSourceType(ctx, getDDLRequestSource(job.Type))
s := sctx.(sqlexec.SQLExecutor)
switch job.Type {
case model.ActionDropSchema:
@@ -307,9 +307,13 @@ func insertJobIntoDeleteRangeTable(ctx context.Context, sctx sessionctx.Context,
endKey := tablecodec.EncodeTablePrefix(tableID + 1)
elemID := ea.allocForPhysicalID(tableID)
return doInsert(ctx, s, job.ID, elemID, startKey, endKey, now, fmt.Sprintf("table ID is %d", tableID))
- case model.ActionDropTablePartition, model.ActionTruncateTablePartition:
+ case model.ActionDropTablePartition, model.ActionTruncateTablePartition, model.ActionReorganizePartition:
var physicalTableIDs []int64
- if err := job.DecodeArgs(&physicalTableIDs); err != nil {
+ // partInfo is not used, but is set in ReorgPartition.
+ // Better to have an additional argument in job.DecodeArgs since it is ignored,
+ // instead of having one to few, which will remove the data from the job arguments...
+ var partInfo model.PartitionInfo
+ if err := job.DecodeArgs(&physicalTableIDs, &partInfo); err != nil {
return errors.Trace(err)
}
for _, physicalTableID := range physicalTableIDs {
diff --git a/ddl/dist_backfilling.go b/ddl/dist_backfilling.go
new file mode 100644
index 0000000000000..9b2524beb86c5
--- /dev/null
+++ b/ddl/dist_backfilling.go
@@ -0,0 +1,272 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "sync"
+ "time"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/tidb/ddl/ingest"
+ "github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/resourcemanager/pooltask"
+ "github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/util"
+ "github.com/pingcap/tidb/util/dbterror"
+ "github.com/pingcap/tidb/util/gpool"
+ "github.com/pingcap/tidb/util/gpool/spmc"
+ "github.com/pingcap/tidb/util/logutil"
+ "go.uber.org/zap"
+)
+
+const getJobWithoutPartition = -1
+
+type backfillWorkerContext struct {
+ currID int
+ mu sync.Mutex
+ sessCtxs []sessionctx.Context
+ backfillWorkers []*backfillWorker
+}
+
+type newBackfillerFunc func(bfCtx *backfillCtx) (bf backfiller, err error)
+
+func newBackfillWorkerContext(d *ddl, schemaName string, tbl table.Table, workerCnt int, jobID int64, bfMeta *model.BackfillMeta,
+ bfFunc newBackfillerFunc) (*backfillWorkerContext, error) {
+ if workerCnt <= 0 {
+ return nil, nil
+ }
+
+ bwCtx := &backfillWorkerContext{backfillWorkers: make([]*backfillWorker, 0, workerCnt), sessCtxs: make([]sessionctx.Context, 0, workerCnt)}
+ var err error
+ defer func() {
+ if err != nil {
+ bwCtx.close(d)
+ }
+ }()
+
+ for i := 0; i < workerCnt; i++ {
+ var se sessionctx.Context
+ se, err = d.sessPool.get()
+ if err != nil {
+ logutil.BgLogger().Error("[ddl] new backfill worker context, get a session failed", zap.Int64("jobID", jobID), zap.Error(err))
+ return nil, errors.Trace(err)
+ }
+ bwCtx.sessCtxs = append(bwCtx.sessCtxs, se)
+ err = initSessCtx(se, bfMeta.SQLMode, bfMeta.Location)
+ if err != nil {
+ logutil.BgLogger().Error("[ddl] new backfill worker context, init the session ctx failed", zap.Int64("jobID", jobID), zap.Error(err))
+ return nil, errors.Trace(err)
+ }
+
+ var bf backfiller
+ bf, err = bfFunc(newBackfillCtx(d.ddlCtx, 0, se, bfMeta.ReorgTp, schemaName, tbl))
+ if err != nil {
+ if canSkipError(jobID, len(bwCtx.backfillWorkers), err) {
+ err = nil
+ continue
+ }
+ logutil.BgLogger().Error("[ddl] new backfill worker context, do bfFunc failed", zap.Int64("jobID", jobID), zap.Error(err))
+ return nil, errors.Trace(err)
+ }
+ var bCtx *backfillWorker
+ bCtx, err = d.backfillCtxPool.get()
+ if err != nil || bCtx == nil {
+ logutil.BgLogger().Info("[ddl] new backfill worker context, get backfill context failed", zap.Int64("jobID", jobID), zap.Error(err))
+ err = nil
+ break
+ }
+ bCtx.backfiller = bf
+ bwCtx.backfillWorkers = append(bwCtx.backfillWorkers, bCtx)
+ }
+ return bwCtx, nil
+}
+
+func (bwCtx *backfillWorkerContext) GetContext() *backfillWorker {
+ bwCtx.mu.Lock()
+ // TODO: Special considerations are required if the number of consumers we get from the backfillWorkerPool is increased.
+ offset := bwCtx.currID % len(bwCtx.backfillWorkers)
+ // To prevent different workers from using the same session.
+ bw := bwCtx.backfillWorkers[offset]
+ logutil.BgLogger().Info("[ddl] backfill worker get context", zap.Int("workerCount", len(bwCtx.backfillWorkers)),
+ zap.Int("currID", bwCtx.currID), zap.Int("offset", offset), zap.Stringer("backfill worker", bw))
+ bwCtx.currID++
+ bwCtx.mu.Unlock()
+ return bw
+}
+
+func runBackfillJobs(d *ddl, sess *session, ingestBackendCtx *ingest.BackendContext, bJob *BackfillJob, jobCtx *JobContext) (table.Table, error) {
+ dbInfo, tbl, err := d.getTableByTxn(d.store, bJob.Meta.SchemaID, bJob.Meta.TableID)
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] runBackfillJobs gets table failed", zap.String("bfJob", bJob.AbbrStr()), zap.Error(err))
+ return nil, err
+ }
+
+ workerCnt := int(variable.GetDDLReorgWorkerCounter())
+ // TODO: Different worker using different newBackfillerFunc.
+ workerCtx, err := newAddIndexWorkerContext(d, dbInfo.Name, tbl, workerCnt, bJob, jobCtx)
+ if err != nil || workerCtx == nil {
+ logutil.BgLogger().Info("[ddl] new adding index worker context failed", zap.Reflect("workerCtx", workerCtx), zap.Error(err))
+ return nil, errors.Trace(err)
+ }
+ workerCnt = len(workerCtx.backfillWorkers)
+ bwMgr := newBackfilWorkerManager(workerCtx)
+ d.backfillWorkerPool.SetConsumerFunc(func(task *reorgBackfillTask, _ int, bfWorker *backfillWorker) *backfillResult {
+ return bfWorker.runTask(task)
+ })
+
+ runningPID := int64(0)
+ // If txn-merge we needn't to claim the backfill job through the partition table
+ if ingestBackendCtx == nil {
+ runningPID = getJobWithoutPartition
+ }
+ proFunc := func() ([]*reorgBackfillTask, error) {
+ // TODO: After BackfillJob replaces reorgBackfillTask, use backfiller's GetTasks instead of it.
+ return GetTasks(d.ddlCtx, sess, tbl, bJob.JobID, &runningPID, workerCnt+5)
+ }
+ // add new task
+ resultCh, control := d.backfillWorkerPool.AddProduceBySlice(proFunc, 0, workerCtx, spmc.WithConcurrency(workerCnt))
+ bwMgr.waitFinalResult(resultCh, ingestBackendCtx, bJob.EleID, control)
+
+ // waiting task finishing
+ control.Wait()
+ err = bwMgr.close(d)
+
+ return tbl, err
+}
+
+func (bwCtx *backfillWorkerContext) close(d *ddl) {
+ for _, s := range bwCtx.sessCtxs {
+ d.sessPool.put(s)
+ }
+ for _, w := range bwCtx.backfillWorkers {
+ d.backfillCtxPool.put(w)
+ }
+}
+
+type backfilWorkerManager struct {
+ bwCtx *backfillWorkerContext
+ wg util.WaitGroupWrapper
+ unsyncErr error
+ exitCh chan struct{}
+}
+
+func newBackfilWorkerManager(bwCtx *backfillWorkerContext) *backfilWorkerManager {
+ return &backfilWorkerManager{
+ bwCtx: bwCtx,
+ exitCh: make(chan struct{}),
+ }
+}
+
+func (bwm *backfilWorkerManager) waitFinalResult(resultCh <-chan *backfillResult, ingestBackendCtx *ingest.BackendContext, eleID int64,
+ tControl pooltask.TaskController[*reorgBackfillTask, *backfillResult, int, *backfillWorker, *backfillWorkerContext]) {
+ bwm.wg.Run(func() {
+ i := 0
+ workerCnt := len(bwm.bwCtx.backfillWorkers)
+
+ for {
+ select {
+ case result, ok := <-resultCh:
+ if !ok {
+ return
+ }
+ if result.err != nil {
+ logutil.BgLogger().Warn("handle backfill task failed", zap.Error(result.err))
+ bwm.unsyncErr = result.err
+ tControl.Stop()
+ return
+ }
+
+ if ingestBackendCtx != nil && i%workerCnt == 0 {
+ err := ingestBackendCtx.Flush(eleID)
+ if err != nil {
+ bwm.unsyncErr = err
+ return
+ }
+ }
+ i++
+ case <-bwm.exitCh:
+ return
+ }
+ }
+ })
+}
+
+func (bwm *backfilWorkerManager) close(d *ddl) error {
+ close(bwm.exitCh)
+ bwm.wg.Wait()
+
+ bwm.bwCtx.close(d)
+
+ return bwm.unsyncErr
+}
+
+// backfillJob2Task builds reorg task.
+func (dc *ddlCtx) backfillJob2Task(t table.Table, bfJob *BackfillJob) (*reorgBackfillTask, error) {
+ pt := t.(table.PhysicalTable)
+ if tbl, ok := t.(table.PartitionedTable); ok {
+ pt = tbl.GetPartition(bfJob.PhysicalTableID)
+ if pt == nil {
+ return nil, dbterror.ErrCancelledDDLJob.GenWithStack("Can not find partition id %d for table %d", bfJob.PhysicalTableID, t.Meta().ID)
+ }
+ }
+ return &reorgBackfillTask{
+ bfJob: bfJob,
+ physicalTable: pt,
+ // TODO: Remove these fields after remove the old logic.
+ sqlQuery: bfJob.Meta.Query,
+ startKey: bfJob.Meta.StartKey,
+ endKey: bfJob.Meta.EndKey,
+ endInclude: bfJob.Meta.EndInclude,
+ priority: bfJob.Meta.Priority}, nil
+}
+
+// GetTasks gets the backfill tasks associated with the non-runningJobID.
+func GetTasks(d *ddlCtx, sess *session, tbl table.Table, runningJobID int64, runningPID *int64, concurrency int) ([]*reorgBackfillTask, error) {
+ // TODO: At present, only add index is processed. In the future, different elements need to be distinguished.
+ var err error
+ for i := 0; i < retrySQLTimes; i++ {
+ bJobs, err := GetAndMarkBackfillJobsForOneEle(sess, concurrency, runningJobID, d.uuid, *runningPID, InstanceLease)
+ if err != nil {
+ // TODO: add test: if all tidbs can't get the unmark backfill job(a tidb mark a backfill job, other tidbs returned, then the tidb can't handle this job.)
+ if dbterror.ErrDDLJobNotFound.Equal(err) {
+ logutil.BgLogger().Info("no backfill job, handle backfill task finished")
+ return nil, gpool.ErrProducerClosed
+ }
+ if kv.ErrWriteConflict.Equal(err) {
+ logutil.BgLogger().Info("GetAndMarkBackfillJobsForOneEle failed", zap.Error(err))
+ time.Sleep(RetrySQLInterval)
+ continue
+ }
+ }
+
+ if *runningPID != getJobWithoutPartition {
+ *runningPID = bJobs[0].PhysicalTableID
+ }
+ tasks := make([]*reorgBackfillTask, 0, len(bJobs))
+ for _, bJ := range bJobs {
+ task, err := d.backfillJob2Task(tbl, bJ)
+ if err != nil {
+ return nil, err
+ }
+ tasks = append(tasks, task)
+ }
+ return tasks, nil
+ }
+
+ return nil, err
+}
diff --git a/ddl/dist_owner.go b/ddl/dist_owner.go
new file mode 100644
index 0000000000000..43a6dff87bc97
--- /dev/null
+++ b/ddl/dist_owner.go
@@ -0,0 +1,532 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "context"
+ "encoding/hex"
+ "fmt"
+ "strconv"
+ "time"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/meta"
+ "github.com/pingcap/tidb/metrics"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/parser/mysql"
+ "github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/table"
+ tidbutil "github.com/pingcap/tidb/util"
+ "github.com/pingcap/tidb/util/dbterror"
+ "github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/mathutil"
+ atomicutil "go.uber.org/atomic"
+ "go.uber.org/zap"
+)
+
+// CheckBackfillJobFinishInterval is export for test.
+var (
+ CheckBackfillJobFinishInterval = 300 * time.Millisecond
+ telemetryDistReorgUsage = metrics.TelemetryDistReorgCnt
+)
+
+const (
+ distPhysicalTableConcurrency = 16
+)
+
+func initDistReorg(reorgMeta *model.DDLReorgMeta) {
+ isDistReorg := variable.DDLEnableDistributeReorg.Load()
+ reorgMeta.IsDistReorg = isDistReorg
+ if isDistReorg {
+ metrics.TelemetryDistReorgCnt.Inc()
+ }
+}
+
+// BackfillJobRangeMeta is export for test.
+type BackfillJobRangeMeta struct {
+ ID int64
+ PhyTblID int64
+ PhyTbl table.PhysicalTable
+ StartKey []byte
+ EndKey []byte
+}
+
+func (m *BackfillJobRangeMeta) String() string {
+ physicalID := strconv.FormatInt(m.PhyTblID, 10)
+ startKey := hex.EncodeToString(m.StartKey)
+ endKey := hex.EncodeToString(m.EndKey)
+ rangeStr := "taskID_" + strconv.Itoa(int(m.ID)) + "_physicalTableID_" + physicalID + "_" + "[" + startKey + "," + endKey + ")"
+ return rangeStr
+}
+
+type splitJobContext struct {
+ ctx context.Context
+ cancel context.CancelFunc
+ isMultiPhyTbl bool
+ bfWorkerType backfillerType
+ isUnique bool
+ batchSize int
+ minBatchSize int
+ currBackfillJobID *atomicutil.Int64
+ currPhysicalID int64
+ phyTblMetaCh chan *BackfillJobRangeMeta
+ resultCh chan error
+}
+
+func getRunningPhysicalTableMetas(sess *session, sJobCtx *splitJobContext, reorgInfo *reorgInfo) ([]*BackfillJobRangeMeta, error) {
+ ddlJobID, eleID, eleKey, currPID := reorgInfo.Job.ID, reorgInfo.currElement.ID, reorgInfo.currElement.TypeKey, reorgInfo.PhysicalTableID
+ pTblMetas, err := GetPhysicalTableMetas(sess, ddlJobID, eleID, eleKey)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ currBfJobID := int64(1)
+ physicalTIDs := make([]int64, 0, len(pTblMetas))
+ phyTblMetas := make([]*BackfillJobRangeMeta, 0, len(pTblMetas))
+ if len(pTblMetas) == 0 {
+ bfJM := &BackfillJobRangeMeta{PhyTblID: currPID, StartKey: reorgInfo.StartKey, EndKey: reorgInfo.EndKey}
+ phyTblMetas = append(phyTblMetas, bfJM)
+ physicalTIDs = append(physicalTIDs, bfJM.PhyTblID)
+ } else {
+ for _, pMeta := range pTblMetas {
+ phyTblMetas = append(phyTblMetas, pMeta)
+ currPID = mathutil.Max(pMeta.PhyTblID, currPID)
+ currBfJobID = mathutil.Max(pMeta.ID, currBfJobID)
+ physicalTIDs = append(physicalTIDs, pMeta.PhyTblID)
+ }
+ }
+ sJobCtx.currPhysicalID = currPID
+ sJobCtx.currBackfillJobID = atomicutil.NewInt64(currBfJobID)
+ logutil.BgLogger().Info("[ddl] unprocessed physical table ranges get from table", zap.Int64("jobID", ddlJobID),
+ zap.Int64("eleID", eleID), zap.ByteString("eleKey", eleKey),
+ zap.Int64("currPID", sJobCtx.currPhysicalID), zap.Int64s("phyTblIDs", physicalTIDs))
+ return phyTblMetas, nil
+}
+
+func (dc *ddlCtx) sendPhysicalTableMetas(reorgInfo *reorgInfo, t table.Table, sJobCtx *splitJobContext, runningPTblMetas []*BackfillJobRangeMeta) {
+ var err error
+ physicalTIDs := make([]int64, 0, distPhysicalTableConcurrency)
+ defer func() {
+ logutil.BgLogger().Info("[ddl] send physical table ranges to split finished", zap.Int64("jobID", reorgInfo.Job.ID),
+ zap.Stringer("ele", reorgInfo.currElement), zap.Int64s("phyTblIDs", physicalTIDs), zap.Error(err))
+ if err != nil {
+ sJobCtx.cancel()
+ } else {
+ close(sJobCtx.phyTblMetaCh)
+ }
+ }()
+
+ for _, pTblM := range runningPTblMetas {
+ err = dc.isReorgRunnable(reorgInfo.Job.ID, false)
+ if err != nil {
+ return
+ }
+
+ if tbl, ok := t.(table.PartitionedTable); ok {
+ pTblM.PhyTbl = tbl.GetPartition(pTblM.PhyTblID)
+ sJobCtx.phyTblMetaCh <- pTblM
+ } else {
+ //nolint:forcetypeassert
+ phyTbl := t.(table.PhysicalTable)
+ pTblM.PhyTbl = phyTbl
+ sJobCtx.phyTblMetaCh <- pTblM
+ }
+ physicalTIDs = append(physicalTIDs, pTblM.PhyTblID)
+ }
+
+ if tbl, ok := t.(table.PartitionedTable); ok {
+ currPhysicalID := sJobCtx.currPhysicalID
+ for {
+ err = dc.isReorgRunnable(reorgInfo.Job.ID, false)
+ if err != nil {
+ return
+ }
+ select {
+ case <-sJobCtx.ctx.Done():
+ err = sJobCtx.ctx.Err()
+ return
+ default:
+ }
+
+ pID, startKey, endKey, err1 := getNextPartitionInfo(reorgInfo, tbl, currPhysicalID)
+ if err1 != nil {
+ err = err1
+ return
+ }
+ if pID == 0 {
+ // Next partition does not exist, all the job done.
+ return
+ }
+ pTbl := tbl.GetPartition(pID)
+ if pTbl == nil {
+ err = dbterror.ErrCancelledDDLJob.GenWithStack("Can not find partition id %d for table %d", pID, t.Meta().ID)
+ return
+ }
+ bfJM := &BackfillJobRangeMeta{PhyTblID: pID, PhyTbl: pTbl, StartKey: startKey, EndKey: endKey}
+ sJobCtx.phyTblMetaCh <- bfJM
+ currPhysicalID = pID
+
+ physicalTIDs = append(physicalTIDs, pID)
+ }
+ }
+}
+
+func (dc *ddlCtx) controlWriteTableRecord(sessPool *sessionPool, t table.Table, bfWorkerType backfillerType, reorgInfo *reorgInfo) error {
+ startKey, endKey := reorgInfo.StartKey, reorgInfo.EndKey
+ if startKey == nil && endKey == nil {
+ return nil
+ }
+
+ ddlJobID := reorgInfo.Job.ID
+ currEle := reorgInfo.currElement
+ logutil.BgLogger().Info("[ddl] control write table record start",
+ zap.Int64("jobID", ddlJobID), zap.Stringer("ele", currEle),
+ zap.Int64("tblID", t.Meta().ID), zap.Int64("currPID", reorgInfo.PhysicalTableID))
+ sCtx, err := sessPool.get()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ defer sessPool.put(sCtx)
+ sess := newSession(sCtx)
+
+ if err := dc.isReorgRunnable(ddlJobID, true); err != nil {
+ return errors.Trace(err)
+ }
+ var isUnique bool
+ if bfWorkerType == typeAddIndexWorker {
+ idxInfo := model.FindIndexInfoByID(t.Meta().Indices, currEle.ID)
+ isUnique = idxInfo.Unique
+ }
+
+ wg := tidbutil.WaitGroupWrapper{}
+ sJobCtx := &splitJobContext{
+ bfWorkerType: bfWorkerType,
+ isUnique: isUnique,
+ batchSize: genTaskBatch,
+ minBatchSize: minGenTaskBatch,
+ phyTblMetaCh: make(chan *BackfillJobRangeMeta, 1),
+ resultCh: make(chan error, distPhysicalTableConcurrency),
+ }
+ ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
+ sJobCtx.ctx, sJobCtx.cancel = context.WithCancel(ctx)
+ concurrency := 1
+ if tbl, ok := t.(table.PartitionedTable); ok {
+ ids := len(tbl.GetAllPartitionIDs())
+ if ids > 1 {
+ sJobCtx.isMultiPhyTbl = true
+ concurrency = ids
+ }
+ if ids > distPhysicalTableConcurrency {
+ concurrency = distPhysicalTableConcurrency
+ }
+ sJobCtx.batchSize = genPhysicalTableTaskBatch
+ sJobCtx.minBatchSize = minGenPhysicalTableTaskBatch
+ }
+
+ err = checkAndHandleInterruptedBackfillJobs(sess, ddlJobID, currEle.ID, currEle.TypeKey)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ phyTblMetas, err := getRunningPhysicalTableMetas(sess, sJobCtx, reorgInfo)
+ if err != nil {
+ return err
+ }
+
+ sCtxs := make([]sessionctx.Context, 0, concurrency)
+ for i := 0; i < concurrency; i++ {
+ sCtx, err := sessPool.get()
+ if err != nil {
+ return err
+ }
+ sCtxs = append(sCtxs, sCtx)
+ }
+
+ wg.Run(func() {
+ defer tidbutil.Recover(metrics.LabelDistReorg, "sendPhysicalTableMeta", nil, false)
+ dc.sendPhysicalTableMetas(reorgInfo, t, sJobCtx, phyTblMetas)
+ })
+ for _, sCtx := range sCtxs {
+ func(ctx sessionctx.Context) {
+ wg.Run(func() {
+ defer func() {
+ tidbutil.Recover(metrics.LabelDistReorg, "splitTableToBackfillJobs", nil, false)
+ }()
+ se := newSession(ctx)
+ dc.splitPhysicalTableToBackfillJobs(se, reorgInfo, sJobCtx)
+ })
+ }(sCtx)
+ }
+ wg.Wait()
+ for _, sCtx := range sCtxs {
+ sessPool.put(sCtx)
+ }
+ return checkReorgJobFinished(dc.ctx, sess, &dc.reorgCtx, ddlJobID, currEle)
+}
+
+func (dc *ddlCtx) splitPhysicalTableToBackfillJobs(sess *session, reorgInfo *reorgInfo, sJobCtx *splitJobContext) {
+ defaultSQLMode := sess.GetSessionVars().SQLMode
+ defer func() { sess.GetSessionVars().SQLMode = defaultSQLMode }()
+ // Make timestamp type can be inserted ZeroTimestamp.
+ sess.GetSessionVars().SQLMode = mysql.ModeNone
+
+ var err error
+ var pTblMetaCnt int
+ var pTblMeta *BackfillJobRangeMeta
+ defer func() {
+ if err != nil {
+ sJobCtx.cancel()
+ }
+ logutil.BgLogger().Info("[ddl] split backfill jobs to table finish", zap.Int64("jobID", reorgInfo.Job.ID),
+ zap.Stringer("ele", reorgInfo.currElement), zap.Int("donePTbls", pTblMetaCnt), zap.Stringer("physical_tbl", pTblMeta), zap.Error(err))
+ }()
+
+ var ok bool
+ for {
+ select {
+ case <-sJobCtx.ctx.Done():
+ err = sJobCtx.ctx.Err()
+ case pTblMeta, ok = <-sJobCtx.phyTblMetaCh:
+ if !ok {
+ return
+ }
+ if err = dc.isReorgRunnable(reorgInfo.Job.ID, false); err != nil {
+ return
+ }
+
+ err = dc.splitTableToBackfillJobs(sess, reorgInfo, sJobCtx, pTblMeta)
+ if err != nil {
+ return
+ }
+ pTblMetaCnt++
+ }
+ }
+}
+
+func checkReorgJobFinished(ctx context.Context, sess *session, reorgCtxs *reorgContexts, ddlJobID int64, currEle *meta.Element) error {
+ var times int64
+ var bfJob *BackfillJob
+ var backfillJobFinished bool
+ ticker := time.NewTicker(CheckBackfillJobFinishInterval)
+ defer ticker.Stop()
+ for {
+ if getReorgCtx(reorgCtxs, ddlJobID).isReorgCanceled() {
+ // Job is cancelled. So it can't be done.
+ return dbterror.ErrCancelledDDLJob
+ }
+
+ select {
+ case <-ticker.C:
+ times++
+ // Print this log every 5 min.
+ if times%1000 == 0 {
+ logutil.BgLogger().Info("[ddl] check all backfill jobs is finished",
+ zap.Int64("job ID", ddlJobID), zap.Bool("isFinished", backfillJobFinished), zap.Reflect("bfJob", bfJob))
+ }
+ if !backfillJobFinished {
+ err := checkAndHandleInterruptedBackfillJobs(sess, ddlJobID, currEle.ID, currEle.TypeKey)
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] finish interrupted backfill jobs", zap.Int64("job ID", ddlJobID), zap.Stringer("ele", currEle), zap.Error(err))
+ return errors.Trace(err)
+ }
+
+ bfJobs, err := getBackfillJobWithRetry(sess, BackgroundSubtaskTable, ddlJobID, currEle.ID, currEle.TypeKey)
+ if err != nil {
+ logutil.BgLogger().Info("[ddl] getBackfillJobWithRetry failed", zap.Int64("job ID", ddlJobID), zap.Stringer("ele", currEle), zap.Error(err))
+ return errors.Trace(err)
+ }
+ if len(bfJobs) == 0 {
+ backfillJobFinished = true
+ logutil.BgLogger().Info("[ddl] finish all backfill jobs", zap.Int64("job ID", ddlJobID), zap.Stringer("ele", currEle))
+ }
+ }
+ if backfillJobFinished {
+ // TODO: Consider whether these backfill jobs are always out of sync.
+ isSynced, err := checkJobIsFinished(sess, ddlJobID)
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] checkJobIsFinished failed", zap.Int64("job ID", ddlJobID), zap.Stringer("ele", currEle), zap.Error(err))
+ return errors.Trace(err)
+ }
+ if isSynced {
+ logutil.BgLogger().Info("[ddl] finish all backfill jobs and put them to history", zap.Int64("job ID", ddlJobID), zap.Stringer("ele", currEle))
+ return GetBackfillErr(sess, ddlJobID, currEle.ID, currEle.TypeKey)
+ }
+ }
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+}
+
+func checkJobIsFinished(sess *session, ddlJobID int64) (bool, error) {
+ var err error
+ var unsyncedInstanceIDs []string
+ for i := 0; i < retrySQLTimes; i++ {
+ unsyncedInstanceIDs, err = getUnsyncedInstanceIDs(sess, ddlJobID, "check_backfill_history_job_sync")
+ if err == nil && len(unsyncedInstanceIDs) == 0 {
+ return true, nil
+ }
+
+ time.Sleep(RetrySQLInterval)
+ }
+ logutil.BgLogger().Info("[ddl] checkJobIsSynced failed",
+ zap.Strings("unsyncedInstanceIDs", unsyncedInstanceIDs), zap.Int("tryTimes", retrySQLTimes), zap.Error(err))
+
+ return false, errors.Trace(err)
+}
+
+// GetBackfillErr gets the error in backfill job.
+func GetBackfillErr(sess *session, ddlJobID, currEleID int64, currEleKey []byte) error {
+ var err error
+ var metas []*model.BackfillMeta
+ for i := 0; i < retrySQLTimes; i++ {
+ metas, err = GetBackfillMetas(sess, BackgroundSubtaskHistoryTable, fmt.Sprintf("task_key like \"%d_%s_%d_%%\"",
+ ddlJobID, hex.EncodeToString(currEleKey), currEleID), "get_backfill_job_metas")
+ if err == nil {
+ for _, m := range metas {
+ if m.Error != nil {
+ return m.Error
+ }
+ }
+ return nil
+ }
+
+ logutil.BgLogger().Info("[ddl] GetBackfillMetas failed in checkJobIsSynced", zap.Int("tryTimes", i), zap.Error(err))
+ time.Sleep(RetrySQLInterval)
+ }
+
+ return errors.Trace(err)
+}
+
+func checkAndHandleInterruptedBackfillJobs(sess *session, ddlJobID, currEleID int64, currEleKey []byte) (err error) {
+ var bJobs []*BackfillJob
+ for i := 0; i < retrySQLTimes; i++ {
+ bJobs, err = GetInterruptedBackfillJobForOneEle(sess, ddlJobID, currEleID, currEleKey)
+ if err == nil {
+ break
+ }
+ logutil.BgLogger().Info("[ddl] getInterruptedBackfillJobsForOneEle failed", zap.Error(err))
+ time.Sleep(RetrySQLInterval)
+ }
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if len(bJobs) == 0 {
+ return nil
+ }
+
+ for i := 0; i < retrySQLTimes; i++ {
+ err = MoveBackfillJobsToHistoryTable(sess, bJobs[0])
+ if err == nil {
+ return bJobs[0].Meta.Error
+ }
+ logutil.BgLogger().Info("[ddl] MoveBackfillJobsToHistoryTable failed", zap.Error(err))
+ time.Sleep(RetrySQLInterval)
+ }
+ return errors.Trace(err)
+}
+
+func checkBackfillJobCount(sess *session, ddlJobID, currEleID int64, currEleKey []byte, pTblID int64) (backfillJobCnt int, err error) {
+ err = checkAndHandleInterruptedBackfillJobs(sess, ddlJobID, currEleID, currEleKey)
+ if err != nil {
+ return 0, errors.Trace(err)
+ }
+
+ backfillJobCnt, err = GetBackfillJobCount(sess, BackgroundSubtaskTable,
+ fmt.Sprintf("task_key like \"%d_%s_%d_%%\"",
+ ddlJobID, hex.EncodeToString(currEleKey), currEleID), "check_backfill_job_count")
+ if err != nil {
+ return 0, errors.Trace(err)
+ }
+
+ return backfillJobCnt, nil
+}
+
+func getBackfillJobWithRetry(sess *session, tableName string, ddlJobID, currEleID int64, currEleKey []byte) ([]*BackfillJob, error) {
+ var err error
+ var bJobs []*BackfillJob
+ for i := 0; i < retrySQLTimes; i++ {
+ bJobs, err = GetBackfillJobs(sess, tableName, fmt.Sprintf("task_key like \"%d_%s_%d_%%\" limit 1",
+ ddlJobID, hex.EncodeToString(currEleKey), currEleID), "check_backfill_job_state")
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] GetBackfillJobs failed", zap.Error(err))
+ time.Sleep(RetrySQLInterval)
+ continue
+ }
+ return bJobs, nil
+ }
+ return nil, errors.Trace(err)
+}
+
+// GetPhysicalTableMetas gets the max backfill metas per physical table in BackgroundSubtaskTable and BackgroundSubtaskHistoryTable.
+func GetPhysicalTableMetas(sess *session, ddlJobID, currEleID int64, currEleKey []byte) (map[int64]*BackfillJobRangeMeta, error) {
+ condition := fmt.Sprintf("task_key like \"%d_%s_%d_%%\"", ddlJobID, hex.EncodeToString(currEleKey), currEleID)
+ pTblMs, err := GetBackfillIDAndMetas(sess, BackgroundSubtaskTable, condition, "get_ptbl_metas")
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ hPTblMs, err := GetBackfillIDAndMetas(sess, BackgroundSubtaskHistoryTable, condition, "get_ptbl_metas")
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ metaMap := make(map[int64]*BackfillJobRangeMeta, len(pTblMs)+len(hPTblMs))
+ for _, m := range pTblMs {
+ metaMap[m.PhyTblID] = m
+ }
+ for _, m := range hPTblMs {
+ val, ok := metaMap[m.PhyTblID]
+ if !ok || (ok && m.ID > val.ID) {
+ metaMap[m.PhyTblID] = m
+ }
+ }
+ return metaMap, nil
+}
+
+// MoveBackfillJobsToHistoryTable moves backfill table jobs to the backfill history table.
+func MoveBackfillJobsToHistoryTable(sctx sessionctx.Context, bfJob *BackfillJob) error {
+ s, ok := sctx.(*session)
+ if !ok {
+ return errors.Errorf("sess ctx:%#v convert session failed", sctx)
+ }
+
+ return s.runInTxn(func(se *session) error {
+ // TODO: Consider batch by batch update backfill jobs and insert backfill history jobs.
+ bJobs, err := GetBackfillJobs(se, BackgroundSubtaskTable, fmt.Sprintf("task_key like \"%d_%s_%d_%%\"",
+ bfJob.JobID, hex.EncodeToString(bfJob.EleKey), bfJob.EleID), "update_backfill_job")
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if len(bJobs) == 0 {
+ return nil
+ }
+
+ txn, err := se.txn()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ startTS := txn.StartTS()
+ err = RemoveBackfillJob(se, true, bJobs[0])
+ if err == nil {
+ for _, bj := range bJobs {
+ bj.State = model.JobStateCancelled
+ bj.StateUpdateTS = startTS
+ }
+ err = AddBackfillHistoryJob(se, bJobs)
+ }
+ logutil.BgLogger().Info("[ddl] move backfill jobs to history table", zap.Int("job count", len(bJobs)))
+ return errors.Trace(err)
+ })
+}
diff --git a/ddl/export_test.go b/ddl/export_test.go
index 708b3474515c5..d83549609c890 100644
--- a/ddl/export_test.go
+++ b/ddl/export_test.go
@@ -14,6 +14,44 @@
package ddl
+import (
+ "context"
+
+ "github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/types"
+)
+
func SetBatchInsertDeleteRangeSize(i int) {
batchInsertDeleteRangeSize = i
}
+
+var NewCopContext4Test = newCopContext
+
+func FetchRowsFromCop4Test(copCtx *copContext, tbl table.PhysicalTable, startKey, endKey kv.Key, store kv.Storage,
+ batchSize int) ([]*indexRecord, bool, error) {
+ variable.SetDDLReorgBatchSize(int32(batchSize))
+ task := &reorgBackfillTask{
+ id: 1,
+ startKey: startKey,
+ endKey: endKey,
+ physicalTable: tbl,
+ }
+ pool := newCopReqSenderPool(context.Background(), copCtx, store)
+ pool.adjustSize(1)
+ pool.tasksCh <- task
+ idxRec, _, _, done, err := pool.fetchRowColValsFromCop(*task)
+ pool.close()
+ return idxRec, done, err
+}
+
+type IndexRecord4Test = *indexRecord
+
+func (i IndexRecord4Test) GetHandle() kv.Handle {
+ return i.handle
+}
+
+func (i IndexRecord4Test) GetIndexValues() []types.Datum {
+ return i.vals
+}
diff --git a/ddl/fail_test.go b/ddl/fail_test.go
index 39437b43a2b73..3c4ca0769bc1e 100644
--- a/ddl/fail_test.go
+++ b/ddl/fail_test.go
@@ -18,7 +18,7 @@ import (
"testing"
"github.com/pingcap/failpoint"
- "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/testkit"
"github.com/stretchr/testify/require"
@@ -38,7 +38,7 @@ func TestFailBeforeDecodeArgs(t *testing.T) {
tableID = int64(tableIDi)
d := dom.DDL()
- tc := &ddl.TestDDLCallback{Do: dom}
+ tc := &callback.TestDDLCallback{Do: dom}
first := true
stateCnt := 0
diff --git a/ddl/failtest/BUILD.bazel b/ddl/failtest/BUILD.bazel
index 025f6097ae8be..4b7980412adc0 100644
--- a/ddl/failtest/BUILD.bazel
+++ b/ddl/failtest/BUILD.bazel
@@ -2,7 +2,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test")
go_test(
name = "failtest_test",
- timeout = "short",
+ timeout = "moderate",
srcs = [
"fail_db_test.go",
"main_test.go",
diff --git a/ddl/failtest/fail_db_test.go b/ddl/failtest/fail_db_test.go
index bde5e9b1b9569..6f33f6b2107dd 100644
--- a/ddl/failtest/fail_db_test.go
+++ b/ddl/failtest/fail_db_test.go
@@ -18,6 +18,7 @@ import (
"context"
"fmt"
"math/rand"
+ "strings"
"sync/atomic"
"testing"
"time"
@@ -332,6 +333,10 @@ func TestRunDDLJobPanicDisableClusteredIndex(t *testing.T) {
}
func testAddIndexWorkerNum(t *testing.T, s *failedSuite, test func(*testkit.TestKit)) {
+ if variable.DDLEnableDistributeReorg.Load() {
+ t.Skip("dist reorg didn't support checkBackfillWorkerNum, skip this test")
+ }
+
tk := testkit.NewTestKit(t, s.store)
tk.MustExec("create database if not exists test_db")
tk.MustExec("use test_db")
@@ -493,6 +498,8 @@ func TestModifyColumn(t *testing.T) {
tk.MustExec("admin check table t")
// Test unsupported statements.
+ tk.MustExec("create table t1(a int) partition by hash (a) partitions 2")
+ tk.MustGetErrMsg("alter table t1 modify column a mediumint", "[ddl:8200]Unsupported modify column: table is partition table")
tk.MustExec("create table t2(id int, a int, b int generated always as (abs(a)) virtual, c int generated always as (a+1) stored)")
tk.MustGetErrMsg("alter table t2 modify column b mediumint", "[ddl:8200]Unsupported modify column: newCol IsGenerated false, oldCol IsGenerated true")
tk.MustGetErrMsg("alter table t2 modify column c mediumint", "[ddl:8200]Unsupported modify column: newCol IsGenerated false, oldCol IsGenerated true")
@@ -529,7 +536,35 @@ func TestModifyColumn(t *testing.T) {
tk.MustExec("insert into t5 values (1,1),(2,2),(3,3),(4,4),(5,5);")
tk.MustExec("alter table t5 modify a int not null;")
- tk.MustExec("drop table t, t2, t3, t4, t5")
+ tk.MustExec("drop table t, t1, t2, t3, t4, t5")
+}
+
+func TestIssue38699(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ //Test multi records
+ tk.MustExec("USE test")
+ tk.MustExec("set sql_mode=''")
+ tk.MustExec("DROP TABLE IF EXISTS t;")
+ tk.MustExec("CREATE TABLE t (a int)")
+ tk.MustExec("insert into t values (1000000000), (2000000)")
+ tk.MustExec("alter table t modify a tinyint")
+ result := tk.MustQuery("show warnings")
+ require.Len(t, result.Rows(), 1)
+ result.CheckWithFunc(testkit.Rows("Warning 1690 2 warnings with this error code"), func(actual []string, expected []interface{}) bool {
+ //Check if it starts with x warning(s)
+ return strings.EqualFold(actual[0], expected[0].(string)) && strings.EqualFold(actual[1], expected[1].(string)) && strings.HasPrefix(actual[2], expected[2].(string))
+ })
+
+ //Test single record
+ tk.MustExec("DROP TABLE IF EXISTS t;")
+ tk.MustExec("CREATE TABLE t (a int)")
+ tk.MustExec("insert into t values (1000000000)")
+ tk.MustExec("alter table t modify a tinyint")
+ result = tk.MustQuery("show warnings")
+ require.Len(t, result.Rows(), 1)
+ result.Check(testkit.Rows("Warning 1690 constant 1000000000 overflows tinyint"))
}
func TestPartitionAddPanic(t *testing.T) {
diff --git a/ddl/fktest/BUILD.bazel b/ddl/fktest/BUILD.bazel
new file mode 100644
index 0000000000000..9e2c14fe0718a
--- /dev/null
+++ b/ddl/fktest/BUILD.bazel
@@ -0,0 +1,30 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+ name = "fktest_test",
+ timeout = "short",
+ srcs = [
+ "foreign_key_test.go",
+ "main_test.go",
+ ],
+ flaky = True,
+ shard_count = 4,
+ deps = [
+ "//config",
+ "//ddl",
+ "//domain",
+ "//infoschema",
+ "//meta",
+ "//meta/autoid",
+ "//parser/auth",
+ "//parser/model",
+ "//planner/core",
+ "//sessiontxn",
+ "//testkit",
+ "//testkit/testsetup",
+ "//util/dbterror",
+ "@com_github_stretchr_testify//require",
+ "@com_github_tikv_client_go_v2//tikv",
+ "@org_uber_go_goleak//:goleak",
+ ],
+)
diff --git a/ddl/fktest/foreign_key_test.go b/ddl/fktest/foreign_key_test.go
new file mode 100644
index 0000000000000..489e125dc8a3c
--- /dev/null
+++ b/ddl/fktest/foreign_key_test.go
@@ -0,0 +1,1837 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl_test
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/pingcap/tidb/domain"
+ "github.com/pingcap/tidb/infoschema"
+ "github.com/pingcap/tidb/meta"
+ "github.com/pingcap/tidb/parser/auth"
+ "github.com/pingcap/tidb/parser/model"
+ plannercore "github.com/pingcap/tidb/planner/core"
+ "github.com/pingcap/tidb/sessiontxn"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/pingcap/tidb/util/dbterror"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCreateTableWithForeignKeyMetaInfo(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, a int,b int as (a) virtual);")
+ tk.MustExec("create database test2")
+ tk.MustExec("use test2")
+ tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references test.t1(id) ON UPDATE RESTRICT ON DELETE CASCADE)")
+ tb1Info := getTableInfo(t, dom, "test", "t1")
+ tb2Info := getTableInfo(t, dom, "test2", "t2")
+ require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test", "t1")))
+ require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t2")))
+ require.Equal(t, 0, len(tb1Info.ForeignKeys))
+ tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test2"),
+ ChildTable: model.NewCIStr("t2"),
+ ChildFKName: model.NewCIStr("fk_b"),
+ }, *tb1ReferredFKs[0])
+ tb2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "t2")
+ require.Equal(t, 0, len(tb2ReferredFKs))
+ require.Equal(t, 1, len(tb2Info.ForeignKeys))
+ require.Equal(t, model.FKInfo{
+ ID: 1,
+ Name: model.NewCIStr("fk_b"),
+ RefSchema: model.NewCIStr("test"),
+ RefTable: model.NewCIStr("t1"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("b")},
+ OnDelete: 2,
+ OnUpdate: 1,
+ State: model.StatePublic,
+ Version: 1,
+ }, *tb2Info.ForeignKeys[0])
+ // Auto create index for foreign key usage.
+ require.Equal(t, 1, len(tb2Info.Indices))
+ require.Equal(t, "fk_b", tb2Info.Indices[0].Name.L)
+ require.Equal(t, "`test2`.`t2`, CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `test`.`t1` (`id`) ON DELETE CASCADE ON UPDATE RESTRICT", tb2Info.ForeignKeys[0].String("test2", "t2"))
+
+ tk.MustExec("create table t3 (id int, b int, index idx_b(b), foreign key fk_b(b) references t2(id) ON UPDATE SET NULL ON DELETE NO ACTION)")
+ tb2Info = getTableInfo(t, dom, "test2", "t2")
+ tb3Info := getTableInfo(t, dom, "test2", "t3")
+ require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t2")))
+ require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t3")))
+ require.Equal(t, 1, len(tb2Info.ForeignKeys))
+ tb2ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test2", "t2")
+ require.Equal(t, 1, len(tb2ReferredFKs))
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test2"),
+ ChildTable: model.NewCIStr("t3"),
+ ChildFKName: model.NewCIStr("fk_b"),
+ }, *tb2ReferredFKs[0])
+ tb3ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "t3")
+ require.Equal(t, 0, len(tb3ReferredFKs))
+ require.Equal(t, 1, len(tb3Info.ForeignKeys))
+ require.Equal(t, model.FKInfo{
+ ID: 1,
+ Name: model.NewCIStr("fk_b"),
+ RefSchema: model.NewCIStr("test2"),
+ RefTable: model.NewCIStr("t2"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("b")},
+ OnDelete: 4,
+ OnUpdate: 3,
+ State: model.StatePublic,
+ Version: 1,
+ }, *tb3Info.ForeignKeys[0])
+ require.Equal(t, 1, len(tb3Info.Indices))
+ require.Equal(t, "idx_b", tb3Info.Indices[0].Name.L)
+ require.Equal(t, "`test2`.`t3`, CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `t2` (`id`) ON DELETE NO ACTION ON UPDATE SET NULL", tb3Info.ForeignKeys[0].String("test2", "t3"))
+
+ tk.MustExec("create table t5 (id int key, a int, b int, foreign key (a) references t5(id));")
+ tb5Info := getTableInfo(t, dom, "test2", "t5")
+ require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t5")))
+ require.Equal(t, 1, len(tb5Info.ForeignKeys))
+ tb5ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "t5")
+ require.Equal(t, 1, len(tb5ReferredFKs))
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test2"),
+ ChildTable: model.NewCIStr("t5"),
+ ChildFKName: model.NewCIStr("fk_1"),
+ }, *tb5ReferredFKs[0])
+ require.Equal(t, model.FKInfo{
+ ID: 1,
+ Name: model.NewCIStr("fk_1"),
+ RefSchema: model.NewCIStr("test2"),
+ RefTable: model.NewCIStr("t5"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("a")},
+ State: model.StatePublic,
+ Version: 1,
+ }, *tb5Info.ForeignKeys[0])
+ require.Equal(t, 1, len(tb5Info.Indices))
+ require.Equal(t, "fk_1", tb5Info.Indices[0].Name.L)
+ require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test", "t1")))
+ require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t2")))
+ require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t3")))
+ require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t5")))
+
+ tk.MustExec("set @@global.tidb_enable_foreign_key=0")
+ tk.MustExec("drop database test2")
+ require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t2")))
+ require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t3")))
+ require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t5")))
+}
+
+func TestCreateTableWithForeignKeyMetaInfo2(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("create database test2")
+ tk.MustExec("set @@foreign_key_checks=0")
+ tk.MustExec("use test2")
+ tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references test.t1(id) ON UPDATE RESTRICT ON DELETE CASCADE)")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, a int, b int as (a) virtual);")
+ tb1Info := getTableInfo(t, dom, "test", "t1")
+ tb2Info := getTableInfo(t, dom, "test2", "t2")
+ require.Equal(t, 0, len(tb1Info.ForeignKeys))
+ tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test2"),
+ ChildTable: model.NewCIStr("t2"),
+ ChildFKName: model.NewCIStr("fk_b"),
+ }, *tb1ReferredFKs[0])
+ tb2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "t2")
+ require.Equal(t, 0, len(tb2ReferredFKs))
+ require.Equal(t, 1, len(tb2Info.ForeignKeys))
+ require.Equal(t, model.FKInfo{
+ ID: 1,
+ Name: model.NewCIStr("fk_b"),
+ RefSchema: model.NewCIStr("test"),
+ RefTable: model.NewCIStr("t1"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("b")},
+ OnDelete: 2,
+ OnUpdate: 1,
+ State: model.StatePublic,
+ Version: 1,
+ }, *tb2Info.ForeignKeys[0])
+ // Auto create index for foreign key usage.
+ require.Equal(t, 1, len(tb2Info.Indices))
+ require.Equal(t, "fk_b", tb2Info.Indices[0].Name.L)
+ require.Equal(t, "`test2`.`t2`, CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `test`.`t1` (`id`) ON DELETE CASCADE ON UPDATE RESTRICT", tb2Info.ForeignKeys[0].String("test2", "t2"))
+
+ tk.MustExec("create table t3 (id int key, a int, foreign key fk_a(a) references test.t1(id) ON DELETE CASCADE ON UPDATE RESTRICT, foreign key fk_a2(a) references test2.t2(id))")
+ tb1Info = getTableInfo(t, dom, "test", "t1")
+ tb3Info := getTableInfo(t, dom, "test", "t3")
+ require.Equal(t, 0, len(tb1Info.ForeignKeys))
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 2, len(tb1ReferredFKs))
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test"),
+ ChildTable: model.NewCIStr("t3"),
+ ChildFKName: model.NewCIStr("fk_a"),
+ }, *tb1ReferredFKs[0])
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test2"),
+ ChildTable: model.NewCIStr("t2"),
+ ChildFKName: model.NewCIStr("fk_b"),
+ }, *tb1ReferredFKs[1])
+ tb3ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t3")
+ require.Equal(t, 0, len(tb3ReferredFKs))
+ require.Equal(t, 2, len(tb3Info.ForeignKeys))
+ require.Equal(t, model.FKInfo{
+ ID: 1,
+ Name: model.NewCIStr("fk_a"),
+ RefSchema: model.NewCIStr("test"),
+ RefTable: model.NewCIStr("t1"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("a")},
+ OnDelete: 2,
+ OnUpdate: 1,
+ State: model.StatePublic,
+ Version: 1,
+ }, *tb3Info.ForeignKeys[0])
+ require.Equal(t, model.FKInfo{
+ ID: 2,
+ Name: model.NewCIStr("fk_a2"),
+ RefSchema: model.NewCIStr("test2"),
+ RefTable: model.NewCIStr("t2"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("a")},
+ State: model.StatePublic,
+ Version: 1,
+ }, *tb3Info.ForeignKeys[1])
+ // Auto create index for foreign key usage.
+ require.Equal(t, 1, len(tb3Info.Indices))
+ require.Equal(t, "fk_a", tb3Info.Indices[0].Name.L)
+ require.Equal(t, "`test`.`t3`, CONSTRAINT `fk_a` FOREIGN KEY (`a`) REFERENCES `t1` (`id`) ON DELETE CASCADE ON UPDATE RESTRICT", tb3Info.ForeignKeys[0].String("test", "t3"))
+ require.Equal(t, "`test`.`t3`, CONSTRAINT `fk_a2` FOREIGN KEY (`a`) REFERENCES `test2`.`t2` (`id`)", tb3Info.ForeignKeys[1].String("test", "t3"))
+
+ tk.MustExec("set @@foreign_key_checks=0")
+ tk.MustExec("drop table test2.t2")
+ tb1Info = getTableInfo(t, dom, "test", "t1")
+ tb3Info = getTableInfo(t, dom, "test", "t3")
+ require.Equal(t, 0, len(tb1Info.ForeignKeys))
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test"),
+ ChildTable: model.NewCIStr("t3"),
+ ChildFKName: model.NewCIStr("fk_a"),
+ }, *tb1ReferredFKs[0])
+ tb3ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t3")
+ require.Equal(t, 0, len(tb3ReferredFKs))
+ require.Equal(t, 2, len(tb3Info.ForeignKeys))
+ require.Equal(t, model.FKInfo{
+ ID: 1,
+ Name: model.NewCIStr("fk_a"),
+ RefSchema: model.NewCIStr("test"),
+ RefTable: model.NewCIStr("t1"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("a")},
+ OnDelete: 2,
+ OnUpdate: 1,
+ State: model.StatePublic,
+ Version: 1,
+ }, *tb3Info.ForeignKeys[0])
+ require.Equal(t, model.FKInfo{
+ ID: 2,
+ Name: model.NewCIStr("fk_a2"),
+ RefSchema: model.NewCIStr("test2"),
+ RefTable: model.NewCIStr("t2"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("a")},
+ State: model.StatePublic,
+ Version: 1,
+ }, *tb3Info.ForeignKeys[1])
+}
+
+func TestCreateTableWithForeignKeyMetaInfo3(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, a int, b int as (a) virtual);")
+ tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
+ tk.MustExec("create table t3 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
+ tk.MustExec("create table t4 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
+ tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ tk.MustExec("drop table t3")
+ tk.MustExec("create table t5 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
+ require.Equal(t, 3, len(tb1ReferredFKs))
+ require.Equal(t, "t2", tb1ReferredFKs[0].ChildTable.L)
+ require.Equal(t, "t3", tb1ReferredFKs[1].ChildTable.L)
+ require.Equal(t, "t4", tb1ReferredFKs[2].ChildTable.L)
+}
+
+func TestCreateTableWithForeignKeyPrivilegeCheck(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk.MustExec("create user 'u1'@'%' identified by '';")
+ tk.MustExec("grant create on *.* to 'u1'@'%';")
+ tk.MustExec("create table t1 (id int key);")
+
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk2.Session().Auth(&auth.UserIdentity{Username: "u1", Hostname: "localhost", CurrentUser: true, AuthUsername: "u1", AuthHostname: "%"}, nil, []byte("012345678901234567890"))
+ err := tk2.ExecToErr("create table t2 (a int, foreign key fk(a) references t1(id));")
+ require.Error(t, err)
+ require.Equal(t, "[planner:1142]REFERENCES command denied to user 'u1'@'%' for table 't1'", err.Error())
+
+ tk.MustExec("grant references on test.t1 to 'u1'@'%';")
+ tk2.MustExec("create table t2 (a int, foreign key fk(a) references t1(id));")
+ tk2.MustExec("create table t3 (id int key)")
+ err = tk2.ExecToErr("create table t4 (a int, foreign key fk(a) references t1(id), foreign key (a) references t3(id));")
+ require.Error(t, err)
+ require.Equal(t, "[planner:1142]REFERENCES command denied to user 'u1'@'%' for table 't3'", err.Error())
+
+ tk.MustExec("grant references on test.t3 to 'u1'@'%';")
+ tk2.MustExec("create table t4 (a int, foreign key fk(a) references t1(id), foreign key (a) references t3(id));")
+}
+
+func TestAlterTableWithForeignKeyPrivilegeCheck(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create user 'u1'@'%' identified by '';")
+ tk.MustExec("grant create,alter on *.* to 'u1'@'%';")
+ tk.MustExec("create table t1 (id int key);")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk2.Session().Auth(&auth.UserIdentity{Username: "u1", Hostname: "localhost", CurrentUser: true, AuthUsername: "u1", AuthHostname: "%"}, nil, []byte("012345678901234567890"))
+ tk2.MustExec("create table t2 (a int)")
+ err := tk2.ExecToErr("alter table t2 add foreign key (a) references t1 (id) on update cascade")
+ require.Error(t, err)
+ require.Equal(t, "[planner:1142]REFERENCES command denied to user 'u1'@'%' for table 't1'", err.Error())
+ tk.MustExec("grant references on test.t1 to 'u1'@'%';")
+ tk2.MustExec("alter table t2 add foreign key (a) references t1 (id) on update cascade")
+}
+
+func TestRenameTableWithForeignKeyMetaInfo(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("create database test2")
+ tk.MustExec("create database test3")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, a int, b int, foreign key fk(a) references t1(id))")
+ tk.MustExec("rename table test.t1 to test2.t2")
+ // check the schema diff
+ diff := getLatestSchemaDiff(t, tk)
+ require.Equal(t, model.ActionRenameTable, diff.Type)
+ require.Equal(t, 0, len(diff.AffectedOpts))
+ tk.MustQuery("show create table test2.t2").Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" +
+ " `id` int(11) NOT NULL,\n" +
+ " `a` int(11) DEFAULT NULL,\n" +
+ " `b` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `fk` (`a`),\n" +
+ " CONSTRAINT `fk` FOREIGN KEY (`a`) REFERENCES `test2`.`t2` (`id`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+ tblInfo := getTableInfo(t, dom, "test2", "t2")
+ tbReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "t2")
+ require.Equal(t, 1, len(tblInfo.ForeignKeys))
+ require.Equal(t, 1, len(tbReferredFKs))
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test2"),
+ ChildTable: model.NewCIStr("t2"),
+ ChildFKName: model.NewCIStr("fk"),
+ }, *tbReferredFKs[0])
+ require.Equal(t, model.FKInfo{
+ ID: 1,
+ Name: model.NewCIStr("fk"),
+ RefSchema: model.NewCIStr("test2"),
+ RefTable: model.NewCIStr("t2"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("a")},
+ State: model.StatePublic,
+ Version: 1,
+ }, *tblInfo.ForeignKeys[0])
+
+ tk.MustExec("drop table test2.t2")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, a int, b int as (a) virtual);")
+ tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
+ tk.MustExec("use test2")
+ tk.MustExec("rename table test.t2 to test2.tt2")
+ // check the schema diff
+ diff = getLatestSchemaDiff(t, tk)
+ require.Equal(t, model.ActionRenameTable, diff.Type)
+ require.Equal(t, 0, len(diff.AffectedOpts))
+ tb1Info := getTableInfo(t, dom, "test", "t1")
+ tb2Info := getTableInfo(t, dom, "test2", "tt2")
+ require.Equal(t, 0, len(tb1Info.ForeignKeys))
+ tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test2"),
+ ChildTable: model.NewCIStr("tt2"),
+ ChildFKName: model.NewCIStr("fk_b"),
+ }, *tb1ReferredFKs[0])
+ tb2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "tt2")
+ require.Equal(t, 0, len(tb2ReferredFKs))
+ require.Equal(t, 1, len(tb2Info.ForeignKeys))
+ require.Equal(t, model.FKInfo{
+ ID: 1,
+ Name: model.NewCIStr("fk_b"),
+ RefSchema: model.NewCIStr("test"),
+ RefTable: model.NewCIStr("t1"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("b")},
+ State: model.StatePublic,
+ Version: 1,
+ }, *tb2Info.ForeignKeys[0])
+ // Auto create index for foreign key usage.
+ require.Equal(t, 1, len(tb2Info.Indices))
+ require.Equal(t, "fk_b", tb2Info.Indices[0].Name.L)
+ require.Equal(t, "`test2`.`tt2`, CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `test`.`t1` (`id`)", tb2Info.ForeignKeys[0].String("test2", "tt2"))
+
+ tk.MustExec("rename table test.t1 to test3.tt1")
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test3", "tt1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ require.Equal(t, 1, len(tb1ReferredFKs[0].Cols))
+ // check the schema diff
+ diff = getLatestSchemaDiff(t, tk)
+ require.Equal(t, model.ActionRenameTable, diff.Type)
+ require.Equal(t, 1, len(diff.AffectedOpts))
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test2"),
+ ChildTable: model.NewCIStr("tt2"),
+ ChildFKName: model.NewCIStr("fk_b"),
+ }, *tb1ReferredFKs[0])
+ tbl2Info := getTableInfo(t, dom, "test2", "tt2")
+ tb2ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test2", "tt2")
+ require.Equal(t, 0, len(tb2ReferredFKs))
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys))
+ require.Equal(t, model.FKInfo{
+ ID: 1,
+ Name: model.NewCIStr("fk_b"),
+ RefSchema: model.NewCIStr("test3"),
+ RefTable: model.NewCIStr("tt1"),
+ RefCols: []model.CIStr{model.NewCIStr("id")},
+ Cols: []model.CIStr{model.NewCIStr("b")},
+ State: model.StatePublic,
+ Version: 1,
+ }, *tbl2Info.ForeignKeys[0])
+ tk.MustQuery("show create table test2.tt2").Check(testkit.Rows("tt2 CREATE TABLE `tt2` (\n" +
+ " `id` int(11) NOT NULL,\n" +
+ " `b` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `fk_b` (`b`),\n" +
+ " CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `test3`.`tt1` (`id`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+}
+
+func TestCreateTableWithForeignKeyDML(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, a int);")
+ tk.MustExec("begin")
+ tk.MustExec("insert into t1 values (1, 1)")
+ tk.MustExec("update t1 set a = 2 where id = 1")
+
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk2.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
+
+ tk.MustExec("commit")
+}
+
+func TestCreateTableWithForeignKeyError(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("use test")
+
+ cases := []struct {
+ prepare []string
+ refer string
+ create string
+ err string
+ }{
+ {
+ refer: "create table t1 (id int, a int, b int);",
+ create: "create table t2 (a int, b int, foreign key fk_b(b) references T_unknown(b));",
+ err: "[schema:1824]Failed to open the referenced table 'T_unknown'",
+ },
+ {
+ refer: "create table t1 (id int, a int, b int);",
+ create: "create table t2 (a int, b int, foreign key fk_b(b) references t1(c_unknown));",
+ err: "[schema:3734]Failed to add the foreign key constraint. Missing column 'c_unknown' for constraint 'fk_b' in the referenced table 't1'",
+ },
+ {
+ refer: "create table t1 (id int key, a int, b int);",
+ create: "create table t2 (a int, b int, foreign key fk(c_unknown) references t1(id));",
+ err: "[ddl:1072]Key column 'c_unknown' doesn't exist in table",
+ },
+ {
+ refer: "create table t1 (id int, a int, b int);",
+ create: "create table t2 (a int, b int, foreign key fk_b(b) references t1(b));",
+ err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
+ },
+ {
+ refer: "create table t1 (id int, a int, b int not null, index(b));",
+ create: "create table t2 (a int, b int not null, foreign key fk_b(b) references t1(b) on update set null);",
+ err: "[schema:1830]Column 'b' cannot be NOT NULL: needed in a foreign key constraint 'fk_b' SET NULL",
+ },
+ {
+ refer: "create table t1 (id int, a int, b int not null, index(b));",
+ create: "create table t2 (a int, b int not null, foreign key fk_b(b) references t1(b) on delete set null);",
+ err: "[schema:1830]Column 'b' cannot be NOT NULL: needed in a foreign key constraint 'fk_b' SET NULL",
+ },
+ {
+ refer: "create table t1 (id int key, a int, b int as (a) virtual, index(b));",
+ create: "create table t2 (a int, b int, foreign key fk_b(b) references t1(b));",
+ err: "[schema:3733]Foreign key 'fk_b' uses virtual column 'b' which is not supported.",
+ },
+ {
+ refer: "create table t1 (id int key, a int, b int, index(b));",
+ create: "create table t2 (a int, b int as (a) virtual, foreign key fk_b(b) references t1(b));",
+ err: "[schema:3733]Foreign key 'fk_b' uses virtual column 'b' which is not supported.",
+ },
+ {
+ refer: "create table t1 (id int key, a int);",
+ create: "create table t2 (a int, b varchar(10), foreign key fk(b) references t1(id));",
+ err: "[ddl:3780]Referencing column 'b' and referenced column 'id' in foreign key constraint 'fk' are incompatible.",
+ },
+ {
+ refer: "create table t1 (id int key, a int not null, index(a));",
+ create: "create table t2 (a int, b int unsigned, foreign key fk_b(b) references t1(a));",
+ err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
+ },
+ {
+ refer: "create table t1 (id int key, a bigint, index(a));",
+ create: "create table t2 (a int, b int, foreign key fk_b(b) references t1(a));",
+ err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
+ },
+ {
+ refer: "create table t1 (id int key, a varchar(10) charset utf8, index(a));",
+ create: "create table t2 (a int, b varchar(10) charset utf8mb4, foreign key fk_b(b) references t1(a));",
+ err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
+ },
+ {
+ refer: "create table t1 (id int key, a varchar(10) collate utf8_bin, index(a));",
+ create: "create table t2 (a int, b varchar(10) collate utf8mb4_bin, foreign key fk_b(b) references t1(a));",
+ err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
+ },
+ {
+ refer: "create table t1 (id int key, a varchar(10));",
+ create: "create table t2 (a int, b varchar(10), foreign key fk_b(b) references t1(a));",
+ err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
+ },
+ {
+ refer: "create table t1 (id int key, a varchar(10), index (a(5)));",
+ create: "create table t2 (a int, b varchar(10), foreign key fk_b(b) references t1(a));",
+ err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
+ },
+ {
+ refer: "create table t1 (id int key, a int, index(a));",
+ create: "create table t2 (a int, b int, foreign key fk_b(b) references t1(id, a));",
+ err: "[schema:1239]Incorrect foreign key definition for 'fk_b': Key reference and table reference don't match",
+ },
+ {
+ create: "create table t2 (a int key, foreign key (a) references t2(a));",
+ err: "[schema:1215]Cannot add foreign key constraint",
+ },
+ {
+ create: "create table t2 (a int, b int, index(a,b), index(b,a), foreign key (a,b) references t2(a,b));",
+ err: "[schema:1215]Cannot add foreign key constraint",
+ },
+ {
+ create: "create table t2 (a int, b int, index(a,b), foreign key (a,b) references t2(b,a));",
+ err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_1' in the referenced table 't2'",
+ },
+ {
+ prepare: []string{
+ "set @@foreign_key_checks=0;",
+ "create table t2 (a int, b int, index(a), foreign key (a) references t1(id));",
+ },
+ create: "create table t1 (id int, a int);",
+ err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_1' in the referenced table 't1'",
+ },
+ {
+ prepare: []string{
+ "set @@foreign_key_checks=0;",
+ "create table t2 (a int, b int, index(a), foreign key (a) references t1(id));",
+ },
+ create: "create table t1 (id bigint key, a int);",
+ err: "[ddl:3780]Referencing column 'a' and referenced column 'id' in foreign key constraint 'fk_1' are incompatible.",
+ },
+ {
+ // foreign key is not support in temporary table.
+ refer: "create temporary table t1 (id int key, b int, index(b))",
+ create: "create table t2 (a int, b int, foreign key fk(b) references t1(b))",
+ err: "[schema:1824]Failed to open the referenced table 't1'",
+ },
+ {
+ // foreign key is not support in temporary table.
+ refer: "create global temporary table t1 (id int key, b int, index(b)) on commit delete rows",
+ create: "create table t2 (a int, b int, foreign key fk(b) references t1(b))",
+ err: "[schema:1215]Cannot add foreign key constraint",
+ },
+ {
+ // foreign key is not support in temporary table.
+ refer: "create table t1 (id int key, b int, index(b))",
+ create: "create temporary table t2 (a int, b int, foreign key fk(b) references t1(b))",
+ err: "[schema:1215]Cannot add foreign key constraint",
+ },
+ {
+ // foreign key is not support in temporary table.
+ refer: "create table t1 (id int key, b int, index(b))",
+ create: "create global temporary table t2 (a int, b int, foreign key fk(b) references t1(b)) on commit delete rows",
+ err: "[schema:1215]Cannot add foreign key constraint",
+ },
+ {
+ create: "create table t1 (a int, foreign key ``(a) references t1(a));",
+ err: "[ddl:1280]Incorrect index name ''",
+ },
+ {
+ create: "create table t1 (a int, constraint `` foreign key (a) references t1(a));",
+ err: "[ddl:1280]Incorrect index name ''",
+ },
+ {
+ create: "create table t1 (a int, constraint `fk` foreign key (a,a) references t1(a, b));",
+ err: "[schema:1060]Duplicate column name 'a'",
+ },
+ {
+ refer: "create table t1(a int, b int, index(a,b));",
+ create: "create table t2 (a int, b int, foreign key (a,b) references t1(a,a));",
+ err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_1' in the referenced table 't1'",
+ },
+ {
+ refer: "create table t1 (id int key, b int, index(b))",
+ create: "create table t2 (a int, b int, index fk_1(a), foreign key (b) references t1(b));",
+ err: "[ddl:1061]duplicate key name fk_1",
+ },
+ {
+ refer: "create table t1 (id int key);",
+ create: "create table t2 (id int key, foreign key name5678901234567890123456789012345678901234567890123456789012345(id) references t1(id));",
+ err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
+ },
+ {
+ refer: "create table t1 (id int key);",
+ create: "create table t2 (id int key, constraint name5678901234567890123456789012345678901234567890123456789012345 foreign key (id) references t1(id));",
+ err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
+ },
+ {
+ create: "create table t2 (id int key, constraint fk foreign key (id) references name5678901234567890123456789012345678901234567890123456789012345.t1(id));",
+ err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
+ },
+ {
+ prepare: []string{
+ "set @@foreign_key_checks=0;",
+ },
+ create: "create table t2 (id int key, constraint fk foreign key (id) references name5678901234567890123456789012345678901234567890123456789012345(id));",
+ err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
+ },
+ {
+ prepare: []string{
+ "set @@foreign_key_checks=0;",
+ },
+ create: "create table t2 (id int key, constraint fk foreign key (id) references t1(name5678901234567890123456789012345678901234567890123456789012345));",
+ err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
+ },
+ // Test foreign key with temporary table
+ {
+ refer: "create temporary table t1 (id int key);",
+ create: "create table t2 (id int key, constraint fk foreign key (id) references t1(id));",
+ err: "[schema:1824]Failed to open the referenced table 't1'",
+ },
+ {
+ refer: "create table t1 (id int key);",
+ create: "create temporary table t2 (id int key, constraint fk foreign key (id) references t1(id));",
+ err: "[schema:1215]Cannot add foreign key constraint",
+ },
+ // Test foreign key with partition table
+ {
+ refer: "create table t1 (id int key) partition by hash(id) partitions 3;",
+ create: "create table t2 (id int key, constraint fk foreign key (id) references t1(id));",
+ err: "[schema:1506]Foreign key clause is not yet supported in conjunction with partitioning",
+ },
+ {
+ refer: "create table t1 (id int key);",
+ create: "create table t2 (id int key, constraint fk foreign key (id) references t1(id)) partition by hash(id) partitions 3;",
+ err: "[schema:1506]Foreign key clause is not yet supported in conjunction with partitioning",
+ },
+ }
+ for _, ca := range cases {
+ tk.MustExec("drop table if exists t2")
+ tk.MustExec("drop table if exists t1")
+ tk.MustExec("set @@foreign_key_checks=1")
+ for _, sql := range ca.prepare {
+ tk.MustExec(sql)
+ }
+ if ca.refer != "" {
+ tk.MustExec(ca.refer)
+ }
+ err := tk.ExecToErr(ca.create)
+ require.Error(t, err, ca.create)
+ require.Equal(t, ca.err, err.Error(), ca.create)
+ }
+
+ passCases := [][]string{
+ {
+ "create table t1 (id int key, a int, b int, foreign key fk(a) references t1(id))",
+ },
+ {
+ "create table t1 (id int key, b int not null, index(b))",
+ "create table t2 (a int, b int, foreign key fk_b(b) references t1(b));",
+ },
+ {
+ "create table t1 (id int key, a varchar(10), index(a));",
+ "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
+ },
+ {
+ "create table t1 (id int key, a decimal(10,5), index(a));",
+ "create table t2 (a int, b decimal(20, 10), foreign key fk_b(b) references t1(a));",
+ },
+ {
+ "create table t1 (id int key, a varchar(10), index (a(10)));",
+ "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
+ },
+ {
+ "set @@foreign_key_checks=0;",
+ "create table t2 (a int, b int, foreign key fk_b(b) references t_unknown(b));",
+ "set @@foreign_key_checks=1;",
+ },
+ {
+ "create table t2 (a int, b int, index(a,b), index(b,a), foreign key (a,b) references t2(b,a));",
+ },
+ {
+ "create table t1 (a int key, b int, index(b))",
+ "create table t2 (a int, b int, foreign key (a) references t1(a), foreign key (b) references t1(b));",
+ },
+ {
+ "create table t1 (id int key);",
+ "create table t2 (id int key, foreign key name567890123456789012345678901234567890123456789012345678901234(id) references t1(id));",
+ },
+ }
+ for _, ca := range passCases {
+ tk.MustExec("drop table if exists t2")
+ tk.MustExec("drop table if exists t1")
+ for _, sql := range ca {
+ tk.MustExec(sql)
+ }
+ }
+}
+
+func TestModifyColumnWithForeignKey(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+
+ tk.MustExec("create table t1 (id int key, b varchar(10), index(b));")
+ tk.MustExec("create table t2 (a varchar(10), constraint fk foreign key (a) references t1(b));")
+ tk.MustExec("insert into t1 values (1, '123456789');")
+ tk.MustExec("insert into t2 values ('123456789');")
+ tk.MustGetErrMsg("alter table t1 modify column b varchar(5);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
+ tk.MustGetErrMsg("alter table t1 modify column b bigint;", "[ddl:3780]Referencing column 'a' and referenced column 'b' in foreign key constraint 'fk' are incompatible.")
+ tk.MustExec("alter table t1 modify column b varchar(20);")
+ tk.MustGetErrMsg("alter table t1 modify column b varchar(10);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
+ tk.MustExec("alter table t2 modify column a varchar(20);")
+ tk.MustExec("alter table t2 modify column a varchar(21);")
+ tk.MustGetErrMsg("alter table t2 modify column a varchar(5);", "[ddl:1832]Cannot change column 'a': used in a foreign key constraint 'fk'")
+ tk.MustGetErrMsg("alter table t2 modify column a bigint;", "[ddl:3780]Referencing column 'a' and referenced column 'b' in foreign key constraint 'fk' are incompatible.")
+
+ tk.MustExec("drop table t2")
+ tk.MustExec("drop table t1")
+ tk.MustExec("create table t1 (id int key, b decimal(10, 5), index(b));")
+ tk.MustExec("create table t2 (a decimal(10, 5), constraint fk foreign key (a) references t1(b));")
+ tk.MustExec("insert into t1 values (1, 12345.67891);")
+ tk.MustExec("insert into t2 values (12345.67891);")
+ tk.MustGetErrMsg("alter table t1 modify column b decimal(10, 6);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
+ tk.MustGetErrMsg("alter table t1 modify column b decimal(10, 3);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
+ tk.MustGetErrMsg("alter table t1 modify column b decimal(5, 2);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
+ tk.MustGetErrMsg("alter table t1 modify column b decimal(20, 10);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
+ tk.MustGetErrMsg("alter table t2 modify column a decimal(30, 15);", "[ddl:1832]Cannot change column 'a': used in a foreign key constraint 'fk'")
+ tk.MustGetErrMsg("alter table t2 modify column a decimal(5, 2);", "[ddl:1832]Cannot change column 'a': used in a foreign key constraint 'fk'")
+}
+
+func TestDropChildTableForeignKeyMetaInfo(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, a int, b int, CONSTRAINT fk foreign key (a) references t1(id))")
+ tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ tk.MustExec("drop table t1")
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 0, len(tb1ReferredFKs))
+
+ tk.MustExec("create table t1 (id int key, b int, index(b))")
+ tk.MustExec("create table t2 (a int, b int, foreign key fk (a) references t1(b));")
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ tk.MustExec("drop table t2")
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 0, len(tb1ReferredFKs))
+}
+
+func TestDropForeignKeyMetaInfo(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, a int, b int, CONSTRAINT fk foreign key (a) references t1(id))")
+ tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ tk.MustExec("alter table t1 drop foreign key fk")
+ tbl1Info := getTableInfo(t, dom, "test", "t1")
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 0, len(tbl1Info.ForeignKeys))
+ require.Equal(t, 0, len(tb1ReferredFKs))
+
+ tk.MustExec("drop table t1")
+ tk.MustExec("create table t1 (id int key, b int, index(b))")
+ tk.MustExec("create table t2 (a int, b int, foreign key fk (a) references t1(b));")
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ tk.MustExec("alter table t2 drop foreign key fk")
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 0, len(tb1ReferredFKs))
+ tbl2Info := getTableInfo(t, dom, "test", "t2")
+ require.Equal(t, 0, len(tbl2Info.ForeignKeys))
+}
+
+func TestTruncateOrDropTableWithForeignKeyReferred(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("use test")
+
+ cases := []struct {
+ prepares []string
+ tbl string
+ truncateErr string
+ dropErr string
+ }{
+ {
+ prepares: []string{
+ "create table t1 (id int key, b int not null, index(b))",
+ "create table t2 (a int, b int, foreign key fk_b(b) references t1(b));",
+ },
+ tbl: "t1",
+ truncateErr: "[ddl:1701]Cannot truncate a table referenced in a foreign key constraint (`test`.`t2` CONSTRAINT `fk_b`)",
+ dropErr: "[ddl:3730]Cannot drop table 't1' referenced by a foreign key constraint 'fk_b' on table 't2'.",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a varchar(10), index(a));",
+ "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
+ },
+ tbl: "t1",
+ truncateErr: "[ddl:1701]Cannot truncate a table referenced in a foreign key constraint (`test`.`t2` CONSTRAINT `fk_b`)",
+ dropErr: "[ddl:3730]Cannot drop table 't1' referenced by a foreign key constraint 'fk_b' on table 't2'.",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a varchar(10), index (a(10)));",
+ "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
+ },
+ tbl: "t1",
+ truncateErr: "[ddl:1701]Cannot truncate a table referenced in a foreign key constraint (`test`.`t2` CONSTRAINT `fk_b`)",
+ dropErr: "[ddl:3730]Cannot drop table 't1' referenced by a foreign key constraint 'fk_b' on table 't2'.",
+ },
+ }
+
+ for _, ca := range cases {
+ tk.MustExec("drop table if exists t2")
+ tk.MustExec("drop table if exists t1")
+ for _, sql := range ca.prepares {
+ tk.MustExec(sql)
+ }
+ truncateSQL := fmt.Sprintf("truncate table %v", ca.tbl)
+ tk.MustExec("set @@foreign_key_checks=1;")
+ err := tk.ExecToErr(truncateSQL)
+ require.Error(t, err)
+ require.Equal(t, ca.truncateErr, err.Error())
+ dropSQL := fmt.Sprintf("drop table %v", ca.tbl)
+ err = tk.ExecToErr(dropSQL)
+ require.Error(t, err)
+ require.Equal(t, ca.dropErr, err.Error())
+
+ tk.MustExec("set @@foreign_key_checks=0;")
+ tk.MustExec(truncateSQL)
+ }
+ passCases := [][]string{
+ {
+ "create table t1 (id int key, a int, b int, foreign key fk(a) references t1(id))",
+ "truncate table t1",
+ "drop table t1",
+ },
+ {
+ "create table t1 (id int key, a varchar(10), index (a(10)));",
+ "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
+ "drop table t1, t2",
+ },
+ {
+ "set @@foreign_key_checks=0;",
+ "create table t1 (id int key, a varchar(10), index (a(10)));",
+ "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
+ "truncate table t1",
+ "drop table t1",
+ },
+ }
+ for _, ca := range passCases {
+ tk.MustExec("drop table if exists t1, t2")
+ tk.MustExec("set @@foreign_key_checks=1;")
+ for _, sql := range ca {
+ tk.MustExec(sql)
+ }
+ }
+}
+
+func TestDropTableWithForeignKeyReferred(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+
+ tk.MustExec("create table t1 (id int key, b int, index(b));")
+ tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references t1(id));")
+ tk.MustExec("create table t3 (id int key, b int, foreign key fk_b(b) references t2(id));")
+ err := tk.ExecToErr("drop table if exists t1,t2;")
+ require.Error(t, err)
+ require.Equal(t, "[ddl:3730]Cannot drop table 't2' referenced by a foreign key constraint 'fk_b' on table 't3'.", err.Error())
+ tk.MustQuery("show tables").Check(testkit.Rows("t1", "t2", "t3"))
+}
+
+func TestDropIndexNeededInForeignKey(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+
+ cases := []struct {
+ prepares []string
+ drops []string
+ err string
+ }{
+ {
+ prepares: []string{
+ "create table t1 (id int key, b int, index idx (b))",
+ "create table t2 (a int, b int, index idx (b), foreign key fk_b(b) references t1(b));",
+ },
+ drops: []string{
+ "alter table t1 drop index idx",
+ "alter table t2 drop index idx",
+ },
+ err: "[ddl:1553]Cannot drop index 'idx': needed in a foreign key constraint",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int, b int, index idx (id, b))",
+ "create table t2 (a int, b int, index idx (b, a), foreign key fk_b(b) references t1(id));",
+ },
+ drops: []string{
+ "alter table t1 drop index idx",
+ "alter table t2 drop index idx",
+ },
+ err: "[ddl:1553]Cannot drop index 'idx': needed in a foreign key constraint",
+ },
+ }
+
+ for _, ca := range cases {
+ tk.MustExec("drop table if exists t2")
+ tk.MustExec("drop table if exists t1")
+ for _, sql := range ca.prepares {
+ tk.MustExec(sql)
+ }
+ for _, drop := range ca.drops {
+ // even disable foreign key check, still can't drop the index used by foreign key.
+ tk.MustExec("set @@foreign_key_checks=0;")
+ err := tk.ExecToErr(drop)
+ require.Error(t, err)
+ require.Equal(t, ca.err, err.Error())
+ tk.MustExec("set @@foreign_key_checks=1;")
+ err = tk.ExecToErr(drop)
+ require.Error(t, err)
+ require.Equal(t, ca.err, err.Error())
+ }
+ }
+ passCases := [][]string{
+ {
+ "create table t1 (id int key, b int, index idxb (b))",
+ "create table t2 (a int, b int key, index idxa (a),index idxb (b), foreign key fk_b(b) references t1(id));",
+ "alter table t1 drop index idxb",
+ "alter table t2 drop index idxa",
+ "alter table t2 drop index idxb",
+ },
+ {
+ "create table t1 (id int key, b int, index idxb (b), unique index idx(b, id))",
+ "create table t2 (a int, b int key, index idx (b, a),index idxb (b), index idxab(a, b), foreign key fk_b(b) references t1(b));",
+ "alter table t1 drop index idxb",
+ "alter table t1 add index idxb (b)",
+ "alter table t1 drop index idx",
+ "alter table t2 drop index idx",
+ "alter table t2 add index idx (b, a)",
+ "alter table t2 drop index idxb",
+ "alter table t2 drop index idxab",
+ },
+ }
+ tk.MustExec("set @@foreign_key_checks=1;")
+ for _, ca := range passCases {
+ tk.MustExec("drop table if exists t2")
+ tk.MustExec("drop table if exists t1")
+ for _, sql := range ca {
+ tk.MustExec(sql)
+ }
+ }
+}
+
+func getTableInfo(t *testing.T, dom *domain.Domain, db, tb string) *model.TableInfo {
+ err := dom.Reload()
+ require.NoError(t, err)
+ is := dom.InfoSchema()
+ tbl, err := is.TableByName(model.NewCIStr(db), model.NewCIStr(tb))
+ require.NoError(t, err)
+ _, exist := is.TableByID(tbl.Meta().ID)
+ require.True(t, exist)
+ return tbl.Meta()
+}
+
+func getTableInfoReferredForeignKeys(t *testing.T, dom *domain.Domain, db, tb string) []*model.ReferredFKInfo {
+ err := dom.Reload()
+ require.NoError(t, err)
+ return dom.InfoSchema().GetTableReferredForeignKeys(db, tb)
+}
+
+func TestDropColumnWithForeignKey(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+
+ tk.MustExec("create table t1 (id int key, a int, b int, index(b), CONSTRAINT fk foreign key (a) references t1(b))")
+ tk.MustGetErrMsg("alter table t1 drop column a;", "[ddl:1828]Cannot drop column 'a': needed in a foreign key constraint 'fk'")
+ tk.MustGetErrMsg("alter table t1 drop column b;", "[ddl:1829]Cannot drop column 'b': needed in a foreign key constraint 'fk' of table 't1'")
+
+ tk.MustExec("drop table t1")
+ tk.MustExec("create table t1 (id int key, b int, index(b));")
+ tk.MustExec("create table t2 (a int, b int, constraint fk foreign key (a) references t1(b));")
+ tk.MustGetErrMsg("alter table t1 drop column b;", "[ddl:1829]Cannot drop column 'b': needed in a foreign key constraint 'fk' of table 't2'")
+ tk.MustGetErrMsg("alter table t2 drop column a;", "[ddl:1828]Cannot drop column 'a': needed in a foreign key constraint 'fk'")
+}
+
+func TestRenameColumnWithForeignKeyMetaInfo(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+
+ tk.MustExec("create table t1 (id int key, a int, b int, foreign key fk(a) references t1(id))")
+ tk.MustExec("alter table t1 change id kid int")
+ tk.MustExec("alter table t1 rename column a to aa")
+ tbl1Info := getTableInfo(t, dom, "test", "t1")
+ tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tbl1Info.ForeignKeys))
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ require.Equal(t, "kid", tb1ReferredFKs[0].Cols[0].L)
+ require.Equal(t, "kid", tbl1Info.ForeignKeys[0].RefCols[0].L)
+ require.Equal(t, "aa", tbl1Info.ForeignKeys[0].Cols[0].L)
+
+ tk.MustExec("drop table t1")
+ tk.MustExec("create table t1 (id int key, b int, index(b))")
+ tk.MustExec("create table t2 (a int, b int, foreign key fk(a) references t1(b));")
+ tk.MustExec("alter table t2 change a aa int")
+ tbl1Info = getTableInfo(t, dom, "test", "t1")
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ require.Equal(t, 1, len(tb1ReferredFKs[0].Cols))
+ require.Equal(t, "b", tb1ReferredFKs[0].Cols[0].L)
+ tbl2Info := getTableInfo(t, dom, "test", "t2")
+ tb2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t2")
+ require.Equal(t, 0, len(tb2ReferredFKs))
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys))
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].Cols))
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].RefCols))
+ require.Equal(t, "aa", tbl2Info.ForeignKeys[0].Cols[0].L)
+ require.Equal(t, "b", tbl2Info.ForeignKeys[0].RefCols[0].L)
+
+ tk.MustExec("alter table t1 change id kid int")
+ tk.MustExec("alter table t1 change b bb int")
+ tbl1Info = getTableInfo(t, dom, "test", "t1")
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 1, len(tb1ReferredFKs))
+ require.Equal(t, 1, len(tb1ReferredFKs[0].Cols))
+ require.Equal(t, "bb", tb1ReferredFKs[0].Cols[0].L)
+ tbl2Info = getTableInfo(t, dom, "test", "t2")
+ tb2ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t2")
+ require.Equal(t, 0, len(tb2ReferredFKs))
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys))
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].Cols))
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].RefCols))
+ require.Equal(t, "aa", tbl2Info.ForeignKeys[0].Cols[0].L)
+ require.Equal(t, "bb", tbl2Info.ForeignKeys[0].RefCols[0].L)
+
+ tk.MustExec("drop table t1, t2")
+ tk.MustExec("create table t1 (id int key, b int, index(b))")
+ tk.MustExec("create table t2 (a int, b int, foreign key (a) references t1(b), foreign key (b) references t1(b));")
+ tk.MustExec("alter table t1 change b bb int")
+ tbl1Info = getTableInfo(t, dom, "test", "t1")
+ tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ require.Equal(t, 2, len(tb1ReferredFKs))
+ require.Equal(t, 1, len(tb1ReferredFKs[0].Cols))
+ require.Equal(t, 1, len(tb1ReferredFKs[1].Cols))
+ require.Equal(t, "bb", tb1ReferredFKs[0].Cols[0].L)
+ require.Equal(t, "bb", tb1ReferredFKs[1].Cols[0].L)
+ tbl2Info = getTableInfo(t, dom, "test", "t2")
+ tb2ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t2")
+ require.Equal(t, 0, len(tb2ReferredFKs))
+ require.Equal(t, 2, len(tbl2Info.ForeignKeys))
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].Cols))
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].RefCols))
+ require.Equal(t, "a", tbl2Info.ForeignKeys[0].Cols[0].L)
+ require.Equal(t, "bb", tbl2Info.ForeignKeys[0].RefCols[0].L)
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys[1].Cols))
+ require.Equal(t, 1, len(tbl2Info.ForeignKeys[1].RefCols))
+ require.Equal(t, "b", tbl2Info.ForeignKeys[1].Cols[0].L)
+ require.Equal(t, "bb", tbl2Info.ForeignKeys[1].RefCols[0].L)
+ tk.MustExec("alter table t2 rename column a to aa")
+ tk.MustExec("alter table t2 change b bb int")
+ tk.MustQuery("show create table t2").
+ Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" +
+ " `aa` int(11) DEFAULT NULL,\n" +
+ " `bb` int(11) DEFAULT NULL,\n" +
+ " KEY `fk_1` (`aa`),\n KEY `fk_2` (`bb`),\n" +
+ " CONSTRAINT `fk_1` FOREIGN KEY (`aa`) REFERENCES `test`.`t1` (`bb`),\n" +
+ " CONSTRAINT `fk_2` FOREIGN KEY (`bb`) REFERENCES `test`.`t1` (`bb`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+}
+
+func TestDropDatabaseWithForeignKeyReferred(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+
+ tk.MustExec("create table t1 (id int key, b int, index(b));")
+ tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references t1(id));")
+ tk.MustExec("create database test2")
+ tk.MustExec("create table test2.t3 (id int key, b int, foreign key fk_b(b) references test.t2(id));")
+ err := tk.ExecToErr("drop database test;")
+ require.Error(t, err)
+ require.Equal(t, "[ddl:3730]Cannot drop table 't2' referenced by a foreign key constraint 'fk_b' on table 't3'.", err.Error())
+ tk.MustExec("set @@foreign_key_checks=0;")
+ tk.MustExec("drop database test")
+
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("create database test")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, b int, index(b));")
+ tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references t1(id));")
+ err = tk.ExecToErr("drop database test;")
+ require.Error(t, err)
+ require.Equal(t, "[ddl:3730]Cannot drop table 't2' referenced by a foreign key constraint 'fk_b' on table 't3'.", err.Error())
+ tk.MustExec("drop table test2.t3")
+ tk.MustExec("drop database test")
+}
+
+func TestAddForeignKey(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, b int);")
+ tk.MustExec("create table t2 (id int key, b int);")
+ tk.MustExec("alter table t2 add index(b)")
+ tk.MustExec("alter table t2 add foreign key (b) references t1(id);")
+ tbl2Info := getTableInfo(t, dom, "test", "t2")
+ require.Equal(t, int64(1), tbl2Info.MaxForeignKeyID)
+ tk.MustGetDBError("alter table t2 add foreign key (b) references t1(b);", infoschema.ErrForeignKeyNoIndexInParent)
+ tk.MustExec("alter table t1 add index(b)")
+ tk.MustExec("alter table t2 add foreign key (b) references t1(b);")
+ tk.MustGetDBError("alter table t2 add foreign key (b) references t2(b);", infoschema.ErrCannotAddForeign)
+ // Test auto-create index when create foreign key constraint.
+ tk.MustExec("drop table if exists t1,t2")
+ tk.MustExec("create table t1 (id int key, b int, index(b));")
+ tk.MustExec("create table t2 (id int key, b int);")
+ tk.MustExec("alter table t2 add constraint fk foreign key (b) references t1(b);")
+ tbl2Info = getTableInfo(t, dom, "test", "t2")
+ require.Equal(t, 1, len(tbl2Info.Indices))
+ require.Equal(t, "fk", tbl2Info.Indices[0].Name.L)
+ require.Equal(t, model.StatePublic, tbl2Info.Indices[0].State)
+ tk.MustQuery("select b from t2 use index(fk)").Check(testkit.Rows())
+ res := tk.MustQuery("explain select b from t2 use index(fk)")
+ plan := bytes.NewBuffer(nil)
+ rows := res.Rows()
+ for _, row := range rows {
+ for _, c := range row {
+ plan.WriteString(c.(string))
+ plan.WriteString(" ")
+ }
+ }
+ require.Regexp(t, ".*IndexReader.*index:fk.*", plan.String())
+
+ // Test add multiple foreign key constraint in one statement.
+ tk.MustExec("alter table t2 add column c int, add column d int, add column e int;")
+ tk.MustExec("alter table t2 add index idx_c(c, d, e)")
+ tk.MustExec("alter table t2 add constraint fk_c foreign key (c) references t1(b), " +
+ "add constraint fk_d foreign key (d) references t1(b)," +
+ "add constraint fk_e foreign key (e) references t1(b)")
+ tbl2Info = getTableInfo(t, dom, "test", "t2")
+ require.Equal(t, 4, len(tbl2Info.Indices))
+ names := []string{"fk", "idx_c", "fk_d", "fk_e"}
+ for i, idx := range tbl2Info.Indices {
+ require.Equal(t, names[i], idx.Name.L)
+ require.Equal(t, model.StatePublic, idx.State)
+ }
+ names = []string{"fk", "fk_c", "fk_d", "fk_e"}
+ for i, fkInfo := range tbl2Info.ForeignKeys {
+ require.Equal(t, names[i], fkInfo.Name.L)
+ require.Equal(t, model.StatePublic, fkInfo.State)
+ }
+ tk.MustGetDBError("insert into t2 (id, b) values (1,1)", plannercore.ErrNoReferencedRow2)
+ tk.MustGetDBError("insert into t2 (id, c) values (1,1)", plannercore.ErrNoReferencedRow2)
+ tk.MustGetDBError("insert into t2 (id, d) values (1,1)", plannercore.ErrNoReferencedRow2)
+ tk.MustGetDBError("insert into t2 (id, e) values (1,1)", plannercore.ErrNoReferencedRow2)
+
+ // Test add multiple foreign key constraint in one statement but failed.
+ tk.MustExec("alter table t2 drop foreign key fk")
+ tk.MustExec("alter table t2 drop foreign key fk_c")
+ tk.MustExec("alter table t2 drop foreign key fk_d")
+ tk.MustExec("alter table t2 drop foreign key fk_e")
+ tk.MustGetDBError("alter table t2 add constraint fk_c foreign key (c) references t1(b), "+
+ "add constraint fk_d foreign key (d) references t1(b),"+
+ "add constraint fk_e foreign key (e) references t1(unknown_col)", infoschema.ErrForeignKeyNoColumnInParent)
+ tbl2Info = getTableInfo(t, dom, "test", "t2")
+ require.Equal(t, 0, len(tbl2Info.ForeignKeys))
+ tk.MustGetDBError("alter table t2 drop index idx_c, add constraint fk_c foreign key (c) references t1(b)", dbterror.ErrDropIndexNeededInForeignKey)
+
+ // Test circular dependency add foreign key failed.
+ tk.MustExec("drop table if exists t1,t2")
+ tk.MustExec("create table t1 (id int key,a int, index(a));")
+ tk.MustExec("create table t2 (id int key,a int, foreign key fk(a) references t1(id) ON DELETE CASCADE);")
+ tk.MustExec("insert into t1 values (1,1);")
+ err := tk.ExecToErr("ALTER TABLE t1 ADD foreign key fk(a) references t2(id) ON DELETE CASCADE;")
+ require.Error(t, err)
+ require.Equal(t, "[ddl:1452]Cannot add or update a child row: a foreign key constraint fails (`test`.`t1`, CONSTRAINT `fk` FOREIGN KEY (`a`) REFERENCES `t2` (`id`) ON DELETE CASCADE)", err.Error())
+ tbl1Info := getTableInfo(t, dom, "test", "t1")
+ require.Equal(t, 0, len(tbl1Info.ForeignKeys))
+ referredFKs := dom.InfoSchema().GetTableReferredForeignKeys("test", "t2")
+ require.Equal(t, 0, len(referredFKs))
+ tk.MustQuery("show create table t1").Check(testkit.Rows("t1 CREATE TABLE `t1` (\n" +
+ " `id` int(11) NOT NULL,\n" +
+ " `a` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `a` (`a`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+ // Test add foreign key with auto-create index failed.
+ tk.MustExec("drop table if exists t1,t2")
+ tk.MustExec("create table t1 (id int key,a int);")
+ tk.MustExec("create table t2 (id int key);")
+ tk.MustExec("insert into t1 values (1,1);")
+ err = tk.ExecToErr("ALTER TABLE t1 ADD foreign key fk(a) references t2(id) ON DELETE CASCADE;")
+ require.Error(t, err)
+ require.Equal(t, "[ddl:1452]Cannot add or update a child row: a foreign key constraint fails (`test`.`t1`, CONSTRAINT `fk` FOREIGN KEY (`a`) REFERENCES `t2` (`id`) ON DELETE CASCADE)", err.Error())
+ tbl1Info = getTableInfo(t, dom, "test", "t1")
+ require.Equal(t, 0, len(tbl1Info.ForeignKeys))
+ referredFKs = dom.InfoSchema().GetTableReferredForeignKeys("test", "t2")
+ require.Equal(t, 0, len(referredFKs))
+ tk.MustQuery("show create table t1").Check(testkit.Rows("t1 CREATE TABLE `t1` (\n" +
+ " `id` int(11) NOT NULL,\n" +
+ " `a` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+}
+
+func TestAlterTableAddForeignKeyError(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+ cases := []struct {
+ prepares []string
+ alter string
+ err string
+ }{
+ {
+ prepares: []string{
+ "create table t1 (id int, a int, b int);",
+ "create table t2 (a int, b int);",
+ },
+ alter: "alter table t2 add foreign key fk(b) references t_unknown(id)",
+ err: "[schema:1824]Failed to open the referenced table 't_unknown'",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int, a int, b int);",
+ "create table t2 (a int, b int);",
+ },
+ alter: "alter table t2 add foreign key fk(b) references t1(c_unknown)",
+ err: "[schema:3734]Failed to add the foreign key constraint. Missing column 'c_unknown' for constraint 'fk' in the referenced table 't1'",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int, a int, b int);",
+ "create table t2 (a int, b int);",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(b)",
+ err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int, a int, b int not null, index(b));",
+ "create table t2 (a int, b int not null);",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(b) on update set null",
+ err: "[schema:1830]Column 'b' cannot be NOT NULL: needed in a foreign key constraint 'fk_b' SET NULL",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int, a int, b int not null, index(b));",
+ "create table t2 (a int, b int not null);",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(b) on delete set null",
+ err: "[schema:1830]Column 'b' cannot be NOT NULL: needed in a foreign key constraint 'fk_b' SET NULL",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a int, b int as (a) virtual, index(b));",
+ "create table t2 (a int, b int);",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(b)",
+ err: "[schema:3733]Foreign key 'fk_b' uses virtual column 'b' which is not supported.",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a int, b int, index(b));",
+ "create table t2 (a int, b int as (a) virtual);",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(b)",
+ err: "[schema:3733]Foreign key 'fk_b' uses virtual column 'b' which is not supported.",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a int);",
+ "create table t2 (a int, b varchar(10));",
+ },
+ alter: "alter table t2 add foreign key fk(b) references t1(id)",
+ err: "[ddl:3780]Referencing column 'b' and referenced column 'id' in foreign key constraint 'fk' are incompatible.",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a int not null, index(a));",
+ "create table t2 (a int, b int unsigned);",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
+ err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a bigint, index(a));",
+ "create table t2 (a int, b int);",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
+ err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a varchar(10) charset utf8, index(a));",
+ "create table t2 (a int, b varchar(10) charset utf8mb4);",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
+ err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a varchar(10) collate utf8_bin, index(a));",
+ "create table t2 (a int, b varchar(10) collate utf8mb4_bin);",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
+ err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a varchar(10));",
+ "create table t2 (a int, b varchar(10));",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
+ err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a varchar(10), index (a(5)));",
+ "create table t2 (a int, b varchar(10));",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
+ err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key, a int)",
+ "create table t2 (id int, b int, index(b))",
+ "insert into t2 values (1,1)",
+ },
+ alter: "alter table t2 add foreign key fk_b(b) references t1(id)",
+ err: "[ddl:1452]Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `t1` (`id`))",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int, a int, b int, index(a,b))",
+ "create table t2 (id int, a int, b int, index(a,b))",
+ "insert into t2 values (1, 1, null), (2, null, 1), (3, null, null), (4, 1, 1)",
+ },
+ alter: "alter table t2 add foreign key fk_b(a, b) references t1(a, b)",
+ err: "[ddl:1452]Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `fk_b` FOREIGN KEY (`a`, `b`) REFERENCES `t1` (`a`, `b`))",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key);",
+ "create table t2 (a int, b int unique);",
+ },
+ alter: "alter table t2 add foreign key name5678901234567890123456789012345678901234567890123456789012345(b) references t1(id)",
+ err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key);",
+ "create table t2 (a int, b int unique);",
+ },
+ alter: "alter table t2 add constraint name5678901234567890123456789012345678901234567890123456789012345 foreign key (b) references t1(id)",
+ err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
+ },
+ // Test foreign key with temporary table.
+ {
+ prepares: []string{
+ "create temporary table t1 (id int key);",
+ "create table t2 (a int, b int unique);",
+ },
+ alter: "alter table t2 add constraint fk foreign key (b) references t1(id)",
+ err: "[schema:1824]Failed to open the referenced table 't1'",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key);",
+ "create temporary table t2 (a int, b int unique);",
+ },
+ alter: "alter table t2 add constraint fk foreign key (b) references t1(id)",
+ err: "[ddl:8200]TiDB doesn't support ALTER TABLE for local temporary table",
+ },
+ // Test foreign key with partition table
+ {
+ prepares: []string{
+ "create table t1 (id int key) partition by hash(id) partitions 3;",
+ "create table t2 (id int key);",
+ },
+ alter: "alter table t2 add constraint fk foreign key (id) references t1(id)",
+ err: "[schema:1506]Foreign key clause is not yet supported in conjunction with partitioning",
+ },
+ {
+ prepares: []string{
+ "create table t1 (id int key);",
+ "create table t2 (id int key) partition by hash(id) partitions 3;;",
+ },
+ alter: "alter table t2 add constraint fk foreign key (id) references t1(id)",
+ err: "[schema:1506]Foreign key clause is not yet supported in conjunction with partitioning",
+ },
+ }
+ for i, ca := range cases {
+ tk.MustExec("drop table if exists t2")
+ tk.MustExec("drop table if exists t1")
+ for _, sql := range ca.prepares {
+ tk.MustExec(sql)
+ }
+ err := tk.ExecToErr(ca.alter)
+ require.Error(t, err, fmt.Sprintf("%v, %v", i, ca.err))
+ require.Equal(t, ca.err, err.Error())
+ }
+
+ passCases := [][]string{
+ {
+ "create table t1 (id int key, a int, b int, index(a))",
+ "alter table t1 add foreign key fk(a) references t1(id)",
+ },
+ {
+ "create table t1 (id int key, b int not null, index(b))",
+ "create table t2 (a int, b int, index(b));",
+ "alter table t2 add foreign key fk_b(b) references t1(b)",
+ },
+ {
+ "create table t1 (id int key, a varchar(10), index(a));",
+ "create table t2 (a int, b varchar(20), index(b));",
+ "alter table t2 add foreign key fk_b(b) references t1(a)",
+ },
+ {
+ "create table t1 (id int key, a decimal(10,5), index(a));",
+ "create table t2 (a int, b decimal(20, 10), index(b));",
+ "alter table t2 add foreign key fk_b(b) references t1(a)",
+ },
+ {
+ "create table t1 (id int key, a varchar(10), index (a(10)));",
+ "create table t2 (a int, b varchar(20), index(b));",
+ "alter table t2 add foreign key fk_b(b) references t1(a)",
+ },
+ {
+ "create table t1 (id int key, a int)",
+ "create table t2 (id int, b int, index(b))",
+ "insert into t2 values (1, null)",
+ "alter table t2 add foreign key fk_b(b) references t1(id)",
+ },
+ {
+ "create table t1 (id int, a int, b int, index(a,b))",
+ "create table t2 (id int, a int, b int, index(a,b))",
+ "insert into t2 values (1, 1, null), (2, null, 1), (3, null, null)",
+ "alter table t2 add foreign key fk_b(a, b) references t1(a, b)",
+ },
+ {
+ "set @@foreign_key_checks=0;",
+ "create table t1 (id int, a int, b int, index(a,b))",
+ "create table t2 (id int, a int, b int, index(a,b))",
+ "insert into t2 values (1, 1, 1)",
+ "alter table t2 add foreign key fk_b(a, b) references t1(a, b)",
+ "set @@foreign_key_checks=1;",
+ },
+ {
+ "set @@foreign_key_checks=0;",
+ "create table t2 (a int, b int, index(b));",
+ "alter table t2 add foreign key fk_b(b) references t_unknown(a)",
+ "set @@foreign_key_checks=1;",
+ },
+ {
+ "create table t1 (id int key);",
+ "create table t2 (a int, b int unique);",
+ "alter table t2 add foreign key name567890123456789012345678901234567890123456789012345678901234(b) references t1(id)",
+ },
+ }
+ for _, ca := range passCases {
+ tk.MustExec("drop table if exists t2")
+ tk.MustExec("drop table if exists t1")
+ for _, sql := range ca {
+ tk.MustExec(sql)
+ }
+ }
+}
+
+func TestRenameTablesWithForeignKey(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=0;")
+ tk.MustExec("create database test1")
+ tk.MustExec("create database test2")
+ tk.MustExec("use test")
+ tk.MustExec("create table t0 (id int key, b int);")
+ tk.MustExec("create table t1 (id int key, b int, index(b), foreign key fk(b) references t2(id));")
+ tk.MustExec("create table t2 (id int key, b int, index(b), foreign key fk(b) references t1(id));")
+ tk.MustExec("rename table test.t1 to test1.tt1, test.t2 to test2.tt2, test.t0 to test.tt0")
+
+ // check the schema diff
+ diff := getLatestSchemaDiff(t, tk)
+ require.Equal(t, model.ActionRenameTables, diff.Type)
+ require.Equal(t, 3, len(diff.AffectedOpts))
+
+ // check referred foreign key information.
+ t1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
+ t2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t2")
+ require.Equal(t, 0, len(t1ReferredFKs))
+ require.Equal(t, 0, len(t2ReferredFKs))
+ tt1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test1", "tt1")
+ tt2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "tt2")
+ require.Equal(t, 1, len(tt1ReferredFKs))
+ require.Equal(t, 1, len(tt2ReferredFKs))
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test2"),
+ ChildTable: model.NewCIStr("tt2"),
+ ChildFKName: model.NewCIStr("fk"),
+ }, *tt1ReferredFKs[0])
+ require.Equal(t, model.ReferredFKInfo{
+ Cols: []model.CIStr{model.NewCIStr("id")},
+ ChildSchema: model.NewCIStr("test1"),
+ ChildTable: model.NewCIStr("tt1"),
+ ChildFKName: model.NewCIStr("fk"),
+ }, *tt2ReferredFKs[0])
+
+ // check show create table information
+ tk.MustQuery("show create table test1.tt1").Check(testkit.Rows("tt1 CREATE TABLE `tt1` (\n" +
+ " `id` int(11) NOT NULL,\n" +
+ " `b` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " CONSTRAINT `fk` FOREIGN KEY (`b`) REFERENCES `test2`.`tt2` (`id`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+ tk.MustQuery("show create table test2.tt2").Check(testkit.Rows("tt2 CREATE TABLE `tt2` (\n" +
+ " `id` int(11) NOT NULL,\n" +
+ " `b` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " CONSTRAINT `fk` FOREIGN KEY (`b`) REFERENCES `test1`.`tt1` (`id`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+}
+
+func getLatestSchemaDiff(t *testing.T, tk *testkit.TestKit) *model.SchemaDiff {
+ ctx := tk.Session()
+ err := sessiontxn.NewTxn(context.Background(), ctx)
+ require.NoError(t, err)
+ txn, err := ctx.Txn(true)
+ require.NoError(t, err)
+ m := meta.NewMeta(txn)
+ ver, err := m.GetSchemaVersion()
+ require.NoError(t, err)
+ diff, err := m.GetSchemaDiff(ver)
+ require.NoError(t, err)
+ return diff
+}
+
+func TestMultiSchemaAddForeignKey(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key);")
+ tk.MustExec("create table t2 (a int, b int);")
+ tk.MustExec("alter table t2 add foreign key (a) references t1(id), add foreign key (b) references t1(id)")
+ tk.MustExec("alter table t2 add column c int, add column d int")
+ tk.MustExec("alter table t2 add foreign key (c) references t1(id), add foreign key (d) references t1(id), add index(c), add index(d)")
+ tk.MustExec("drop table t2")
+ tk.MustExec("create table t2 (a int, b int, index idx1(a), index idx2(b));")
+ tk.MustGetErrMsg("alter table t2 drop index idx1, drop index idx2, add foreign key (a) references t1(id), add foreign key (b) references t1(id)",
+ "[ddl:1553]Cannot drop index 'idx1': needed in a foreign key constraint")
+ tk.MustExec("alter table t2 drop index idx1, drop index idx2")
+ tk.MustExec("alter table t2 add foreign key (a) references t1(id), add foreign key (b) references t1(id)")
+ tk.MustQuery("show create table t2").Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" +
+ " `a` int(11) DEFAULT NULL,\n" +
+ " `b` int(11) DEFAULT NULL,\n" +
+ " KEY `fk_1` (`a`),\n" +
+ " KEY `fk_2` (`b`),\n" +
+ " CONSTRAINT `fk_1` FOREIGN KEY (`a`) REFERENCES `test`.`t1` (`id`),\n" +
+ " CONSTRAINT `fk_2` FOREIGN KEY (`b`) REFERENCES `test`.`t1` (`id`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+ tk.MustExec("drop table t2")
+ tk.MustExec("create table t2 (a int, b int, index idx0(a,b), index idx1(a), index idx2(b));")
+ tk.MustExec("alter table t2 drop index idx1, add foreign key (a) references t1(id), add foreign key (b) references t1(id)")
+}
+
+func TestAddForeignKeyInBigTable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+ tk.MustExec("create table employee (id bigint auto_increment key, pid bigint)")
+ tk.MustExec("insert into employee (id) values (1),(2),(3),(4),(5),(6),(7),(8)")
+ for i := 0; i < 14; i++ {
+ tk.MustExec("insert into employee (pid) select pid from employee")
+ }
+ tk.MustExec("update employee set pid=id-1 where id>1")
+ start := time.Now()
+ tk.MustExec("alter table employee add foreign key fk_1(pid) references employee(id)")
+ require.Less(t, time.Since(start), time.Minute)
+}
+
+func TestForeignKeyWithCacheTable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+ // Test foreign key refer cache table.
+ tk.MustExec("create table t1 (id int key);")
+ tk.MustExec("insert into t1 values (1),(2),(3),(4)")
+ tk.MustExec("alter table t1 cache;")
+ tk.MustExec("create table t2 (b int);")
+ tk.MustExec("alter table t2 add constraint fk foreign key (b) references t1(id) on delete cascade on update cascade")
+ tk.MustExec("insert into t2 values (1),(2),(3),(4)")
+ tk.MustGetDBError("insert into t2 values (5)", plannercore.ErrNoReferencedRow2)
+ tk.MustExec("update t1 set id = id+10 where id=1")
+ tk.MustExec("delete from t1 where id<10")
+ tk.MustQuery("select * from t1").Check(testkit.Rows("11"))
+ tk.MustQuery("select * from t2").Check(testkit.Rows("11"))
+ tk.MustExec("alter table t1 nocache;")
+ tk.MustExec("drop table t1,t2;")
+
+ // Test add foreign key on cache table.
+ tk.MustExec("create table t1 (id int key);")
+ tk.MustExec("create table t2 (b int);")
+ tk.MustExec("alter table t2 add constraint fk foreign key (b) references t1(id) on delete cascade on update cascade")
+ tk.MustExec("alter table t2 cache;")
+ tk.MustExec("insert into t1 values (1),(2),(3),(4)")
+ tk.MustExec("insert into t2 values (1),(2),(3),(4)")
+ tk.MustGetDBError("insert into t2 values (5)", plannercore.ErrNoReferencedRow2)
+ tk.MustExec("update t1 set id = id+10 where id=1")
+ tk.MustExec("delete from t1 where id<10")
+ tk.MustQuery("select * from t1").Check(testkit.Rows("11"))
+ tk.MustQuery("select * from t2").Check(testkit.Rows("11"))
+ tk.MustExec("alter table t2 nocache;")
+ tk.MustExec("drop table t1,t2;")
+}
+
+func TestForeignKeyAndConcurrentDDL(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+ // Test foreign key refer cache table.
+ tk.MustExec("create table t1 (a int, b int, c int, index(a), index(b), index(c));")
+ tk.MustExec("create table t2 (a int, b int, c int, index(a), index(b), index(c));")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("set @@foreign_key_checks=1;")
+ tk2.MustExec("use test")
+ passCases := []struct {
+ prepare []string
+ ddl1 string
+ ddl2 string
+ }{
+ {
+ ddl1: "alter table t2 add constraint fk_1 foreign key (a) references t1(a)",
+ ddl2: "alter table t2 add constraint fk_2 foreign key (b) references t1(b)",
+ },
+ {
+ ddl1: "alter table t2 drop foreign key fk_1",
+ ddl2: "alter table t2 drop foreign key fk_2",
+ },
+ {
+ prepare: []string{
+ "alter table t2 drop index a",
+ },
+ ddl1: "alter table t2 add index(a)",
+ ddl2: "alter table t2 add constraint fk_1 foreign key (a) references t1(a)",
+ },
+ {
+ ddl1: "alter table t2 drop index c",
+ ddl2: "alter table t2 add constraint fk_2 foreign key (b) references t1(b)",
+ },
+ }
+ for _, ca := range passCases {
+ var wg sync.WaitGroup
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ tk.MustExec(ca.ddl1)
+ }()
+ go func() {
+ defer wg.Done()
+ tk2.MustExec(ca.ddl2)
+ }()
+ wg.Wait()
+ }
+ errorCases := []struct {
+ prepare []string
+ ddl1 string
+ err1 string
+ ddl2 string
+ err2 string
+ }{
+ {
+ ddl1: "alter table t2 add constraint fk foreign key (a) references t1(a)",
+ err1: "[ddl:1826]Duplicate foreign key constraint name 'fk'",
+ ddl2: "alter table t2 add constraint fk foreign key (b) references t1(b)",
+ err2: "[ddl:1826]Duplicate foreign key constraint name 'fk'",
+ },
+ {
+ prepare: []string{
+ "alter table t2 add constraint fk_1 foreign key (a) references t1(a)",
+ },
+ ddl1: "alter table t2 drop foreign key fk_1",
+ err1: "[schema:1091]Can't DROP 'fk_1'; check that column/key exists",
+ ddl2: "alter table t2 drop foreign key fk_1",
+ err2: "[schema:1091]Can't DROP 'fk_1'; check that column/key exists",
+ },
+ {
+ ddl1: "alter table t2 drop index a",
+ err1: "[ddl:1553]Cannot drop index 'a': needed in a foreign key constraint",
+ ddl2: "alter table t2 add constraint fk_1 foreign key (a) references t1(a)",
+ err2: "[ddl:-1]Failed to add the foreign key constraint. Missing index for 'fk_1' foreign key columns in the table 't2'",
+ },
+ }
+ tk.MustExec("drop table t1,t2")
+ tk.MustExec("create table t1 (a int, b int, c int, index(a), index(b), index(c));")
+ tk.MustExec("create table t2 (a int, b int, c int, index(a), index(b), index(c));")
+ for i, ca := range errorCases {
+ for _, sql := range ca.prepare {
+ tk.MustExec(sql)
+ }
+ var wg sync.WaitGroup
+ var err1, err2 error
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ err1 = tk.ExecToErr(ca.ddl1)
+ }()
+ go func() {
+ defer wg.Done()
+ err2 = tk2.ExecToErr(ca.ddl2)
+ }()
+ wg.Wait()
+ if (err1 == nil && err2 == nil) || (err1 != nil && err2 != nil) {
+ require.Failf(t, "both ddl1 and ddl2 execute success, but expect 1 error", fmt.Sprintf("idx: %v, err1: %v, err2: %v", i, err1, err2))
+ }
+ if err1 != nil {
+ require.Equal(t, ca.err1, err1.Error())
+ }
+ if err2 != nil {
+ require.Equal(t, ca.err2, err2.Error())
+ }
+ }
+}
+
+func TestForeignKeyAndRenameIndex(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1;")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, b int, index idx1(b));")
+ tk.MustExec("create table t2 (id int key, b int, constraint fk foreign key (b) references t1(b));")
+ tk.MustExec("insert into t1 values (1,1),(2,2)")
+ tk.MustExec("insert into t2 values (1,1),(2,2)")
+ tk.MustGetDBError("insert into t2 values (3,3)", plannercore.ErrNoReferencedRow2)
+ tk.MustGetDBError("delete from t1 where id=1", plannercore.ErrRowIsReferenced2)
+ tk.MustExec("alter table t1 rename index idx1 to idx2")
+ tk.MustExec("alter table t2 rename index fk to idx")
+ tk.MustGetDBError("insert into t2 values (3,3)", plannercore.ErrNoReferencedRow2)
+ tk.MustGetDBError("delete from t1 where id=1", plannercore.ErrRowIsReferenced2)
+ tk.MustExec("alter table t2 drop foreign key fk")
+ tk.MustExec("alter table t2 add foreign key fk (b) references t1(b) on delete cascade on update cascade")
+ tk.MustExec("alter table t1 rename index idx2 to idx3")
+ tk.MustExec("alter table t2 rename index idx to idx0")
+ tk.MustExec("delete from t1 where id=1")
+ tk.MustQuery("select * from t1").Check(testkit.Rows("2 2"))
+ tk.MustQuery("select * from t2").Check(testkit.Rows("2 2"))
+ tk.MustExec("admin check table t1,t2")
+}
diff --git a/ddl/fktest/main_test.go b/ddl/fktest/main_test.go
new file mode 100644
index 0000000000000..36f34049a9d03
--- /dev/null
+++ b/ddl/fktest/main_test.go
@@ -0,0 +1,56 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl_test
+
+import (
+ "testing"
+ "time"
+
+ "github.com/pingcap/tidb/config"
+ "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/domain"
+ "github.com/pingcap/tidb/meta/autoid"
+ "github.com/pingcap/tidb/testkit/testsetup"
+ "github.com/tikv/client-go/v2/tikv"
+ "go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+ testsetup.SetupForCommonTest()
+ tikv.EnableFailpoints()
+
+ domain.SchemaOutOfDateRetryInterval.Store(50 * time.Millisecond)
+ domain.SchemaOutOfDateRetryTimes.Store(50)
+
+ autoid.SetStep(5000)
+ ddl.RunInGoTest = true
+
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.Instance.SlowThreshold = 10000
+ conf.TiKVClient.AsyncCommit.SafeWindow = 0
+ conf.TiKVClient.AsyncCommit.AllowedClockDrift = 0
+ conf.Experimental.AllowsExpressionIndex = true
+ })
+
+ opts := []goleak.Option{
+ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
+ goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
+ goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
+ goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"),
+ goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
+ }
+
+ goleak.VerifyTestMain(m, opts...)
+}
diff --git a/ddl/foreign_key.go b/ddl/foreign_key.go
index 6f1d02d464ac8..1a06719cb404b 100644
--- a/ddl/foreign_key.go
+++ b/ddl/foreign_key.go
@@ -44,6 +44,9 @@ func (w *worker) onCreateForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
+ if job.IsRollingback() {
+ return dropForeignKey(d, t, job, tblInfo, fkInfo.Name)
+ }
switch job.SchemaState {
case model.StateNone:
err = checkAddForeignKeyValidInOwner(d, t, job.SchemaName, tblInfo, &fkInfo, fkCheck)
@@ -63,7 +66,7 @@ func (w *worker) onCreateForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
case model.StateWriteOnly:
err = checkForeignKeyConstrain(w, job.SchemaName, tblInfo.Name.L, &fkInfo, fkCheck)
if err != nil {
- job.State = model.JobStateCancelled
+ job.State = model.JobStateRollingback
return ver, err
}
tblInfo.ForeignKeys[len(tblInfo.ForeignKeys)-1].State = model.StateWriteReorganization
@@ -94,29 +97,27 @@ func onDropForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ err
return ver, errors.Trace(err)
}
- var (
- fkName model.CIStr
- found bool
- fkInfo model.FKInfo
- )
+ var fkName model.CIStr
err = job.DecodeArgs(&fkName)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
+ return dropForeignKey(d, t, job, tblInfo, fkName)
+}
+func dropForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, fkName model.CIStr) (ver int64, err error) {
+ var fkInfo *model.FKInfo
for _, fk := range tblInfo.ForeignKeys {
if fk.Name.L == fkName.L {
- found = true
- fkInfo = *fk
+ fkInfo = fk
+ break
}
}
-
- if !found {
+ if fkInfo == nil {
job.State = model.JobStateCancelled
return ver, infoschema.ErrForeignKeyNotExists.GenWithStackByArgs(fkName)
}
-
nfks := tblInfo.ForeignKeys[:0]
for _, fk := range tblInfo.ForeignKeys {
if fk.Name.L != fkName.L {
@@ -124,24 +125,18 @@ func onDropForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ err
}
}
tblInfo.ForeignKeys = nfks
-
- originalState := fkInfo.State
- switch fkInfo.State {
- case model.StatePublic:
- // We just support record the foreign key, so we just make it none.
- // public -> none
- fkInfo.State = model.StateNone
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != fkInfo.State)
- if err != nil {
- return ver, errors.Trace(err)
- }
- // Finish this job.
+ ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ // Finish this job.
+ if job.IsRollingback() {
+ job.FinishTableJob(model.JobStateRollbackDone, model.StateNone, ver, tblInfo)
+ } else {
job.FinishTableJob(model.JobStateDone, model.StateNone, ver, tblInfo)
- job.SchemaState = fkInfo.State
- return ver, nil
- default:
- return ver, dbterror.ErrInvalidDDLState.GenWithStackByArgs("foreign key", fkInfo.State)
}
+ job.SchemaState = model.StateNone
+ return ver, err
}
func allocateFKIndexID(tblInfo *model.TableInfo) int64 {
@@ -268,6 +263,12 @@ func checkTableForeignKey(referTblInfo, tblInfo *model.TableInfo, fkInfo *model.
if referTblInfo.TempTableType != model.TempTableNone || tblInfo.TempTableType != model.TempTableNone {
return infoschema.ErrCannotAddForeign
}
+ if referTblInfo.TTLInfo != nil {
+ return dbterror.ErrUnsupportedTTLReferencedByFK
+ }
+ if referTblInfo.GetPartitionInfo() != nil || tblInfo.GetPartitionInfo() != nil {
+ return infoschema.ErrForeignKeyOnPartitioned
+ }
// check refer columns in parent table.
for i := range fkInfo.RefCols {
@@ -293,7 +294,7 @@ func checkTableForeignKey(referTblInfo, tblInfo *model.TableInfo, fkInfo *model.
}
}
// check refer columns should have index.
- if model.FindIndexByColumns(referTblInfo, fkInfo.RefCols...) == nil {
+ if model.FindIndexByColumns(referTblInfo, referTblInfo.Indices, fkInfo.RefCols...) == nil {
return infoschema.ErrForeignKeyNoIndexInParent.GenWithStackByArgs(fkInfo.Name, fkInfo.RefTable)
}
return nil
@@ -662,7 +663,7 @@ func checkAddForeignKeyValidInOwner(d *ddlCtx, t *meta.Meta, schema string, tbIn
return nil
}
}
- if model.FindIndexByColumns(tbInfo, fk.Cols...) == nil {
+ if model.FindIndexByColumns(tbInfo, tbInfo.Indices, fk.Cols...) == nil {
return errors.Errorf("Failed to add the foreign key constraint. Missing index for '%s' foreign key columns in the table '%s'", fk.Name, tbInfo.Name)
}
return nil
@@ -676,7 +677,12 @@ func checkForeignKeyConstrain(w *worker, schema, table string, fkInfo *model.FKI
if err != nil {
return errors.Trace(err)
}
- defer w.sessPool.put(sctx)
+ originValue := sctx.GetSessionVars().OptimizerEnableNAAJ
+ sctx.GetSessionVars().OptimizerEnableNAAJ = true
+ defer func() {
+ sctx.GetSessionVars().OptimizerEnableNAAJ = originValue
+ w.sessPool.put(sctx)
+ }()
var buf strings.Builder
buf.WriteString("select 1 from %n.%n where ")
@@ -711,7 +717,7 @@ func checkForeignKeyConstrain(w *worker, schema, table string, fkInfo *model.FKI
}
buf.WriteString(" from %n.%n ) limit 1")
paramsList = append(paramsList, fkInfo.RefSchema.L, fkInfo.RefTable.L)
- rows, _, err := sctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(w.ctx, nil, buf.String(), paramsList...)
+ rows, _, err := sctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(w.ctx, []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseCurSession}, buf.String(), paramsList...)
if err != nil {
return errors.Trace(err)
}
diff --git a/ddl/foreign_key_test.go b/ddl/foreign_key_test.go
index a50fd80e03681..032adfb296120 100644
--- a/ddl/foreign_key_test.go
+++ b/ddl/foreign_key_test.go
@@ -15,9 +15,7 @@
package ddl_test
import (
- "bytes"
"context"
- "fmt"
"strings"
"sync"
"testing"
@@ -25,17 +23,13 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/ddl"
- "github.com/pingcap/tidb/domain"
- "github.com/pingcap/tidb/infoschema"
- "github.com/pingcap/tidb/meta"
- "github.com/pingcap/tidb/parser/auth"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/parser/model"
- plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessiontxn"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/testkit"
- "github.com/pingcap/tidb/util/dbterror"
+ "github.com/pingcap/tidb/types"
"github.com/stretchr/testify/require"
)
@@ -115,14 +109,24 @@ func TestForeignKey(t *testing.T) {
testCreateSchema(t, testkit.NewTestKit(t, store).Session(), dom.DDL(), dbInfo)
tblInfo, err := testTableInfo(store, "t", 3)
require.NoError(t, err)
-
+ tblInfo.Indices = append(tblInfo.Indices, &model.IndexInfo{
+ ID: 1,
+ Name: model.NewCIStr("idx_fk"),
+ Table: model.NewCIStr("t"),
+ Columns: []*model.IndexColumn{{
+ Name: model.NewCIStr("c1"),
+ Offset: 0,
+ Length: types.UnspecifiedLength,
+ }},
+ State: model.StatePublic,
+ })
testCreateTable(t, testkit.NewTestKit(t, store).Session(), d, dbInfo, tblInfo)
// fix data race
var mu sync.Mutex
checkOK := false
var hookErr error
- tc := &ddl.TestDDLCallback{}
+ tc := &callback.TestDDLCallback{}
onJobUpdatedExportedFunc := func(job *model.Job) {
if job.State != model.JobStateDone {
return
@@ -164,7 +168,7 @@ func TestForeignKey(t *testing.T) {
checkOK = false
mu.Unlock()
// fix data race pr/#9491
- tc2 := &ddl.TestDDLCallback{}
+ tc2 := &callback.TestDDLCallback{}
onJobUpdatedExportedFunc2 := func(job *model.Job) {
if job.State != model.JobStateDone {
return
@@ -204,853 +208,6 @@ func TestForeignKey(t *testing.T) {
require.NoError(t, err)
}
-func TestCreateTableWithForeignKeyMetaInfo(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (id int key, a int,b int as (a) virtual);")
- tk.MustExec("create database test2")
- tk.MustExec("use test2")
- tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references test.t1(id) ON UPDATE RESTRICT ON DELETE CASCADE)")
- tb1Info := getTableInfo(t, dom, "test", "t1")
- tb2Info := getTableInfo(t, dom, "test2", "t2")
- require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test", "t1")))
- require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t2")))
- require.Equal(t, 0, len(tb1Info.ForeignKeys))
- tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test2"),
- ChildTable: model.NewCIStr("t2"),
- ChildFKName: model.NewCIStr("fk_b"),
- }, *tb1ReferredFKs[0])
- tb2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "t2")
- require.Equal(t, 0, len(tb2ReferredFKs))
- require.Equal(t, 1, len(tb2Info.ForeignKeys))
- require.Equal(t, model.FKInfo{
- ID: 1,
- Name: model.NewCIStr("fk_b"),
- RefSchema: model.NewCIStr("test"),
- RefTable: model.NewCIStr("t1"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("b")},
- OnDelete: 2,
- OnUpdate: 1,
- State: model.StatePublic,
- Version: 1,
- }, *tb2Info.ForeignKeys[0])
- // Auto create index for foreign key usage.
- require.Equal(t, 1, len(tb2Info.Indices))
- require.Equal(t, "fk_b", tb2Info.Indices[0].Name.L)
- require.Equal(t, "`test2`.`t2`, CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `test`.`t1` (`id`) ON DELETE CASCADE ON UPDATE RESTRICT", tb2Info.ForeignKeys[0].String("test2", "t2"))
-
- tk.MustExec("create table t3 (id int, b int, index idx_b(b), foreign key fk_b(b) references t2(id) ON UPDATE SET NULL ON DELETE NO ACTION)")
- tb2Info = getTableInfo(t, dom, "test2", "t2")
- tb3Info := getTableInfo(t, dom, "test2", "t3")
- require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t2")))
- require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t3")))
- require.Equal(t, 1, len(tb2Info.ForeignKeys))
- tb2ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test2", "t2")
- require.Equal(t, 1, len(tb2ReferredFKs))
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test2"),
- ChildTable: model.NewCIStr("t3"),
- ChildFKName: model.NewCIStr("fk_b"),
- }, *tb2ReferredFKs[0])
- tb3ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "t3")
- require.Equal(t, 0, len(tb3ReferredFKs))
- require.Equal(t, 1, len(tb3Info.ForeignKeys))
- require.Equal(t, model.FKInfo{
- ID: 1,
- Name: model.NewCIStr("fk_b"),
- RefSchema: model.NewCIStr("test2"),
- RefTable: model.NewCIStr("t2"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("b")},
- OnDelete: 4,
- OnUpdate: 3,
- State: model.StatePublic,
- Version: 1,
- }, *tb3Info.ForeignKeys[0])
- require.Equal(t, 1, len(tb3Info.Indices))
- require.Equal(t, "idx_b", tb3Info.Indices[0].Name.L)
- require.Equal(t, "`test2`.`t3`, CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `t2` (`id`) ON DELETE NO ACTION ON UPDATE SET NULL", tb3Info.ForeignKeys[0].String("test2", "t3"))
-
- tk.MustExec("create table t5 (id int key, a int, b int, foreign key (a) references t5(id));")
- tb5Info := getTableInfo(t, dom, "test2", "t5")
- require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t5")))
- require.Equal(t, 1, len(tb5Info.ForeignKeys))
- tb5ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "t5")
- require.Equal(t, 1, len(tb5ReferredFKs))
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test2"),
- ChildTable: model.NewCIStr("t5"),
- ChildFKName: model.NewCIStr("fk_1"),
- }, *tb5ReferredFKs[0])
- require.Equal(t, model.FKInfo{
- ID: 1,
- Name: model.NewCIStr("fk_1"),
- RefSchema: model.NewCIStr("test2"),
- RefTable: model.NewCIStr("t5"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("a")},
- State: model.StatePublic,
- Version: 1,
- }, *tb5Info.ForeignKeys[0])
- require.Equal(t, 1, len(tb5Info.Indices))
- require.Equal(t, "fk_1", tb5Info.Indices[0].Name.L)
- require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test", "t1")))
- require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t2")))
- require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t3")))
- require.Equal(t, 1, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t5")))
-
- tk.MustExec("set @@global.tidb_enable_foreign_key=0")
- tk.MustExec("drop database test2")
- require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t2")))
- require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t3")))
- require.Equal(t, 0, len(dom.InfoSchema().GetTableReferredForeignKeys("test2", "t5")))
-}
-
-func TestCreateTableWithForeignKeyMetaInfo2(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("create database test2")
- tk.MustExec("set @@foreign_key_checks=0")
- tk.MustExec("use test2")
- tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references test.t1(id) ON UPDATE RESTRICT ON DELETE CASCADE)")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (id int key, a int, b int as (a) virtual);")
- tb1Info := getTableInfo(t, dom, "test", "t1")
- tb2Info := getTableInfo(t, dom, "test2", "t2")
- require.Equal(t, 0, len(tb1Info.ForeignKeys))
- tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test2"),
- ChildTable: model.NewCIStr("t2"),
- ChildFKName: model.NewCIStr("fk_b"),
- }, *tb1ReferredFKs[0])
- tb2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "t2")
- require.Equal(t, 0, len(tb2ReferredFKs))
- require.Equal(t, 1, len(tb2Info.ForeignKeys))
- require.Equal(t, model.FKInfo{
- ID: 1,
- Name: model.NewCIStr("fk_b"),
- RefSchema: model.NewCIStr("test"),
- RefTable: model.NewCIStr("t1"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("b")},
- OnDelete: 2,
- OnUpdate: 1,
- State: model.StatePublic,
- Version: 1,
- }, *tb2Info.ForeignKeys[0])
- // Auto create index for foreign key usage.
- require.Equal(t, 1, len(tb2Info.Indices))
- require.Equal(t, "fk_b", tb2Info.Indices[0].Name.L)
- require.Equal(t, "`test2`.`t2`, CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `test`.`t1` (`id`) ON DELETE CASCADE ON UPDATE RESTRICT", tb2Info.ForeignKeys[0].String("test2", "t2"))
-
- tk.MustExec("create table t3 (id int key, a int, foreign key fk_a(a) references test.t1(id) ON DELETE CASCADE ON UPDATE RESTRICT, foreign key fk_a2(a) references test2.t2(id))")
- tb1Info = getTableInfo(t, dom, "test", "t1")
- tb3Info := getTableInfo(t, dom, "test", "t3")
- require.Equal(t, 0, len(tb1Info.ForeignKeys))
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 2, len(tb1ReferredFKs))
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test"),
- ChildTable: model.NewCIStr("t3"),
- ChildFKName: model.NewCIStr("fk_a"),
- }, *tb1ReferredFKs[0])
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test2"),
- ChildTable: model.NewCIStr("t2"),
- ChildFKName: model.NewCIStr("fk_b"),
- }, *tb1ReferredFKs[1])
- tb3ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t3")
- require.Equal(t, 0, len(tb3ReferredFKs))
- require.Equal(t, 2, len(tb3Info.ForeignKeys))
- require.Equal(t, model.FKInfo{
- ID: 1,
- Name: model.NewCIStr("fk_a"),
- RefSchema: model.NewCIStr("test"),
- RefTable: model.NewCIStr("t1"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("a")},
- OnDelete: 2,
- OnUpdate: 1,
- State: model.StatePublic,
- Version: 1,
- }, *tb3Info.ForeignKeys[0])
- require.Equal(t, model.FKInfo{
- ID: 2,
- Name: model.NewCIStr("fk_a2"),
- RefSchema: model.NewCIStr("test2"),
- RefTable: model.NewCIStr("t2"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("a")},
- State: model.StatePublic,
- Version: 1,
- }, *tb3Info.ForeignKeys[1])
- // Auto create index for foreign key usage.
- require.Equal(t, 1, len(tb3Info.Indices))
- require.Equal(t, "fk_a", tb3Info.Indices[0].Name.L)
- require.Equal(t, "`test`.`t3`, CONSTRAINT `fk_a` FOREIGN KEY (`a`) REFERENCES `t1` (`id`) ON DELETE CASCADE ON UPDATE RESTRICT", tb3Info.ForeignKeys[0].String("test", "t3"))
- require.Equal(t, "`test`.`t3`, CONSTRAINT `fk_a2` FOREIGN KEY (`a`) REFERENCES `test2`.`t2` (`id`)", tb3Info.ForeignKeys[1].String("test", "t3"))
-
- tk.MustExec("set @@foreign_key_checks=0")
- tk.MustExec("drop table test2.t2")
- tb1Info = getTableInfo(t, dom, "test", "t1")
- tb3Info = getTableInfo(t, dom, "test", "t3")
- require.Equal(t, 0, len(tb1Info.ForeignKeys))
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test"),
- ChildTable: model.NewCIStr("t3"),
- ChildFKName: model.NewCIStr("fk_a"),
- }, *tb1ReferredFKs[0])
- tb3ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t3")
- require.Equal(t, 0, len(tb3ReferredFKs))
- require.Equal(t, 2, len(tb3Info.ForeignKeys))
- require.Equal(t, model.FKInfo{
- ID: 1,
- Name: model.NewCIStr("fk_a"),
- RefSchema: model.NewCIStr("test"),
- RefTable: model.NewCIStr("t1"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("a")},
- OnDelete: 2,
- OnUpdate: 1,
- State: model.StatePublic,
- Version: 1,
- }, *tb3Info.ForeignKeys[0])
- require.Equal(t, model.FKInfo{
- ID: 2,
- Name: model.NewCIStr("fk_a2"),
- RefSchema: model.NewCIStr("test2"),
- RefTable: model.NewCIStr("t2"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("a")},
- State: model.StatePublic,
- Version: 1,
- }, *tb3Info.ForeignKeys[1])
-}
-
-func TestCreateTableWithForeignKeyMetaInfo3(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("set @@foreign_key_checks=1")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (id int key, a int, b int as (a) virtual);")
- tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
- tk.MustExec("create table t3 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
- tk.MustExec("create table t4 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
- tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- tk.MustExec("drop table t3")
- tk.MustExec("create table t5 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
- require.Equal(t, 3, len(tb1ReferredFKs))
- require.Equal(t, "t2", tb1ReferredFKs[0].ChildTable.L)
- require.Equal(t, "t3", tb1ReferredFKs[1].ChildTable.L)
- require.Equal(t, "t4", tb1ReferredFKs[2].ChildTable.L)
-}
-
-func TestCreateTableWithForeignKeyPrivilegeCheck(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
-
- tk.MustExec("create user 'u1'@'%' identified by '';")
- tk.MustExec("grant create on *.* to 'u1'@'%';")
- tk.MustExec("create table t1 (id int key);")
-
- tk2 := testkit.NewTestKit(t, store)
- tk2.MustExec("use test")
- tk2.Session().Auth(&auth.UserIdentity{Username: "u1", Hostname: "localhost", CurrentUser: true, AuthUsername: "u1", AuthHostname: "%"}, nil, []byte("012345678901234567890"))
- err := tk2.ExecToErr("create table t2 (a int, foreign key fk(a) references t1(id));")
- require.Error(t, err)
- require.Equal(t, "[planner:1142]REFERENCES command denied to user 'u1'@'%' for table 't1'", err.Error())
-
- tk.MustExec("grant references on test.t1 to 'u1'@'%';")
- tk2.MustExec("create table t2 (a int, foreign key fk(a) references t1(id));")
- tk2.MustExec("create table t3 (id int key)")
- err = tk2.ExecToErr("create table t4 (a int, foreign key fk(a) references t1(id), foreign key (a) references t3(id));")
- require.Error(t, err)
- require.Equal(t, "[planner:1142]REFERENCES command denied to user 'u1'@'%' for table 't3'", err.Error())
-
- tk.MustExec("grant references on test.t3 to 'u1'@'%';")
- tk2.MustExec("create table t4 (a int, foreign key fk(a) references t1(id), foreign key (a) references t3(id));")
-}
-
-func TestRenameTableWithForeignKeyMetaInfo(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("create database test2")
- tk.MustExec("create database test3")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (id int key, a int, b int, foreign key fk(a) references t1(id))")
- tk.MustExec("rename table test.t1 to test2.t2")
- // check the schema diff
- diff := getLatestSchemaDiff(t, tk)
- require.Equal(t, model.ActionRenameTable, diff.Type)
- require.Equal(t, 0, len(diff.AffectedOpts))
- tk.MustQuery("show create table test2.t2").Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" +
- " `id` int(11) NOT NULL,\n" +
- " `a` int(11) DEFAULT NULL,\n" +
- " `b` int(11) DEFAULT NULL,\n" +
- " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" +
- " KEY `fk` (`a`),\n" +
- " CONSTRAINT `fk` FOREIGN KEY (`a`) REFERENCES `test2`.`t2` (`id`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
- tblInfo := getTableInfo(t, dom, "test2", "t2")
- tbReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "t2")
- require.Equal(t, 1, len(tblInfo.ForeignKeys))
- require.Equal(t, 1, len(tbReferredFKs))
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test2"),
- ChildTable: model.NewCIStr("t2"),
- ChildFKName: model.NewCIStr("fk"),
- }, *tbReferredFKs[0])
- require.Equal(t, model.FKInfo{
- ID: 1,
- Name: model.NewCIStr("fk"),
- RefSchema: model.NewCIStr("test2"),
- RefTable: model.NewCIStr("t2"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("a")},
- State: model.StatePublic,
- Version: 1,
- }, *tblInfo.ForeignKeys[0])
-
- tk.MustExec("drop table test2.t2")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (id int key, a int, b int as (a) virtual);")
- tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
- tk.MustExec("use test2")
- tk.MustExec("rename table test.t2 to test2.tt2")
- // check the schema diff
- diff = getLatestSchemaDiff(t, tk)
- require.Equal(t, model.ActionRenameTable, diff.Type)
- require.Equal(t, 0, len(diff.AffectedOpts))
- tb1Info := getTableInfo(t, dom, "test", "t1")
- tb2Info := getTableInfo(t, dom, "test2", "tt2")
- require.Equal(t, 0, len(tb1Info.ForeignKeys))
- tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test2"),
- ChildTable: model.NewCIStr("tt2"),
- ChildFKName: model.NewCIStr("fk_b"),
- }, *tb1ReferredFKs[0])
- tb2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "tt2")
- require.Equal(t, 0, len(tb2ReferredFKs))
- require.Equal(t, 1, len(tb2Info.ForeignKeys))
- require.Equal(t, model.FKInfo{
- ID: 1,
- Name: model.NewCIStr("fk_b"),
- RefSchema: model.NewCIStr("test"),
- RefTable: model.NewCIStr("t1"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("b")},
- State: model.StatePublic,
- Version: 1,
- }, *tb2Info.ForeignKeys[0])
- // Auto create index for foreign key usage.
- require.Equal(t, 1, len(tb2Info.Indices))
- require.Equal(t, "fk_b", tb2Info.Indices[0].Name.L)
- require.Equal(t, "`test2`.`tt2`, CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `test`.`t1` (`id`)", tb2Info.ForeignKeys[0].String("test2", "tt2"))
-
- tk.MustExec("rename table test.t1 to test3.tt1")
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test3", "tt1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- require.Equal(t, 1, len(tb1ReferredFKs[0].Cols))
- // check the schema diff
- diff = getLatestSchemaDiff(t, tk)
- require.Equal(t, model.ActionRenameTable, diff.Type)
- require.Equal(t, 1, len(diff.AffectedOpts))
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test2"),
- ChildTable: model.NewCIStr("tt2"),
- ChildFKName: model.NewCIStr("fk_b"),
- }, *tb1ReferredFKs[0])
- tbl2Info := getTableInfo(t, dom, "test2", "tt2")
- tb2ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test2", "tt2")
- require.Equal(t, 0, len(tb2ReferredFKs))
- require.Equal(t, 1, len(tbl2Info.ForeignKeys))
- require.Equal(t, model.FKInfo{
- ID: 1,
- Name: model.NewCIStr("fk_b"),
- RefSchema: model.NewCIStr("test3"),
- RefTable: model.NewCIStr("tt1"),
- RefCols: []model.CIStr{model.NewCIStr("id")},
- Cols: []model.CIStr{model.NewCIStr("b")},
- State: model.StatePublic,
- Version: 1,
- }, *tbl2Info.ForeignKeys[0])
- tk.MustQuery("show create table test2.tt2").Check(testkit.Rows("tt2 CREATE TABLE `tt2` (\n" +
- " `id` int(11) NOT NULL,\n" +
- " `b` int(11) DEFAULT NULL,\n" +
- " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" +
- " KEY `fk_b` (`b`),\n" +
- " CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `test3`.`tt1` (`id`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
-}
-
-func TestCreateTableWithForeignKeyDML(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (id int key, a int);")
- tk.MustExec("begin")
- tk.MustExec("insert into t1 values (1, 1)")
- tk.MustExec("update t1 set a = 2 where id = 1")
-
- tk2 := testkit.NewTestKit(t, store)
- tk2.MustExec("use test")
- tk2.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references test.t1(id))")
-
- tk.MustExec("commit")
-}
-
-func TestCreateTableWithForeignKeyError(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("use test")
-
- cases := []struct {
- prepare []string
- refer string
- create string
- err string
- }{
- {
- refer: "create table t1 (id int, a int, b int);",
- create: "create table t2 (a int, b int, foreign key fk_b(b) references T_unknown(b));",
- err: "[schema:1824]Failed to open the referenced table 'T_unknown'",
- },
- {
- refer: "create table t1 (id int, a int, b int);",
- create: "create table t2 (a int, b int, foreign key fk_b(b) references t1(c_unknown));",
- err: "[schema:3734]Failed to add the foreign key constraint. Missing column 'c_unknown' for constraint 'fk_b' in the referenced table 't1'",
- },
- {
- refer: "create table t1 (id int key, a int, b int);",
- create: "create table t2 (a int, b int, foreign key fk(c_unknown) references t1(id));",
- err: "[ddl:1072]Key column 'c_unknown' doesn't exist in table",
- },
- {
- refer: "create table t1 (id int, a int, b int);",
- create: "create table t2 (a int, b int, foreign key fk_b(b) references t1(b));",
- err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
- },
- {
- refer: "create table t1 (id int, a int, b int not null, index(b));",
- create: "create table t2 (a int, b int not null, foreign key fk_b(b) references t1(b) on update set null);",
- err: "[schema:1830]Column 'b' cannot be NOT NULL: needed in a foreign key constraint 'fk_b' SET NULL",
- },
- {
- refer: "create table t1 (id int, a int, b int not null, index(b));",
- create: "create table t2 (a int, b int not null, foreign key fk_b(b) references t1(b) on delete set null);",
- err: "[schema:1830]Column 'b' cannot be NOT NULL: needed in a foreign key constraint 'fk_b' SET NULL",
- },
- {
- refer: "create table t1 (id int key, a int, b int as (a) virtual, index(b));",
- create: "create table t2 (a int, b int, foreign key fk_b(b) references t1(b));",
- err: "[schema:3733]Foreign key 'fk_b' uses virtual column 'b' which is not supported.",
- },
- {
- refer: "create table t1 (id int key, a int, b int, index(b));",
- create: "create table t2 (a int, b int as (a) virtual, foreign key fk_b(b) references t1(b));",
- err: "[schema:3733]Foreign key 'fk_b' uses virtual column 'b' which is not supported.",
- },
- {
- refer: "create table t1 (id int key, a int);",
- create: "create table t2 (a int, b varchar(10), foreign key fk(b) references t1(id));",
- err: "[ddl:3780]Referencing column 'b' and referenced column 'id' in foreign key constraint 'fk' are incompatible.",
- },
- {
- refer: "create table t1 (id int key, a int not null, index(a));",
- create: "create table t2 (a int, b int unsigned, foreign key fk_b(b) references t1(a));",
- err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
- },
- {
- refer: "create table t1 (id int key, a bigint, index(a));",
- create: "create table t2 (a int, b int, foreign key fk_b(b) references t1(a));",
- err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
- },
- {
- refer: "create table t1 (id int key, a varchar(10) charset utf8, index(a));",
- create: "create table t2 (a int, b varchar(10) charset utf8mb4, foreign key fk_b(b) references t1(a));",
- err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
- },
- {
- refer: "create table t1 (id int key, a varchar(10) collate utf8_bin, index(a));",
- create: "create table t2 (a int, b varchar(10) collate utf8mb4_bin, foreign key fk_b(b) references t1(a));",
- err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
- },
- {
- refer: "create table t1 (id int key, a varchar(10));",
- create: "create table t2 (a int, b varchar(10), foreign key fk_b(b) references t1(a));",
- err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
- },
- {
- refer: "create table t1 (id int key, a varchar(10), index (a(5)));",
- create: "create table t2 (a int, b varchar(10), foreign key fk_b(b) references t1(a));",
- err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
- },
- {
- refer: "create table t1 (id int key, a int, index(a));",
- create: "create table t2 (a int, b int, foreign key fk_b(b) references t1(id, a));",
- err: "[schema:1239]Incorrect foreign key definition for 'fk_b': Key reference and table reference don't match",
- },
- {
- create: "create table t2 (a int key, foreign key (a) references t2(a));",
- err: "[schema:1215]Cannot add foreign key constraint",
- },
- {
- create: "create table t2 (a int, b int, index(a,b), index(b,a), foreign key (a,b) references t2(a,b));",
- err: "[schema:1215]Cannot add foreign key constraint",
- },
- {
- create: "create table t2 (a int, b int, index(a,b), foreign key (a,b) references t2(b,a));",
- err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_1' in the referenced table 't2'",
- },
- {
- prepare: []string{
- "set @@foreign_key_checks=0;",
- "create table t2 (a int, b int, index(a), foreign key (a) references t1(id));",
- },
- create: "create table t1 (id int, a int);",
- err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_1' in the referenced table 't1'",
- },
- {
- prepare: []string{
- "set @@foreign_key_checks=0;",
- "create table t2 (a int, b int, index(a), foreign key (a) references t1(id));",
- },
- create: "create table t1 (id bigint key, a int);",
- err: "[ddl:3780]Referencing column 'a' and referenced column 'id' in foreign key constraint 'fk_1' are incompatible.",
- },
- {
- // foreign key is not support in temporary table.
- refer: "create temporary table t1 (id int key, b int, index(b))",
- create: "create table t2 (a int, b int, foreign key fk(b) references t1(b))",
- err: "[schema:1824]Failed to open the referenced table 't1'",
- },
- {
- // foreign key is not support in temporary table.
- refer: "create global temporary table t1 (id int key, b int, index(b)) on commit delete rows",
- create: "create table t2 (a int, b int, foreign key fk(b) references t1(b))",
- err: "[schema:1215]Cannot add foreign key constraint",
- },
- {
- // foreign key is not support in temporary table.
- refer: "create table t1 (id int key, b int, index(b))",
- create: "create temporary table t2 (a int, b int, foreign key fk(b) references t1(b))",
- err: "[schema:1215]Cannot add foreign key constraint",
- },
- {
- // foreign key is not support in temporary table.
- refer: "create table t1 (id int key, b int, index(b))",
- create: "create global temporary table t2 (a int, b int, foreign key fk(b) references t1(b)) on commit delete rows",
- err: "[schema:1215]Cannot add foreign key constraint",
- },
- {
- create: "create table t1 (a int, foreign key ``(a) references t1(a));",
- err: "[ddl:1280]Incorrect index name ''",
- },
- {
- create: "create table t1 (a int, constraint `` foreign key (a) references t1(a));",
- err: "[ddl:1280]Incorrect index name ''",
- },
- {
- create: "create table t1 (a int, constraint `fk` foreign key (a,a) references t1(a, b));",
- err: "[schema:1060]Duplicate column name 'a'",
- },
- {
- refer: "create table t1(a int, b int, index(a,b));",
- create: "create table t2 (a int, b int, foreign key (a,b) references t1(a,a));",
- err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_1' in the referenced table 't1'",
- },
- {
- refer: "create table t1 (id int key, b int, index(b))",
- create: "create table t2 (a int, b int, index fk_1(a), foreign key (b) references t1(b));",
- err: "[ddl:1061]duplicate key name fk_1",
- },
- {
- refer: "create table t1 (id int key);",
- create: "create table t2 (id int key, foreign key name5678901234567890123456789012345678901234567890123456789012345(id) references t1(id));",
- err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
- },
- {
- refer: "create table t1 (id int key);",
- create: "create table t2 (id int key, constraint name5678901234567890123456789012345678901234567890123456789012345 foreign key (id) references t1(id));",
- err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
- },
- {
- create: "create table t2 (id int key, constraint fk foreign key (id) references name5678901234567890123456789012345678901234567890123456789012345.t1(id));",
- err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
- },
- {
- prepare: []string{
- "set @@foreign_key_checks=0;",
- },
- create: "create table t2 (id int key, constraint fk foreign key (id) references name5678901234567890123456789012345678901234567890123456789012345(id));",
- err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
- },
- {
- prepare: []string{
- "set @@foreign_key_checks=0;",
- },
- create: "create table t2 (id int key, constraint fk foreign key (id) references t1(name5678901234567890123456789012345678901234567890123456789012345));",
- err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
- },
- }
- for _, ca := range cases {
- tk.MustExec("drop table if exists t2")
- tk.MustExec("drop table if exists t1")
- tk.MustExec("set @@foreign_key_checks=1")
- for _, sql := range ca.prepare {
- tk.MustExec(sql)
- }
- if ca.refer != "" {
- tk.MustExec(ca.refer)
- }
- err := tk.ExecToErr(ca.create)
- require.Error(t, err, ca.create)
- require.Equal(t, ca.err, err.Error(), ca.create)
- }
-
- passCases := [][]string{
- {
- "create table t1 (id int key, a int, b int, foreign key fk(a) references t1(id))",
- },
- {
- "create table t1 (id int key, b int not null, index(b))",
- "create table t2 (a int, b int, foreign key fk_b(b) references t1(b));",
- },
- {
- "create table t1 (id int key, a varchar(10), index(a));",
- "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
- },
- {
- "create table t1 (id int key, a decimal(10,5), index(a));",
- "create table t2 (a int, b decimal(20, 10), foreign key fk_b(b) references t1(a));",
- },
- {
- "create table t1 (id int key, a varchar(10), index (a(10)));",
- "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
- },
- {
- "set @@foreign_key_checks=0;",
- "create table t2 (a int, b int, foreign key fk_b(b) references t_unknown(b));",
- "set @@foreign_key_checks=1;",
- },
- {
- "create table t2 (a int, b int, index(a,b), index(b,a), foreign key (a,b) references t2(b,a));",
- },
- {
- "create table t1 (a int key, b int, index(b))",
- "create table t2 (a int, b int, foreign key (a) references t1(a), foreign key (b) references t1(b));",
- },
- {
- "create table t1 (id int key);",
- "create table t2 (id int key, foreign key name567890123456789012345678901234567890123456789012345678901234(id) references t1(id));",
- },
- }
- for _, ca := range passCases {
- tk.MustExec("drop table if exists t2")
- tk.MustExec("drop table if exists t1")
- for _, sql := range ca {
- tk.MustExec(sql)
- }
- }
-}
-
-func TestModifyColumnWithForeignKey(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("set @@foreign_key_checks=1;")
- tk.MustExec("use test")
-
- tk.MustExec("create table t1 (id int key, b varchar(10), index(b));")
- tk.MustExec("create table t2 (a varchar(10), constraint fk foreign key (a) references t1(b));")
- tk.MustExec("insert into t1 values (1, '123456789');")
- tk.MustExec("insert into t2 values ('123456789');")
- tk.MustGetErrMsg("alter table t1 modify column b varchar(5);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
- tk.MustGetErrMsg("alter table t1 modify column b bigint;", "[ddl:3780]Referencing column 'a' and referenced column 'b' in foreign key constraint 'fk' are incompatible.")
- tk.MustExec("alter table t1 modify column b varchar(20);")
- tk.MustGetErrMsg("alter table t1 modify column b varchar(10);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
- tk.MustExec("alter table t2 modify column a varchar(20);")
- tk.MustExec("alter table t2 modify column a varchar(21);")
- tk.MustGetErrMsg("alter table t2 modify column a varchar(5);", "[ddl:1832]Cannot change column 'a': used in a foreign key constraint 'fk'")
- tk.MustGetErrMsg("alter table t2 modify column a bigint;", "[ddl:3780]Referencing column 'a' and referenced column 'b' in foreign key constraint 'fk' are incompatible.")
-
- tk.MustExec("drop table t2")
- tk.MustExec("drop table t1")
- tk.MustExec("create table t1 (id int key, b decimal(10, 5), index(b));")
- tk.MustExec("create table t2 (a decimal(10, 5), constraint fk foreign key (a) references t1(b));")
- tk.MustExec("insert into t1 values (1, 12345.67891);")
- tk.MustExec("insert into t2 values (12345.67891);")
- tk.MustGetErrMsg("alter table t1 modify column b decimal(10, 6);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
- tk.MustGetErrMsg("alter table t1 modify column b decimal(10, 3);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
- tk.MustGetErrMsg("alter table t1 modify column b decimal(5, 2);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
- tk.MustGetErrMsg("alter table t1 modify column b decimal(20, 10);", "[ddl:1833]Cannot change column 'b': used in a foreign key constraint 'fk' of table 'test.t2'")
- tk.MustGetErrMsg("alter table t2 modify column a decimal(30, 15);", "[ddl:1832]Cannot change column 'a': used in a foreign key constraint 'fk'")
- tk.MustGetErrMsg("alter table t2 modify column a decimal(5, 2);", "[ddl:1832]Cannot change column 'a': used in a foreign key constraint 'fk'")
-}
-
-func TestDropChildTableForeignKeyMetaInfo(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (id int key, a int, b int, CONSTRAINT fk foreign key (a) references t1(id))")
- tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- tk.MustExec("drop table t1")
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 0, len(tb1ReferredFKs))
-
- tk.MustExec("create table t1 (id int key, b int, index(b))")
- tk.MustExec("create table t2 (a int, b int, foreign key fk (a) references t1(b));")
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- tk.MustExec("drop table t2")
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 0, len(tb1ReferredFKs))
-}
-
-func TestDropForeignKeyMetaInfo(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (id int key, a int, b int, CONSTRAINT fk foreign key (a) references t1(id))")
- tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- tk.MustExec("alter table t1 drop foreign key fk")
- tbl1Info := getTableInfo(t, dom, "test", "t1")
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 0, len(tbl1Info.ForeignKeys))
- require.Equal(t, 0, len(tb1ReferredFKs))
-
- tk.MustExec("drop table t1")
- tk.MustExec("create table t1 (id int key, b int, index(b))")
- tk.MustExec("create table t2 (a int, b int, foreign key fk (a) references t1(b));")
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- tk.MustExec("alter table t2 drop foreign key fk")
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 0, len(tb1ReferredFKs))
- tbl2Info := getTableInfo(t, dom, "test", "t2")
- require.Equal(t, 0, len(tbl2Info.ForeignKeys))
-}
-
-func TestTruncateOrDropTableWithForeignKeyReferred(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("use test")
-
- cases := []struct {
- prepares []string
- tbl string
- truncateErr string
- dropErr string
- }{
- {
- prepares: []string{
- "create table t1 (id int key, b int not null, index(b))",
- "create table t2 (a int, b int, foreign key fk_b(b) references t1(b));",
- },
- tbl: "t1",
- truncateErr: "[ddl:1701]Cannot truncate a table referenced in a foreign key constraint (`test`.`t2` CONSTRAINT `fk_b`)",
- dropErr: "[ddl:3730]Cannot drop table 't1' referenced by a foreign key constraint 'fk_b' on table 't2'.",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a varchar(10), index(a));",
- "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
- },
- tbl: "t1",
- truncateErr: "[ddl:1701]Cannot truncate a table referenced in a foreign key constraint (`test`.`t2` CONSTRAINT `fk_b`)",
- dropErr: "[ddl:3730]Cannot drop table 't1' referenced by a foreign key constraint 'fk_b' on table 't2'.",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a varchar(10), index (a(10)));",
- "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
- },
- tbl: "t1",
- truncateErr: "[ddl:1701]Cannot truncate a table referenced in a foreign key constraint (`test`.`t2` CONSTRAINT `fk_b`)",
- dropErr: "[ddl:3730]Cannot drop table 't1' referenced by a foreign key constraint 'fk_b' on table 't2'.",
- },
- }
-
- for _, ca := range cases {
- tk.MustExec("drop table if exists t2")
- tk.MustExec("drop table if exists t1")
- for _, sql := range ca.prepares {
- tk.MustExec(sql)
- }
- truncateSQL := fmt.Sprintf("truncate table %v", ca.tbl)
- tk.MustExec("set @@foreign_key_checks=1;")
- err := tk.ExecToErr(truncateSQL)
- require.Error(t, err)
- require.Equal(t, ca.truncateErr, err.Error())
- dropSQL := fmt.Sprintf("drop table %v", ca.tbl)
- err = tk.ExecToErr(dropSQL)
- require.Error(t, err)
- require.Equal(t, ca.dropErr, err.Error())
-
- tk.MustExec("set @@foreign_key_checks=0;")
- tk.MustExec(truncateSQL)
- }
- passCases := [][]string{
- {
- "create table t1 (id int key, a int, b int, foreign key fk(a) references t1(id))",
- "truncate table t1",
- "drop table t1",
- },
- {
- "create table t1 (id int key, a varchar(10), index (a(10)));",
- "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
- "drop table t1, t2",
- },
- {
- "set @@foreign_key_checks=0;",
- "create table t1 (id int key, a varchar(10), index (a(10)));",
- "create table t2 (a int, b varchar(20), foreign key fk_b(b) references t1(a));",
- "truncate table t1",
- "drop table t1",
- },
- }
- for _, ca := range passCases {
- tk.MustExec("drop table if exists t1, t2")
- tk.MustExec("set @@foreign_key_checks=1;")
- for _, sql := range ca {
- tk.MustExec(sql)
- }
- }
-}
-
func TestTruncateOrDropTableWithForeignKeyReferred2(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
d := dom.DDL()
@@ -1068,7 +225,7 @@ func TestTruncateOrDropTableWithForeignKeyReferred2(t *testing.T) {
var wg sync.WaitGroup
var truncateErr, dropErr error
testTruncate := true
- tc := &ddl.TestDDLCallback{}
+ tc := &callback.TestDDLCallback{}
tc.OnJobRunBeforeExported = func(job *model.Job) {
if job.SchemaState != model.StateNone {
return
@@ -1108,106 +265,6 @@ func TestTruncateOrDropTableWithForeignKeyReferred2(t *testing.T) {
require.Equal(t, "[ddl:1701]Cannot truncate a table referenced in a foreign key constraint (`test`.`t2` CONSTRAINT `fk`)", dropErr.Error())
}
-func TestDropTableWithForeignKeyReferred(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("set @@foreign_key_checks=1;")
- tk.MustExec("use test")
-
- tk.MustExec("create table t1 (id int key, b int, index(b));")
- tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references t1(id));")
- tk.MustExec("create table t3 (id int key, b int, foreign key fk_b(b) references t2(id));")
- err := tk.ExecToErr("drop table if exists t1,t2;")
- require.Error(t, err)
- require.Equal(t, "[ddl:3730]Cannot drop table 't2' referenced by a foreign key constraint 'fk_b' on table 't3'.", err.Error())
- tk.MustQuery("show tables").Check(testkit.Rows("t1", "t2", "t3"))
-}
-
-func TestDropIndexNeededInForeignKey(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("set @@foreign_key_checks=1")
- tk.MustExec("use test")
-
- cases := []struct {
- prepares []string
- drops []string
- err string
- }{
- {
- prepares: []string{
- "create table t1 (id int key, b int, index idx (b))",
- "create table t2 (a int, b int, index idx (b), foreign key fk_b(b) references t1(b));",
- },
- drops: []string{
- "alter table t1 drop index idx",
- "alter table t2 drop index idx",
- },
- err: "[ddl:1553]Cannot drop index 'idx': needed in a foreign key constraint",
- },
- {
- prepares: []string{
- "create table t1 (id int, b int, index idx (id, b))",
- "create table t2 (a int, b int, index idx (b, a), foreign key fk_b(b) references t1(id));",
- },
- drops: []string{
- "alter table t1 drop index idx",
- "alter table t2 drop index idx",
- },
- err: "[ddl:1553]Cannot drop index 'idx': needed in a foreign key constraint",
- },
- }
-
- for _, ca := range cases {
- tk.MustExec("drop table if exists t2")
- tk.MustExec("drop table if exists t1")
- for _, sql := range ca.prepares {
- tk.MustExec(sql)
- }
- for _, drop := range ca.drops {
- // even disable foreign key check, still can't drop the index used by foreign key.
- tk.MustExec("set @@foreign_key_checks=0;")
- err := tk.ExecToErr(drop)
- require.Error(t, err)
- require.Equal(t, ca.err, err.Error())
- tk.MustExec("set @@foreign_key_checks=1;")
- err = tk.ExecToErr(drop)
- require.Error(t, err)
- require.Equal(t, ca.err, err.Error())
- }
- }
- passCases := [][]string{
- {
- "create table t1 (id int key, b int, index idxb (b))",
- "create table t2 (a int, b int key, index idxa (a),index idxb (b), foreign key fk_b(b) references t1(id));",
- "alter table t1 drop index idxb",
- "alter table t2 drop index idxa",
- "alter table t2 drop index idxb",
- },
- {
- "create table t1 (id int key, b int, index idxb (b), unique index idx(b, id))",
- "create table t2 (a int, b int key, index idx (b, a),index idxb (b), index idxab(a, b), foreign key fk_b(b) references t1(b));",
- "alter table t1 drop index idxb",
- "alter table t1 add index idxb (b)",
- "alter table t1 drop index idx",
- "alter table t2 drop index idx",
- "alter table t2 add index idx (b, a)",
- "alter table t2 drop index idxb",
- "alter table t2 drop index idxab",
- },
- }
- tk.MustExec("set @@foreign_key_checks=1;")
- for _, ca := range passCases {
- tk.MustExec("drop table if exists t2")
- tk.MustExec("drop table if exists t1")
- for _, sql := range ca {
- tk.MustExec(sql)
- }
- }
-}
-
func TestDropIndexNeededInForeignKey2(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
d := dom.DDL()
@@ -1224,7 +281,7 @@ func TestDropIndexNeededInForeignKey2(t *testing.T) {
var wg sync.WaitGroup
var dropErr error
- tc := &ddl.TestDDLCallback{}
+ tc := &callback.TestDDLCallback{}
tc.OnJobRunBeforeExported = func(job *model.Job) {
if job.SchemaState != model.StatePublic || job.Type != model.ActionDropIndex {
return
@@ -1247,157 +304,6 @@ func TestDropIndexNeededInForeignKey2(t *testing.T) {
require.Equal(t, "[ddl:1553]Cannot drop index 'idx2': needed in a foreign key constraint", dropErr.Error())
}
-func getTableInfo(t *testing.T, dom *domain.Domain, db, tb string) *model.TableInfo {
- err := dom.Reload()
- require.NoError(t, err)
- is := dom.InfoSchema()
- tbl, err := is.TableByName(model.NewCIStr(db), model.NewCIStr(tb))
- require.NoError(t, err)
- _, exist := is.TableByID(tbl.Meta().ID)
- require.True(t, exist)
- return tbl.Meta()
-}
-
-func getTableInfoReferredForeignKeys(t *testing.T, dom *domain.Domain, db, tb string) []*model.ReferredFKInfo {
- err := dom.Reload()
- require.NoError(t, err)
- return dom.InfoSchema().GetTableReferredForeignKeys(db, tb)
-}
-
-func TestDropColumnWithForeignKey(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("set @@foreign_key_checks=1;")
- tk.MustExec("use test")
-
- tk.MustExec("create table t1 (id int key, a int, b int, index(b), CONSTRAINT fk foreign key (a) references t1(b))")
- tk.MustGetErrMsg("alter table t1 drop column a;", "[ddl:1828]Cannot drop column 'a': needed in a foreign key constraint 'fk'")
- tk.MustGetErrMsg("alter table t1 drop column b;", "[ddl:1829]Cannot drop column 'b': needed in a foreign key constraint 'fk' of table 't1'")
-
- tk.MustExec("drop table t1")
- tk.MustExec("create table t1 (id int key, b int, index(b));")
- tk.MustExec("create table t2 (a int, b int, constraint fk foreign key (a) references t1(b));")
- tk.MustGetErrMsg("alter table t1 drop column b;", "[ddl:1829]Cannot drop column 'b': needed in a foreign key constraint 'fk' of table 't2'")
- tk.MustGetErrMsg("alter table t2 drop column a;", "[ddl:1828]Cannot drop column 'a': needed in a foreign key constraint 'fk'")
-}
-
-func TestRenameColumnWithForeignKeyMetaInfo(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("set @@foreign_key_checks=1;")
- tk.MustExec("use test")
-
- tk.MustExec("create table t1 (id int key, a int, b int, foreign key fk(a) references t1(id))")
- tk.MustExec("alter table t1 change id kid int")
- tk.MustExec("alter table t1 rename column a to aa")
- tbl1Info := getTableInfo(t, dom, "test", "t1")
- tb1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tbl1Info.ForeignKeys))
- require.Equal(t, 1, len(tb1ReferredFKs))
- require.Equal(t, "kid", tb1ReferredFKs[0].Cols[0].L)
- require.Equal(t, "kid", tbl1Info.ForeignKeys[0].RefCols[0].L)
- require.Equal(t, "aa", tbl1Info.ForeignKeys[0].Cols[0].L)
-
- tk.MustExec("drop table t1")
- tk.MustExec("create table t1 (id int key, b int, index(b))")
- tk.MustExec("create table t2 (a int, b int, foreign key fk(a) references t1(b));")
- tk.MustExec("alter table t2 change a aa int")
- tbl1Info = getTableInfo(t, dom, "test", "t1")
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- require.Equal(t, 1, len(tb1ReferredFKs[0].Cols))
- require.Equal(t, "b", tb1ReferredFKs[0].Cols[0].L)
- tbl2Info := getTableInfo(t, dom, "test", "t2")
- tb2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t2")
- require.Equal(t, 0, len(tb2ReferredFKs))
- require.Equal(t, 1, len(tbl2Info.ForeignKeys))
- require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].Cols))
- require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].RefCols))
- require.Equal(t, "aa", tbl2Info.ForeignKeys[0].Cols[0].L)
- require.Equal(t, "b", tbl2Info.ForeignKeys[0].RefCols[0].L)
-
- tk.MustExec("alter table t1 change id kid int")
- tk.MustExec("alter table t1 change b bb int")
- tbl1Info = getTableInfo(t, dom, "test", "t1")
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 1, len(tb1ReferredFKs))
- require.Equal(t, 1, len(tb1ReferredFKs[0].Cols))
- require.Equal(t, "bb", tb1ReferredFKs[0].Cols[0].L)
- tbl2Info = getTableInfo(t, dom, "test", "t2")
- tb2ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t2")
- require.Equal(t, 0, len(tb2ReferredFKs))
- require.Equal(t, 1, len(tbl2Info.ForeignKeys))
- require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].Cols))
- require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].RefCols))
- require.Equal(t, "aa", tbl2Info.ForeignKeys[0].Cols[0].L)
- require.Equal(t, "bb", tbl2Info.ForeignKeys[0].RefCols[0].L)
-
- tk.MustExec("drop table t1, t2")
- tk.MustExec("create table t1 (id int key, b int, index(b))")
- tk.MustExec("create table t2 (a int, b int, foreign key (a) references t1(b), foreign key (b) references t1(b));")
- tk.MustExec("alter table t1 change b bb int")
- tbl1Info = getTableInfo(t, dom, "test", "t1")
- tb1ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- require.Equal(t, 2, len(tb1ReferredFKs))
- require.Equal(t, 1, len(tb1ReferredFKs[0].Cols))
- require.Equal(t, 1, len(tb1ReferredFKs[1].Cols))
- require.Equal(t, "bb", tb1ReferredFKs[0].Cols[0].L)
- require.Equal(t, "bb", tb1ReferredFKs[1].Cols[0].L)
- tbl2Info = getTableInfo(t, dom, "test", "t2")
- tb2ReferredFKs = getTableInfoReferredForeignKeys(t, dom, "test", "t2")
- require.Equal(t, 0, len(tb2ReferredFKs))
- require.Equal(t, 2, len(tbl2Info.ForeignKeys))
- require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].Cols))
- require.Equal(t, 1, len(tbl2Info.ForeignKeys[0].RefCols))
- require.Equal(t, "a", tbl2Info.ForeignKeys[0].Cols[0].L)
- require.Equal(t, "bb", tbl2Info.ForeignKeys[0].RefCols[0].L)
- require.Equal(t, 1, len(tbl2Info.ForeignKeys[1].Cols))
- require.Equal(t, 1, len(tbl2Info.ForeignKeys[1].RefCols))
- require.Equal(t, "b", tbl2Info.ForeignKeys[1].Cols[0].L)
- require.Equal(t, "bb", tbl2Info.ForeignKeys[1].RefCols[0].L)
- tk.MustExec("alter table t2 rename column a to aa")
- tk.MustExec("alter table t2 change b bb int")
- tk.MustQuery("show create table t2").
- Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" +
- " `aa` int(11) DEFAULT NULL,\n" +
- " `bb` int(11) DEFAULT NULL,\n" +
- " KEY `fk_1` (`aa`),\n KEY `fk_2` (`bb`),\n" +
- " CONSTRAINT `fk_1` FOREIGN KEY (`aa`) REFERENCES `test`.`t1` (`bb`),\n" +
- " CONSTRAINT `fk_2` FOREIGN KEY (`bb`) REFERENCES `test`.`t1` (`bb`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
-}
-
-func TestDropDatabaseWithForeignKeyReferred(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("set @@foreign_key_checks=1;")
- tk.MustExec("use test")
-
- tk.MustExec("create table t1 (id int key, b int, index(b));")
- tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references t1(id));")
- tk.MustExec("create database test2")
- tk.MustExec("create table test2.t3 (id int key, b int, foreign key fk_b(b) references test.t2(id));")
- err := tk.ExecToErr("drop database test;")
- require.Error(t, err)
- require.Equal(t, "[ddl:3730]Cannot drop table 't2' referenced by a foreign key constraint 'fk_b' on table 't3'.", err.Error())
- tk.MustExec("set @@foreign_key_checks=0;")
- tk.MustExec("drop database test")
-
- tk.MustExec("set @@foreign_key_checks=1;")
- tk.MustExec("create database test")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (id int key, b int, index(b));")
- tk.MustExec("create table t2 (id int key, b int, foreign key fk_b(b) references t1(id));")
- err = tk.ExecToErr("drop database test;")
- require.Error(t, err)
- require.Equal(t, "[ddl:3730]Cannot drop table 't2' referenced by a foreign key constraint 'fk_b' on table 't3'.", err.Error())
- tk.MustExec("drop table test2.t3")
- tk.MustExec("drop database test")
-}
-
func TestDropDatabaseWithForeignKeyReferred2(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
d := dom.DDL()
@@ -1414,7 +320,7 @@ func TestDropDatabaseWithForeignKeyReferred2(t *testing.T) {
tk.MustExec("create database test2")
var wg sync.WaitGroup
var dropErr error
- tc := &ddl.TestDDLCallback{}
+ tc := &callback.TestDDLCallback{}
tc.OnJobRunBeforeExported = func(job *model.Job) {
if job.SchemaState != model.StateNone {
return
@@ -1442,79 +348,6 @@ func TestDropDatabaseWithForeignKeyReferred2(t *testing.T) {
tk.MustExec("drop database test")
}
-func TestAddForeignKey(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("set @@foreign_key_checks=1;")
- tk.MustExec("use test")
- tk.MustExec("create table t1 (id int key, b int);")
- tk.MustExec("create table t2 (id int key, b int);")
- tk.MustExec("alter table t2 add index(b)")
- tk.MustExec("alter table t2 add foreign key (b) references t1(id);")
- tbl2Info := getTableInfo(t, dom, "test", "t2")
- require.Equal(t, int64(1), tbl2Info.MaxForeignKeyID)
- tk.MustGetDBError("alter table t2 add foreign key (b) references t1(b);", infoschema.ErrForeignKeyNoIndexInParent)
- tk.MustExec("alter table t1 add index(b)")
- tk.MustExec("alter table t2 add foreign key (b) references t1(b);")
- tk.MustGetDBError("alter table t2 add foreign key (b) references t2(b);", infoschema.ErrCannotAddForeign)
- // Test auto-create index when create foreign key constraint.
- tk.MustExec("drop table if exists t1,t2")
- tk.MustExec("create table t1 (id int key, b int, index(b));")
- tk.MustExec("create table t2 (id int key, b int);")
- tk.MustExec("alter table t2 add constraint fk foreign key (b) references t1(b);")
- tbl2Info = getTableInfo(t, dom, "test", "t2")
- require.Equal(t, 1, len(tbl2Info.Indices))
- require.Equal(t, "fk", tbl2Info.Indices[0].Name.L)
- require.Equal(t, model.StatePublic, tbl2Info.Indices[0].State)
- tk.MustQuery("select b from t2 use index(fk)").Check(testkit.Rows())
- res := tk.MustQuery("explain select b from t2 use index(fk)")
- plan := bytes.NewBuffer(nil)
- rows := res.Rows()
- for _, row := range rows {
- for _, c := range row {
- plan.WriteString(c.(string))
- plan.WriteString(" ")
- }
- }
- require.Regexp(t, ".*IndexReader.*index:fk.*", plan.String())
-
- // Test add multiple foreign key constraint in one statement.
- tk.MustExec("alter table t2 add column c int, add column d int, add column e int;")
- tk.MustExec("alter table t2 add index idx_c(c, d, e)")
- tk.MustExec("alter table t2 add constraint fk_c foreign key (c) references t1(b), " +
- "add constraint fk_d foreign key (d) references t1(b)," +
- "add constraint fk_e foreign key (e) references t1(b)")
- tbl2Info = getTableInfo(t, dom, "test", "t2")
- require.Equal(t, 4, len(tbl2Info.Indices))
- names := []string{"fk", "idx_c", "fk_d", "fk_e"}
- for i, idx := range tbl2Info.Indices {
- require.Equal(t, names[i], idx.Name.L)
- require.Equal(t, model.StatePublic, idx.State)
- }
- names = []string{"fk", "fk_c", "fk_d", "fk_e"}
- for i, fkInfo := range tbl2Info.ForeignKeys {
- require.Equal(t, names[i], fkInfo.Name.L)
- require.Equal(t, model.StatePublic, fkInfo.State)
- }
- tk.MustGetDBError("insert into t2 (id, b) values (1,1)", plannercore.ErrNoReferencedRow2)
- tk.MustGetDBError("insert into t2 (id, c) values (1,1)", plannercore.ErrNoReferencedRow2)
- tk.MustGetDBError("insert into t2 (id, d) values (1,1)", plannercore.ErrNoReferencedRow2)
- tk.MustGetDBError("insert into t2 (id, e) values (1,1)", plannercore.ErrNoReferencedRow2)
-
- // Test add multiple foreign key constraint in one statement but failed.
- tk.MustExec("alter table t2 drop foreign key fk")
- tk.MustExec("alter table t2 drop foreign key fk_c")
- tk.MustExec("alter table t2 drop foreign key fk_d")
- tk.MustExec("alter table t2 drop foreign key fk_e")
- tk.MustGetDBError("alter table t2 add constraint fk_c foreign key (c) references t1(b), "+
- "add constraint fk_d foreign key (d) references t1(b),"+
- "add constraint fk_e foreign key (e) references t1(unknown_col)", infoschema.ErrForeignKeyNoColumnInParent)
- tbl2Info = getTableInfo(t, dom, "test", "t2")
- require.Equal(t, 0, len(tbl2Info.ForeignKeys))
- tk.MustGetDBError("alter table t2 drop index idx_c, add constraint fk_c foreign key (c) references t1(b)", dbterror.ErrDropIndexNeededInForeignKey)
-}
-
func TestAddForeignKey2(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
d := dom.DDL()
@@ -1528,7 +361,7 @@ func TestAddForeignKey2(t *testing.T) {
tk.MustExec("create table t2 (id int key, b int, index(b));")
var wg sync.WaitGroup
var addErr error
- tc := &ddl.TestDDLCallback{}
+ tc := &callback.TestDDLCallback{}
tc.OnJobRunBeforeExported = func(job *model.Job) {
if job.SchemaState != model.StatePublic || job.Type != model.ActionDropIndex {
return
@@ -1568,7 +401,7 @@ func TestAddForeignKey3(t *testing.T) {
var insertErrs []error
var deleteErrs []error
- tc := &ddl.TestDDLCallback{}
+ tc := &callback.TestDDLCallback{}
tc.OnJobRunBeforeExported = func(job *model.Job) {
if job.Type != model.ActionAddForeignKey {
return
@@ -1597,309 +430,3 @@ func TestAddForeignKey3(t *testing.T) {
tk.MustQuery("select * from t1 order by id").Check(testkit.Rows("1 1", "2 2", "3 3"))
tk.MustQuery("select * from t2 order by id").Check(testkit.Rows("1 1", "2 2", "3 3"))
}
-
-func TestAlterTableAddForeignKeyError(t *testing.T) {
- store, _ := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("set @@foreign_key_checks=1;")
- tk.MustExec("use test")
- cases := []struct {
- prepares []string
- alter string
- err string
- }{
- {
- prepares: []string{
- "create table t1 (id int, a int, b int);",
- "create table t2 (a int, b int);",
- },
- alter: "alter table t2 add foreign key fk(b) references t_unknown(id)",
- err: "[schema:1824]Failed to open the referenced table 't_unknown'",
- },
- {
- prepares: []string{
- "create table t1 (id int, a int, b int);",
- "create table t2 (a int, b int);",
- },
- alter: "alter table t2 add foreign key fk(b) references t1(c_unknown)",
- err: "[schema:3734]Failed to add the foreign key constraint. Missing column 'c_unknown' for constraint 'fk' in the referenced table 't1'",
- },
- {
- prepares: []string{
- "create table t1 (id int, a int, b int);",
- "create table t2 (a int, b int);",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(b)",
- err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
- },
- {
- prepares: []string{
- "create table t1 (id int, a int, b int not null, index(b));",
- "create table t2 (a int, b int not null);",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(b) on update set null",
- err: "[schema:1830]Column 'b' cannot be NOT NULL: needed in a foreign key constraint 'fk_b' SET NULL",
- },
- {
- prepares: []string{
- "create table t1 (id int, a int, b int not null, index(b));",
- "create table t2 (a int, b int not null);",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(b) on delete set null",
- err: "[schema:1830]Column 'b' cannot be NOT NULL: needed in a foreign key constraint 'fk_b' SET NULL",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a int, b int as (a) virtual, index(b));",
- "create table t2 (a int, b int);",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(b)",
- err: "[schema:3733]Foreign key 'fk_b' uses virtual column 'b' which is not supported.",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a int, b int, index(b));",
- "create table t2 (a int, b int as (a) virtual);",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(b)",
- err: "[schema:3733]Foreign key 'fk_b' uses virtual column 'b' which is not supported.",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a int);",
- "create table t2 (a int, b varchar(10));",
- },
- alter: "alter table t2 add foreign key fk(b) references t1(id)",
- err: "[ddl:3780]Referencing column 'b' and referenced column 'id' in foreign key constraint 'fk' are incompatible.",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a int not null, index(a));",
- "create table t2 (a int, b int unsigned);",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
- err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a bigint, index(a));",
- "create table t2 (a int, b int);",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
- err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a varchar(10) charset utf8, index(a));",
- "create table t2 (a int, b varchar(10) charset utf8mb4);",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
- err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a varchar(10) collate utf8_bin, index(a));",
- "create table t2 (a int, b varchar(10) collate utf8mb4_bin);",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
- err: "[ddl:3780]Referencing column 'b' and referenced column 'a' in foreign key constraint 'fk_b' are incompatible.",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a varchar(10));",
- "create table t2 (a int, b varchar(10));",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
- err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a varchar(10), index (a(5)));",
- "create table t2 (a int, b varchar(10));",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(a)",
- err: "[schema:1822]Failed to add the foreign key constraint. Missing index for constraint 'fk_b' in the referenced table 't1'",
- },
- {
- prepares: []string{
- "create table t1 (id int key, a int)",
- "create table t2 (id int, b int, index(b))",
- "insert into t2 values (1,1)",
- },
- alter: "alter table t2 add foreign key fk_b(b) references t1(id)",
- err: "[ddl:1452]Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `fk_b` FOREIGN KEY (`b`) REFERENCES `t1` (`id`))",
- },
- {
- prepares: []string{
- "create table t1 (id int, a int, b int, index(a,b))",
- "create table t2 (id int, a int, b int, index(a,b))",
- "insert into t2 values (1, 1, null), (2, null, 1), (3, null, null), (4, 1, 1)",
- },
- alter: "alter table t2 add foreign key fk_b(a, b) references t1(a, b)",
- err: "[ddl:1452]Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `fk_b` FOREIGN KEY (`a`, `b`) REFERENCES `t1` (`a`, `b`))",
- },
- {
- prepares: []string{
- "create table t1 (id int key);",
- "create table t2 (a int, b int unique);",
- },
- alter: "alter table t2 add foreign key name5678901234567890123456789012345678901234567890123456789012345(b) references t1(id)",
- err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
- },
- {
- prepares: []string{
- "create table t1 (id int key);",
- "create table t2 (a int, b int unique);",
- },
- alter: "alter table t2 add constraint name5678901234567890123456789012345678901234567890123456789012345 foreign key (b) references t1(id)",
- err: "[ddl:1059]Identifier name 'name5678901234567890123456789012345678901234567890123456789012345' is too long",
- },
- }
- for i, ca := range cases {
- tk.MustExec("drop table if exists t2")
- tk.MustExec("drop table if exists t1")
- for _, sql := range ca.prepares {
- tk.MustExec(sql)
- }
- err := tk.ExecToErr(ca.alter)
- require.Error(t, err, fmt.Sprintf("%v, %v", i, ca.err))
- require.Equal(t, ca.err, err.Error())
- }
-
- passCases := [][]string{
- {
- "create table t1 (id int key, a int, b int, index(a))",
- "alter table t1 add foreign key fk(a) references t1(id)",
- },
- {
- "create table t1 (id int key, b int not null, index(b))",
- "create table t2 (a int, b int, index(b));",
- "alter table t2 add foreign key fk_b(b) references t1(b)",
- },
- {
- "create table t1 (id int key, a varchar(10), index(a));",
- "create table t2 (a int, b varchar(20), index(b));",
- "alter table t2 add foreign key fk_b(b) references t1(a)",
- },
- {
- "create table t1 (id int key, a decimal(10,5), index(a));",
- "create table t2 (a int, b decimal(20, 10), index(b));",
- "alter table t2 add foreign key fk_b(b) references t1(a)",
- },
- {
- "create table t1 (id int key, a varchar(10), index (a(10)));",
- "create table t2 (a int, b varchar(20), index(b));",
- "alter table t2 add foreign key fk_b(b) references t1(a)",
- },
- {
- "create table t1 (id int key, a int)",
- "create table t2 (id int, b int, index(b))",
- "insert into t2 values (1, null)",
- "alter table t2 add foreign key fk_b(b) references t1(id)",
- },
- {
- "create table t1 (id int, a int, b int, index(a,b))",
- "create table t2 (id int, a int, b int, index(a,b))",
- "insert into t2 values (1, 1, null), (2, null, 1), (3, null, null)",
- "alter table t2 add foreign key fk_b(a, b) references t1(a, b)",
- },
- {
- "set @@foreign_key_checks=0;",
- "create table t1 (id int, a int, b int, index(a,b))",
- "create table t2 (id int, a int, b int, index(a,b))",
- "insert into t2 values (1, 1, 1)",
- "alter table t2 add foreign key fk_b(a, b) references t1(a, b)",
- "set @@foreign_key_checks=1;",
- },
- {
- "set @@foreign_key_checks=0;",
- "create table t2 (a int, b int, index(b));",
- "alter table t2 add foreign key fk_b(b) references t_unknown(a)",
- "set @@foreign_key_checks=1;",
- },
- {
- "create table t1 (id int key);",
- "create table t2 (a int, b int unique);",
- "alter table t2 add foreign key name567890123456789012345678901234567890123456789012345678901234(b) references t1(id)",
- },
- }
- for _, ca := range passCases {
- tk.MustExec("drop table if exists t2")
- tk.MustExec("drop table if exists t1")
- for _, sql := range ca {
- tk.MustExec(sql)
- }
- }
-}
-
-func TestRenameTablesWithForeignKey(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@global.tidb_enable_foreign_key=1")
- tk.MustExec("set @@foreign_key_checks=0;")
- tk.MustExec("create database test1")
- tk.MustExec("create database test2")
- tk.MustExec("use test")
- tk.MustExec("create table t0 (id int key, b int);")
- tk.MustExec("create table t1 (id int key, b int, index(b), foreign key fk(b) references t2(id));")
- tk.MustExec("create table t2 (id int key, b int, index(b), foreign key fk(b) references t1(id));")
- tk.MustExec("rename table test.t1 to test1.tt1, test.t2 to test2.tt2, test.t0 to test.tt0")
-
- // check the schema diff
- diff := getLatestSchemaDiff(t, tk)
- require.Equal(t, model.ActionRenameTables, diff.Type)
- require.Equal(t, 3, len(diff.AffectedOpts))
-
- // check referred foreign key information.
- t1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t1")
- t2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test", "t2")
- require.Equal(t, 0, len(t1ReferredFKs))
- require.Equal(t, 0, len(t2ReferredFKs))
- tt1ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test1", "tt1")
- tt2ReferredFKs := getTableInfoReferredForeignKeys(t, dom, "test2", "tt2")
- require.Equal(t, 1, len(tt1ReferredFKs))
- require.Equal(t, 1, len(tt2ReferredFKs))
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test2"),
- ChildTable: model.NewCIStr("tt2"),
- ChildFKName: model.NewCIStr("fk"),
- }, *tt1ReferredFKs[0])
- require.Equal(t, model.ReferredFKInfo{
- Cols: []model.CIStr{model.NewCIStr("id")},
- ChildSchema: model.NewCIStr("test1"),
- ChildTable: model.NewCIStr("tt1"),
- ChildFKName: model.NewCIStr("fk"),
- }, *tt2ReferredFKs[0])
-
- // check show create table information
- tk.MustQuery("show create table test1.tt1").Check(testkit.Rows("tt1 CREATE TABLE `tt1` (\n" +
- " `id` int(11) NOT NULL,\n" +
- " `b` int(11) DEFAULT NULL,\n" +
- " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" +
- " KEY `b` (`b`),\n" +
- " CONSTRAINT `fk` FOREIGN KEY (`b`) REFERENCES `test2`.`tt2` (`id`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
- tk.MustQuery("show create table test2.tt2").Check(testkit.Rows("tt2 CREATE TABLE `tt2` (\n" +
- " `id` int(11) NOT NULL,\n" +
- " `b` int(11) DEFAULT NULL,\n" +
- " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" +
- " KEY `b` (`b`),\n" +
- " CONSTRAINT `fk` FOREIGN KEY (`b`) REFERENCES `test1`.`tt1` (`id`)\n" +
- ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
-}
-
-func getLatestSchemaDiff(t *testing.T, tk *testkit.TestKit) *model.SchemaDiff {
- ctx := tk.Session()
- err := sessiontxn.NewTxn(context.Background(), ctx)
- require.NoError(t, err)
- txn, err := ctx.Txn(true)
- require.NoError(t, err)
- m := meta.NewMeta(txn)
- ver, err := m.GetSchemaVersion()
- require.NoError(t, err)
- diff, err := m.GetSchemaDiff(ver)
- require.NoError(t, err)
- return diff
-}
diff --git a/ddl/generated_column.go b/ddl/generated_column.go
index 2f4ceee8b60a9..0e877a3e63eb3 100644
--- a/ddl/generated_column.go
+++ b/ddl/generated_column.go
@@ -122,13 +122,19 @@ func findPositionRelativeColumn(cols []*table.Column, pos *ast.ColumnPosition) (
// findDependedColumnNames returns a set of string, which indicates
// the names of the columns that are depended by colDef.
-func findDependedColumnNames(colDef *ast.ColumnDef) (generated bool, colsMap map[string]struct{}) {
+func findDependedColumnNames(schemaName model.CIStr, tableName model.CIStr, colDef *ast.ColumnDef) (generated bool, colsMap map[string]struct{}, err error) {
colsMap = make(map[string]struct{})
for _, option := range colDef.Options {
if option.Tp == ast.ColumnOptionGenerated {
generated = true
colNames := FindColumnNamesInExpr(option.Expr)
for _, depCol := range colNames {
+ if depCol.Schema.L != "" && schemaName.L != "" && depCol.Schema.L != schemaName.L {
+ return false, nil, dbterror.ErrWrongDBName.GenWithStackByArgs(depCol.Schema.O)
+ }
+ if depCol.Table.L != "" && tableName.L != "" && depCol.Table.L != tableName.L {
+ return false, nil, dbterror.ErrWrongTableName.GenWithStackByArgs(depCol.Table.O)
+ }
colsMap[depCol.Name.L] = struct{}{}
}
break
@@ -192,7 +198,7 @@ func (c *generatedColumnChecker) Leave(inNode ast.Node) (node ast.Node, ok bool)
// 3. check if the modified expr contains non-deterministic functions
// 4. check whether new column refers to any auto-increment columns.
// 5. check if the new column is indexed or stored
-func checkModifyGeneratedColumn(sctx sessionctx.Context, tbl table.Table, oldCol, newCol *table.Column, newColDef *ast.ColumnDef, pos *ast.ColumnPosition) error {
+func checkModifyGeneratedColumn(sctx sessionctx.Context, schemaName model.CIStr, tbl table.Table, oldCol, newCol *table.Column, newColDef *ast.ColumnDef, pos *ast.ColumnPosition) error {
// rule 1.
oldColIsStored := !oldCol.IsGenerated() || oldCol.GeneratedStored
newColIsStored := !newCol.IsGenerated() || newCol.GeneratedStored
@@ -252,7 +258,10 @@ func checkModifyGeneratedColumn(sctx sessionctx.Context, tbl table.Table, oldCol
}
// rule 4.
- _, dependColNames := findDependedColumnNames(newColDef)
+ _, dependColNames, err := findDependedColumnNames(schemaName, tbl.Meta().Name, newColDef)
+ if err != nil {
+ return errors.Trace(err)
+ }
if !sctx.GetSessionVars().EnableAutoIncrementInGenerated {
if err := checkAutoIncrementRef(newColDef.Name.Name.L, dependColNames, tbl.Meta()); err != nil {
return errors.Trace(err)
@@ -268,12 +277,14 @@ func checkModifyGeneratedColumn(sctx sessionctx.Context, tbl table.Table, oldCol
}
type illegalFunctionChecker struct {
- hasIllegalFunc bool
- hasAggFunc bool
- hasRowVal bool // hasRowVal checks whether the functional index refers to a row value
- hasWindowFunc bool
- hasNotGAFunc4ExprIdx bool
- otherErr error
+ hasIllegalFunc bool
+ hasAggFunc bool
+ hasRowVal bool // hasRowVal checks whether the functional index refers to a row value
+ hasWindowFunc bool
+ hasNotGAFunc4ExprIdx bool
+ hasCastArrayFunc bool
+ disallowCastArrayFunc bool
+ otherErr error
}
func (c *illegalFunctionChecker) Enter(inNode ast.Node) (outNode ast.Node, skipChildren bool) {
@@ -308,7 +319,16 @@ func (c *illegalFunctionChecker) Enter(inNode ast.Node) (outNode ast.Node, skipC
case *ast.WindowFuncExpr:
c.hasWindowFunc = true
return inNode, true
+ case *ast.FuncCastExpr:
+ c.hasCastArrayFunc = c.hasCastArrayFunc || node.Tp.IsArray()
+ if c.disallowCastArrayFunc && node.Tp.IsArray() {
+ c.otherErr = expression.ErrNotSupportedYet.GenWithStackByArgs("Use of CAST( .. AS .. ARRAY) outside of functional index in CREATE(non-SELECT)/ALTER TABLE or in general expressions")
+ return inNode, true
+ }
+ case *ast.ParenthesesExpr:
+ return inNode, false
}
+ c.disallowCastArrayFunc = true
return inNode, false
}
@@ -355,6 +375,9 @@ func checkIllegalFn4Generated(name string, genType int, expr ast.ExprNode) error
if genType == typeIndex && c.hasNotGAFunc4ExprIdx && !config.GetGlobalConfig().Experimental.AllowsExpressionIndex {
return dbterror.ErrUnsupportedExpressionIndex
}
+ if genType == typeColumn && c.hasCastArrayFunc {
+ return expression.ErrNotSupportedYet.GenWithStackByArgs("Use of CAST( .. AS .. ARRAY) outside of functional index in CREATE(non-SELECT)/ALTER TABLE or in general expressions")
+ }
return nil
}
diff --git a/ddl/index.go b/ddl/index.go
index 69622ec3b5bef..f7430cc864cdf 100644
--- a/ddl/index.go
+++ b/ddl/index.go
@@ -26,6 +26,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/kvrpcpb"
+ "github.com/pingcap/tidb/br/pkg/lightning/common"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/ddl/ingest"
@@ -38,12 +39,14 @@ import (
"github.com/pingcap/tidb/parser/charset"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
+ "github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util"
+ "github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/logutil"
decoder "github.com/pingcap/tidb/util/rowDecoder"
@@ -63,26 +66,33 @@ var (
telemetryAddIndexIngestUsage = metrics.TelemetryAddIndexIngestCnt
)
-func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, indexPartSpecifications []*ast.IndexPartSpecification) ([]*model.IndexColumn, error) {
+func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, indexPartSpecifications []*ast.IndexPartSpecification) ([]*model.IndexColumn, bool, error) {
// Build offsets.
idxParts := make([]*model.IndexColumn, 0, len(indexPartSpecifications))
var col *model.ColumnInfo
+ var mvIndex bool
maxIndexLength := config.GetGlobalConfig().MaxIndexLength
// The sum of length of all index columns.
sumLength := 0
for _, ip := range indexPartSpecifications {
col = model.FindColumnInfo(columns, ip.Column.Name.L)
if col == nil {
- return nil, dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ip.Column.Name)
+ return nil, false, dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ip.Column.Name)
}
if err := checkIndexColumn(ctx, col, ip.Length); err != nil {
- return nil, err
+ return nil, false, err
+ }
+ if col.FieldType.IsArray() {
+ if mvIndex {
+ return nil, false, dbterror.ErrNotSupportedYet.GenWithStackByArgs("more than one multi-valued key part per index")
+ }
+ mvIndex = true
}
indexColLen := ip.Length
indexColumnLength, err := getIndexColumnLength(col, ip.Length)
if err != nil {
- return nil, err
+ return nil, false, err
}
sumLength += indexColumnLength
@@ -91,16 +101,16 @@ func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, inde
// The multiple column index and the unique index in which the length sum exceeds the maximum size
// will return an error instead produce a warning.
if ctx == nil || ctx.GetSessionVars().StrictSQLMode || mysql.HasUniKeyFlag(col.GetFlag()) || len(indexPartSpecifications) > 1 {
- return nil, dbterror.ErrTooLongKey.GenWithStackByArgs(maxIndexLength)
+ return nil, false, dbterror.ErrTooLongKey.GenWithStackByArgs(sumLength, maxIndexLength)
}
// truncate index length and produce warning message in non-restrict sql mode.
colLenPerUint, err := getIndexColumnLength(col, 1)
if err != nil {
- return nil, err
+ return nil, false, err
}
indexColLen = maxIndexLength / colLenPerUint
// produce warning message
- ctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrTooLongKey.FastGenByArgs(maxIndexLength))
+ ctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrTooLongKey.FastGenByArgs(sumLength, maxIndexLength))
}
idxParts = append(idxParts, &model.IndexColumn{
@@ -110,7 +120,7 @@ func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, inde
})
}
- return idxParts, nil
+ return idxParts, mvIndex, nil
}
// CheckPKOnGeneratedColumn checks the specification of PK is valid.
@@ -139,7 +149,7 @@ func checkIndexPrefixLength(columns []*model.ColumnInfo, idxColumns []*model.Ind
return err
}
if idxLen > config.GetGlobalConfig().MaxIndexLength {
- return dbterror.ErrTooLongKey.GenWithStackByArgs(config.GetGlobalConfig().MaxIndexLength)
+ return dbterror.ErrTooLongKey.GenWithStackByArgs(idxLen, config.GetGlobalConfig().MaxIndexLength)
}
return nil
}
@@ -153,7 +163,7 @@ func checkIndexColumn(ctx sessionctx.Context, col *model.ColumnInfo, indexColumn
}
// JSON column cannot index.
- if col.FieldType.GetType() == mysql.TypeJSON {
+ if col.FieldType.GetType() == mysql.TypeJSON && !col.FieldType.IsArray() {
if col.Hidden {
return dbterror.ErrFunctionalIndexOnJSONOrGeometryFunction
}
@@ -201,7 +211,7 @@ func checkIndexColumn(ctx sessionctx.Context, col *model.ColumnInfo, indexColumn
maxIndexLength := config.GetGlobalConfig().MaxIndexLength
if indexColumnLen > maxIndexLength && (ctx == nil || ctx.GetSessionVars().StrictSQLMode) {
// return error in strict sql mode
- return dbterror.ErrTooLongKey.GenWithStackByArgs(maxIndexLength)
+ return dbterror.ErrTooLongKey.GenWithStackByArgs(indexColumnLen, maxIndexLength)
}
return nil
}
@@ -262,7 +272,7 @@ func BuildIndexInfo(
return nil, errors.Trace(err)
}
- idxColumns, err := buildIndexColumns(ctx, allTableColumns, indexPartSpecifications)
+ idxColumns, mvIndex, err := buildIndexColumns(ctx, allTableColumns, indexPartSpecifications)
if err != nil {
return nil, errors.Trace(err)
}
@@ -275,6 +285,7 @@ func BuildIndexInfo(
Primary: isPrimary,
Unique: isUnique,
Global: isGlobal,
+ MVIndex: mvIndex,
}
if indexOption != nil {
@@ -644,6 +655,10 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo
// Initialize SnapshotVer to 0 for later reorganization check.
job.SnapshotVer = 0
job.SchemaState = model.StateWriteReorganization
+
+ if job.MultiSchemaInfo == nil {
+ initDistReorg(job.ReorgMeta)
+ }
case model.StateWriteReorganization:
// reorganization -> public
tbl, err := getTable(d.store, schemaID, tblInfo)
@@ -655,7 +670,11 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo
if job.MultiSchemaInfo != nil {
done, ver, err = doReorgWorkForCreateIndexMultiSchema(w, d, t, job, tbl, indexInfo)
} else {
- done, ver, err = doReorgWorkForCreateIndex(w, d, t, job, tbl, indexInfo)
+ if job.ReorgMeta.IsDistReorg {
+ done, ver, err = doReorgWorkForCreateIndexWithDistReorg(w, d, t, job, tbl, indexInfo)
+ } else {
+ done, ver, err = doReorgWorkForCreateIndex(w, d, t, job, tbl, indexInfo)
+ }
}
if !done {
return ver, err
@@ -676,6 +695,9 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo
job.Args = []interface{}{indexInfo.ID, false /*if exists*/, getPartitionIDs(tbl.Meta())}
// Finish this job.
job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo)
+ if job.ReorgMeta.ReorgTp == model.ReorgTypeLitMerge {
+ ingest.LitBackCtxMgr.Unregister(job.ID)
+ }
default:
err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("index", tblInfo.State)
}
@@ -691,15 +713,18 @@ func pickBackfillType(w *worker, job *model.Job) model.ReorgType {
return job.ReorgMeta.ReorgTp
}
if IsEnableFastReorg() {
- canUseIngest := canUseIngest(w)
- if ingest.LitInitialized && canUseIngest {
- job.ReorgMeta.ReorgTp = model.ReorgTypeLitMerge
- return model.ReorgTypeLitMerge
+ var useIngest bool
+ if ingest.LitInitialized {
+ useIngest = canUseIngest(w)
+ if useIngest {
+ job.ReorgMeta.ReorgTp = model.ReorgTypeLitMerge
+ return model.ReorgTypeLitMerge
+ }
}
// The lightning environment is unavailable, but we can still use the txn-merge backfill.
logutil.BgLogger().Info("[ddl] fallback to txn-merge backfill process",
zap.Bool("lightning env initialized", ingest.LitInitialized),
- zap.Bool("can use ingest", canUseIngest))
+ zap.Bool("can use ingest", useIngest))
job.ReorgMeta.ReorgTp = model.ReorgTypeTxnMerge
return model.ReorgTypeTxnMerge
}
@@ -710,7 +735,10 @@ func pickBackfillType(w *worker, job *model.Job) model.ReorgType {
// canUseIngest indicates whether it can use ingest way to backfill index.
func canUseIngest(w *worker) bool {
// We only allow one task to use ingest at the same time, in order to limit the CPU usage.
- if len(ingest.LitBackCtxMgr.Keys()) > 0 {
+ activeJobIDs := ingest.LitBackCtxMgr.Keys()
+ if len(activeJobIDs) > 0 {
+ logutil.BgLogger().Info("[ddl-ingest] ingest backfill is already in use by another DDL job",
+ zap.Int64("job ID", activeJobIDs[0]))
return false
}
ctx, err := w.sessPool.get()
@@ -753,12 +781,15 @@ func IngestJobsNotExisted(ctx sessionctx.Context) bool {
}
// tryFallbackToTxnMerge changes the reorg type to txn-merge if the lightning backfill meets something wrong.
-func tryFallbackToTxnMerge(job *model.Job, err error) {
+func tryFallbackToTxnMerge(job *model.Job, err error) error {
if job.State != model.JobStateRollingback {
logutil.BgLogger().Info("[ddl] fallback to txn-merge backfill process", zap.Error(err))
job.ReorgMeta.ReorgTp = model.ReorgTypeTxnMerge
job.SnapshotVer = 0
+ job.RowCount = 0
+ return nil
}
+ return err
}
func doReorgWorkForCreateIndexMultiSchema(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job,
@@ -767,6 +798,9 @@ func doReorgWorkForCreateIndexMultiSchema(w *worker, d *ddlCtx, t *meta.Meta, jo
done, ver, err = doReorgWorkForCreateIndex(w, d, t, job, tbl, indexInfo)
if done {
job.MarkNonRevertible()
+ if err == nil {
+ ver, err = updateVersionAndTableInfo(d, t, job, tbl.Meta(), true)
+ }
}
// We need another round to wait for all the others sub-jobs to finish.
return false, ver, err
@@ -782,8 +816,10 @@ func doReorgWorkForCreateIndex(w *worker, d *ddlCtx, t *meta.Meta, job *model.Jo
}
switch indexInfo.BackfillState {
case model.BackfillStateRunning:
- logutil.BgLogger().Info("[ddl] index backfill state running", zap.Int64("job ID", job.ID),
- zap.String("table", tbl.Meta().Name.O), zap.String("index", indexInfo.Name.O))
+ logutil.BgLogger().Info("[ddl] index backfill state running",
+ zap.Int64("job ID", job.ID), zap.String("table", tbl.Meta().Name.O),
+ zap.Bool("ingest mode", bfProcess == model.ReorgTypeLitMerge),
+ zap.String("index", indexInfo.Name.O))
switch bfProcess {
case model.ReorgTypeLitMerge:
bc, ok := ingest.LitBackCtxMgr.Load(job.ID)
@@ -793,17 +829,18 @@ func doReorgWorkForCreateIndex(w *worker, d *ddlCtx, t *meta.Meta, job *model.Jo
if !ok && job.SnapshotVer != 0 {
// The owner is crashed or changed, we need to restart the backfill.
job.SnapshotVer = 0
+ job.RowCount = 0
return false, ver, nil
}
bc, err = ingest.LitBackCtxMgr.Register(w.ctx, indexInfo.Unique, job.ID, job.ReorgMeta.SQLMode)
if err != nil {
- tryFallbackToTxnMerge(job, err)
+ err = tryFallbackToTxnMerge(job, err)
return false, ver, errors.Trace(err)
}
done, ver, err = runReorgJobAndHandleErr(w, d, t, job, tbl, indexInfo, false)
if err != nil {
ingest.LitBackCtxMgr.Unregister(job.ID)
- tryFallbackToTxnMerge(job, err)
+ err = tryFallbackToTxnMerge(job, err)
return false, ver, errors.Trace(err)
}
if !done {
@@ -811,12 +848,15 @@ func doReorgWorkForCreateIndex(w *worker, d *ddlCtx, t *meta.Meta, job *model.Jo
}
err = bc.FinishImport(indexInfo.ID, indexInfo.Unique, tbl)
if err != nil {
- if kv.ErrKeyExists.Equal(err) {
+ if kv.ErrKeyExists.Equal(err) || common.ErrFoundDuplicateKeys.Equal(err) {
logutil.BgLogger().Warn("[ddl] import index duplicate key, convert job to rollback", zap.String("job", job.String()), zap.Error(err))
+ if common.ErrFoundDuplicateKeys.Equal(err) {
+ err = convertToKeyExistsErr(err, indexInfo, tbl.Meta())
+ }
ver, err = convertAddIdxJob2RollbackJob(d, t, job, tbl.Meta(), indexInfo, err)
} else {
logutil.BgLogger().Warn("[ddl] lightning import error", zap.Error(err))
- tryFallbackToTxnMerge(job, err)
+ err = tryFallbackToTxnMerge(job, err)
}
ingest.LitBackCtxMgr.Unregister(job.ID)
return false, ver, errors.Trace(err)
@@ -841,6 +881,56 @@ func doReorgWorkForCreateIndex(w *worker, d *ddlCtx, t *meta.Meta, job *model.Jo
job.SnapshotVer = 0 // Reset the snapshot version for merge index reorg.
ver, err = updateVersionAndTableInfo(d, t, job, tbl.Meta(), true)
return false, ver, errors.Trace(err)
+ case model.BackfillStateMerging:
+ done, ver, err = runReorgJobAndHandleErr(w, d, t, job, tbl, indexInfo, true)
+ if !done {
+ return false, ver, err
+ }
+ indexInfo.BackfillState = model.BackfillStateInapplicable // Prevent double-write on this index.
+ return true, ver, err
+ default:
+ return false, 0, dbterror.ErrInvalidDDLState.GenWithStackByArgs("backfill", indexInfo.BackfillState)
+ }
+}
+
+func doReorgWorkForCreateIndexWithDistReorg(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job,
+ tbl table.Table, indexInfo *model.IndexInfo) (done bool, ver int64, err error) {
+ bfProcess := pickBackfillType(w, job)
+ if !bfProcess.NeedMergeProcess() {
+ return runReorgJobAndHandleErr(w, d, t, job, tbl, indexInfo, false)
+ }
+ switch indexInfo.BackfillState {
+ case model.BackfillStateRunning:
+ logutil.BgLogger().Info("[ddl] index backfill state running",
+ zap.Int64("job ID", job.ID), zap.String("table", tbl.Meta().Name.O),
+ zap.Bool("ingest mode", bfProcess == model.ReorgTypeLitMerge),
+ zap.String("index", indexInfo.Name.O))
+ switch bfProcess {
+ case model.ReorgTypeLitMerge:
+ done, ver, err = runReorgJobAndHandleErr(w, d, t, job, tbl, indexInfo, false)
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] dist lightning import error", zap.Error(err))
+ return false, ver, errors.Trace(err)
+ }
+ if !done {
+ return false, ver, nil
+ }
+ case model.ReorgTypeTxnMerge:
+ done, ver, err = runReorgJobAndHandleErr(w, d, t, job, tbl, indexInfo, false)
+ if err != nil || !done {
+ return false, ver, errors.Trace(err)
+ }
+ }
+ indexInfo.BackfillState = model.BackfillStateReadyToMerge
+ ver, err = updateVersionAndTableInfo(d, t, job, tbl.Meta(), true)
+ return false, ver, errors.Trace(err)
+ case model.BackfillStateReadyToMerge:
+ logutil.BgLogger().Info("[ddl] index backfill state ready to merge", zap.Int64("job ID", job.ID),
+ zap.String("table", tbl.Meta().Name.O), zap.String("index", indexInfo.Name.O))
+ indexInfo.BackfillState = model.BackfillStateMerging
+ job.SnapshotVer = 0 // Reset the snapshot version for merge index reorg.
+ ver, err = updateVersionAndTableInfo(d, t, job, tbl.Meta(), true)
+ return false, ver, errors.Trace(err)
case model.BackfillStateMerging:
done, ver, err = runReorgJobAndHandleErr(w, d, t, job, tbl, indexInfo, true)
if !done {
@@ -853,11 +943,46 @@ func doReorgWorkForCreateIndex(w *worker, d *ddlCtx, t *meta.Meta, job *model.Jo
}
}
+func convertToKeyExistsErr(originErr error, idxInfo *model.IndexInfo, tblInfo *model.TableInfo) error {
+ tErr, ok := errors.Cause(originErr).(*terror.Error)
+ if !ok {
+ return originErr
+ }
+ if len(tErr.Args()) != 2 {
+ return originErr
+ }
+ key, keyIsByte := tErr.Args()[0].([]byte)
+ value, valIsByte := tErr.Args()[1].([]byte)
+ if !keyIsByte || !valIsByte {
+ return originErr
+ }
+ return genKeyExistsErr(key, value, idxInfo, tblInfo)
+}
+
func runReorgJobAndHandleErr(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job,
tbl table.Table, indexInfo *model.IndexInfo, mergingTmpIdx bool) (done bool, ver int64, err error) {
elements := []*meta.Element{{ID: indexInfo.ID, TypeKey: meta.IndexElementKey}}
- rh := newReorgHandler(t, w.sess, w.concurrentDDL)
- reorgInfo, err := getReorgInfo(d.jobContext(job), d, rh, job, tbl, elements, mergingTmpIdx)
+
+ failpoint.Inject("mockDMLExecutionStateMerging", func(val failpoint.Value) {
+ //nolint:forcetypeassert
+ if val.(bool) && indexInfo.BackfillState == model.BackfillStateMerging &&
+ MockDMLExecutionStateMerging != nil {
+ MockDMLExecutionStateMerging()
+ }
+ })
+
+ sctx, err1 := w.sessPool.get()
+ if err1 != nil {
+ err = err1
+ return
+ }
+ defer w.sessPool.put(sctx)
+ rh := newReorgHandler(newSession(sctx))
+ dbInfo, err := t.GetDatabase(job.SchemaID)
+ if err != nil {
+ return false, ver, errors.Trace(err)
+ }
+ reorgInfo, err := getReorgInfo(d.jobContext(job.ID), d, rh, job, dbInfo, tbl, elements, mergingTmpIdx)
if err != nil || reorgInfo.first {
// If we run reorg firstly, we should update the job snapshot version
// and then run the reorg next time.
@@ -875,7 +1000,9 @@ func runReorgJobAndHandleErr(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job,
// if timeout, we should return, check for the owner and re-wait job done.
return false, ver, nil
}
- if kv.ErrKeyExists.Equal(err) || dbterror.ErrCancelledDDLJob.Equal(err) || dbterror.ErrCantDecodeRecord.Equal(err) {
+ if kv.ErrKeyExists.Equal(err) || dbterror.ErrCancelledDDLJob.Equal(err) || dbterror.ErrCantDecodeRecord.Equal(err) ||
+ // TODO: Remove this check make it can be retry. Related test is TestModifyColumnReorgInfo.
+ job.ReorgMeta.IsDistReorg {
logutil.BgLogger().Warn("[ddl] run add index job failed, convert job to rollback", zap.String("job", job.String()), zap.Error(err))
ver, err = convertAddIdxJob2RollbackJob(d, t, job, tbl.Meta(), indexInfo, err)
if err1 := rh.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil {
@@ -953,6 +1080,9 @@ func onDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
// Finish this job.
if job.IsRollingback() {
job.FinishTableJob(model.JobStateRollbackDone, model.StateNone, ver, tblInfo)
+ if job.ReorgMeta.ReorgTp == model.ReorgTypeLitMerge {
+ ingest.LitBackCtxMgr.Unregister(job.ID)
+ }
job.Args[0] = indexInfo.ID
} else {
// the partition ids were append by convertAddIdxJob2RollbackJob, it is weird, but for the compatibility,
@@ -1159,9 +1289,10 @@ type indexRecord struct {
}
type baseIndexWorker struct {
- *backfillWorker
+ *backfillCtx
indexes []table.Index
+ tp backfillerType
metricCounter prometheus.Counter
// The following attributes are used to reduce memory allocation.
@@ -1170,43 +1301,43 @@ type baseIndexWorker struct {
rowMap map[int64]types.Datum
rowDecoder *decoder.RowDecoder
- sqlMode mysql.SQLMode
jobContext *JobContext
}
type addIndexWorker struct {
baseIndexWorker
- index table.Index
- writerCtx *ingest.WriterContext
+ index table.Index
+ writerCtx *ingest.WriterContext
+ copReqSenderPool *copReqSenderPool
// The following attributes are used to reduce memory allocation.
idxKeyBufs [][]byte
batchCheckKeys []kv.Key
+ batchCheckValues [][]byte
distinctCheckFlags []bool
}
-func newAddIndexWorker(sessCtx sessionctx.Context, id int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column,
- reorgInfo *reorgInfo, jc *JobContext, job *model.Job) (*addIndexWorker, error) {
- if !bytes.Equal(reorgInfo.currElement.TypeKey, meta.IndexElementKey) {
- logutil.BgLogger().Error("Element type for addIndexWorker incorrect", zap.String("jobQuery", reorgInfo.Query),
- zap.String("reorgInfo", reorgInfo.String()))
- return nil, errors.Errorf("element type is not index, typeKey: %v", reorgInfo.currElement.TypeKey)
+func newAddIndexWorker(decodeColMap map[int64]decoder.Column, t table.PhysicalTable, bfCtx *backfillCtx, jc *JobContext, jobID, eleID int64, eleTypeKey []byte) (*addIndexWorker, error) {
+ if !bytes.Equal(eleTypeKey, meta.IndexElementKey) {
+ logutil.BgLogger().Error("Element type for addIndexWorker incorrect", zap.String("jobQuery", jc.cacheSQL),
+ zap.Int64("job ID", jobID), zap.ByteString("element type", eleTypeKey), zap.Int64("element ID", eleID))
+ return nil, errors.Errorf("element type is not index, typeKey: %v", eleTypeKey)
}
- indexInfo := model.FindIndexInfoByID(t.Meta().Indices, reorgInfo.currElement.ID)
+ indexInfo := model.FindIndexInfoByID(t.Meta().Indices, eleID)
index := tables.NewIndex(t.GetPhysicalID(), t.Meta(), indexInfo)
rowDecoder := decoder.NewRowDecoder(t, t.WritableCols(), decodeColMap)
var lwCtx *ingest.WriterContext
- if job.ReorgMeta.ReorgTp == model.ReorgTypeLitMerge {
- bc, ok := ingest.LitBackCtxMgr.Load(job.ID)
+ if bfCtx.reorgTp == model.ReorgTypeLitMerge {
+ bc, ok := ingest.LitBackCtxMgr.Load(jobID)
if !ok {
return nil, errors.Trace(errors.New(ingest.LitErrGetBackendFail))
}
- ei, err := bc.EngMgr.Register(bc, job, reorgInfo.currElement.ID)
+ ei, err := bc.EngMgr.Register(bc, jobID, eleID, bfCtx.schemaName, t.Meta().Name.O)
if err != nil {
- return nil, errors.Trace(errors.New(ingest.LitErrCreateEngineFail))
+ return nil, errors.Trace(err)
}
- lwCtx, err = ei.NewWriterCtx(id)
+ lwCtx, err = ei.NewWriterCtx(bfCtx.id, indexInfo.Unique)
if err != nil {
return nil, err
}
@@ -1214,14 +1345,13 @@ func newAddIndexWorker(sessCtx sessionctx.Context, id int, t table.PhysicalTable
return &addIndexWorker{
baseIndexWorker: baseIndexWorker{
- backfillWorker: newBackfillWorker(jc.ddlJobCtx, sessCtx, id, t, reorgInfo, typeAddIndexWorker),
- indexes: []table.Index{index},
- rowDecoder: rowDecoder,
- defaultVals: make([]types.Datum, len(t.WritableCols())),
- rowMap: make(map[int64]types.Datum, len(decodeColMap)),
- metricCounter: metrics.BackfillTotalCounter.WithLabelValues(metrics.GenerateReorgLabel("add_idx_rate", reorgInfo.SchemaName, t.Meta().Name.String())),
- sqlMode: reorgInfo.ReorgMeta.SQLMode,
- jobContext: jc,
+ backfillCtx: bfCtx,
+ indexes: []table.Index{index},
+ rowDecoder: rowDecoder,
+ defaultVals: make([]types.Datum, len(t.WritableCols())),
+ rowMap: make(map[int64]types.Datum, len(decodeColMap)),
+ metricCounter: metrics.BackfillTotalCounter.WithLabelValues(metrics.GenerateReorgLabel("add_idx_rate", bfCtx.schemaName, t.Meta().Name.String())),
+ jobContext: jc,
},
index: index,
writerCtx: lwCtx,
@@ -1232,6 +1362,75 @@ func (w *baseIndexWorker) AddMetricInfo(cnt float64) {
w.metricCounter.Add(cnt)
}
+func (*baseIndexWorker) GetTasks() ([]*BackfillJob, error) {
+ return nil, nil
+}
+
+func (w *baseIndexWorker) String() string {
+ return w.tp.String()
+}
+
+func (w *baseIndexWorker) UpdateTask(bfJob *BackfillJob) error {
+ s := newSession(w.backfillCtx.sessCtx)
+
+ return s.runInTxn(func(se *session) error {
+ jobs, err := GetBackfillJobs(se, BackgroundSubtaskTable, fmt.Sprintf("task_key = '%d_%s_%d_%d'",
+ bfJob.JobID, hex.EncodeToString(bfJob.EleKey), bfJob.EleID, bfJob.ID), "update_backfill_task")
+ if err != nil {
+ return err
+ }
+ if len(jobs) == 0 {
+ return dbterror.ErrDDLJobNotFound.FastGen("get zero backfill job")
+ }
+ if jobs[0].InstanceID != bfJob.InstanceID {
+ return dbterror.ErrDDLJobNotFound.FastGenByArgs(fmt.Sprintf("get a backfill job %v, want instance ID %s", jobs[0], bfJob.InstanceID))
+ }
+
+ currTime, err := GetOracleTimeWithStartTS(se)
+ if err != nil {
+ return err
+ }
+ bfJob.InstanceLease = GetLeaseGoTime(currTime, InstanceLease)
+ return updateBackfillJob(se, BackgroundSubtaskTable, bfJob, "update_backfill_task")
+ })
+}
+
+func (w *baseIndexWorker) FinishTask(bfJob *BackfillJob) error {
+ s := newSession(w.backfillCtx.sessCtx)
+ return s.runInTxn(func(se *session) error {
+ txn, err := se.txn()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ bfJob.StateUpdateTS = txn.StartTS()
+ err = RemoveBackfillJob(se, false, bfJob)
+ if err != nil {
+ return err
+ }
+ return AddBackfillHistoryJob(se, []*BackfillJob{bfJob})
+ })
+}
+
+func (w *baseIndexWorker) GetCtx() *backfillCtx {
+ return w.backfillCtx
+}
+
+func newAddIndexWorkerContext(d *ddl, schemaName model.CIStr, tbl table.Table, workerCnt int,
+ bfJob *BackfillJob, jobCtx *JobContext) (*backfillWorkerContext, error) {
+ //nolint:forcetypeassert
+ phyTbl := tbl.(table.PhysicalTable)
+ return newBackfillWorkerContext(d, schemaName.O, tbl, workerCnt, bfJob.JobID, bfJob.Meta,
+ func(bfCtx *backfillCtx) (backfiller, error) {
+ decodeColMap, err := makeupDecodeColMap(bfCtx.sessCtx, schemaName, phyTbl)
+ if err != nil {
+ logutil.BgLogger().Error("[ddl] make up decode column map failed", zap.Error(err))
+ return nil, errors.Trace(err)
+ }
+ bf, err1 := newAddIndexWorker(decodeColMap, phyTbl, bfCtx, jobCtx, bfJob.JobID, bfJob.EleID, bfJob.EleKey)
+ return bf, err1
+ })
+}
+
// mockNotOwnerErrOnce uses to make sure `notOwnerErr` only mock error once.
var mockNotOwnerErrOnce uint32
@@ -1245,7 +1444,7 @@ func (w *baseIndexWorker) getIndexRecord(idxInfo *model.IndexInfo, handle kv.Han
failpoint.Return(nil, errors.Trace(dbterror.ErrCantDecodeRecord.GenWithStackByArgs("index",
errors.New("mock can't decode record error"))))
case "modifyColumnNotOwnerErr":
- if idxInfo.Name.O == "_Idx$_idx" && handle.IntValue() == 7168 && atomic.CompareAndSwapUint32(&mockNotOwnerErrOnce, 0, 1) {
+ if idxInfo.Name.O == "_Idx$_idx_0" && handle.IntValue() == 7168 && atomic.CompareAndSwapUint32(&mockNotOwnerErrOnce, 0, 1) {
failpoint.Return(nil, errors.Trace(dbterror.ErrNotOwner))
}
case "addIdxNotOwnerErr":
@@ -1290,7 +1489,7 @@ func (w *baseIndexWorker) getNextKey(taskRange reorgBackfillTask, taskDone bool)
if !taskDone {
// The task is not done. So we need to pick the last processed entry's handle and add one.
lastHandle := w.idxRecords[len(w.idxRecords)-1].handle
- recordKey := tablecodec.EncodeRecordKey(w.table.RecordPrefix(), lastHandle)
+ recordKey := tablecodec.EncodeRecordKey(taskRange.physicalTable.RecordPrefix(), lastHandle)
return recordKey.Next()
}
if taskRange.endInclude {
@@ -1322,8 +1521,9 @@ func (w *baseIndexWorker) fetchRowColVals(txn kv.Transaction, taskRange reorgBac
// taskDone means that the reorged handle is out of taskRange.endHandle.
taskDone := false
oprStartTime := startTime
- err := iterateSnapshotKeys(w.reorgInfo.d.jobContext(w.reorgInfo.Job), w.sessCtx.GetStore(), w.priority, w.table.RecordPrefix(), txn.StartTS(), taskRange.startKey, taskRange.endKey,
- func(handle kv.Handle, recordKey kv.Key, rawRow []byte) (bool, error) {
+ jobID := taskRange.getJobID()
+ err := iterateSnapshotKeys(w.GetCtx().jobContext(jobID), w.sessCtx.GetStore(), taskRange.priority, taskRange.physicalTable.RecordPrefix(), txn.StartTS(),
+ taskRange.startKey, taskRange.endKey, func(handle kv.Handle, recordKey kv.Key, rawRow []byte) (bool, error) {
oprEndTime := time.Now()
logSlowOperations(oprEndTime.Sub(oprStartTime), "iterateSnapshotKeys in baseIndexWorker fetchRowColVals", 0)
oprStartTime = oprEndTime
@@ -1365,7 +1565,7 @@ func (w *baseIndexWorker) fetchRowColVals(txn kv.Transaction, taskRange reorgBac
taskDone = true
}
- logutil.BgLogger().Debug("[ddl] txn fetches handle info", zap.Uint64("txnStartTS", txn.StartTS()),
+ logutil.BgLogger().Debug("[ddl] txn fetches handle info", zap.Stringer("worker", w), zap.Uint64("txnStartTS", txn.StartTS()),
zap.String("taskRange", taskRange.String()), zap.Duration("takeTime", time.Since(startTime)))
return w.idxRecords, w.getNextKey(taskRange, taskDone), taskDone, errors.Trace(err)
}
@@ -1376,6 +1576,7 @@ func (w *addIndexWorker) initBatchCheckBufs(batchCount int) {
}
w.batchCheckKeys = w.batchCheckKeys[:0]
+ w.batchCheckValues = w.batchCheckValues[:0]
w.distinctCheckFlags = w.distinctCheckFlags[:0]
}
@@ -1391,22 +1592,34 @@ func (w *addIndexWorker) checkHandleExists(key kv.Key, value []byte, handle kv.H
if hasBeenBackFilled {
return nil
}
+ return genKeyExistsErr(key, value, idxInfo, tblInfo)
+}
+
+func genKeyExistsErr(key, value []byte, idxInfo *model.IndexInfo, tblInfo *model.TableInfo) error {
+ idxColLen := len(idxInfo.Columns)
+ indexName := fmt.Sprintf("%s.%s", tblInfo.Name.String(), idxInfo.Name.String())
colInfos := tables.BuildRowcodecColInfoForIndexColumns(idxInfo, tblInfo)
values, err := tablecodec.DecodeIndexKV(key, value, idxColLen, tablecodec.HandleNotNeeded, colInfos)
if err != nil {
- return err
+ logutil.BgLogger().Warn("decode index key value failed", zap.String("index", indexName),
+ zap.String("key", hex.EncodeToString(key)), zap.String("value", hex.EncodeToString(value)), zap.Error(err))
+ return kv.ErrKeyExists.FastGenByArgs(key, indexName)
}
- indexName := fmt.Sprintf("%s.%s", w.index.TableMeta().Name.String(), w.index.Meta().Name.String())
valueStr := make([]string, 0, idxColLen)
for i, val := range values[:idxColLen] {
d, err := tablecodec.DecodeColumnValue(val, colInfos[i].Ft, time.Local)
if err != nil {
- return kv.ErrKeyExists.FastGenByArgs(key.String(), indexName)
+ logutil.BgLogger().Warn("decode column value failed", zap.String("index", indexName),
+ zap.String("key", hex.EncodeToString(key)), zap.String("value", hex.EncodeToString(value)), zap.Error(err))
+ return kv.ErrKeyExists.FastGenByArgs(key, indexName)
}
str, err := d.ToString()
if err != nil {
str = string(val)
}
+ if types.IsBinaryStr(colInfos[i].Ft) || types.IsTypeBit(colInfos[i].Ft) {
+ str = util.FmtNonASCIIPrintableCharToHex(str)
+ }
valueStr = append(valueStr, str)
}
return kv.ErrKeyExists.FastGenByArgs(strings.Join(valueStr, "-"), indexName)
@@ -1422,16 +1635,28 @@ func (w *addIndexWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords []*i
w.initBatchCheckBufs(len(idxRecords))
stmtCtx := w.sessCtx.GetSessionVars().StmtCtx
+ cnt := 0
for i, record := range idxRecords {
- idxKey, distinct, err := w.index.GenIndexKey(stmtCtx, record.vals, record.handle, w.idxKeyBufs[i])
- if err != nil {
- return errors.Trace(err)
+ iter := w.index.GenIndexKVIter(stmtCtx, record.vals, record.handle, idxRecords[i].rsData)
+ for iter.Valid() {
+ var buf []byte
+ if cnt < len(w.idxKeyBufs) {
+ buf = w.idxKeyBufs[cnt]
+ }
+ key, val, distinct, err := iter.Next(buf)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if cnt < len(w.idxKeyBufs) {
+ w.idxKeyBufs[cnt] = key
+ } else {
+ w.idxKeyBufs = append(w.idxKeyBufs, key)
+ }
+ cnt++
+ w.batchCheckKeys = append(w.batchCheckKeys, key)
+ w.batchCheckValues = append(w.batchCheckValues, val)
+ w.distinctCheckFlags = append(w.distinctCheckFlags, distinct)
}
- // save the buffer to reduce memory allocations.
- w.idxKeyBufs[i] = idxKey
-
- w.batchCheckKeys = append(w.batchCheckKeys, idxKey)
- w.distinctCheckFlags = append(w.distinctCheckFlags, distinct)
}
batchVals, err := txn.BatchGet(context.Background(), w.batchCheckKeys)
@@ -1453,12 +1678,7 @@ func (w *addIndexWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords []*i
} else if w.distinctCheckFlags[i] {
// The keys in w.batchCheckKeys also maybe duplicate,
// so we need to backfill the not found key into `batchVals` map.
- needRsData := tables.NeedRestoredData(w.index.Meta().Columns, w.table.Meta().Columns)
- val, err := tablecodec.GenIndexValuePortal(stmtCtx, w.table.Meta(), w.index.Meta(), needRsData, w.distinctCheckFlags[i], false, idxRecords[i].vals, idxRecords[i].handle, 0, idxRecords[i].rsData)
- if err != nil {
- return errors.Trace(err)
- }
- batchVals[string(key)] = val
+ batchVals[string(key)] = w.batchCheckValues[i]
}
}
// Constrains is already checked.
@@ -1480,16 +1700,29 @@ func (w *addIndexWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (taskC
needMergeTmpIdx := w.index.Meta().BackfillState != model.BackfillStateInapplicable
oprStartTime := time.Now()
+ jobID := handleRange.getJobID()
ctx := kv.WithInternalSourceType(context.Background(), w.jobContext.ddlJobSourceType())
- errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(ctx context.Context, txn kv.Transaction) error {
+ errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(ctx context.Context, txn kv.Transaction) (err error) {
+ taskCtx.finishTS = txn.StartTS()
taskCtx.addedCount = 0
taskCtx.scanCount = 0
- txn.SetOption(kv.Priority, w.priority)
- if tagger := w.reorgInfo.d.getResourceGroupTaggerForTopSQL(w.reorgInfo.Job); tagger != nil {
+ txn.SetOption(kv.Priority, handleRange.priority)
+ if tagger := w.GetCtx().getResourceGroupTaggerForTopSQL(jobID); tagger != nil {
txn.SetOption(kv.ResourceGroupTagger, tagger)
}
- idxRecords, nextKey, taskDone, err := w.fetchRowColVals(txn, handleRange)
+ var (
+ idxRecords []*indexRecord
+ copChunk *chunk.Chunk // only used by the coprocessor request sender.
+ nextKey kv.Key
+ taskDone bool
+ )
+ if w.copReqSenderPool != nil {
+ idxRecords, copChunk, nextKey, taskDone, err = w.copReqSenderPool.fetchRowColValsFromCop(handleRange)
+ defer w.copReqSenderPool.recycleIdxRecordsAndChunk(idxRecords, copChunk)
+ } else {
+ idxRecords, nextKey, taskDone, err = w.fetchRowColVals(txn, handleRange)
+ }
if err != nil {
return errors.Trace(err)
}
@@ -1534,19 +1767,18 @@ func (w *addIndexWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (taskC
} else { // The lightning environment is ready.
vars := w.sessCtx.GetSessionVars()
sCtx, writeBufs := vars.StmtCtx, vars.GetWriteStmtBufs()
- key, distinct, err := w.index.GenIndexKey(sCtx, idxRecord.vals, idxRecord.handle, writeBufs.IndexKeyBuf)
- if err != nil {
- return errors.Trace(err)
- }
- idxVal, err := w.index.GenIndexValue(sCtx, distinct, idxRecord.vals, idxRecord.handle, idxRecord.rsData)
- if err != nil {
- return errors.Trace(err)
- }
- err = w.writerCtx.WriteRow(key, idxVal)
- if err != nil {
- return errors.Trace(err)
+ iter := w.index.GenIndexKVIter(sCtx, idxRecord.vals, idxRecord.handle, idxRecord.rsData)
+ for iter.Valid() {
+ key, idxVal, _, err := iter.Next(writeBufs.IndexKeyBuf)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ err = w.writerCtx.WriteRow(key, idxVal)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ writeBufs.IndexKeyBuf = key
}
- writeBufs.IndexKeyBuf = key
}
taskCtx.addedCount++
}
@@ -1554,10 +1786,24 @@ func (w *addIndexWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (taskC
return nil
})
logSlowOperations(time.Since(oprStartTime), "AddIndexBackfillDataInTxn", 3000)
-
+ failpoint.Inject("mockDMLExecution", func(val failpoint.Value) {
+ //nolint:forcetypeassert
+ if val.(bool) && MockDMLExecution != nil {
+ MockDMLExecution()
+ }
+ })
return
}
+// MockDMLExecution is only used for test.
+var MockDMLExecution func()
+
+// MockDMLExecutionMerging is only used for test.
+var MockDMLExecutionMerging func()
+
+// MockDMLExecutionStateMerging is only used for test.
+var MockDMLExecutionStateMerging func()
+
func (w *worker) addPhysicalTableIndex(t table.PhysicalTable, reorgInfo *reorgInfo) error {
if reorgInfo.mergingTmpIdx {
logutil.BgLogger().Info("[ddl] start to merge temp index", zap.String("job", reorgInfo.Job.String()), zap.String("reorgInfo", reorgInfo.String()))
@@ -1569,6 +1815,11 @@ func (w *worker) addPhysicalTableIndex(t table.PhysicalTable, reorgInfo *reorgIn
// addTableIndex handles the add index reorganization state for a table.
func (w *worker) addTableIndex(t table.Table, reorgInfo *reorgInfo) error {
+ // TODO: Support typeAddIndexMergeTmpWorker.
+ if reorgInfo.Job.ReorgMeta.IsDistReorg && !reorgInfo.mergingTmpIdx {
+ return w.controlWriteTableRecord(w.sessPool, t, typeAddIndexWorker, reorgInfo)
+ }
+
var err error
if tbl, ok := t.(table.PartitionedTable); ok {
var finish bool
@@ -1581,36 +1832,43 @@ func (w *worker) addTableIndex(t table.Table, reorgInfo *reorgInfo) error {
if err != nil {
break
}
- finish, err = w.updateReorgInfo(tbl, reorgInfo)
+ finish, err = updateReorgInfo(w.sessPool, tbl, reorgInfo)
if err != nil {
return errors.Trace(err)
}
}
} else {
//nolint:forcetypeassert
- err = w.addPhysicalTableIndex(t.(table.PhysicalTable), reorgInfo)
+ phyTbl := t.(table.PhysicalTable)
+ err = w.addPhysicalTableIndex(phyTbl, reorgInfo)
}
return errors.Trace(err)
}
-// updateReorgInfo will find the next partition according to current reorgInfo.
-// If no more partitions, or table t is not a partitioned table, returns true to
-// indicate that the reorganize work is finished.
-func (w *worker) updateReorgInfo(t table.PartitionedTable, reorg *reorgInfo) (bool, error) {
+func getNextPartitionInfo(reorg *reorgInfo, t table.PartitionedTable, currPhysicalTableID int64) (int64, kv.Key, kv.Key, error) {
pi := t.Meta().GetPartitionInfo()
if pi == nil {
- return true, nil
+ return 0, nil, nil, nil
}
- pid, err := findNextPartitionID(reorg.PhysicalTableID, pi.Definitions)
+ // During data copying, copy data from partitions to be dropped
+ nextPartitionDefs := pi.DroppingDefinitions
+ if bytes.Equal(reorg.currElement.TypeKey, meta.IndexElementKey) {
+ // During index re-creation, process data from partitions to be added
+ nextPartitionDefs = pi.AddingDefinitions
+ }
+ if nextPartitionDefs == nil {
+ nextPartitionDefs = pi.Definitions
+ }
+ pid, err := findNextPartitionID(currPhysicalTableID, nextPartitionDefs)
if err != nil {
// Fatal error, should not run here.
logutil.BgLogger().Error("[ddl] find next partition ID failed", zap.Reflect("table", t), zap.Error(err))
- return false, errors.Trace(err)
+ return 0, nil, nil, errors.Trace(err)
}
if pid == 0 {
// Next partition does not exist, all the job done.
- return true, nil
+ return 0, nil, nil, nil
}
failpoint.Inject("mockUpdateCachedSafePoint", func(val failpoint.Value) {
@@ -1623,28 +1881,43 @@ func (w *worker) updateReorgInfo(t table.PartitionedTable, reorg *reorgInfo) (bo
time.Sleep(time.Second * 3)
}
})
+
+ var startKey, endKey kv.Key
if reorg.mergingTmpIdx {
indexID := reorg.currElement.ID
- reorg.StartKey, reorg.EndKey = tablecodec.GetTableIndexKeyRange(pid, tablecodec.TempIndexPrefix|indexID)
+ startKey, endKey = tablecodec.GetTableIndexKeyRange(pid, tablecodec.TempIndexPrefix|indexID)
} else {
currentVer, err := getValidCurrentVersion(reorg.d.store)
if err != nil {
- return false, errors.Trace(err)
+ return 0, nil, nil, errors.Trace(err)
}
- start, end, err := getTableRange(reorg.d.jobContext(reorg.Job), reorg.d, t.GetPartition(pid), currentVer.Ver, reorg.Job.Priority)
+ startKey, endKey, err = getTableRange(reorg.d.jobContext(reorg.Job.ID), reorg.d, t.GetPartition(pid), currentVer.Ver, reorg.Job.Priority)
if err != nil {
- return false, errors.Trace(err)
+ return 0, nil, nil, errors.Trace(err)
}
- reorg.StartKey, reorg.EndKey = start, end
}
- reorg.PhysicalTableID = pid
+ return pid, startKey, endKey, nil
+}
+
+// updateReorgInfo will find the next partition according to current reorgInfo.
+// If no more partitions, or table t is not a partitioned table, returns true to
+// indicate that the reorganize work is finished.
+func updateReorgInfo(sessPool *sessionPool, t table.PartitionedTable, reorg *reorgInfo) (bool, error) {
+ pid, startKey, endKey, err := getNextPartitionInfo(reorg, t, reorg.PhysicalTableID)
+ if err != nil {
+ return false, errors.Trace(err)
+ }
+ if pid == 0 {
+ // Next partition does not exist, all the job done.
+ return true, nil
+ }
+ reorg.PhysicalTableID, reorg.StartKey, reorg.EndKey = pid, startKey, endKey
// Write the reorg info to store so the whole reorganize process can recover from panic.
- err = reorg.UpdateReorgMeta(reorg.StartKey, w.sessPool)
+ err = reorg.UpdateReorgMeta(reorg.StartKey, sessPool)
logutil.BgLogger().Info("[ddl] job update reorgInfo",
zap.Int64("jobID", reorg.Job.ID),
- zap.ByteString("elementType", reorg.currElement.TypeKey),
- zap.Int64("elementID", reorg.currElement.ID),
+ zap.Stringer("element", reorg.currElement),
zap.Int64("partitionTableID", pid),
zap.String("startKey", hex.EncodeToString(reorg.StartKey)),
zap.String("endKey", hex.EncodeToString(reorg.EndKey)), zap.Error(err))
@@ -1712,14 +1985,13 @@ func newCleanUpIndexWorker(sessCtx sessionctx.Context, id int, t table.PhysicalT
}
return &cleanUpIndexWorker{
baseIndexWorker: baseIndexWorker{
- backfillWorker: newBackfillWorker(jc.ddlJobCtx, sessCtx, id, t, reorgInfo, typeCleanUpIndexWorker),
- indexes: indexes,
- rowDecoder: rowDecoder,
- defaultVals: make([]types.Datum, len(t.WritableCols())),
- rowMap: make(map[int64]types.Datum, len(decodeColMap)),
- metricCounter: metrics.BackfillTotalCounter.WithLabelValues(metrics.GenerateReorgLabel("cleanup_idx_rate", reorgInfo.SchemaName, t.Meta().Name.String())),
- sqlMode: reorgInfo.ReorgMeta.SQLMode,
- jobContext: jc,
+ backfillCtx: newBackfillCtx(reorgInfo.d, id, sessCtx, reorgInfo.ReorgMeta.ReorgTp, reorgInfo.SchemaName, t),
+ indexes: indexes,
+ rowDecoder: rowDecoder,
+ defaultVals: make([]types.Datum, len(t.WritableCols())),
+ rowMap: make(map[int64]types.Datum, len(decodeColMap)),
+ metricCounter: metrics.BackfillTotalCounter.WithLabelValues(metrics.GenerateReorgLabel("cleanup_idx_rate", reorgInfo.SchemaName, t.Meta().Name.String())),
+ jobContext: jc,
},
}
}
@@ -1737,8 +2009,8 @@ func (w *cleanUpIndexWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (t
errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(ctx context.Context, txn kv.Transaction) error {
taskCtx.addedCount = 0
taskCtx.scanCount = 0
- txn.SetOption(kv.Priority, w.priority)
- if tagger := w.reorgInfo.d.getResourceGroupTaggerForTopSQL(w.reorgInfo.Job); tagger != nil {
+ txn.SetOption(kv.Priority, handleRange.priority)
+ if tagger := w.GetCtx().getResourceGroupTaggerForTopSQL(handleRange.getJobID()); tagger != nil {
txn.SetOption(kv.ResourceGroupTagger, tagger)
}
@@ -1821,7 +2093,7 @@ func (w *worker) updateReorgInfoForPartitions(t table.PartitionedTable, reorg *r
if err != nil {
return false, errors.Trace(err)
}
- start, end, err := getTableRange(reorg.d.jobContext(reorg.Job), reorg.d, t.GetPartition(pid), currentVer.Ver, reorg.Job.Priority)
+ start, end, err := getTableRange(reorg.d.jobContext(reorg.Job.ID), reorg.d, t.GetPartition(pid), currentVer.Ver, reorg.Job.Priority)
if err != nil {
return false, errors.Trace(err)
}
@@ -1829,13 +2101,39 @@ func (w *worker) updateReorgInfoForPartitions(t table.PartitionedTable, reorg *r
// Write the reorg info to store so the whole reorganize process can recover from panic.
err = reorg.UpdateReorgMeta(reorg.StartKey, w.sessPool)
- logutil.BgLogger().Info("[ddl] job update reorgInfo", zap.Int64("jobID", reorg.Job.ID),
- zap.ByteString("elementType", reorg.currElement.TypeKey), zap.Int64("elementID", reorg.currElement.ID),
- zap.Int64("partitionTableID", pid), zap.String("startHandle", tryDecodeToHandleString(start)),
- zap.String("endHandle", tryDecodeToHandleString(end)), zap.Error(err))
+ logutil.BgLogger().Info("[ddl] job update reorg info", zap.Int64("jobID", reorg.Job.ID),
+ zap.Stringer("element", reorg.currElement),
+ zap.Int64("partition table ID", pid), zap.String("start key", hex.EncodeToString(start)),
+ zap.String("end key", hex.EncodeToString(end)), zap.Error(err))
return false, errors.Trace(err)
}
+func runBackfillJobsWithLightning(d *ddl, sess *session, bfJob *BackfillJob, jobCtx *JobContext) error {
+ bc, err := ingest.LitBackCtxMgr.Register(d.ctx, bfJob.Meta.IsUnique, bfJob.JobID, bfJob.Meta.SQLMode)
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] lightning register error", zap.Error(err))
+ return err
+ }
+
+ tbl, err := runBackfillJobs(d, sess, bc, bfJob, jobCtx)
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] runBackfillJobs error", zap.Error(err))
+ ingest.LitBackCtxMgr.Unregister(bfJob.JobID)
+ return err
+ }
+
+ bc.EngMgr.ResetWorkers(bc, bfJob.JobID, bfJob.EleID)
+ err = bc.FinishImport(bfJob.EleID, bfJob.Meta.IsUnique, tbl)
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] lightning import error", zap.String("first backfill job", bfJob.AbbrStr()), zap.Error(err))
+ ingest.LitBackCtxMgr.Unregister(bfJob.JobID)
+ return err
+ }
+ ingest.LitBackCtxMgr.Unregister(bfJob.ID)
+ bc.SetDone()
+ return nil
+}
+
// changingIndex is used to store the index that need to be changed during modifying column.
type changingIndex struct {
IndexInfo *model.IndexInfo
diff --git a/ddl/index_change_test.go b/ddl/index_change_test.go
index b5d2c9d6ce983..dc1b98f205f08 100644
--- a/ddl/index_change_test.go
+++ b/ddl/index_change_test.go
@@ -21,6 +21,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/sessionctx"
@@ -41,7 +42,7 @@ func TestIndexChange(t *testing.T) {
tk.MustExec("insert t values (1, 1), (2, 2), (3, 3);")
d := dom.DDL()
- tc := &ddl.TestDDLCallback{Do: dom}
+ tc := &callback.TestDDLCallback{Do: dom}
// set up hook
prevState := model.StateNone
addIndexDone := false
@@ -219,6 +220,7 @@ func checkAddWriteOnlyForAddIndex(ctx sessionctx.Context, delOnlyTbl, writeOnlyT
}
func checkAddPublicForAddIndex(ctx sessionctx.Context, writeTbl, publicTbl table.Table) error {
+ var err1 error
// WriteOnlyTable: insert t values (6, 6)
err := sessiontxn.NewTxn(context.Background(), ctx)
if err != nil {
@@ -229,7 +231,11 @@ func checkAddPublicForAddIndex(ctx sessionctx.Context, writeTbl, publicTbl table
return errors.Trace(err)
}
err = checkIndexExists(ctx, publicTbl, 6, 6, true)
- if err != nil {
+ if ddl.IsEnableFastReorg() {
+ // Need check temp index also.
+ err1 = checkIndexExists(ctx, writeTbl, 6, 6, true)
+ }
+ if err != nil && err1 != nil {
return errors.Trace(err)
}
// PublicTable: insert t values (7, 7)
@@ -248,10 +254,18 @@ func checkAddPublicForAddIndex(ctx sessionctx.Context, writeTbl, publicTbl table
return errors.Trace(err)
}
err = checkIndexExists(ctx, publicTbl, 5, 7, true)
- if err != nil {
+ if ddl.IsEnableFastReorg() {
+ // Need check temp index also.
+ err1 = checkIndexExists(ctx, writeTbl, 5, 7, true)
+ }
+ if err != nil && err1 != nil {
return errors.Trace(err)
}
- err = checkIndexExists(ctx, publicTbl, 7, 7, false)
+ if ddl.IsEnableFastReorg() {
+ err = checkIndexExists(ctx, writeTbl, 7, 7, false)
+ } else {
+ err = checkIndexExists(ctx, publicTbl, 7, 7, false)
+ }
if err != nil {
return errors.Trace(err)
}
@@ -281,7 +295,11 @@ func checkAddPublicForAddIndex(ctx sessionctx.Context, writeTbl, publicTbl table
idxVal := row[1].GetInt64()
handle := row[0].GetInt64()
err = checkIndexExists(ctx, publicTbl, idxVal, handle, true)
- if err != nil {
+ if ddl.IsEnableFastReorg() {
+ // Need check temp index also.
+ err1 = checkIndexExists(ctx, writeTbl, idxVal, handle, true)
+ }
+ if err != nil && err1 != nil {
return errors.Trace(err)
}
}
diff --git a/ddl/index_cop.go b/ddl/index_cop.go
new file mode 100644
index 0000000000000..fab097727139b
--- /dev/null
+++ b/ddl/index_cop.go
@@ -0,0 +1,546 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/distsql"
+ "github.com/pingcap/tidb/expression"
+ "github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/metrics"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/parser/terror"
+ "github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/sessionctx/stmtctx"
+ "github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/statistics"
+ "github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/table/tables"
+ "github.com/pingcap/tidb/tablecodec"
+ "github.com/pingcap/tidb/types"
+ "github.com/pingcap/tidb/util"
+ "github.com/pingcap/tidb/util/chunk"
+ "github.com/pingcap/tidb/util/codec"
+ "github.com/pingcap/tidb/util/collate"
+ "github.com/pingcap/tidb/util/dbterror"
+ "github.com/pingcap/tidb/util/generic"
+ "github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/timeutil"
+ "github.com/pingcap/tipb/go-tipb"
+ "go.uber.org/zap"
+)
+
+// copReadBatchSize is the batch size of coprocessor read.
+// It multiplies the tidb_ddl_reorg_batch_size by 10 to avoid
+// sending too many cop requests for the same handle range.
+func copReadBatchSize() int {
+ return 10 * int(variable.GetDDLReorgBatchSize())
+}
+
+// copReadChunkPoolSize is the size of chunk pool, which
+// represents the max concurrent ongoing coprocessor requests.
+// It multiplies the tidb_ddl_reorg_worker_cnt by 10.
+func copReadChunkPoolSize() int {
+ return 10 * int(variable.GetDDLReorgWorkerCounter())
+}
+
+func (c *copReqSenderPool) fetchRowColValsFromCop(handleRange reorgBackfillTask) ([]*indexRecord, *chunk.Chunk, kv.Key, bool, error) {
+ ticker := time.NewTicker(500 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ select {
+ case rs, ok := <-c.resultsCh:
+ if !ok {
+ logutil.BgLogger().Info("[ddl-ingest] cop-response channel is closed",
+ zap.Int("id", handleRange.id), zap.String("task", handleRange.String()))
+ return nil, nil, handleRange.endKey, true, nil
+ }
+ if rs.err != nil {
+ return nil, nil, handleRange.startKey, false, rs.err
+ }
+ if rs.done {
+ logutil.BgLogger().Info("[ddl-ingest] finish a cop-request task",
+ zap.Int("id", rs.id), zap.Int("total", rs.total))
+ c.results.Store(rs.id, struct{}{})
+ }
+ if _, found := c.results.Load(handleRange.id); found {
+ logutil.BgLogger().Info("[ddl-ingest] task is found in results",
+ zap.Int("id", handleRange.id), zap.String("task", handleRange.String()))
+ c.results.Delete(handleRange.id)
+ return rs.records, rs.chunk, handleRange.endKey, true, nil
+ }
+ return rs.records, rs.chunk, handleRange.startKey, false, nil
+ case <-ticker.C:
+ logutil.BgLogger().Info("[ddl-ingest] cop-request result channel is empty",
+ zap.Int("id", handleRange.id))
+ if _, found := c.results.Load(handleRange.id); found {
+ c.results.Delete(handleRange.id)
+ return nil, nil, handleRange.endKey, true, nil
+ }
+ }
+ }
+}
+
+type copReqSenderPool struct {
+ tasksCh chan *reorgBackfillTask
+ resultsCh chan idxRecResult
+ results generic.SyncMap[int, struct{}]
+
+ ctx context.Context
+ copCtx *copContext
+ store kv.Storage
+
+ senders []*copReqSender
+ wg sync.WaitGroup
+
+ idxBufPool chan []*indexRecord
+ srcChkPool chan *chunk.Chunk
+}
+
+type copReqSender struct {
+ senderPool *copReqSenderPool
+
+ ctx context.Context
+ cancel context.CancelFunc
+}
+
+func (c *copReqSender) run() {
+ p := c.senderPool
+ defer p.wg.Done()
+ var curTaskID int
+ defer util.Recover(metrics.LabelDDL, "copReqSender.run", func() {
+ p.resultsCh <- idxRecResult{id: curTaskID, err: dbterror.ErrReorgPanic}
+ }, false)
+ for {
+ if util.HasCancelled(c.ctx) {
+ return
+ }
+ task, ok := <-p.tasksCh
+ if !ok {
+ return
+ }
+ curTaskID = task.id
+ logutil.BgLogger().Info("[ddl-ingest] start a cop-request task",
+ zap.Int("id", task.id), zap.String("task", task.String()))
+ ver, err := p.store.CurrentVersion(kv.GlobalTxnScope)
+ if err != nil {
+ p.resultsCh <- idxRecResult{id: task.id, err: err}
+ return
+ }
+ rs, err := p.copCtx.buildTableScan(p.ctx, ver.Ver, task.startKey, task.excludedEndKey())
+ if err != nil {
+ p.resultsCh <- idxRecResult{id: task.id, err: err}
+ return
+ }
+ failpoint.Inject("MockCopSenderPanic", func(val failpoint.Value) {
+ if val.(bool) {
+ panic("mock panic")
+ }
+ })
+ var done bool
+ var total int
+ for !done {
+ idxRec, srcChk := p.getIndexRecordsAndChunks()
+ idxRec, done, err = p.copCtx.fetchTableScanResult(p.ctx, rs, srcChk, idxRec)
+ if err != nil {
+ p.resultsCh <- idxRecResult{id: task.id, err: err}
+ p.recycleIdxRecordsAndChunk(idxRec, srcChk)
+ terror.Call(rs.Close)
+ return
+ }
+ total += len(idxRec)
+ p.resultsCh <- idxRecResult{id: task.id, records: idxRec, chunk: srcChk, done: done, total: total}
+ }
+ terror.Call(rs.Close)
+ }
+}
+
+func newCopReqSenderPool(ctx context.Context, copCtx *copContext, store kv.Storage) *copReqSenderPool {
+ poolSize := copReadChunkPoolSize()
+ idxBufPool := make(chan []*indexRecord, poolSize)
+ srcChkPool := make(chan *chunk.Chunk, poolSize)
+ for i := 0; i < poolSize; i++ {
+ idxBufPool <- make([]*indexRecord, 0, copReadBatchSize())
+ srcChkPool <- chunk.NewChunkWithCapacity(copCtx.fieldTps, copReadBatchSize())
+ }
+ return &copReqSenderPool{
+ tasksCh: make(chan *reorgBackfillTask, backfillTaskChanSize),
+ resultsCh: make(chan idxRecResult, backfillTaskChanSize),
+ results: generic.NewSyncMap[int, struct{}](10),
+ ctx: ctx,
+ copCtx: copCtx,
+ store: store,
+ senders: make([]*copReqSender, 0, variable.GetDDLReorgWorkerCounter()),
+ wg: sync.WaitGroup{},
+ idxBufPool: idxBufPool,
+ srcChkPool: srcChkPool,
+ }
+}
+
+func (c *copReqSenderPool) sendTask(task *reorgBackfillTask) {
+ c.tasksCh <- task
+}
+
+func (c *copReqSenderPool) adjustSize(n int) {
+ // Add some senders.
+ for i := len(c.senders); i < n; i++ {
+ ctx, cancel := context.WithCancel(c.ctx)
+ c.senders = append(c.senders, &copReqSender{
+ senderPool: c,
+ ctx: ctx,
+ cancel: cancel,
+ })
+ c.wg.Add(1)
+ go c.senders[i].run()
+ }
+ // Remove some senders.
+ if n < len(c.senders) {
+ for i := n; i < len(c.senders); i++ {
+ c.senders[i].cancel()
+ }
+ c.senders = c.senders[:n]
+ }
+}
+
+func (c *copReqSenderPool) close() {
+ logutil.BgLogger().Info("[ddl-ingest] close cop-request sender pool", zap.Int("results not handled", len(c.results.Keys())))
+ close(c.tasksCh)
+ for _, w := range c.senders {
+ w.cancel()
+ }
+ cleanupWg := util.WaitGroupWrapper{}
+ cleanupWg.Run(c.drainResults)
+ // Wait for all cop-req senders to exit.
+ c.wg.Wait()
+ close(c.resultsCh)
+ cleanupWg.Wait()
+ close(c.idxBufPool)
+ close(c.srcChkPool)
+}
+
+func (c *copReqSenderPool) drainResults() {
+ // Consume the rest results because the writers are inactive anymore.
+ for rs := range c.resultsCh {
+ c.recycleIdxRecordsAndChunk(rs.records, rs.chunk)
+ }
+}
+
+func (c *copReqSenderPool) getIndexRecordsAndChunks() ([]*indexRecord, *chunk.Chunk) {
+ ir := <-c.idxBufPool
+ chk := <-c.srcChkPool
+ newCap := copReadBatchSize()
+ if chk.Capacity() != newCap {
+ chk = chunk.NewChunkWithCapacity(c.copCtx.fieldTps, newCap)
+ }
+ chk.Reset()
+ return ir[:0], chk
+}
+
+// recycleIdxRecordsAndChunk puts the index record slice and the chunk back to the pool for reuse.
+func (c *copReqSenderPool) recycleIdxRecordsAndChunk(idxRecs []*indexRecord, chk *chunk.Chunk) {
+ if idxRecs == nil || chk == nil {
+ return
+ }
+ c.idxBufPool <- idxRecs
+ c.srcChkPool <- chk
+}
+
+// copContext contains the information that is needed when building a coprocessor request.
+// It is unchanged after initialization.
+type copContext struct {
+ tblInfo *model.TableInfo
+ idxInfo *model.IndexInfo
+ pkInfo *model.IndexInfo
+ colInfos []*model.ColumnInfo
+ fieldTps []*types.FieldType
+ sessCtx sessionctx.Context
+
+ expColInfos []*expression.Column
+ idxColOutputOffsets []int
+ handleOutputOffsets []int
+ virtualColOffsets []int
+ virtualColFieldTps []*types.FieldType
+}
+
+func newCopContext(tblInfo *model.TableInfo, idxInfo *model.IndexInfo, sessCtx sessionctx.Context) (*copContext, error) {
+ var err error
+ usedColumnIDs := make(map[int64]struct{}, len(idxInfo.Columns))
+ usedColumnIDs, err = fillUsedColumns(usedColumnIDs, idxInfo, tblInfo)
+ var handleIDs []int64
+ if err != nil {
+ return nil, err
+ }
+ var primaryIdx *model.IndexInfo
+ if tblInfo.PKIsHandle {
+ pkCol := tblInfo.GetPkColInfo()
+ usedColumnIDs[pkCol.ID] = struct{}{}
+ handleIDs = []int64{pkCol.ID}
+ } else if tblInfo.IsCommonHandle {
+ primaryIdx = tables.FindPrimaryIndex(tblInfo)
+ handleIDs = make([]int64, 0, len(primaryIdx.Columns))
+ for _, pkCol := range primaryIdx.Columns {
+ col := tblInfo.Columns[pkCol.Offset]
+ handleIDs = append(handleIDs, col.ID)
+ }
+ usedColumnIDs, err = fillUsedColumns(usedColumnIDs, primaryIdx, tblInfo)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // Only collect the columns that are used by the index.
+ colInfos := make([]*model.ColumnInfo, 0, len(idxInfo.Columns))
+ fieldTps := make([]*types.FieldType, 0, len(idxInfo.Columns))
+ for i := range tblInfo.Columns {
+ col := tblInfo.Columns[i]
+ if _, found := usedColumnIDs[col.ID]; found {
+ colInfos = append(colInfos, col)
+ fieldTps = append(fieldTps, &col.FieldType)
+ }
+ }
+
+ // Append the extra handle column when _tidb_rowid is used.
+ if !tblInfo.HasClusteredIndex() {
+ extra := model.NewExtraHandleColInfo()
+ colInfos = append(colInfos, extra)
+ fieldTps = append(fieldTps, &extra.FieldType)
+ handleIDs = []int64{extra.ID}
+ }
+
+ expColInfos, _, err := expression.ColumnInfos2ColumnsAndNames(sessCtx,
+ model.CIStr{} /* unused */, tblInfo.Name, colInfos, tblInfo)
+ if err != nil {
+ return nil, err
+ }
+ idxOffsets := resolveIndicesForIndex(expColInfos, idxInfo, tblInfo)
+ hdColOffsets := resolveIndicesForHandle(expColInfos, handleIDs)
+ vColOffsets, vColFts := collectVirtualColumnOffsetsAndTypes(expColInfos)
+
+ copCtx := &copContext{
+ tblInfo: tblInfo,
+ idxInfo: idxInfo,
+ pkInfo: primaryIdx,
+ colInfos: colInfos,
+ fieldTps: fieldTps,
+ sessCtx: sessCtx,
+
+ expColInfos: expColInfos,
+ idxColOutputOffsets: idxOffsets,
+ handleOutputOffsets: hdColOffsets,
+ virtualColOffsets: vColOffsets,
+ virtualColFieldTps: vColFts,
+ }
+ return copCtx, nil
+}
+
+func fillUsedColumns(usedCols map[int64]struct{}, idxInfo *model.IndexInfo, tblInfo *model.TableInfo) (map[int64]struct{}, error) {
+ colsToChecks := make([]*model.ColumnInfo, 0, len(idxInfo.Columns))
+ for _, idxCol := range idxInfo.Columns {
+ colsToChecks = append(colsToChecks, tblInfo.Columns[idxCol.Offset])
+ }
+ for len(colsToChecks) > 0 {
+ next := colsToChecks[0]
+ colsToChecks = colsToChecks[1:]
+ usedCols[next.ID] = struct{}{}
+ for depColName := range next.Dependences {
+ // Expand the virtual generated columns.
+ depCol := model.FindColumnInfo(tblInfo.Columns, depColName)
+ if depCol == nil {
+ return nil, errors.Trace(errors.Errorf("dependent column %s not found", depColName))
+ }
+ if _, ok := usedCols[depCol.ID]; !ok {
+ colsToChecks = append(colsToChecks, depCol)
+ }
+ }
+ }
+ return usedCols, nil
+}
+
+func resolveIndicesForIndex(outputCols []*expression.Column, idxInfo *model.IndexInfo, tblInfo *model.TableInfo) []int {
+ offsets := make([]int, 0, len(idxInfo.Columns))
+ for _, idxCol := range idxInfo.Columns {
+ hid := tblInfo.Columns[idxCol.Offset].ID
+ for j, col := range outputCols {
+ if col.ID == hid {
+ offsets = append(offsets, j)
+ break
+ }
+ }
+ }
+ return offsets
+}
+
+func resolveIndicesForHandle(cols []*expression.Column, handleIDs []int64) []int {
+ offsets := make([]int, 0, len(handleIDs))
+ for _, hid := range handleIDs {
+ for j, col := range cols {
+ if col.ID == hid {
+ offsets = append(offsets, j)
+ break
+ }
+ }
+ }
+ return offsets
+}
+
+func collectVirtualColumnOffsetsAndTypes(cols []*expression.Column) ([]int, []*types.FieldType) {
+ var offsets []int
+ var fts []*types.FieldType
+ for i, col := range cols {
+ if col.VirtualExpr != nil {
+ offsets = append(offsets, i)
+ fts = append(fts, col.GetType())
+ }
+ }
+ return offsets, fts
+}
+
+func (c *copContext) buildTableScan(ctx context.Context, startTS uint64, start, end kv.Key) (distsql.SelectResult, error) {
+ dagPB, err := buildDAGPB(c.sessCtx, c.tblInfo, c.colInfos)
+ if err != nil {
+ return nil, err
+ }
+
+ var builder distsql.RequestBuilder
+ kvReq, err := builder.
+ SetDAGRequest(dagPB).
+ SetStartTS(startTS).
+ SetKeyRanges([]kv.KeyRange{{StartKey: start, EndKey: end}}).
+ SetKeepOrder(true).
+ SetFromSessionVars(c.sessCtx.GetSessionVars()).
+ SetFromInfoSchema(c.sessCtx.GetDomainInfoSchema()).
+ SetConcurrency(1).
+ Build()
+ if err != nil {
+ return nil, err
+ }
+ return distsql.Select(ctx, c.sessCtx, kvReq, c.fieldTps, statistics.NewQueryFeedback(0, nil, 0, false))
+}
+
+func (c *copContext) fetchTableScanResult(ctx context.Context, result distsql.SelectResult,
+ chk *chunk.Chunk, buf []*indexRecord) ([]*indexRecord, bool, error) {
+ sctx := c.sessCtx.GetSessionVars().StmtCtx
+ err := result.Next(ctx, chk)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ if chk.NumRows() == 0 {
+ return buf, true, nil
+ }
+ iter := chunk.NewIterator4Chunk(chk)
+ err = table.FillVirtualColumnValue(c.virtualColFieldTps, c.virtualColOffsets, c.expColInfos, c.colInfos, c.sessCtx, chk)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ for row := iter.Begin(); row != iter.End(); row = iter.Next() {
+ idxDt := extractDatumByOffsets(row, c.idxColOutputOffsets, c.expColInfos)
+ hdDt := extractDatumByOffsets(row, c.handleOutputOffsets, c.expColInfos)
+ handle, err := buildHandle(hdDt, c.tblInfo, c.pkInfo, sctx)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ rsData := getRestoreData(c.tblInfo, c.idxInfo, c.pkInfo, hdDt)
+ buf = append(buf, &indexRecord{handle: handle, key: nil, vals: idxDt, rsData: rsData, skip: false})
+ }
+ return buf, false, nil
+}
+
+func getRestoreData(tblInfo *model.TableInfo, targetIdx, pkIdx *model.IndexInfo, handleDts []types.Datum) []types.Datum {
+ if !collate.NewCollationEnabled() || !tblInfo.IsCommonHandle || tblInfo.CommonHandleVersion == 0 {
+ return nil
+ }
+ if pkIdx == nil {
+ return nil
+ }
+ for i, pkIdxCol := range pkIdx.Columns {
+ pkCol := tblInfo.Columns[pkIdxCol.Offset]
+ if !types.NeedRestoredData(&pkCol.FieldType) {
+ // Since the handle data cannot be null, we can use SetNull to
+ // indicate that this column does not need to be restored.
+ handleDts[i].SetNull()
+ continue
+ }
+ tables.TryTruncateRestoredData(&handleDts[i], pkCol, pkIdxCol, targetIdx)
+ tables.ConvertDatumToTailSpaceCount(&handleDts[i], pkCol)
+ }
+ dtToRestored := handleDts[:0]
+ for _, handleDt := range handleDts {
+ if !handleDt.IsNull() {
+ dtToRestored = append(dtToRestored, handleDt)
+ }
+ }
+ return dtToRestored
+}
+
+func buildDAGPB(sCtx sessionctx.Context, tblInfo *model.TableInfo, colInfos []*model.ColumnInfo) (*tipb.DAGRequest, error) {
+ dagReq := &tipb.DAGRequest{}
+ dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(sCtx.GetSessionVars().Location())
+ sc := sCtx.GetSessionVars().StmtCtx
+ dagReq.Flags = sc.PushDownFlags()
+ for i := range colInfos {
+ dagReq.OutputOffsets = append(dagReq.OutputOffsets, uint32(i))
+ }
+ execPB, err := constructTableScanPB(sCtx, tblInfo, colInfos)
+ if err != nil {
+ return nil, err
+ }
+ dagReq.Executors = append(dagReq.Executors, execPB)
+ distsql.SetEncodeType(sCtx, dagReq)
+ return dagReq, nil
+}
+
+func constructTableScanPB(sCtx sessionctx.Context, tblInfo *model.TableInfo, colInfos []*model.ColumnInfo) (*tipb.Executor, error) {
+ tblScan := tables.BuildTableScanFromInfos(tblInfo, colInfos)
+ tblScan.TableId = tblInfo.ID
+ err := tables.SetPBColumnsDefaultValue(sCtx, tblScan.Columns, colInfos)
+ return &tipb.Executor{Tp: tipb.ExecType_TypeTableScan, TblScan: tblScan}, err
+}
+
+func extractDatumByOffsets(row chunk.Row, offsets []int, expCols []*expression.Column) []types.Datum {
+ datumBuf := make([]types.Datum, 0, len(offsets))
+ for _, offset := range offsets {
+ c := expCols[offset]
+ rowDt := row.GetDatum(offset, c.GetType())
+ datumBuf = append(datumBuf, rowDt)
+ }
+ return datumBuf
+}
+
+func buildHandle(pkDts []types.Datum, tblInfo *model.TableInfo,
+ pkInfo *model.IndexInfo, stmtCtx *stmtctx.StatementContext) (kv.Handle, error) {
+ if tblInfo.IsCommonHandle {
+ tablecodec.TruncateIndexValues(tblInfo, pkInfo, pkDts)
+ handleBytes, err := codec.EncodeKey(stmtCtx, nil, pkDts...)
+ if err != nil {
+ return nil, err
+ }
+ return kv.NewCommonHandle(handleBytes)
+ }
+ return kv.IntHandle(pkDts[0].GetInt64()), nil
+}
+
+type idxRecResult struct {
+ id int
+ records []*indexRecord
+ chunk *chunk.Chunk
+ err error
+ done bool
+ total int
+}
diff --git a/ddl/index_cop_test.go b/ddl/index_cop_test.go
new file mode 100644
index 0000000000000..5edc1680b2308
--- /dev/null
+++ b/ddl/index_cop_test.go
@@ -0,0 +1,102 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl_test
+
+import (
+ "fmt"
+ "strconv"
+ "testing"
+
+ "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/pingcap/tidb/types"
+ "github.com/stretchr/testify/require"
+)
+
+func TestAddIndexFetchRowsFromCoprocessor(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ testFetchRows := func(db, tb, idx string) ([]kv.Handle, [][]types.Datum) {
+ tbl, err := dom.InfoSchema().TableByName(model.NewCIStr(db), model.NewCIStr(tb))
+ require.NoError(t, err)
+ tblInfo := tbl.Meta()
+ idxInfo := tblInfo.FindIndexByName(idx)
+ copCtx, err := ddl.NewCopContext4Test(tblInfo, idxInfo, tk.Session())
+ require.NoError(t, err)
+ startKey := tbl.RecordPrefix()
+ endKey := startKey.PrefixNext()
+ txn, err := store.Begin()
+ require.NoError(t, err)
+ idxRec, done, err := ddl.FetchRowsFromCop4Test(copCtx, tbl.(table.PhysicalTable), startKey, endKey, store, 10)
+ require.NoError(t, err)
+ require.False(t, done)
+ require.NoError(t, txn.Rollback())
+
+ handles := make([]kv.Handle, 0, len(idxRec))
+ values := make([][]types.Datum, 0, len(idxRec))
+ for _, rec := range idxRec {
+ handles = append(handles, rec.GetHandle())
+ values = append(values, rec.GetIndexValues())
+ }
+ return handles, values
+ }
+
+ // Test nonclustered primary key table.
+ tk.MustExec("drop table if exists t;")
+ tk.MustExec("create table t (a bigint, b int, index idx (b));")
+ for i := 0; i < 8; i++ {
+ tk.MustExec("insert into t values (?, ?)", i, i)
+ }
+ hds, vals := testFetchRows("test", "t", "idx")
+ require.Len(t, hds, 8)
+ for i := 0; i < 8; i++ {
+ require.Equal(t, hds[i].IntValue(), int64(i+1))
+ require.Len(t, vals[i], 1)
+ require.Equal(t, vals[i][0].GetInt64(), int64(i))
+ }
+
+ // Test clustered primary key table(pk_is_handle).
+ tk.MustExec("drop table if exists t;")
+ tk.MustExec("create table t (a bigint primary key, b int, index idx (b));")
+ for i := 0; i < 8; i++ {
+ tk.MustExec("insert into t values (?, ?)", i, i)
+ }
+ hds, vals = testFetchRows("test", "t", "idx")
+ require.Len(t, hds, 8)
+ for i := 0; i < 8; i++ {
+ require.Equal(t, hds[i].IntValue(), int64(i))
+ require.Len(t, vals[i], 1)
+ require.Equal(t, vals[i][0].GetInt64(), int64(i))
+ }
+
+ // Test clustered primary key table(common_handle).
+ tk.MustExec("drop table if exists t;")
+ tk.MustExec("create table t (a varchar(10), b int, c char(10), primary key (a, c) clustered, index idx (b));")
+ for i := 0; i < 8; i++ {
+ tk.MustExec("insert into t values (?, ?, ?)", strconv.Itoa(i), i, strconv.Itoa(i))
+ }
+ hds, vals = testFetchRows("test", "t", "idx")
+ require.Len(t, hds, 8)
+ for i := 0; i < 8; i++ {
+ require.Equal(t, hds[i].String(), fmt.Sprintf("{%d, %d}", i, i))
+ require.Len(t, vals[i], 1)
+ require.Equal(t, vals[i][0].GetInt64(), int64(i))
+ }
+}
diff --git a/ddl/index_merge_tmp.go b/ddl/index_merge_tmp.go
index b33111119facc..029c87542de11 100644
--- a/ddl/index_merge_tmp.go
+++ b/ddl/index_merge_tmp.go
@@ -20,9 +20,9 @@ import (
"time"
"github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/model"
- "github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/table/tables"
@@ -49,16 +49,12 @@ func (w *mergeIndexWorker) batchCheckTemporaryUniqueKey(txn kv.Transaction, idxR
return errors.Trace(err)
}
- // 1. unique-key/primary-key is duplicate and the handle is equal, skip it.
- // 2. unique-key/primary-key is duplicate and the handle is not equal, return duplicate error.
- // 3. non-unique-key is duplicate, skip it.
for i, key := range w.originIdxKeys {
if val, found := batchVals[string(key)]; found {
- if idxRecords[i].distinct && !bytes.Equal(val, idxRecords[i].vals) {
- return kv.ErrKeyExists
- }
- if !idxRecords[i].delete {
- idxRecords[i].skip = true
+ // Found a value in the original index key.
+ err := checkTempIndexKey(txn, idxRecords[i], val, w.table)
+ if err != nil {
+ return errors.Trace(err)
}
} else if idxRecords[i].distinct {
// The keys in w.batchCheckKeys also maybe duplicate,
@@ -69,6 +65,48 @@ func (w *mergeIndexWorker) batchCheckTemporaryUniqueKey(txn kv.Transaction, idxR
return nil
}
+func checkTempIndexKey(txn kv.Transaction, tmpRec *temporaryIndexRecord, originIdxVal []byte, tblInfo table.Table) error {
+ if !tmpRec.delete {
+ if tmpRec.distinct && !bytes.Equal(originIdxVal, tmpRec.vals) {
+ return kv.ErrKeyExists
+ }
+ // The key has been found in the original index, skip merging it.
+ tmpRec.skip = true
+ return nil
+ }
+ // Delete operation.
+ distinct := tablecodec.IndexKVIsUnique(originIdxVal)
+ if !distinct {
+ // For non-distinct key, it is consist of a null value and the handle.
+ // Same as the non-unique indexes, replay the delete operation on non-distinct keys.
+ return nil
+ }
+ // For distinct index key values, prevent deleting an unexpected index KV in original index.
+ hdInVal, err := tablecodec.DecodeHandleInUniqueIndexValue(originIdxVal, tblInfo.Meta().IsCommonHandle)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if !tmpRec.handle.Equal(hdInVal) {
+ // The inequality means multiple modifications happened in the same key.
+ // We use the handle in origin index value to check if the row exists.
+ rowKey := tablecodec.EncodeRecordKey(tblInfo.RecordPrefix(), hdInVal)
+ _, err := txn.Get(context.Background(), rowKey)
+ if err != nil {
+ if kv.IsErrNotFound(err) {
+ // The row is deleted, so we can merge the delete operation to the origin index.
+ tmpRec.skip = false
+ return nil
+ }
+ // Unexpected errors.
+ return errors.Trace(err)
+ }
+ // Don't delete the index key if the row exists.
+ tmpRec.skip = true
+ return nil
+ }
+ return nil
+}
+
// temporaryIndexRecord is the record information of an index.
type temporaryIndexRecord struct {
vals []byte
@@ -76,10 +114,12 @@ type temporaryIndexRecord struct {
delete bool
unique bool
distinct bool
+ handle kv.Handle
+ rowKey kv.Key
}
type mergeIndexWorker struct {
- *backfillWorker
+ *backfillCtx
index table.Index
@@ -89,15 +129,15 @@ type mergeIndexWorker struct {
jobContext *JobContext
}
-func newMergeTempIndexWorker(sessCtx sessionctx.Context, id int, t table.PhysicalTable, reorgInfo *reorgInfo, jc *JobContext) *mergeIndexWorker {
- indexInfo := model.FindIndexInfoByID(t.Meta().Indices, reorgInfo.currElement.ID)
+func newMergeTempIndexWorker(bfCtx *backfillCtx, id int, t table.PhysicalTable, eleID int64, jc *JobContext) *mergeIndexWorker {
+ indexInfo := model.FindIndexInfoByID(t.Meta().Indices, eleID)
index := tables.NewIndex(t.GetPhysicalID(), t.Meta(), indexInfo)
return &mergeIndexWorker{
- backfillWorker: newBackfillWorker(jc.ddlJobCtx, sessCtx, id, t, reorgInfo, typeAddIndexMergeTmpWorker),
- index: index,
- jobContext: jc,
+ backfillCtx: bfCtx,
+ index: index,
+ jobContext: jc,
}
}
@@ -108,8 +148,8 @@ func (w *mergeIndexWorker) BackfillDataInTxn(taskRange reorgBackfillTask) (taskC
errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(ctx context.Context, txn kv.Transaction) error {
taskCtx.addedCount = 0
taskCtx.scanCount = 0
- txn.SetOption(kv.Priority, w.priority)
- if tagger := w.reorgInfo.d.getResourceGroupTaggerForTopSQL(w.reorgInfo.Job); tagger != nil {
+ txn.SetOption(kv.Priority, taskRange.priority)
+ if tagger := w.GetCtx().getResourceGroupTaggerForTopSQL(taskRange.getJobID()); tagger != nil {
txn.SetOption(kv.ResourceGroupTagger, tagger)
}
@@ -133,6 +173,15 @@ func (w *mergeIndexWorker) BackfillDataInTxn(taskRange reorgBackfillTask) (taskC
if idxRecord.skip {
continue
}
+
+ // Lock the corresponding row keys so that it doesn't modify the index KVs
+ // that are changing by a pessimistic transaction.
+ rowKey := tablecodec.EncodeRecordKey(w.table.RecordPrefix(), idxRecord.handle)
+ err := txn.LockKeys(context.Background(), new(kv.LockCtx), rowKey)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
if idxRecord.delete {
if idxRecord.unique {
err = txn.GetMemBuffer().DeleteWithFlags(w.originIdxKeys[i], kv.SetNeedLocked)
@@ -149,11 +198,38 @@ func (w *mergeIndexWorker) BackfillDataInTxn(taskRange reorgBackfillTask) (taskC
}
return nil
})
+
+ failpoint.Inject("mockDMLExecutionMerging", func(val failpoint.Value) {
+ //nolint:forcetypeassert
+ if val.(bool) && MockDMLExecutionMerging != nil {
+ MockDMLExecutionMerging()
+ }
+ })
logSlowOperations(time.Since(oprStartTime), "AddIndexMergeDataInTxn", 3000)
return
}
-func (w *mergeIndexWorker) AddMetricInfo(cnt float64) {
+func (*mergeIndexWorker) AddMetricInfo(float64) {
+}
+
+func (*mergeIndexWorker) String() string {
+ return typeAddIndexMergeTmpWorker.String()
+}
+
+func (*mergeIndexWorker) GetTasks() ([]*BackfillJob, error) {
+ panic("[ddl] merge index worker GetTask function doesn't implement")
+}
+
+func (*mergeIndexWorker) UpdateTask(*BackfillJob) error {
+ panic("[ddl] merge index worker UpdateTask function doesn't implement")
+}
+
+func (*mergeIndexWorker) FinishTask(*BackfillJob) error {
+ panic("[ddl] merge index worker FinishTask function doesn't implement")
+}
+
+func (w *mergeIndexWorker) GetCtx() *backfillCtx {
+ return w.backfillCtx
}
func (w *mergeIndexWorker) fetchTempIndexVals(txn kv.Transaction, taskRange reorgBackfillTask) ([]*temporaryIndexRecord, kv.Key, bool, error) {
@@ -166,7 +242,8 @@ func (w *mergeIndexWorker) fetchTempIndexVals(txn kv.Transaction, taskRange reor
oprStartTime := startTime
idxPrefix := w.table.IndexPrefix()
var lastKey kv.Key
- err := iterateSnapshotKeys(w.reorgInfo.d.jobContext(w.reorgInfo.Job), w.sessCtx.GetStore(), w.priority, idxPrefix, txn.StartTS(),
+ isCommonHandle := w.table.Meta().IsCommonHandle
+ err := iterateSnapshotKeys(w.GetCtx().jobContext(taskRange.getJobID()), w.sessCtx.GetStore(), taskRange.priority, idxPrefix, txn.StartTS(),
taskRange.startKey, taskRange.endKey, func(_ kv.Handle, indexKey kv.Key, rawValue []byte) (more bool, err error) {
oprEndTime := time.Now()
logSlowOperations(oprEndTime.Sub(oprStartTime), "iterate temporary index in merge process", 0)
@@ -182,38 +259,49 @@ func (w *mergeIndexWorker) fetchTempIndexVals(txn kv.Transaction, taskRange reor
return false, nil
}
- isDelete := false
- unique := false
- length := len(rawValue)
- keyVer := rawValue[length-1]
- if keyVer == tables.TempIndexKeyTypeMerge {
- // The kv is written in the merging state. It has been written to the origin index, we can skip it.
- return true, nil
- }
- rawValue = rawValue[:length-1]
- if bytes.Equal(rawValue, tables.DeleteMarker) {
- isDelete = true
- } else if bytes.Equal(rawValue, tables.DeleteMarkerUnique) {
- isDelete = true
- unique = true
+ tempIdxVal, err := tablecodec.DecodeTempIndexValue(rawValue, isCommonHandle)
+ if err != nil {
+ return false, err
}
+ tempIdxVal = tempIdxVal.FilterOverwritten()
- originIdxKey := make([]byte, len(indexKey))
- copy(originIdxKey, indexKey)
- tablecodec.TempIndexKey2IndexKey(w.index.Meta().ID, originIdxKey)
+ // Extract the operations on the original index and replay them later.
+ for _, elem := range tempIdxVal {
+ if elem.KeyVer == tables.TempIndexKeyTypeMerge || elem.KeyVer == tables.TempIndexKeyTypeDelete {
+ // For 'm' version kvs, they are double-written.
+ // For 'd' version kvs, they are written in the delete-only state and can be dropped safely.
+ continue
+ }
- idxRecord := &temporaryIndexRecord{
- delete: isDelete,
- unique: unique,
- skip: false,
- }
- if !isDelete {
- idxRecord.vals = rawValue
- idxRecord.distinct = tablecodec.IndexKVIsUnique(rawValue)
+ if elem.Handle == nil {
+ // If the handle is not found in the value of the temp index, it means
+ // 1) This is not a deletion marker, the handle is in the key or the origin value.
+ // 2) This is a deletion marker, but the handle is in the key of temp index.
+ elem.Handle, err = tablecodec.DecodeIndexHandle(indexKey, elem.Value, len(w.index.Meta().Columns))
+ if err != nil {
+ return false, err
+ }
+ }
+
+ originIdxKey := make([]byte, len(indexKey))
+ copy(originIdxKey, indexKey)
+ tablecodec.TempIndexKey2IndexKey(w.index.Meta().ID, originIdxKey)
+
+ idxRecord := &temporaryIndexRecord{
+ handle: elem.Handle,
+ delete: elem.Delete,
+ unique: elem.Distinct,
+ skip: false,
+ }
+ if !elem.Delete {
+ idxRecord.vals = elem.Value
+ idxRecord.distinct = tablecodec.IndexKVIsUnique(elem.Value)
+ }
+ w.tmpIdxRecords = append(w.tmpIdxRecords, idxRecord)
+ w.originIdxKeys = append(w.originIdxKeys, originIdxKey)
+ w.tmpIdxKeys = append(w.tmpIdxKeys, indexKey)
}
- w.tmpIdxRecords = append(w.tmpIdxRecords, idxRecord)
- w.originIdxKeys = append(w.originIdxKeys, originIdxKey)
- w.tmpIdxKeys = append(w.tmpIdxKeys, indexKey)
+
lastKey = indexKey
return true, nil
})
diff --git a/ddl/index_merge_tmp_test.go b/ddl/index_merge_tmp_test.go
deleted file mode 100644
index eb0a935690068..0000000000000
--- a/ddl/index_merge_tmp_test.go
+++ /dev/null
@@ -1,195 +0,0 @@
-// Copyright 2022 PingCAP, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ddl_test
-
-import (
- "testing"
-
- "github.com/pingcap/tidb/ddl"
- "github.com/pingcap/tidb/ddl/ingest"
- "github.com/pingcap/tidb/domain"
- "github.com/pingcap/tidb/kv"
- "github.com/pingcap/tidb/parser/model"
- "github.com/pingcap/tidb/tablecodec"
- "github.com/pingcap/tidb/testkit"
- "github.com/pingcap/tidb/util/logutil"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- "go.uber.org/zap"
-)
-
-func TestAddIndexMergeProcess(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk2 := testkit.NewTestKit(t, store)
- tk2.MustExec("use test")
- tk.MustExec("create table t (c1 int primary key, c2 int, c3 int)")
- tk.MustExec("insert into t values (1, 2, 3), (4, 5, 6);")
- // Force onCreateIndex use the txn-merge process.
- ingest.LitInitialized = false
- tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = 1;")
-
- var checkErr error
- var runDML, backfillDone bool
- originHook := dom.DDL().GetHook()
- callback := &ddl.TestDDLCallback{
- Do: dom,
- }
- onJobUpdatedExportedFunc := func(job *model.Job) {
- if !runDML && job.Type == model.ActionAddIndex && job.SchemaState == model.StateWriteReorganization {
- idx := findIdxInfo(dom, "test", "t", "idx")
- if idx == nil || idx.BackfillState != model.BackfillStateRunning {
- return
- }
- if !backfillDone {
- // Wait another round so that the backfill range is determined(1-4).
- backfillDone = true
- return
- }
- runDML = true
- // Write record 7 to the temporary index.
- _, checkErr = tk2.Exec("insert into t values (7, 8, 9);")
- }
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(callback)
- tk.MustExec("alter table t add index idx(c1);")
- dom.DDL().SetHook(originHook)
- require.True(t, backfillDone)
- require.True(t, runDML)
- require.NoError(t, checkErr)
- tk.MustExec("admin check table t;")
- tk.MustQuery("select * from t use index (idx);").Check(testkit.Rows("1 2 3", "4 5 6", "7 8 9"))
- tk.MustQuery("select * from t ignore index (idx);").Check(testkit.Rows("1 2 3", "4 5 6", "7 8 9"))
-}
-
-func TestAddPrimaryKeyMergeProcess(t *testing.T) {
- // Disable auto schema reload.
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 0)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk2 := testkit.NewTestKit(t, store)
- tk2.MustExec("use test")
- tk.MustExec("create table t (c1 int, c2 int, c3 int)")
- tk.MustExec("insert into t values (1, 2, 3), (4, 5, 6);")
- // Force onCreateIndex use the backfill-merge process.
- ingest.LitInitialized = false
- tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = 1;")
-
- var checkErr error
- var runDML, backfillDone bool
- originHook := dom.DDL().GetHook()
- callback := &ddl.TestDDLCallback{
- Do: nil, // We'll reload the schema manually.
-
- }
- onJobUpdatedExportedFunc := func(job *model.Job) {
- if !runDML && job.Type == model.ActionAddPrimaryKey && job.SchemaState == model.StateWriteReorganization {
- idx := findIdxInfo(dom, "test", "t", "primary")
- if idx == nil || idx.BackfillState != model.BackfillStateRunning || job.SnapshotVer == 0 {
- return
- }
- if !backfillDone {
- // Wait another round so that the backfill process is finished, but
- // the info schema is not updated.
- backfillDone = true
- return
- }
- runDML = true
- // Add delete record 4 to the temporary index.
- _, checkErr = tk2.Exec("delete from t where c1 = 4;")
- }
- assert.NoError(t, dom.Reload())
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(callback)
- tk.MustExec("alter table t add primary key idx(c1);")
- dom.DDL().SetHook(originHook)
- require.True(t, backfillDone)
- require.True(t, runDML)
- require.NoError(t, checkErr)
- tk.MustExec("admin check table t;")
- tk.MustQuery("select * from t use index (primary);").Check(testkit.Rows("1 2 3"))
- tk.MustQuery("select * from t ignore index (primary);").Check(testkit.Rows("1 2 3"))
-}
-
-func TestAddIndexMergeVersionIndexValue(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk2 := testkit.NewTestKit(t, store)
- tk2.MustExec("use test")
- tk.MustExec("create table t (c1 int);")
- // Force onCreateIndex use the txn-merge process.
- ingest.LitInitialized = false
- tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = 1;")
-
- var checkErr error
- var runDML bool
- var tblID, idxID int64
- originHook := dom.DDL().GetHook()
- callback := &ddl.TestDDLCallback{
- Do: dom,
- }
- onJobUpdatedExportedFunc := func(job *model.Job) {
- if !runDML && job.Type == model.ActionAddIndex && job.SchemaState == model.StateWriteReorganization {
- idx := findIdxInfo(dom, "test", "t", "idx")
- if idx == nil || idx.BackfillState != model.BackfillStateReadyToMerge {
- return
- }
- runDML = true
- tblID = job.TableID
- idxID = idx.ID
- _, checkErr = tk2.Exec("insert into t values (1);")
- }
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(callback)
- tk.MustExec("alter table t add unique index idx(c1);")
- dom.DDL().SetHook(originHook)
- require.True(t, runDML)
- require.NoError(t, checkErr)
- tk.MustExec("admin check table t;")
- tk.MustQuery("select * from t use index (idx);").Check(testkit.Rows("1"))
- tk.MustQuery("select * from t ignore index (idx);").Check(testkit.Rows("1"))
-
- snap := store.GetSnapshot(kv.MaxVersion)
- iter, err := snap.Iter(tablecodec.GetTableIndexKeyRange(tblID, idxID))
- require.NoError(t, err)
- require.True(t, iter.Valid())
- // The origin index value should not have 'm' version appended.
- require.Equal(t, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, iter.Value())
-}
-
-func findIdxInfo(dom *domain.Domain, dbName, tbName, idxName string) *model.IndexInfo {
- tbl, err := dom.InfoSchema().TableByName(model.NewCIStr(dbName), model.NewCIStr(tbName))
- if err != nil {
- logutil.BgLogger().Warn("cannot find table", zap.String("dbName", dbName), zap.String("tbName", tbName))
- return nil
- }
- return tbl.Meta().FindIndexByName(idxName)
-}
-
-func TestPessimisticAmendIncompatibleWithFastReorg(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk.MustExec("set global tidb_ddl_enable_fast_reorg = 1;")
- tk.MustExec("set global tidb_ddl_enable_fast_reorg = 1;")
-
- tk.MustGetErrMsg("set @@tidb_enable_amend_pessimistic_txn = 1;",
- "amend pessimistic transactions is not compatible with tidb_ddl_enable_fast_reorg")
-}
diff --git a/ddl/index_modify_test.go b/ddl/index_modify_test.go
index c24f2dfb1b783..2caf54c31c157 100644
--- a/ddl/index_modify_test.go
+++ b/ddl/index_modify_test.go
@@ -416,7 +416,7 @@ func testAddIndexRollback(t *testing.T, idxName, addIdxSQL, errMsg string, hasNu
}
done := make(chan error, 1)
- go backgroundExec(store, addIdxSQL, done)
+ go backgroundExec(store, "test", addIdxSQL, done)
times := 0
ticker := time.NewTicker(indexModifyLease / 2)
diff --git a/ddl/indexmergetest/BUILD.bazel b/ddl/indexmergetest/BUILD.bazel
new file mode 100644
index 0000000000000..b70146ae8d461
--- /dev/null
+++ b/ddl/indexmergetest/BUILD.bazel
@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+ name = "indexmergetest_test",
+ timeout = "moderate",
+ srcs = [
+ "main_test.go",
+ "merge_test.go",
+ ],
+ flaky = True,
+ race = "on",
+ shard_count = 4,
+ deps = [
+ "//config",
+ "//ddl",
+ "//ddl/ingest",
+ "//ddl/internal/callback",
+ "//ddl/testutil",
+ "//domain",
+ "//errno",
+ "//kv",
+ "//meta/autoid",
+ "//parser/model",
+ "//tablecodec",
+ "//testkit",
+ "//testkit/testsetup",
+ "@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_stretchr_testify//assert",
+ "@com_github_stretchr_testify//require",
+ "@com_github_tikv_client_go_v2//tikv",
+ "@org_uber_go_goleak//:goleak",
+ ],
+)
diff --git a/ddl/indexmergetest/main_test.go b/ddl/indexmergetest/main_test.go
new file mode 100644
index 0000000000000..b4de8700ce167
--- /dev/null
+++ b/ddl/indexmergetest/main_test.go
@@ -0,0 +1,56 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package indexmergetest
+
+import (
+ "testing"
+ "time"
+
+ "github.com/pingcap/tidb/config"
+ "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/domain"
+ "github.com/pingcap/tidb/meta/autoid"
+ "github.com/pingcap/tidb/testkit/testsetup"
+ "github.com/tikv/client-go/v2/tikv"
+ "go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+ testsetup.SetupForCommonTest()
+ tikv.EnableFailpoints()
+
+ domain.SchemaOutOfDateRetryInterval.Store(50 * time.Millisecond)
+ domain.SchemaOutOfDateRetryTimes.Store(50)
+
+ autoid.SetStep(5000)
+ ddl.RunInGoTest = true
+
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.Instance.SlowThreshold = 10000
+ conf.TiKVClient.AsyncCommit.SafeWindow = 0
+ conf.TiKVClient.AsyncCommit.AllowedClockDrift = 0
+ conf.Experimental.AllowsExpressionIndex = true
+ })
+
+ opts := []goleak.Option{
+ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
+ goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
+ goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
+ goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"),
+ goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
+ }
+
+ goleak.VerifyTestMain(m, opts...)
+}
diff --git a/ddl/indexmergetest/merge_test.go b/ddl/indexmergetest/merge_test.go
new file mode 100644
index 0000000000000..f74db4e0b9eb9
--- /dev/null
+++ b/ddl/indexmergetest/merge_test.go
@@ -0,0 +1,860 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package indexmergetest
+
+import (
+ "testing"
+ "time"
+
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/ingest"
+ "github.com/pingcap/tidb/ddl/internal/callback"
+ "github.com/pingcap/tidb/ddl/testutil"
+ "github.com/pingcap/tidb/errno"
+ "github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/tablecodec"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestAddIndexMergeProcess(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk.MustExec("create table t (c1 int primary key, c2 int, c3 int)")
+ tk.MustExec("insert into t values (1, 2, 3), (4, 5, 6);")
+ // Force onCreateIndex use the txn-merge process.
+ ingest.LitInitialized = false
+ tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = 1;")
+
+ var checkErr error
+ var runDML, backfillDone bool
+ originHook := dom.DDL().GetHook()
+ callback := &callback.TestDDLCallback{
+ Do: dom,
+ }
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if !runDML && job.Type == model.ActionAddIndex && job.SchemaState == model.StateWriteReorganization {
+ idx := testutil.FindIdxInfo(dom, "test", "t", "idx")
+ if idx == nil || idx.BackfillState != model.BackfillStateRunning {
+ return
+ }
+ if !backfillDone {
+ // Wait another round so that the backfill range is determined(1-4).
+ backfillDone = true
+ return
+ }
+ runDML = true
+ // Write record 7 to the temporary index.
+ _, checkErr = tk2.Exec("insert into t values (7, 8, 9);")
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ dom.DDL().SetHook(callback)
+ tk.MustExec("alter table t add index idx(c1);")
+ dom.DDL().SetHook(originHook)
+ require.True(t, backfillDone)
+ require.True(t, runDML)
+ require.NoError(t, checkErr)
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t use index (idx);").Check(testkit.Rows("1 2 3", "4 5 6", "7 8 9"))
+ tk.MustQuery("select * from t ignore index (idx);").Check(testkit.Rows("1 2 3", "4 5 6", "7 8 9"))
+}
+
+func TestAddPrimaryKeyMergeProcess(t *testing.T) {
+ // Disable auto schema reload.
+ store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 0)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk.MustExec("create table t (c1 int, c2 int, c3 int)")
+ tk.MustExec("insert into t values (1, 2, 3), (4, 5, 6);")
+ // Force onCreateIndex use the backfill-merge process.
+ ingest.LitInitialized = false
+ tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = 1;")
+
+ var checkErr error
+ var runDML, backfillDone bool
+ originHook := dom.DDL().GetHook()
+ callback := &callback.TestDDLCallback{
+ Do: nil, // We'll reload the schema manually.
+
+ }
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if !runDML && job.Type == model.ActionAddPrimaryKey && job.SchemaState == model.StateWriteReorganization {
+ idx := testutil.FindIdxInfo(dom, "test", "t", "primary")
+ if idx == nil || idx.BackfillState != model.BackfillStateRunning || job.SnapshotVer == 0 {
+ return
+ }
+ if !backfillDone {
+ // Wait another round so that the backfill process is finished, but
+ // the info schema is not updated.
+ backfillDone = true
+ return
+ }
+ runDML = true
+ // Add delete record 4 to the temporary index.
+ _, checkErr = tk2.Exec("delete from t where c1 = 4;")
+ }
+ assert.NoError(t, dom.Reload())
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ dom.DDL().SetHook(callback)
+ tk.MustExec("alter table t add primary key idx(c1);")
+ dom.DDL().SetHook(originHook)
+ require.True(t, backfillDone)
+ require.True(t, runDML)
+ require.NoError(t, checkErr)
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t use index (primary);").Check(testkit.Rows("1 2 3"))
+ tk.MustQuery("select * from t ignore index (primary);").Check(testkit.Rows("1 2 3"))
+}
+
+func TestAddIndexMergeVersionIndexValue(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk.MustExec("create table t (c1 int);")
+ // Force onCreateIndex use the txn-merge process.
+ ingest.LitInitialized = false
+ tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = 1;")
+
+ var checkErr error
+ var runDML bool
+ var tblID, idxID int64
+ originHook := dom.DDL().GetHook()
+ callback := &callback.TestDDLCallback{
+ Do: dom,
+ }
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if !runDML && job.Type == model.ActionAddIndex && job.SchemaState == model.StateWriteReorganization {
+ idx := testutil.FindIdxInfo(dom, "test", "t", "idx")
+ if idx == nil || idx.BackfillState != model.BackfillStateReadyToMerge {
+ return
+ }
+ runDML = true
+ tblID = job.TableID
+ idxID = idx.ID
+ _, checkErr = tk2.Exec("insert into t values (1);")
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ dom.DDL().SetHook(callback)
+ tk.MustExec("alter table t add unique index idx(c1);")
+ dom.DDL().SetHook(originHook)
+ require.True(t, runDML)
+ require.NoError(t, checkErr)
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t use index (idx);").Check(testkit.Rows("1"))
+ tk.MustQuery("select * from t ignore index (idx);").Check(testkit.Rows("1"))
+
+ snap := store.GetSnapshot(kv.MaxVersion)
+ iter, err := snap.Iter(tablecodec.GetTableIndexKeyRange(tblID, idxID))
+ require.NoError(t, err)
+ require.True(t, iter.Valid())
+ // The origin index value should not have 'm' version appended.
+ require.Equal(t, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, iter.Value())
+}
+
+func TestAddIndexMergeIndexUntouchedValue(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk.MustExec(`create table t (
+ id int not null auto_increment,
+ k int not null default '0',
+ c char(120) not null default '',
+ pad char(60) not null default '',
+ primary key (id) clustered,
+ key k_1(k));`)
+ tk.MustExec("insert into t values (1, 1, 'a', 'a')")
+ // Force onCreateIndex use the txn-merge process.
+ ingest.LitInitialized = false
+ tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = 1;")
+
+ var checkErrs []error
+ var runInsert bool
+ var runUpdate bool
+ originHook := dom.DDL().GetHook()
+ callback := &callback.TestDDLCallback{
+ Do: dom,
+ }
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if job.Type != model.ActionAddIndex || job.SchemaState != model.StateWriteReorganization {
+ return
+ }
+ idx := testutil.FindIdxInfo(dom, "test", "t", "idx")
+ if idx == nil {
+ return
+ }
+ if !runInsert {
+ if idx.BackfillState != model.BackfillStateRunning || job.SnapshotVer == 0 {
+ return
+ }
+ runInsert = true
+ _, err := tk2.Exec("insert into t values (100, 1, 'a', 'a');")
+ checkErrs = append(checkErrs, err)
+ }
+ if !runUpdate {
+ if idx.BackfillState != model.BackfillStateReadyToMerge {
+ return
+ }
+ runUpdate = true
+ _, err := tk2.Exec("begin;")
+ checkErrs = append(checkErrs, err)
+ _, err = tk2.Exec("update t set k=k+1 where id = 100;")
+ checkErrs = append(checkErrs, err)
+ _, err = tk2.Exec("commit;")
+ checkErrs = append(checkErrs, err)
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ dom.DDL().SetHook(callback)
+ tk.MustExec("alter table t add index idx(c);")
+ dom.DDL().SetHook(originHook)
+ require.True(t, runUpdate)
+ for _, err := range checkErrs {
+ require.NoError(t, err)
+ }
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t use index (idx);").Check(testkit.Rows("1 1 a a", "100 2 a a"))
+ tk.MustQuery("select * from t ignore index (idx);").Check(testkit.Rows("1 1 a a", "100 2 a a"))
+}
+
+// TestCreateUniqueIndexKeyExist this case will test below things:
+// Create one unique index idx((a*b+1));
+// insert (0, 6) and delete it;
+// insert (0, 9), it should be successful;
+// Should check temp key exist and skip deleted mark
+// The error returned below:
+// Error: Received unexpected error:
+//
+// [kv:1062]Duplicate entry '1' for key 't.idx'
+func TestCreateUniqueIndexKeyExist(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int default 0, b int default 0)")
+ tk.MustExec("insert into t values (1, 1), (2, 2), (3, 3), (4, 4)")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ stateDeleteOnlySQLs := []string{"insert into t values (5, 5)", "begin pessimistic;", "insert into t select * from t", "rollback", "insert into t set b = 6", "update t set b = 7 where a = 1", "delete from t where b = 4"}
+
+ // If waitReorg timeout, the worker may enter writeReorg more than 2 times.
+ reorgTime := 0
+ d := dom.DDL()
+ originalCallback := d.GetHook()
+ defer d.SetHook(originalCallback)
+ callback := &callback.TestDDLCallback{}
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if t.Failed() {
+ return
+ }
+ var err error
+ switch job.SchemaState {
+ case model.StateDeleteOnly:
+ for _, sql := range stateDeleteOnlySQLs {
+ _, err = tk1.Exec(sql)
+ assert.NoError(t, err)
+ }
+ // (1, 7), (2, 2), (3, 3), (5, 5), (0, 6)
+ case model.StateWriteOnly:
+ _, err = tk1.Exec("insert into t values (8, 8)")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("update t set b = 7 where a = 2")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("delete from t where b = 3")
+ assert.NoError(t, err)
+ // (1, 7), (2, 7), (5, 5), (0, 6), (8, 8)
+ case model.StateWriteReorganization:
+ if reorgTime < 1 {
+ reorgTime++
+ } else {
+ return
+ }
+ _, err = tk1.Exec("insert into t values (10, 10)")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("delete from t where b = 6")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("insert into t set b = 9")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("update t set b = 7 where a = 5")
+ assert.NoError(t, err)
+ // (1, 7), (2, 7), (5, 7), (8, 8), (10, 10), (0, 9)
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ d.SetHook(callback)
+ tk.MustExec("alter table t add unique index idx((a*b+1))")
+ tk.MustExec("admin check table t")
+ tk.MustQuery("select * from t order by a, b").Check(testkit.Rows("0 9", "1 7", "2 7", "5 7", "8 8", "10 10"))
+}
+
+func TestAddIndexMergeIndexUpdateOnDeleteOnly(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk.MustExec(`CREATE TABLE t (a DATE NULL DEFAULT '1619-01-18', b BOOL NULL DEFAULT '0') CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_bin';`)
+ tk.MustExec(`INSERT INTO t SET b = '1';`)
+
+ updateSQLs := []string{
+ "UPDATE t SET a = '9432-05-10', b = '0';",
+ "UPDATE t SET a = '9432-05-10', b = '1';",
+ }
+
+ // Force onCreateIndex use the txn-merge process.
+ ingest.LitInitialized = false
+ tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = 1;")
+ tk.MustExec("set @@global.tidb_enable_mutation_checker = 1;")
+ tk.MustExec("set @@global.tidb_txn_assertion_level = 'STRICT';")
+
+ var checkErrs []error
+ originHook := dom.DDL().GetHook()
+ callback := &callback.TestDDLCallback{
+ Do: dom,
+ }
+ onJobUpdatedBefore := func(job *model.Job) {
+ if job.SchemaState == model.StateDeleteOnly {
+ for _, sql := range updateSQLs {
+ _, err := tk2.Exec(sql)
+ if err != nil {
+ checkErrs = append(checkErrs, err)
+ }
+ }
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedBefore)
+ dom.DDL().SetHook(callback)
+ tk.MustExec("alter table t add index idx(b);")
+ dom.DDL().SetHook(originHook)
+ for _, err := range checkErrs {
+ require.NoError(t, err)
+ }
+ tk.MustExec("admin check table t;")
+}
+
+func TestAddIndexMergeDeleteUniqueOnWriteOnly(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int default 0, b int default 0);")
+ tk.MustExec("insert into t values (1, 1), (2, 2), (3, 3), (4, 4);")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ d := dom.DDL()
+ originalCallback := d.GetHook()
+ defer d.SetHook(originalCallback)
+ callback := &callback.TestDDLCallback{}
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if t.Failed() {
+ return
+ }
+ var err error
+ switch job.SchemaState {
+ case model.StateDeleteOnly:
+ _, err = tk1.Exec("insert into t values (5, 5);")
+ assert.NoError(t, err)
+ case model.StateWriteOnly:
+ _, err = tk1.Exec("insert into t values (5, 7);")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("delete from t where b = 7;")
+ assert.NoError(t, err)
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ d.SetHook(callback)
+ tk.MustExec("alter table t add unique index idx(a);")
+ tk.MustExec("admin check table t;")
+}
+
+func TestAddIndexMergeDeleteNullUnique(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(id int primary key, a int default 0);")
+ tk.MustExec("insert into t values (1, 1), (2, null);")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ ddl.MockDMLExecution = func() {
+ _, err := tk1.Exec("delete from t where id = 2;")
+ assert.NoError(t, err)
+ }
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockDMLExecution", "1*return(true)->return(false)"))
+ tk.MustExec("alter table t add unique index idx(a);")
+ tk.MustQuery("select count(1) from t;").Check(testkit.Rows("1"))
+ tk.MustExec("admin check table t;")
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockDMLExecution"))
+}
+
+func TestAddIndexMergeDoubleDelete(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(id int primary key, a int default 0);")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ d := dom.DDL()
+ originalCallback := d.GetHook()
+ defer d.SetHook(originalCallback)
+ callback := &callback.TestDDLCallback{}
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if t.Failed() {
+ return
+ }
+ switch job.SchemaState {
+ case model.StateWriteOnly:
+ _, err := tk1.Exec("insert into t values (1, 1);")
+ assert.NoError(t, err)
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ d.SetHook(callback)
+
+ ddl.MockDMLExecution = func() {
+ _, err := tk1.Exec("delete from t where id = 1;")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("insert into t values (2, 1);")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("delete from t where id = 2;")
+ assert.NoError(t, err)
+ }
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockDMLExecution", "1*return(true)->return(false)"))
+ tk.MustExec("alter table t add unique index idx(a);")
+ tk.MustQuery("select count(1) from t;").Check(testkit.Rows("0"))
+ tk.MustExec("admin check table t;")
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockDMLExecution"))
+}
+
+func TestAddIndexMergeConflictWithPessimistic(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk.MustExec(`CREATE TABLE t (id int primary key, a int);`)
+ tk.MustExec(`INSERT INTO t VALUES (1, 1);`)
+
+ // Make shorten the conversion time from ReorgTypeLitMerge to BackfillStateReadyToMerge.
+ interval := ddl.CheckBackfillJobFinishInterval
+ ddl.CheckBackfillJobFinishInterval = 50 * time.Millisecond
+ defer func() { ddl.CheckBackfillJobFinishInterval = interval }()
+
+ // Force onCreateIndex use the txn-merge process.
+ ingest.LitInitialized = false
+ tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = 1;")
+ tk.MustExec("set @@global.tidb_enable_metadata_lock = 0;")
+
+ originHook := dom.DDL().GetHook()
+ callback := &callback.TestDDLCallback{Do: dom}
+
+ runPessimisticTxn := false
+ afterPessDML := make(chan struct{}, 1)
+ callback.OnJobRunBeforeExported = func(job *model.Job) {
+ if t.Failed() {
+ return
+ }
+ if job.SchemaState == model.StateWriteOnly {
+ // Write a record to the temp index.
+ _, err := tk2.Exec("update t set a = 2 where id = 1;")
+ assert.NoError(t, err)
+ }
+ if !runPessimisticTxn && job.SchemaState == model.StateWriteReorganization {
+ idx := testutil.FindIdxInfo(dom, "test", "t", "idx")
+ if idx == nil {
+ return
+ }
+ if idx.BackfillState != model.BackfillStateReadyToMerge {
+ return
+ }
+ runPessimisticTxn = true
+ _, err := tk2.Exec("begin pessimistic;")
+ assert.NoError(t, err)
+ _, err = tk2.Exec("update t set a = 3 where id = 1;")
+ assert.NoError(t, err)
+ afterPessDML <- struct{}{}
+ }
+ }
+ dom.DDL().SetHook(callback)
+ afterCommit := make(chan struct{}, 1)
+ go func() {
+ tk.MustExec("alter table t add index idx(a);")
+ afterCommit <- struct{}{}
+ }()
+ timer := time.NewTimer(300 * time.Millisecond)
+ select {
+ case <-timer.C:
+ break
+ case <-afterCommit:
+ require.Fail(t, "should be blocked by the pessimistic txn")
+ }
+ <-afterPessDML
+ tk2.MustExec("rollback;")
+ <-afterCommit
+ dom.DDL().SetHook(originHook)
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("1 2"))
+}
+
+func TestAddIndexMergeInsertOnMerging(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int default 0, b int default 0)")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ d := dom.DDL()
+ originalCallback := d.GetHook()
+ defer d.SetHook(originalCallback)
+ callback := &callback.TestDDLCallback{}
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if t.Failed() {
+ return
+ }
+ var err error
+ switch job.SchemaState {
+ case model.StateDeleteOnly:
+ _, err = tk1.Exec("insert into t values (5, 5)")
+ assert.NoError(t, err)
+ case model.StateWriteOnly:
+ _, err = tk1.Exec("insert into t values (5, 7)")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("delete from t where b = 7")
+ assert.NoError(t, err)
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ d.SetHook(callback)
+
+ ddl.MockDMLExecutionStateMerging = func() {
+ _, err := tk1.Exec("insert into t values (5, 8);")
+ assert.Error(t, err) // [kv:1062]Duplicate entry '5' for key 't.idx'
+ _, err = tk1.Exec("insert into t values (5, 8) on duplicate key update a = 6;")
+ assert.NoError(t, err) // The row should be normally updated to (6, 5).
+ ddl.MockDMLExecutionStateMerging = nil
+ }
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockDMLExecutionStateMerging", "return(true)"))
+ tk.MustExec("alter table t add unique index idx(a);")
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("6 5"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockDMLExecutionStateMerging"))
+}
+
+func TestAddIndexMergeReplaceOnMerging(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int default 0, b int default 0);")
+ tk.MustExec("insert into t values (5, 5);")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ ddl.MockDMLExecution = func() {
+ _, err := tk1.Exec("delete from t where b = 5;")
+ assert.NoError(t, err)
+ }
+
+ ddl.MockDMLExecutionStateMerging = func() {
+ _, err := tk1.Exec("replace into t values (5, 8);")
+ assert.NoError(t, err)
+ ddl.MockDMLExecutionStateMerging = nil
+ }
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockDMLExecution", "1*return(true)->return(false)"))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockDMLExecutionStateMerging", "return(true)"))
+ tk.MustExec("alter table t add unique index idx(a);")
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("5 8"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockDMLExecution"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockDMLExecutionStateMerging"))
+}
+
+func TestAddIndexMergeInsertToDeletedTempIndex(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int default 0, b int default 0)")
+ tk.MustExec("insert into t values (5, 5);")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ d := dom.DDL()
+ originalCallback := d.GetHook()
+ defer d.SetHook(originalCallback)
+ callback := &callback.TestDDLCallback{}
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if t.Failed() {
+ return
+ }
+ var err error
+ switch job.SchemaState {
+ case model.StateWriteOnly:
+ _, err = tk1.Exec("delete from t where b = 5")
+ assert.NoError(t, err)
+ _, err := tk1.Exec("set @@tidb_constraint_check_in_place = true;")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("insert into t values (5, 8);")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("insert into t values (5, 8);")
+ assert.Error(t, err)
+ _, err = tk1.Exec("set @@tidb_constraint_check_in_place = false;")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("insert into t values (5, 8);")
+ assert.Error(t, err)
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ d.SetHook(callback)
+
+ tk.MustExec("alter table t add unique index idx(a);")
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("5 8"))
+}
+
+func TestAddIndexMergeReplaceDelete(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(id int primary key, a int default 0);")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ d := dom.DDL()
+ originalCallback := d.GetHook()
+ defer d.SetHook(originalCallback)
+ callback := &callback.TestDDLCallback{}
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if t.Failed() {
+ return
+ }
+ switch job.SchemaState {
+ case model.StateDeleteOnly:
+ _, err := tk1.Exec("insert into t values (1, 1);")
+ assert.NoError(t, err)
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ d.SetHook(callback)
+
+ ddl.MockDMLExecutionMerging = func() {
+ _, err := tk1.Exec("replace into t values (2, 1);")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("delete from t where id = 2;")
+ assert.NoError(t, err)
+ }
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockDMLExecutionMerging", "1*return(true)->return(false)"))
+ tk.MustExec("alter table t add unique index idx(a);")
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t;").Check(testkit.Rows())
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockDMLExecutionMerging"))
+}
+
+func TestAddIndexMergeDeleteDifferentHandle(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(id int primary key, c char(10));")
+ tk.MustExec("insert into t values (1, 'a');")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ d := dom.DDL()
+ originalCallback := d.GetHook()
+ defer d.SetHook(originalCallback)
+ callback := &callback.TestDDLCallback{}
+ runDML := false
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if t.Failed() || runDML {
+ return
+ }
+ if job.SnapshotVer == 0 {
+ return
+ }
+ switch job.SchemaState {
+ case model.StateWriteReorganization:
+ _, err := tk1.Exec("insert into t values (2, 'a');")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("replace into t values (3, 'a');")
+ assert.NoError(t, err)
+ runDML = true
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ d.SetHook(callback)
+
+ ddl.MockDMLExecution = func() {
+ // It is too late to remove the duplicated index value.
+ _, err := tk1.Exec("delete from t where id = 1;")
+ assert.NoError(t, err)
+ }
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockDMLExecution", "1*return(true)->return(false)"))
+ tk.MustGetErrCode("alter table t add unique index idx(c);", errno.ErrDupEntry)
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("3 a"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockDMLExecution"))
+}
+
+func TestAddIndexDecodeTempIndexCommonHandle(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(id_a bigint, id_b char(20), c char(20), primary key (id_a, id_b));")
+ tk.MustExec("insert into t values (1, 'id_1', 'char_1');")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ d := dom.DDL()
+ originalCallback := d.GetHook()
+ defer d.SetHook(originalCallback)
+ callback := &callback.TestDDLCallback{}
+ runDML := false
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if t.Failed() || runDML {
+ return
+ }
+ if job.SnapshotVer == 0 {
+ return
+ }
+ switch job.SchemaState {
+ case model.StateWriteReorganization:
+ _, err := tk1.Exec("insert into t values (2, 'id_2', 'char_2');")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("insert into t values (3, 'id_3', 'char_3');")
+ assert.NoError(t, err)
+ runDML = true
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ d.SetHook(callback)
+
+ tk.MustExec("alter table t add unique index idx(c);")
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("1 id_1 char_1", "2 id_2 char_2", "3 id_3 char_3"))
+}
+
+func TestAddIndexInsertIgnoreOnBackfill(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(id int primary key, b int);")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ d := dom.DDL()
+ originalCallback := d.GetHook()
+ defer d.SetHook(originalCallback)
+ callback := &callback.TestDDLCallback{}
+ runDML := false
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if t.Failed() || runDML {
+ return
+ }
+ switch job.SchemaState {
+ case model.StateWriteReorganization:
+ _, err := tk1.Exec("insert ignore into t values (1, 1);")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("insert ignore into t values (2, 2);")
+ assert.NoError(t, err)
+ _, err = tk1.Exec("update t set b = null where id = 1;")
+ assert.NoError(t, err)
+ runDML = true
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ d.SetHook(callback)
+
+ tk.MustExec("alter table t add unique index idx(b);")
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("1 ", "2 2"))
+}
+
+func TestAddIndexMultipleDelete(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(id int primary key, b int);")
+ tk.MustExec("insert into t values (1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1);")
+
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+
+ d := dom.DDL()
+ originalCallback := d.GetHook()
+ defer d.SetHook(originalCallback)
+ callback := &callback.TestDDLCallback{}
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ if t.Failed() {
+ return
+ }
+ switch job.SchemaState {
+ case model.StateDeleteOnly:
+ _, err := tk1.Exec("delete from t where id in (4, 5, 6);")
+ assert.NoError(t, err)
+ case model.StateWriteOnly:
+ _, err := tk1.Exec("delete from t where id in (2, 3);")
+ assert.NoError(t, err)
+ }
+ }
+ callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ d.SetHook(callback)
+
+ ddl.MockDMLExecution = func() {
+ _, err := tk1.Exec("delete from t where id = 1;")
+ assert.NoError(t, err)
+ }
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockDMLExecution", "1*return(true)->return(false)"))
+ tk.MustExec("alter table t add unique index idx(b);")
+ tk.MustExec("admin check table t;")
+ tk.MustQuery("select * from t;").Check(testkit.Rows())
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockDMLExecution"))
+}
diff --git a/ddl/ingest/BUILD.bazel b/ddl/ingest/BUILD.bazel
index 3fd286e450b25..902900c316a12 100644
--- a/ddl/ingest/BUILD.bazel
+++ b/ddl/ingest/BUILD.bazel
@@ -33,19 +33,20 @@ go_library(
"//sessionctx/variable",
"//table",
"//util",
+ "//util/dbterror",
"//util/generic",
"//util/logutil",
"//util/mathutil",
"//util/size",
"@com_github_google_uuid//:uuid",
"@com_github_pingcap_errors//:errors",
- "@com_github_pkg_errors//:errors",
"@org_uber_go_zap//:zap",
],
)
go_test(
name = "ingest_test",
+ timeout = "short",
srcs = [
"env_test.go",
"mem_root_test.go",
diff --git a/ddl/ingest/backend.go b/ddl/ingest/backend.go
index 63034f0be3a22..a94639d89d6bb 100644
--- a/ddl/ingest/backend.go
+++ b/ddl/ingest/backend.go
@@ -17,13 +17,13 @@ package ingest
import (
"context"
- "github.com/pingcap/errors"
"github.com/pingcap/tidb/br/pkg/lightning/backend"
"github.com/pingcap/tidb/br/pkg/lightning/backend/kv"
- "github.com/pingcap/tidb/br/pkg/lightning/config"
+ lightning "github.com/pingcap/tidb/br/pkg/lightning/config"
tikv "github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/logutil"
"go.uber.org/zap"
)
@@ -33,7 +33,7 @@ type BackendContext struct {
jobID int64
backend *backend.Backend
ctx context.Context
- cfg *config.Config
+ cfg *lightning.Config
EngMgr engineManager
sysVars map[string]string
diskRoot DiskRoot
@@ -45,7 +45,7 @@ type BackendContext struct {
func (bc *BackendContext) FinishImport(indexID int64, unique bool, tbl table.Table) error {
ei, exist := bc.EngMgr.Load(indexID)
if !exist {
- return errors.New(LitErrGetEngineFail)
+ return dbterror.ErrIngestFailed.FastGenByArgs("ingest engine not found")
}
err := ei.ImportAndClean()
@@ -63,7 +63,7 @@ func (bc *BackendContext) FinishImport(indexID int64, unique bool, tbl table.Tab
if err != nil {
logutil.BgLogger().Error(LitInfoRemoteDupCheck, zap.Error(err),
zap.String("table", tbl.Meta().Name.O), zap.Int64("index ID", indexID))
- return errors.New(LitInfoRemoteDupCheck)
+ return err
} else if hasDupe {
logutil.BgLogger().Error(LitErrRemoteDupExistErr,
zap.String("table", tbl.Meta().Name.O), zap.Int64("index ID", indexID))
@@ -80,7 +80,7 @@ func (bc *BackendContext) Flush(indexID int64) error {
ei, exist := bc.EngMgr.Load(indexID)
if !exist {
logutil.BgLogger().Error(LitErrGetEngineFail, zap.Int64("index ID", indexID))
- return errors.New(LitErrGetEngineFail)
+ return dbterror.ErrIngestFailed.FastGenByArgs("ingest engine not found")
}
err := bc.diskRoot.UpdateUsageAndQuota()
@@ -99,7 +99,7 @@ func (bc *BackendContext) Flush(indexID int64) error {
logutil.BgLogger().Info(LitInfoUnsafeImport, zap.Int64("index ID", indexID),
zap.Uint64("current disk usage", bc.diskRoot.CurrentUsage()),
zap.Uint64("max disk quota", bc.diskRoot.MaxQuota()))
- err = bc.backend.UnsafeImportAndReset(bc.ctx, ei.uuid, int64(config.SplitRegionSize)*int64(config.MaxSplitRegionSizeRatio), int64(config.SplitRegionKeys))
+ err = bc.backend.UnsafeImportAndReset(bc.ctx, ei.uuid, int64(lightning.SplitRegionSize)*int64(lightning.MaxSplitRegionSizeRatio), int64(lightning.SplitRegionKeys))
if err != nil {
logutil.BgLogger().Error(LitErrIngestDataErr, zap.Int64("index ID", indexID),
zap.Error(err), zap.Uint64("current disk usage", bc.diskRoot.CurrentUsage()),
diff --git a/ddl/ingest/backend_mgr.go b/ddl/ingest/backend_mgr.go
index 14bb4fb3aa67a..49788c9760b58 100644
--- a/ddl/ingest/backend_mgr.go
+++ b/ddl/ingest/backend_mgr.go
@@ -56,7 +56,7 @@ func (m *backendCtxManager) Register(ctx context.Context, unique bool, jobID int
if !ok {
return nil, genBackendAllocMemFailedErr(m.memRoot, jobID)
}
- cfg, err := generateLightningConfig(m.memRoot, jobID, unique)
+ cfg, err := genConfig(m.memRoot, jobID, unique)
if err != nil {
logutil.BgLogger().Warn(LitWarnConfigError, zap.Int64("job ID", jobID), zap.Error(err))
return nil, err
@@ -67,7 +67,7 @@ func (m *backendCtxManager) Register(ctx context.Context, unique bool, jobID int
return nil, err
}
- bcCtx := newBackendContext(ctx, jobID, &bd, cfg, defaultImportantVariables, m.memRoot, m.diskRoot)
+ bcCtx := newBackendContext(ctx, jobID, &bd, cfg.Lightning, defaultImportantVariables, m.memRoot, m.diskRoot)
m.Store(jobID, bcCtx)
m.memRoot.Consume(StructSizeBackendCtx)
@@ -80,15 +80,16 @@ func (m *backendCtxManager) Register(ctx context.Context, unique bool, jobID int
return bc, nil
}
-func createLocalBackend(ctx context.Context, cfg *config.Config, glue glue.Glue) (backend.Backend, error) {
- tls, err := cfg.ToTLS()
+func createLocalBackend(ctx context.Context, cfg *Config, glue glue.Glue) (backend.Backend, error) {
+ tls, err := cfg.Lightning.ToTLS()
if err != nil {
logutil.BgLogger().Error(LitErrCreateBackendFail, zap.Error(err))
return backend.Backend{}, err
}
- errorMgr := errormanager.New(nil, cfg, log.Logger{Logger: logutil.BgLogger()})
- return local.NewLocalBackend(ctx, tls, cfg, glue, int(LitRLimit), errorMgr)
+ logutil.BgLogger().Info("[ddl-ingest] create local backend for adding index", zap.String("keyspaceName", cfg.KeyspaceName))
+ errorMgr := errormanager.New(nil, cfg.Lightning, log.Logger{Logger: logutil.BgLogger()})
+ return local.NewLocalBackend(ctx, tls, cfg.Lightning, glue, int(LitRLimit), errorMgr, cfg.KeyspaceName)
}
func newBackendContext(ctx context.Context, jobID int64, be *backend.Backend,
diff --git a/ddl/ingest/config.go b/ddl/ingest/config.go
index 3a96e8ae5201b..7362c2fbc5823 100644
--- a/ddl/ingest/config.go
+++ b/ddl/ingest/config.go
@@ -16,22 +16,35 @@ package ingest
import (
"path/filepath"
+ "sync/atomic"
"github.com/pingcap/tidb/br/pkg/lightning/backend"
"github.com/pingcap/tidb/br/pkg/lightning/checkpoints"
- "github.com/pingcap/tidb/br/pkg/lightning/config"
- tidbconf "github.com/pingcap/tidb/config"
+ lightning "github.com/pingcap/tidb/br/pkg/lightning/config"
+ tidb "github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/size"
"go.uber.org/zap"
)
-func generateLightningConfig(memRoot MemRoot, jobID int64, unique bool) (*config.Config, error) {
- tidbCfg := tidbconf.GetGlobalConfig()
- cfg := config.NewConfig()
- cfg.TikvImporter.Backend = config.BackendLocal
+// ImporterRangeConcurrencyForTest is only used for test.
+var ImporterRangeConcurrencyForTest *atomic.Int32
+
+// Config is the configuration for the lightning local backend used in DDL.
+type Config struct {
+ Lightning *lightning.Config
+ KeyspaceName string
+}
+
+func genConfig(memRoot MemRoot, jobID int64, unique bool) (*Config, error) {
+ tidbCfg := tidb.GetGlobalConfig()
+ cfg := lightning.NewConfig()
+ cfg.TikvImporter.Backend = lightning.BackendLocal
// Each backend will build a single dir in lightning dir.
cfg.TikvImporter.SortedKVDir = filepath.Join(LitSortPath, encodeBackendTag(jobID))
+ if ImporterRangeConcurrencyForTest != nil {
+ cfg.TikvImporter.RangeConcurrency = int(ImporterRangeConcurrencyForTest.Load())
+ }
_, err := cfg.AdjustCommon()
if err != nil {
logutil.BgLogger().Warn(LitWarnConfigError, zap.Error(err))
@@ -40,9 +53,9 @@ func generateLightningConfig(memRoot MemRoot, jobID int64, unique bool) (*config
adjustImportMemory(memRoot, cfg)
cfg.Checkpoint.Enable = true
if unique {
- cfg.TikvImporter.DuplicateResolution = config.DupeResAlgRecord
+ cfg.TikvImporter.DuplicateResolution = lightning.DupeResAlgErr
} else {
- cfg.TikvImporter.DuplicateResolution = config.DupeResAlgNone
+ cfg.TikvImporter.DuplicateResolution = lightning.DupeResAlgNone
}
cfg.TiDB.PdAddr = tidbCfg.Path
cfg.TiDB.Host = "127.0.0.1"
@@ -52,7 +65,12 @@ func generateLightningConfig(memRoot MemRoot, jobID int64, unique bool) (*config
cfg.Security.CertPath = tidbCfg.Security.ClusterSSLCert
cfg.Security.KeyPath = tidbCfg.Security.ClusterSSLKey
- return cfg, err
+ c := &Config{
+ Lightning: cfg,
+ KeyspaceName: tidb.GetGlobalKeyspaceName(),
+ }
+
+ return c, err
}
var (
@@ -76,7 +94,7 @@ func generateLocalEngineConfig(id int64, dbName, tbName string) *backend.EngineC
}
// adjustImportMemory adjusts the lightning memory parameters according to the memory root's max limitation.
-func adjustImportMemory(memRoot MemRoot, cfg *config.Config) {
+func adjustImportMemory(memRoot MemRoot, cfg *lightning.Config) {
var scale int64
// Try aggressive resource usage successful.
if tryAggressiveMemory(memRoot, cfg) {
@@ -97,8 +115,8 @@ func adjustImportMemory(memRoot MemRoot, cfg *config.Config) {
return
}
- cfg.TikvImporter.LocalWriterMemCacheSize /= config.ByteSize(scale)
- cfg.TikvImporter.EngineMemCacheSize /= config.ByteSize(scale)
+ cfg.TikvImporter.LocalWriterMemCacheSize /= lightning.ByteSize(scale)
+ cfg.TikvImporter.EngineMemCacheSize /= lightning.ByteSize(scale)
// TODO: adjust range concurrency number to control total concurrency in the future.
logutil.BgLogger().Info(LitInfoChgMemSetting,
zap.Int64("local writer memory cache size", int64(cfg.TikvImporter.LocalWriterMemCacheSize)),
@@ -107,7 +125,7 @@ func adjustImportMemory(memRoot MemRoot, cfg *config.Config) {
}
// tryAggressiveMemory lightning memory parameters according memory root's max limitation.
-func tryAggressiveMemory(memRoot MemRoot, cfg *config.Config) bool {
+func tryAggressiveMemory(memRoot MemRoot, cfg *lightning.Config) bool {
var defaultMemSize int64
defaultMemSize = int64(int(cfg.TikvImporter.LocalWriterMemCacheSize) * cfg.TikvImporter.RangeConcurrency)
defaultMemSize += int64(cfg.TikvImporter.EngineMemCacheSize)
diff --git a/ddl/ingest/disk_root.go b/ddl/ingest/disk_root.go
index c1c98f3fe681a..e2a1176a76878 100644
--- a/ddl/ingest/disk_root.go
+++ b/ddl/ingest/disk_root.go
@@ -15,7 +15,8 @@
package ingest
import (
- "github.com/pingcap/errors"
+ "sync"
+
lcom "github.com/pingcap/tidb/br/pkg/lightning/common"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/util/logutil"
@@ -38,6 +39,7 @@ type diskRootImpl struct {
currentUsage uint64
maxQuota uint64
bcCtx *backendCtxManager
+ mu sync.RWMutex
}
// NewDiskRootImpl creates a new DiskRoot.
@@ -50,22 +52,32 @@ func NewDiskRootImpl(path string, bcCtx *backendCtxManager) DiskRoot {
// CurrentUsage implements DiskRoot interface.
func (d *diskRootImpl) CurrentUsage() uint64 {
- return d.currentUsage
+ d.mu.RLock()
+ usage := d.currentUsage
+ d.mu.RUnlock()
+ return usage
}
// MaxQuota implements DiskRoot interface.
func (d *diskRootImpl) MaxQuota() uint64 {
- return d.maxQuota
+ d.mu.RLock()
+ quota := d.maxQuota
+ d.mu.RUnlock()
+ return quota
}
// UpdateUsageAndQuota implements DiskRoot interface.
func (d *diskRootImpl) UpdateUsageAndQuota() error {
- d.currentUsage = d.bcCtx.TotalDiskUsage()
+ totalDiskUsage := d.bcCtx.TotalDiskUsage()
sz, err := lcom.GetStorageSize(d.path)
if err != nil {
logutil.BgLogger().Error(LitErrGetStorageQuota, zap.Error(err))
- return errors.New(LitErrGetStorageQuota)
+ return err
}
- d.maxQuota = mathutil.Min(variable.DDLDiskQuota.Load(), uint64(capacityThreshold*float64(sz.Capacity)))
+ maxQuota := mathutil.Min(variable.DDLDiskQuota.Load(), uint64(capacityThreshold*float64(sz.Capacity)))
+ d.mu.Lock()
+ d.currentUsage = totalDiskUsage
+ d.maxQuota = maxQuota
+ d.mu.Unlock()
return nil
}
diff --git a/ddl/ingest/engine.go b/ddl/ingest/engine.go
index d875d78e346d0..24779ef9d7718 100644
--- a/ddl/ingest/engine.go
+++ b/ddl/ingest/engine.go
@@ -17,6 +17,7 @@ package ingest
import (
"context"
"strconv"
+ "sync/atomic"
"github.com/google/uuid"
"github.com/pingcap/tidb/br/pkg/lightning/backend"
@@ -25,7 +26,6 @@ import (
"github.com/pingcap/tidb/br/pkg/lightning/config"
"github.com/pingcap/tidb/util/generic"
"github.com/pingcap/tidb/util/logutil"
- "github.com/pkg/errors"
"go.uber.org/zap"
)
@@ -42,6 +42,7 @@ type engineInfo struct {
writerCache generic.SyncMap[int, *backend.LocalEngineWriter]
memRoot MemRoot
diskRoot DiskRoot
+ rowSeq atomic.Int64
}
// NewEngineInfo create a new EngineInfo struct.
@@ -83,6 +84,11 @@ func (ei *engineInfo) Clean() {
zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
}
ei.openedEngine = nil
+ err = ei.closeWriters()
+ if err != nil {
+ logutil.BgLogger().Error(LitErrCloseWriterErr, zap.Error(err),
+ zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
+ }
// Here the local intermediate files will be removed.
err = closedEngine.Cleanup(ei.ctx)
if err != nil {
@@ -99,11 +105,17 @@ func (ei *engineInfo) ImportAndClean() error {
if err1 != nil {
logutil.BgLogger().Error(LitErrCloseEngineErr, zap.Error(err1),
zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
- return errors.New(LitErrCloseEngineErr)
+ return err1
}
ei.openedEngine = nil
+ err := ei.closeWriters()
+ if err != nil {
+ logutil.BgLogger().Error(LitErrCloseWriterErr, zap.Error(err),
+ zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
+ return err
+ }
- err := ei.diskRoot.UpdateUsageAndQuota()
+ err = ei.diskRoot.UpdateUsageAndQuota()
if err != nil {
logutil.BgLogger().Error(LitErrUpdateDiskStats, zap.Error(err),
zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
@@ -118,7 +130,7 @@ func (ei *engineInfo) ImportAndClean() error {
if err != nil {
logutil.BgLogger().Error(LitErrIngestDataErr, zap.Error(err),
zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
- return errors.New(LitErrIngestDataErr)
+ return err
}
// Clean up the engine local workspace.
@@ -126,7 +138,7 @@ func (ei *engineInfo) ImportAndClean() error {
if err != nil {
logutil.BgLogger().Error(LitErrCloseEngineErr, zap.Error(err),
zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
- return errors.New(LitErrCloseEngineErr)
+ return err
}
return nil
}
@@ -134,17 +146,18 @@ func (ei *engineInfo) ImportAndClean() error {
// WriterContext is used to keep a lightning local writer for each backfill worker.
type WriterContext struct {
ctx context.Context
+ rowSeq func() int64
lWrite *backend.LocalEngineWriter
}
-func (ei *engineInfo) NewWriterCtx(id int) (*WriterContext, error) {
+func (ei *engineInfo) NewWriterCtx(id int, unique bool) (*WriterContext, error) {
ei.memRoot.RefreshConsumption()
ok := ei.memRoot.CheckConsume(StructSizeWriterCtx)
if !ok {
return nil, genEngineAllocMemFailedErr(ei.memRoot, ei.jobID, ei.indexID)
}
- wCtx, err := ei.newWriterContext(id)
+ wCtx, err := ei.newWriterContext(id, unique)
if err != nil {
logutil.BgLogger().Error(LitErrCreateContextFail, zap.Error(err),
zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID),
@@ -165,7 +178,7 @@ func (ei *engineInfo) NewWriterCtx(id int) (*WriterContext, error) {
// If local writer not exist, then create new one and store it into engine info writer cache.
// note: operate ei.writeCache map is not thread safe please make sure there is sync mechanism to
// make sure the safe.
-func (ei *engineInfo) newWriterContext(workerID int) (*WriterContext, error) {
+func (ei *engineInfo) newWriterContext(workerID int, unique bool) (*WriterContext, error) {
lWrite, exist := ei.writerCache.Load(workerID)
if !exist {
var err error
@@ -176,10 +189,32 @@ func (ei *engineInfo) newWriterContext(workerID int) (*WriterContext, error) {
// Cache the local writer.
ei.writerCache.Store(workerID, lWrite)
}
- return &WriterContext{
+ wc := &WriterContext{
ctx: ei.ctx,
lWrite: lWrite,
- }, nil
+ }
+ if unique {
+ wc.rowSeq = func() int64 {
+ return ei.rowSeq.Add(1)
+ }
+ }
+ return wc, nil
+}
+
+func (ei *engineInfo) closeWriters() error {
+ var firstErr error
+ for _, wid := range ei.writerCache.Keys() {
+ if w, ok := ei.writerCache.Load(wid); ok {
+ _, err := w.Close(ei.ctx)
+ if err != nil {
+ if firstErr == nil {
+ firstErr = err
+ }
+ }
+ }
+ ei.writerCache.Delete(wid)
+ }
+ return firstErr
}
// WriteRow Write one row into local writer buffer.
@@ -187,6 +222,9 @@ func (wCtx *WriterContext) WriteRow(key, idxVal []byte) error {
kvs := make([]common.KvPair, 1)
kvs[0].Key = key
kvs[0].Val = idxVal
+ if wCtx.rowSeq != nil {
+ kvs[0].RowID = wCtx.rowSeq()
+ }
row := kv.MakeRowsFromKvPairs(kvs)
return wCtx.lWrite.WriteRows(wCtx.ctx, nil, row)
}
diff --git a/ddl/ingest/engine_mgr.go b/ddl/ingest/engine_mgr.go
index 44ecff9941932..f9b006ec9e369 100644
--- a/ddl/ingest/engine_mgr.go
+++ b/ddl/ingest/engine_mgr.go
@@ -18,7 +18,7 @@ import (
"fmt"
"github.com/pingcap/errors"
- "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/generic"
"github.com/pingcap/tidb/util/logutil"
"go.uber.org/zap"
@@ -37,7 +37,7 @@ func (m *engineManager) init(memRoot MemRoot, diskRoot DiskRoot) {
}
// Register create a new engineInfo and register it to the engineManager.
-func (m *engineManager) Register(bc *BackendContext, job *model.Job, indexID int64) (*engineInfo, error) {
+func (m *engineManager) Register(bc *BackendContext, jobID, indexID int64, schemaName, tableName string) (*engineInfo, error) {
// Calculate lightning concurrency degree and set memory usage
// and pre-allocate memory usage for worker.
m.MemRoot.RefreshConsumption()
@@ -55,29 +55,31 @@ func (m *engineManager) Register(bc *BackendContext, job *model.Job, indexID int
return nil, genEngineAllocMemFailedErr(m.MemRoot, bc.jobID, indexID)
}
- cfg := generateLocalEngineConfig(job.ID, job.SchemaName, job.TableName)
- openedEn, err := bc.backend.OpenEngine(bc.ctx, cfg, job.TableName, int32(indexID))
+ cfg := generateLocalEngineConfig(jobID, schemaName, tableName)
+ openedEn, err := bc.backend.OpenEngine(bc.ctx, cfg, tableName, int32(indexID))
if err != nil {
- return nil, errors.New(LitErrCreateEngineFail)
+ logutil.BgLogger().Warn(LitErrCreateEngineFail, zap.Int64("job ID", jobID),
+ zap.Int64("index ID", indexID), zap.Error(err))
+ return nil, errors.Trace(err)
}
id := openedEn.GetEngineUUID()
- en = NewEngineInfo(bc.ctx, job.ID, indexID, cfg, openedEn, id, 1, m.MemRoot, m.DiskRoot)
+ en = NewEngineInfo(bc.ctx, jobID, indexID, cfg, openedEn, id, 1, m.MemRoot, m.DiskRoot)
m.Store(indexID, en)
m.MemRoot.Consume(StructSizeEngineInfo)
- m.MemRoot.ConsumeWithTag(encodeEngineTag(job.ID, indexID), engineCacheSize)
+ m.MemRoot.ConsumeWithTag(encodeEngineTag(jobID, indexID), engineCacheSize)
info = LitInfoOpenEngine
} else {
if en.writerCount+1 > bc.cfg.TikvImporter.RangeConcurrency {
- logutil.BgLogger().Warn(LitErrExceedConcurrency, zap.Int64("job ID", job.ID),
+ logutil.BgLogger().Warn(LitErrExceedConcurrency, zap.Int64("job ID", jobID),
zap.Int64("index ID", indexID),
zap.Int("concurrency", bc.cfg.TikvImporter.RangeConcurrency))
- return nil, errors.New(LitErrExceedConcurrency)
+ return nil, dbterror.ErrIngestFailed.FastGenByArgs("concurrency quota exceeded")
}
en.writerCount++
info = LitInfoAddWriter
}
- m.MemRoot.ConsumeWithTag(encodeEngineTag(job.ID, indexID), int64(bc.cfg.TikvImporter.LocalWriterMemCacheSize))
- logutil.BgLogger().Info(info, zap.Int64("job ID", job.ID),
+ m.MemRoot.ConsumeWithTag(encodeEngineTag(jobID, indexID), int64(bc.cfg.TikvImporter.LocalWriterMemCacheSize))
+ logutil.BgLogger().Info(info, zap.Int64("job ID", jobID),
zap.Int64("index ID", indexID),
zap.Int64("current memory usage", m.MemRoot.CurrentUsage()),
zap.Int64("memory limitation", m.MemRoot.MaxMemoryQuota()),
@@ -99,6 +101,20 @@ func (m *engineManager) Unregister(jobID, indexID int64) {
m.MemRoot.Release(StructSizeEngineInfo)
}
+// ResetWorkers reset the writer count of the engineInfo because
+// the goroutines of backfill workers have been terminated.
+func (m *engineManager) ResetWorkers(bc *BackendContext, jobID, indexID int64) {
+ ei, exist := m.Load(indexID)
+ if !exist {
+ return
+ }
+ m.MemRoot.Release(StructSizeWriterCtx * int64(ei.writerCount))
+ m.MemRoot.ReleaseWithTag(encodeEngineTag(jobID, indexID))
+ engineCacheSize := int64(bc.cfg.TikvImporter.EngineMemCacheSize)
+ m.MemRoot.ConsumeWithTag(encodeEngineTag(jobID, indexID), engineCacheSize)
+ ei.writerCount = 0
+}
+
// UnregisterAll delete all engineInfo from the engineManager.
func (m *engineManager) UnregisterAll(jobID int64) {
for _, idxID := range m.Keys() {
diff --git a/ddl/ingest/env.go b/ddl/ingest/env.go
index 185f873b820a4..864cc61ae4e02 100644
--- a/ddl/ingest/env.go
+++ b/ddl/ingest/env.go
@@ -47,6 +47,14 @@ const maxMemoryQuota = 2 * size.GB
// InitGlobalLightningEnv initialize Lightning backfill environment.
func InitGlobalLightningEnv() {
log.SetAppLogger(logutil.BgLogger())
+ globalCfg := config.GetGlobalConfig()
+ if globalCfg.Store != "tikv" {
+ logutil.BgLogger().Warn(LitWarnEnvInitFail,
+ zap.String("storage limitation", "only support TiKV storage"),
+ zap.String("current storage", globalCfg.Store),
+ zap.Bool("lightning is initialized", LitInitialized))
+ return
+ }
sPath, err := genLightningDataDir()
if err != nil {
logutil.BgLogger().Warn(LitWarnEnvInitFail, zap.Error(err),
@@ -102,8 +110,5 @@ func genLightningDataDir() (string, error) {
return sortPath, nil
}
-// GenRLimitForTest is only used for test.
-var GenRLimitForTest = util.GenRLimit()
-
// GenLightningDataDirForTest is only used for test.
var GenLightningDataDirForTest = genLightningDataDir
diff --git a/ddl/ingest/mem_root.go b/ddl/ingest/mem_root.go
index a36d934c0abcd..522e5ddc1f7cc 100644
--- a/ddl/ingest/mem_root.go
+++ b/ddl/ingest/mem_root.go
@@ -122,7 +122,7 @@ func (m *memRootImpl) ConsumeWithTag(tag string, size int64) {
m.structSize[tag] = size
}
-// TestConsume implements MemRoot.
+// CheckConsume implements MemRoot.
func (m *memRootImpl) CheckConsume(size int64) bool {
m.mu.RLock()
defer m.mu.RUnlock()
diff --git a/ddl/ingest/message.go b/ddl/ingest/message.go
index 3858d40ec4e0a..4996aab49a415 100644
--- a/ddl/ingest/message.go
+++ b/ddl/ingest/message.go
@@ -15,7 +15,7 @@
package ingest
import (
- "github.com/pingcap/errors"
+ "github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/logutil"
"go.uber.org/zap"
)
@@ -23,28 +23,26 @@ import (
// Message const text
const (
LitErrAllocMemFail string = "[ddl-ingest] allocate memory failed"
- LitErrOutMaxMem string = "[ddl-ingest] memory used up for lightning add index"
- LitErrCreateDirFail string = "[ddl-ingest] create lightning sort path error"
- LitErrStatDirFail string = "[ddl-ingest] stat lightning sort path error"
- LitErrDeleteDirFail string = "[ddl-ingest] delete lightning sort path error"
- LitErrCreateBackendFail string = "[ddl-ingest] build lightning backend failed, will use kernel index reorg method to backfill the index"
- LitErrGetBackendFail string = "[ddl-ingest]: Can not get cached backend"
- LitErrCreateEngineFail string = "[ddl-ingest] build lightning engine failed, will use kernel index reorg method to backfill the index"
- LitErrCreateContextFail string = "[ddl-ingest] build lightning worker context failed, will use kernel index reorg method to backfill the index"
- LitErrGetEngineFail string = "[ddl-ingest] can not get cached engine info"
+ LitErrCreateDirFail string = "[ddl-ingest] create ingest sort path error"
+ LitErrStatDirFail string = "[ddl-ingest] stat ingest sort path error"
+ LitErrDeleteDirFail string = "[ddl-ingest] delete ingest sort path error"
+ LitErrCreateBackendFail string = "[ddl-ingest] build ingest backend failed"
+ LitErrGetBackendFail string = "[ddl-ingest] cannot get ingest backend"
+ LitErrCreateEngineFail string = "[ddl-ingest] build ingest engine failed"
+ LitErrCreateContextFail string = "[ddl-ingest] build ingest writer context failed"
+ LitErrGetEngineFail string = "[ddl-ingest] can not get ingest engine info"
LitErrGetStorageQuota string = "[ddl-ingest] get storage quota error"
LitErrCloseEngineErr string = "[ddl-ingest] close engine error"
LitErrCleanEngineErr string = "[ddl-ingest] clean engine error"
LitErrFlushEngineErr string = "[ddl-ingest] flush engine data err"
LitErrIngestDataErr string = "[ddl-ingest] ingest data into storage error"
LitErrRemoteDupExistErr string = "[ddl-ingest] remote duplicate index key exist"
- LitErrExceedConcurrency string = "[ddl-ingest] the concurrency is greater than lightning limit(tikv-importer.range-concurrency)"
+ LitErrExceedConcurrency string = "[ddl-ingest] the concurrency is greater than ingest limit"
LitErrUpdateDiskStats string = "[ddl-ingest] update disk usage error"
LitWarnEnvInitFail string = "[ddl-ingest] initialize environment failed"
LitWarnConfigError string = "[ddl-ingest] build config for backend failed"
- LitWarnGenMemLimit string = "[ddl-ingest] generate memory max limitation"
- LitInfoEnvInitSucc string = "[ddl-ingest] init global lightning backend environment finished"
- LitInfoSortDir string = "[ddl-ingest] the lightning sorted dir"
+ LitInfoEnvInitSucc string = "[ddl-ingest] init global ingest backend environment finished"
+ LitInfoSortDir string = "[ddl-ingest] the ingest sorted directory"
LitInfoCreateBackend string = "[ddl-ingest] create one backend for an DDL job"
LitInfoCloseBackend string = "[ddl-ingest] close one backend for DDL job"
LitInfoOpenEngine string = "[ddl-ingest] open an engine for index reorg task"
@@ -53,17 +51,17 @@ const (
LitInfoCloseEngine string = "[ddl-ingest] flush all writer and get closed engine"
LitInfoRemoteDupCheck string = "[ddl-ingest] start remote duplicate checking"
LitInfoStartImport string = "[ddl-ingest] start to import data"
- LitInfoSetMemLimit string = "[ddl-ingest] set max memory limitation"
- LitInfoChgMemSetting string = "[ddl-ingest] change memory setting for lightning"
- LitInfoInitMemSetting string = "[ddl-ingest] initial memory setting for lightning"
+ LitInfoChgMemSetting string = "[ddl-ingest] change memory setting for ingest"
+ LitInfoInitMemSetting string = "[ddl-ingest] initial memory setting for ingest"
LitInfoUnsafeImport string = "[ddl-ingest] do a partial import data into the storage"
+ LitErrCloseWriterErr string = "[ddl-ingest] close writer error"
)
func genBackendAllocMemFailedErr(memRoot MemRoot, jobID int64) error {
logutil.BgLogger().Warn(LitErrAllocMemFail, zap.Int64("job ID", jobID),
zap.Int64("current memory usage", memRoot.CurrentUsage()),
zap.Int64("max memory quota", memRoot.MaxMemoryQuota()))
- return errors.New(LitErrOutMaxMem)
+ return dbterror.ErrIngestFailed.FastGenByArgs("memory used up")
}
func genEngineAllocMemFailedErr(memRoot MemRoot, jobID, idxID int64) error {
@@ -71,5 +69,5 @@ func genEngineAllocMemFailedErr(memRoot MemRoot, jobID, idxID int64) error {
zap.Int64("index ID", idxID),
zap.Int64("current memory usage", memRoot.CurrentUsage()),
zap.Int64("max memory quota", memRoot.MaxMemoryQuota()))
- return errors.New(LitErrOutMaxMem)
+ return dbterror.ErrIngestFailed.FastGenByArgs("memory used up")
}
diff --git a/ddl/integration_test.go b/ddl/integration_test.go
index 29e69aa855274..264e755889899 100644
--- a/ddl/integration_test.go
+++ b/ddl/integration_test.go
@@ -18,7 +18,7 @@ import (
"fmt"
"testing"
- "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/testkit"
"github.com/stretchr/testify/require"
@@ -86,7 +86,7 @@ func TestDDLStatementsBackFill(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
needReorg := false
- callback := &ddl.TestDDLCallback{
+ callback := &callback.TestDDLCallback{
Do: dom,
}
onJobUpdatedExportedFunc := func(job *model.Job) {
diff --git a/ddl/internal/callback/BUILD.bazel b/ddl/internal/callback/BUILD.bazel
new file mode 100644
index 0000000000000..63a4cf7f4687f
--- /dev/null
+++ b/ddl/internal/callback/BUILD.bazel
@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "callback",
+ srcs = ["callback.go"],
+ importpath = "github.com/pingcap/tidb/ddl/internal/callback",
+ visibility = ["//ddl:__subpackages__"],
+ deps = [
+ "//ddl",
+ "//infoschema",
+ "//parser/model",
+ "//sessionctx",
+ "//util/logutil",
+ "@org_uber_go_zap//:zap",
+ ],
+)
+
+go_test(
+ name = "callback_test",
+ timeout = "short",
+ srcs = ["callback_test.go"],
+ embed = [":callback"],
+ flaky = True,
+ deps = [
+ "//ddl",
+ "@com_github_stretchr_testify//require",
+ ],
+)
diff --git a/ddl/callback_test.go b/ddl/internal/callback/callback.go
similarity index 93%
rename from ddl/callback_test.go
rename to ddl/internal/callback/callback.go
index 5a97e8212689e..a3c84d774dd5c 100644
--- a/ddl/callback_test.go
+++ b/ddl/internal/callback/callback.go
@@ -1,4 +1,4 @@
-// Copyright 2015 PingCAP, Inc.
+// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,27 +12,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package ddl
+package callback
import (
"context"
"sync/atomic"
- "testing"
+ "github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/util/logutil"
- "github.com/stretchr/testify/require"
"go.uber.org/zap"
)
+// TestInterceptor is a test interceptor in the ddl
type TestInterceptor struct {
- *BaseInterceptor
+ *ddl.BaseInterceptor
OnGetInfoSchemaExported func(ctx sessionctx.Context, is infoschema.InfoSchema) infoschema.InfoSchema
}
+// OnGetInfoSchema is to run when to call GetInfoSchema
func (ti *TestInterceptor) OnGetInfoSchema(ctx sessionctx.Context, is infoschema.InfoSchema) infoschema.InfoSchema {
if ti.OnGetInfoSchemaExported != nil {
return ti.OnGetInfoSchemaExported(ctx, is)
@@ -43,10 +44,10 @@ func (ti *TestInterceptor) OnGetInfoSchema(ctx sessionctx.Context, is infoschema
// TestDDLCallback is used to customize user callback themselves.
type TestDDLCallback struct {
- *BaseCallback
+ *ddl.BaseCallback
// We recommended to pass the domain parameter to the test ddl callback, it will ensure
// domain to reload schema before your ddl stepping into the next state change.
- Do DomainReloader
+ Do ddl.DomainReloader
onJobRunBefore func(*model.Job)
OnJobRunBeforeExported func(*model.Job)
@@ -149,11 +150,3 @@ func (tc *TestDDLCallback) OnGetJobAfter(jobType string, job *model.Job) {
func (tc *TestDDLCallback) Clone() *TestDDLCallback {
return &*tc
}
-
-func TestCallback(t *testing.T) {
- cb := &BaseCallback{}
- require.Nil(t, cb.OnChanged(nil))
- cb.OnJobRunBefore(nil)
- cb.OnJobUpdated(nil)
- cb.OnWatched(context.TODO())
-}
diff --git a/ddl/internal/callback/callback_test.go b/ddl/internal/callback/callback_test.go
new file mode 100644
index 0000000000000..f611394909e48
--- /dev/null
+++ b/ddl/internal/callback/callback_test.go
@@ -0,0 +1,31 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package callback
+
+import (
+ "context"
+ "testing"
+
+ "github.com/pingcap/tidb/ddl"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCallback(t *testing.T) {
+ cb := &ddl.BaseCallback{}
+ require.Nil(t, cb.OnChanged(nil))
+ cb.OnJobRunBefore(nil)
+ cb.OnJobUpdated(nil)
+ cb.OnWatched(context.TODO())
+}
diff --git a/ddl/job_table.go b/ddl/job_table.go
index d23f083539e87..710fc7ad9d0ce 100644
--- a/ddl/job_table.go
+++ b/ddl/job_table.go
@@ -17,6 +17,7 @@ package ddl
import (
"bytes"
"context"
+ "encoding/hex"
"fmt"
"math"
"strconv"
@@ -26,11 +27,16 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/kvrpcpb"
+ "github.com/pingcap/tidb/ddl/ingest"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/types"
+ tidbutil "github.com/pingcap/tidb/util"
+ "github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/logutil"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
@@ -39,6 +45,7 @@ import (
var (
addingDDLJobConcurrent = "/tidb/ddl/add_ddl_job_general"
+ addingBackfillJob = "/tidb/ddl/add_backfill_job"
)
func (dc *ddlCtx) insertRunningDDLJobMap(id int64) {
@@ -67,7 +74,7 @@ func (dc *ddlCtx) excludeJobIDs() string {
}
const (
- getJobSQL = "select job_meta, processing from mysql.tidb_ddl_job where job_id in (select min(job_id) from mysql.tidb_ddl_job group by schema_ids, table_ids) and %s reorg %s order by processing desc, job_id"
+ getJobSQL = "select job_meta, processing from mysql.tidb_ddl_job where job_id in (select min(job_id) from mysql.tidb_ddl_job group by schema_ids, table_ids, processing) and %s reorg %s order by processing desc, job_id"
)
type jobType int
@@ -173,7 +180,7 @@ func (d *ddl) startDispatchLoop() {
if isChanClosed(d.ctx.Done()) {
return
}
- if !variable.EnableConcurrentDDL.Load() || !d.isOwner() || d.waiting.Load() {
+ if !d.isOwner() || d.waiting.Load() {
d.once.Store(true)
time.Sleep(time.Second)
continue
@@ -236,7 +243,7 @@ func (d *ddl) delivery2worker(wk *worker, pool *workerPool, job *model.Job) {
// check if this ddl job is synced to all servers.
if !d.isSynced(job) || d.once.Load() {
if variable.EnableMDL.Load() {
- exist, err := checkMDLInfo(job.ID, d.sessPool)
+ exist, version, err := checkMDLInfo(job.ID, d.sessPool)
if err != nil {
logutil.BgLogger().Warn("[ddl] check MDL info failed", zap.Error(err), zap.String("job", job.String()))
// Release the worker resource.
@@ -245,14 +252,12 @@ func (d *ddl) delivery2worker(wk *worker, pool *workerPool, job *model.Job) {
} else if exist {
// Release the worker resource.
pool.put(wk)
- err = waitSchemaSynced(d.ddlCtx, job, 2*d.lease)
+ err = waitSchemaSyncedForMDL(d.ddlCtx, job, version)
if err != nil {
- logutil.BgLogger().Warn("[ddl] wait ddl job sync failed", zap.Error(err), zap.String("job", job.String()))
- time.Sleep(time.Second)
return
}
d.once.Store(false)
- cleanMDLInfo(d.sessPool, job.ID)
+ cleanMDLInfo(d.sessPool, job.ID, d.etcdCli)
// Don't have a worker now.
return
}
@@ -287,7 +292,7 @@ func (d *ddl) delivery2worker(wk *worker, pool *workerPool, job *model.Job) {
// If the job is done or still running or rolling back, we will wait 2 * lease time to guarantee other servers to update
// the newest schema.
waitSchemaChanged(context.Background(), d.ddlCtx, d.lease*2, schemaVer, job)
- cleanMDLInfo(d.sessPool, job.ID)
+ cleanMDLInfo(d.sessPool, job.ID, d.etcdCli)
d.synced(job)
if RunInGoTest {
@@ -310,6 +315,122 @@ func (d *ddl) markJobProcessing(sess *session, job *model.Job) error {
return errors.Trace(err)
}
+func (d *ddl) startDispatchBackfillJobsLoop() {
+ d.backfillCtx.jobCtxMap = make(map[int64]*JobContext)
+
+ var notifyBackfillJobByEtcdCh clientv3.WatchChan
+ if d.etcdCli != nil {
+ notifyBackfillJobByEtcdCh = d.etcdCli.Watch(d.ctx, addingBackfillJob)
+ }
+ ticker := time.NewTicker(500 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ if isChanClosed(d.ctx.Done()) {
+ return
+ }
+ select {
+ case <-d.backfillJobCh:
+ case <-ticker.C:
+ case _, ok := <-notifyBackfillJobByEtcdCh:
+ if !ok {
+ logutil.BgLogger().Warn("[ddl] start backfill worker watch channel closed", zap.String("watch key", addingBackfillJob))
+ notifyBackfillJobByEtcdCh = d.etcdCli.Watch(d.ctx, addingBackfillJob)
+ time.Sleep(time.Second)
+ continue
+ }
+ case <-d.ctx.Done():
+ return
+ }
+ d.loadBackfillJobAndRun()
+ }
+}
+
+func (d *ddl) getTableByTxn(store kv.Storage, schemaID, tableID int64) (*model.DBInfo, table.Table, error) {
+ var tbl table.Table
+ var dbInfo *model.DBInfo
+ err := kv.RunInNewTxn(d.ctx, store, false, func(ctx context.Context, txn kv.Transaction) error {
+ t := meta.NewMeta(txn)
+ var err1 error
+ dbInfo, err1 = t.GetDatabase(schemaID)
+ if err1 != nil {
+ return errors.Trace(err1)
+ }
+ tblInfo, err1 := getTableInfo(t, tableID, schemaID)
+ if err1 != nil {
+ return errors.Trace(err1)
+ }
+ tbl, err1 = getTable(store, schemaID, tblInfo)
+ return errors.Trace(err1)
+ })
+ return dbInfo, tbl, err
+}
+
+func (d *ddl) loadBackfillJobAndRun() {
+ isDistReorg := variable.DDLEnableDistributeReorg.Load()
+ if !isDistReorg {
+ return
+ }
+ se, err := d.sessPool.get()
+ if err != nil {
+ logutil.BgLogger().Fatal("dispatch backfill jobs loop get session failed, it should not happen, please try restart TiDB", zap.Error(err))
+ }
+ sess := newSession(se)
+
+ runningJobIDs := d.backfillCtxJobIDs()
+ if len(runningJobIDs) >= reorgWorkerCnt {
+ d.sessPool.put(se)
+ return
+ }
+
+ // TODO: Add ele info to distinguish backfill jobs.
+ // Get a Backfill job to get the reorg info like element info, schema ID and so on.
+ bfJob, err := GetBackfillJobForOneEle(sess, runningJobIDs, InstanceLease)
+ if err != nil || bfJob == nil {
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] get backfill jobs failed in this instance", zap.Error(err))
+ } else {
+ logutil.BgLogger().Debug("[ddl] get no backfill job in this instance")
+ }
+ d.sessPool.put(se)
+ return
+ }
+
+ jobCtx, existent := d.setBackfillCtxJobContext(bfJob.JobID, bfJob.Meta.Query, bfJob.Meta.Type)
+ if existent {
+ logutil.BgLogger().Warn("[ddl] get the type of backfill job is running in this instance", zap.String("backfill job", bfJob.AbbrStr()))
+ d.sessPool.put(se)
+ return
+ }
+ // TODO: Adjust how the non-owner uses ReorgCtx.
+ d.setReorgCtxForBackfill(bfJob)
+ d.wg.Run(func() {
+ defer func() {
+ tidbutil.Recover(metrics.LabelDistReorg, "runBackfillJobs", nil, false)
+ d.removeBackfillCtxJobCtx(bfJob.JobID)
+ d.removeReorgCtx(bfJob.JobID)
+ d.sessPool.put(se)
+ }()
+
+ if bfJob.Meta.ReorgTp == model.ReorgTypeLitMerge {
+ if !ingest.LitInitialized {
+ logutil.BgLogger().Warn("[ddl] we can't do ingest in this instance",
+ zap.Bool("LitInitialized", ingest.LitInitialized), zap.String("bfJob", bfJob.AbbrStr()))
+ return
+ }
+ logutil.BgLogger().Info("[ddl] run backfill jobs with ingest in this instance", zap.String("bfJob", bfJob.AbbrStr()))
+ err = runBackfillJobsWithLightning(d, sess, bfJob, jobCtx)
+ } else {
+ logutil.BgLogger().Info("[ddl] run backfill jobs with txn-merge in this instance", zap.String("bfJob", bfJob.AbbrStr()))
+ _, err = runBackfillJobs(d, sess, nil, bfJob, jobCtx)
+ }
+
+ if err == nil {
+ err = syncBackfillHistoryJobs(sess, d.uuid, bfJob)
+ }
+ logutil.BgLogger().Info("[ddl] run backfill jobs finished in this instance", zap.Stringer("reorg type", bfJob.Meta.ReorgTp), zap.Error(err))
+ })
+}
+
const (
addDDLJobSQL = "insert into mysql.tidb_ddl_job(job_id, reorg, schema_ids, table_ids, job_meta, type, processing) values"
updateDDLJobSQL = "update mysql.tidb_ddl_job set job_meta = %s where job_id = %d"
@@ -371,6 +492,8 @@ func job2UniqueIDs(job *model.Job, schema bool) string {
}
slices.Sort(s)
return strings.Join(s, ",")
+ case model.ActionTruncateTable:
+ return strconv.FormatInt(job.TableID, 10) + "," + strconv.FormatInt(job.Args[0].(int64), 10)
}
if schema {
return strconv.FormatInt(job.SchemaID, 10)
@@ -397,7 +520,8 @@ func updateDDLJob2Table(sctx *session, job *model.Job, updateRawArgs bool) error
// getDDLReorgHandle gets DDL reorg handle.
func getDDLReorgHandle(sess *session, job *model.Job) (element *meta.Element, startKey, endKey kv.Key, physicalTableID int64, err error) {
sql := fmt.Sprintf("select ele_id, ele_type, start_key, end_key, physical_id from mysql.tidb_ddl_reorg where job_id = %d", job.ID)
- rows, err := sess.execute(context.Background(), sql, "get_handle")
+ ctx := kv.WithInternalSourceType(context.Background(), getDDLRequestSource(job.Type))
+ rows, err := sess.execute(ctx, sql, "get_handle")
if err != nil {
return nil, nil, nil, 0, err
}
@@ -430,15 +554,8 @@ func getDDLReorgHandle(sess *session, job *model.Job) (element *meta.Element, st
return
}
-// updateDDLReorgStartHandle update the startKey of the handle.
-func updateDDLReorgStartHandle(sess *session, job *model.Job, element *meta.Element, startKey kv.Key) error {
- sql := fmt.Sprintf("update mysql.tidb_ddl_reorg set ele_id = %d, ele_type = %s, start_key = %s where job_id = %d",
- element.ID, wrapKey2String(element.TypeKey), wrapKey2String(startKey), job.ID)
- _, err := sess.execute(context.Background(), sql, "update_start_handle")
- return err
-}
-
// updateDDLReorgHandle update startKey, endKey physicalTableID and element of the handle.
+// Caller should wrap this in a separate transaction, to avoid conflicts.
func updateDDLReorgHandle(sess *session, jobID int64, startKey kv.Key, endKey kv.Key, physicalTableID int64, element *meta.Element) error {
sql := fmt.Sprintf("update mysql.tidb_ddl_reorg set ele_id = %d, ele_type = %s, start_key = %s, end_key = %s, physical_id = %d where job_id = %d",
element.ID, wrapKey2String(element.TypeKey), wrapKey2String(startKey), wrapKey2String(endKey), physicalTableID, jobID)
@@ -447,28 +564,48 @@ func updateDDLReorgHandle(sess *session, jobID int64, startKey kv.Key, endKey kv
}
// initDDLReorgHandle initializes the handle for ddl reorg.
-func initDDLReorgHandle(sess *session, jobID int64, startKey kv.Key, endKey kv.Key, physicalTableID int64, element *meta.Element) error {
- sql := fmt.Sprintf("insert into mysql.tidb_ddl_reorg(job_id, ele_id, ele_type, start_key, end_key, physical_id) values (%d, %d, %s, %s, %s, %d)",
+func initDDLReorgHandle(s *session, jobID int64, startKey kv.Key, endKey kv.Key, physicalTableID int64, element *meta.Element) error {
+ del := fmt.Sprintf("delete from mysql.tidb_ddl_reorg where job_id = %d", jobID)
+ ins := fmt.Sprintf("insert into mysql.tidb_ddl_reorg(job_id, ele_id, ele_type, start_key, end_key, physical_id) values (%d, %d, %s, %s, %s, %d)",
jobID, element.ID, wrapKey2String(element.TypeKey), wrapKey2String(startKey), wrapKey2String(endKey), physicalTableID)
- _, err := sess.execute(context.Background(), sql, "update_handle")
- return err
+ return s.runInTxn(func(se *session) error {
+ _, err := se.execute(context.Background(), del, "init_handle")
+ if err != nil {
+ logutil.BgLogger().Info("initDDLReorgHandle failed to delete", zap.Int64("jobID", jobID), zap.Error(err))
+ }
+ _, err = se.execute(context.Background(), ins, "init_handle")
+ return err
+ })
}
// deleteDDLReorgHandle deletes the handle for ddl reorg.
-func removeDDLReorgHandle(sess *session, job *model.Job, elements []*meta.Element) error {
+func removeDDLReorgHandle(s *session, job *model.Job, elements []*meta.Element) error {
if len(elements) == 0 {
return nil
}
sql := fmt.Sprintf("delete from mysql.tidb_ddl_reorg where job_id = %d", job.ID)
- _, err := sess.execute(context.Background(), sql, "remove_handle")
- return err
+ return s.runInTxn(func(se *session) error {
+ _, err := se.execute(context.Background(), sql, "remove_handle")
+ return err
+ })
}
// removeReorgElement removes the element from ddl reorg, it is the same with removeDDLReorgHandle, only used in failpoint
-func removeReorgElement(sess *session, job *model.Job) error {
+func removeReorgElement(s *session, job *model.Job) error {
sql := fmt.Sprintf("delete from mysql.tidb_ddl_reorg where job_id = %d", job.ID)
- _, err := sess.execute(context.Background(), sql, "remove_handle")
- return err
+ return s.runInTxn(func(se *session) error {
+ _, err := se.execute(context.Background(), sql, "remove_handle")
+ return err
+ })
+}
+
+// cleanDDLReorgHandles removes handles that are no longer needed.
+func cleanDDLReorgHandles(s *session, job *model.Job) error {
+ sql := "delete from mysql.tidb_ddl_reorg where job_id = " + strconv.FormatInt(job.ID, 10)
+ return s.runInTxn(func(se *session) error {
+ _, err := se.execute(context.Background(), sql, "clean_handle")
+ return err
+ })
}
func wrapKey2String(key []byte) string {
@@ -496,134 +633,329 @@ func getJobsBySQL(sess *session, tbl, condition string) ([]*model.Job, error) {
return jobs, nil
}
-// MoveJobFromQueue2Table move existing DDLs in queue to table.
-func (d *ddl) MoveJobFromQueue2Table(inBootstrap bool) error {
- sess, err := d.sessPool.get()
+func syncBackfillHistoryJobs(sess *session, uuid string, backfillJob *BackfillJob) error {
+ sql := fmt.Sprintf("update mysql.%s set state = '%s' where task_key like \"%d_%s_%d_%%\" and exec_id = '%s' limit 1;",
+ BackgroundSubtaskHistoryTable, model.JobStateSynced.String(), backfillJob.JobID, hex.EncodeToString(backfillJob.EleKey), backfillJob.EleID, uuid)
+ _, err := sess.execute(context.Background(), sql, "sync_backfill_history_job")
+ return err
+}
+
+func generateInsertBackfillJobSQL(tableName string, backfillJobs []*BackfillJob) (string, error) {
+ sqlBuilder := strings.Builder{}
+ sqlBuilder.WriteString("insert into mysql.")
+ sqlBuilder.WriteString(tableName)
+ sqlBuilder.WriteString("(task_key, ddl_physical_tid, type, exec_id, exec_expired, state, checkpoint, start_time, state_update_time, meta) values")
+ for i, bj := range backfillJobs {
+ mateByte, err := bj.Meta.Encode()
+ if err != nil {
+ return "", errors.Trace(err)
+ }
+
+ if i != 0 {
+ sqlBuilder.WriteString(", ")
+ }
+ sqlBuilder.WriteString(fmt.Sprintf("('%d_%s_%d_%d', %d, %d, '%s', '%s', '%s', %s, %d, %d, %s)",
+ bj.JobID, hex.EncodeToString(bj.EleKey), bj.EleID, bj.ID, bj.PhysicalTableID, bj.Tp, bj.InstanceID, bj.InstanceLease, bj.State.String(), wrapKey2String(bj.Meta.CurrKey),
+ bj.StartTS, bj.StateUpdateTS, wrapKey2String(mateByte)))
+ }
+ return sqlBuilder.String(), nil
+}
+
+// AddBackfillHistoryJob adds the backfill jobs to the tidb_background_subtask_history table.
+func AddBackfillHistoryJob(sess *session, backfillJobs []*BackfillJob) error {
+ label := fmt.Sprintf("add_%s_job", BackgroundSubtaskHistoryTable)
+ sql, err := generateInsertBackfillJobSQL(BackgroundSubtaskHistoryTable, backfillJobs)
if err != nil {
return err
}
- defer d.sessPool.put(sess)
- return runInTxn(newSession(sess), func(se *session) error {
+ _, err = sess.execute(context.Background(), sql, label)
+ return errors.Trace(err)
+}
+
+// AddBackfillJobs adds the backfill jobs to the tidb_background_subtask table.
+func AddBackfillJobs(s *session, backfillJobs []*BackfillJob) error {
+ label := fmt.Sprintf("add_%s_job", BackgroundSubtaskTable)
+ // Do runInTxn to get StartTS.
+ return s.runInTxn(func(se *session) error {
txn, err := se.txn()
if err != nil {
return errors.Trace(err)
}
- t := meta.NewMeta(txn)
- isConcurrentDDL, err := t.IsConcurrentDDL()
- if !inBootstrap && (isConcurrentDDL || err != nil) {
- return errors.Trace(err)
+ startTS := txn.StartTS()
+ for _, bj := range backfillJobs {
+ bj.StartTS = startTS
}
- systemDBID, err := t.GetSystemDBID()
+
+ sql, err := generateInsertBackfillJobSQL(BackgroundSubtaskTable, backfillJobs)
if err != nil {
- return errors.Trace(err)
+ return err
}
- for _, tp := range []workerType{addIdxWorker, generalWorker} {
- t := newMetaWithQueueTp(txn, tp)
- jobs, err := t.GetAllDDLJobsInQueue()
- if err != nil {
- return errors.Trace(err)
- }
- for _, job := range jobs {
- // In bootstrap, we can ignore the internal DDL.
- if inBootstrap && job.SchemaID == systemDBID {
- continue
- }
- err = insertDDLJobs2Table(se, false, job)
- if err != nil {
- return errors.Trace(err)
- }
- if tp == generalWorker {
- // General job do not have reorg info.
- continue
- }
- element, start, end, pid, err := t.GetDDLReorgHandle(job)
- if meta.ErrDDLReorgElementNotExist.Equal(err) {
- continue
- }
+ _, err = se.execute(context.Background(), sql, label)
+ return errors.Trace(err)
+ })
+}
+
+// GetBackfillJobForOneEle gets the backfill jobs in the tblName table that contains only one element.
+func GetBackfillJobForOneEle(s *session, excludedJobIDs []int64, lease time.Duration) (*BackfillJob, error) {
+ eJobIDsBuilder := strings.Builder{}
+ for _, id := range excludedJobIDs {
+ eJobIDsBuilder.WriteString(fmt.Sprintf(" and task_key not like \"%d_%%\"", id))
+ }
+
+ var err error
+ var bJobs []*BackfillJob
+ err = s.runInTxn(func(se *session) error {
+ currTime, err := GetOracleTimeWithStartTS(se)
+ if err != nil {
+ return err
+ }
+ leaseStr := currTime.Add(-lease).Format(types.TimeFormat)
+ bJobs, err = GetBackfillJobs(se, BackgroundSubtaskTable,
+ fmt.Sprintf("(exec_id = '' or exec_expired < '%v') %s order by task_key limit 1",
+ leaseStr, eJobIDsBuilder.String()), "get_backfill_job")
+ return err
+ })
+ if err != nil || len(bJobs) == 0 {
+ return nil, err
+ }
+
+ return bJobs[0], nil
+}
+
+// GetAndMarkBackfillJobsForOneEle batch gets the backfill jobs in the tblName table that contains only one element,
+// and update these jobs with instance ID and lease.
+func GetAndMarkBackfillJobsForOneEle(s *session, batch int, jobID int64, uuid string, pTblID int64, lease time.Duration) ([]*BackfillJob, error) {
+ var validLen int
+ var bJobs []*BackfillJob
+ err := s.runInTxn(func(se *session) error {
+ currTime, err := GetOracleTimeWithStartTS(se)
+ if err != nil {
+ return err
+ }
+ leaseStr := currTime.Add(-lease).Format(types.TimeFormat)
+
+ getJobsSQL := fmt.Sprintf("(exec_id = '' or exec_expired < '%v') and task_key like \"%d_%%\" order by task_key limit %d",
+ leaseStr, jobID, batch)
+ if pTblID != getJobWithoutPartition {
+ if pTblID == 0 {
+ rows, err := s.execute(context.Background(),
+ fmt.Sprintf("select ddl_physical_tid from mysql.%s group by substring_index(task_key,\"_\",3), ddl_physical_tid having max(length(exec_id)) = 0 or max(exec_expired) < '%s' order by substring_index(task_key,\"_\",3), ddl_physical_tid limit 1",
+ BackgroundSubtaskTable, leaseStr), "get_mark_backfill_job")
if err != nil {
return errors.Trace(err)
}
- err = initDDLReorgHandle(se, job.ID, start, end, pid, element)
- if err != nil {
- return errors.Trace(err)
+
+ if len(rows) == 0 {
+ return dbterror.ErrDDLJobNotFound.FastGen("get zero backfill job")
}
+
+ pTblID = rows[0].GetInt64(0)
}
+ getJobsSQL = fmt.Sprintf("(exec_id = '' or exec_expired < '%s') and task_key like \"%d_%%\" and ddl_physical_tid = %d order by task_key limit %d",
+ leaseStr, jobID, pTblID, batch)
}
- if err = t.ClearALLDDLJob(); err != nil {
- return errors.Trace(err)
+ bJobs, err = GetBackfillJobs(se, BackgroundSubtaskTable, getJobsSQL, "get_mark_backfill_job")
+ if err != nil {
+ return err
}
- if err = t.ClearAllDDLReorgHandle(); err != nil {
- return errors.Trace(err)
+ if len(bJobs) == 0 {
+ return dbterror.ErrDDLJobNotFound.FastGen("get zero backfill job")
}
- return t.SetConcurrentDDL(true)
+
+ validLen = 0
+ firstJobID, firstEleID, firstEleKey := bJobs[0].JobID, bJobs[0].EleID, bJobs[0].EleKey
+ for i := 0; i < len(bJobs); i++ {
+ if bJobs[i].JobID != firstJobID || bJobs[i].EleID != firstEleID || !bytes.Equal(bJobs[i].EleKey, firstEleKey) {
+ break
+ }
+ validLen++
+
+ bJobs[i].InstanceID = uuid
+ bJobs[i].InstanceLease = GetLeaseGoTime(currTime, lease)
+ // TODO: batch update
+ if err = updateBackfillJob(se, BackgroundSubtaskTable, bJobs[i], "get_mark_backfill_job"); err != nil {
+ return err
+ }
+ }
+ return nil
})
+ if validLen == 0 {
+ return nil, err
+ }
+
+ return bJobs[:validLen], err
}
-// MoveJobFromTable2Queue move existing DDLs in table to queue.
-func (d *ddl) MoveJobFromTable2Queue() error {
- sess, err := d.sessPool.get()
+// GetInterruptedBackfillJobForOneEle gets an interrupted backfill job that contains only one element.
+func GetInterruptedBackfillJobForOneEle(sess *session, jobID, eleID int64, eleKey []byte) ([]*BackfillJob, error) {
+ bJobs, err := GetBackfillJobs(sess, BackgroundSubtaskHistoryTable, fmt.Sprintf("task_key like \"%d_%s_%d_%%\" and state = \"%s\" limit 1",
+ jobID, hex.EncodeToString(eleKey), eleID, model.JobStateCancelled.String()), "get_interrupt_backfill_job")
+ if err != nil || len(bJobs) == 0 {
+ return nil, err
+ }
+ return bJobs, nil
+}
+
+// GetBackfillJobCount gets the number of rows in the tblName table according to condition.
+func GetBackfillJobCount(sess *session, tblName, condition string, label string) (int, error) {
+ rows, err := sess.execute(context.Background(), fmt.Sprintf("select count(1) from mysql.%s where %s", tblName, condition), label)
if err != nil {
- return err
+ return 0, errors.Trace(err)
}
- defer d.sessPool.put(sess)
- return runInTxn(newSession(sess), func(se *session) error {
- txn, err := se.txn()
+ if len(rows) == 0 {
+ return 0, dbterror.ErrDDLJobNotFound.FastGenByArgs(fmt.Sprintf("get wrong result cnt:%d", len(rows)))
+ }
+
+ return int(rows[0].GetInt64(0)), nil
+}
+
+// GetBackfillMetas gets the backfill metas in the tblName table according to condition.
+func GetBackfillMetas(sess *session, tblName, condition string, label string) ([]*model.BackfillMeta, error) {
+ rows, err := sess.execute(context.Background(), fmt.Sprintf("select meta from mysql.%s where %s", tblName, condition), label)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if len(rows) == 0 {
+ return nil, dbterror.ErrDDLJobNotFound.FastGenByArgs(fmt.Sprintf("get wrong result cnt:%d", len(rows)))
+ }
+
+ metas := make([]*model.BackfillMeta, 0, len(rows))
+ for _, r := range rows {
+ meta := &model.BackfillMeta{}
+ err = meta.Decode(r.GetBytes(0))
if err != nil {
- return errors.Trace(err)
+ return nil, errors.Trace(err)
}
- t := meta.NewMeta(txn)
- isConcurrentDDL, err := t.IsConcurrentDDL()
- if !isConcurrentDDL || err != nil {
- return errors.Trace(err)
+ metas = append(metas, meta)
+ }
+
+ return metas, nil
+}
+
+// GetBackfillIDAndMetas gets the backfill IDs and metas in the tblName table according to condition.
+func GetBackfillIDAndMetas(sess *session, tblName, condition string, label string) ([]*BackfillJobRangeMeta, error) {
+ sql := "select tbl.task_key, tbl.meta, tbl.ddl_physical_tid from (select max(task_key) max_id, ddl_physical_tid " +
+ fmt.Sprintf(" from mysql.%s tbl where %s group by ddl_physical_tid) tmp join mysql.%s tbl",
+ tblName, condition, tblName) + " on tbl.task_key=tmp.max_id and tbl.ddl_physical_tid=tmp.ddl_physical_tid;"
+ rows, err := sess.execute(context.Background(), sql, label)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if len(rows) == 0 {
+ return nil, nil
+ }
+
+ pTblMetas := make([]*BackfillJobRangeMeta, 0, len(rows))
+ for _, r := range rows {
+ key := r.GetString(0)
+ keySlice := strings.Split(key, "_")
+ id, err := strconv.ParseInt(keySlice[3], 10, 64)
+ if err != nil {
+ return nil, err
}
- jobs, err := getJobsBySQL(se, "tidb_ddl_job", "1 order by job_id")
+ meta := &model.BackfillMeta{}
+ err = meta.Decode(r.GetBytes(1))
if err != nil {
- return errors.Trace(err)
+ return nil, err
}
-
- for _, job := range jobs {
- jobListKey := meta.DefaultJobListKey
- if job.MayNeedReorg() {
- jobListKey = meta.AddIndexJobListKey
- }
- if err := t.EnQueueDDLJobNoUpdate(job, jobListKey); err != nil {
- return errors.Trace(err)
- }
+ pTblMeta := BackfillJobRangeMeta{
+ ID: id,
+ StartKey: meta.StartKey,
+ EndKey: meta.EndKey,
+ PhyTblID: r.GetInt64(2),
}
+ pTblMetas = append(pTblMetas, &pTblMeta)
+ }
+ return pTblMetas, nil
+}
+
+func getUnsyncedInstanceIDs(sess *session, jobID int64, label string) ([]string, error) {
+ sql := fmt.Sprintf("select sum((state='%s') + (state='%s')) as tmp, exec_id from mysql.tidb_background_subtask_history where task_key like \"%d_%%\" group by exec_id having tmp = 0;",
+ model.JobStateSynced.String(), model.JobStateCancelled.String(), jobID)
+ rows, err := sess.execute(context.Background(), sql, label)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ InstanceIDs := make([]string, 0, len(rows))
+ for _, row := range rows {
+ InstanceID := row.GetString(1)
+ InstanceIDs = append(InstanceIDs, InstanceID)
+ }
+ return InstanceIDs, nil
+}
- reorgHandle, err := se.execute(context.Background(), "select job_id, start_key, end_key, physical_id, ele_id, ele_type from mysql.tidb_ddl_reorg", "get_handle")
+// GetBackfillJobs gets the backfill jobs in the tblName table according to condition.
+func GetBackfillJobs(sess *session, tblName, condition string, label string) ([]*BackfillJob, error) {
+ rows, err := sess.execute(context.Background(), fmt.Sprintf("select * from mysql.%s where %s", tblName, condition), label)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ bJobs := make([]*BackfillJob, 0, len(rows))
+ for _, row := range rows {
+ key := row.GetString(2)
+ keySlice := strings.Split(key, "_")
+ jobID, err := strconv.ParseInt(keySlice[0], 10, 64)
if err != nil {
- return errors.Trace(err)
+ return nil, err
}
- for _, row := range reorgHandle {
- if err := t.UpdateDDLReorgHandle(row.GetInt64(0), row.GetBytes(1), row.GetBytes(2), row.GetInt64(3), &meta.Element{ID: row.GetInt64(4), TypeKey: row.GetBytes(5)}); err != nil {
- return errors.Trace(err)
- }
+ eleKey, err := hex.DecodeString(keySlice[1])
+ if err != nil {
+ return nil, err
}
-
- // clean up these 2 tables.
- _, err = se.execute(context.Background(), "delete from mysql.tidb_ddl_job", "delete_old_ddl")
+ eleID, err := strconv.ParseInt(keySlice[2], 10, 64)
if err != nil {
- return errors.Trace(err)
+ return nil, err
}
- _, err = se.execute(context.Background(), "delete from mysql.tidb_ddl_reorg", "delete_old_reorg")
+ id, err := strconv.ParseInt(keySlice[3], 10, 64)
if err != nil {
- return errors.Trace(err)
+ return nil, err
+ }
+ bfJob := BackfillJob{
+ ID: id,
+ JobID: jobID,
+ EleID: eleID,
+ EleKey: eleKey,
+ PhysicalTableID: row.GetInt64(3),
+ Tp: backfillerType(row.GetInt64(4)),
+ InstanceID: row.GetString(5),
+ InstanceLease: row.GetTime(6),
+ State: model.StrToJobState(row.GetString(7)),
+ StartTS: row.GetUint64(9),
+ StateUpdateTS: row.GetUint64(10),
+ }
+ bfJob.Meta = &model.BackfillMeta{}
+ err = bfJob.Meta.Decode(row.GetBytes(11))
+ if err != nil {
+ return nil, errors.Trace(err)
}
- return t.SetConcurrentDDL(false)
- })
+ bfJob.Meta.CurrKey = row.GetBytes(8)
+ bJobs = append(bJobs, &bfJob)
+ }
+ return bJobs, nil
}
-func runInTxn(se *session, f func(*session) error) (err error) {
- err = se.begin()
- if err != nil {
- return err
+// RemoveBackfillJob removes the backfill jobs from the tidb_background_subtask table.
+// If isOneEle is true, removes all jobs with backfillJob's ddl_job_id, ele_id and ele_key. Otherwise, removes the backfillJob.
+func RemoveBackfillJob(sess *session, isOneEle bool, backfillJob *BackfillJob) error {
+ sql := "delete from mysql.tidb_background_subtask"
+ if !isOneEle {
+ sql += fmt.Sprintf(" where task_key like \"%d_%s_%d_%d\"", backfillJob.JobID, hex.EncodeToString(backfillJob.EleKey), backfillJob.EleID, backfillJob.ID)
+ } else {
+ sql += fmt.Sprintf(" where task_key like \"%d_%s_%d_%%\"", backfillJob.JobID, hex.EncodeToString(backfillJob.EleKey), backfillJob.EleID)
}
- err = f(se)
+ _, err := sess.execute(context.Background(), sql, "remove_backfill_job")
+ return err
+}
+
+func updateBackfillJob(sess *session, tableName string, backfillJob *BackfillJob, label string) error {
+ mate, err := backfillJob.Meta.Encode()
if err != nil {
- se.rollback()
- return
+ return err
}
- return errors.Trace(se.commit())
+ sql := fmt.Sprintf("update mysql.%s set exec_id = '%s', exec_expired = '%s', state = '%s', checkpoint = %s, meta = %s where task_key = '%d_%s_%d_%d'",
+ tableName, backfillJob.InstanceID, backfillJob.InstanceLease, backfillJob.State.String(), wrapKey2String(backfillJob.Meta.CurrKey),
+ wrapKey2String(mate), backfillJob.JobID, hex.EncodeToString(backfillJob.EleKey), backfillJob.EleID, backfillJob.ID)
+ _, err = sess.execute(context.Background(), sql, label)
+ return err
}
diff --git a/ddl/job_table_test.go b/ddl/job_table_test.go
index 8f9e59ae31084..2f7a180f7b9ca 100644
--- a/ddl/job_table_test.go
+++ b/ddl/job_table_test.go
@@ -15,16 +15,28 @@
package ddl_test
import (
+ "context"
+ "encoding/hex"
+ "fmt"
+ "strconv"
+ "strings"
"sync"
"testing"
"time"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
+ "github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/parser/mysql"
+ "github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/sessiontxn"
"github.com/pingcap/tidb/testkit"
+ "github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util"
+ "github.com/pingcap/tidb/util/dbterror"
"github.com/stretchr/testify/require"
"golang.org/x/exp/slices"
)
@@ -33,9 +45,6 @@ import (
// This test checks the chosen job records to see if there are wrong scheduling, if job A and job B cannot run concurrently,
// then the all the record of job A must before or after job B, no cross record between these 2 jobs should be in between.
func TestDDLScheduling(t *testing.T) {
- if !variable.EnableConcurrentDDL.Load() {
- t.Skipf("test requires concurrent ddl")
- }
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
@@ -59,7 +68,7 @@ func TestDDLScheduling(t *testing.T) {
"ALTER TABLE e EXCHANGE PARTITION p1 WITH TABLE e3;",
}
- hook := &ddl.TestDDLCallback{}
+ hook := &callback.TestDDLCallback{}
var wg util.WaitGroupWrapper
wg.Add(1)
var once sync.Once
@@ -177,3 +186,640 @@ func check(t *testing.T, record []int64, ids ...int64) {
}
}
}
+
+func makeAddIdxBackfillJobs(schemaID, tblID, jobID, eleID int64, cnt int, query string) []*ddl.BackfillJob {
+ bJobs := make([]*ddl.BackfillJob, 0, cnt)
+ for i := 0; i < cnt; i++ {
+ sKey := []byte(fmt.Sprintf("%d", i))
+ eKey := []byte(fmt.Sprintf("%d", i+1))
+ bm := &model.BackfillMeta{
+ EndInclude: true,
+ JobMeta: &model.JobMeta{
+ SchemaID: schemaID,
+ TableID: tblID,
+ Query: query,
+ },
+ StartKey: sKey,
+ EndKey: eKey,
+ }
+ bj := &ddl.BackfillJob{
+ ID: int64(i),
+ JobID: jobID,
+ EleID: eleID,
+ EleKey: meta.IndexElementKey,
+ State: model.JobStateNone,
+ PhysicalTableID: 1,
+ InstanceLease: types.ZeroTimestamp,
+ Meta: bm,
+ }
+ bj.Meta.CurrKey = sKey
+ bJobs = append(bJobs, bj)
+ }
+ return bJobs
+}
+
+func equalBackfillJob(t *testing.T, a, b *ddl.BackfillJob, lessTime types.Time) {
+ require.Equal(t, a.ID, b.ID)
+ require.Equal(t, a.JobID, b.JobID)
+ require.Equal(t, a.EleID, b.EleID)
+ require.Equal(t, a.EleKey, b.EleKey)
+ require.Equal(t, a.PhysicalTableID, b.PhysicalTableID)
+ require.Equal(t, a.InstanceID, b.InstanceID)
+ require.GreaterOrEqual(t, b.InstanceLease.Compare(lessTime), 0)
+ require.Equal(t, a.State, b.State)
+ require.Equal(t, a.Meta, b.Meta)
+}
+
+func getIdxConditionStr(jobID, eleID int64) string {
+ return fmt.Sprintf("task_key like \"%d_%s_%d_%%\"",
+ jobID, hex.EncodeToString(meta.IndexElementKey), eleID)
+}
+
+func readInTxn(se sessionctx.Context, f func(sessionctx.Context)) (err error) {
+ err = sessiontxn.NewTxn(context.Background(), se)
+ if err != nil {
+ return err
+ }
+ f(se)
+ se.RollbackTxn(context.Background())
+ return nil
+}
+
+func backfillJob2PTblMetaMap(bJob *ddl.BackfillJob) map[int64]*ddl.BackfillJobRangeMeta {
+ m := &ddl.BackfillJobRangeMeta{
+ ID: bJob.ID,
+ PhyTblID: bJob.PhysicalTableID,
+ StartKey: bJob.Meta.StartKey,
+ EndKey: bJob.Meta.EndKey,
+ }
+ mMap := make(map[int64]*ddl.BackfillJobRangeMeta)
+ mMap[m.PhyTblID] = m
+ return mMap
+}
+
+func TestSimpleExecBackfillJobs(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ d := dom.DDL()
+ se := ddl.NewSession(tk.Session())
+
+ jobID1 := int64(1)
+ jobID2 := int64(2)
+ eleID1 := int64(11)
+ eleID2 := int64(22)
+ eleID3 := int64(33)
+ noPID := int64(0)
+ uuid := d.GetID()
+ eleKey := meta.IndexElementKey
+ instanceLease := ddl.InstanceLease
+
+ // test no backfill job
+ bJob, err := ddl.GetBackfillJobForOneEle(se, []int64{jobID1, jobID2}, instanceLease)
+ require.NoError(t, err)
+ require.Nil(t, bJob)
+ bJobs, err := ddl.GetAndMarkBackfillJobsForOneEle(se, 1, jobID1, uuid, noPID, instanceLease)
+ require.EqualError(t, err, dbterror.ErrDDLJobNotFound.FastGen("get zero backfill job").Error())
+ require.Nil(t, bJobs)
+ allCnt, err := ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskTable, getIdxConditionStr(jobID1, eleID2), "check_backfill_job_count")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, 0)
+ // Test some backfill jobs, add backfill jobs to the table.
+ cnt := 2
+ bjTestCases := make([]*ddl.BackfillJob, 0, cnt*3)
+ bJobs1 := makeAddIdxBackfillJobs(1, 2, jobID1, eleID1, cnt, "alter table t add index idx(a)")
+ bJobs2 := makeAddIdxBackfillJobs(1, 2, jobID2, eleID2, cnt, "alter table t add index idx(b)")
+ bJobs3 := makeAddIdxBackfillJobs(1, 2, jobID2, eleID3, cnt, "alter table t add index idx(c)")
+ bjTestCases = append(bjTestCases, bJobs1...)
+ bjTestCases = append(bjTestCases, bJobs2...)
+ bjTestCases = append(bjTestCases, bJobs3...)
+ err = ddl.AddBackfillJobs(se, bjTestCases)
+ require.Equal(t, err.Error(), "[table:1292]Incorrect timestamp value: '0000-00-00 00:00:00' for column 'exec_expired' at row 1")
+ tk.Session().GetSessionVars().SQLMode = mysql.ModeNone
+ err = ddl.AddBackfillJobs(se, bjTestCases)
+ // ID jobID eleID InstanceID PhysicalTableID
+ // --------------------------------------------------
+ // 0 jobID1 eleID1 uuid 1
+ // 1 jobID1 eleID1 "" 1
+ // 0 jobID2 eleID2 "" 1
+ // 1 jobID2 eleID2 "" 1
+ // 0 jobID2 eleID3 "" 1
+ // 1 jobID2 eleID3 "" 1
+ require.NoError(t, err)
+ // test get some backfill jobs
+ bJob, err = ddl.GetBackfillJobForOneEle(se, []int64{jobID2 - 1, jobID2 + 1}, instanceLease)
+ require.NoError(t, err)
+ expectJob := bjTestCases[2]
+ if expectJob.ID != bJob.ID {
+ expectJob = bjTestCases[3]
+ }
+ require.Equal(t, expectJob, bJob)
+ previousTime, err := ddl.GetOracleTimeWithStartTS(se)
+ require.EqualError(t, err, "[kv:8024]invalid transaction")
+ readInTxn(se, func(sessionctx.Context) {
+ previousTime, err = ddl.GetOracleTimeWithStartTS(se)
+ require.NoError(t, err)
+ })
+
+ bJobs, err = ddl.GetAndMarkBackfillJobsForOneEle(se, 1, jobID2, uuid, noPID, instanceLease)
+ require.NoError(t, err)
+ require.Len(t, bJobs, 1)
+ expectJob = bjTestCases[2]
+ if expectJob.ID != bJobs[0].ID {
+ expectJob = bjTestCases[3]
+ }
+ expectJob.InstanceID = uuid
+ equalBackfillJob(t, expectJob, bJobs[0], ddl.GetLeaseGoTime(previousTime, instanceLease))
+ var currTime time.Time
+ readInTxn(se, func(sessionctx.Context) {
+ currTime, err = ddl.GetOracleTimeWithStartTS(se)
+ require.NoError(t, err)
+ })
+ currGoTime := ddl.GetLeaseGoTime(currTime, instanceLease)
+ require.GreaterOrEqual(t, currGoTime.Compare(bJobs[0].InstanceLease), 0)
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskTable, getIdxConditionStr(jobID2, eleID2), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, cnt)
+ // test physical table
+ err = ddl.RemoveBackfillJob(se, true, bJobs1[0])
+ require.NoError(t, err)
+ // ID jobID eleID InstanceID PhysicalTableID
+ // --------------------------------------------------
+ // 0 jobID2 eleID2 "" 1
+ // 1 jobID2 eleID2 "" 1
+ // 0 jobID2 eleID3 "" 1
+ // 1 jobID2 eleID3 "" 1
+ bPhyJobs := makeAddIdxBackfillJobs(1, 2, jobID1, eleID1, 10, "alter table t add index idx(a)")
+ bPhyJobs[1].InstanceID = "uuid_1"
+ bPhyJobs[2].PhysicalTableID = 2
+ bPhyJobs[6].PhysicalTableID = 2
+ bPhyJobs[4].PhysicalTableID = 3
+ bPhyJobs[5].PhysicalTableID = 3
+ bPhyJobs[8].PhysicalTableID = 3
+ bPhyJobs[7].PhysicalTableID = 4
+ bPhyJobs[9].PhysicalTableID = 4
+ // ID jobID eleID InstanceID InstanceLease PhysicalTableID
+ // -----------------------------------------------------------------------
+ // 0 jobID2 eleID2 "" 1
+ // 1 jobID2 eleID2 "" 1
+ // 0 jobID2 eleID3 "" 1
+ // 1 jobID2 eleID3 "" 1
+ // 0 jobID1 eleID1 "" 1
+ // 1 jobID1 eleID1 "uuid_1" 1
+ // 2 jobID1 eleID1 "" 2
+ // 3 jobID1 eleID1 "" 1
+ // 4 jobID1 eleID1 "" 3
+ // 5 jobID1 eleID1 "" 3
+ // 6 jobID1 eleID1 "" 2
+ // 7 jobID1 eleID1 "" 4
+ // 8 jobID1 eleID1 "" 3
+ // 9 jobID1 eleID1 "" 4
+ simpleCheck := func(batch, jobCnt int, bfJobIDs []int64, pID int64) {
+ err = ddl.AddBackfillJobs(se, bPhyJobs)
+ require.NoError(t, err)
+ bJobs, err = ddl.GetAndMarkBackfillJobsForOneEle(se, batch, jobID1, uuid, pID, instanceLease)
+ require.NoError(t, err)
+ require.Len(t, bJobs, jobCnt)
+ isExist := false
+ for _, id := range bfJobIDs {
+ if id == bJobs[0].ID {
+ isExist = true
+ }
+ }
+ require.True(t, isExist, fmt.Sprintf("expected ids:%v, actual id:%d", bfJobIDs, bJobs[0].ID))
+ err = ddl.RemoveBackfillJob(se, true, bJobs1[0])
+ require.NoError(t, err)
+ }
+ type cntAndID struct {
+ batch int
+ bfJobCnt int
+ bfJobID []int64
+ }
+ checkAndClean := func(expectRet1, expectRet2 cntAndID) {
+ simpleCheck(expectRet1.batch, expectRet1.bfJobCnt, expectRet1.bfJobID, noPID)
+ simpleCheck(expectRet2.batch, expectRet2.bfJobCnt, expectRet2.bfJobID, ddl.GetJobWithoutPartition)
+ }
+ checkAndClean(cntAndID{3, 3, []int64{0, 1, 3}},
+ cntAndID{3, 3, []int64{0, 1, 3}})
+ bPhyJobs[1].InstanceLease = types.NewTime(types.FromGoTime(time.Now().Add(-time.Hour).UTC()), 0, 0)
+ // ID jobID eleID InstanceID InstanceLease PhysicalTableID
+ // -----------------------------------------------------------------------
+ // 0 jobID2 eleID2 "" 1
+ // 1 jobID2 eleID2 "" 1
+ // 0 jobID2 eleID3 "" 1
+ // 1 jobID2 eleID3 "" 1
+ // 0 jobID1 eleID1 "" 1
+ // 1 jobID1 eleID1 "uuid_1" currentTime-hour 1
+ // 2 jobID1 eleID1 "" 2
+ // 3 jobID1 eleID1 "" 1
+ // 4 jobID1 eleID1 "" 3
+ // 5 jobID1 eleID1 "" 3
+ // 6 jobID1 eleID1 "" 2
+ // 7 jobID1 eleID1 "" 4
+ // 8 jobID1 eleID1 "" 3
+ // 9 jobID1 eleID1 "" 4
+ checkAndClean(cntAndID{3, 3, []int64{0, 1, 3}},
+ cntAndID{3, 3, []int64{0, 1, 3}})
+ bPhyJobs[3].InstanceLease = types.NewTime(types.FromGoTime(time.Now().UTC()), 0, 0)
+ // ID jobID eleID InstanceID InstanceLease PhysicalTableID
+ // -----------------------------------------------------------------------
+ // 0 jobID2 eleID2 "" 1
+ // 1 jobID2 eleID2 "" 1
+ // 0 jobID2 eleID3 "" 1
+ // 1 jobID2 eleID3 "" 1
+ // 0 jobID1 eleID1 "" 1
+ // 1 jobID1 eleID1 "uuid_1" currentTime-hour 1
+ // 2 jobID1 eleID1 "" 2
+ // 3 jobID1 eleID1 "" currentTime 1 // should not exist
+ // 4 jobID1 eleID1 "" 3
+ // 5 jobID1 eleID1 "" 3
+ // 6 jobID1 eleID1 "" 2
+ // 7 jobID1 eleID1 "" 4
+ // 8 jobID1 eleID1 "" 3
+ // 9 jobID1 eleID1 "" 4
+ checkAndClean(cntAndID{3, 2, []int64{2, 6}},
+ cntAndID{3, 3, []int64{0, 1, 3}})
+ bPhyJobs[6].InstanceLease = types.NewTime(types.FromGoTime(time.Now().UTC()), 0, 0)
+ // ID jobID eleID InstanceID InstanceLease PhysicalTableID
+ // -----------------------------------------------------------------------
+ // 0 jobID2 eleID2 "" 1
+ // 1 jobID2 eleID2 "" 1
+ // 0 jobID2 eleID3 "" 1
+ // 1 jobID2 eleID3 "" 1
+ // 0 jobID1 eleID1 "" 1
+ // 1 jobID1 eleID1 "uuid_1" currentTime-hour 1
+ // 2 jobID1 eleID1 "" 2
+ // 3 jobID1 eleID1 "" currentTime 1 // should not exist
+ // 4 jobID1 eleID1 "" 3
+ // 5 jobID1 eleID1 "" 3
+ // 6 jobID1 eleID1 "" currentTime 2 // should not exist
+ // 7 jobID1 eleID1 "" 4
+ // 8 jobID1 eleID1 "" 3
+ // 9 jobID1 eleID1 "" 4
+ checkAndClean(cntAndID{3, 2, []int64{2, 6}},
+ cntAndID{10, 10, []int64{0, 1, 3}})
+ bPhyJobs[6].InstanceID = "uuid_2"
+ // ID jobID eleID InstanceID InstanceLease PhysicalTableID
+ // -----------------------------------------------------------------------
+ // 0 jobID2 eleID2 "" 1
+ // 1 jobID2 eleID2 "" 1
+ // 0 jobID2 eleID3 "" 1
+ // 1 jobID2 eleID3 "" 1
+ // 0 jobID1 eleID1 "" 1
+ // 1 jobID1 eleID1 "uuid_1" currentTime-hour 1
+ // 2 jobID1 eleID1 "" 2
+ // 3 jobID1 eleID1 "" currentTime 1 // should not exist
+ // 4 jobID1 eleID1 "" 3
+ // 5 jobID1 eleID1 "" 3
+ // 6 jobID1 eleID1 "uuid_2" currentTime 2 // should not exist
+ // 7 jobID1 eleID1 "" 4
+ // 8 jobID1 eleID1 "" 3
+ // 9 jobID1 eleID1 "" 4
+ checkAndClean(cntAndID{3, 3, []int64{4, 5, 8}},
+ cntAndID{10, 9, []int64{0, 1, 3}})
+ // ID jobID eleID InstanceID InstanceLease PhysicalTableID
+ // -----------------------------------------------------------------------
+ // 0 jobID2 eleID2 "" 1
+ // 1 jobID2 eleID2 "" 1
+ // 0 jobID2 eleID3 "" 1
+ // 1 jobID2 eleID3 "" 1
+
+ err = ddl.AddBackfillJobs(se, bJobs1)
+ require.NoError(t, err)
+ // ID jobID eleID
+ // ------------------------
+ // 0 jobID1 eleID1
+ // 1 jobID1 eleID1
+ // 0 jobID2 eleID2
+ // 1 jobID2 eleID2
+ // 0 jobID2 eleID3
+ // 1 jobID2 eleID3
+ // remove a backfill job
+ err = ddl.RemoveBackfillJob(se, false, bJobs1[0])
+ // ID jobID eleID
+ // ------------------------
+ // 1 jobID1 eleID1
+ // 0 jobID2 eleID2
+ // 1 jobID2 eleID2
+ // 0 jobID2 eleID3
+ // 1 jobID2 eleID3
+ require.NoError(t, err)
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskTable, getIdxConditionStr(jobID1, eleID1), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, 1)
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskTable, getIdxConditionStr(jobID2, eleID2), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, cnt)
+ // remove all backfill jobs
+ err = ddl.RemoveBackfillJob(se, true, bJobs2[0])
+ // ID jobID eleID
+ // ------------------------
+ // 1 jobID1 eleID1
+ // 0 jobID2 eleID3
+ // 1 jobID2 eleID3
+ require.NoError(t, err)
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskTable, getIdxConditionStr(jobID1, eleID1), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, 1)
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskTable, getIdxConditionStr(jobID2, eleID2), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, 0)
+ // clean backfill job
+ err = ddl.RemoveBackfillJob(se, true, bJobs1[1])
+ require.NoError(t, err)
+ err = ddl.RemoveBackfillJob(se, true, bJobs3[0])
+ require.NoError(t, err)
+ // ID jobID eleID
+ // ------------------------
+
+ // test history backfill jobs
+ err = ddl.AddBackfillHistoryJob(se, []*ddl.BackfillJob{bJobs2[0]})
+ require.NoError(t, err)
+ // ID jobID eleID
+ // ------------------------
+ // 0 jobID2 eleID2
+ readInTxn(se, func(sessionctx.Context) {
+ currTime, err = ddl.GetOracleTimeWithStartTS(se)
+ require.NoError(t, err)
+ })
+ condition := fmt.Sprintf("exec_id = '' or exec_expired < '%v' and task_key like \"%d_%%\"", currTime.Add(-instanceLease), jobID2)
+ bJobs, err = ddl.GetBackfillJobs(se, ddl.BackgroundSubtaskHistoryTable, condition, "test_get_bj")
+ require.NoError(t, err)
+ require.Len(t, bJobs, 1)
+ require.Equal(t, bJobs[0].StateUpdateTS, uint64(0))
+
+ // test GetMaxBackfillJob
+ pTblMeta, err := ddl.GetPhysicalTableMetas(se, bJobs3[0].JobID, bJobs3[0].EleID, eleKey)
+ require.NoError(t, err)
+ require.Len(t, pTblMeta, 0)
+ err = ddl.AddBackfillJobs(se, bjTestCases)
+ require.NoError(t, err)
+ // ID jobID eleID
+ // ------------------------
+ // 0 jobID1 eleID1
+ // 1 jobID1 eleID1
+ // 0 jobID2 eleID2
+ // 1 jobID2 eleID2
+ // 0 jobID2 eleID3
+ // 1 jobID2 eleID3
+ pTblMeta, err = ddl.GetPhysicalTableMetas(se, jobID2, eleID2, eleKey)
+ require.NoError(t, err)
+ require.Equal(t, backfillJob2PTblMetaMap(bJobs2[1]), pTblMeta)
+ bJobs1[0].State = model.JobStateRollingback
+ bJobs1[0].ID = 2
+ bJobs1[0].InstanceID = uuid
+ bJobs1[1].State = model.JobStateCancelled
+ bJobs1[1].ID = 3
+ bJobs1[1].Meta.Error = dbterror.ErrCancelledDDLJob
+ err = ddl.AddBackfillJobs(se, bJobs1)
+ require.NoError(t, err)
+ // ID jobID eleID state
+ // --------------------------------
+ // 0 jobID1 eleID1 JobStateNone
+ // 1 jobID1 eleID1 JobStateNone
+ // 0 jobID2 eleID2 JobStateNone
+ // 1 jobID2 eleID2 JobStateNone
+ // 0 jobID2 eleID3 JobStateNone
+ // 1 jobID2 eleID3 JobStateNone
+ // 2 jobID1 eleID1 JobStateRollingback
+ // 3 jobID1 eleID1 JobStateCancelled
+ pTblMeta, err = ddl.GetPhysicalTableMetas(se, jobID1, eleID1, eleKey)
+ require.NoError(t, err)
+ require.Equal(t, backfillJob2PTblMetaMap(bJobs1[1]), pTblMeta)
+ // test the BackfillJob's AbbrStr
+ require.Equal(t, fmt.Sprintf("ID:2, JobID:1, EleID:11, Type:add index, State:rollingback, InstanceID:%s, InstanceLease:0000-00-00 00:00:00", uuid), bJobs1[0].AbbrStr())
+ require.Equal(t, "ID:3, JobID:1, EleID:11, Type:add index, State:cancelled, InstanceID:, InstanceLease:0000-00-00 00:00:00", bJobs1[1].AbbrStr())
+ require.Equal(t, "ID:0, JobID:2, EleID:33, Type:add index, State:none, InstanceID:, InstanceLease:0000-00-00 00:00:00", bJobs3[0].AbbrStr())
+ require.Equal(t, "ID:1, JobID:2, EleID:33, Type:add index, State:none, InstanceID:, InstanceLease:0000-00-00 00:00:00", bJobs3[1].AbbrStr())
+ // test select tidb_background_subtask
+ tk.MustQuery(fmt.Sprintf("select exec_id, exec_expired from mysql.tidb_background_subtask where task_key like \"%%%d\" and %s", bJobs1[0].ID, getIdxConditionStr(jobID1, eleID1))).
+ Check(testkit.Rows(fmt.Sprintf("%s 0000-00-00 00:00:00", uuid)))
+ tk.MustQuery(fmt.Sprintf("select exec_id, exec_expired from mysql.tidb_background_subtask where task_key like \"%%%d\" and %s", bJobs1[1].ID, getIdxConditionStr(jobID1, eleID1))).
+ Check(testkit.Rows(" 0000-00-00 00:00:00"))
+ // test GetBackfillMetas
+ bfErr := ddl.GetBackfillErr(se, jobID1, eleID1, eleKey)
+ require.Error(t, bfErr, dbterror.ErrCancelledDDLJob)
+ bfErr = ddl.GetBackfillErr(se, jobID2, eleID2, eleKey)
+ require.NoError(t, bfErr)
+
+ bJobs1[0].State = model.JobStateNone
+ bJobs1[0].ID = 5
+ bJobs1[1].State = model.JobStateNone
+ bJobs1[1].ID = 4
+ err = ddl.AddBackfillHistoryJob(se, bJobs1)
+ // BackgroundSubtaskTable
+ // ID jobID eleID state
+ // --------------------------------
+ // 0 jobID1 eleID1 JobStateNone
+ // 1 jobID1 eleID1 JobStateNone
+ // 0 jobID2 eleID2 JobStateNone
+ // 1 jobID2 eleID2 JobStateNone
+ // 0 jobID2 eleID3 JobStateNone
+ // 1 jobID2 eleID3 JobStateNone
+ // 2 jobID1 eleID1 JobStateRollingback
+ // 3 jobID1 eleID1 JobStateCancelled
+ //
+ // BackgroundSubtaskHistoryTable
+ // ID jobID eleID state
+ // --------------------------------
+ // 5 jobID1 eleID1 JobStateNone
+ // 4 jobID1 eleID1 JobStateNone
+ pTblMeta, err = ddl.GetPhysicalTableMetas(se, jobID1, eleID1, eleKey)
+ require.NoError(t, err)
+ require.Equal(t, backfillJob2PTblMetaMap(bJobs1[0]), pTblMeta) // ???????????
+ bJobs1[0].ID = 6
+ bJobs1[1].ID = 7
+ err = ddl.AddBackfillJobs(se, bJobs1)
+ // BackgroundSubtaskTable
+ // ID jobID eleID state
+ // --------------------------------
+ // 0 jobID1 eleID1 JobStateNone
+ // 1 jobID1 eleID1 JobStateNone
+ // 0 jobID2 eleID2 JobStateNone
+ // 1 jobID2 eleID2 JobStateNone
+ // 0 jobID2 eleID3 JobStateNone
+ // 1 jobID2 eleID3 JobStateNone
+ // 2 jobID1 eleID1 JobStateRollingback
+ // 3 jobID1 eleID1 JobStateCancelled
+ // 6 jobID1 eleID1 JobStateNone
+ // 7 jobID1 eleID1 JobStateNone
+ //
+ // BackgroundSubtaskHistoryTable
+ // ID jobID eleID state
+ // --------------------------------
+ // 5 jobID1 eleID1 JobStateNone
+ // 4 jobID1 eleID1 JobStateNone
+ pTblMeta, err = ddl.GetPhysicalTableMetas(se, jobID1, eleID1, eleKey)
+ require.NoError(t, err)
+ require.Equal(t, backfillJob2PTblMetaMap(bJobs1[1]), pTblMeta)
+
+ // test MoveBackfillJobsToHistoryTable and GetInterruptedBackfillJobForOneEle
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskTable, getIdxConditionStr(jobID2, eleID3), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, 2)
+ err = ddl.MoveBackfillJobsToHistoryTable(se, bJobs3[0])
+ require.NoError(t, err)
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskTable, getIdxConditionStr(jobID2, eleID3), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, 0)
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskHistoryTable, getIdxConditionStr(jobID2, eleID3), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, 2)
+ // BackgroundSubtaskTable
+ // ID jobID eleID state
+ // --------------------------------
+ // 0 jobID1 eleID1 JobStateNone
+ // 1 jobID1 eleID1 JobStateNone
+ // 0 jobID2 eleID2 JobStateNone
+ // 1 jobID2 eleID2 JobStateNone
+ // 2 jobID1 eleID1 JobStateRollingback
+ // 3 jobID1 eleID1 JobStateCancelled
+ // 6 jobID1 eleID1 JobStateNone
+ // 7 jobID1 eleID1 JobStateNone
+ //
+ // BackgroundSubtaskHistoryTable
+ // ID jobID eleID state
+ // --------------------------------
+ // 5 jobID1 eleID1 JobStateNone
+ // 4 jobID1 eleID1 JobStateNone
+ // 0 jobID2 eleID3 JobStateCancelled
+ // 1 jobID2 eleID3 JobStateCancelled
+ bJobs, err = ddl.GetInterruptedBackfillJobForOneEle(se, jobID1, eleID1, eleKey)
+ require.NoError(t, err)
+ require.Len(t, bJobs, 0)
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskTable, getIdxConditionStr(jobID1, eleID1), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, 6)
+ err = ddl.MoveBackfillJobsToHistoryTable(se, bJobs1[0])
+ require.NoError(t, err)
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskTable, getIdxConditionStr(jobID1, eleID1), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, 0)
+ allCnt, err = ddl.GetBackfillJobCount(se, ddl.BackgroundSubtaskHistoryTable, getIdxConditionStr(jobID1, eleID1), "test_get_bj")
+ require.NoError(t, err)
+ require.Equal(t, allCnt, 8)
+ bJobs, err = ddl.GetInterruptedBackfillJobForOneEle(se, jobID2, eleID3, eleKey)
+ require.NoError(t, err)
+ require.Len(t, bJobs, 1)
+ expectJob = bJobs3[0]
+ if expectJob.ID != bJob.ID {
+ expectJob = bJobs3[1]
+ }
+ expectJob.State = model.JobStateCancelled
+ equalBackfillJob(t, bJobs3[0], bJobs[0], types.ZeroTimestamp)
+ // BackgroundSubtaskTable
+ // ID jobID eleID state
+ // --------------------------------
+ // 0 jobID2 eleID2 JobStateNone
+ // 1 jobID2 eleID2 JobStateNone
+ //
+ // BackgroundSubtaskHistoryTable
+ // ID jobID eleID state
+ // --------------------------------
+ // 5 jobID1 eleID1 JobStateNone
+ // 4 jobID1 eleID1 JobStateNone
+ // 0 jobID2 eleID3 JobStateCancelled
+ // 1 jobID2 eleID3 JobStateCancelled
+ // 0 jobID1 eleID1 JobStateCancelled
+ // 1 jobID1 eleID1 JobStateCancelled
+ // 2 jobID1 eleID1 JobStateCancelled
+ // 3 jobID1 eleID1 JobStateCancelled
+ // 6 jobID1 eleID1 JobStateCancelled
+ // 7 jobID1 eleID1 JobStateCancelled
+}
+
+func TestGetTasks(t *testing.T) {
+ // TODO: update the variable of `enableDistReorg`
+ isDistReorg := variable.DDLEnableDistributeReorg.Load()
+ variable.DDLEnableDistributeReorg.Store(false)
+ defer func() { variable.DDLEnableDistributeReorg.Store(isDistReorg) }()
+
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ se := ddl.NewSession(tk.Session())
+ se.GetSessionVars().SQLMode = mysql.ModeNone
+ d := dom.DDL()
+
+ jobID1 := int64(1)
+ eleID1 := int64(11)
+ uuid := d.GetID()
+ cnt := 3
+ instanceLease := ddl.InstanceLease
+ bJobsTestCases := makeAddIdxBackfillJobs(1, 2, jobID1, eleID1, cnt, "alter table t add index idx(a)")
+ err := ddl.AddBackfillJobs(se, bJobsTestCases)
+ require.NoError(t, err)
+
+ var wg util.WaitGroupWrapper
+ // Mock GetAndMarkBackfillJobsForOneEle gets a writing conflict error.
+ // Step 1: se1 begins txn1.
+ // Step 2: se2 begins txn2.
+ // Step 3: execute txn1 and txn2, then txn1 or txn2 returns a writing conflict error.
+ var err1 error
+ ch := make(chan struct{}, 1)
+ wg.Run(func() {
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/NotifyBeginTxnCh", `return(1)`))
+ ch <- struct{}{}
+ var bJobs []*ddl.BackfillJob
+ bJobs, err = ddl.GetAndMarkBackfillJobsForOneEle(se, 1, jobID1, uuid, 1, instanceLease)
+ require.Len(t, bJobs, 1)
+ })
+ <-ch
+ defer func() { require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/NotifyBeginTxnCh")) }()
+ wg.Run(func() {
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+ se1 := ddl.NewSession(tk1.Session())
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/NotifyBeginTxnCh", `return(2)`))
+ var bJobs1 []*ddl.BackfillJob
+ bJobs1, err1 = ddl.GetAndMarkBackfillJobsForOneEle(se1, 1, jobID1, uuid, 1, instanceLease)
+ require.Len(t, bJobs1, 1)
+ })
+ wg.Wait()
+ if err == nil {
+ require.NotNil(t, err1)
+ require.True(t, strings.Contains(err1.Error(), "[kv:9007]Write conflict"))
+ } else {
+ require.Nil(t, err1)
+ require.True(t, strings.Contains(err.Error(), "[kv:9007]Write conflict"))
+ }
+
+ err = ddl.RemoveBackfillJob(se, true, bJobsTestCases[0])
+ require.NoError(t, err)
+ err = ddl.AddBackfillJobs(se, bJobsTestCases)
+ require.NoError(t, err)
+ // get tbl
+ tk.MustExec("create table t(a int, b int)")
+ var tableID int64
+ rs := tk.MustQuery("select TIDB_TABLE_ID from information_schema.tables where table_name='t' and table_schema='test';")
+ tableIDi, err := strconv.Atoi(rs.Rows()[0][0].(string))
+ require.Nil(t, err)
+ tableID = int64(tableIDi)
+ tbl := testGetTable(t, dom, tableID)
+ pID := int64(0)
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/NotifyBeginTxnCh", `return(0)`))
+ // Mock GetAndMarkBackfillJobsForOneEle gets a writing conflict error, but getTasks is successful.
+ // Step 1: se1 begins txn1.
+ // Step 2: se2 begins txn2.
+ // Step 3: execute txn1 and txn2, then txn1 or txn2 returns a writing conflict error.
+ // Step 4: se2 begin txn3.
+ // Step 5: getTasks(txn3) executes successfully.
+ wg.Run(func() {
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/NotifyBeginTxnCh", `return(1)`))
+ ch <- struct{}{}
+ bJobs, err := ddl.GetTasks(ddl.GetDDLCtx(d), se, tbl, jobID1, &pID, 1)
+ require.Nil(t, err)
+ require.Len(t, bJobs, 1)
+ })
+ <-ch
+ wg.Run(func() {
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+ se1 := ddl.NewSession(tk1.Session())
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/NotifyBeginTxnCh", `return(2)`))
+ bJobs1, err1 := ddl.GetTasks(ddl.GetDDLCtx(d), se1, tbl, jobID1, &pID, 1)
+ require.Nil(t, err1)
+ require.Len(t, bJobs1, 1)
+ })
+ wg.Wait()
+}
diff --git a/ddl/main_test.go b/ddl/main_test.go
index 3418d16a23ece..b1bbd8516fa94 100644
--- a/ddl/main_test.go
+++ b/ddl/main_test.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/domain/infosync"
+ "github.com/pingcap/tidb/keyspace"
"github.com/pingcap/tidb/meta/autoid"
"github.com/pingcap/tidb/testkit/testsetup"
"github.com/tikv/client-go/v2/tikv"
@@ -40,6 +41,8 @@ func TestMain(m *testing.M) {
autoid.SetStep(5000)
ddl.ReorgWaitTimeout = 30 * time.Millisecond
+ ddl.RetrySQLInterval = 30 * time.Millisecond
+ ddl.CheckBackfillJobFinishInterval = 50 * time.Millisecond
ddl.RunInGoTest = true
ddl.SetBatchInsertDeleteRangeSize(2)
@@ -52,7 +55,7 @@ func TestMain(m *testing.M) {
conf.Experimental.AllowsExpressionIndex = true
})
- _, err := infosync.GlobalInfoSyncerInit(context.Background(), "t", func() uint64 { return 1 }, nil, true)
+ _, err := infosync.GlobalInfoSyncerInit(context.Background(), "t", func() uint64 { return 1 }, nil, nil, nil, keyspace.CodecV1, true)
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "ddl: infosync.GlobalInfoSyncerInit: %v\n", err)
os.Exit(1)
@@ -64,6 +67,8 @@ func TestMain(m *testing.M) {
goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"),
goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
+ goleak.IgnoreTopFunction("internal/poll.runtime_pollWait"),
+ goleak.IgnoreTopFunction("net/http.(*persistConn).writeLoop"),
}
goleak.VerifyTestMain(m, opts...)
diff --git a/ddl/metadatalocktest/BUILD.bazel b/ddl/metadatalocktest/BUILD.bazel
index 7f1fc31f9987a..4e36531bd5a6a 100644
--- a/ddl/metadatalocktest/BUILD.bazel
+++ b/ddl/metadatalocktest/BUILD.bazel
@@ -2,6 +2,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test")
go_test(
name = "metadatalocktest_test",
+ timeout = "short",
srcs = [
"main_test.go",
"mdl_test.go",
diff --git a/ddl/metadatalocktest/mdl_test.go b/ddl/metadatalocktest/mdl_test.go
index 6f4a0012cdfff..d7c05fa334508 100644
--- a/ddl/metadatalocktest/mdl_test.go
+++ b/ddl/metadatalocktest/mdl_test.go
@@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-//go:build !featuretag
-
package metadatalocktest
import (
@@ -257,6 +255,47 @@ func TestMDLBasicBatchPointGet(t *testing.T) {
require.Less(t, ts1, ts2)
}
+func TestMDLAddForeignKey(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ sv := server.CreateMockServer(t, store)
+
+ sv.SetDomain(dom)
+ dom.InfoSyncer().SetSessionManager(sv)
+ defer sv.Close()
+
+ conn1 := server.CreateMockConn(t, sv)
+ tk := testkit.NewTestKitWithSession(t, store, conn1.Context().Session)
+ conn2 := server.CreateMockConn(t, sv)
+ tkDDL := testkit.NewTestKitWithSession(t, store, conn2.Context().Session)
+ tk.MustExec("use test")
+ tk.MustExec("set global tidb_enable_metadata_lock=1")
+ tk.MustExec("create table t1(id int key);")
+ tk.MustExec("create table t2(id int key);")
+
+ tk.MustExec("begin")
+ tk.MustExec("insert into t2 values(1);")
+
+ var wg sync.WaitGroup
+ var ddlErr error
+ wg.Add(1)
+ var ts2 time.Time
+ go func() {
+ defer wg.Done()
+ ddlErr = tkDDL.ExecToErr("alter table test.t2 add foreign key (id) references t1(id)")
+ ts2 = time.Now()
+ }()
+
+ time.Sleep(2 * time.Second)
+
+ ts1 := time.Now()
+ tk.MustExec("commit")
+
+ wg.Wait()
+ require.Error(t, ddlErr)
+ require.Equal(t, "[ddl:1452]Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `fk_1` FOREIGN KEY (`id`) REFERENCES `t1` (`id`))", ddlErr.Error())
+ require.Less(t, ts1, ts2)
+}
+
func TestMDLRRUpdateSchema(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
sv := server.CreateMockServer(t, store)
@@ -1105,3 +1144,16 @@ func TestMDLRenameTable(t *testing.T) {
tk.MustGetErrCode("select * from test2.t1;", mysql.ErrNoSuchTable)
tk.MustExec("commit")
}
+
+func TestMDLPrepareFail(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk2 := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int);")
+ _, _, _, err := tk.Session().PrepareStmt("select b from t")
+ require.Error(t, err)
+
+ tk2.MustExec("alter table test.t add column c int")
+}
diff --git a/ddl/modify_column_test.go b/ddl/modify_column_test.go
index 10a9835bc8215..7aa23d526bc65 100644
--- a/ddl/modify_column_test.go
+++ b/ddl/modify_column_test.go
@@ -17,12 +17,14 @@ package ddl_test
import (
"context"
"fmt"
+ "strconv"
"sync"
"testing"
"time"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/parser/ast"
@@ -50,6 +52,14 @@ func batchInsert(tk *testkit.TestKit, tbl string, start, end int) {
func TestModifyColumnReorgInfo(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
+ originalTimeout := ddl.ReorgWaitTimeout
+ ddl.ReorgWaitTimeout = 10 * time.Millisecond
+ limit := variable.GetDDLErrorCountLimit()
+ variable.SetDDLErrorCountLimit(5)
+ defer func() {
+ ddl.ReorgWaitTimeout = originalTimeout
+ variable.SetDDLErrorCountLimit(limit)
+ }()
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t1")
@@ -66,7 +76,7 @@ func TestModifyColumnReorgInfo(t *testing.T) {
tbl := external.GetTableByName(t, tk, "test", "t1")
// Check insert null before job first update.
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
var currJob *model.Job
var elements []*meta.Element
@@ -112,14 +122,18 @@ func TestModifyColumnReorgInfo(t *testing.T) {
require.NoError(t, checkErr)
// Check whether the reorg information is cleaned up when executing "modify column" failed.
checkReorgHandle := func(gotElements, expectedElements []*meta.Element) {
+ require.Equal(t, len(expectedElements), len(gotElements))
for i, e := range gotElements {
require.Equal(t, expectedElements[i], e)
}
+ // check the consistency of the tables.
+ currJobID := strconv.FormatInt(currJob.ID, 10)
+ tk.MustQuery("select job_id, reorg, schema_ids, table_ids, type, processing from mysql.tidb_ddl_job where job_id = " + currJobID).Check(testkit.Rows())
+ tk.MustQuery("select job_id from mysql.tidb_ddl_history where job_id = " + currJobID).Check(testkit.Rows(currJobID))
+ tk.MustQuery("select job_id, ele_id, ele_type, physical_id from mysql.tidb_ddl_reorg where job_id = " + currJobID).Check(testkit.Rows())
require.NoError(t, sessiontxn.NewTxn(context.Background(), ctx))
- txn, err := ctx.Txn(true)
- require.NoError(t, err)
- m := meta.NewMeta(txn)
- e, start, end, physicalID, err := ddl.NewReorgHandlerForTest(m, testkit.NewTestKit(t, store).Session()).GetDDLReorgHandle(currJob)
+ e, start, end, physicalID, err := ddl.NewReorgHandlerForTest(testkit.NewTestKit(t, store).Session()).GetDDLReorgHandle(currJob)
+ require.Error(t, err, "Error not ErrDDLReorgElementNotExists, found orphan row in tidb_ddl_reorg for job.ID %d: e: '%s', physicalID: %d, start: 0x%x end: 0x%x", currJob.ID, e, physicalID, start, end)
require.True(t, meta.ErrDDLReorgElementNotExist.Equal(err))
require.Nil(t, e)
require.Nil(t, start)
@@ -149,10 +163,16 @@ func TestModifyColumnReorgInfo(t *testing.T) {
// Test encountering a "notOwnerErr" error which caused the processing backfill job to exit halfway.
// During the period, the old TiDB version(do not exist the element information) is upgraded to the new TiDB version.
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/MockGetIndexRecordErr", `return("addIdxNotOwnerErr")`))
- tk.MustExec("alter table t1 add index idx2(c1)")
- expectedElements = []*meta.Element{
- {ID: 7, TypeKey: meta.IndexElementKey}}
- checkReorgHandle(elements, expectedElements)
+ // TODO: Remove this check after "err" isn't nil in runReorgJobAndHandleErr.
+ if variable.DDLEnableDistributeReorg.Load() {
+ err = tk.ExecToErr("alter table t1 add index idx2(c1)")
+ require.EqualError(t, err, "[ddl:8201]TiDB server is not a DDL owner")
+ } else {
+ tk.MustExec("alter table t1 add index idx2(c1)")
+ expectedElements = []*meta.Element{
+ {ID: 7, TypeKey: meta.IndexElementKey}}
+ checkReorgHandle(elements, expectedElements)
+ }
tk.MustExec("admin check table t1")
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/MockGetIndexRecordErr"))
}
@@ -188,7 +208,7 @@ func TestModifyColumnNullToNotNull(t *testing.T) {
tbl := external.GetTableByName(t, tk1, "test", "t1")
// Check insert null before job first update.
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
tk1.MustExec("delete from t1")
once := sync.Once{}
var checkErr error
@@ -243,7 +263,7 @@ func TestModifyColumnNullToNotNullWithChangingVal(t *testing.T) {
tbl := external.GetTableByName(t, tk1, "test", "t1")
// Check insert null before job first update.
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
tk1.MustExec("delete from t1")
once := sync.Once{}
var checkErr error
@@ -813,10 +833,7 @@ func TestModifyColumnTypeWithWarnings(t *testing.T) {
// 111.22 will be truncated the fraction .22 as .2 with truncated warning for each row.
tk.MustExec("alter table t modify column a decimal(4,1)")
// there should 4 rows of warnings corresponding to the origin rows.
- tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1292 Truncated incorrect DECIMAL value: '111.22'",
- "Warning 1292 Truncated incorrect DECIMAL value: '111.22'",
- "Warning 1292 Truncated incorrect DECIMAL value: '111.22'",
- "Warning 1292 Truncated incorrect DECIMAL value: '111.22'"))
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1292 4 warnings with this error code, first warning: Truncated incorrect DECIMAL value: '111.22'"))
// Test the strict warnings is treated as errors under the strict mode.
tk.MustExec("drop table if exists t")
@@ -829,15 +846,13 @@ func TestModifyColumnTypeWithWarnings(t *testing.T) {
// Test the strict warnings is treated as warnings under the non-strict mode.
tk.MustExec("set @@sql_mode=\"\"")
tk.MustExec("alter table t modify column a decimal(3,1)")
- tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1690 DECIMAL value is out of range in '(3, 1)'",
- "Warning 1690 DECIMAL value is out of range in '(3, 1)'",
- "Warning 1690 DECIMAL value is out of range in '(3, 1)'"))
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1690 3 warnings with this error code, first warning: DECIMAL value is out of range in '(3, 1)'"))
}
// TestModifyColumnTypeWhenInterception is to test modifying column type with warnings intercepted by
// reorg timeout, not owner error and so on.
func TestModifyColumnTypeWhenInterception(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store, _ := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -858,42 +873,10 @@ func TestModifyColumnTypeWhenInterception(t *testing.T) {
// Make the regions scale like: [1, 1024), [1024, 2048), [2048, 3072), [3072, 4096]
tk.MustQuery("split table t between(0) and (4096) regions 4")
- d := dom.DDL()
- hook := &ddl.TestDDLCallback{}
- var checkMiddleWarningCount bool
- var checkMiddleAddedCount bool
- // Since the `DefTiDBDDLReorgWorkerCount` is 4, every worker will be assigned with one region
- // for the first time. Here we mock the insert failure/reorg timeout in region [2048, 3072)
- // which will lead next handle be set to 2048 and partial warnings be stored into ddl job.
- // Since the existence of reorg batch size, only the last reorg batch [2816, 3072) of kv
- // range [2048, 3072) fail to commit, the rest of them all committed successfully. So the
- // addedCount and warnings count in the job are all equal to `4096 - reorg batch size`.
- // In the next round of this ddl job, the last reorg batch will be finished.
- var middleWarningsCount = int64(defaultBatchSize*4 - defaultReorgBatchSize)
- onJobUpdatedExportedFunc := func(job *model.Job) {
- if job.SchemaState == model.StateWriteReorganization || job.SnapshotVer != 0 {
- if len(job.ReorgMeta.WarningsCount) == len(job.ReorgMeta.Warnings) {
- for _, v := range job.ReorgMeta.WarningsCount {
- if v == middleWarningsCount {
- checkMiddleWarningCount = true
- }
- }
- }
- if job.RowCount == middleWarningsCount {
- checkMiddleAddedCount = true
- }
- }
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(hook)
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/MockReorgTimeoutInOneRegion", `return(true)`))
defer func() {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/MockReorgTimeoutInOneRegion"))
}()
tk.MustExec("alter table t modify column b decimal(3,1)")
- require.True(t, checkMiddleWarningCount)
- require.True(t, checkMiddleAddedCount)
-
- res := tk.MustQuery("show warnings")
- require.Len(t, res.Rows(), count)
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1292 4096 warnings with this error code, first warning: Truncated incorrect DECIMAL value: '11.22'"))
}
diff --git a/ddl/multi_schema_change.go b/ddl/multi_schema_change.go
index ab306fe546932..19c355ebfa8da 100644
--- a/ddl/multi_schema_change.go
+++ b/ddl/multi_schema_change.go
@@ -190,6 +190,10 @@ func appendToSubJobs(m *model.MultiSchemaInfo, job *model.Job) error {
if err != nil {
return err
}
+ var reorgTp model.ReorgType
+ if job.ReorgMeta != nil {
+ reorgTp = job.ReorgMeta.ReorgTp
+ }
m.SubJobs = append(m.SubJobs, &model.SubJob{
Type: job.Type,
Args: job.Args,
@@ -198,6 +202,7 @@ func appendToSubJobs(m *model.MultiSchemaInfo, job *model.Job) error {
SnapshotVer: job.SnapshotVer,
Revertible: true,
CtxVars: job.CtxVars,
+ ReorgTp: reorgTp,
})
return nil
}
@@ -261,7 +266,10 @@ func fillMultiSchemaInfo(info *model.MultiSchemaInfo, job *model.Job) (err error
case model.ActionRebaseAutoID, model.ActionModifyTableComment, model.ActionModifyTableCharsetAndCollate:
case model.ActionAddForeignKey:
fkInfo := job.Args[0].(*model.FKInfo)
- info.ForeignKeys = append(info.ForeignKeys, fkInfo.Name)
+ info.AddForeignKeys = append(info.AddForeignKeys, model.AddForeignKeyInfo{
+ Name: fkInfo.Name,
+ Cols: fkInfo.Cols,
+ })
default:
return dbterror.ErrRunMultiSchemaChanges.FastGenByArgs(job.Type.String())
}
@@ -323,6 +331,32 @@ func checkOperateSameColAndIdx(info *model.MultiSchemaInfo) error {
return checkIndexes(info.AlterIndexes, true)
}
+func checkOperateDropIndexUseByForeignKey(info *model.MultiSchemaInfo, t table.Table) error {
+ var remainIndexes, droppingIndexes []*model.IndexInfo
+ tbInfo := t.Meta()
+ for _, idx := range tbInfo.Indices {
+ dropping := false
+ for _, name := range info.DropIndexes {
+ if name.L == idx.Name.L {
+ dropping = true
+ break
+ }
+ }
+ if dropping {
+ droppingIndexes = append(droppingIndexes, idx)
+ } else {
+ remainIndexes = append(remainIndexes, idx)
+ }
+ }
+
+ for _, fk := range info.AddForeignKeys {
+ if droppingIdx := model.FindIndexByColumns(tbInfo, droppingIndexes, fk.Cols...); droppingIdx != nil && model.FindIndexByColumns(tbInfo, remainIndexes, fk.Cols...) == nil {
+ return dbterror.ErrDropIndexNeededInForeignKey.GenWithStackByArgs(droppingIdx.Name)
+ }
+ }
+ return nil
+}
+
func checkMultiSchemaInfo(info *model.MultiSchemaInfo, t table.Table) error {
err := checkOperateSameColAndIdx(info)
if err != nil {
@@ -334,6 +368,11 @@ func checkMultiSchemaInfo(info *model.MultiSchemaInfo, t table.Table) error {
return err
}
+ err = checkOperateDropIndexUseByForeignKey(info, t)
+ if err != nil {
+ return err
+ }
+
return checkAddColumnTooManyColumns(len(t.Cols()) + len(info.AddColumns) - len(info.DropColumns))
}
diff --git a/ddl/multi_schema_change_test.go b/ddl/multi_schema_change_test.go
index 250fb692f3500..1f6a52bcce244 100644
--- a/ddl/multi_schema_change_test.go
+++ b/ddl/multi_schema_change_test.go
@@ -20,6 +20,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/kv"
@@ -141,6 +142,9 @@ func TestMultiSchemaChangeAddColumnsCancelled(t *testing.T) {
tk.MustExec("insert into t values (1);")
hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
// Cancel job when the column 'c' is in write-reorg.
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StateWriteReorganization
})
@@ -221,6 +225,9 @@ func TestMultiSchemaChangeDropColumnsCancelled(t *testing.T) {
tk.MustExec("insert into t values ();")
hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
// Cancel job when the column 'a' is in delete-reorg.
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StateDeleteReorganization
})
@@ -236,6 +243,9 @@ func TestMultiSchemaChangeDropColumnsCancelled(t *testing.T) {
tk.MustExec("insert into t values ();")
hook = newCancelJobHook(t, store, dom, func(job *model.Job) bool {
// Cancel job when the column 'a' is in public.
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StatePublic
})
@@ -258,6 +268,9 @@ func TestMultiSchemaChangeDropIndexedColumnsCancelled(t *testing.T) {
tk.MustExec("insert into t values ();")
hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
// Cancel job when the column 'a' is in delete-reorg.
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StateDeleteReorganization
})
@@ -358,6 +371,9 @@ func TestMultiSchemaChangeRenameColumns(t *testing.T) {
tk.MustExec("insert into t values ()")
hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
// Cancel job when the column 'c' is in write-reorg.
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 2)
return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization
})
@@ -371,7 +387,7 @@ func TestMultiSchemaChangeRenameColumns(t *testing.T) {
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (a int default 1, b int default 2)")
tk.MustExec("insert into t values ()")
- hook1 := &ddl.TestDDLCallback{Do: dom}
+ hook1 := &callback.TestDDLCallback{Do: dom}
hook1.OnJobRunBeforeExported = func(job *model.Job) {
assert.Equal(t, model.ActionMultiSchemaChange, job.Type)
if job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization {
@@ -427,6 +443,9 @@ func TestMultiSchemaChangeAlterColumns(t *testing.T) {
tk.MustExec("create table t (a int default 1, b int default 2)")
hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
// Cancel job when the column 'a' is in write-reorg.
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 2)
return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization
})
@@ -439,7 +458,7 @@ func TestMultiSchemaChangeAlterColumns(t *testing.T) {
// Test dml stmts when do alter
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (a int default 1, b int default 2)")
- hook1 := &ddl.TestDDLCallback{Do: dom}
+ hook1 := &callback.TestDDLCallback{Do: dom}
hook1.OnJobRunBeforeExported = func(job *model.Job) {
assert.Equal(t, model.ActionMultiSchemaChange, job.Type)
if job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteOnly {
@@ -496,6 +515,9 @@ func TestMultiSchemaChangeChangeColumns(t *testing.T) {
tk.MustExec("insert into t values ()")
hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
// Cancel job when the column 'c' is in write-reorg.
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 2)
return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization
})
@@ -555,6 +577,9 @@ func TestMultiSchemaChangeAddIndexesCancelled(t *testing.T) {
tk.MustExec("insert into t values (1, 2, 3);")
cancelHook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
// Cancel the job when index 't2' is in write-reorg.
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 4)
return job.MultiSchemaInfo.SubJobs[2].SchemaState == model.StateWriteReorganization
})
@@ -574,6 +599,9 @@ func TestMultiSchemaChangeAddIndexesCancelled(t *testing.T) {
tk.MustExec("insert into t values (1, 2, 3);")
cancelHook = newCancelJobHook(t, store, dom, func(job *model.Job) bool {
// Cancel the job when index 't1' is in public.
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 4)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StatePublic
})
@@ -623,6 +651,9 @@ func TestMultiSchemaChangeDropIndexesCancelled(t *testing.T) {
// Test for cancelling the job in a middle state.
tk.MustExec("create table t (a int, b int, index(a), unique index(b), index idx(a, b));")
hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StateDeleteOnly
})
@@ -638,6 +669,9 @@ func TestMultiSchemaChangeDropIndexesCancelled(t *testing.T) {
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (a int, b int, index(a), unique index(b), index idx(a, b));")
hook = newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StatePublic
})
@@ -686,8 +720,8 @@ func TestMultiSchemaChangeAddDropIndexes(t *testing.T) {
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (a int, b int, c int, index (a), index(b), index(c));")
tk.MustExec("insert into t values (1, 2, 3);")
- tk.MustExec("alter table t add index aa(a), drop index a, add index cc(c), drop index b, drop index c, add index bb(b);")
- tk.MustQuery("select * from t use index(aa, bb, cc);").Check(testkit.Rows("1 2 3"))
+ tk.MustExec("alter table t add index xa(a), drop index a, add index xc(c), drop index b, drop index c, add index xb(b);")
+ tk.MustQuery("select * from t use index(xa, xb, xc);").Check(testkit.Rows("1 2 3"))
tk.MustGetErrCode("select * from t use index(a);", errno.ErrKeyDoesNotExist)
tk.MustGetErrCode("select * from t use index(b);", errno.ErrKeyDoesNotExist)
tk.MustGetErrCode("select * from t use index(c);", errno.ErrKeyDoesNotExist)
@@ -733,6 +767,9 @@ func TestMultiSchemaChangeRenameIndexes(t *testing.T) {
tk.MustExec("insert into t values ()")
hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
// Cancel job when the column 'c' is in write-reorg.
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 2)
return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization
})
@@ -881,6 +918,9 @@ func TestMultiSchemaChangeModifyColumnsCancelled(t *testing.T) {
tk.MustExec("create table t (a int, b int, c int, index i1(a), unique index i2(b), index i3(a, b));")
tk.MustExec("insert into t values (1, 2, 3);")
hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ if job.Type != model.ActionMultiSchemaChange {
+ return false
+ }
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[2].SchemaState == model.StateWriteReorganization
})
@@ -933,7 +973,7 @@ func TestMultiSchemaChangeAlterIndex(t *testing.T) {
tk.MustExec("insert into t values (1, 2);")
originHook := dom.DDL().GetHook()
var checked bool
- callback := &ddl.TestDDLCallback{Do: dom}
+ callback := &callback.TestDDLCallback{Do: dom}
onJobUpdatedExportedFunc := func(job *model.Job) {
assert.NotNil(t, job.MultiSchemaInfo)
// "modify column a tinyint" in write-reorg.
@@ -975,6 +1015,7 @@ func TestMultiSchemaChangeMixCancelled(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
+ tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("create table t (a int, b int, c int, index i1(c), index i2(c));")
tk.MustExec("insert into t values (1, 2, 3);")
@@ -1002,7 +1043,7 @@ func TestMultiSchemaChangeAdminShowDDLJobs(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
originHook := dom.DDL().GetHook()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
hook.OnJobRunBeforeExported = func(job *model.Job) {
assert.Equal(t, model.ActionMultiSchemaChange, job.Type)
if job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateDeleteOnly {
@@ -1012,7 +1053,7 @@ func TestMultiSchemaChangeAdminShowDDLJobs(t *testing.T) {
assert.Equal(t, len(rows), 4)
assert.Equal(t, rows[1][1], "test")
assert.Equal(t, rows[1][2], "t")
- assert.Equal(t, rows[1][3], "add index /* subjob */")
+ assert.Equal(t, rows[1][3], "add index /* subjob */ /* txn-merge */")
assert.Equal(t, rows[1][4], "delete only")
assert.Equal(t, rows[1][len(rows[1])-1], "running")
@@ -1089,7 +1130,7 @@ func TestMultiSchemaChangeWithExpressionIndex(t *testing.T) {
tk.MustQuery("select * from t;").Check(testkit.Rows("1 2", "2 1"))
originHook := dom.DDL().GetHook()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
hook.OnJobRunBeforeExported = func(job *model.Job) {
if checkErr != nil {
@@ -1137,7 +1178,7 @@ func TestMultiSchemaChangeUnsupportedType(t *testing.T) {
tk.MustExec("use test;")
tk.MustExec("create table t (a int, b int);")
- tk.MustGetErrMsg("alter table t add column c int, auto_id_cache = 1;",
+ tk.MustGetErrMsg("alter table t add column c int, auto_id_cache = 10;",
"[ddl:8200]Unsupported multi schema change for modify auto id cache")
}
@@ -1151,7 +1192,7 @@ func TestMultiSchemaChangeSchemaVersion(t *testing.T) {
schemaVerMap := map[int64]struct{}{}
originHook := dom.DDL().GetHook()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
hook.OnJobSchemaStateChanged = func(schemaVer int64) {
if schemaVer != 0 {
// No same return schemaVer during multi-schema change
@@ -1168,6 +1209,17 @@ func TestMultiSchemaChangeSchemaVersion(t *testing.T) {
dom.DDL().SetHook(originHook)
}
+func TestMultiSchemaChangeAddIndexChangeColumn(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test;")
+ tk.MustExec("CREATE TABLE t (a SMALLINT DEFAULT '30219', b TIME NULL DEFAULT '02:45:06', PRIMARY KEY (a));")
+ tk.MustExec("ALTER TABLE t ADD unique INDEX idx4 (b), change column a e MEDIUMINT DEFAULT '5280454' FIRST;")
+ tk.MustExec("insert ignore into t (e) values (5586359),(501788),(-5961048),(220083),(-4917129),(-7267211),(7750448);")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("5586359 02:45:06"))
+ tk.MustExec("admin check table t;")
+}
+
func TestMultiSchemaChangeMixedWithUpdate(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
@@ -1180,7 +1232,7 @@ func TestMultiSchemaChangeMixedWithUpdate(t *testing.T) {
"'2020-01-01 10:00:00', 'wer', '10:00:00', 2.1, 12, 'qwer', 12, 'asdf');")
originHook := dom.DDL().GetHook()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
hook.OnJobRunBeforeExported = func(job *model.Job) {
if checkErr != nil {
@@ -1215,7 +1267,7 @@ type cancelOnceHook struct {
pred func(job *model.Job) bool
s sessionctx.Context
- ddl.TestDDLCallback
+ callback.TestDDLCallback
}
func (c *cancelOnceHook) OnJobUpdated(job *model.Job) {
@@ -1223,7 +1275,7 @@ func (c *cancelOnceHook) OnJobUpdated(job *model.Job) {
return
}
c.triggered = true
- errs, err := ddl.CancelJobs(c.s, c.store, []int64{job.ID})
+ errs, err := ddl.CancelJobs(c.s, []int64{job.ID})
if errs[0] != nil {
c.cancelErr = errs[0]
return
@@ -1248,7 +1300,7 @@ func newCancelJobHook(t *testing.T, store kv.Storage, dom *domain.Domain,
return &cancelOnceHook{
store: store,
pred: pred,
- TestDDLCallback: ddl.TestDDLCallback{Do: dom},
+ TestDDLCallback: callback.TestDDLCallback{Do: dom},
s: tk.Session(),
}
}
@@ -1262,7 +1314,6 @@ func putTheSameDDLJobTwice(t *testing.T, fn func()) {
}
func assertMultiSchema(t *testing.T, job *model.Job, subJobLen int) {
- assert.Equal(t, model.ActionMultiSchemaChange, job.Type, job)
assert.NotNil(t, job.MultiSchemaInfo, job)
assert.Len(t, job.MultiSchemaInfo.SubJobs, subJobLen, job)
}
diff --git a/ddl/mv_index_test.go b/ddl/mv_index_test.go
new file mode 100644
index 0000000000000..10fbe2971377a
--- /dev/null
+++ b/ddl/mv_index_test.go
@@ -0,0 +1,71 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl_test
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/pingcap/tidb/ddl/internal/callback"
+ "github.com/pingcap/tidb/errno"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/testkit"
+)
+
+func TestMultiValuedIndexOnlineDDL(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t (pk int primary key, a json) partition by hash(pk) partitions 32;")
+ var sb strings.Builder
+ sb.WriteString("insert into t values ")
+ for i := 0; i < 100; i++ {
+ sb.WriteString(fmt.Sprintf("(%d, '[%d, %d, %d]')", i, i+1, i+2, i+3))
+ if i != 99 {
+ sb.WriteString(",")
+ }
+ }
+ tk.MustExec(sb.String())
+
+ internalTK := testkit.NewTestKit(t, store)
+ internalTK.MustExec("use test")
+
+ hook := &callback.TestDDLCallback{Do: dom}
+ n := 100
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ internalTK.MustExec(fmt.Sprintf("insert into t values (%d, '[%d, %d, %d]')", n, n, n+1, n+2))
+ internalTK.MustExec(fmt.Sprintf("delete from t where pk = %d", n-4))
+ internalTK.MustExec(fmt.Sprintf("update t set a = '[%d, %d, %d]' where pk = %d", n-3, n-2, n+1000, n-3))
+ n++
+ }
+ o := dom.DDL().GetHook()
+ dom.DDL().SetHook(hook)
+
+ tk.MustExec("alter table t add index idx((cast(a as signed array)))")
+ tk.MustExec("admin check table t")
+ dom.DDL().SetHook(o)
+
+ tk.MustExec("drop table if exists t;")
+ tk.MustExec("create table t (pk int primary key, a json);")
+ tk.MustExec("insert into t values (1, '[1,2,3]');")
+ tk.MustExec("insert into t values (2, '[2,3,4]');")
+ tk.MustExec("insert into t values (3, '[3,4,5]');")
+ tk.MustExec("insert into t values (4, '[-4,5,6]');")
+ tk.MustGetErrCode("alter table t add unique index idx((cast(a as signed array)));", errno.ErrDupEntry)
+ tk.MustGetErrMsg("alter table t add index idx((cast(a as unsigned array)));", "[ddl:8202]Cannot decode index value, because [types:1690]constant -4 overflows bigint")
+}
diff --git a/ddl/partition.go b/ddl/partition.go
index 7bba0b1006332..00a5582be9a54 100644
--- a/ddl/partition.go
+++ b/ddl/partition.go
@@ -34,6 +34,7 @@ import (
"github.com/pingcap/tidb/domain/infosync"
"github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/infoschema"
+ "github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/parser"
@@ -42,8 +43,10 @@ import (
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/opcode"
+ "github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
driver "github.com/pingcap/tidb/types/parser_driver"
@@ -55,9 +58,11 @@ import (
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/mathutil"
"github.com/pingcap/tidb/util/mock"
+ decoder "github.com/pingcap/tidb/util/rowDecoder"
"github.com/pingcap/tidb/util/slice"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/pingcap/tidb/util/stringutil"
+ "github.com/prometheus/client_golang/prometheus"
"github.com/tikv/client-go/v2/tikv"
"go.uber.org/zap"
)
@@ -161,7 +166,7 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v
for _, p := range tblInfo.Partition.AddingDefinitions {
ids = append(ids, p.ID)
}
- if err := alterTableLabelRule(job.SchemaName, tblInfo, ids); err != nil {
+ if _, err := alterTableLabelRule(job.SchemaName, tblInfo, ids); err != nil {
job.State = model.JobStateCancelled
return ver, err
}
@@ -170,6 +175,10 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v
job.SchemaState = model.StateReplicaOnly
case model.StateReplicaOnly:
// replica only -> public
+ failpoint.Inject("sleepBeforeReplicaOnly", func(val failpoint.Value) {
+ sleepSecond := val.(int)
+ time.Sleep(time.Duration(sleepSecond) * time.Second)
+ })
// Here need do some tiflash replica complement check.
// TODO: If a table is with no TiFlashReplica or it is not available, the replica-only state can be eliminated.
if tblInfo.TiFlashReplica != nil && tblInfo.TiFlashReplica.Available {
@@ -177,8 +186,7 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v
// be finished. Otherwise the query to this partition will be blocked.
needRetry, err := checkPartitionReplica(tblInfo.TiFlashReplica.Count, addingDefinitions, d)
if err != nil {
- ver, err = convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
- return ver, err
+ return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
}
if needRetry {
// The new added partition hasn't been replicated.
@@ -193,6 +201,15 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v
if tblInfo.TiFlashReplica != nil && tblInfo.TiFlashReplica.Available {
for _, d := range partInfo.Definitions {
tblInfo.TiFlashReplica.AvailablePartitionIDs = append(tblInfo.TiFlashReplica.AvailablePartitionIDs, d.ID)
+ err = infosync.UpdateTiFlashProgressCache(d.ID, 1)
+ if err != nil {
+ // just print log, progress will be updated in `refreshTiFlashTicker`
+ logutil.BgLogger().Error("update tiflash sync progress cache failed",
+ zap.Error(err),
+ zap.Int64("tableID", tblInfo.ID),
+ zap.Int64("partitionID", d.ID),
+ )
+ }
}
}
// For normal and replica finished table, move the `addingDefinitions` into `Definitions`.
@@ -213,14 +230,16 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v
return ver, errors.Trace(err)
}
-func alterTableLabelRule(schemaName string, meta *model.TableInfo, ids []int64) error {
+// alterTableLabelRule updates Label Rules if they exists
+// returns true if changed.
+func alterTableLabelRule(schemaName string, meta *model.TableInfo, ids []int64) (bool, error) {
tableRuleID := fmt.Sprintf(label.TableIDFormat, label.IDPrefix, schemaName, meta.Name.L)
oldRule, err := infosync.GetLabelRules(context.TODO(), []string{tableRuleID})
if err != nil {
- return errors.Trace(err)
+ return false, errors.Trace(err)
}
if len(oldRule) == 0 {
- return nil
+ return false, nil
}
r, ok := oldRule[tableRuleID]
@@ -228,10 +247,11 @@ func alterTableLabelRule(schemaName string, meta *model.TableInfo, ids []int64)
rule := r.Reset(schemaName, meta.Name.L, "", ids...)
err = infosync.PutLabelRule(context.TODO(), rule)
if err != nil {
- return errors.Wrapf(err, "failed to notify PD label rule")
+ return false, errors.Wrapf(err, "failed to notify PD label rule")
}
+ return true, nil
}
- return nil
+ return false, nil
}
func alterTablePartitionBundles(t *meta.Meta, tblInfo *model.TableInfo, addingDefinitions []model.PartitionDefinition) ([]*placement.Bundle, error) {
@@ -1294,6 +1314,28 @@ func checkAddPartitionNameUnique(tbInfo *model.TableInfo, pi *model.PartitionInf
return nil
}
+func checkReorgPartitionNames(p *model.PartitionInfo, droppedNames []model.CIStr, pi *model.PartitionInfo) error {
+ partNames := make(map[string]struct{})
+ oldDefs := p.Definitions
+ for _, oldDef := range oldDefs {
+ partNames[oldDef.Name.L] = struct{}{}
+ }
+ for _, delName := range droppedNames {
+ if _, ok := partNames[delName.L]; !ok {
+ return dbterror.ErrSameNamePartition.GenWithStackByArgs(delName)
+ }
+ delete(partNames, delName.L)
+ }
+ newDefs := pi.Definitions
+ for _, newDef := range newDefs {
+ if _, ok := partNames[newDef.Name.L]; ok {
+ return dbterror.ErrSameNamePartition.GenWithStackByArgs(newDef.Name)
+ }
+ partNames[newDef.Name.L] = struct{}{}
+ }
+ return nil
+}
+
func checkAndOverridePartitionID(newTableInfo, oldTableInfo *model.TableInfo) error {
// If any old partitionInfo has lost, that means the partition ID lost too, so did the data, repair failed.
if newTableInfo.Partition == nil {
@@ -1362,7 +1404,7 @@ func checkPartitionFuncType(ctx sessionctx.Context, expr ast.ExprNode, tblInfo *
return nil
}
- e, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, expr)
+ e, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, expr, false)
if err != nil {
return errors.Trace(err)
}
@@ -1663,7 +1705,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
if err != nil {
return ver, errors.Trace(err)
}
- if job.Type == model.ActionAddTablePartition {
+ if job.Type == model.ActionAddTablePartition || job.Type == model.ActionReorganizePartition {
// It is rollbacked from adding table partition, just remove addingDefinitions from tableInfo.
physicalTableIDs, pNames, rollbackBundles := rollbackAddingPartitionInfo(tblInfo)
err = infosync.PutRuleBundlesWithDefaultRetry(context.TODO(), rollbackBundles)
@@ -1677,7 +1719,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
return ver, errors.Wrapf(err, "failed to notify PD the label rules")
}
- if err := alterTableLabelRule(job.SchemaName, tblInfo, getIDs([]*model.TableInfo{tblInfo})); err != nil {
+ if _, err := alterTableLabelRule(job.SchemaName, tblInfo, getIDs([]*model.TableInfo{tblInfo})); err != nil {
job.State = model.JobStateCancelled
return ver, err
}
@@ -1710,7 +1752,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
return ver, errors.Wrapf(err, "failed to notify PD the label rules")
}
- if err := alterTableLabelRule(job.SchemaName, tblInfo, getIDs([]*model.TableInfo{tblInfo})); err != nil {
+ if _, err := alterTableLabelRule(job.SchemaName, tblInfo, getIDs([]*model.TableInfo{tblInfo})); err != nil {
job.State = model.JobStateCancelled
return ver, err
}
@@ -1730,6 +1772,10 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
if err != nil {
return ver, errors.Trace(err)
}
+ dbInfo, err := t.GetDatabase(job.SchemaID)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
// If table has global indexes, we need reorg to clean up them.
if pt, ok := tbl.(table.PartitionedTable); ok && hasGlobalIndex(tblInfo) {
// Build elements for compatible with modify column type. elements will not be used when reorganizing.
@@ -1739,8 +1785,13 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
elements = append(elements, &meta.Element{ID: idxInfo.ID, TypeKey: meta.IndexElementKey})
}
}
- rh := newReorgHandler(t, w.sess, w.concurrentDDL)
- reorgInfo, err := getReorgInfoFromPartitions(d.jobContext(job), d, rh, job, tbl, physicalTableIDs, elements)
+ sctx, err1 := w.sessPool.get()
+ if err1 != nil {
+ return ver, err1
+ }
+ defer w.sessPool.put(sctx)
+ rh := newReorgHandler(newSession(sctx))
+ reorgInfo, err := getReorgInfoFromPartitions(d.jobContext(job.ID), d, rh, job, dbInfo, pt, physicalTableIDs, elements)
if err != nil || reorgInfo.first {
// If we run reorg firstly, we should update the job snapshot version
@@ -2064,7 +2115,7 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
failpoint.Return(ver, err)
}
sess := newSession(se)
- _, err = sess.execute(context.Background(), "insert into test.pt values (40000000)", "exchange_partition_test")
+ _, err = sess.execute(context.Background(), "insert ignore into test.pt values (40000000)", "exchange_partition_test")
if err != nil {
failpoint.Return(ver, err)
}
@@ -2137,6 +2188,665 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
return ver, nil
}
+func checkReorgPartition(t *meta.Meta, job *model.Job) (*model.TableInfo, []model.CIStr, *model.PartitionInfo, []model.PartitionDefinition, []model.PartitionDefinition, error) {
+ schemaID := job.SchemaID
+ tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, schemaID)
+ if err != nil {
+ return nil, nil, nil, nil, nil, errors.Trace(err)
+ }
+ partInfo := &model.PartitionInfo{}
+ var partNames []model.CIStr
+ err = job.DecodeArgs(&partNames, &partInfo)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return nil, nil, nil, nil, nil, errors.Trace(err)
+ }
+ addingDefs := tblInfo.Partition.AddingDefinitions
+ droppingDefs := tblInfo.Partition.DroppingDefinitions
+ if len(addingDefs) == 0 {
+ addingDefs = []model.PartitionDefinition{}
+ }
+ if len(droppingDefs) == 0 {
+ droppingDefs = []model.PartitionDefinition{}
+ }
+ return tblInfo, partNames, partInfo, droppingDefs, addingDefs, nil
+}
+
+func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ // Handle the rolling back job
+ if job.IsRollingback() {
+ ver, err := w.onDropTablePartition(d, t, job)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ return ver, nil
+ }
+
+ tblInfo, partNamesCIStr, partInfo, _, addingDefinitions, err := checkReorgPartition(t, job)
+ if err != nil {
+ return ver, err
+ }
+ partNames := make([]string, len(partNamesCIStr))
+ for i := range partNamesCIStr {
+ partNames[i] = partNamesCIStr[i].L
+ }
+
+ // In order to skip maintaining the state check in partitionDefinition, TiDB use dropping/addingDefinition instead of state field.
+ // So here using `job.SchemaState` to judge what the stage of this job is.
+ originalState := job.SchemaState
+ switch job.SchemaState {
+ case model.StateNone:
+ // job.SchemaState == model.StateNone means the job is in the initial state of reorg partition.
+ // Here should use partInfo from job directly and do some check action.
+ // In case there was a race for queueing different schema changes on the same
+ // table and the checks was not done on the current schema version.
+ // The partInfo may have been checked against an older schema version for example.
+ // If the check is done here, it does not need to be repeated, since no other
+ // DDL on the same table can be run concurrently.
+ err = checkAddPartitionTooManyPartitions(uint64(len(tblInfo.Partition.Definitions) +
+ len(partInfo.Definitions) -
+ len(partNames)))
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+
+ err = checkReorgPartitionNames(tblInfo.Partition, partNamesCIStr, partInfo)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+
+ // Re-check that the dropped/added partitions are compatible with current definition
+ firstPartIdx, lastPartIdx, idMap, err := getReplacedPartitionIDs(partNamesCIStr, tblInfo.Partition)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, err
+ }
+ sctx := w.sess.Context
+ if err = checkReorgPartitionDefs(sctx, tblInfo, partInfo, firstPartIdx, lastPartIdx, idMap); err != nil {
+ job.State = model.JobStateCancelled
+ return ver, err
+ }
+
+ // move the adding definition into tableInfo.
+ updateAddingPartitionInfo(partInfo, tblInfo)
+ orgDefs := tblInfo.Partition.Definitions
+ _ = updateDroppingPartitionInfo(tblInfo, partNames)
+ // Reset original partitions, and keep DroppedDefinitions
+ tblInfo.Partition.Definitions = orgDefs
+
+ // modify placement settings
+ for _, def := range tblInfo.Partition.AddingDefinitions {
+ if _, err = checkPlacementPolicyRefValidAndCanNonValidJob(t, job, def.PlacementPolicyRef); err != nil {
+ // job.State = model.JobStateCancelled may be set depending on error in function above.
+ return ver, errors.Trace(err)
+ }
+ }
+
+ // From now on we cannot just cancel the DDL, we must roll back if changesMade!
+ changesMade := false
+ if tblInfo.TiFlashReplica != nil {
+ // Must set placement rule, and make sure it succeeds.
+ if err := infosync.ConfigureTiFlashPDForPartitions(true, &tblInfo.Partition.AddingDefinitions, tblInfo.TiFlashReplica.Count, &tblInfo.TiFlashReplica.LocationLabels, tblInfo.ID); err != nil {
+ logutil.BgLogger().Error("ConfigureTiFlashPDForPartitions fails", zap.Error(err))
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ changesMade = true
+ // In the next step, StateDeleteOnly, wait to verify the TiFlash replicas are OK
+ }
+
+ bundles, err := alterTablePartitionBundles(t, tblInfo, tblInfo.Partition.AddingDefinitions)
+ if err != nil {
+ if !changesMade {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
+ }
+
+ if len(bundles) > 0 {
+ if err = infosync.PutRuleBundlesWithDefaultRetry(context.TODO(), bundles); err != nil {
+ if !changesMade {
+ job.State = model.JobStateCancelled
+ return ver, errors.Wrapf(err, "failed to notify PD the placement rules")
+ }
+ return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
+ }
+ changesMade = true
+ }
+
+ ids := getIDs([]*model.TableInfo{tblInfo})
+ for _, p := range tblInfo.Partition.AddingDefinitions {
+ ids = append(ids, p.ID)
+ }
+ changed, err := alterTableLabelRule(job.SchemaName, tblInfo, ids)
+ changesMade = changesMade || changed
+ if err != nil {
+ if !changesMade {
+ job.State = model.JobStateCancelled
+ return ver, err
+ }
+ return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
+ }
+
+ // Doing the preSplitAndScatter here, since all checks are completed,
+ // and we will soon start writing to the new partitions.
+ if s, ok := d.store.(kv.SplittableStore); ok && s != nil {
+ // partInfo only contains the AddingPartitions
+ splitPartitionTableRegion(w.sess.Context, s, tblInfo, partInfo, true)
+ }
+
+ // Assume we cannot have more than MaxUint64 rows, set the progress to 1/10 of that.
+ metrics.GetBackfillProgressByLabel(metrics.LblReorgPartition, job.SchemaName, tblInfo.Name.String()).Set(0.1 / float64(math.MaxUint64))
+ job.SchemaState = model.StateDeleteOnly
+ tblInfo.Partition.DDLState = model.StateDeleteOnly
+ ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ // Is really both StateDeleteOnly AND StateWriteOnly needed?
+ // If transaction A in WriteOnly inserts row 1 (into both new and old partition set)
+ // and then transaction B in DeleteOnly deletes that row (in both new and old)
+ // does really transaction B need to do the delete in the new partition?
+ // Yes, otherwise it would still be there when the WriteReorg happens,
+ // and WriteReorg would only copy existing rows to the new table, so unless it is
+ // deleted it would result in a ghost row!
+ // What about update then?
+ // Updates also need to be handled for new partitions in DeleteOnly,
+ // since it would not be overwritten during Reorganize phase.
+ // BUT if the update results in adding in one partition and deleting in another,
+ // THEN only the delete must happen in the new partition set, not the insert!
+ case model.StateDeleteOnly:
+ // This state is to confirm all servers can not see the new partitions when reorg is running,
+ // so that all deletes will be done in both old and new partitions when in either DeleteOnly
+ // or WriteOnly state.
+ // Also using the state for checking that the optional TiFlash replica is available, making it
+ // in a state without (much) data and easy to retry without side effects.
+
+ // Reason for having it here, is to make it easy for retry, and better to make sure it is in-sync
+ // as early as possible, to avoid a long wait after the data copying.
+ if tblInfo.TiFlashReplica != nil && tblInfo.TiFlashReplica.Available {
+ // For available state, the new added partition should wait its replica to
+ // be finished, otherwise the query to this partition will be blocked.
+ count := tblInfo.TiFlashReplica.Count
+ needRetry, err := checkPartitionReplica(count, addingDefinitions, d)
+ if err != nil {
+ // need to rollback, since we tried to register the new
+ // partitions before!
+ return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
+ }
+ if needRetry {
+ // The new added partition hasn't been replicated.
+ // Do nothing to the job this time, wait next worker round.
+ time.Sleep(tiflashCheckTiDBHTTPAPIHalfInterval)
+ // Set the error here which will lead this job exit when it's retry times beyond the limitation.
+ return ver, errors.Errorf("[ddl] add partition wait for tiflash replica to complete")
+ }
+
+ // When TiFlash Replica is ready, we must move them into `AvailablePartitionIDs`.
+ // Since onUpdateFlashReplicaStatus cannot see the partitions yet (not public)
+ for _, d := range addingDefinitions {
+ tblInfo.TiFlashReplica.AvailablePartitionIDs = append(tblInfo.TiFlashReplica.AvailablePartitionIDs, d.ID)
+ }
+ }
+
+ job.SchemaState = model.StateWriteOnly
+ tblInfo.Partition.DDLState = model.StateWriteOnly
+ metrics.GetBackfillProgressByLabel(metrics.LblReorgPartition, job.SchemaName, tblInfo.Name.String()).Set(0.2 / float64(math.MaxUint64))
+ ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != job.SchemaState)
+ case model.StateWriteOnly:
+ // Insert this state to confirm all servers can see the new partitions when reorg is running,
+ // so that new data will be updated in both old and new partitions when reorganizing.
+ job.SnapshotVer = 0
+ job.SchemaState = model.StateWriteReorganization
+ tblInfo.Partition.DDLState = model.StateWriteReorganization
+ metrics.GetBackfillProgressByLabel(metrics.LblReorgPartition, job.SchemaName, tblInfo.Name.String()).Set(0.3 / float64(math.MaxUint64))
+ ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != job.SchemaState)
+ case model.StateWriteReorganization:
+ physicalTableIDs := getPartitionIDsFromDefinitions(tblInfo.Partition.DroppingDefinitions)
+ tbl, err2 := getTable(d.store, job.SchemaID, tblInfo)
+ if err2 != nil {
+ return ver, errors.Trace(err2)
+ }
+ // TODO: If table has global indexes, we need reorg to clean up them.
+ // and then add the new partition ids back...
+ if _, ok := tbl.(table.PartitionedTable); ok && hasGlobalIndex(tblInfo) {
+ err = errors.Trace(dbterror.ErrCancelledDDLJob.GenWithStack("global indexes is not supported yet for reorganize partition"))
+ return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
+ }
+ var done bool
+ done, ver, err = doPartitionReorgWork(w, d, t, job, tbl, physicalTableIDs)
+
+ if !done {
+ return ver, err
+ }
+
+ firstPartIdx, lastPartIdx, idMap, err2 := getReplacedPartitionIDs(partNamesCIStr, tblInfo.Partition)
+ failpoint.Inject("reorgPartWriteReorgReplacedPartIDsFail", func(val failpoint.Value) {
+ if val.(bool) {
+ err2 = errors.New("Injected error by reorgPartWriteReorgReplacedPartIDsFail")
+ }
+ })
+ if err2 != nil {
+ return ver, err2
+ }
+ newDefs := getReorganizedDefinitions(tblInfo.Partition, firstPartIdx, lastPartIdx, idMap)
+
+ // From now on, use the new definitions, but keep the Adding and Dropping for double write
+ tblInfo.Partition.Definitions = newDefs
+ tblInfo.Partition.Num = uint64(len(newDefs))
+
+ // Now all the data copying is done, but we cannot simply remove the droppingDefinitions
+ // since they are a part of the normal Definitions that other nodes with
+ // the current schema version. So we need to double write for one more schema version
+ job.SchemaState = model.StateDeleteReorganization
+ tblInfo.Partition.DDLState = model.StateDeleteReorganization
+ ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != job.SchemaState)
+
+ case model.StateDeleteReorganization:
+ // Drop the droppingDefinitions and finish the DDL
+ // This state is needed for the case where client A sees the schema
+ // with version of StateWriteReorg and would not see updates of
+ // client B that writes to the new partitions, previously
+ // addingDefinitions, since it would not double write to
+ // the droppingDefinitions during this time
+ // By adding StateDeleteReorg state, client B will write to both
+ // the new (previously addingDefinitions) AND droppingDefinitions
+
+ // Register the droppingDefinitions ids for rangeDelete
+ // and the addingDefinitions for handling in the updateSchemaVersion
+ physicalTableIDs := getPartitionIDsFromDefinitions(tblInfo.Partition.DroppingDefinitions)
+ newIDs := getPartitionIDsFromDefinitions(partInfo.Definitions)
+ job.CtxVars = []interface{}{physicalTableIDs, newIDs}
+ definitionsToDrop := tblInfo.Partition.DroppingDefinitions
+ tblInfo.Partition.DroppingDefinitions = nil
+ tblInfo.Partition.AddingDefinitions = nil
+ ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ failpoint.Inject("reorgPartWriteReorgSchemaVersionUpdateFail", func(val failpoint.Value) {
+ if val.(bool) {
+ err = errors.New("Injected error by reorgPartWriteReorgSchemaVersionUpdateFail")
+ }
+ })
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ job.SchemaState = model.StateNone
+ tblInfo.Partition.DDLState = model.StateNone
+ job.FinishTableJob(model.JobStateDone, model.StateNone, ver, tblInfo)
+ // How to handle this?
+ // Seems to only trigger asynchronous update of statistics.
+ // Should it actually be synchronous?
+ asyncNotifyEvent(d, &util.Event{Tp: model.ActionReorganizePartition, TableInfo: tblInfo, PartInfo: &model.PartitionInfo{Definitions: definitionsToDrop}})
+ // A background job will be created to delete old partition data.
+ job.Args = []interface{}{physicalTableIDs}
+
+ default:
+ err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("partition", job.SchemaState)
+ }
+
+ return ver, errors.Trace(err)
+}
+
+func doPartitionReorgWork(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, tbl table.Table, physTblIDs []int64) (done bool, ver int64, err error) {
+ job.ReorgMeta.ReorgTp = model.ReorgTypeTxn
+ sctx, err1 := w.sessPool.get()
+ if err1 != nil {
+ return done, ver, err1
+ }
+ defer w.sessPool.put(sctx)
+ rh := newReorgHandler(newSession(sctx))
+ elements := BuildElements(tbl.Meta().Columns[0], tbl.Meta().Indices)
+ partTbl, ok := tbl.(table.PartitionedTable)
+ if !ok {
+ return false, ver, dbterror.ErrUnsupportedReorganizePartition.GenWithStackByArgs()
+ }
+ dbInfo, err := t.GetDatabase(job.SchemaID)
+ if err != nil {
+ return false, ver, errors.Trace(err)
+ }
+ reorgInfo, err := getReorgInfoFromPartitions(d.jobContext(job.ID), d, rh, job, dbInfo, partTbl, physTblIDs, elements)
+ err = w.runReorgJob(rh, reorgInfo, tbl.Meta(), d.lease, func() (reorgErr error) {
+ defer tidbutil.Recover(metrics.LabelDDL, "doPartitionReorgWork",
+ func() {
+ reorgErr = dbterror.ErrCancelledDDLJob.GenWithStack("reorganize partition for table `%v` panic", tbl.Meta().Name)
+ }, false)
+ return w.reorgPartitionDataAndIndex(tbl, reorgInfo)
+ })
+ if err != nil {
+ if dbterror.ErrWaitReorgTimeout.Equal(err) {
+ // If timeout, we should return, check for the owner and re-wait job done.
+ return false, ver, nil
+ }
+ if kv.IsTxnRetryableError(err) {
+ return false, ver, errors.Trace(err)
+ }
+ if err1 := rh.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil {
+ logutil.BgLogger().Warn("[ddl] reorg partition job failed, RemoveDDLReorgHandle failed, can't convert job to rollback",
+ zap.String("job", job.String()), zap.Error(err1))
+ }
+ logutil.BgLogger().Warn("[ddl] reorg partition job failed, convert job to rollback", zap.String("job", job.String()), zap.Error(err))
+ ver, err = convertAddTablePartitionJob2RollbackJob(d, t, job, err, tbl.Meta())
+ return false, ver, errors.Trace(err)
+ }
+ return true, ver, err
+}
+
+type reorgPartitionWorker struct {
+ *backfillCtx
+ metricCounter prometheus.Counter
+
+ // Static allocated to limit memory allocations
+ rowRecords []*rowRecord
+ rowDecoder *decoder.RowDecoder
+ rowMap map[int64]types.Datum
+ writeColOffsetMap map[int64]int
+ maxOffset int
+ reorgedTbl table.PartitionedTable
+
+ jobContext *JobContext
+}
+
+func newReorgPartitionWorker(sessCtx sessionctx.Context, i int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) (*reorgPartitionWorker, error) {
+ reorgedTbl, err := tables.GetReorganizedPartitionedTable(t)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ pt := t.GetPartitionedTable()
+ if pt == nil {
+ return nil, dbterror.ErrUnsupportedReorganizePartition.GenWithStackByArgs()
+ }
+ partColIDs := pt.GetPartitionColumnIDs()
+ writeColOffsetMap := make(map[int64]int, len(partColIDs))
+ maxOffset := 0
+ for _, col := range pt.Cols() {
+ found := false
+ for _, id := range partColIDs {
+ if col.ID == id {
+ found = true
+ break
+ }
+ }
+ if !found {
+ continue
+ }
+ writeColOffsetMap[col.ID] = col.Offset
+ maxOffset = mathutil.Max[int](maxOffset, col.Offset)
+ }
+ return &reorgPartitionWorker{
+ backfillCtx: newBackfillCtx(reorgInfo.d, i, sessCtx, reorgInfo.ReorgMeta.ReorgTp, reorgInfo.SchemaName, t),
+ metricCounter: metrics.BackfillTotalCounter.WithLabelValues(metrics.GenerateReorgLabel("reorg_partition_rate", reorgInfo.SchemaName, t.Meta().Name.String())),
+ rowDecoder: decoder.NewRowDecoder(t, t.WritableCols(), decodeColMap),
+ rowMap: make(map[int64]types.Datum, len(decodeColMap)),
+ jobContext: jc,
+ writeColOffsetMap: writeColOffsetMap,
+ maxOffset: maxOffset,
+ reorgedTbl: reorgedTbl,
+ }, nil
+}
+
+func (w *reorgPartitionWorker) GetTasks() ([]*BackfillJob, error) {
+ panic("[ddl] reorg partition worker GetTask function doesn't implement")
+}
+
+func (w *reorgPartitionWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (taskCtx backfillTaskContext, errInTxn error) {
+ oprStartTime := time.Now()
+ ctx := kv.WithInternalSourceType(context.Background(), w.jobContext.ddlJobSourceType())
+ errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(ctx context.Context, txn kv.Transaction) error {
+ taskCtx.addedCount = 0
+ taskCtx.scanCount = 0
+ txn.SetOption(kv.Priority, handleRange.priority)
+ if tagger := w.GetCtx().getResourceGroupTaggerForTopSQL(handleRange.getJobID()); tagger != nil {
+ txn.SetOption(kv.ResourceGroupTagger, tagger)
+ }
+
+ rowRecords, nextKey, taskDone, err := w.fetchRowColVals(txn, handleRange)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ taskCtx.nextKey = nextKey
+ taskCtx.done = taskDone
+
+ warningsMap := make(map[errors.ErrorID]*terror.Error)
+ warningsCountMap := make(map[errors.ErrorID]int64)
+ for _, prr := range rowRecords {
+ taskCtx.scanCount++
+
+ err = txn.Set(prr.key, prr.vals)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ taskCtx.addedCount++
+ if prr.warning != nil {
+ if _, ok := warningsCountMap[prr.warning.ID()]; ok {
+ warningsCountMap[prr.warning.ID()]++
+ } else {
+ warningsCountMap[prr.warning.ID()] = 1
+ warningsMap[prr.warning.ID()] = prr.warning
+ }
+ }
+ // TODO: Future optimization: also write the indexes here?
+ // What if the transaction limit is just enough for a single row, without index?
+ // Hmm, how could that be in the first place?
+ // For now, implement the batch-txn w.addTableIndex,
+ // since it already exists and is in use
+ }
+
+ // Collect the warnings.
+ taskCtx.warnings, taskCtx.warningsCount = warningsMap, warningsCountMap
+
+ // also add the index entries here? And make sure they are not added somewhere else
+
+ return nil
+ })
+ logSlowOperations(time.Since(oprStartTime), "BackfillDataInTxn", 3000)
+
+ return
+}
+
+func (w *reorgPartitionWorker) fetchRowColVals(txn kv.Transaction, taskRange reorgBackfillTask) ([]*rowRecord, kv.Key, bool, error) {
+ w.rowRecords = w.rowRecords[:0]
+ startTime := time.Now()
+
+ // taskDone means that the added handle is out of taskRange.endHandle.
+ taskDone := false
+ sysTZ := w.sessCtx.GetSessionVars().StmtCtx.TimeZone
+
+ tmpRow := make([]types.Datum, w.maxOffset+1)
+ var lastAccessedHandle kv.Key
+ oprStartTime := startTime
+ err := iterateSnapshotKeys(w.GetCtx().jobContext(taskRange.getJobID()), w.sessCtx.GetStore(), taskRange.priority, w.table.RecordPrefix(), txn.StartTS(), taskRange.startKey, taskRange.endKey,
+ func(handle kv.Handle, recordKey kv.Key, rawRow []byte) (bool, error) {
+ oprEndTime := time.Now()
+ logSlowOperations(oprEndTime.Sub(oprStartTime), "iterateSnapshotKeys in reorgPartitionWorker fetchRowColVals", 0)
+ oprStartTime = oprEndTime
+
+ if taskRange.endInclude {
+ taskDone = recordKey.Cmp(taskRange.endKey) > 0
+ } else {
+ taskDone = recordKey.Cmp(taskRange.endKey) >= 0
+ }
+
+ if taskDone || len(w.rowRecords) >= w.batchCnt {
+ return false, nil
+ }
+
+ _, err := w.rowDecoder.DecodeTheExistedColumnMap(w.sessCtx, handle, rawRow, sysTZ, w.rowMap)
+ if err != nil {
+ return false, errors.Trace(err)
+ }
+
+ // Set the partitioning columns and calculate which partition to write to
+ for colID, offset := range w.writeColOffsetMap {
+ if d, ok := w.rowMap[colID]; ok {
+ tmpRow[offset] = d
+ } else {
+ return false, dbterror.ErrUnsupportedReorganizePartition.GenWithStackByArgs()
+ }
+ }
+ p, err := w.reorgedTbl.GetPartitionByRow(w.sessCtx, tmpRow)
+ if err != nil {
+ return false, errors.Trace(err)
+ }
+ pid := p.GetPhysicalID()
+ newKey := tablecodec.EncodeTablePrefix(pid)
+ newKey = append(newKey, recordKey[len(newKey):]...)
+ w.rowRecords = append(w.rowRecords, &rowRecord{
+ key: newKey, vals: rawRow,
+ })
+
+ w.cleanRowMap()
+ lastAccessedHandle = recordKey
+ if recordKey.Cmp(taskRange.endKey) == 0 {
+ taskDone = true
+ return false, nil
+ }
+ return true, nil
+ })
+
+ if len(w.rowRecords) == 0 {
+ taskDone = true
+ }
+
+ logutil.BgLogger().Debug("[ddl] txn fetches handle info", zap.Uint64("txnStartTS", txn.StartTS()), zap.String("taskRange", taskRange.String()), zap.Duration("takeTime", time.Since(startTime)))
+ return w.rowRecords, getNextHandleKey(taskRange, taskDone, lastAccessedHandle), taskDone, errors.Trace(err)
+}
+
+func (w *reorgPartitionWorker) cleanRowMap() {
+ for id := range w.rowMap {
+ delete(w.rowMap, id)
+ }
+}
+
+func (w *reorgPartitionWorker) AddMetricInfo(cnt float64) {
+ w.metricCounter.Add(cnt)
+}
+
+func (w *reorgPartitionWorker) String() string {
+ return typeReorgPartitionWorker.String()
+}
+
+func (w *reorgPartitionWorker) GetTask() (*BackfillJob, error) {
+ panic("[ddl] partition reorg worker does not implement GetTask function")
+}
+
+func (w *reorgPartitionWorker) UpdateTask(*BackfillJob) error {
+ panic("[ddl] partition reorg worker does not implement UpdateTask function")
+}
+
+func (w *reorgPartitionWorker) FinishTask(*BackfillJob) error {
+ panic("[ddl] partition reorg worker does not implement FinishTask function")
+}
+
+func (w *reorgPartitionWorker) GetCtx() *backfillCtx {
+ return w.backfillCtx
+}
+
+func (w *worker) reorgPartitionDataAndIndex(t table.Table, reorgInfo *reorgInfo) error {
+ // First copy all table data to the new partitions
+ // from each of the DroppingDefinitions partitions.
+ // Then create all indexes on the AddingDefinitions partitions
+ // for each new index, one partition at a time.
+
+ // Copy the data from the DroppingDefinitions to the AddingDefinitions
+ if bytes.Equal(reorgInfo.currElement.TypeKey, meta.ColumnElementKey) {
+ err := w.updatePhysicalTableRow(t, reorgInfo)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
+
+ failpoint.Inject("reorgPartitionAfterDataCopy", func(val failpoint.Value) {
+ //nolint:forcetypeassert
+ if val.(bool) {
+ panic("panic test in reorgPartitionAfterDataCopy")
+ }
+ })
+
+ // Rewrite this to do all indexes at once in addTableIndex
+ // instead of calling it once per index (meaning reading the table multiple times)
+ // But for now, try to understand how it works...
+ firstNewPartitionID := t.Meta().Partition.AddingDefinitions[0].ID
+ startElementOffset := 0
+ //startElementOffsetToResetHandle := -1
+ // This backfill job starts with backfilling index data, whose index ID is currElement.ID.
+ if !bytes.Equal(reorgInfo.currElement.TypeKey, meta.IndexElementKey) {
+ // First run, have not yet started backfilling index data
+ // Restart with the first new partition.
+ // TODO: handle remove partitioning
+ reorgInfo.PhysicalTableID = firstNewPartitionID
+ } else {
+ // The job was interrupted and has been restarted,
+ // reset and start from where it was done
+ for i, element := range reorgInfo.elements[1:] {
+ if reorgInfo.currElement.ID == element.ID {
+ startElementOffset = i
+ //startElementOffsetToResetHandle = i
+ break
+ }
+ }
+ }
+
+ for i := startElementOffset; i < len(reorgInfo.elements[1:]); i++ {
+ // Now build the indexes in the new partitions
+ var physTbl table.PhysicalTable
+ if tbl, ok := t.(table.PartitionedTable); ok {
+ physTbl = tbl.GetPartition(reorgInfo.PhysicalTableID)
+ } else if tbl, ok := t.(table.PhysicalTable); ok {
+ // This may be used when partitioning a non-partitioned table
+ physTbl = tbl
+ }
+ // Get the original start handle and end handle.
+ currentVer, err := getValidCurrentVersion(reorgInfo.d.store)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ // TODO: Can we improve this in case of a crash?
+ // like where the regInfo PhysicalTableID and element is the same,
+ // and the tableid in the key-prefix regInfo.StartKey and regInfo.EndKey matches with PhysicalTableID
+ // do not change the reorgInfo start/end key
+ startHandle, endHandle, err := getTableRange(reorgInfo.d.jobContext(reorgInfo.Job.ID), reorgInfo.d, physTbl, currentVer.Ver, reorgInfo.Job.Priority)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ // Always (re)start with the full PhysicalTable range
+ reorgInfo.StartKey, reorgInfo.EndKey = startHandle, endHandle
+
+ // Update the element in the reorgCtx to keep the atomic access for daemon-worker.
+ w.getReorgCtx(reorgInfo.Job.ID).setCurrentElement(reorgInfo.elements[i+1])
+
+ // Update the element in the reorgInfo for updating the reorg meta below.
+ reorgInfo.currElement = reorgInfo.elements[i+1]
+ // Write the reorg info to store so the whole reorganize process can recover from panic.
+ err = reorgInfo.UpdateReorgMeta(reorgInfo.StartKey, w.sessPool)
+ logutil.BgLogger().Info("[ddl] update column and indexes",
+ zap.Int64("jobID", reorgInfo.Job.ID),
+ zap.ByteString("elementType", reorgInfo.currElement.TypeKey),
+ zap.Int64("elementID", reorgInfo.currElement.ID),
+ zap.Int64("partitionTableId", physTbl.GetPhysicalID()),
+ zap.String("startHandle", hex.EncodeToString(reorgInfo.StartKey)),
+ zap.String("endHandle", hex.EncodeToString(reorgInfo.EndKey)))
+ if err != nil {
+ return errors.Trace(err)
+ }
+ err = w.addTableIndex(t, reorgInfo)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ reorgInfo.PhysicalTableID = firstNewPartitionID
+ }
+ failpoint.Inject("reorgPartitionAfterIndex", func(val failpoint.Value) {
+ //nolint:forcetypeassert
+ if val.(bool) {
+ panic("panic test in reorgPartitionAfterIndex")
+ }
+ })
+ return nil
+}
+
func bundlesForExchangeTablePartition(t *meta.Meta, job *model.Job, pt *model.TableInfo, newPar *model.PartitionDefinition, nt *model.TableInfo) ([]*placement.Bundle, error) {
bundles := make([]*placement.Bundle, 0, 3)
diff --git a/ddl/placement/bundle.go b/ddl/placement/bundle.go
index d8f24f08583f5..62faeb5fe8a3a 100644
--- a/ddl/placement/bundle.go
+++ b/ddl/placement/bundle.go
@@ -29,6 +29,7 @@ import (
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/util/codec"
"golang.org/x/exp/slices"
+ "gopkg.in/yaml.v2"
)
// Refer to https://github.com/tikv/pd/issues/2701 .
@@ -123,7 +124,13 @@ func NewBundleFromConstraintsOptions(options *model.PlacementSettings) (*Bundle,
rules = append(rules, rule)
}
}
-
+ labels, err := newLocationLabelsFromSurvivalPreferences(options.SurvivalPreferences)
+ if err != nil {
+ return nil, err
+ }
+ for _, rule := range rules {
+ rule.LocationLabels = labels
+ }
return &Bundle{Rules: rules}, nil
}
@@ -155,9 +162,17 @@ func NewBundleFromSugarOptions(options *model.PlacementSettings) (*Bundle, error
var rules []*Rule
+ locationLabels, err := newLocationLabelsFromSurvivalPreferences(options.SurvivalPreferences)
+ if err != nil {
+ return nil, err
+ }
+
// in case empty primaryRegion and regions, just return an empty bundle
if primaryRegion == "" && len(regions) == 0 {
rules = append(rules, NewRule(Voter, followers+1, NewConstraintsDirect()))
+ for _, rule := range rules {
+ rule.LocationLabels = locationLabels
+ }
return &Bundle{Rules: rules}, nil
}
@@ -195,6 +210,11 @@ func NewBundleFromSugarOptions(options *model.PlacementSettings) (*Bundle, error
}
}
+ // set location labels
+ for _, rule := range rules {
+ rule.LocationLabels = locationLabels
+ }
+
return &Bundle{Rules: rules}, nil
}
@@ -223,6 +243,19 @@ func newBundleFromOptions(options *model.PlacementSettings) (bundle *Bundle, err
return bundle, err
}
+// newLocationLabelsFromSurvivalPreferences will parse the survival preferences into location labels.
+func newLocationLabelsFromSurvivalPreferences(survivalPreferenceStr string) ([]string, error) {
+ if len(survivalPreferenceStr) > 0 {
+ labels := []string{}
+ err := yaml.UnmarshalStrict([]byte(survivalPreferenceStr), &labels)
+ if err != nil {
+ return nil, ErrInvalidSurvivalPreferenceFormat
+ }
+ return labels, nil
+ }
+ return nil, nil
+}
+
// NewBundleFromOptions will transform options into the bundle.
func NewBundleFromOptions(options *model.PlacementSettings) (bundle *Bundle, err error) {
bundle, err = newBundleFromOptions(options)
@@ -257,6 +290,15 @@ func (b *Bundle) String() string {
func (b *Bundle) Tidy() error {
extraCnt := map[PeerRoleType]int{}
newRules := b.Rules[:0]
+
+ // One Bundle is from one PlacementSettings, rule share same location labels, so we can use the first rule's location labels.
+ var locationLabels []string
+ for _, rule := range b.Rules {
+ if len(rule.LocationLabels) > 0 {
+ locationLabels = rule.LocationLabels
+ break
+ }
+ }
for i, rule := range b.Rules {
// useless Rule
if rule.Count <= 0 {
@@ -300,6 +342,8 @@ func (b *Bundle) Tidy() error {
Key: EngineLabelKey,
Values: []string{EngineLabelTiFlash},
}},
+ // the merged rule should have the same location labels with the original rules.
+ LocationLabels: locationLabels,
})
}
b.Rules = newRules
diff --git a/ddl/placement/bundle_test.go b/ddl/placement/bundle_test.go
index a219fc0589789..0dc9032da8562 100644
--- a/ddl/placement/bundle_test.go
+++ b/ddl/placement/bundle_test.go
@@ -887,6 +887,9 @@ func TestTidy(t *testing.T) {
bundle.Rules = append(bundle.Rules, rules3...)
bundle.Rules = append(bundle.Rules, rules4...)
+ for _, r := range bundle.Rules {
+ r.LocationLabels = []string{"zone", "host"}
+ }
chkfunc := func() {
require.NoError(t, err)
require.Len(t, bundle.Rules, 3)
@@ -901,6 +904,7 @@ func TestTidy(t *testing.T) {
Values: []string{EngineLabelTiFlash},
},
}, bundle.Rules[2].Constraints)
+ require.Equal(t, []string{"zone", "host"}, bundle.Rules[2].LocationLabels)
}
err = bundle.Tidy()
chkfunc()
diff --git a/ddl/placement/common.go b/ddl/placement/common.go
index cd02622dd0562..7c77ead97e30e 100644
--- a/ddl/placement/common.go
+++ b/ddl/placement/common.go
@@ -54,4 +54,8 @@ const (
// EngineLabelTiKV is the label value used in some tests. And possibly TiKV will
// set the engine label with a value of EngineLabelTiKV.
EngineLabelTiKV = "tikv"
+
+ // EngineLabelTiFlashCompute is for disaggregated tiflash mode,
+ // it's the lable of tiflash_compute nodes.
+ EngineLabelTiFlashCompute = "tiflash_compute"
)
diff --git a/ddl/placement/errors.go b/ddl/placement/errors.go
index b609827bd4ce7..a93d0e2bd45b2 100644
--- a/ddl/placement/errors.go
+++ b/ddl/placement/errors.go
@@ -29,6 +29,8 @@ var (
ErrInvalidConstraintsMapcnt = errors.New("label constraints in map syntax have invalid replicas")
// ErrInvalidConstraintsFormat is from rule.go.
ErrInvalidConstraintsFormat = errors.New("invalid label constraints format")
+ // ErrInvalidSurvivalPreferenceFormat is from rule.go.
+ ErrInvalidSurvivalPreferenceFormat = errors.New("survival preference format should be in format [xxx=yyy, ...]")
// ErrInvalidConstraintsRelicas is from rule.go.
ErrInvalidConstraintsRelicas = errors.New("label constraints with invalid REPLICAS")
// ErrInvalidBundleID is from bundle.go.
diff --git a/ddl/placement/rule.go b/ddl/placement/rule.go
index 2e91df421dc21..bea4710236338 100644
--- a/ddl/placement/rule.go
+++ b/ddl/placement/rule.go
@@ -45,15 +45,16 @@ type RuleGroupConfig struct {
// Rule is the core placement rule struct. Check https://github.com/tikv/pd/blob/master/server/schedule/placement/rule.go.
type Rule struct {
- GroupID string `json:"group_id"`
- ID string `json:"id"`
- Index int `json:"index,omitempty"`
- Override bool `json:"override,omitempty"`
- StartKeyHex string `json:"start_key"`
- EndKeyHex string `json:"end_key"`
- Role PeerRoleType `json:"role"`
- Count int `json:"count"`
- Constraints Constraints `json:"label_constraints,omitempty"`
+ GroupID string `json:"group_id"`
+ ID string `json:"id"`
+ Index int `json:"index,omitempty"`
+ Override bool `json:"override,omitempty"`
+ StartKeyHex string `json:"start_key"`
+ EndKeyHex string `json:"end_key"`
+ Role PeerRoleType `json:"role"`
+ Count int `json:"count"`
+ Constraints Constraints `json:"label_constraints,omitempty"`
+ LocationLabels []string `json:"location_labels,omitempty"`
}
// TiFlashRule extends Rule with other necessary fields.
diff --git a/ddl/placement_policy_ddl_test.go b/ddl/placement_policy_ddl_test.go
index f5d1392018c84..0c0793e711d8e 100644
--- a/ddl/placement_policy_ddl_test.go
+++ b/ddl/placement_policy_ddl_test.go
@@ -121,6 +121,7 @@ func TestPlacementPolicyInUse(t *testing.T) {
builder, err := infoschema.NewBuilder(store, nil).InitWithDBInfos(
[]*model.DBInfo{db1, db2, dbP},
[]*model.PolicyInfo{p1, p2, p3, p4, p5},
+ nil,
1,
)
require.NoError(t, err)
diff --git a/ddl/placement_policy_test.go b/ddl/placement_policy_test.go
index 559cc0ff59a46..0b2c3f29b21ca 100644
--- a/ddl/placement_policy_test.go
+++ b/ddl/placement_policy_test.go
@@ -24,6 +24,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/ddl/placement"
"github.com/pingcap/tidb/ddl/util"
"github.com/pingcap/tidb/domain"
@@ -135,7 +136,7 @@ func TestPlacementPolicy(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("drop placement policy if exists x")
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var policyID int64
onJobUpdatedExportedFunc := func(job *model.Job) {
if policyID != 0 {
@@ -154,7 +155,8 @@ func TestPlacementPolicy(t *testing.T) {
"LEARNERS=1 " +
"LEARNER_CONSTRAINTS=\"[+region=cn-west-1]\" " +
"FOLLOWERS=3 " +
- "FOLLOWER_CONSTRAINTS=\"[+disk=ssd]\"")
+ "FOLLOWER_CONSTRAINTS=\"[+disk=ssd]\"" +
+ "SURVIVAL_PREFERENCES=\"[region, zone]\"")
checkFunc := func(policyInfo *model.PolicyInfo) {
require.Equal(t, true, policyInfo.ID != 0)
@@ -167,6 +169,7 @@ func TestPlacementPolicy(t *testing.T) {
require.Equal(t, "[+region=cn-west-1]", policyInfo.LearnerConstraints)
require.Equal(t, model.StatePublic, policyInfo.State)
require.Equal(t, "", policyInfo.Schedule)
+ require.Equal(t, "[region, zone]", policyInfo.SurvivalPreferences)
}
// Check the policy is correctly reloaded in the information schema.
@@ -589,11 +592,16 @@ func TestCreateTableWithPlacementPolicy(t *testing.T) {
tk.MustExec("create placement policy x " +
"FOLLOWERS=2 " +
"CONSTRAINTS=\"[+disk=ssd]\" ")
+ tk.MustExec("create placement policy z " +
+ "FOLLOWERS=1 " +
+ "SURVIVAL_PREFERENCES=\"[region, zone]\"")
tk.MustExec("create placement policy y " +
"FOLLOWERS=3 " +
"CONSTRAINTS=\"[+region=bj]\" ")
tk.MustExec("create table t(a int)" +
"PLACEMENT POLICY=\"x\"")
+ tk.MustExec("create table tt(a int)" +
+ "PLACEMENT POLICY=\"z\"")
tk.MustQuery("SELECT TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TIDB_PLACEMENT_POLICY_NAME FROM information_schema.Tables WHERE TABLE_SCHEMA='test' AND TABLE_NAME = 't'").Check(testkit.Rows(`def test t x`))
tk.MustExec("create table t_range_p(id int) placement policy x partition by range(id) (" +
"PARTITION p0 VALUES LESS THAN (100)," +
@@ -616,7 +624,18 @@ func TestCreateTableWithPlacementPolicy(t *testing.T) {
require.Equal(t, "y", policyY.Name.L)
require.Equal(t, true, policyY.ID != 0)
- tbl := external.GetTableByName(t, tk, "test", "t")
+ policyZ := testGetPolicyByName(t, tk.Session(), "z", true)
+ require.Equal(t, "z", policyZ.Name.L)
+ require.Equal(t, true, policyZ.ID != 0)
+ require.Equal(t, "[region, zone]", policyZ.SurvivalPreferences)
+
+ tbl := external.GetTableByName(t, tk, "test", "tt")
+ require.NotNil(t, tbl)
+ require.NotNil(t, tbl.Meta().PlacementPolicyRef)
+ require.Equal(t, "z", tbl.Meta().PlacementPolicyRef.Name.L)
+ require.Equal(t, policyZ.ID, tbl.Meta().PlacementPolicyRef.ID)
+
+ tbl = external.GetTableByName(t, tk, "test", "t")
require.NotNil(t, tbl)
require.NotNil(t, tbl.Meta().PlacementPolicyRef)
require.Equal(t, "x", tbl.Meta().PlacementPolicyRef.Name.L)
diff --git a/ddl/placement_sql_test.go b/ddl/placement_sql_test.go
index 5daca69feabec..b9af2af16e19d 100644
--- a/ddl/placement_sql_test.go
+++ b/ddl/placement_sql_test.go
@@ -588,7 +588,7 @@ func checkTiflashReplicaSet(t *testing.T, do *domain.Domain, db, tb string, cnt
return
}
- CheckPlacementRule(infosync.GetMockTiFlash(), *infosync.MakeNewRule(tbl.Meta().ID, 1, nil))
+ infosync.GetMockTiFlash().CheckPlacementRule(*infosync.MakeNewRule(tbl.Meta().ID, 1, nil))
require.NotNil(t, tiflashReplica)
require.Equal(t, cnt, tiflashReplica.Count)
}
diff --git a/ddl/reorg.go b/ddl/reorg.go
index 2c7508d24b38f..3bde3fdaa844b 100644
--- a/ddl/reorg.go
+++ b/ddl/reorg.go
@@ -18,6 +18,7 @@ import (
"encoding/hex"
"fmt"
"strconv"
+ "strings"
"sync"
"sync/atomic"
"time"
@@ -34,7 +35,6 @@ import (
"github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/stmtctx"
- "github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/statistics"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/table/tables"
@@ -48,6 +48,7 @@ import (
"github.com/pingcap/tidb/util/ranger"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/pingcap/tipb/go-tipb"
+ atomicutil "go.uber.org/atomic"
"go.uber.org/zap"
)
@@ -77,6 +78,8 @@ type reorgCtx struct {
warnings map[errors.ErrorID]*terror.Error
warningsCount map[errors.ErrorID]int64
}
+
+ references atomicutil.Int32
}
// nullableKey can store kv.Key.
@@ -142,11 +145,9 @@ func (rc *reorgCtx) increaseRowCount(count int64) {
atomic.AddInt64(&rc.rowCount, count)
}
-func (rc *reorgCtx) getRowCountAndKey() (int64, kv.Key, *meta.Element) {
+func (rc *reorgCtx) getRowCount() int64 {
row := atomic.LoadInt64(&rc.rowCount)
- h, _ := (rc.doneKey.Load()).(nullableKey)
- element, _ := (rc.element.Load()).(*meta.Element)
- return row, h.key, element
+ return row
}
// runReorgJob is used as a portal to do the reorganization work.
@@ -154,6 +155,7 @@ func (rc *reorgCtx) getRowCountAndKey() (int64, kv.Key, *meta.Element) {
// 1: add index
// 2: alter column type
// 3: clean global index
+// 4: reorganize partitions
/*
ddl goroutine >---------+
^ |
@@ -197,7 +199,7 @@ func (w *worker) runReorgJob(rh *reorgHandler, reorgInfo *reorgInfo, tblInfo *mo
}
}
- rc := w.getReorgCtx(job)
+ rc := w.getReorgCtx(job.ID)
if rc == nil {
// This job is cancelling, we should return ErrCancelledDDLJob directly.
// Q: Is there any possibility that the job is cancelling and has no reorgCtx?
@@ -207,7 +209,7 @@ func (w *worker) runReorgJob(rh *reorgHandler, reorgInfo *reorgInfo, tblInfo *mo
if job.IsCancelling() {
return dbterror.ErrCancelledDDLJob
}
- rc = w.newReorgCtx(reorgInfo)
+ rc = w.newReorgCtx(reorgInfo.Job.ID, reorgInfo.StartKey, reorgInfo.currElement, reorgInfo.Job.GetRowCount())
w.wg.Add(1)
go func() {
defer w.wg.Done()
@@ -230,17 +232,22 @@ func (w *worker) runReorgJob(rh *reorgHandler, reorgInfo *reorgInfo, tblInfo *mo
case err := <-rc.doneCh:
// Since job is cancelled，we don't care about its partial counts.
if rc.isReorgCanceled() || terror.ErrorEqual(err, dbterror.ErrCancelledDDLJob) {
- d.removeReorgCtx(job)
+ d.removeReorgCtx(job.ID)
return dbterror.ErrCancelledDDLJob
}
- rowCount, _, _ := rc.getRowCountAndKey()
- logutil.BgLogger().Info("[ddl] run reorg job done", zap.Int64("handled rows", rowCount))
+ rowCount := rc.getRowCount()
+ if err != nil {
+ logutil.BgLogger().Warn("[ddl] run reorg job done", zap.Int64("handled rows", rowCount), zap.Error(err))
+ } else {
+ logutil.BgLogger().Info("[ddl] run reorg job done", zap.Int64("handled rows", rowCount))
+ }
+
job.SetRowCount(rowCount)
// Update a job's warnings.
w.mergeWarningsIntoJob(job)
- d.removeReorgCtx(job)
+ d.removeReorgCtx(job.ID)
// For other errors, even err is not nil here, we still wait the partial counts to be collected.
// since in the next round, the startKey is brand new which is stored by last time.
if err != nil {
@@ -248,17 +255,13 @@ func (w *worker) runReorgJob(rh *reorgHandler, reorgInfo *reorgInfo, tblInfo *mo
}
updateBackfillProgress(w, reorgInfo, tblInfo, 0)
- if err1 := rh.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil {
- logutil.BgLogger().Warn("[ddl] run reorg job done, removeDDLReorgHandle failed", zap.Error(err1))
- return errors.Trace(err1)
- }
case <-w.ctx.Done():
logutil.BgLogger().Info("[ddl] run reorg job quit")
- d.removeReorgCtx(job)
+ d.removeReorgCtx(job.ID)
// We return dbterror.ErrWaitReorgTimeout here too, so that outer loop will break.
return dbterror.ErrWaitReorgTimeout
case <-time.After(waitTimeout):
- rowCount, doneKey, currentElement := rc.getRowCountAndKey()
+ rowCount := rc.getRowCount()
job.SetRowCount(rowCount)
updateBackfillProgress(w, reorgInfo, tblInfo, rowCount)
@@ -267,18 +270,9 @@ func (w *worker) runReorgJob(rh *reorgHandler, reorgInfo *reorgInfo, tblInfo *mo
rc.resetWarnings()
- // Update a reorgInfo's handle.
- // Since daemon-worker is triggered by timer to store the info half-way.
- // you should keep these infos is read-only (like job) / atomic (like doneKey & element) / concurrent safe.
- err := rh.UpdateDDLReorgStartHandle(job, currentElement, doneKey)
-
logutil.BgLogger().Info("[ddl] run reorg job wait timeout",
- zap.Duration("waitTime", waitTimeout),
- zap.ByteString("elementType", currentElement.TypeKey),
- zap.Int64("elementID", currentElement.ID),
- zap.Int64("totalAddedRowCount", rowCount),
- zap.String("doneKey", tryDecodeToHandleString(doneKey)),
- zap.Error(err))
+ zap.Duration("wait time", waitTimeout),
+ zap.Int64("total added row count", rowCount))
// If timeout, we will return, check the owner and retry to wait job done again.
return dbterror.ErrWaitReorgTimeout
}
@@ -286,12 +280,14 @@ func (w *worker) runReorgJob(rh *reorgHandler, reorgInfo *reorgInfo, tblInfo *mo
}
func (w *worker) mergeWarningsIntoJob(job *model.Job) {
- rc := w.getReorgCtx(job)
+ rc := w.getReorgCtx(job.ID)
rc.mu.Lock()
- defer rc.mu.Unlock()
partWarnings := rc.mu.warnings
partWarningsCount := rc.mu.warningsCount
- job.SetWarnings(mergeWarningsAndWarningsCount(partWarnings, job.ReorgMeta.Warnings, partWarningsCount, job.ReorgMeta.WarningsCount))
+ rc.mu.Unlock()
+ warnings, warningsCount := job.GetWarnings()
+ warnings, warningsCount = mergeWarningsAndWarningsCount(partWarnings, warnings, partWarningsCount, warningsCount)
+ job.SetWarnings(warnings, warningsCount)
}
func updateBackfillProgress(w *worker, reorgInfo *reorgInfo, tblInfo *model.TableInfo,
@@ -310,6 +306,10 @@ func updateBackfillProgress(w *worker, reorgInfo *reorgInfo, tblInfo *model.Tabl
if progress > 1 {
progress = 1
}
+ logutil.BgLogger().Debug("[ddl] update progress",
+ zap.Float64("progress", progress),
+ zap.Int64("addedRowCount", addedRowCount),
+ zap.Int64("totalCount", totalCount))
}
switch reorgInfo.Type {
case model.ActionAddIndex, model.ActionAddPrimaryKey:
@@ -322,6 +322,8 @@ func updateBackfillProgress(w *worker, reorgInfo *reorgInfo, tblInfo *model.Tabl
metrics.GetBackfillProgressByLabel(label, reorgInfo.SchemaName, tblInfo.Name.String()).Set(progress * 100)
case model.ActionModifyColumn:
metrics.GetBackfillProgressByLabel(metrics.LblModifyColumn, reorgInfo.SchemaName, tblInfo.Name.String()).Set(progress * 100)
+ case model.ActionReorganizePartition:
+ metrics.GetBackfillProgressByLabel(metrics.LblReorgPartition, reorgInfo.SchemaName, tblInfo.Name.String()).Set(progress * 100)
}
}
@@ -338,8 +340,20 @@ func getTableTotalCount(w *worker, tblInfo *model.TableInfo) int64 {
if !ok {
return statistics.PseudoRowCount
}
- sql := "select table_rows from information_schema.tables where tidb_table_id=%?;"
- rows, _, err := executor.ExecRestrictedSQL(w.ctx, nil, sql, tblInfo.ID)
+ var rows []chunk.Row
+ if tblInfo.Partition != nil && len(tblInfo.Partition.DroppingDefinitions) > 0 {
+ // if Reorganize Partition, only select number of rows from the selected partitions!
+ defs := tblInfo.Partition.DroppingDefinitions
+ partIDs := make([]string, 0, len(defs))
+ for _, def := range defs {
+ partIDs = append(partIDs, strconv.FormatInt(def.ID, 10))
+ }
+ sql := "select sum(table_rows) from information_schema.partitions where tidb_partition_id in (%?);"
+ rows, _, err = executor.ExecRestrictedSQL(w.ctx, nil, sql, strings.Join(partIDs, ","))
+ } else {
+ sql := "select table_rows from information_schema.tables where tidb_table_id=%?;"
+ rows, _, err = executor.ExecRestrictedSQL(w.ctx, nil, sql, tblInfo.ID)
+ }
if err != nil {
return statistics.PseudoRowCount
}
@@ -349,17 +363,21 @@ func getTableTotalCount(w *worker, tblInfo *model.TableInfo) int64 {
return rows[0].GetInt64(0)
}
-func (dc *ddlCtx) isReorgRunnable(job *model.Job) error {
+func (dc *ddlCtx) isReorgRunnable(jobID int64, isDistReorg bool) error {
if isChanClosed(dc.ctx.Done()) {
// Worker is closed. So it can't do the reorganization.
return dbterror.ErrInvalidWorker.GenWithStack("worker is closed")
}
- if dc.getReorgCtx(job).isReorgCanceled() {
+ if dc.getReorgCtx(jobID).isReorgCanceled() {
// Job is cancelled. So it can't be done.
return dbterror.ErrCancelledDDLJob
}
+ // If isDistReorg is true, we needn't check if it is owner.
+ if isDistReorg {
+ return nil
+ }
if !dc.isOwner() {
// If it's not the owner, we will try later, so here just returns an error.
logutil.BgLogger().Info("[ddl] DDL is not the DDL owner", zap.String("ID", dc.uuid))
@@ -381,6 +399,7 @@ type reorgInfo struct {
// PhysicalTableID is used to trace the current partition we are handling.
// If the table is not partitioned, PhysicalTableID would be TableID.
PhysicalTableID int64
+ dbInfo *model.DBInfo
elements []*meta.Element
currElement *meta.Element
}
@@ -559,10 +578,12 @@ func getTableRange(ctx *JobContext, d *ddlCtx, tbl table.PhysicalTable, snapshot
endHandleKey = tablecodec.EncodeRecordKey(tbl.RecordPrefix(), maxHandle)
}
if isEmptyTable || endHandleKey.Cmp(startHandleKey) < 0 {
- logutil.BgLogger().Info("[ddl] get table range, endHandle < startHandle", zap.String("table", fmt.Sprintf("%v", tbl.Meta())),
+ logutil.BgLogger().Info("[ddl] get noop table range",
+ zap.String("table", fmt.Sprintf("%v", tbl.Meta())),
zap.Int64("table/partition ID", tbl.GetPhysicalID()),
- zap.String("endHandle", tryDecodeToHandleString(endHandleKey)),
- zap.String("startHandle", tryDecodeToHandleString(startHandleKey)))
+ zap.String("start key", hex.EncodeToString(startHandleKey)),
+ zap.String("end key", hex.EncodeToString(endHandleKey)),
+ zap.Bool("is empty table", isEmptyTable))
endHandleKey = startHandleKey
}
return
@@ -578,7 +599,7 @@ func getValidCurrentVersion(store kv.Storage) (ver kv.Version, err error) {
return ver, nil
}
-func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job,
+func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, dbInfo *model.DBInfo,
tbl table.Table, elements []*meta.Element, mergingTmpIdx bool) (*reorgInfo, error) {
var (
element *meta.Element
@@ -634,10 +655,6 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job,
failpoint.Inject("errorUpdateReorgHandle", func() (*reorgInfo, error) {
return &info, errors.New("occur an error when update reorg handle")
})
- err = rh.RemoveDDLReorgHandle(job, elements)
- if err != nil {
- return &info, errors.Trace(err)
- }
err = rh.InitDDLReorgHandle(job, start, end, pid, elements[0])
if err != nil {
return &info, errors.Trace(err)
@@ -665,7 +682,7 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job,
// We'll try to remove it in the next major TiDB version.
if meta.ErrDDLReorgElementNotExist.Equal(err) {
job.SnapshotVer = 0
- logutil.BgLogger().Warn("[ddl] get reorg info, the element does not exist", zap.String("job", job.String()), zap.Bool("enableConcurrentDDL", rh.enableConcurrentDDL))
+ logutil.BgLogger().Warn("[ddl] get reorg info, the element does not exist", zap.String("job", job.String()))
}
return &info, errors.Trace(err)
}
@@ -678,11 +695,12 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job,
info.currElement = element
info.elements = elements
info.mergingTmpIdx = mergingTmpIdx
+ info.dbInfo = dbInfo
return &info, nil
}
-func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, tbl table.Table, partitionIDs []int64, elements []*meta.Element) (*reorgInfo, error) {
+func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, dbInfo *model.DBInfo, tbl table.PartitionedTable, partitionIDs []int64, elements []*meta.Element) (*reorgInfo, error) {
var (
element *meta.Element
start kv.Key
@@ -700,15 +718,16 @@ func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, jo
return nil, errors.Trace(err)
}
pid = partitionIDs[0]
- tb := tbl.(table.PartitionedTable).GetPartition(pid)
- start, end, err = getTableRange(ctx, d, tb, ver.Ver, job.Priority)
+ physTbl := tbl.GetPartition(pid)
+
+ start, end, err = getTableRange(ctx, d, physTbl, ver.Ver, job.Priority)
if err != nil {
return nil, errors.Trace(err)
}
logutil.BgLogger().Info("[ddl] job get table range",
- zap.Int64("jobID", job.ID), zap.Int64("physicalTableID", pid),
- zap.String("startHandle", tryDecodeToHandleString(start)),
- zap.String("endHandle", tryDecodeToHandleString(end)))
+ zap.Int64("job ID", job.ID), zap.Int64("physical table ID", pid),
+ zap.String("start key", hex.EncodeToString(start)),
+ zap.String("end key", hex.EncodeToString(end)))
err = rh.InitDDLReorgHandle(job, start, end, pid, elements[0])
if err != nil {
@@ -738,32 +757,30 @@ func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, jo
info.PhysicalTableID = pid
info.currElement = element
info.elements = elements
+ info.dbInfo = dbInfo
return &info, nil
}
+// UpdateReorgMeta creates a new transaction and updates tidb_ddl_reorg table,
+// so the reorg can restart in case of issues.
func (r *reorgInfo) UpdateReorgMeta(startKey kv.Key, pool *sessionPool) (err error) {
if startKey == nil && r.EndKey == nil {
return nil
}
- se, err := pool.get()
+ sctx, err := pool.get()
if err != nil {
return
}
- defer pool.put(se)
+ defer pool.put(sctx)
- sess := newSession(se)
+ sess := newSession(sctx)
err = sess.begin()
if err != nil {
return
}
- txn, err := sess.txn()
- if err != nil {
- sess.rollback()
- return err
- }
- rh := newReorgHandler(meta.NewMeta(txn), sess, variable.EnableConcurrentDDL.Load())
- err = rh.UpdateDDLReorgHandle(r.Job, startKey, r.EndKey, r.PhysicalTableID, r.currElement)
+ rh := newReorgHandler(sess)
+ err = updateDDLReorgHandle(rh.s, r.Job.ID, startKey, r.EndKey, r.PhysicalTableID, r.currElement)
err1 := sess.commit()
if err == nil {
err = err1
@@ -773,65 +790,48 @@ func (r *reorgInfo) UpdateReorgMeta(startKey kv.Key, pool *sessionPool) (err err
// reorgHandler is used to handle the reorg information duration reorganization DDL job.
type reorgHandler struct {
- m *meta.Meta
s *session
-
- enableConcurrentDDL bool
}
// NewReorgHandlerForTest creates a new reorgHandler, only used in test.
-func NewReorgHandlerForTest(t *meta.Meta, sess sessionctx.Context) *reorgHandler {
- return newReorgHandler(t, newSession(sess), variable.EnableConcurrentDDL.Load())
-}
-
-func newReorgHandler(t *meta.Meta, sess *session, enableConcurrentDDL bool) *reorgHandler {
- return &reorgHandler{m: t, s: sess, enableConcurrentDDL: enableConcurrentDDL}
+func NewReorgHandlerForTest(sess sessionctx.Context) *reorgHandler {
+ return newReorgHandler(newSession(sess))
}
-// UpdateDDLReorgStartHandle saves the job reorganization latest processed element and start handle for later resuming.
-func (r *reorgHandler) UpdateDDLReorgStartHandle(job *model.Job, element *meta.Element, startKey kv.Key) error {
- if r.enableConcurrentDDL {
- return updateDDLReorgStartHandle(r.s, job, element, startKey)
- }
- return r.m.UpdateDDLReorgStartHandle(job, element, startKey)
-}
-
-// UpdateDDLReorgHandle saves the job reorganization latest processed information for later resuming.
-func (r *reorgHandler) UpdateDDLReorgHandle(job *model.Job, startKey, endKey kv.Key, physicalTableID int64, element *meta.Element) error {
- if r.enableConcurrentDDL {
- return updateDDLReorgHandle(r.s, job.ID, startKey, endKey, physicalTableID, element)
- }
- return r.m.UpdateDDLReorgHandle(job.ID, startKey, endKey, physicalTableID, element)
+func newReorgHandler(sess *session) *reorgHandler {
+ return &reorgHandler{s: sess}
}
// InitDDLReorgHandle initializes the job reorganization information.
func (r *reorgHandler) InitDDLReorgHandle(job *model.Job, startKey, endKey kv.Key, physicalTableID int64, element *meta.Element) error {
- if r.enableConcurrentDDL {
- return initDDLReorgHandle(r.s, job.ID, startKey, endKey, physicalTableID, element)
- }
- return r.m.UpdateDDLReorgHandle(job.ID, startKey, endKey, physicalTableID, element)
+ return initDDLReorgHandle(r.s, job.ID, startKey, endKey, physicalTableID, element)
}
// RemoveReorgElementFailPoint removes the element of the reorganization information.
func (r *reorgHandler) RemoveReorgElementFailPoint(job *model.Job) error {
- if r.enableConcurrentDDL {
- return removeReorgElement(r.s, job)
- }
- return r.m.RemoveReorgElement(job)
+ return removeReorgElement(r.s, job)
}
// RemoveDDLReorgHandle removes the job reorganization related handles.
func (r *reorgHandler) RemoveDDLReorgHandle(job *model.Job, elements []*meta.Element) error {
- if r.enableConcurrentDDL {
- return removeDDLReorgHandle(r.s, job, elements)
+ return removeDDLReorgHandle(r.s, job, elements)
+}
+
+// CleanupDDLReorgHandles removes the job reorganization related handles.
+func CleanupDDLReorgHandles(job *model.Job, s *session) {
+ if job != nil && !job.IsFinished() && !job.IsSynced() {
+ // Job is given, but it is neither finished nor synced; do nothing
+ return
+ }
+
+ err := cleanDDLReorgHandles(s, job)
+ if err != nil {
+ // ignore error, cleanup is not that critical
+ logutil.BgLogger().Warn("Failed removing the DDL reorg entry in tidb_ddl_reorg", zap.String("job", job.String()), zap.Error(err))
}
- return r.m.RemoveDDLReorgHandle(job, elements)
}
// GetDDLReorgHandle gets the latest processed DDL reorganize position.
func (r *reorgHandler) GetDDLReorgHandle(job *model.Job) (element *meta.Element, startKey, endKey kv.Key, physicalTableID int64, err error) {
- if r.enableConcurrentDDL {
- return getDDLReorgHandle(r.s, job)
- }
- return r.m.GetDDLReorgHandle(job)
+ return getDDLReorgHandle(r.s, job)
}
diff --git a/ddl/reorg_partition_test.go b/ddl/reorg_partition_test.go
new file mode 100644
index 0000000000000..fb0ae51f29b25
--- /dev/null
+++ b/ddl/reorg_partition_test.go
@@ -0,0 +1,822 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl_test
+
+import (
+ "bytes"
+ "context"
+ "encoding/hex"
+ "fmt"
+ "testing"
+
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/ddl/internal/callback"
+ "github.com/pingcap/tidb/domain"
+ "github.com/pingcap/tidb/errno"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/sessiontxn"
+ "github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/tablecodec"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/mathutil"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/zap"
+)
+
+type allTableData struct {
+ keys [][]byte
+ vals [][]byte
+ tp []string
+}
+
+// TODO: Create a more generic function that gets all accessible table ids
+// from all schemas, and checks the full key space so that there are no
+// keys for non-existing table IDs. Also figure out how to wait for deleteRange
+// Checks that there are no accessible data after an existing table
+// assumes that tableIDs are only increasing.
+// To be used during failure testing of ALTER, to make sure cleanup is done.
+func noNewTablesAfter(t *testing.T, tk *testkit.TestKit, ctx sessionctx.Context, tbl table.Table) {
+ waitForGC := tk.MustQuery(`select start_key, end_key from mysql.gc_delete_range`).Rows()
+ require.NoError(t, sessiontxn.NewTxn(context.Background(), ctx))
+ txn, err := ctx.Txn(true)
+ require.NoError(t, err)
+ defer func() {
+ err := txn.Rollback()
+ require.NoError(t, err)
+ }()
+ // Get max tableID (if partitioned)
+ tblID := tbl.Meta().ID
+ if pt := tbl.GetPartitionedTable(); pt != nil {
+ defs := pt.Meta().Partition.Definitions
+ {
+ for i := range defs {
+ tblID = mathutil.Max[int64](tblID, defs[i].ID)
+ }
+ }
+ }
+ prefix := tablecodec.EncodeTablePrefix(tblID + 1)
+ it, err := txn.Iter(prefix, nil)
+ require.NoError(t, err)
+ROW:
+ for it.Valid() {
+ for _, rowGC := range waitForGC {
+ // OK if queued for range delete / GC
+ hexString := fmt.Sprintf("%v", rowGC[0])
+ start, err := hex.DecodeString(hexString)
+ require.NoError(t, err)
+ hexString = fmt.Sprintf("%v", rowGC[1])
+ end, err := hex.DecodeString(hexString)
+ require.NoError(t, err)
+ if bytes.Compare(start, it.Key()) >= 0 && bytes.Compare(it.Key(), end) < 0 {
+ it.Close()
+ it, err = txn.Iter(end, nil)
+ require.NoError(t, err)
+ continue ROW
+ }
+ }
+ foundTblID := tablecodec.DecodeTableID(it.Key())
+ // There are internal table ids starting from MaxInt48 -1 and allocating decreasing ids
+ // Allow 0xFF of them, See JobTableID, ReorgTableID, HistoryTableID, MDLTableID
+ require.False(t, it.Key()[0] == 't' && foundTblID < 0xFFFFFFFFFF00, "Found table data after highest physical Table ID %d < %d", tblID, foundTblID)
+ break
+ }
+}
+
+func getAllDataForPhysicalTable(t *testing.T, ctx sessionctx.Context, physTable table.PhysicalTable) allTableData {
+ require.NoError(t, sessiontxn.NewTxn(context.Background(), ctx))
+ txn, err := ctx.Txn(true)
+ require.NoError(t, err)
+ defer func() {
+ err := txn.Rollback()
+ require.NoError(t, err)
+ }()
+
+ all := allTableData{
+ keys: make([][]byte, 0),
+ vals: make([][]byte, 0),
+ tp: make([]string, 0),
+ }
+ pid := physTable.GetPhysicalID()
+ prefix := tablecodec.EncodeTablePrefix(pid)
+ it, err := txn.Iter(prefix, nil)
+ require.NoError(t, err)
+ for it.Valid() {
+ if !it.Key().HasPrefix(prefix) {
+ break
+ }
+ all.keys = append(all.keys, it.Key())
+ all.vals = append(all.vals, it.Value())
+ if tablecodec.IsRecordKey(it.Key()) {
+ all.tp = append(all.tp, "Record")
+ tblID, kv, _ := tablecodec.DecodeRecordKey(it.Key())
+ require.Equal(t, pid, tblID)
+ vals, _ := tablecodec.DecodeValuesBytesToStrings(it.Value())
+ logutil.BgLogger().Info("Record",
+ zap.Int64("pid", tblID),
+ zap.Stringer("key", kv),
+ zap.Strings("values", vals))
+ } else if tablecodec.IsIndexKey(it.Key()) {
+ all.tp = append(all.tp, "Index")
+ } else {
+ all.tp = append(all.tp, "Other")
+ }
+ err = it.Next()
+ require.NoError(t, err)
+ }
+ return all
+}
+
+func TestReorganizeRangePartition(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("create database ReorgPartition")
+ tk.MustExec("use ReorgPartition")
+ tk.MustExec(`create table t (a int unsigned PRIMARY KEY, b varchar(255), c int, key (b), key (c,b)) partition by range (a) ` +
+ `(partition p0 values less than (10),` +
+ ` partition p1 values less than (20),` +
+ ` partition pMax values less than (MAXVALUE))`)
+ tk.MustExec(`insert into t values (1,"1",1), (12,"12",21),(23,"23",32),(34,"34",43),(45,"45",54),(56,"56",65)`)
+ tk.MustQuery(`select * from t where c < 40`).Sort().Check(testkit.Rows(""+
+ "1 1 1",
+ "12 12 21",
+ "23 23 32"))
+ tk.MustExec(`alter table t reorganize partition pMax into (partition p2 values less than (30), partition pMax values less than (MAXVALUE))`)
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p0` VALUES LESS THAN (10),\n" +
+ " PARTITION `p1` VALUES LESS THAN (20),\n" +
+ " PARTITION `p2` VALUES LESS THAN (30),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+ tk.MustQuery(`select * from t`).Sort().Check(testkit.Rows(""+
+ "1 1 1",
+ "12 12 21",
+ "23 23 32",
+ "34 34 43",
+ "45 45 54",
+ "56 56 65"))
+ tk.MustQuery(`select * from t partition (p0)`).Sort().Check(testkit.Rows("" +
+ "1 1 1"))
+ tk.MustQuery(`select * from t partition (p1)`).Sort().Check(testkit.Rows("" +
+ "12 12 21"))
+ tk.MustQuery(`select * from t partition (p2)`).Sort().Check(testkit.Rows("" +
+ "23 23 32"))
+ tk.MustQuery(`select * from t partition (pMax)`).Sort().Check(testkit.Rows(""+
+ "34 34 43",
+ "45 45 54",
+ "56 56 65"))
+ tk.MustQuery(`select * from t where b > "1"`).Sort().Check(testkit.Rows(""+
+ "12 12 21",
+ "23 23 32",
+ "34 34 43",
+ "45 45 54",
+ "56 56 65"))
+ tk.MustQuery(`select * from t where c < 40`).Sort().Check(testkit.Rows(""+
+ "1 1 1",
+ "12 12 21",
+ "23 23 32"))
+ tk.MustExec(`alter table t reorganize partition p2,pMax into (partition p2 values less than (35),partition p3 values less than (47), partition pMax values less than (MAXVALUE))`)
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`select * from t`).Sort().Check(testkit.Rows(""+
+ "1 1 1",
+ "12 12 21",
+ "23 23 32",
+ "34 34 43",
+ "45 45 54",
+ "56 56 65"))
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p0` VALUES LESS THAN (10),\n" +
+ " PARTITION `p1` VALUES LESS THAN (20),\n" +
+ " PARTITION `p2` VALUES LESS THAN (35),\n" +
+ " PARTITION `p3` VALUES LESS THAN (47),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+ tk.MustQuery(`select * from t partition (p0)`).Sort().Check(testkit.Rows("" +
+ "1 1 1"))
+ tk.MustQuery(`select * from t partition (p1)`).Sort().Check(testkit.Rows("" +
+ "12 12 21"))
+ tk.MustQuery(`select * from t partition (p2)`).Sort().Check(testkit.Rows(""+
+ "23 23 32",
+ "34 34 43"))
+ tk.MustQuery(`select * from t partition (p3)`).Sort().Check(testkit.Rows("" +
+ "45 45 54"))
+ tk.MustQuery(`select * from t partition (pMax)`).Sort().Check(testkit.Rows("" +
+ "56 56 65"))
+ tk.MustExec(`alter table t reorganize partition p0,p1 into (partition p1 values less than (20))`)
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p1` VALUES LESS THAN (20),\n" +
+ " PARTITION `p2` VALUES LESS THAN (35),\n" +
+ " PARTITION `p3` VALUES LESS THAN (47),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+ tk.MustQuery(`select * from t`).Sort().Check(testkit.Rows(""+
+ "1 1 1",
+ "12 12 21",
+ "23 23 32",
+ "34 34 43",
+ "45 45 54",
+ "56 56 65"))
+ tk.MustExec(`alter table t drop index b`)
+ tk.MustExec(`alter table t drop index c`)
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p1` VALUES LESS THAN (20),\n" +
+ " PARTITION `p2` VALUES LESS THAN (35),\n" +
+ " PARTITION `p3` VALUES LESS THAN (47),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+ tk.MustExec(`create table t2 (a int unsigned not null, b varchar(255), c int, key (b), key (c,b)) partition by range (a) ` +
+ "(PARTITION `p1` VALUES LESS THAN (20),\n" +
+ " PARTITION `p2` VALUES LESS THAN (35),\n" +
+ " PARTITION `p3` VALUES LESS THAN (47),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))")
+ tk.MustExec(`insert into t2 select * from t`)
+ // Not allowed to change the start range!
+ tk.MustGetErrCode(`alter table t2 reorganize partition p2 into (partition p2a values less than (20), partition p2b values less than (36))`,
+ errno.ErrRangeNotIncreasing)
+ // Not allowed to change the end range!
+ tk.MustGetErrCode(`alter table t2 reorganize partition p2 into (partition p2a values less than (30), partition p2b values less than (36))`, errno.ErrRangeNotIncreasing)
+ tk.MustGetErrCode(`alter table t2 reorganize partition p2 into (partition p2a values less than (30), partition p2b values less than (34))`, errno.ErrRangeNotIncreasing)
+ // Also not allowed to change from MAXVALUE to something else IF there are values in the removed range!
+ tk.MustContainErrMsg(`alter table t2 reorganize partition pMax into (partition p2b values less than (50))`, "[table:1526]Table has no partition for value 56")
+ tk.MustQuery(`show create table t2`).Check(testkit.Rows("" +
+ "t2 CREATE TABLE `t2` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p1` VALUES LESS THAN (20),\n" +
+ " PARTITION `p2` VALUES LESS THAN (35),\n" +
+ " PARTITION `p3` VALUES LESS THAN (47),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+ // But allowed to change from MAXVALUE if no existing values is outside the new range!
+ tk.MustExec(`alter table t2 reorganize partition pMax into (partition p4 values less than (90))`)
+ tk.MustExec(`admin check table t2`)
+ tk.MustQuery(`show create table t2`).Check(testkit.Rows("" +
+ "t2 CREATE TABLE `t2` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p1` VALUES LESS THAN (20),\n" +
+ " PARTITION `p2` VALUES LESS THAN (35),\n" +
+ " PARTITION `p3` VALUES LESS THAN (47),\n" +
+ " PARTITION `p4` VALUES LESS THAN (90))"))
+}
+
+func TestReorganizeListPartition(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("create database ReorgListPartition")
+ tk.MustExec("use ReorgListPartition")
+ tk.MustExec(`create table t (a int, b varchar(55), c int) partition by list (a)` +
+ ` (partition p1 values in (12,23,51,14), partition p2 values in (24,63), partition p3 values in (45))`)
+ tk.MustExec(`insert into t values (12,"12",21), (24,"24",42),(51,"51",15),(23,"23",32),(63,"63",36),(45,"45",54)`)
+ tk.MustExec(`alter table t reorganize partition p1 into (partition p0 values in (12,51,13), partition p1 values in (23))`)
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(11) DEFAULT NULL,\n" +
+ " `b` varchar(55) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY LIST (`a`)\n" +
+ "(PARTITION `p0` VALUES IN (12,51,13),\n" +
+ " PARTITION `p1` VALUES IN (23),\n" +
+ " PARTITION `p2` VALUES IN (24,63),\n" +
+ " PARTITION `p3` VALUES IN (45))"))
+ tk.MustExec(`alter table t add primary key (a), add key (b), add key (c,b)`)
+
+ // Note: MySQL cannot reorganize two non-consecutive list partitions :)
+ // ERROR 1519 (HY000): When reorganizing a set of partitions they must be in consecutive order
+ // https://bugs.mysql.com/bug.php?id=106011
+ // https://bugs.mysql.com/bug.php?id=109939
+ tk.MustExec(`alter table t reorganize partition p1, p3 into (partition pa values in (45,23,15))`)
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(11) NOT NULL,\n" +
+ " `b` varchar(55) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] NONCLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY LIST (`a`)\n" +
+ "(PARTITION `p0` VALUES IN (12,51,13),\n" +
+ " PARTITION `pa` VALUES IN (45,23,15),\n" +
+ " PARTITION `p2` VALUES IN (24,63))"))
+ tk.MustGetErrCode(`alter table t modify a varchar(20)`, errno.ErrUnsupportedDDLOperation)
+}
+
+type TestReorgDDLCallback struct {
+ *callback.TestDDLCallback
+ syncChan chan bool
+}
+
+func (tc *TestReorgDDLCallback) OnChanged(err error) error {
+ err = tc.TestDDLCallback.OnChanged(err)
+ <-tc.syncChan
+ // We want to wait here
+ <-tc.syncChan
+ return err
+}
+
+func TestReorgPartitionConcurrent(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ schemaName := "ReorgPartConcurrent"
+ tk.MustExec("create database " + schemaName)
+ tk.MustExec("use " + schemaName)
+ tk.MustExec(`create table t (a int unsigned PRIMARY KEY, b varchar(255), c int, key (b), key (c,b))` +
+ ` partition by range (a) ` +
+ `(partition p0 values less than (10),` +
+ ` partition p1 values less than (20),` +
+ ` partition pMax values less than (MAXVALUE))`)
+ tk.MustExec(`insert into t values (1,"1",1), (10,"10",10),(23,"23",32),(34,"34",43),(45,"45",54),(56,"56",65)`)
+ dom := domain.GetDomain(tk.Session())
+ originHook := dom.DDL().GetHook()
+ defer dom.DDL().SetHook(originHook)
+ syncOnChanged := make(chan bool)
+ defer close(syncOnChanged)
+ hook := &TestReorgDDLCallback{TestDDLCallback: &callback.TestDDLCallback{Do: dom}, syncChan: syncOnChanged}
+ dom.DDL().SetHook(hook)
+
+ wait := make(chan bool)
+ defer close(wait)
+
+ currState := model.StateNone
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.Type == model.ActionReorganizePartition &&
+ (job.SchemaState == model.StateDeleteOnly ||
+ job.SchemaState == model.StateWriteOnly ||
+ job.SchemaState == model.StateWriteReorganization ||
+ job.SchemaState == model.StateDeleteReorganization) &&
+ currState != job.SchemaState {
+ currState = job.SchemaState
+ <-wait
+ <-wait
+ }
+ }
+ alterErr := make(chan error, 1)
+ go backgroundExec(store, schemaName, "alter table t reorganize partition p1 into (partition p1a values less than (15), partition p1b values less than (20))", alterErr)
+
+ wait <- true
+ // StateDeleteOnly
+ deleteOnlyInfoSchema := sessiontxn.GetTxnManager(tk.Session()).GetTxnInfoSchema()
+ wait <- true
+
+ // StateWriteOnly
+ wait <- true
+ tk.MustExec(`insert into t values (11, "11", 11),(12,"12",21)`)
+ tk.MustExec(`admin check table t`)
+ writeOnlyInfoSchema := sessiontxn.GetTxnManager(tk.Session()).GetTxnInfoSchema()
+ require.Equal(t, int64(1), writeOnlyInfoSchema.SchemaMetaVersion()-deleteOnlyInfoSchema.SchemaMetaVersion())
+ deleteOnlyTbl, err := deleteOnlyInfoSchema.TableByName(model.NewCIStr(schemaName), model.NewCIStr("t"))
+ require.NoError(t, err)
+ writeOnlyTbl, err := writeOnlyInfoSchema.TableByName(model.NewCIStr(schemaName), model.NewCIStr("t"))
+ require.NoError(t, err)
+ writeOnlyParts := writeOnlyTbl.Meta().Partition
+ writeOnlyTbl.Meta().Partition = deleteOnlyTbl.Meta().Partition
+ // If not DeleteOnly is working, then this would show up when reorg is done
+ tk.MustExec(`delete from t where a = 11`)
+ tk.MustExec(`update t set b = "12b", c = 12 where a = 12`)
+ tk.MustExec(`admin check table t`)
+ writeOnlyTbl.Meta().Partition = writeOnlyParts
+ tk.MustExec(`admin check table t`)
+ wait <- true
+
+ // StateWriteReorganization
+ wait <- true
+ tk.MustExec(`insert into t values (14, "14", 14),(15, "15",15)`)
+ writeReorgInfoSchema := sessiontxn.GetTxnManager(tk.Session()).GetTxnInfoSchema()
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p0` VALUES LESS THAN (10),\n" +
+ " PARTITION `p1` VALUES LESS THAN (20),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+ wait <- true
+
+ // StateDeleteReorganization
+ wait <- true
+ tk.MustQuery(`select * from t where c between 10 and 22`).Sort().Check(testkit.Rows(""+
+ "10 10 10",
+ "12 12b 12",
+ "14 14 14",
+ "15 15 15"))
+ deleteReorgInfoSchema := sessiontxn.GetTxnManager(tk.Session()).GetTxnInfoSchema()
+ require.Equal(t, int64(1), deleteReorgInfoSchema.SchemaMetaVersion()-writeReorgInfoSchema.SchemaMetaVersion())
+ tk.MustExec(`insert into t values (16, "16", 16)`)
+ oldTbl, err := writeReorgInfoSchema.TableByName(model.NewCIStr(schemaName), model.NewCIStr("t"))
+ require.NoError(t, err)
+ partDef := oldTbl.Meta().Partition.Definitions[1]
+ require.Equal(t, "p1", partDef.Name.O)
+ rows := getNumRowsFromPartitionDefs(t, tk, oldTbl, oldTbl.Meta().Partition.Definitions[1:2])
+ require.Equal(t, 5, rows)
+ currTbl, err := deleteReorgInfoSchema.TableByName(model.NewCIStr(schemaName), model.NewCIStr("t"))
+ require.NoError(t, err)
+ currPart := currTbl.Meta().Partition
+ currTbl.Meta().Partition = oldTbl.Meta().Partition
+ tk.MustQuery(`select * from t where b = "16"`).Sort().Check(testkit.Rows("16 16 16"))
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p0` VALUES LESS THAN (10),\n" +
+ " PARTITION `p1` VALUES LESS THAN (20),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+ tk.MustQuery(`select * from t partition (p1)`).Sort().Check(testkit.Rows(""+
+ "10 10 10",
+ "12 12b 12",
+ "14 14 14",
+ "15 15 15",
+ "16 16 16"))
+ currTbl.Meta().Partition = currPart
+ wait <- true
+ syncOnChanged <- true
+ // This reads the new schema (Schema update completed)
+ tk.MustQuery(`select * from t where c between 10 and 22`).Sort().Check(testkit.Rows(""+
+ "10 10 10",
+ "12 12b 12",
+ "14 14 14",
+ "15 15 15",
+ "16 16 16"))
+ tk.MustExec(`admin check table t`)
+ newInfoSchema := sessiontxn.GetTxnManager(tk.Session()).GetTxnInfoSchema()
+ require.Equal(t, int64(1), newInfoSchema.SchemaMetaVersion()-deleteReorgInfoSchema.SchemaMetaVersion())
+ oldTbl, err = deleteReorgInfoSchema.TableByName(model.NewCIStr(schemaName), model.NewCIStr("t"))
+ require.NoError(t, err)
+ partDef = oldTbl.Meta().Partition.Definitions[1]
+ require.Equal(t, "p1a", partDef.Name.O)
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p0` VALUES LESS THAN (10),\n" +
+ " PARTITION `p1a` VALUES LESS THAN (15),\n" +
+ " PARTITION `p1b` VALUES LESS THAN (20),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+ newTbl, err := deleteReorgInfoSchema.TableByName(model.NewCIStr(schemaName), model.NewCIStr("t"))
+ require.NoError(t, err)
+ newPart := newTbl.Meta().Partition
+ newTbl.Meta().Partition = oldTbl.Meta().Partition
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p0` VALUES LESS THAN (10),\n" +
+ " PARTITION `p1a` VALUES LESS THAN (15),\n" +
+ " PARTITION `p1b` VALUES LESS THAN (20),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+ tk.MustExec(`admin check table t`)
+ newTbl.Meta().Partition = newPart
+ syncOnChanged <- true
+ require.NoError(t, <-alterErr)
+}
+
+func TestReorgPartitionFailConcurrent(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ schemaName := "ReorgPartFailConcurrent"
+ tk.MustExec("create database " + schemaName)
+ tk.MustExec("use " + schemaName)
+ tk.MustExec(`create table t (a int unsigned PRIMARY KEY, b varchar(255), c int, key (b), key (c,b))` +
+ ` partition by range (a) ` +
+ `(partition p0 values less than (10),` +
+ ` partition p1 values less than (20),` +
+ ` partition pMax values less than (MAXVALUE))`)
+ tk.MustExec(`insert into t values (1,"1",1), (12,"12",21),(23,"23",32),(34,"34",43),(45,"45",54),(56,"56",65)`)
+ dom := domain.GetDomain(tk.Session())
+ originHook := dom.DDL().GetHook()
+ defer dom.DDL().SetHook(originHook)
+ hook := &callback.TestDDLCallback{Do: dom}
+ dom.DDL().SetHook(hook)
+
+ wait := make(chan bool)
+ defer close(wait)
+
+ // Test insert of duplicate key during copy phase
+ injected := false
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.Type == model.ActionReorganizePartition && job.SchemaState == model.StateWriteReorganization && !injected {
+ injected = true
+ <-wait
+ <-wait
+ }
+ }
+ alterErr := make(chan error, 1)
+ go backgroundExec(store, schemaName, "alter table t reorganize partition p1 into (partition p1a values less than (15), partition p1b values less than (20))", alterErr)
+ wait <- true
+ tk.MustExec(`insert into t values (14, "14", 14),(15, "15",15)`)
+ tk.MustGetErrCode(`insert into t values (11, "11", 11),(12,"duplicate PK 💥", 13)`, errno.ErrDupEntry)
+ tk.MustExec(`admin check table t`)
+ wait <- true
+ require.NoError(t, <-alterErr)
+ tk.MustQuery(`select * from t where c between 10 and 22`).Sort().Check(testkit.Rows(""+
+ "12 12 21",
+ "14 14 14",
+ "15 15 15"))
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p0` VALUES LESS THAN (10),\n" +
+ " PARTITION `p1a` VALUES LESS THAN (15),\n" +
+ " PARTITION `p1b` VALUES LESS THAN (20),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+
+ // Test reorg of duplicate key
+ prevState := model.StateNone
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.Type == model.ActionReorganizePartition &&
+ job.SchemaState == model.StateWriteReorganization &&
+ job.SnapshotVer == 0 &&
+ prevState != job.SchemaState {
+ prevState = job.SchemaState
+ <-wait
+ <-wait
+ }
+ if job.Type == model.ActionReorganizePartition &&
+ job.SchemaState == model.StateDeleteReorganization &&
+ prevState != job.SchemaState {
+ prevState = job.SchemaState
+ <-wait
+ <-wait
+ }
+ }
+ go backgroundExec(store, schemaName, "alter table t reorganize partition p1a,p1b into (partition p1a values less than (14), partition p1b values less than (17), partition p1c values less than (20))", alterErr)
+ wait <- true
+ infoSchema := sessiontxn.GetTxnManager(tk.Session()).GetTxnInfoSchema()
+ tbl, err := infoSchema.TableByName(model.NewCIStr(schemaName), model.NewCIStr("t"))
+ require.NoError(t, err)
+ require.Equal(t, 0, getNumRowsFromPartitionDefs(t, tk, tbl, tbl.Meta().Partition.AddingDefinitions))
+ tk.MustExec(`delete from t where a = 14`)
+ tk.MustExec(`insert into t values (13, "13", 31),(14,"14b",14),(16, "16",16)`)
+ tk.MustExec(`admin check table t`)
+ wait <- true
+ wait <- true
+ tbl, err = infoSchema.TableByName(model.NewCIStr(schemaName), model.NewCIStr("t"))
+ require.NoError(t, err)
+ require.Equal(t, 5, getNumRowsFromPartitionDefs(t, tk, tbl, tbl.Meta().Partition.AddingDefinitions))
+ tk.MustExec(`delete from t where a = 15`)
+ tk.MustExec(`insert into t values (11, "11", 11),(15,"15b",15),(17, "17",17)`)
+ tk.MustExec(`admin check table t`)
+ wait <- true
+ require.NoError(t, <-alterErr)
+
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`select * from t where a between 10 and 22`).Sort().Check(testkit.Rows(""+
+ "11 11 11",
+ "12 12 21",
+ "13 13 31",
+ "14 14b 14",
+ "15 15b 15",
+ "16 16 16",
+ "17 17 17"))
+ tk.MustQuery(`select * from t where c between 10 and 22`).Sort().Check(testkit.Rows(""+
+ "11 11 11",
+ "12 12 21",
+ "14 14b 14",
+ "15 15b 15",
+ "16 16 16",
+ "17 17 17"))
+ tk.MustQuery(`select * from t where b between "10" and "22"`).Sort().Check(testkit.Rows(""+
+ "11 11 11",
+ "12 12 21",
+ "13 13 31",
+ "14 14b 14",
+ "15 15b 15",
+ "16 16 16",
+ "17 17 17"))
+}
+
+func getNumRowsFromPartitionDefs(t *testing.T, tk *testkit.TestKit, tbl table.Table, defs []model.PartitionDefinition) int {
+ ctx := tk.Session()
+ pt := tbl.GetPartitionedTable()
+ require.NotNil(t, pt)
+ cnt := 0
+ for _, def := range defs {
+ data := getAllDataForPhysicalTable(t, ctx, pt.GetPartition(def.ID))
+ require.True(t, len(data.keys) == len(data.vals))
+ require.True(t, len(data.keys) == len(data.tp))
+ for _, s := range data.tp {
+ if s == "Record" {
+ cnt++
+ }
+ }
+ }
+ return cnt
+}
+
+func TestReorgPartitionFailInject(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ schemaName := "ReorgPartFailInjectConcurrent"
+ tk.MustExec("create database " + schemaName)
+ tk.MustExec("use " + schemaName)
+ tk.MustExec(`create table t (a int unsigned PRIMARY KEY, b varchar(255), c int, key (b), key (c,b))` +
+ ` partition by range (a) ` +
+ `(partition p0 values less than (10),` +
+ ` partition p1 values less than (20),` +
+ ` partition pMax values less than (MAXVALUE))`)
+ tk.MustExec(`insert into t values (1,"1",1), (12,"12",21),(23,"23",32),(34,"34",43),(45,"45",54),(56,"56",65)`)
+
+ dom := domain.GetDomain(tk.Session())
+ originHook := dom.DDL().GetHook()
+ defer dom.DDL().SetHook(originHook)
+ hook := &callback.TestDDLCallback{Do: dom}
+ dom.DDL().SetHook(hook)
+
+ wait := make(chan bool)
+ defer close(wait)
+
+ injected := false
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.Type == model.ActionReorganizePartition && job.SchemaState == model.StateWriteReorganization && !injected {
+ injected = true
+ <-wait
+ <-wait
+ }
+ }
+ alterErr := make(chan error, 1)
+ go backgroundExec(store, schemaName, "alter table t reorganize partition p1 into (partition p1a values less than (15), partition p1b values less than (20))", alterErr)
+ wait <- true
+ tk.MustExec(`insert into t values (14, "14", 14),(15, "15",15)`)
+ tk.MustGetErrCode(`insert into t values (11, "11", 11),(12,"duplicate PK 💥", 13)`, errno.ErrDupEntry)
+ tk.MustExec(`admin check table t`)
+ wait <- true
+ require.NoError(t, <-alterErr)
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`select * from t where c between 10 and 22`).Sort().Check(testkit.Rows(""+
+ "12 12 21",
+ "14 14 14",
+ "15 15 15"))
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p0` VALUES LESS THAN (10),\n" +
+ " PARTITION `p1a` VALUES LESS THAN (15),\n" +
+ " PARTITION `p1b` VALUES LESS THAN (20),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+}
+
+func TestReorgPartitionRollback(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ schemaName := "ReorgPartRollback"
+ tk.MustExec("create database " + schemaName)
+ tk.MustExec("use " + schemaName)
+ tk.MustExec(`create table t (a int unsigned PRIMARY KEY, b varchar(255), c int, key (b), key (c,b))` +
+ ` partition by range (a) ` +
+ `(partition p0 values less than (10),` +
+ ` partition p1 values less than (20),` +
+ ` partition pMax values less than (MAXVALUE))`)
+ tk.MustExec(`insert into t values (1,"1",1), (12,"12",21),(23,"23",32),(34,"34",43),(45,"45",54),(56,"56",65)`)
+ // TODO: Check that there are no additional placement rules,
+ // bundles, or ranges with non-completed tableIDs
+ // (partitions used during reorg, but was dropped)
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockUpdateVersionAndTableInfoErr", `return(true)`))
+ tk.MustExecToErr("alter table t reorganize partition p1 into (partition p1a values less than (15), partition p1b values less than (20))")
+ tk.MustExec(`admin check table t`)
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockUpdateVersionAndTableInfoErr"))
+ ctx := tk.Session()
+ is := domain.GetDomain(ctx).InfoSchema()
+ tbl, err := is.TableByName(model.NewCIStr(schemaName), model.NewCIStr("t"))
+ require.NoError(t, err)
+ noNewTablesAfter(t, tk, ctx, tbl)
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/reorgPartitionAfterDataCopy", `return(true)`))
+ defer func() {
+ err := failpoint.Disable("github.com/pingcap/tidb/ddl/reorgPartitionAfterDataCopy")
+ require.NoError(t, err)
+ }()
+ tk.MustExecToErr("alter table t reorganize partition p1 into (partition p1a values less than (15), partition p1b values less than (20))")
+ tk.MustExec(`admin check table t`)
+ tk.MustQuery(`show create table t`).Check(testkit.Rows("" +
+ "t CREATE TABLE `t` (\n" +
+ " `a` int(10) unsigned NOT NULL,\n" +
+ " `b` varchar(255) DEFAULT NULL,\n" +
+ " `c` int(11) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */,\n" +
+ " KEY `b` (`b`),\n" +
+ " KEY `c` (`c`,`b`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
+ "PARTITION BY RANGE (`a`)\n" +
+ "(PARTITION `p0` VALUES LESS THAN (10),\n" +
+ " PARTITION `p1` VALUES LESS THAN (20),\n" +
+ " PARTITION `pMax` VALUES LESS THAN (MAXVALUE))"))
+
+ tbl, err = is.TableByName(model.NewCIStr(schemaName), model.NewCIStr("t"))
+ require.NoError(t, err)
+ noNewTablesAfter(t, tk, ctx, tbl)
+}
+
+func TestReorgPartitionData(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ schemaName := "ReorgPartData"
+ tk.MustExec("create database " + schemaName)
+ tk.MustExec("use " + schemaName)
+ tk.MustExec(`SET @@session.sql_mode = default`)
+ tk.MustExec(`create table t (a int PRIMARY KEY AUTO_INCREMENT, b varchar(255), c int, d datetime, key (b), key (c,b)) partition by range (a) (partition p1 values less than (0), partition p1M values less than (1000000))`)
+ tk.MustContainErrMsg(`insert into t values (0, "Zero value!", 0, '2022-02-30')`, "[table:1292]Incorrect datetime value: '2022-02-30' for column 'd' at row 1")
+ tk.MustExec(`SET @@session.sql_mode = 'ALLOW_INVALID_DATES,NO_AUTO_VALUE_ON_ZERO'`)
+ tk.MustExec(`insert into t values (0, "Zero value!", 0, '2022-02-30')`)
+ tk.MustQuery(`show warnings`).Check(testkit.Rows())
+ tk.MustQuery(`select * from t`).Sort().Check(testkit.Rows("0 Zero value! 0 2022-02-30 00:00:00"))
+ tk.MustExec(`SET @@session.sql_mode = default`)
+ tk.MustExec(`alter table t reorganize partition p1M into (partition p0 values less than (1), partition p2M values less than (2000000))`)
+ tk.MustQuery(`select * from t`).Sort().Check(testkit.Rows("0 Zero value! 0 2022-02-30 00:00:00"))
+ tk.MustExec(`admin check table t`)
+}
diff --git a/ddl/repair_table_test.go b/ddl/repair_table_test.go
index 6881c6ce5f019..8b16f9bfbc69d 100644
--- a/ddl/repair_table_test.go
+++ b/ddl/repair_table_test.go
@@ -20,7 +20,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
- "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
@@ -101,7 +101,7 @@ func TestRepairTable(t *testing.T) {
// Repaired tableInfo has been filtered by `domain.InfoSchema()`, so get it in repairInfo.
originTableInfo, _ := domainutil.RepairInfo.GetRepairedTableInfoByTableName("test", "origin")
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var repairErr error
hook.OnJobRunBeforeExported = func(job *model.Job) {
if job.Type != model.ActionRepairTable {
diff --git a/ddl/resource_group.go b/ddl/resource_group.go
new file mode 100644
index 0000000000000..514e8cf30c2b8
--- /dev/null
+++ b/ddl/resource_group.go
@@ -0,0 +1,157 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "context"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/tidb/ddl/resourcegroup"
+ "github.com/pingcap/tidb/domain/infosync"
+ "github.com/pingcap/tidb/infoschema"
+ "github.com/pingcap/tidb/meta"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/util/dbterror"
+ "github.com/pingcap/tidb/util/logutil"
+ "go.uber.org/zap"
+)
+
+func onCreateResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ groupInfo := &model.ResourceGroupInfo{}
+ if err := job.DecodeArgs(groupInfo); err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ groupInfo.State = model.StateNone
+
+ // check if resource group value is valid and convert to proto format.
+ protoGroup, err := resourcegroup.NewGroupFromOptions(groupInfo.Name.L, groupInfo.ResourceGroupSettings)
+ if err != nil {
+ logutil.BgLogger().Warn("convert to resource group failed", zap.Error(err))
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+
+ switch groupInfo.State {
+ case model.StateNone:
+ // none -> public
+ groupInfo.State = model.StatePublic
+ err := t.AddResourceGroup(groupInfo)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ err = infosync.AddResourceGroup(context.TODO(), protoGroup)
+ if err != nil {
+ logutil.BgLogger().Warn("create resource group failed", zap.Error(err))
+ return ver, errors.Trace(err)
+ }
+ job.SchemaID = groupInfo.ID
+ ver, err = updateSchemaVersion(d, t, job)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ // Finish this job.
+ job.FinishDBJob(model.JobStateDone, model.StatePublic, ver, nil)
+ return ver, nil
+ default:
+ return ver, dbterror.ErrInvalidDDLState.GenWithStackByArgs("resource_group", groupInfo.State)
+ }
+}
+
+func onAlterResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ alterGroupInfo := &model.ResourceGroupInfo{}
+ if err := job.DecodeArgs(alterGroupInfo); err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ // check if resource group value is valid and convert to proto format.
+ protoGroup, err := resourcegroup.NewGroupFromOptions(alterGroupInfo.Name.L, alterGroupInfo.ResourceGroupSettings)
+ if err != nil {
+ logutil.BgLogger().Warn("convert to resource group failed", zap.Error(err))
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+
+ oldGroup, err := checkResourceGroupExist(t, job, alterGroupInfo.ID)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ newGroup := *oldGroup
+ newGroup.ResourceGroupSettings = alterGroupInfo.ResourceGroupSettings
+
+ // TODO: check the group validation
+ err = t.UpdateResourceGroup(&newGroup)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ err = infosync.ModifyResourceGroup(context.TODO(), protoGroup)
+ if err != nil {
+ logutil.BgLogger().Warn("update resource group failed", zap.Error(err))
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+
+ ver, err = updateSchemaVersion(d, t, job)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ // Finish this job.
+ job.FinishDBJob(model.JobStateDone, model.StatePublic, ver, nil)
+ return ver, nil
+}
+
+func checkResourceGroupExist(t *meta.Meta, job *model.Job, groupID int64) (*model.ResourceGroupInfo, error) {
+ groupInfo, err := t.GetResourceGroup(groupID)
+ if err == nil {
+ return groupInfo, nil
+ }
+ if infoschema.ErrResourceGroupNotExists.Equal(err) {
+ job.State = model.JobStateCancelled
+ }
+ return nil, err
+}
+
+func onDropResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ groupInfo, err := checkResourceGroupExist(t, job, job.SchemaID)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ // TODO: check the resource group not in use.
+ switch groupInfo.State {
+ case model.StatePublic:
+ // public -> none
+ // resource group not influence the correctness of the data, so we can directly remove it.
+ groupInfo.State = model.StateNone
+ err = t.DropResourceGroup(groupInfo.ID)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ err = infosync.DeleteResourceGroup(context.TODO(), groupInfo.Name.L)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ ver, err = updateSchemaVersion(d, t, job)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ // Finish this job.
+ job.FinishDBJob(model.JobStateDone, model.StateNone, ver, nil)
+ default:
+ err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("resource_group", groupInfo.State)
+ }
+ return ver, errors.Trace(err)
+}
diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go
new file mode 100644
index 0000000000000..162a3ac233a05
--- /dev/null
+++ b/ddl/resource_group_test.go
@@ -0,0 +1,178 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl_test
+
+import (
+ "context"
+ "testing"
+
+ "github.com/pingcap/tidb/ddl/internal/callback"
+ "github.com/pingcap/tidb/domain"
+ "github.com/pingcap/tidb/domain/infosync"
+ mysql "github.com/pingcap/tidb/errno"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/stretchr/testify/require"
+)
+
+func TestResourceGroupBasic(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ re := require.New(t)
+
+ hook := &callback.TestDDLCallback{Do: dom}
+ var groupID int64
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ // job.SchemaID will be assigned when the group is created.
+ if (job.SchemaName == "x" || job.SchemaName == "y") && job.Type == model.ActionCreateResourceGroup && job.SchemaID != 0 {
+ groupID = job.SchemaID
+ return
+ }
+ }
+ hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ dom.DDL().SetHook(hook)
+
+ tk.MustExec("set global tidb_enable_resource_control = 'off'")
+ tk.MustGetErrCode("create user usr1 resource group rg1", mysql.ErrResourceGroupSupportDisabled)
+ tk.MustExec("create user usr1")
+ tk.MustGetErrCode("alter user usr1 resource group rg1", mysql.ErrResourceGroupSupportDisabled)
+ tk.MustGetErrCode("create resource group x RU_PER_SEC=1000 ", mysql.ErrResourceGroupSupportDisabled)
+
+ tk.MustExec("set global tidb_enable_resource_control = 'on'")
+
+ tk.MustExec("create resource group x RU_PER_SEC=1000")
+ checkFunc := func(groupInfo *model.ResourceGroupInfo) {
+ require.Equal(t, true, groupInfo.ID != 0)
+ require.Equal(t, "x", groupInfo.Name.L)
+ require.Equal(t, groupID, groupInfo.ID)
+ require.Equal(t, uint64(1000), groupInfo.RURate)
+ }
+ // Check the group is correctly reloaded in the information schema.
+ g := testResourceGroupNameFromIS(t, tk.Session(), "x")
+ checkFunc(g)
+
+ // test create if not exists
+ tk.MustExec("create resource group if not exists x RU_PER_SEC=10000")
+ // Check the resource group is not changed
+ g = testResourceGroupNameFromIS(t, tk.Session(), "x")
+ checkFunc(g)
+ // Check warning message
+ res := tk.MustQuery("show warnings")
+ res.Check(testkit.Rows("Note 8248 Resource group 'x' already exists"))
+
+ tk.MustExec("set global tidb_enable_resource_control = DEFAULT")
+ tk.MustGetErrCode("alter resource group x RU_PER_SEC=2000 ", mysql.ErrResourceGroupSupportDisabled)
+ tk.MustGetErrCode("drop resource group x ", mysql.ErrResourceGroupSupportDisabled)
+
+ tk.MustExec("set global tidb_enable_resource_control = 'on'")
+
+ tk.MustGetErrCode("create resource group x RU_PER_SEC=1000 ", mysql.ErrResourceGroupExists)
+
+ tk.MustExec("alter resource group x RU_PER_SEC=2000 BURSTABLE")
+ g = testResourceGroupNameFromIS(t, tk.Session(), "x")
+ re.Equal(uint64(2000), g.RURate)
+ re.Equal(int64(-1), g.BurstLimit)
+
+ tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x 2000 YES"))
+
+ tk.MustExec("drop resource group x")
+ g = testResourceGroupNameFromIS(t, tk.Session(), "x")
+ re.Nil(g)
+
+ tk.MustExec("alter resource group if exists not_exists RU_PER_SEC=2000")
+ // Check warning message
+ res = tk.MustQuery("show warnings")
+ res.Check(testkit.Rows("Note 8249 Unknown resource group 'not_exists'"))
+
+ tk.MustExec("create resource group y RU_PER_SEC=4000")
+ checkFunc = func(groupInfo *model.ResourceGroupInfo) {
+ require.Equal(t, true, groupInfo.ID != 0)
+ require.Equal(t, "y", groupInfo.Name.L)
+ require.Equal(t, groupID, groupInfo.ID)
+ require.Equal(t, uint64(4000), groupInfo.RURate)
+ require.Equal(t, int64(4000), groupInfo.BurstLimit)
+ }
+ g = testResourceGroupNameFromIS(t, tk.Session(), "y")
+ checkFunc(g)
+ tk.MustExec("alter resource group y BURSTABLE RU_PER_SEC=5000")
+ checkFunc = func(groupInfo *model.ResourceGroupInfo) {
+ require.Equal(t, true, groupInfo.ID != 0)
+ require.Equal(t, "y", groupInfo.Name.L)
+ require.Equal(t, groupID, groupInfo.ID)
+ require.Equal(t, uint64(5000), groupInfo.RURate)
+ require.Equal(t, int64(-1), groupInfo.BurstLimit)
+ }
+ g = testResourceGroupNameFromIS(t, tk.Session(), "y")
+ checkFunc(g)
+ tk.MustExec("drop resource group y")
+ g = testResourceGroupNameFromIS(t, tk.Session(), "y")
+ re.Nil(g)
+
+ tk.MustGetErrCode("create resource group x ru_per_sec=1000 ru_per_sec=200", mysql.ErrParse)
+ tk.MustContainErrMsg("create resource group x ru_per_sec=1000 ru_per_sec=200, ru_per_sec=300", "Dupliated options specified")
+ tk.MustGetErrCode("create resource group x burstable, burstable", mysql.ErrParse)
+ tk.MustContainErrMsg("create resource group x burstable, burstable", "Dupliated options specified")
+ tk.MustGetErrCode("create resource group x ru_per_sec=1000, burstable, burstable", mysql.ErrParse)
+ tk.MustContainErrMsg("create resource group x ru_per_sec=1000, burstable, burstable", "Dupliated options specified")
+ tk.MustGetErrCode("create resource group x burstable, ru_per_sec=1000, burstable", mysql.ErrParse)
+ tk.MustContainErrMsg("create resource group x burstable, ru_per_sec=1000, burstable", "Dupliated options specified")
+ groups, err := infosync.ListResourceGroups(context.TODO())
+ require.Equal(t, 0, len(groups))
+ require.NoError(t, err)
+
+ // Check information schema table information_schema.resource_groups
+ tk.MustExec("create resource group x RU_PER_SEC=1000")
+ tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x 1000 NO"))
+ tk.MustExec("alter resource group x RU_PER_SEC=2000 BURSTABLE")
+ tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x 2000 YES"))
+ tk.MustExec("alter resource group x BURSTABLE RU_PER_SEC=3000")
+ tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x 3000 YES"))
+ tk.MustQuery("show create resource group x").Check(testkit.Rows("x CREATE RESOURCE GROUP `x` RU_PER_SEC=3000 BURSTABLE"))
+
+ tk.MustExec("create resource group y BURSTABLE RU_PER_SEC=2000")
+ tk.MustQuery("select * from information_schema.resource_groups where name = 'y'").Check(testkit.Rows("y 2000 YES"))
+ tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RU_PER_SEC=2000 BURSTABLE"))
+
+ tk.MustExec("alter resource group y RU_PER_SEC=4000 BURSTABLE")
+ tk.MustQuery("select * from information_schema.resource_groups where name = 'y'").Check(testkit.Rows("y 4000 YES"))
+ tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RU_PER_SEC=4000 BURSTABLE"))
+
+ tk.MustQuery("select count(*) from information_schema.resource_groups").Check(testkit.Rows("2"))
+ tk.MustGetErrCode("create user usr_fail resource group nil_group", mysql.ErrResourceGroupNotExists)
+ tk.MustContainErrMsg("create user usr_fail resource group nil_group", "Unknown resource group 'nil_group'")
+ tk.MustExec("create user user2")
+ tk.MustGetErrCode("alter user user2 resource group nil_group", mysql.ErrResourceGroupNotExists)
+ tk.MustContainErrMsg("alter user user2 resource group nil_group", "Unknown resource group 'nil_group'")
+
+ tk.MustExec("create resource group do_not_delete_rg ru_per_sec=100")
+ tk.MustExec("create user usr3 resource group do_not_delete_rg")
+ tk.MustQuery("select user_attributes from mysql.user where user = 'usr3'").Check(testkit.Rows(`{"resource_group": "do_not_delete_rg"}`))
+ tk.MustContainErrMsg("drop resource group do_not_delete_rg", "user [usr3] depends on the resource group to drop")
+ tk.MustExec("alter user usr3 resource group `default`")
+ tk.MustExec("alter user usr3 resource group ``")
+ tk.MustExec("alter user usr3 resource group `DeFault`")
+ tk.MustQuery("select user_attributes from mysql.user where user = 'usr3'").Check(testkit.Rows(`{"resource_group": "default"}`))
+}
+
+func testResourceGroupNameFromIS(t *testing.T, ctx sessionctx.Context, name string) *model.ResourceGroupInfo {
+ dom := domain.GetDomain(ctx)
+ // Make sure the table schema is the new schema.
+ err := dom.Reload()
+ require.NoError(t, err)
+ g, _ := dom.InfoSchema().ResourceGroupByName(model.NewCIStr(name))
+ return g
+}
diff --git a/ddl/resourcegroup/BUILD.bazel b/ddl/resourcegroup/BUILD.bazel
new file mode 100644
index 0000000000000..ae19c6f654e86
--- /dev/null
+++ b/ddl/resourcegroup/BUILD.bazel
@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "resourcegroup",
+ srcs = [
+ "errors.go",
+ "group.go",
+ ],
+ importpath = "github.com/pingcap/tidb/ddl/resourcegroup",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//parser/model",
+ "@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_kvproto//pkg/resource_manager",
+ ],
+)
+
+go_test(
+ name = "resourcegroup_test",
+ timeout = "short",
+ srcs = ["group_test.go"],
+ embed = [":resourcegroup"],
+ flaky = True,
+ deps = [
+ "//parser/model",
+ "@com_github_pingcap_kvproto//pkg/resource_manager",
+ "@com_github_stretchr_testify//require",
+ ],
+)
diff --git a/ddl/resourcegroup/errors.go b/ddl/resourcegroup/errors.go
new file mode 100644
index 0000000000000..0893f59876f6d
--- /dev/null
+++ b/ddl/resourcegroup/errors.go
@@ -0,0 +1,32 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resourcegroup
+
+import (
+ "github.com/pingcap/errors"
+)
+
+var (
+ // ErrInvalidGroupSettings is from group.go.
+ ErrInvalidGroupSettings = errors.New("invalid group settings")
+ // ErrTooLongResourceGroupName is from group.go.
+ ErrTooLongResourceGroupName = errors.New("resource group name too long")
+ // ErrInvalidResourceGroupFormat is from group.go.
+ ErrInvalidResourceGroupFormat = errors.New("group settings with invalid format")
+ // ErrInvalidResourceGroupDuplicatedMode is from group.go.
+ ErrInvalidResourceGroupDuplicatedMode = errors.New("cannot set RU mode and Raw mode options at the same time")
+ // ErrUnknownResourceGroupMode is from group.go.
+ ErrUnknownResourceGroupMode = errors.New("unknown resource group mode")
+)
diff --git a/ddl/resourcegroup/group.go b/ddl/resourcegroup/group.go
new file mode 100644
index 0000000000000..6e371a501d3ab
--- /dev/null
+++ b/ddl/resourcegroup/group.go
@@ -0,0 +1,53 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resourcegroup
+
+import (
+ rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
+ "github.com/pingcap/tidb/parser/model"
+)
+
+// MaxGroupNameLength is max length of the name of a resource group
+const MaxGroupNameLength = 32
+
+// NewGroupFromOptions creates a new resource group from the given options.
+func NewGroupFromOptions(groupName string, options *model.ResourceGroupSettings) (*rmpb.ResourceGroup, error) {
+ if options == nil {
+ return nil, ErrInvalidGroupSettings
+ }
+ if len(groupName) > MaxGroupNameLength {
+ return nil, ErrTooLongResourceGroupName
+ }
+ group := &rmpb.ResourceGroup{
+ Name: groupName,
+ }
+ if options.RURate > 0 {
+ group.Mode = rmpb.GroupMode_RUMode
+ group.RUSettings = &rmpb.GroupRequestUnitSettings{
+ RU: &rmpb.TokenBucket{
+ Settings: &rmpb.TokenLimitSettings{
+ FillRate: options.RURate,
+ BurstLimit: options.BurstLimit,
+ },
+ },
+ }
+ if len(options.CPULimiter) > 0 || len(options.IOReadBandwidth) > 0 || len(options.IOWriteBandwidth) > 0 {
+ return nil, ErrInvalidResourceGroupDuplicatedMode
+ }
+ return group, nil
+ }
+ // Only support RU mode now
+ return nil, ErrUnknownResourceGroupMode
+}
diff --git a/ddl/resourcegroup/group_test.go b/ddl/resourcegroup/group_test.go
new file mode 100644
index 0000000000000..7d79f00aa6184
--- /dev/null
+++ b/ddl/resourcegroup/group_test.go
@@ -0,0 +1,143 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resourcegroup
+
+import (
+ "fmt"
+ "testing"
+
+ rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNewResourceGroupFromOptions(t *testing.T) {
+ type TestCase struct {
+ name string
+ groupName string
+ input *model.ResourceGroupSettings
+ output *rmpb.ResourceGroup
+ err error
+ }
+ var tests []TestCase
+ groupName := "test"
+ tests = append(tests, TestCase{
+ name: "empty 1",
+ input: &model.ResourceGroupSettings{},
+ err: ErrUnknownResourceGroupMode,
+ })
+
+ tests = append(tests, TestCase{
+ name: "empty 2",
+ input: nil,
+ err: ErrInvalidGroupSettings,
+ })
+
+ tests = append(tests, TestCase{
+ name: "normal case: ru case 1",
+ input: &model.ResourceGroupSettings{
+ RURate: 2000,
+ },
+ output: &rmpb.ResourceGroup{
+ Name: groupName,
+ Mode: rmpb.GroupMode_RUMode,
+ RUSettings: &rmpb.GroupRequestUnitSettings{
+ RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 2000}},
+ },
+ },
+ })
+
+ tests = append(tests, TestCase{
+ name: "normal case: ru case 2",
+ input: &model.ResourceGroupSettings{
+ RURate: 5000,
+ },
+ output: &rmpb.ResourceGroup{
+ Name: groupName,
+ Mode: rmpb.GroupMode_RUMode,
+ RUSettings: &rmpb.GroupRequestUnitSettings{
+ RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 5000}},
+ },
+ },
+ })
+
+ tests = append(tests, TestCase{
+ name: "error case: native case 1",
+ input: &model.ResourceGroupSettings{
+ CPULimiter: "8",
+ IOReadBandwidth: "3000MB/s",
+ IOWriteBandwidth: "3000Mi",
+ },
+ err: ErrUnknownResourceGroupMode,
+ })
+
+ tests = append(tests, TestCase{
+ name: "error case: native case 2",
+ input: &model.ResourceGroupSettings{
+ CPULimiter: "8c",
+ IOReadBandwidth: "3000Mi",
+ IOWriteBandwidth: "3000Mi",
+ },
+ err: ErrUnknownResourceGroupMode,
+ })
+
+ tests = append(tests, TestCase{
+ name: "error case: native case 3",
+ input: &model.ResourceGroupSettings{
+ CPULimiter: "8",
+ IOReadBandwidth: "3000G",
+ IOWriteBandwidth: "3000MB",
+ },
+ err: ErrUnknownResourceGroupMode,
+ })
+
+ tests = append(tests, TestCase{
+ name: "error case: duplicated mode",
+ input: &model.ResourceGroupSettings{
+ CPULimiter: "8",
+ IOReadBandwidth: "3000Mi",
+ IOWriteBandwidth: "3000Mi",
+ RURate: 1000,
+ },
+ err: ErrInvalidResourceGroupDuplicatedMode,
+ })
+
+ tests = append(tests, TestCase{
+ name: "error case: duplicated mode",
+ groupName: "test_group_too_looooooooooooooooooooooooooooooooooooooooooooooooong",
+ input: &model.ResourceGroupSettings{
+ CPULimiter: "8",
+ IOReadBandwidth: "3000Mi",
+ IOWriteBandwidth: "3000Mi",
+ RURate: 1000,
+ },
+ err: ErrTooLongResourceGroupName,
+ })
+
+ for _, test := range tests {
+ name := groupName
+ if len(test.groupName) > 0 {
+ name = test.groupName
+ }
+ group, err := NewGroupFromOptions(name, test.input)
+ comment := fmt.Sprintf("[%s]\nerr1 %s\nerr2 %s", test.name, err, test.err)
+ if test.err != nil {
+ require.ErrorIs(t, err, test.err, comment)
+ } else {
+ require.NoError(t, err, comment)
+ require.Equal(t, test.output, group)
+ }
+ }
+}
diff --git a/ddl/restart_test.go b/ddl/restart_test.go
index 450624f7dfe97..fa4e21b5f1c05 100644
--- a/ddl/restart_test.go
+++ b/ddl/restart_test.go
@@ -141,7 +141,7 @@ func TestStat(t *testing.T) {
SchemaID: dbInfo.ID,
Type: model.ActionDropSchema,
BinlogInfo: &model.HistoryInfo{},
- Args: []interface{}{dbInfo.Name},
+ Args: []interface{}{true},
}
done := make(chan error, 1)
diff --git a/ddl/rollingback_test.go b/ddl/rollingback_test.go
index f1850eb80dcbc..ee5894441175f 100644
--- a/ddl/rollingback_test.go
+++ b/ddl/rollingback_test.go
@@ -21,6 +21,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/testkit/external"
@@ -50,7 +51,7 @@ func TestCancelAddIndexJobError(t *testing.T) {
require.NotNil(t, tbl)
d := dom.DDL()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var (
checkErr error
jobID int64
diff --git a/ddl/sanity_check.go b/ddl/sanity_check.go
index e005eee6a9856..63eb94d4592a4 100644
--- a/ddl/sanity_check.go
+++ b/ddl/sanity_check.go
@@ -16,7 +16,6 @@ package ddl
import (
"context"
- "flag"
"fmt"
"strings"
@@ -26,6 +25,7 @@ import (
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/util/intest"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/mathutil"
"github.com/pingcap/tidb/util/sqlexec"
@@ -98,7 +98,8 @@ func expectedDeleteRangeCnt(ctx delRangeCntCtx, job *model.Job) (int, error) {
return 0, errors.Trace(err)
}
return mathutil.Max(len(physicalTableIDs), 1), nil
- case model.ActionDropTablePartition, model.ActionTruncateTablePartition:
+ case model.ActionDropTablePartition, model.ActionTruncateTablePartition,
+ model.ActionReorganizePartition:
var physicalTableIDs []int64
if err := job.DecodeArgs(&physicalTableIDs); err != nil {
return 0, errors.Trace(err)
@@ -182,7 +183,7 @@ func (ctx *delRangeCntCtx) deduplicateIdxCnt(indexIDs []int64) int {
// It's only check during the test environment, so it would panic directly.
// These checks may be controlled by configuration in the future.
func (d *ddl) checkHistoryJobInTest(ctx sessionctx.Context, historyJob *model.Job) {
- if !(flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil) {
+ if !intest.InTest {
return
}
diff --git a/ddl/schema.go b/ddl/schema.go
index d9e86c30c5eaf..e9cb1e6579635 100644
--- a/ddl/schema.go
+++ b/ddl/schema.go
@@ -312,6 +312,10 @@ func (w *worker) onRecoverSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver i
return ver, errors.Trace(err)
}
for _, recoverInfo := range recoverSchemaInfo.RecoverTabsInfo {
+ if recoverInfo.TableInfo.TTLInfo != nil {
+ // force disable TTL job schedule for recovered table
+ recoverInfo.TableInfo.TTLInfo.Enable = false
+ }
ver, err = w.recoverTable(t, job, recoverInfo)
if err != nil {
return ver, errors.Trace(err)
diff --git a/ddl/schema_test.go b/ddl/schema_test.go
index 70206ed2f179f..3be4fb4e4d278 100644
--- a/ddl/schema_test.go
+++ b/ddl/schema_test.go
@@ -163,13 +163,13 @@ func testDropSchema(t *testing.T, ctx sessionctx.Context, d ddl.DDL, dbInfo *mod
return job, ver
}
-func isDDLJobDone(test *testing.T, t *meta.Meta) bool {
- job, err := t.GetDDLJobByIdx(0)
- require.NoError(test, err)
- if job == nil {
+func isDDLJobDone(test *testing.T, t *meta.Meta, store kv.Storage) bool {
+ tk := testkit.NewTestKit(test, store)
+ rows := tk.MustQuery("select * from mysql.tidb_ddl_job").Rows()
+
+ if len(rows) == 0 {
return true
}
-
time.Sleep(testLease)
return false
}
@@ -185,7 +185,7 @@ func testCheckSchemaState(test *testing.T, store kv.Storage, dbInfo *model.DBInf
require.NoError(test, err)
if state == model.StateNone {
- isDropped = isDDLJobDone(test, t)
+ isDropped = isDDLJobDone(test, t, store)
if !isDropped {
return nil
}
diff --git a/ddl/schematracker/checker.go b/ddl/schematracker/checker.go
index b1533d0246fb1..9477d478388ad 100644
--- a/ddl/schematracker/checker.go
+++ b/ddl/schematracker/checker.go
@@ -433,6 +433,22 @@ func (d Checker) AlterPlacementPolicy(ctx sessionctx.Context, stmt *ast.AlterPla
panic("implement me")
}
+// AddResourceGroup implements the DDL interface.
+// ResourceGroup do not affect the transaction.
+func (d Checker) AddResourceGroup(ctx sessionctx.Context, stmt *ast.CreateResourceGroupStmt) error {
+ return nil
+}
+
+// DropResourceGroup implements the DDL interface.
+func (d Checker) DropResourceGroup(ctx sessionctx.Context, stmt *ast.DropResourceGroupStmt) error {
+ return nil
+}
+
+// AlterResourceGroup implements the DDL interface.
+func (d Checker) AlterResourceGroup(ctx sessionctx.Context, stmt *ast.AlterResourceGroupStmt) error {
+ return nil
+}
+
// CreateSchemaWithInfo implements the DDL interface.
func (d Checker) CreateSchemaWithInfo(ctx sessionctx.Context, info *model.DBInfo, onExist ddl.OnExist) error {
err := d.realDDL.CreateSchemaWithInfo(ctx, info, onExist)
@@ -541,16 +557,6 @@ func (d Checker) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
return d.realDDL.DoDDLJob(ctx, job)
}
-// MoveJobFromQueue2Table implements the DDL interface.
-func (d Checker) MoveJobFromQueue2Table(bool) error {
- panic("implement me")
-}
-
-// MoveJobFromTable2Queue implements the DDL interface.
-func (d Checker) MoveJobFromTable2Queue() error {
- panic("implement me")
-}
-
// StorageDDLInjector wraps kv.Storage to inject checker to domain's DDL in bootstrap time.
type StorageDDLInjector struct {
kv.Storage
diff --git a/ddl/schematracker/dm_tracker.go b/ddl/schematracker/dm_tracker.go
index 75f8fa35b429d..323d3b04962f7 100644
--- a/ddl/schematracker/dm_tracker.go
+++ b/ddl/schematracker/dm_tracker.go
@@ -1173,6 +1173,21 @@ func (SchemaTracker) AlterPlacementPolicy(ctx sessionctx.Context, stmt *ast.Alte
return nil
}
+// AddResourceGroup implements the DDL interface, it's no-op in DM's case.
+func (SchemaTracker) AddResourceGroup(_ sessionctx.Context, _ *ast.CreateResourceGroupStmt) error {
+ return nil
+}
+
+// DropResourceGroup implements the DDL interface, it's no-op in DM's case.
+func (SchemaTracker) DropResourceGroup(_ sessionctx.Context, _ *ast.DropResourceGroupStmt) error {
+ return nil
+}
+
+// AlterResourceGroup implements the DDL interface, it's no-op in DM's case.
+func (SchemaTracker) AlterResourceGroup(ctx sessionctx.Context, stmt *ast.AlterResourceGroupStmt) error {
+ return nil
+}
+
// BatchCreateTableWithInfo implements the DDL interface, it will call CreateTableWithInfo for each table.
func (d SchemaTracker) BatchCreateTableWithInfo(ctx sessionctx.Context, schema model.CIStr, info []*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error {
for _, tableInfo := range info {
@@ -1256,13 +1271,3 @@ func (SchemaTracker) GetInfoSchemaWithInterceptor(ctx sessionctx.Context) infosc
func (SchemaTracker) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
return nil
}
-
-// MoveJobFromQueue2Table implements the DDL interface, it's no-op in DM's case.
-func (SchemaTracker) MoveJobFromQueue2Table(b bool) error {
- panic("implement me")
-}
-
-// MoveJobFromTable2Queue implements the DDL interface, it's no-op in DM's case.
-func (SchemaTracker) MoveJobFromTable2Queue() error {
- panic("implement me")
-}
diff --git a/ddl/sequence_test.go b/ddl/sequence_test.go
index df58df12b0ebd..9b798c9f45eea 100644
--- a/ddl/sequence_test.go
+++ b/ddl/sequence_test.go
@@ -62,8 +62,7 @@ func TestCreateSequence(t *testing.T) {
// test unsupported table option in sequence.
tk.MustGetErrCode("create sequence seq CHARSET=utf8", mysql.ErrSequenceUnsupportedTableOption)
- _, err := tk.Exec("create sequence seq comment=\"test\"")
- require.NoError(t, err)
+ tk.MustExec("create sequence seq comment=\"test\"")
sequenceTable := external.GetTableByName(t, tk, "test", "seq")
diff --git a/ddl/serial_test.go b/ddl/serial_test.go
index d99036ed4dc51..4d11db2b3409c 100644
--- a/ddl/serial_test.go
+++ b/ddl/serial_test.go
@@ -27,6 +27,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/ddl/util"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/errno"
@@ -101,7 +102,7 @@ func TestChangeMaxIndexLength(t *testing.T) {
tk.MustExec("create table t (c1 varchar(3073), index(c1)) charset = ascii")
tk.MustExec(fmt.Sprintf("create table t1 (c1 varchar(%d), index(c1)) charset = ascii;", config.DefMaxOfMaxIndexLength))
err := tk.ExecToErr(fmt.Sprintf("create table t2 (c1 varchar(%d), index(c1)) charset = ascii;", config.DefMaxOfMaxIndexLength+1))
- require.EqualError(t, err, "[ddl:1071]Specified key was too long; max key length is 12288 bytes")
+ require.EqualError(t, err, "[ddl:1071]Specified key was too long (12289 bytes); max key length is 12288 bytes")
}
func TestCreateTableWithLike(t *testing.T) {
@@ -112,6 +113,7 @@ func TestCreateTableWithLike(t *testing.T) {
tk.MustExec("use ctwl_db")
tk.MustExec("create table tt(id int primary key)")
tk.MustExec("create table t (c1 int not null auto_increment, c2 int, constraint cc foreign key (c2) references tt(id), primary key(c1)) auto_increment = 10")
+ tk.MustExec("set @@foreign_key_checks=0")
tk.MustExec("insert into t set c2=1")
tk.MustExec("create table t1 like ctwl_db.t")
tk.MustExec("insert into t1 set c2=11")
@@ -297,7 +299,7 @@ func TestCreateTableWithLikeAtTemporaryMode(t *testing.T) {
// Test foreign key.
tk.MustExec("drop table if exists test_foreign_key, t1")
- tk.MustExec("create table t1 (a int, b int)")
+ tk.MustExec("create table t1 (a int, b int, index(b))")
tk.MustExec("create table test_foreign_key (c int,d int,foreign key (d) references t1 (b))")
defer tk.MustExec("drop table if exists test_foreign_key, t1")
tk.MustExec("create global temporary table test_foreign_key_temp like test_foreign_key on commit delete rows")
@@ -382,7 +384,7 @@ func TestCreateTableWithLikeAtTemporaryMode(t *testing.T) {
defer tk.MustExec("drop table if exists partition_table, tmp_partition_table")
tk.MustExec("drop table if exists foreign_key_table1, foreign_key_table2, foreign_key_tmp")
- tk.MustExec("create table foreign_key_table1 (a int, b int)")
+ tk.MustExec("create table foreign_key_table1 (a int, b int, index(b))")
tk.MustExec("create table foreign_key_table2 (c int,d int,foreign key (d) references foreign_key_table1 (b))")
tk.MustExec("create temporary table foreign_key_tmp like foreign_key_table2")
is = sessiontxn.GetTxnManager(tk.Session()).GetTxnInfoSchema()
@@ -444,7 +446,7 @@ func TestCancelAddIndexPanic(t *testing.T) {
oldReorgWaitTimeout := ddl.ReorgWaitTimeout
ddl.ReorgWaitTimeout = 50 * time.Millisecond
defer func() { ddl.ReorgWaitTimeout = oldReorgWaitTimeout }()
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
hook.OnJobRunBeforeExported = func(job *model.Job) {
if job.Type == model.ActionAddIndex && job.State == model.JobStateRunning && job.SchemaState == model.StateWriteReorganization && job.SnapshotVer != 0 {
tkCancel.MustQuery(fmt.Sprintf("admin cancel ddl jobs %d", job.ID))
@@ -461,6 +463,70 @@ func TestCancelAddIndexPanic(t *testing.T) {
require.Truef(t, strings.HasPrefix(errMsg, "[ddl:8214]Cancelled DDL job"), "%v", errMsg)
}
+func TestRecoverTableWithTTL(t *testing.T) {
+ store, _ := createMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("create database if not exists test_recover")
+ tk.MustExec("use test_recover")
+ defer func(originGC bool) {
+ if originGC {
+ util.EmulatorGCEnable()
+ } else {
+ util.EmulatorGCDisable()
+ }
+ }(util.IsEmulatorGCEnable())
+
+ // disable emulator GC.
+ // Otherwise emulator GC will delete table record as soon as possible after execute drop table ddl.
+ util.EmulatorGCDisable()
+ gcTimeFormat := "20060102-15:04:05 -0700 MST"
+ safePointSQL := `INSERT HIGH_PRIORITY INTO mysql.tidb VALUES ('tikv_gc_safe_point', '%[1]s', '')
+ ON DUPLICATE KEY
+ UPDATE variable_value = '%[1]s'`
+ tk.MustExec(fmt.Sprintf(safePointSQL, time.Now().Add(-time.Hour).Format(gcTimeFormat)))
+ getDDLJobID := func(table, tp string) int64 {
+ rs, err := tk.Exec("admin show ddl jobs")
+ require.NoError(t, err)
+ rows, err := session.GetRows4Test(context.Background(), tk.Session(), rs)
+ require.NoError(t, err)
+ for _, row := range rows {
+ if row.GetString(2) == table && row.GetString(3) == tp {
+ return row.GetInt64(0)
+ }
+ }
+ require.FailNowf(t, "can't find %s table of %s", tp, table)
+ return -1
+ }
+
+ // recover table
+ tk.MustExec("create table t_recover1 (t timestamp) TTL=`t`+INTERVAL 1 DAY")
+ tk.MustExec("drop table t_recover1")
+ tk.MustExec("recover table t_recover1")
+ tk.MustQuery("show create table t_recover1").Check(testkit.Rows("t_recover1 CREATE TABLE `t_recover1` (\n `t` timestamp NULL DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`t` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+
+ // recover table with job id
+ tk.MustExec("create table t_recover2 (t timestamp) TTL=`t`+INTERVAL 1 DAY")
+ tk.MustExec("drop table t_recover2")
+ jobID := getDDLJobID("t_recover2", "drop table")
+ tk.MustExec(fmt.Sprintf("recover table BY JOB %d", jobID))
+ tk.MustQuery("show create table t_recover2").Check(testkit.Rows("t_recover2 CREATE TABLE `t_recover2` (\n `t` timestamp NULL DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`t` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+
+ // flashback table
+ tk.MustExec("create table t_recover3 (t timestamp) TTL=`t`+INTERVAL 1 DAY")
+ tk.MustExec("drop table t_recover3")
+ tk.MustExec("flashback table t_recover3")
+ tk.MustQuery("show create table t_recover3").Check(testkit.Rows("t_recover3 CREATE TABLE `t_recover3` (\n `t` timestamp NULL DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`t` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+
+ // flashback database
+ tk.MustExec("create database if not exists test_recover2")
+ tk.MustExec("create table test_recover2.t1 (t timestamp) TTL=`t`+INTERVAL 1 DAY")
+ tk.MustExec("create table test_recover2.t2 (t timestamp) TTL=`t`+INTERVAL 1 DAY")
+ tk.MustExec("drop database test_recover2")
+ tk.MustExec("flashback database test_recover2")
+ tk.MustQuery("show create table test_recover2.t1").Check(testkit.Rows("t1 CREATE TABLE `t1` (\n `t` timestamp NULL DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`t` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+ tk.MustQuery("show create table test_recover2.t2").Check(testkit.Rows("t2 CREATE TABLE `t2` (\n `t` timestamp NULL DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`t` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+}
+
func TestRecoverTableByJobID(t *testing.T) {
store, _ := createMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
@@ -619,7 +685,7 @@ func TestRecoverTableByJobIDFail(t *testing.T) {
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
// set hook
- hook := &ddl.TestDDLCallback{}
+ hook := &callback.TestDDLCallback{}
hook.OnJobRunBeforeExported = func(job *model.Job) {
if job.Type == model.ActionRecoverTable {
require.NoError(t, failpoint.Enable("tikvclient/mockCommitError", `return(true)`))
@@ -678,7 +744,7 @@ func TestRecoverTableByTableNameFail(t *testing.T) {
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
// set hook
- hook := &ddl.TestDDLCallback{}
+ hook := &callback.TestDDLCallback{}
hook.OnJobRunBeforeExported = func(job *model.Job) {
if job.Type == model.ActionRecoverTable {
require.NoError(t, failpoint.Enable("tikvclient/mockCommitError", `return(true)`))
@@ -751,7 +817,7 @@ func TestCanceledJobTakeTime(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("create table t_cjtt(a int)")
- hook := &ddl.TestDDLCallback{}
+ hook := &callback.TestDDLCallback{}
once := sync.Once{}
hook.OnJobRunBeforeExported = func(job *model.Job) {
once.Do(func() {
diff --git a/ddl/split_region.go b/ddl/split_region.go
index b201bf65538f3..ffbcb7439292d 100644
--- a/ddl/split_region.go
+++ b/ddl/split_region.go
@@ -31,7 +31,7 @@ import (
)
func splitPartitionTableRegion(ctx sessionctx.Context, store kv.SplittableStore, tbInfo *model.TableInfo, pi *model.PartitionInfo, scatter bool) {
- // Max partition count is 4096, should we sample and just choose some of the partition to split?
+ // Max partition count is 8192, should we sample and just choose some partitions to split?
regionIDs := make([]uint64, 0, len(pi.Definitions))
ctxWithTimeout, cancel := context.WithTimeout(context.Background(), ctx.GetSessionVars().GetSplitRegionTimeout())
defer cancel()
diff --git a/ddl/stat.go b/ddl/stat.go
index 24462f9bb141a..15be82d6719ae 100644
--- a/ddl/stat.go
+++ b/ddl/stat.go
@@ -15,6 +15,8 @@
package ddl
import (
+ "encoding/hex"
+
"github.com/pingcap/errors"
"github.com/pingcap/tidb/sessionctx/variable"
)
@@ -79,7 +81,7 @@ func (d *ddl) Stats(vars *variable.SessionVars) (map[string]interface{}, error)
m[ddlJobSchemaID] = job.SchemaID
m[ddlJobTableID] = job.TableID
m[ddlJobSnapshotVer] = job.SnapshotVer
- m[ddlJobReorgHandle] = tryDecodeToHandleString(ddlInfo.ReorgHandle)
+ m[ddlJobReorgHandle] = hex.EncodeToString(ddlInfo.ReorgHandle)
m[ddlJobArgs] = job.Args
return m, nil
}
diff --git a/ddl/stat_test.go b/ddl/stat_test.go
index 67d64c7c6cfa5..4280c2254c40a 100644
--- a/ddl/stat_test.go
+++ b/ddl/stat_test.go
@@ -16,6 +16,7 @@ package ddl_test
import (
"context"
+ "encoding/hex"
"fmt"
"strconv"
"testing"
@@ -24,15 +25,14 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/kv"
- "github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx"
- "github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/sessiontxn"
+ "github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/types"
"github.com/stretchr/testify/require"
@@ -42,13 +42,14 @@ func TestDDLStatsInfo(t *testing.T) {
store, domain := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
d := domain.DDL()
+ tk := testkit.NewTestKit(t, store)
+ ctx := tk.Session()
dbInfo, err := testSchemaInfo(store, "test_stat")
require.NoError(t, err)
- testCreateSchema(t, testkit.NewTestKit(t, store).Session(), d, dbInfo)
+ testCreateSchema(t, ctx, d, dbInfo)
tblInfo, err := testTableInfo(store, "t", 2)
require.NoError(t, err)
- testCreateTable(t, testkit.NewTestKit(t, store).Session(), d, dbInfo, tblInfo)
- ctx := testkit.NewTestKit(t, store).Session()
+ testCreateTable(t, ctx, d, dbInfo, tblInfo)
err = sessiontxn.NewTxn(context.Background(), ctx)
require.NoError(t, err)
@@ -60,7 +61,7 @@ func TestDDLStatsInfo(t *testing.T) {
require.NoError(t, err)
_, err = m.AddRecord(ctx, types.MakeDatums(3, 3))
require.NoError(t, err)
- ctx.StmtCommit()
+ ctx.StmtCommit(context.Background())
require.NoError(t, ctx.CommitTxn(context.Background()))
job := buildCreateIdxJob(dbInfo, tblInfo, true, "idx", "c1")
@@ -89,7 +90,11 @@ func TestDDLStatsInfo(t *testing.T) {
varMap, err := d.Stats(nil)
wg.Done()
require.NoError(t, err)
- require.Equal(t, "1", varMap[ddlJobReorgHandle])
+ key, err := hex.DecodeString(varMap[ddlJobReorgHandle].(string))
+ require.NoError(t, err)
+ _, h, err := tablecodec.DecodeRecordKey(key)
+ require.NoError(t, err)
+ require.Equal(t, h.IntValue(), int64(1))
}
}
}
@@ -145,20 +150,13 @@ func TestGetDDLInfo(t *testing.T) {
}
func addDDLJobs(sess session.Session, txn kv.Transaction, job *model.Job) error {
- if variable.EnableConcurrentDDL.Load() {
- b, err := job.Encode(true)
- if err != nil {
- return err
- }
- _, err = sess.Execute(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), fmt.Sprintf("insert into mysql.tidb_ddl_job(job_id, reorg, schema_ids, table_ids, job_meta, type, processing) values (%d, %t, %s, %s, %s, %d, %t)",
- job.ID, job.MayNeedReorg(), strconv.Quote(strconv.FormatInt(job.SchemaID, 10)), strconv.Quote(strconv.FormatInt(job.TableID, 10)), wrapKey2String(b), job.Type, false))
+ b, err := job.Encode(true)
+ if err != nil {
return err
}
- m := meta.NewMeta(txn)
- if job.MayNeedReorg() {
- return m.EnQueueDDLJob(job, meta.AddIndexJobListKey)
- }
- return m.EnQueueDDLJob(job)
+ _, err = sess.Execute(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), fmt.Sprintf("insert into mysql.tidb_ddl_job(job_id, reorg, schema_ids, table_ids, job_meta, type, processing) values (%d, %t, %s, %s, %s, %d, %t)",
+ job.ID, job.MayNeedReorg(), strconv.Quote(strconv.FormatInt(job.SchemaID, 10)), strconv.Quote(strconv.FormatInt(job.TableID, 10)), wrapKey2String(b), job.Type, false))
+ return err
}
func wrapKey2String(key []byte) string {
diff --git a/ddl/syncer/BUILD.bazel b/ddl/syncer/BUILD.bazel
index a608822051aaf..7c32da69cd5ce 100644
--- a/ddl/syncer/BUILD.bazel
+++ b/ddl/syncer/BUILD.bazel
@@ -22,6 +22,7 @@ go_library(
go_test(
name = "syncer_test",
+ timeout = "short",
srcs = ["syncer_test.go"],
flaky = True,
deps = [
diff --git a/ddl/syncer/syncer.go b/ddl/syncer/syncer.go
index ff7eaaa446893..e28d3d4954ca0 100644
--- a/ddl/syncer/syncer.go
+++ b/ddl/syncer/syncer.go
@@ -359,12 +359,6 @@ func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID i
}
if succ {
- if variable.EnableMDL.Load() {
- _, err = s.etcdCli.Delete(ctx, path, clientv3.WithPrefix())
- if err != nil {
- logutil.BgLogger().Warn("[ddl] syncer delete versions failed", zap.Any("job id", jobID), zap.Error(err))
- }
- }
return nil
}
time.Sleep(checkVersInterval)
diff --git a/ddl/table.go b/ddl/table.go
index df329fd973c06..e867bd832c102 100644
--- a/ddl/table.go
+++ b/ddl/table.go
@@ -301,6 +301,7 @@ func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
if oldTbInfoID > 0 && orReplace {
err = t.DropTableOrView(schemaID, oldTbInfoID)
if err != nil {
+ job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
err = t.GetAutoIDAccessors(schemaID, oldTbInfoID).Del()
@@ -404,6 +405,11 @@ func (w *worker) onRecoverTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in
schemaID := recoverInfo.SchemaID
tblInfo := recoverInfo.TableInfo
+ if tblInfo.TTLInfo != nil {
+ // force disable TTL job schedule for recovered table
+ tblInfo.TTLInfo.Enable = false
+ }
+
// check GC and safe point
gcEnable, err := checkGCEnable(w)
if err != nil {
@@ -812,8 +818,8 @@ func onTruncateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ erro
return ver, nil
}
-func onRebaseRowIDType(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
- return onRebaseAutoID(d, d.store, t, job, autoid.RowIDAllocType)
+func onRebaseAutoIncrementIDType(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ return onRebaseAutoID(d, d.store, t, job, autoid.AutoIncrementType)
}
func onRebaseAutoRandomType(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
@@ -862,7 +868,7 @@ func onRebaseAutoID(d *ddlCtx, store kv.Storage, t *meta.Meta, job *model.Job, t
newBase = newBaseTemp
}
- if tp == autoid.RowIDAllocType {
+ if tp == autoid.AutoIncrementType {
tblInfo.AutoIncID = newBase
} else {
tblInfo.AutoRandID = newBase
@@ -1341,6 +1347,7 @@ func onUpdateFlashReplicaStatus(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in
newIDs = append(newIDs, tblInfo.TiFlashReplica.AvailablePartitionIDs[i+1:]...)
tblInfo.TiFlashReplica.AvailablePartitionIDs = newIDs
tblInfo.TiFlashReplica.Available = false
+ logutil.BgLogger().Info("TiFlash replica become unavailable", zap.Int64("tableID", tblInfo.ID), zap.Int64("partitionID", id))
break
}
}
@@ -1350,6 +1357,9 @@ func onUpdateFlashReplicaStatus(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in
return ver, errors.Errorf("unknown physical ID %v in table %v", physicalID, tblInfo.Name.O)
}
+ if tblInfo.TiFlashReplica.Available {
+ logutil.BgLogger().Info("TiFlash replica available", zap.Int64("tableID", tblInfo.ID))
+ }
ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
diff --git a/ddl/table_modify_test.go b/ddl/table_modify_test.go
index f4b273771fd46..7f0b23e3fd894 100644
--- a/ddl/table_modify_test.go
+++ b/ddl/table_modify_test.go
@@ -20,6 +20,7 @@ import (
"time"
"github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/infoschema"
@@ -117,7 +118,6 @@ func TestLockTableReadOnly(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- tk1.MustExec("set global tidb_enable_metadata_lock=0")
tk2.MustExec("use test")
tk1.MustExec("drop table if exists t1,t2")
defer func() {
@@ -162,17 +162,6 @@ func TestLockTableReadOnly(t *testing.T) {
require.True(t, terror.ErrorEqual(tk2.ExecToErr("lock tables t1 write local"), infoschema.ErrTableLocked))
tk1.MustExec("admin cleanup table lock t1")
tk2.MustExec("insert into t1 set a=1, b=2")
-
- tk1.MustExec("set tidb_enable_amend_pessimistic_txn = 1")
- tk1.MustExec("begin pessimistic")
- tk1.MustQuery("select * from t1 where a = 1").Check(testkit.Rows("1 2"))
- tk2.MustExec("update t1 set b = 3")
- tk2.MustExec("alter table t1 read only")
- tk2.MustQuery("select * from t1 where a = 1").Check(testkit.Rows("1 3"))
- tk1.MustQuery("select * from t1 where a = 1").Check(testkit.Rows("1 2"))
- tk1.MustExec("update t1 set b = 4")
- require.True(t, terror.ErrorEqual(tk1.ExecToErr("commit"), domain.ErrInfoSchemaChanged))
- tk2.MustExec("alter table t1 read write")
}
// TestConcurrentLockTables test concurrent lock/unlock tables.
@@ -217,7 +206,7 @@ func TestConcurrentLockTables(t *testing.T) {
}
func testParallelExecSQL(t *testing.T, store kv.Storage, dom *domain.Domain, sql1, sql2 string, se1, se2 session.Session, f func(t *testing.T, err1, err2 error)) {
- callback := &ddl.TestDDLCallback{}
+ callback := &callback.TestDDLCallback{}
times := 0
callback.OnJobRunBeforeExported = func(job *model.Job) {
if times != 0 {
diff --git a/ddl/table_test.go b/ddl/table_test.go
index a9320e01cc781..cf7fdbf1cd7af 100644
--- a/ddl/table_test.go
+++ b/ddl/table_test.go
@@ -24,7 +24,9 @@ import (
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/meta/autoid"
+ "github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessiontxn"
"github.com/pingcap/tidb/table"
@@ -158,7 +160,7 @@ func testGetTableWithError(store kv.Storage, schemaID, tableID int64) (table.Tab
return nil, errors.New("table not found")
}
alloc := autoid.NewAllocator(store, schemaID, tblInfo.ID, false, autoid.RowIDAllocType)
- tbl, err := table.TableFromMeta(autoid.NewAllocators(alloc), tblInfo)
+ tbl, err := table.TableFromMeta(autoid.NewAllocators(false, alloc), tblInfo)
if err != nil {
return nil, errors.Trace(err)
}
@@ -235,6 +237,79 @@ func TestTable(t *testing.T) {
testDropSchema(t, testkit.NewTestKit(t, store).Session(), d, dbInfo)
}
+func TestCreateView(t *testing.T) {
+ store, domain := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
+
+ d := domain.DDL()
+ dbInfo, err := testSchemaInfo(store, "test_table")
+ require.NoError(t, err)
+ testCreateSchema(t, testkit.NewTestKit(t, store).Session(), domain.DDL(), dbInfo)
+
+ ctx := testkit.NewTestKit(t, store).Session()
+
+ tblInfo, err := testTableInfo(store, "t", 3)
+ require.NoError(t, err)
+ job := testCreateTable(t, ctx, d, dbInfo, tblInfo)
+ testCheckTableState(t, store, dbInfo, tblInfo, model.StatePublic)
+ testCheckJobDone(t, store, job.ID, true)
+
+ // Create a view
+ newTblInfo0, err := testTableInfo(store, "v", 3)
+ require.NoError(t, err)
+ job = &model.Job{
+ SchemaID: dbInfo.ID,
+ TableID: tblInfo.ID,
+ Type: model.ActionCreateView,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{newTblInfo0},
+ }
+ ctx.SetValue(sessionctx.QueryString, "skip")
+ err = d.DoDDLJob(ctx, job)
+ require.NoError(t, err)
+
+ v := getSchemaVer(t, ctx)
+ tblInfo.State = model.StatePublic
+ checkHistoryJobArgs(t, ctx, job.ID, &historyJobArgs{ver: v, tbl: newTblInfo0})
+ tblInfo.State = model.StateNone
+ testCheckTableState(t, store, dbInfo, tblInfo, model.StatePublic)
+ testCheckJobDone(t, store, job.ID, true)
+
+ // Replace a view
+ newTblInfo1, err := testTableInfo(store, "v", 3)
+ require.NoError(t, err)
+ job = &model.Job{
+ SchemaID: dbInfo.ID,
+ TableID: tblInfo.ID,
+ Type: model.ActionCreateView,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{newTblInfo1, true, newTblInfo0.ID},
+ }
+ ctx.SetValue(sessionctx.QueryString, "skip")
+ err = d.DoDDLJob(ctx, job)
+ require.NoError(t, err)
+
+ v = getSchemaVer(t, ctx)
+ tblInfo.State = model.StatePublic
+ checkHistoryJobArgs(t, ctx, job.ID, &historyJobArgs{ver: v, tbl: newTblInfo1})
+ tblInfo.State = model.StateNone
+ testCheckTableState(t, store, dbInfo, tblInfo, model.StatePublic)
+ testCheckJobDone(t, store, job.ID, true)
+
+ // Replace a view with a non-existing table id
+ newTblInfo2, err := testTableInfo(store, "v", 3)
+ require.NoError(t, err)
+ job = &model.Job{
+ SchemaID: dbInfo.ID,
+ TableID: tblInfo.ID,
+ Type: model.ActionCreateView,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{newTblInfo2, true, newTblInfo0.ID},
+ }
+ ctx.SetValue(sessionctx.QueryString, "skip")
+ err = d.DoDDLJob(ctx, job)
+ require.Error(t, err)
+}
+
func checkTableCacheTest(t *testing.T, store kv.Storage, dbInfo *model.DBInfo, tblInfo *model.TableInfo) {
ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
require.NoError(t, kv.RunInNewTxn(ctx, store, false, func(ctx context.Context, txn kv.Transaction) error {
@@ -371,3 +446,82 @@ func TestCreateTables(t *testing.T) {
testGetTable(t, domain, genIDs[1])
testGetTable(t, domain, genIDs[2])
}
+
+func TestAlterTTL(t *testing.T) {
+ store, domain := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
+
+ d := domain.DDL()
+
+ dbInfo, err := testSchemaInfo(store, "test_table")
+ require.NoError(t, err)
+ testCreateSchema(t, testkit.NewTestKit(t, store).Session(), d, dbInfo)
+
+ ctx := testkit.NewTestKit(t, store).Session()
+
+ // initialize a table with ttlInfo
+ tableName := "t"
+ tblInfo, err := testTableInfo(store, tableName, 2)
+ require.NoError(t, err)
+ tblInfo.Columns[0].FieldType = *types.NewFieldType(mysql.TypeDatetime)
+ tblInfo.Columns[1].FieldType = *types.NewFieldType(mysql.TypeDatetime)
+ tblInfo.TTLInfo = &model.TTLInfo{
+ ColumnName: tblInfo.Columns[0].Name,
+ IntervalExprStr: "5",
+ IntervalTimeUnit: int(ast.TimeUnitDay),
+ }
+
+ // create table
+ job := testCreateTable(t, ctx, d, dbInfo, tblInfo)
+ testCheckTableState(t, store, dbInfo, tblInfo, model.StatePublic)
+ testCheckJobDone(t, store, job.ID, true)
+
+ // submit ddl job to modify ttlInfo
+ tableInfoAfterAlterTTLInfo := tblInfo.Clone()
+ require.NoError(t, err)
+ tableInfoAfterAlterTTLInfo.TTLInfo = &model.TTLInfo{
+ ColumnName: tblInfo.Columns[1].Name,
+ IntervalExprStr: "1",
+ IntervalTimeUnit: int(ast.TimeUnitYear),
+ }
+
+ job = &model.Job{
+ SchemaID: dbInfo.ID,
+ TableID: tblInfo.ID,
+ Type: model.ActionAlterTTLInfo,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{&model.TTLInfo{
+ ColumnName: tblInfo.Columns[1].Name,
+ IntervalExprStr: "1",
+ IntervalTimeUnit: int(ast.TimeUnitYear),
+ }},
+ }
+ ctx.SetValue(sessionctx.QueryString, "skip")
+ require.NoError(t, d.DoDDLJob(ctx, job))
+
+ v := getSchemaVer(t, ctx)
+ checkHistoryJobArgs(t, ctx, job.ID, &historyJobArgs{ver: v, tbl: nil})
+
+ // assert the ddlInfo as expected
+ historyJob, err := ddl.GetHistoryJobByID(testkit.NewTestKit(t, store).Session(), job.ID)
+ require.NoError(t, err)
+ require.Equal(t, tableInfoAfterAlterTTLInfo.TTLInfo, historyJob.BinlogInfo.TableInfo.TTLInfo)
+
+ // submit a ddl job to modify ttlEnabled
+ job = &model.Job{
+ SchemaID: dbInfo.ID,
+ TableID: tblInfo.ID,
+ Type: model.ActionAlterTTLRemove,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{true},
+ }
+ ctx.SetValue(sessionctx.QueryString, "skip")
+ require.NoError(t, d.DoDDLJob(ctx, job))
+
+ v = getSchemaVer(t, ctx)
+ checkHistoryJobArgs(t, ctx, job.ID, &historyJobArgs{ver: v, tbl: nil})
+
+ // assert the ddlInfo as expected
+ historyJob, err = ddl.GetHistoryJobByID(testkit.NewTestKit(t, store).Session(), job.ID)
+ require.NoError(t, err)
+ require.Empty(t, historyJob.BinlogInfo.TableInfo.TTLInfo)
+}
diff --git a/ddl/testutil/BUILD.bazel b/ddl/testutil/BUILD.bazel
index 3562ca3b34571..052a747f427ba 100644
--- a/ddl/testutil/BUILD.bazel
+++ b/ddl/testutil/BUILD.bazel
@@ -14,6 +14,8 @@ go_library(
"//table",
"//table/tables",
"//types",
+ "//util/logutil",
"@com_github_pingcap_errors//:errors",
+ "@org_uber_go_zap//:zap",
],
)
diff --git a/ddl/testutil/testutil.go b/ddl/testutil/testutil.go
index 642579ba00ea7..52adf6b750f73 100644
--- a/ddl/testutil/testutil.go
+++ b/ddl/testutil/testutil.go
@@ -26,6 +26,8 @@ import (
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/types"
+ "github.com/pingcap/tidb/util/logutil"
+ "go.uber.org/zap"
)
// SessionExecInGoroutine export for testing.
@@ -82,3 +84,13 @@ func ExtractAllTableHandles(se session.Session, dbName, tbName string) ([]int64,
})
return allHandles, err
}
+
+// FindIdxInfo is to get IndexInfo by index name.
+func FindIdxInfo(dom *domain.Domain, dbName, tbName, idxName string) *model.IndexInfo {
+ tbl, err := dom.InfoSchema().TableByName(model.NewCIStr(dbName), model.NewCIStr(tbName))
+ if err != nil {
+ logutil.BgLogger().Warn("cannot find table", zap.String("dbName", dbName), zap.String("tbName", tbName))
+ return nil
+ }
+ return tbl.Meta().FindIndexByName(idxName)
+}
diff --git a/ddl/tiflash_replica_test.go b/ddl/tiflash_replica_test.go
index 874ab05359d3b..abd7275e4669b 100644
--- a/ddl/tiflash_replica_test.go
+++ b/ddl/tiflash_replica_test.go
@@ -26,7 +26,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/config"
- "github.com/pingcap/tidb/ddl"
+ "github.com/pingcap/tidb/ddl/internal/callback"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/kv"
@@ -286,7 +286,7 @@ func TestCreateTableWithLike2(t *testing.T) {
tbl1 := external.GetTableByName(t, tk, "test", "t1")
doneCh := make(chan error, 2)
- hook := &ddl.TestDDLCallback{Do: dom}
+ hook := &callback.TestDDLCallback{Do: dom}
var onceChecker sync.Map
hook.OnJobRunBeforeExported = func(job *model.Job) {
if job.Type != model.ActionAddColumn && job.Type != model.ActionDropColumn &&
@@ -303,7 +303,7 @@ func TestCreateTableWithLike2(t *testing.T) {
}
onceChecker.Store(job.ID, true)
- go backgroundExec(store, "create table t2 like t1", doneCh)
+ go backgroundExec(store, "test", "create table t2 like t1", doneCh)
}
}
originalHook := dom.DDL().GetHook()
diff --git a/ddl/tiflashtest/BUILD.bazel b/ddl/tiflashtest/BUILD.bazel
new file mode 100644
index 0000000000000..4e24ea074d7f4
--- /dev/null
+++ b/ddl/tiflashtest/BUILD.bazel
@@ -0,0 +1,38 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+ name = "tiflashtest_test",
+ timeout = "short",
+ srcs = [
+ "ddl_tiflash_test.go",
+ "main_test.go",
+ ],
+ flaky = True,
+ shard_count = 30,
+ deps = [
+ "//config",
+ "//ddl",
+ "//ddl/placement",
+ "//ddl/util",
+ "//domain",
+ "//domain/infosync",
+ "//kv",
+ "//parser/model",
+ "//session",
+ "//store/gcworker",
+ "//store/mockstore",
+ "//store/mockstore/unistore",
+ "//table",
+ "//testkit",
+ "//testkit/testsetup",
+ "//util",
+ "//util/logutil",
+ "@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_pingcap_kvproto//pkg/metapb",
+ "@com_github_stretchr_testify//require",
+ "@com_github_tikv_client_go_v2//oracle",
+ "@com_github_tikv_client_go_v2//testutils",
+ "@org_uber_go_goleak//:goleak",
+ "@org_uber_go_zap//:zap",
+ ],
+)
diff --git a/ddl/ddl_tiflash_test.go b/ddl/tiflashtest/ddl_tiflash_test.go
similarity index 91%
rename from ddl/ddl_tiflash_test.go
rename to ddl/tiflashtest/ddl_tiflash_test.go
index 27d05112df483..6526dbc01553d 100644
--- a/ddl/ddl_tiflash_test.go
+++ b/ddl/tiflashtest/ddl_tiflash_test.go
@@ -16,7 +16,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSES/QL-LICENSE file.
-package ddl_test
+package tiflashtest
import (
"context"
@@ -45,6 +45,7 @@ import (
"github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/logutil"
"github.com/stretchr/testify/require"
+ "github.com/tikv/client-go/v2/oracle"
"github.com/tikv/client-go/v2/testutils"
"go.uber.org/zap"
)
@@ -126,10 +127,6 @@ func ChangeGCSafePoint(tk *testkit.TestKit, t time.Time, enable string, lifeTime
tk.MustExec(s)
}
-func CheckPlacementRule(tiflash *infosync.MockTiFlash, rule placement.TiFlashRule) bool {
- return tiflash.CheckPlacementRule(rule)
-}
-
func (s *tiflashContext) CheckFlashback(tk *testkit.TestKit, t *testing.T) {
// If table is dropped after tikv_gc_safe_point, it can be recovered
ChangeGCSafePoint(tk, time.Now().Add(-time.Hour), "false", "10m0s")
@@ -442,6 +439,41 @@ func TestTiFlashDropPartition(t *testing.T) {
CheckTableAvailableWithTableName(s.dom, t, 1, []string{}, "test", "ddltiflash")
}
+func TestTiFlashFlashbackCluster(t *testing.T) {
+ s, teardown := createTiFlashContext(t)
+ defer teardown()
+ tk := testkit.NewTestKit(t, s.store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int)")
+ tk.MustExec("insert into t values (1), (2), (3)")
+
+ ts, err := tk.Session().GetStore().GetOracle().GetTimestamp(context.Background(), &oracle.Option{})
+ require.NoError(t, err)
+
+ tk.MustExec("alter table t set tiflash replica 1")
+ time.Sleep(ddl.PollTiFlashInterval * RoundToBeAvailable)
+ CheckTableAvailableWithTableName(s.dom, t, 1, []string{}, "test", "t")
+
+ injectSafeTS := oracle.GoTimeToTS(oracle.GetTimeFromTS(ts).Add(10 * time.Second))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockFlashbackTest", `return(true)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/injectSafeTS",
+ fmt.Sprintf("return(%v)", injectSafeTS)))
+
+ ChangeGCSafePoint(tk, time.Now().Add(-10*time.Second), "true", "10m0s")
+ defer func() {
+ ChangeGCSafePoint(tk, time.Now(), "true", "10m0s")
+ }()
+
+ errorMsg := fmt.Sprintf("[ddl:-1]Detected unsupported DDL job type(%s) during [%s, now), can't do flashback",
+ model.ActionSetTiFlashReplica.String(), oracle.GetTimeFromTS(ts).String())
+ tk.MustGetErrMsg(fmt.Sprintf("flashback cluster to timestamp '%s'", oracle.GetTimeFromTS(ts)), errorMsg)
+
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockFlashbackTest"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/injectSafeTS"))
+}
+
func CheckTableAvailableWithTableName(dom *domain.Domain, t *testing.T, count uint64, labels []string, db string, table string) {
tb, err := dom.InfoSchema().TableByName(model.NewCIStr(db), model.NewCIStr(table))
require.NoError(t, err)
@@ -524,7 +556,7 @@ func TestSetPlacementRuleNormal(t *testing.T) {
tb, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("ddltiflash"))
require.NoError(t, err)
expectRule := infosync.MakeNewRule(tb.Meta().ID, 1, []string{"a", "b"})
- res := CheckPlacementRule(s.tiflash, *expectRule)
+ res := s.tiflash.CheckPlacementRule(*expectRule)
require.True(t, res)
// Set lastSafePoint to a timepoint in future, so all dropped table can be reckon as gc-ed.
@@ -536,7 +568,7 @@ func TestSetPlacementRuleNormal(t *testing.T) {
defer fCancelPD()
tk.MustExec("drop table ddltiflash")
expectRule = infosync.MakeNewRule(tb.Meta().ID, 1, []string{"a", "b"})
- res = CheckPlacementRule(s.tiflash, *expectRule)
+ res = s.tiflash.CheckPlacementRule(*expectRule)
require.True(t, res)
}
@@ -580,7 +612,7 @@ func TestSetPlacementRuleWithGCWorker(t *testing.T) {
require.NoError(t, err)
expectRule := infosync.MakeNewRule(tb.Meta().ID, 1, []string{"a", "b"})
- res := CheckPlacementRule(s.tiflash, *expectRule)
+ res := s.tiflash.CheckPlacementRule(*expectRule)
require.True(t, res)
ChangeGCSafePoint(tk, time.Now().Add(-time.Hour), "true", "10m0s")
@@ -590,7 +622,7 @@ func TestSetPlacementRuleWithGCWorker(t *testing.T) {
// Wait GC
time.Sleep(ddl.PollTiFlashInterval * RoundToBeAvailable)
- res = CheckPlacementRule(s.tiflash, *expectRule)
+ res = s.tiflash.CheckPlacementRule(*expectRule)
require.False(t, res)
}
@@ -611,7 +643,7 @@ func TestSetPlacementRuleFail(t *testing.T) {
require.NoError(t, err)
expectRule := infosync.MakeNewRule(tb.Meta().ID, 1, []string{})
- res := CheckPlacementRule(s.tiflash, *expectRule)
+ res := s.tiflash.CheckPlacementRule(*expectRule)
require.False(t, res)
}
@@ -1264,3 +1296,58 @@ func TestTiFlashPartitionNotAvailable(t *testing.T) {
require.NotNil(t, replica)
require.True(t, replica.Available)
}
+
+func TestTiFlashAvailableAfterAddPartition(t *testing.T) {
+ s, teardown := createTiFlashContext(t)
+ defer teardown()
+ tk := testkit.NewTestKit(t, s.store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists ddltiflash")
+ tk.MustExec("create table ddltiflash(z int) PARTITION BY RANGE(z) (PARTITION p0 VALUES LESS THAN (10))")
+ tk.MustExec("alter table ddltiflash set tiflash replica 1")
+ time.Sleep(ddl.PollTiFlashInterval * RoundToBeAvailable * 3)
+ CheckTableAvailable(s.dom, t, 1, []string{})
+
+ tb, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("ddltiflash"))
+ require.NoError(t, err)
+ require.NotNil(t, tb)
+
+ // still available after adding partition.
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/sleepBeforeReplicaOnly", `return(2)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/waitForAddPartition", `return(3)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/PollTiFlashReplicaStatusReplaceCurAvailableValue", `return(false)`))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/sleepBeforeReplicaOnly"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/waitForAddPartition"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/PollTiFlashReplicaStatusReplaceCurAvailableValue"))
+ }()
+ tk.MustExec("ALTER TABLE ddltiflash ADD PARTITION (PARTITION pn VALUES LESS THAN (20))")
+ time.Sleep(ddl.PollTiFlashInterval * RoundToBeAvailable * 3)
+ CheckTableAvailable(s.dom, t, 1, []string{})
+ tb, err = s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("ddltiflash"))
+ require.NoError(t, err)
+ pi := tb.Meta().GetPartitionInfo()
+ require.NotNil(t, pi)
+ require.Equal(t, len(pi.Definitions), 2)
+}
+
+func TestTiFlashAvailableAfterDownOneStore(t *testing.T) {
+ s, teardown := createTiFlashContext(t)
+ defer teardown()
+ tk := testkit.NewTestKit(t, s.store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists ddltiflash")
+ tk.MustExec("create table ddltiflash(z int) PARTITION BY RANGE(z) (PARTITION p0 VALUES LESS THAN (10))")
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/OneTiFlashStoreDown", `return`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/domain/infosync/OneTiFlashStoreDown", `return`))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/OneTiFlashStoreDown"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/domain/infosync/OneTiFlashStoreDown"))
+ }()
+
+ tk.MustExec("alter table ddltiflash set tiflash replica 1")
+ time.Sleep(ddl.PollTiFlashInterval * RoundToBeAvailable * 3)
+ CheckTableAvailable(s.dom, t, 1, []string{})
+}
diff --git a/ddl/concurrentddltest/main_test.go b/ddl/tiflashtest/main_test.go
similarity index 88%
rename from ddl/concurrentddltest/main_test.go
rename to ddl/tiflashtest/main_test.go
index 4ab7e96eab2ae..68063ce27071b 100644
--- a/ddl/concurrentddltest/main_test.go
+++ b/ddl/tiflashtest/main_test.go
@@ -12,7 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package concurrentddltest
+// Copyright 2013 The ql Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSES/QL-LICENSE file.
+
+package tiflashtest
import (
"testing"
diff --git a/ddl/ttl.go b/ddl/ttl.go
new file mode 100644
index 0000000000000..58011ee9e79f9
--- /dev/null
+++ b/ddl/ttl.go
@@ -0,0 +1,234 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/tidb/expression"
+ "github.com/pingcap/tidb/meta"
+ "github.com/pingcap/tidb/parser"
+ "github.com/pingcap/tidb/parser/ast"
+ "github.com/pingcap/tidb/parser/format"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/parser/mysql"
+ "github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/sessiontxn"
+ "github.com/pingcap/tidb/types"
+ "github.com/pingcap/tidb/util/dbterror"
+)
+
+func onTTLInfoRemove(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+ tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, job.SchemaID)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ tblInfo.TTLInfo = nil
+ ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo)
+ return ver, nil
+}
+
+func onTTLInfoChange(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+ // at least one for them is not nil
+ var ttlInfo *model.TTLInfo
+ var ttlInfoEnable *bool
+ var ttlInfoJobInterval *string
+
+ if err := job.DecodeArgs(&ttlInfo, &ttlInfoEnable, &ttlInfoJobInterval); err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+
+ tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, job.SchemaID)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ if ttlInfo != nil {
+ // if the TTL_ENABLE is not set explicitly, use the original value
+ if ttlInfoEnable == nil && tblInfo.TTLInfo != nil {
+ ttlInfo.Enable = tblInfo.TTLInfo.Enable
+ }
+ if ttlInfoJobInterval == nil && tblInfo.TTLInfo != nil {
+ ttlInfo.JobInterval = tblInfo.TTLInfo.JobInterval
+ }
+ tblInfo.TTLInfo = ttlInfo
+ }
+ if ttlInfoEnable != nil {
+ if tblInfo.TTLInfo == nil {
+ return ver, errors.Trace(dbterror.ErrSetTTLOptionForNonTTLTable.FastGenByArgs("TTL_ENABLE"))
+ }
+
+ tblInfo.TTLInfo.Enable = *ttlInfoEnable
+ }
+ if ttlInfoJobInterval != nil {
+ if tblInfo.TTLInfo == nil {
+ return ver, errors.Trace(dbterror.ErrSetTTLOptionForNonTTLTable.FastGenByArgs("TTL_JOB_INTERVAL"))
+ }
+
+ tblInfo.TTLInfo.JobInterval = *ttlInfoJobInterval
+ }
+
+ ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo)
+ return ver, nil
+}
+
+func checkTTLInfoValid(ctx sessionctx.Context, schema model.CIStr, tblInfo *model.TableInfo) error {
+ if err := checkTTLIntervalExpr(ctx, tblInfo.TTLInfo); err != nil {
+ return err
+ }
+
+ if err := checkTTLTableSuitable(ctx, schema, tblInfo); err != nil {
+ return err
+ }
+
+ return checkTTLInfoColumnType(tblInfo)
+}
+
+func checkTTLIntervalExpr(ctx sessionctx.Context, ttlInfo *model.TTLInfo) error {
+ // FIXME: use a better way to validate the interval expression in ttl
+ var nowAddIntervalExpr ast.ExprNode
+
+ unit := ast.TimeUnitType(ttlInfo.IntervalTimeUnit)
+ expr := fmt.Sprintf("select NOW() + INTERVAL %s %s", ttlInfo.IntervalExprStr, unit.String())
+ stmts, _, err := parser.New().ParseSQL(expr)
+ if err != nil {
+ // FIXME: the error information can be wrong, as it could indicate an unknown position to user.
+ return errors.Trace(err)
+ }
+ nowAddIntervalExpr = stmts[0].(*ast.SelectStmt).Fields.Fields[0].Expr
+ _, err = expression.EvalAstExpr(ctx, nowAddIntervalExpr)
+ return err
+}
+
+func checkTTLInfoColumnType(tblInfo *model.TableInfo) error {
+ colInfo := findColumnByName(tblInfo.TTLInfo.ColumnName.L, tblInfo)
+ if colInfo == nil {
+ return dbterror.ErrBadField.GenWithStackByArgs(tblInfo.TTLInfo.ColumnName.O, "TTL config")
+ }
+ if !types.IsTypeTime(colInfo.FieldType.GetType()) {
+ return dbterror.ErrUnsupportedColumnInTTLConfig.GenWithStackByArgs(tblInfo.TTLInfo.ColumnName.O)
+ }
+
+ return nil
+}
+
+// checkTTLTableSuitable returns whether this table is suitable to be a TTL table
+// A temporary table or a parent table referenced by a foreign key cannot be TTL table
+func checkTTLTableSuitable(ctx sessionctx.Context, schema model.CIStr, tblInfo *model.TableInfo) error {
+ if tblInfo.TempTableType != model.TempTableNone {
+ return dbterror.ErrTempTableNotAllowedWithTTL
+ }
+
+ if err := checkPrimaryKeyForTTLTable(tblInfo); err != nil {
+ return err
+ }
+
+ // checks even when the foreign key check is not enabled, to keep safe
+ is := sessiontxn.GetTxnManager(ctx).GetTxnInfoSchema()
+ if referredFK := checkTableHasForeignKeyReferred(is, schema.L, tblInfo.Name.L, nil, true); referredFK != nil {
+ return dbterror.ErrUnsupportedTTLReferencedByFK
+ }
+
+ return nil
+}
+
+func checkDropColumnWithTTLConfig(tblInfo *model.TableInfo, colName string) error {
+ if tblInfo.TTLInfo != nil {
+ if tblInfo.TTLInfo.ColumnName.L == colName {
+ return dbterror.ErrTTLColumnCannotDrop.GenWithStackByArgs(colName)
+ }
+ }
+
+ return nil
+}
+
+// We should forbid creating a TTL table with clustered primary key that contains a column with type float/double.
+// This is because currently we are using SQL to delete expired rows and when the primary key contains float/double column,
+// it is hard to use condition `WHERE PK in (...)` to delete specified rows because some precision will be lost when comparing.
+func checkPrimaryKeyForTTLTable(tblInfo *model.TableInfo) error {
+ if !tblInfo.IsCommonHandle {
+ // only check the primary keys when it is common handle
+ return nil
+ }
+
+ pk := tblInfo.GetPrimaryKey()
+ if pk == nil {
+ return nil
+ }
+
+ for _, colDef := range pk.Columns {
+ col := tblInfo.Columns[colDef.Offset]
+ switch col.GetType() {
+ case mysql.TypeFloat, mysql.TypeDouble:
+ return dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL
+ }
+ }
+
+ return nil
+}
+
+// getTTLInfoInOptions returns the aggregated ttlInfo, the ttlEnable, or an error.
+// if TTL, TTL_ENABLE or TTL_JOB_INTERVAL is not set in the config, the corresponding return value will be nil.
+// if both of TTL and TTL_ENABLE are set, the `ttlInfo.Enable` will be equal with `ttlEnable`.
+// if both of TTL and TTL_JOB_INTERVAL are set, the `ttlInfo.JobInterval` will be equal with `ttlCronJobSchedule`.
+func getTTLInfoInOptions(options []*ast.TableOption) (ttlInfo *model.TTLInfo, ttlEnable *bool, ttlCronJobSchedule *string, err error) {
+ for _, op := range options {
+ switch op.Tp {
+ case ast.TableOptionTTL:
+ var sb strings.Builder
+ restoreFlags := format.RestoreStringSingleQuotes | format.RestoreNameBackQuotes
+ restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
+ err := op.Value.Restore(restoreCtx)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+
+ intervalExpr := sb.String()
+ ttlInfo = &model.TTLInfo{
+ ColumnName: op.ColumnName.Name,
+ IntervalExprStr: intervalExpr,
+ IntervalTimeUnit: int(op.TimeUnitValue.Unit),
+ Enable: true,
+ JobInterval: "1h",
+ }
+ case ast.TableOptionTTLEnable:
+ ttlEnable = &op.BoolValue
+ case ast.TableOptionTTLJobInterval:
+ ttlCronJobSchedule = &op.StrValue
+ }
+ }
+
+ if ttlInfo != nil {
+ if ttlEnable != nil {
+ ttlInfo.Enable = *ttlEnable
+ }
+ if ttlCronJobSchedule != nil {
+ ttlInfo.JobInterval = *ttlCronJobSchedule
+ }
+ }
+ return ttlInfo, ttlEnable, ttlCronJobSchedule, nil
+}
diff --git a/ddl/ttl_test.go b/ddl/ttl_test.go
new file mode 100644
index 0000000000000..fafc9e240a710
--- /dev/null
+++ b/ddl/ttl_test.go
@@ -0,0 +1,150 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "testing"
+
+ "github.com/pingcap/tidb/parser/ast"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/stretchr/testify/assert"
+)
+
+func Test_getTTLInfoInOptions(t *testing.T) {
+ falseValue := false
+ trueValue := true
+ twentyFourHours := "24h"
+
+ cases := []struct {
+ options []*ast.TableOption
+ ttlInfo *model.TTLInfo
+ ttlEnable *bool
+ ttlCronJobSchedule *string
+ err error
+ }{
+ {
+ []*ast.TableOption{},
+ nil,
+ nil,
+ nil,
+ nil,
+ },
+ {
+ []*ast.TableOption{
+ {
+ Tp: ast.TableOptionTTL,
+ ColumnName: &ast.ColumnName{Name: model.NewCIStr("test_column")},
+ Value: ast.NewValueExpr(5, "", ""),
+ TimeUnitValue: &ast.TimeUnitExpr{Unit: ast.TimeUnitYear},
+ },
+ },
+ &model.TTLInfo{
+ ColumnName: model.NewCIStr("test_column"),
+ IntervalExprStr: "5",
+ IntervalTimeUnit: int(ast.TimeUnitYear),
+ Enable: true,
+ JobInterval: "1h",
+ },
+ nil,
+ nil,
+ nil,
+ },
+ {
+ []*ast.TableOption{
+ {
+ Tp: ast.TableOptionTTLEnable,
+ BoolValue: false,
+ },
+ {
+ Tp: ast.TableOptionTTL,
+ ColumnName: &ast.ColumnName{Name: model.NewCIStr("test_column")},
+ Value: ast.NewValueExpr(5, "", ""),
+ TimeUnitValue: &ast.TimeUnitExpr{Unit: ast.TimeUnitYear},
+ },
+ },
+ &model.TTLInfo{
+ ColumnName: model.NewCIStr("test_column"),
+ IntervalExprStr: "5",
+ IntervalTimeUnit: int(ast.TimeUnitYear),
+ Enable: false,
+ JobInterval: "1h",
+ },
+ &falseValue,
+ nil,
+ nil,
+ },
+ {
+ []*ast.TableOption{
+ {
+ Tp: ast.TableOptionTTLEnable,
+ BoolValue: false,
+ },
+ {
+ Tp: ast.TableOptionTTL,
+ ColumnName: &ast.ColumnName{Name: model.NewCIStr("test_column")},
+ Value: ast.NewValueExpr(5, "", ""),
+ TimeUnitValue: &ast.TimeUnitExpr{Unit: ast.TimeUnitYear},
+ },
+ {
+ Tp: ast.TableOptionTTLEnable,
+ BoolValue: true,
+ },
+ },
+ &model.TTLInfo{
+ ColumnName: model.NewCIStr("test_column"),
+ IntervalExprStr: "5",
+ IntervalTimeUnit: int(ast.TimeUnitYear),
+ Enable: true,
+ JobInterval: "1h",
+ },
+ &trueValue,
+ nil,
+ nil,
+ },
+ {
+ []*ast.TableOption{
+ {
+ Tp: ast.TableOptionTTL,
+ ColumnName: &ast.ColumnName{Name: model.NewCIStr("test_column")},
+ Value: ast.NewValueExpr(5, "", ""),
+ TimeUnitValue: &ast.TimeUnitExpr{Unit: ast.TimeUnitYear},
+ },
+ {
+ Tp: ast.TableOptionTTLJobInterval,
+ StrValue: "24h",
+ },
+ },
+ &model.TTLInfo{
+ ColumnName: model.NewCIStr("test_column"),
+ IntervalExprStr: "5",
+ IntervalTimeUnit: int(ast.TimeUnitYear),
+ Enable: true,
+ JobInterval: "24h",
+ },
+ nil,
+ &twentyFourHours,
+ nil,
+ },
+ }
+
+ for _, c := range cases {
+ ttlInfo, ttlEnable, ttlCronJobSchedule, err := getTTLInfoInOptions(c.options)
+
+ assert.Equal(t, c.ttlInfo, ttlInfo)
+ assert.Equal(t, c.ttlEnable, ttlEnable)
+ assert.Equal(t, c.ttlCronJobSchedule, ttlCronJobSchedule)
+ assert.Equal(t, c.err, err)
+ }
+}
diff --git a/distsql/BUILD.bazel b/distsql/BUILD.bazel
index 5839f55fbc52c..cb9bbc63dd524 100644
--- a/distsql/BUILD.bazel
+++ b/distsql/BUILD.bazel
@@ -34,8 +34,8 @@ go_library(
"//util/logutil",
"//util/memory",
"//util/ranger",
+ "//util/tracing",
"//util/trxevents",
- "@com_github_opentracing_opentracing_go//:opentracing-go",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/metapb",
@@ -64,6 +64,7 @@ go_test(
embed = [":distsql"],
flaky = True,
race = "on",
+ shard_count = 2,
deps = [
"//kv",
"//parser/charset",
diff --git a/distsql/distsql.go b/distsql/distsql.go
index 1f18d084eb0b2..defbeb85ae74f 100644
--- a/distsql/distsql.go
+++ b/distsql/distsql.go
@@ -19,7 +19,6 @@ import (
"strconv"
"unsafe"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/kv"
@@ -30,6 +29,8 @@ import (
"github.com/pingcap/tidb/statistics"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/memory"
+ "github.com/pingcap/tidb/util/tracing"
"github.com/pingcap/tidb/util/trxevents"
"github.com/pingcap/tipb/go-tipb"
"github.com/tikv/client-go/v2/tikvrpc/interceptor"
@@ -38,11 +39,11 @@ import (
)
// DispatchMPPTasks dispatches all tasks and returns an iterator.
-func DispatchMPPTasks(ctx context.Context, sctx sessionctx.Context, tasks []*kv.MPPDispatchRequest, fieldTypes []*types.FieldType, planIDs []int, rootID int, startTs uint64) (SelectResult, error) {
+func DispatchMPPTasks(ctx context.Context, sctx sessionctx.Context, tasks []*kv.MPPDispatchRequest, fieldTypes []*types.FieldType, planIDs []int, rootID int, startTs uint64, mppQueryID kv.MPPQueryID, memTracker *memory.Tracker) (SelectResult, error) {
ctx = WithSQLKvExecCounterInterceptor(ctx, sctx.GetSessionVars().StmtCtx)
_, allowTiFlashFallback := sctx.GetSessionVars().AllowFallbackToTiKV[kv.TiFlash]
ctx = SetTiFlashMaxThreadsInContext(ctx, sctx)
- resp := sctx.GetMPPClient().DispatchMPPTasks(ctx, sctx.GetSessionVars().KVVars, tasks, allowTiFlashFallback, startTs)
+ resp := sctx.GetMPPClient().DispatchMPPTasks(ctx, sctx.GetSessionVars().KVVars, tasks, allowTiFlashFallback, startTs, mppQueryID, sctx.GetSessionVars().ChooseMppVersion(), memTracker)
if resp == nil {
return nil, errors.New("client returns nil response")
}
@@ -63,11 +64,8 @@ func DispatchMPPTasks(ctx context.Context, sctx sessionctx.Context, tasks []*kv.
// Select sends a DAG request, returns SelectResult.
// In kvReq, KeyRanges is required, Concurrency/KeepOrder/Desc/IsolationLevel/Priority are optional.
func Select(ctx context.Context, sctx sessionctx.Context, kvReq *kv.Request, fieldTypes []*types.FieldType, fb *statistics.QueryFeedback) (SelectResult, error) {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("distsql.Select", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "distsql.Select")
+ defer r.End()
// For testing purpose.
if hook := ctx.Value("CheckSelectRequestHook"); hook != nil {
diff --git a/distsql/distsql_test.go b/distsql/distsql_test.go
index 52aa62ba112fa..d7a2447cf35b6 100644
--- a/distsql/distsql_test.go
+++ b/distsql/distsql_test.go
@@ -107,7 +107,8 @@ func TestSelectWithRuntimeStats(t *testing.T) {
}
func TestSelectResultRuntimeStats(t *testing.T) {
- basic := &execdetails.BasicRuntimeStats{}
+ stmtStats := execdetails.NewRuntimeStatsColl(nil)
+ basic := stmtStats.GetBasicRuntimeStats(1)
basic.Record(time.Second, 20)
s1 := &selectResultRuntimeStats{
copRespTime: []time.Duration{time.Second, time.Millisecond},
@@ -120,12 +121,10 @@ func TestSelectResultRuntimeStats(t *testing.T) {
}
s2 := *s1
- stmtStats := execdetails.NewRuntimeStatsColl(nil)
- stmtStats.RegisterStats(1, basic)
stmtStats.RegisterStats(1, s1)
stmtStats.RegisterStats(1, &s2)
stats := stmtStats.GetRootStats(1)
- expect := "time:1s, loops:1, cop_task: {num: 4, max: 1s, min: 1ms, avg: 500.5ms, p95: 1s, max_proc_keys: 200, p95_proc_keys: 200, tot_proc: 2s, tot_wait: 2s, copr_cache_hit_ratio: 0.00, distsql_concurrency: 15}, backoff{RegionMiss: 2ms}"
+ expect := "time:1s, loops:1, cop_task: {num: 4, max: 1s, min: 1ms, avg: 500.5ms, p95: 1s, max_proc_keys: 200, p95_proc_keys: 200, tot_proc: 2s, tot_wait: 2s, copr_cache_hit_ratio: 0.00, max_distsql_concurrency: 15}, backoff{RegionMiss: 2ms}"
require.Equal(t, expect, stats.String())
// Test for idempotence.
require.Equal(t, expect, stats.String())
@@ -136,7 +135,7 @@ func TestSelectResultRuntimeStats(t *testing.T) {
}
stmtStats.RegisterStats(2, s1)
stats = stmtStats.GetRootStats(2)
- expect = "cop_task: {num: 2, max: 1s, min: 1ms, avg: 500.5ms, p95: 1s, max_proc_keys: 200, p95_proc_keys: 200, tot_proc: 1s, tot_wait: 1s, rpc_num: 1, rpc_time: 1s, copr_cache_hit_ratio: 0.00, distsql_concurrency: 15}, backoff{RegionMiss: 1ms}"
+ expect = "cop_task: {num: 2, max: 1s, min: 1ms, avg: 500.5ms, p95: 1s, max_proc_keys: 200, p95_proc_keys: 200, tot_proc: 1s, tot_wait: 1s, rpc_num: 1, rpc_time: 1s, copr_cache_hit_ratio: 0.00, max_distsql_concurrency: 15}, backoff{RegionMiss: 1ms}"
require.Equal(t, expect, stats.String())
// Test for idempotence.
require.Equal(t, expect, stats.String())
diff --git a/distsql/request_builder.go b/distsql/request_builder.go
index 4a8b3ddfeab13..8d0fae3a464f1 100644
--- a/distsql/request_builder.go
+++ b/distsql/request_builder.go
@@ -20,7 +20,6 @@ import (
"sort"
"sync/atomic"
- "github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/tidb/ddl/placement"
@@ -71,6 +70,9 @@ func (builder *RequestBuilder) Build() (*kv.Request, error) {
if err != nil {
builder.err = err
}
+ if builder.Request.KeyRanges == nil {
+ builder.Request.KeyRanges = kv.NewNonParitionedKeyRanges(nil)
+ }
return &builder.Request, builder.err
}
@@ -86,7 +88,7 @@ func (builder *RequestBuilder) SetMemTracker(tracker *memory.Tracker) *RequestBu
// br refers it, so have to keep it.
func (builder *RequestBuilder) SetTableRanges(tid int64, tableRanges []*ranger.Range, fb *statistics.QueryFeedback) *RequestBuilder {
if builder.err == nil {
- builder.Request.KeyRanges = TableRangesToKVRanges(tid, tableRanges, fb)
+ builder.Request.KeyRanges = kv.NewNonParitionedKeyRanges(TableRangesToKVRanges(tid, tableRanges, fb))
}
return builder
}
@@ -112,7 +114,9 @@ func (builder *RequestBuilder) SetIndexRangesForTables(sc *stmtctx.StatementCont
// SetHandleRanges sets "KeyRanges" for "kv.Request" by converting table handle range
// "ranges" to "KeyRanges" firstly.
func (builder *RequestBuilder) SetHandleRanges(sc *stmtctx.StatementContext, tid int64, isCommonHandle bool, ranges []*ranger.Range, fb *statistics.QueryFeedback) *RequestBuilder {
- return builder.SetHandleRangesForTables(sc, []int64{tid}, isCommonHandle, ranges, fb)
+ builder = builder.SetHandleRangesForTables(sc, []int64{tid}, isCommonHandle, ranges, fb)
+ builder.err = builder.Request.KeyRanges.SetToNonPartitioned()
+ return builder
}
// SetHandleRangesForTables sets "KeyRanges" for "kv.Request" by converting table handle range
@@ -127,14 +131,16 @@ func (builder *RequestBuilder) SetHandleRangesForTables(sc *stmtctx.StatementCon
// SetTableHandles sets "KeyRanges" for "kv.Request" by converting table handles
// "handles" to "KeyRanges" firstly.
func (builder *RequestBuilder) SetTableHandles(tid int64, handles []kv.Handle) *RequestBuilder {
- builder.Request.KeyRanges, builder.FixedRowCountHint = TableHandlesToKVRanges(tid, handles)
+ keyRanges, hints := TableHandlesToKVRanges(tid, handles)
+ builder.Request.KeyRanges = kv.NewNonParitionedKeyRangesWithHint(keyRanges, hints)
return builder
}
// SetPartitionsAndHandles sets "KeyRanges" for "kv.Request" by converting ParitionHandles to KeyRanges.
// handles in slice must be kv.PartitionHandle.
func (builder *RequestBuilder) SetPartitionsAndHandles(handles []kv.Handle) *RequestBuilder {
- builder.Request.KeyRanges = PartitionHandlesToKVRanges(handles)
+ keyRanges, hints := PartitionHandlesToKVRanges(handles)
+ builder.Request.KeyRanges = kv.NewNonParitionedKeyRangesWithHint(keyRanges, hints)
return builder
}
@@ -153,6 +159,7 @@ func (builder *RequestBuilder) SetDAGRequest(dag *tipb.DAGRequest) *RequestBuild
if limit != nil && limit.Limit < estimatedRegionRowCount {
builder.Request.Concurrency = 1
}
+ builder.Request.LimitSize = limit.GetLimit()
}
return builder
}
@@ -183,10 +190,28 @@ func (builder *RequestBuilder) SetChecksumRequest(checksum *tipb.ChecksumRequest
// SetKeyRanges sets "KeyRanges" for "kv.Request".
func (builder *RequestBuilder) SetKeyRanges(keyRanges []kv.KeyRange) *RequestBuilder {
+ builder.Request.KeyRanges = kv.NewNonParitionedKeyRanges(keyRanges)
+ return builder
+}
+
+// SetKeyRangesWithHints sets "KeyRanges" for "kv.Request" with row count hints.
+func (builder *RequestBuilder) SetKeyRangesWithHints(keyRanges []kv.KeyRange, hints []int) *RequestBuilder {
+ builder.Request.KeyRanges = kv.NewNonParitionedKeyRangesWithHint(keyRanges, hints)
+ return builder
+}
+
+// SetWrappedKeyRanges sets "KeyRanges" for "kv.Request".
+func (builder *RequestBuilder) SetWrappedKeyRanges(keyRanges *kv.KeyRanges) *RequestBuilder {
builder.Request.KeyRanges = keyRanges
return builder
}
+// SetPartitionKeyRanges sets the "KeyRanges" for "kv.Request" on partitioned table cases.
+func (builder *RequestBuilder) SetPartitionKeyRanges(keyRanges [][]kv.KeyRange) *RequestBuilder {
+ builder.Request.KeyRanges = kv.NewPartitionedKeyRanges(keyRanges)
+ return builder
+}
+
// SetStartTS sets "StartTS" for "kv.Request".
func (builder *RequestBuilder) SetStartTS(startTS uint64) *RequestBuilder {
builder.Request.StartTs = startTS
@@ -243,7 +268,8 @@ func (*RequestBuilder) getKVPriority(sv *variable.SessionVars) int {
}
// SetFromSessionVars sets the following fields for "kv.Request" from session variables:
-// "Concurrency", "IsolationLevel", "NotFillCache", "TaskID", "Priority", "ReplicaRead", "ResourceGroupTagger".
+// "Concurrency", "IsolationLevel", "NotFillCache", "TaskID", "Priority", "ReplicaRead",
+// "ResourceGroupTagger", "ResourceGroupName"
func (builder *RequestBuilder) SetFromSessionVars(sv *variable.SessionVars) *RequestBuilder {
if builder.Request.Concurrency == 0 {
// Concurrency may be set to 1 by SetDAGRequest
@@ -270,6 +296,8 @@ func (builder *RequestBuilder) SetFromSessionVars(sv *variable.SessionVars) *Req
}
builder.RequestSource.RequestSourceInternal = sv.InRestrictedSQL
builder.RequestSource.RequestSourceType = sv.RequestSourceType
+ builder.StoreBatchSize = sv.StoreBatchSize
+ builder.Request.ResourceGroupName = sv.ResourceGroupName
return builder
}
@@ -312,19 +340,24 @@ func (builder *RequestBuilder) SetResourceGroupTagger(tagger tikvrpc.ResourceGro
return builder
}
+// SetResourceGroupName sets the request resource group name.
+func (builder *RequestBuilder) SetResourceGroupName(name string) *RequestBuilder {
+ builder.Request.ResourceGroupName = name
+ return builder
+}
+
func (builder *RequestBuilder) verifyTxnScope() error {
txnScope := builder.TxnScope
if txnScope == "" || txnScope == kv.GlobalReplicaScope || builder.is == nil {
return nil
}
visitPhysicalTableID := make(map[int64]struct{})
- for _, keyRange := range builder.Request.KeyRanges {
- tableID := tablecodec.DecodeTableID(keyRange.StartKey)
- if tableID > 0 {
- visitPhysicalTableID[tableID] = struct{}{}
- } else {
- return errors.New("requestBuilder can't decode tableID from keyRange")
- }
+ tids, err := tablecodec.VerifyTableIDForRanges(builder.Request.KeyRanges)
+ if err != nil {
+ return err
+ }
+ for _, tid := range tids {
+ visitPhysicalTableID[tid] = struct{}{}
}
for phyTableID := range visitPhysicalTableID {
@@ -376,7 +409,7 @@ func (builder *RequestBuilder) SetClosestReplicaReadAdjuster(chkFn kv.CoprReques
}
// TableHandleRangesToKVRanges convert table handle ranges to "KeyRanges" for multiple tables.
-func TableHandleRangesToKVRanges(sc *stmtctx.StatementContext, tid []int64, isCommonHandle bool, ranges []*ranger.Range, fb *statistics.QueryFeedback) ([]kv.KeyRange, error) {
+func TableHandleRangesToKVRanges(sc *stmtctx.StatementContext, tid []int64, isCommonHandle bool, ranges []*ranger.Range, fb *statistics.QueryFeedback) (*kv.KeyRanges, error) {
if !isCommonHandle {
return tablesRangesToKVRanges(tid, ranges, fb), nil
}
@@ -387,14 +420,18 @@ func TableHandleRangesToKVRanges(sc *stmtctx.StatementContext, tid []int64, isCo
// Note this function should not be exported, but currently
// br refers to it, so have to keep it.
func TableRangesToKVRanges(tid int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) []kv.KeyRange {
- return tablesRangesToKVRanges([]int64{tid}, ranges, fb)
+ if len(ranges) == 0 {
+ return []kv.KeyRange{}
+ }
+ return tablesRangesToKVRanges([]int64{tid}, ranges, fb).FirstPartitionRange()
}
// tablesRangesToKVRanges converts table ranges to "KeyRange".
-func tablesRangesToKVRanges(tids []int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) []kv.KeyRange {
+func tablesRangesToKVRanges(tids []int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) *kv.KeyRanges {
if fb == nil || fb.Hist == nil {
return tableRangesToKVRangesWithoutSplit(tids, ranges)
}
+ // The following codes are deprecated since the feedback is deprecated.
krs := make([]kv.KeyRange, 0, len(ranges))
feedbackRanges := make([]*ranger.Range, 0, len(ranges))
for _, ran := range ranges {
@@ -420,20 +457,23 @@ func tablesRangesToKVRanges(tids []int64, ranges []*ranger.Range, fb *statistics
}
}
fb.StoreRanges(feedbackRanges)
- return krs
+ return kv.NewNonParitionedKeyRanges(krs)
}
-func tableRangesToKVRangesWithoutSplit(tids []int64, ranges []*ranger.Range) []kv.KeyRange {
- krs := make([]kv.KeyRange, 0, len(ranges)*len(tids))
+func tableRangesToKVRangesWithoutSplit(tids []int64, ranges []*ranger.Range) *kv.KeyRanges {
+ krs := make([][]kv.KeyRange, len(tids))
+ for i := range krs {
+ krs[i] = make([]kv.KeyRange, 0, len(ranges))
+ }
for _, ran := range ranges {
low, high := encodeHandleKey(ran)
- for _, tid := range tids {
+ for i, tid := range tids {
startKey := tablecodec.EncodeRowKey(tid, low)
endKey := tablecodec.EncodeRowKey(tid, high)
- krs = append(krs, kv.KeyRange{StartKey: startKey, EndKey: endKey})
+ krs[i] = append(krs[i], kv.KeyRange{StartKey: startKey, EndKey: endKey})
}
}
- return krs
+ return kv.NewPartitionedKeyRanges(krs)
}
func encodeHandleKey(ran *ranger.Range) ([]byte, []byte) {
@@ -517,7 +557,7 @@ func SplitRangesAcrossInt64Boundary(ranges []*ranger.Range, keepOrder bool, desc
// For continuous handles, we should merge them to a single key range.
func TableHandlesToKVRanges(tid int64, handles []kv.Handle) ([]kv.KeyRange, []int) {
krs := make([]kv.KeyRange, 0, len(handles))
- hint := make([]int, 0, len(handles))
+ hints := make([]int, 0, len(handles))
i := 0
for i < len(handles) {
if commonHandle, ok := handles[i].(*kv.CommonHandle); ok {
@@ -526,7 +566,7 @@ func TableHandlesToKVRanges(tid int64, handles []kv.Handle) ([]kv.KeyRange, []in
EndKey: tablecodec.EncodeRowKey(tid, kv.Key(commonHandle.Encoded()).Next()),
}
krs = append(krs, ran)
- hint = append(hint, 1)
+ hints = append(hints, 1)
i++
continue
}
@@ -542,16 +582,17 @@ func TableHandlesToKVRanges(tid int64, handles []kv.Handle) ([]kv.KeyRange, []in
startKey := tablecodec.EncodeRowKey(tid, low)
endKey := tablecodec.EncodeRowKey(tid, high)
krs = append(krs, kv.KeyRange{StartKey: startKey, EndKey: endKey})
- hint = append(hint, j-i)
+ hints = append(hints, j-i)
i = j
}
- return krs, hint
+ return krs, hints
}
// PartitionHandlesToKVRanges convert ParitionHandles to kv ranges.
// Handle in slices must be kv.PartitionHandle
-func PartitionHandlesToKVRanges(handles []kv.Handle) []kv.KeyRange {
+func PartitionHandlesToKVRanges(handles []kv.Handle) ([]kv.KeyRange, []int) {
krs := make([]kv.KeyRange, 0, len(handles))
+ hints := make([]int, 0, len(handles))
i := 0
for i < len(handles) {
ph := handles[i].(kv.PartitionHandle)
@@ -563,6 +604,7 @@ func PartitionHandlesToKVRanges(handles []kv.Handle) []kv.KeyRange {
EndKey: tablecodec.EncodeRowKey(pid, append(commonHandle.Encoded(), 0)),
}
krs = append(krs, ran)
+ hints = append(hints, 1)
i++
continue
}
@@ -581,33 +623,40 @@ func PartitionHandlesToKVRanges(handles []kv.Handle) []kv.KeyRange {
startKey := tablecodec.EncodeRowKey(pid, low)
endKey := tablecodec.EncodeRowKey(pid, high)
krs = append(krs, kv.KeyRange{StartKey: startKey, EndKey: endKey})
+ hints = append(hints, j-i)
i = j
}
- return krs
+ return krs, hints
}
// IndexRangesToKVRanges converts index ranges to "KeyRange".
-func IndexRangesToKVRanges(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) ([]kv.KeyRange, error) {
+func IndexRangesToKVRanges(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) (*kv.KeyRanges, error) {
return IndexRangesToKVRangesWithInterruptSignal(sc, tid, idxID, ranges, fb, nil, nil)
}
// IndexRangesToKVRangesWithInterruptSignal converts index ranges to "KeyRange".
// The process can be interrupted by set `interruptSignal` to true.
-func IndexRangesToKVRangesWithInterruptSignal(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback, memTracker *memory.Tracker, interruptSignal *atomic.Value) ([]kv.KeyRange, error) {
- return indexRangesToKVRangesForTablesWithInterruptSignal(sc, []int64{tid}, idxID, ranges, fb, memTracker, interruptSignal)
+func IndexRangesToKVRangesWithInterruptSignal(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback, memTracker *memory.Tracker, interruptSignal *atomic.Value) (*kv.KeyRanges, error) {
+ keyRanges, err := indexRangesToKVRangesForTablesWithInterruptSignal(sc, []int64{tid}, idxID, ranges, fb, memTracker, interruptSignal)
+ if err != nil {
+ return nil, err
+ }
+ err = keyRanges.SetToNonPartitioned()
+ return keyRanges, err
}
// IndexRangesToKVRangesForTables converts indexes ranges to "KeyRange".
-func IndexRangesToKVRangesForTables(sc *stmtctx.StatementContext, tids []int64, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) ([]kv.KeyRange, error) {
+func IndexRangesToKVRangesForTables(sc *stmtctx.StatementContext, tids []int64, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) (*kv.KeyRanges, error) {
return indexRangesToKVRangesForTablesWithInterruptSignal(sc, tids, idxID, ranges, fb, nil, nil)
}
// IndexRangesToKVRangesForTablesWithInterruptSignal converts indexes ranges to "KeyRange".
// The process can be interrupted by set `interruptSignal` to true.
-func indexRangesToKVRangesForTablesWithInterruptSignal(sc *stmtctx.StatementContext, tids []int64, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback, memTracker *memory.Tracker, interruptSignal *atomic.Value) ([]kv.KeyRange, error) {
+func indexRangesToKVRangesForTablesWithInterruptSignal(sc *stmtctx.StatementContext, tids []int64, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback, memTracker *memory.Tracker, interruptSignal *atomic.Value) (*kv.KeyRanges, error) {
if fb == nil || fb.Hist == nil {
return indexRangesToKVWithoutSplit(sc, tids, idxID, ranges, memTracker, interruptSignal)
}
+ // The following code is non maintained since the feedback deprecated.
feedbackRanges := make([]*ranger.Range, 0, len(ranges))
for _, ran := range ranges {
low, high, err := EncodeIndexKey(sc, ran)
@@ -642,11 +691,11 @@ func indexRangesToKVRangesForTablesWithInterruptSignal(sc *stmtctx.StatementCont
}
}
fb.StoreRanges(feedbackRanges)
- return krs, nil
+ return kv.NewNonParitionedKeyRanges(krs), nil
}
// CommonHandleRangesToKVRanges converts common handle ranges to "KeyRange".
-func CommonHandleRangesToKVRanges(sc *stmtctx.StatementContext, tids []int64, ranges []*ranger.Range) ([]kv.KeyRange, error) {
+func CommonHandleRangesToKVRanges(sc *stmtctx.StatementContext, tids []int64, ranges []*ranger.Range) (*kv.KeyRanges, error) {
rans := make([]*ranger.Range, 0, len(ranges))
for _, ran := range ranges {
low, high, err := EncodeIndexKey(sc, ran)
@@ -656,20 +705,23 @@ func CommonHandleRangesToKVRanges(sc *stmtctx.StatementContext, tids []int64, ra
rans = append(rans, &ranger.Range{LowVal: []types.Datum{types.NewBytesDatum(low)},
HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true, Collators: collate.GetBinaryCollatorSlice(1)})
}
- krs := make([]kv.KeyRange, 0, len(rans))
+ krs := make([][]kv.KeyRange, len(tids))
+ for i := range krs {
+ krs[i] = make([]kv.KeyRange, 0, len(ranges))
+ }
for _, ran := range rans {
low, high := ran.LowVal[0].GetBytes(), ran.HighVal[0].GetBytes()
if ran.LowExclude {
low = kv.Key(low).PrefixNext()
}
ran.LowVal[0].SetBytes(low)
- for _, tid := range tids {
+ for i, tid := range tids {
startKey := tablecodec.EncodeRowKey(tid, low)
endKey := tablecodec.EncodeRowKey(tid, high)
- krs = append(krs, kv.KeyRange{StartKey: startKey, EndKey: endKey})
+ krs[i] = append(krs[i], kv.KeyRange{StartKey: startKey, EndKey: endKey})
}
}
- return krs, nil
+ return kv.NewPartitionedKeyRanges(krs), nil
}
// VerifyTxnScope verify whether the txnScope and visited physical table break the leader rule's dcLocation.
@@ -691,8 +743,12 @@ func VerifyTxnScope(txnScope string, physicalTableID int64, is infoschema.InfoSc
return true
}
-func indexRangesToKVWithoutSplit(sc *stmtctx.StatementContext, tids []int64, idxID int64, ranges []*ranger.Range, memTracker *memory.Tracker, interruptSignal *atomic.Value) ([]kv.KeyRange, error) {
- krs := make([]kv.KeyRange, 0, len(ranges))
+func indexRangesToKVWithoutSplit(sc *stmtctx.StatementContext, tids []int64, idxID int64, ranges []*ranger.Range, memTracker *memory.Tracker, interruptSignal *atomic.Value) (*kv.KeyRanges, error) {
+ krs := make([][]kv.KeyRange, len(tids))
+ for i := range krs {
+ krs[i] = make([]kv.KeyRange, 0, len(ranges))
+ }
+
const checkSignalStep = 8
var estimatedMemUsage int64
// encodeIndexKey and EncodeIndexSeekKey is time-consuming, thus we need to
@@ -705,13 +761,13 @@ func indexRangesToKVWithoutSplit(sc *stmtctx.StatementContext, tids []int64, idx
if i == 0 {
estimatedMemUsage += int64(cap(low) + cap(high))
}
- for _, tid := range tids {
+ for j, tid := range tids {
startKey := tablecodec.EncodeIndexSeekKey(tid, idxID, low)
endKey := tablecodec.EncodeIndexSeekKey(tid, idxID, high)
if i == 0 {
estimatedMemUsage += int64(cap(startKey)) + int64(cap(endKey))
}
- krs = append(krs, kv.KeyRange{StartKey: startKey, EndKey: endKey})
+ krs[j] = append(krs[j], kv.KeyRange{StartKey: startKey, EndKey: endKey})
}
if i%checkSignalStep == 0 {
if i == 0 && memTracker != nil {
@@ -719,11 +775,11 @@ func indexRangesToKVWithoutSplit(sc *stmtctx.StatementContext, tids []int64, idx
memTracker.Consume(estimatedMemUsage)
}
if interruptSignal != nil && interruptSignal.Load().(bool) {
- return nil, nil
+ return kv.NewPartitionedKeyRanges(nil), nil
}
}
}
- return krs, nil
+ return kv.NewPartitionedKeyRanges(krs), nil
}
// EncodeIndexKey gets encoded keys containing low and high
@@ -743,20 +799,5 @@ func EncodeIndexKey(sc *stmtctx.StatementContext, ran *ranger.Range) ([]byte, []
if !ran.HighExclude {
high = kv.Key(high).PrefixNext()
}
-
- var hasNull bool
- for _, highVal := range ran.HighVal {
- if highVal.IsNull() {
- hasNull = true
- break
- }
- }
-
- // NOTE: this is a hard-code operation to avoid wrong results when accessing unique index with NULL;
- // Please see https://github.com/pingcap/tidb/issues/29650 for more details
- if hasNull {
- // Append 0 to make unique-key range [null, null] to be a scan rather than point-get.
- high = kv.Key(high).Next()
- }
return low, high, nil
}
diff --git a/distsql/request_builder_test.go b/distsql/request_builder_test.go
index 2ffde4a512c0d..1de6b11dd5244 100644
--- a/distsql/request_builder_test.go
+++ b/distsql/request_builder_test.go
@@ -61,10 +61,11 @@ func TestTableHandlesToKVRanges(t *testing.T) {
// Build key ranges.
expect := getExpectedRanges(1, hrs)
- actual, _ := TableHandlesToKVRanges(1, handles)
+ actual, hints := TableHandlesToKVRanges(1, handles)
// Compare key ranges and expected key ranges.
require.Equal(t, len(expect), len(actual))
+ require.Equal(t, hints, []int{1, 4, 2, 1, 2})
for i := range actual {
require.Equal(t, expect[i].StartKey, actual[i].StartKey)
require.Equal(t, expect[i].EndKey, actual[i].EndKey)
@@ -192,8 +193,8 @@ func TestIndexRangesToKVRanges(t *testing.T) {
actual, err := IndexRangesToKVRanges(new(stmtctx.StatementContext), 12, 15, ranges, nil)
require.NoError(t, err)
- for i := range actual {
- require.Equal(t, expect[i], actual[i])
+ for i := range actual.FirstPartitionRange() {
+ require.Equal(t, expect[i], actual.FirstPartitionRange()[i])
}
}
@@ -242,7 +243,7 @@ func TestRequestBuilder1(t *testing.T) {
Tp: 103,
StartTs: 0x0,
Data: []uint8{0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0},
- KeyRanges: []kv.KeyRange{
+ KeyRanges: kv.NewNonParitionedKeyRanges([]kv.KeyRange{
{
StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
EndKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3},
@@ -263,7 +264,7 @@ func TestRequestBuilder1(t *testing.T) {
StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x23},
EndKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x23},
},
- },
+ }),
Cacheable: true,
KeepOrder: false,
Desc: false,
@@ -325,7 +326,7 @@ func TestRequestBuilder2(t *testing.T) {
Tp: 103,
StartTs: 0x0,
Data: []uint8{0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0},
- KeyRanges: []kv.KeyRange{
+ KeyRanges: kv.NewNonParitionedKeyRanges([]kv.KeyRange{
{
StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
EndKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3},
@@ -346,7 +347,7 @@ func TestRequestBuilder2(t *testing.T) {
StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x23},
EndKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x23},
},
- },
+ }),
Cacheable: true,
KeepOrder: false,
Desc: false,
@@ -378,7 +379,7 @@ func TestRequestBuilder3(t *testing.T) {
Tp: 103,
StartTs: 0x0,
Data: []uint8{0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0},
- KeyRanges: []kv.KeyRange{
+ KeyRanges: kv.NewNonParitionedKeyRangesWithHint([]kv.KeyRange{
{
StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
EndKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
@@ -395,17 +396,16 @@ func TestRequestBuilder3(t *testing.T) {
StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x64},
EndKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x65},
},
- },
- Cacheable: true,
- KeepOrder: false,
- Desc: false,
- Concurrency: variable.DefDistSQLScanConcurrency,
- IsolationLevel: 0,
- Priority: 0,
- NotFillCache: false,
- ReplicaRead: kv.ReplicaReadLeader,
- ReadReplicaScope: kv.GlobalReplicaScope,
- FixedRowCountHint: []int{1, 4, 2, 1},
+ }, []int{1, 4, 2, 1}),
+ Cacheable: true,
+ KeepOrder: false,
+ Desc: false,
+ Concurrency: variable.DefDistSQLScanConcurrency,
+ IsolationLevel: 0,
+ Priority: 0,
+ NotFillCache: false,
+ ReplicaRead: kv.ReplicaReadLeader,
+ ReadReplicaScope: kv.GlobalReplicaScope,
}
expect.Paging.MinPagingSize = paging.MinPagingSize
expect.Paging.MaxPagingSize = paging.MaxPagingSize
@@ -444,7 +444,7 @@ func TestRequestBuilder4(t *testing.T) {
Tp: 103,
StartTs: 0x0,
Data: []uint8{0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0},
- KeyRanges: keyRanges,
+ KeyRanges: kv.NewNonParitionedKeyRanges(keyRanges),
Cacheable: true,
KeepOrder: false,
Desc: false,
@@ -491,7 +491,7 @@ func TestRequestBuilder5(t *testing.T) {
Tp: 104,
StartTs: 0x0,
Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0},
- KeyRanges: keyRanges,
+ KeyRanges: kv.NewNonParitionedKeyRanges(keyRanges),
KeepOrder: true,
Desc: false,
Concurrency: 15,
@@ -520,7 +520,7 @@ func TestRequestBuilder6(t *testing.T) {
Tp: 105,
StartTs: 0x0,
Data: []uint8{0x10, 0x0, 0x18, 0x0},
- KeyRanges: keyRanges,
+ KeyRanges: kv.NewNonParitionedKeyRanges(keyRanges),
KeepOrder: false,
Desc: false,
Concurrency: concurrency,
@@ -557,6 +557,7 @@ func TestRequestBuilder7(t *testing.T) {
Tp: 0,
StartTs: 0x0,
KeepOrder: false,
+ KeyRanges: kv.NewNonParitionedKeyRanges(nil),
Desc: false,
Concurrency: concurrency,
IsolationLevel: 0,
@@ -575,20 +576,23 @@ func TestRequestBuilder7(t *testing.T) {
func TestRequestBuilder8(t *testing.T) {
sv := variable.NewSessionVars(nil)
+ sv.ResourceGroupName = "test"
actual, err := (&RequestBuilder{}).
SetFromSessionVars(sv).
Build()
require.NoError(t, err)
expect := &kv.Request{
- Tp: 0,
- StartTs: 0x0,
- Data: []uint8(nil),
- Concurrency: variable.DefDistSQLScanConcurrency,
- IsolationLevel: 0,
- Priority: 0,
- MemTracker: (*memory.Tracker)(nil),
- SchemaVar: 0,
- ReadReplicaScope: kv.GlobalReplicaScope,
+ Tp: 0,
+ StartTs: 0x0,
+ Data: []uint8(nil),
+ KeyRanges: kv.NewNonParitionedKeyRanges(nil),
+ Concurrency: variable.DefDistSQLScanConcurrency,
+ IsolationLevel: 0,
+ Priority: 0,
+ MemTracker: (*memory.Tracker)(nil),
+ SchemaVar: 0,
+ ReadReplicaScope: kv.GlobalReplicaScope,
+ ResourceGroupName: "test",
}
expect.Paging.MinPagingSize = paging.MinPagingSize
expect.Paging.MaxPagingSize = paging.MaxPagingSize
@@ -635,8 +639,8 @@ func TestIndexRangesToKVRangesWithFbs(t *testing.T) {
EndKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5},
},
}
- for i := 0; i < len(actual); i++ {
- require.Equal(t, expect[i], actual[i])
+ for i := 0; i < len(actual.FirstPartitionRange()); i++ {
+ require.Equal(t, expect[i], actual.FirstPartitionRange()[i])
}
}
@@ -672,6 +676,7 @@ func TestScanLimitConcurrency(t *testing.T) {
Build()
require.NoError(t, err)
require.Equal(t, tt.concurrency, actual.Concurrency)
+ require.Equal(t, actual.LimitSize, tt.limit)
})
}
}
diff --git a/distsql/select_result.go b/distsql/select_result.go
index 0e807b360d0ad..394298e8fa3b0 100644
--- a/distsql/select_result.go
+++ b/distsql/select_result.go
@@ -53,6 +53,12 @@ var (
errQueryInterrupted = dbterror.ClassExecutor.NewStd(errno.ErrQueryInterrupted)
)
+var (
+ telemetryBatchedQueryTaskCnt = metrics.TelemetryBatchedQueryTaskCnt
+ telemetryStoreBatchedCnt = metrics.TelemetryStoreBatchedCnt
+ telemetryStoreBatchedFallbackCnt = metrics.TelemetryStoreBatchedFallbackCnt
+)
+
var (
_ SelectResult = (*selectResult)(nil)
_ SelectResult = (*serialSelectResults)(nil)
@@ -157,7 +163,7 @@ func (r *selectResult) fetchResp(ctx context.Context) error {
if r.stats != nil {
// Ignore internal sql.
if !r.ctx.GetSessionVars().InRestrictedSQL && len(r.stats.copRespTime) > 0 {
- ratio := float64(r.stats.CoprCacheHitNum) / float64(len(r.stats.copRespTime))
+ ratio := r.stats.calcCacheHit()
if ratio >= 1 {
telemetry.CurrentCoprCacheHitRatioGTE100Count.Inc()
}
@@ -359,13 +365,16 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr
}
if r.stats == nil {
- id := r.rootPlanID
r.stats = &selectResultRuntimeStats{
backoffSleep: make(map[string]time.Duration),
rpcStat: tikv.NewRegionRequestRuntimeStats(),
distSQLConcurrency: r.distSQLConcurrency,
}
- r.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(id, r.stats)
+ if ci, ok := r.resp.(copr.CopInfo); ok {
+ conc, extraConc := ci.GetConcurrency()
+ r.stats.distSQLConcurrency = conc
+ r.stats.extraConcurrency = extraConc
+ }
}
r.stats.mergeCopRuntimeStats(copStats, respTime)
@@ -456,24 +465,43 @@ func (r *selectResult) Close() error {
if respSize > 0 {
r.memConsume(-respSize)
}
+ if r.stats != nil {
+ defer func() {
+ if ci, ok := r.resp.(copr.CopInfo); ok {
+ r.stats.buildTaskDuration = ci.GetBuildTaskElapsed()
+ batched, fallback := ci.GetStoreBatchInfo()
+ if batched != 0 || fallback != 0 {
+ r.stats.storeBatchedNum, r.stats.storeBatchedFallbackNum = batched, fallback
+ telemetryStoreBatchedCnt.Add(float64(r.stats.storeBatchedNum))
+ telemetryStoreBatchedFallbackCnt.Add(float64(r.stats.storeBatchedFallbackNum))
+ telemetryBatchedQueryTaskCnt.Add(float64(len(r.stats.copRespTime)))
+ }
+ }
+ r.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(r.rootPlanID, r.stats)
+ }()
+ }
return r.resp.Close()
}
-// CopRuntimeStats is a interface uses to check whether the result has cop runtime stats.
+// CopRuntimeStats is an interface uses to check whether the result has cop runtime stats.
type CopRuntimeStats interface {
// GetCopRuntimeStats gets the cop runtime stats information.
GetCopRuntimeStats() *copr.CopRuntimeStats
}
type selectResultRuntimeStats struct {
- copRespTime []time.Duration
- procKeys []int64
- backoffSleep map[string]time.Duration
- totalProcessTime time.Duration
- totalWaitTime time.Duration
- rpcStat tikv.RegionRequestRuntimeStats
- distSQLConcurrency int
- CoprCacheHitNum int64
+ copRespTime []time.Duration
+ procKeys []int64
+ backoffSleep map[string]time.Duration
+ totalProcessTime time.Duration
+ totalWaitTime time.Duration
+ rpcStat tikv.RegionRequestRuntimeStats
+ distSQLConcurrency int
+ extraConcurrency int
+ CoprCacheHitNum int64
+ storeBatchedNum uint64
+ storeBatchedFallbackNum uint64
+ buildTaskDuration time.Duration
}
func (s *selectResultRuntimeStats) mergeCopRuntimeStats(copStats *copr.CopRuntimeStats, respTime time.Duration) {
@@ -494,12 +522,16 @@ func (s *selectResultRuntimeStats) mergeCopRuntimeStats(copStats *copr.CopRuntim
func (s *selectResultRuntimeStats) Clone() execdetails.RuntimeStats {
newRs := selectResultRuntimeStats{
- copRespTime: make([]time.Duration, 0, len(s.copRespTime)),
- procKeys: make([]int64, 0, len(s.procKeys)),
- backoffSleep: make(map[string]time.Duration, len(s.backoffSleep)),
- rpcStat: tikv.NewRegionRequestRuntimeStats(),
- distSQLConcurrency: s.distSQLConcurrency,
- CoprCacheHitNum: s.CoprCacheHitNum,
+ copRespTime: make([]time.Duration, 0, len(s.copRespTime)),
+ procKeys: make([]int64, 0, len(s.procKeys)),
+ backoffSleep: make(map[string]time.Duration, len(s.backoffSleep)),
+ rpcStat: tikv.NewRegionRequestRuntimeStats(),
+ distSQLConcurrency: s.distSQLConcurrency,
+ extraConcurrency: s.extraConcurrency,
+ CoprCacheHitNum: s.CoprCacheHitNum,
+ storeBatchedNum: s.storeBatchedNum,
+ storeBatchedFallbackNum: s.storeBatchedFallbackNum,
+ buildTaskDuration: s.buildTaskDuration,
}
newRs.copRespTime = append(newRs.copRespTime, s.copRespTime...)
newRs.procKeys = append(newRs.procKeys, s.procKeys...)
@@ -527,6 +559,15 @@ func (s *selectResultRuntimeStats) Merge(rs execdetails.RuntimeStats) {
s.totalWaitTime += other.totalWaitTime
s.rpcStat.Merge(other.rpcStat)
s.CoprCacheHitNum += other.CoprCacheHitNum
+ if other.distSQLConcurrency > s.distSQLConcurrency {
+ s.distSQLConcurrency = other.distSQLConcurrency
+ }
+ if other.extraConcurrency > s.extraConcurrency {
+ s.extraConcurrency = other.extraConcurrency
+ }
+ s.storeBatchedNum += other.storeBatchedNum
+ s.storeBatchedFallbackNum += other.storeBatchedFallbackNum
+ s.buildTaskDuration += other.buildTaskDuration
}
func (s *selectResultRuntimeStats) String() string {
@@ -578,14 +619,30 @@ func (s *selectResultRuntimeStats) String() string {
}
if config.GetGlobalConfig().TiKVClient.CoprCache.CapacityMB > 0 {
buf.WriteString(fmt.Sprintf(", copr_cache_hit_ratio: %v",
- strconv.FormatFloat(float64(s.CoprCacheHitNum)/float64(len(s.copRespTime)), 'f', 2, 64)))
+ strconv.FormatFloat(s.calcCacheHit(), 'f', 2, 64)))
} else {
buf.WriteString(", copr_cache: disabled")
}
+ if s.buildTaskDuration > 0 {
+ buf.WriteString(", build_task_duration: ")
+ buf.WriteString(execdetails.FormatDuration(s.buildTaskDuration))
+ }
if s.distSQLConcurrency > 0 {
- buf.WriteString(", distsql_concurrency: ")
+ buf.WriteString(", max_distsql_concurrency: ")
buf.WriteString(strconv.FormatInt(int64(s.distSQLConcurrency), 10))
}
+ if s.extraConcurrency > 0 {
+ buf.WriteString(", max_extra_concurrency: ")
+ buf.WriteString(strconv.FormatInt(int64(s.extraConcurrency), 10))
+ }
+ if s.storeBatchedNum > 0 {
+ buf.WriteString(", store_batch_num: ")
+ buf.WriteString(strconv.FormatInt(int64(s.storeBatchedNum), 10))
+ }
+ if s.storeBatchedFallbackNum > 0 {
+ buf.WriteString(", store_batch_fallback_num: ")
+ buf.WriteString(strconv.FormatInt(int64(s.storeBatchedFallbackNum), 10))
+ }
buf.WriteString("}")
}
@@ -614,3 +671,15 @@ func (s *selectResultRuntimeStats) String() string {
func (*selectResultRuntimeStats) Tp() int {
return execdetails.TpSelectResultRuntimeStats
}
+
+func (s *selectResultRuntimeStats) calcCacheHit() float64 {
+ hit := s.CoprCacheHitNum
+ tot := len(s.copRespTime)
+ if s.storeBatchedNum > 0 {
+ tot += int(s.storeBatchedNum)
+ }
+ if tot == 0 {
+ return 0
+ }
+ return float64(hit) / float64(tot)
+}
diff --git a/distsql/select_result_test.go b/distsql/select_result_test.go
index c12892083d641..4ec56a286e5ab 100644
--- a/distsql/select_result_test.go
+++ b/distsql/select_result_test.go
@@ -34,7 +34,7 @@ func TestUpdateCopRuntimeStats(t *testing.T) {
require.Nil(t, ctx.GetSessionVars().StmtCtx.RuntimeStatsColl)
sr.rootPlanID = 1234
- sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ExecDetails: execdetails.ExecDetails{CalleeAddress: "a"}}, 0)
+ sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ExecDetails: execdetails.ExecDetails{DetailsNeedP90: execdetails.DetailsNeedP90{CalleeAddress: "a"}}}, 0)
ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil)
i := uint64(1)
@@ -46,13 +46,13 @@ func TestUpdateCopRuntimeStats(t *testing.T) {
require.NotEqual(t, len(sr.copPlanIDs), len(sr.selectResp.GetExecutionSummaries()))
- sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ExecDetails: execdetails.ExecDetails{CalleeAddress: "callee"}}, 0)
+ sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ExecDetails: execdetails.ExecDetails{DetailsNeedP90: execdetails.DetailsNeedP90{CalleeAddress: "callee"}}}, 0)
require.False(t, ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.ExistsCopStats(1234))
sr.copPlanIDs = []int{sr.rootPlanID}
require.NotNil(t, ctx.GetSessionVars().StmtCtx.RuntimeStatsColl)
require.Equal(t, len(sr.copPlanIDs), len(sr.selectResp.GetExecutionSummaries()))
- sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ExecDetails: execdetails.ExecDetails{CalleeAddress: "callee"}}, 0)
+ sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ExecDetails: execdetails.ExecDetails{DetailsNeedP90: execdetails.DetailsNeedP90{CalleeAddress: "callee"}}}, 0)
require.Equal(t, "tikv_task:{time:1ns, loops:1}", ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetOrCreateCopStats(1234, "tikv").String())
}
diff --git a/docs/design/2019-11-05-index-advisor.md b/docs/design/2019-11-05-index-advisor.md
index 53abab5ba0d5a..5606d9bd9a942 100644
--- a/docs/design/2019-11-05-index-advisor.md
+++ b/docs/design/2019-11-05-index-advisor.md
@@ -57,7 +57,7 @@ for {
Note that executing `Swap and Re-evaluate` algorithm is necessary as the `reduced_cost` sometimes is a joint effect of several indexes and it's hard to tell each index's independent contribution to the final `reduced_cost`. For example, assume there is an extremely slow query in input workload and the desired indexes for this query is `a` and `b`. However, the number of allowed recommended indexes for the whole workload is limited and for some reason, `a` ranks top `n` in the final score list while `b` is not. But there are chances that without `b`, `a` can no more optimize that extremely slow query.
----------------------------------------------
-### A quick exmaple for single-column index recommendation
+### A quick example for single-column index recommendation
**Workload**:
diff --git a/docs/design/2020-01-24-collations.md b/docs/design/2020-01-24-collations.md
index a222035745b83..d610514ed76d5 100644
--- a/docs/design/2020-01-24-collations.md
+++ b/docs/design/2020-01-24-collations.md
@@ -105,10 +105,10 @@ The interface is quite similar to the Go [collate package](https://godoc.org/gol
### Row Format
-The encoding layout of TiDB has been described in our [previous article](https://pingcap.com/blog/2017-07-11-tidbinternal2/#map). The row format should be changed to make it memory comparable, this is important to the index lookup. Basic principle is that all keys encoded for strings should use the `sortKeys` result from `Key()`/`KeyFromString()` function. However, most of the `sortKeys` calculations are not reversible.
+The encoding layout of TiDB has been described in our [previous article](https://docs.pingcap.com/tidb/stable/tidb-computing). The row format should be changed to make it memory comparable, this is important to the index lookup. Basic principle is that all keys encoded for strings should use the `sortKeys` result from `Key()`/`KeyFromString()` function. However, most of the `sortKeys` calculations are not reversible.
* For table data, encodings stay unchanged. All strings are compared after decoding with the `Compare()` function.
- * For table indices, we replace current `ColumnValue` with `sortKey` and encode the `ColumnValue` to the value,:
+ * For table indices, we replace current `ColumnValue` with `sortKey` and encode the `ColumnValue` to the value:
- For unique indices:
```
Key: tablePrefix{tableID}_indexPrefixSep{indexID}_sortKey
@@ -233,7 +233,7 @@ The following features of the general collation algorithm will be supported:
* Tertiary Weight i.e. case
* PAD / NOPAD
-All of them are supported by `text/collate` package of Go, so it is possible to map Go collations to some of UCA-based collations in MySQL like `utf8mb4_unicode_ci`/`utf8mb4_0900_ai_ci`, if we ignore the differences between UCA versions: current `text/collate` uses UCA version `6.2.0` and it is not changable. However, the collations in MySQL are with different UCA versions marked in the names, for example, `utf8mb4_0900_ai_ci` uses version `9.0`.
+All of them are supported by `text/collate` package of Go, so it is possible to map Go collations to some of UCA-based collations in MySQL like `utf8mb4_unicode_ci`/`utf8mb4_0900_ai_ci`, if we ignore the differences between UCA versions: current `text/collate` uses UCA version `6.2.0` and it is not changeable. However, the collations in MySQL are with different UCA versions marked in the names, for example, `utf8mb4_0900_ai_ci` uses version `9.0`.
For non-standard UCA implementations in MySQL, i.e. the `utf8mb4_general_ci`. The implementation depends on our choice to the [Compatibility with MySQL](#compatibility-with-mysql) chapter, if a 100% compatibility of `utf8mb4_general_ci` is chosen, we need to implement it by our hands.
diff --git a/docs/design/2020-06-24-placement-rules-in-sql.md b/docs/design/2020-06-24-placement-rules-in-sql.md
index 207e6a8a4a2eb..30191f676badb 100644
--- a/docs/design/2020-06-24-placement-rules-in-sql.md
+++ b/docs/design/2020-06-24-placement-rules-in-sql.md
@@ -413,6 +413,26 @@ If a table is imported when `tidb_placement_mode='IGNORE'`, and the placement po
The default value for `tidb_placement_mode` is `STRICT`. The option is an enum, and in future we may add support for a `WARN` mode.
+
+#### Survival preference
+
+Some important data may need to store multiple copies across availability zones, so as to have high disaster recovery survivability, such as region-level survivability, `SURVIVAL_PREFERENCES` can provide survivability preference settings.
+
+The following example sets a constraint that the data try to satisfy the Survival Preferences setting:
+
+``` sql
+CREATE PLACEMENT POLICY multiregion
+ follower=4
+ PRIMARY_REGION="region1"
+ SURVIVAL_PREFERENCES="[region, zone]";
+```
+
+For tables with this policy set, the data will first meet the survival goal of cross-region data isolation, and then ensure the survival goal of cross-zone data isolation.
+
+> **Note:**
+>
+> `SURVIVAL_PREFERENCES` is equivalent to `LOCATION_LABELS` in PD. For more information, please refer to [Replica scheduling by topology label](https://docs.pingcap.com/tidb/dev/schedule-replicas-by-topology-labels#schedule-replicas-by-topology-labels).
+
#### Ambiguous and edge cases
The following two policies are not identical:
@@ -491,6 +511,7 @@ In this case the default rules will apply to placement, and the output from `SHO
- `FOLLOWER_CONSTRAINTS`
- `LEARNER_CONSTRAINTS`
- `PLACEMENT POLICY`
+- `SURVIVAL_PREFERENCE`
For a more complex rule using partitions, consider the following example:
diff --git a/docs/design/2020-08-04-global-index.md b/docs/design/2020-08-04-global-index.md
index 80078688777b7..f5e2d89f932c4 100644
--- a/docs/design/2020-08-04-global-index.md
+++ b/docs/design/2020-08-04-global-index.md
@@ -183,7 +183,7 @@ In TiDB, operators in the partitioned table will be translated to UnionAll in th
## Compatibility
-MySQL does not support global index, which means this feature may cause some compatibility issues. We add an option `enable_global_index` in `config.Config` to control it. The default value of this option is `false`, so TiDB will keep consistent with MySQL, unless the user open global index feature manually.
+MySQL does not support global index, which means this feature may cause some compatibility issues. We add an option `enable-global-index` in `config.Config` to control it. The default value of this option is `false`, so TiDB will keep consistent with MySQL, unless the user open global index feature manually.
## Implementation
diff --git a/docs/design/2022-06-06-Adding-Index-Acceleration.md b/docs/design/2022-06-07-adding-index-acceleration.md
similarity index 100%
rename from docs/design/2022-06-06-Adding-Index-Acceleration.md
rename to docs/design/2022-06-07-adding-index-acceleration.md
diff --git a/docs/design/2022-06-22-foreign-key.md b/docs/design/2022-06-22-foreign-key.md
new file mode 100644
index 0000000000000..5c6b32d9474e0
--- /dev/null
+++ b/docs/design/2022-06-22-foreign-key.md
@@ -0,0 +1,708 @@
+# Foreign Key Design Doc
+
+- Author(s): [crazycs520](https://github.com/crazycs520)
+- Tracking Issue: https://github.com/pingcap/tidb/issues/18209
+
+## Abstract
+
+This proposes an implementation of supporting foreign key constraints.
+
+## DDL Technical Design
+
+### Table Information Changes
+The table's foreign key information will be stored in `model.TableInfo`:
+
+```go
+// TableInfo provides meta data describing a DB table.
+type TableInfo struct {
+ ...
+ ForeignKeys []*FKInfo `json:"fk_info"`
+ // MaxFKIndexID uses to allocate foreign key ID.
+ MaxForeignKeyID int64 `json:"max_fk_id"`
+ ...
+}
+
+// FKInfo provides meta data describing a foreign key constraint.
+type FKInfo struct {
+ ID int64 `json:"id"`
+ Name CIStr `json:"fk_name"`
+ RefSchema CIStr `json:"ref_schema"`
+ RefTable CIStr `json:"ref_table"`
+ RefCols []CIStr `json:"ref_cols"`
+ Cols []CIStr `json:"cols"`
+ OnDelete int `json:"on_delete"`
+ OnUpdate int `json:"on_update"`
+ State SchemaState `json:"state"`
+ Version int `json:"version"`
+}
+```
+
+Struct `FKInfo` uses for the child table to record the referenced parent table. Struct `FKInfo` has existed for a long time, I just added some fields.
+- `Version`: uses to distinguish between old and new versions. The new version value is 1, the old version value is 0.
+
+Why `FKInfo` record the table/schema name instead of table/schema id? Because we may don't know the table/schema id when building `FKInfo`. Here is an example:
+
+```sql
+>set @@foreign_key_checks=0;
+Query OK, 0 rows affected
+>create table t2 (a int key, foreign key fk(a) references t1(id));
+Query OK, 0 rows affected
+>create table t1 (id int key);
+Query OK, 0 rows affected
+>set @@foreign_key_checks=1;
+Query OK, 0 rows affected
+>insert into t2 values (1);
+(1452, 'Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`id`))')
+```
+
+As you can see, table `t2` refers to table `t1`, and when creating table `t2`, table `t1` still not be created, so we can't know the id of table `t1`.
+
+### Create Table with Foreign Key
+
+#### Build TableInfo
+
+When building `TableInfo`, an index for the foreign key columns is created automatically if there is no index covering the foreign key columns. Here is an example:
+
+```sql
+mysql> create table t (id int key, a int, foreign key fk(a) references t(id));
+Query OK, 0 rows affected
+mysql> show create table t\G
+***************************[ 1. row ]***************************
+Table | t
+Create Table | CREATE TABLE `t` (
+ `id` int NOT NULL,
+ `a` int DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `fk` (`a`),
+ CONSTRAINT `t_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+1 row in set
+```
+
+As you can see, the index `fk` is created automatically for the foreign key.
+
+#### Validate
+
+Create a table with foreign key, check the following conditions when a DDL job is built and the DDL owner received a DDL job(aka Double-Check):
+
+- whether the user has `REFERENCES` privilege to the foreign key references table.
+- Corresponding columns in the foreign key and the referenced key must have similar data types. The size and sign of fixed precision types such as INTEGER and DECIMAL must be the same. The length of string types need not be the same. For nonbinary (character) string columns, the character set and collation must be the same.
+- Supports foreign key references between one column and another within a table. (A column cannot have a foreign key reference to itself.)
+- Indexes on foreign keys and referenced keys are required, because the foreign key check can be fast and do not require a table scan.
+- Prefixed indexes on foreign key columns are not supported. Consequently, BLOB and TEXT columns cannot be included in a foreign key because indexes on those columns must always include a prefix length.
+- Does not currently support foreign keys for the partition table. This includes both reference and child tables.
+- A foreign key constraint cannot reference any virtual generated columns, but the stored generated columns are fine.
+
+#### Handle In DDL Owner
+
+When the DDL Owner handle create table job, the DDL owner needs to create a new table.
+
+At the same point in time, there may be two versions of the schema in the TiDB cluster, so we can't create a new table and
+update all reference tables in one schema version, since this may break foreign key constraints, such as delete reference table
+without foreign key constraint check in the child table.
+
+```sql
+-- In TiDB-1 and Schema Version is 1
+insert into t_has_foreign_key values (1, 1);
+
+-- In TiDB-0 and Schema Version is 0
+delete from t_reference where id = 1; --Since doesn't know foreign key information in old version, so doesn't do foreign key constrain check.
+```
+
+So, when creating a table with foreign key, we need multi-schema version change:
+
+1. None -> Write Only: Create table with the state is `write-only`.
+2. Write Only -> Public: Update the created table state to `public`.
+
+#### Maintain ReferredFKInfo
+
+Why need to maintain `ReferredFKInfo` in the reference table? When executing `UPDATE`/`DELETE` in the reference table, we need the `ReferredFKInfo` of the reference table to do a foreign key check/cascade.
+
+How to maintain `ReferredFKInfo` in the reference table? When we create table with foreign key, we didn't add `ReferredFKInfo` into the reference table, because the reference table may not have been created yet,
+when `foreign_key_checks` variable value is `OFF`, the user can create a child table before the reference table.
+
+We decided to maintain `ReferredFKInfo` while TiDB loading schema. At first, `infoSchema` will record all table's `ReferredFKInfo`:
+
+```sql
+// ReferredFKInfo provides the referred foreign key in the child table.
+type ReferredFKInfo struct {
+ Cols []CIStr `json:"cols"`
+ ChildSchema CIStr `json:"child_schema"`
+ ChildTable CIStr `json:"child_table"`
+ ChildFKName CIStr `json:"child_fk"`
+}
+```
+
+```go
+type infoSchema struct {
+ // referredForeignKeyMap records all table's ReferredFKInfo.
+ // referredSchemaAndTableName => child SchemaAndTableAndForeignKeyName => *model.ReferredFKInfo
+ referredForeignKeyMap map[SchemaAndTableName]map[SchemaAndTableAndForeignKeyName]*model.ReferredFKInfo
+}
+```
+
+Function `applyTableUpdate` uses `applyDropTable` to drop the old table, uses `applyCreateTable` to create the new table.
+
+In the function `applyDropTable`, we will delete the table's foreign key information from `infoSchema.referredForeignKeyMap`.
+
+In the function `applyCreateTable`, we will add the table's foreign key information into `infoSchema.referredForeignKeyMap` first,
+then get the table's `ReferredFKInfo` by schema name and table name, then store the `ReferredFKInfo` into `TableInfo.ReferredForeignKeys`.
+
+Then `applyTableUpdate` will also need to reload the old/new table's referred table information, also uses `applyDropTable` to drop the old reference table, use `applyCreateTable` to create new reference table.
+
+That's all.
+
+### Alter Table Add Foreign Key
+
+Here is an example:
+
+```sql
+create table t1 (id int key,a int, index(a));
+create table t2 (id int key,a int);
+alter table t2 add foreign key fk(a) references t1(id) ON DELETE CASCADE;
+```
+
+Just like create table, we should validate first, and return an error if the conditions for creating foreign keys are not met, and also need to double-check.
+
+When building `TableInfo`, we need to auto-create an index for foreign key columns if there are no index cover foreign key columns.
+And this divides the problem into two cases:
+- Case-1: No need to create an index automatically, and only add the foreign key constraint.
+- Case-2: Need auto-create index for foreign key
+
+#### Case-1: Only add foreign key constrain
+
+The DDL owner handle adds foreign key constrain step is:
+
+1. None -> Write Only: add foreign key constraint which state is `write-only` into the table.
+2. Write Only -> Write Reorg: check all rows in the table whether has related foreign key exists in the reference table, we can use the following SQL to check:
+
+ ```sql
+ select 1 from t2 where t2.a is not null and t2.a not in (select id from t1) limit 1;
+ ```
+ The expected result is `empty`, otherwise, an error is returned and cancels the DDL job.
+
+3. Write Reorg -> Public: update the foreign key constraint state to `public`.
+
+A problem is, How the DML treat the foreign key on delete/update cascade behaviour in which state is non-public?
+Here is an example:
+
+```sql
+create table t1 (id int key,a int, index(a));
+create table t2 (id int key,a int, index(a));
+insert into t1 values (1,1);
+insert into t2 values (1,1);
+alter table t2 add constraint fk_1 foreign key (a) references t1(id) ON DELETE CASCADE;
+```
+
+The schema change of foreign key `fk_1` is from `None` -> `Write-Only` -> `Write-Reorg` -> `Public`。
+When the foreign key `fk_1` in `Write-Only` state, a DML request has come to be processed:
+
+```sql
+delete from t1 where id = 1;
+```
+
+Then, TiDB shouldn't do cascade delete for foreign key `fk_1` in state `Write-Only`, since the `Add Foreign Key` DDL job maybe
+failed in `Write-Reorg` state and rollback the DDL job. But it is hard to rollback the cascade deleted executed before.
+
+So, when executing DML with `non-public` foreign key, TiDB will do foreign key constraint check instead of foreign key cascade behaviour.
+
+#### Case-2: Auto-create index for foreign key and add foreign key constrain
+
+As TiDB support multi-schema change now, we create an `ActionMultiSchemaChange` job that contains the following 2 sub-ddl job.
+- Add Index DDL job
+- Add Foreign Key Constraint DDL job
+
+When TiDB adds foreign key DDL job meet error, TiDB will rollback the `ActionMultiSchemaChange` job and the 2 sub-ddl job will also be rollback.
+
+### Drop Table
+
+If `foreign_key_checks` is `ON`, then drop the table which has foreign key references will be rejected.
+
+```sql
+> drop table t1;
+(3730, "Cannot drop table 't1' referenced by a foreign key constraint 't2_ibfk_1' on table 't2'.")
+```
+
+### Drop Database
+
+If `foreign_key_checks` is `ON`, then drop the database which has foreign key references by another database will be rejected.
+
+```sql
+> drop database test;
+(3730, "Cannot drop table 't1' referenced by a foreign key constraint 't2_ibfk_1' on table 't2'.")
+```
+
+### Drop Index
+
+Drop index used by the foreign key will be rejected.
+
+```sql
+> set @@foreign_key_checks=0; -- Even disable foreign_key_checks, you still can't drop the index used for foreign key constrain.
+Query OK, 0 rows affected
+> alter table t2 drop index fk;
+(1553, "Cannot drop index 'fk': needed in a foreign key constraint")
+```
+
+### Rename Column
+
+Rename column which has foreign keys or references should also need to update the related child/parent table info.
+
+```sql
+create table t1 (id int key,a int, index(a));
+create table t2 (id int key,a int, foreign key fk(a) references t1(id) ON DELETE CASCADE);
+rename table t1 to t11;
+alter table t11 change column id id1 int;
+show create table t2\G
+***************************[ 1. row ]***************************
+ Table | t2
+Create Table | CREATE TABLE `t2` (
+ `id` int NOT NULL,
+ `a` int DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `fk` (`a`),
+ CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t11` (`id1`) ON DELETE CASCADE
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+```
+
+### Truncate Table
+
+A truncate table which has foreign keys or references should also need to update the related child/parent table info.
+
+### Modify Column
+
+Modify column which used by the foreign key will be rejected.
+
+```sql
+> alter table t1 change column id id1 bigint;
+(3780, "Referencing column 'a' and referenced column 'id1' in foreign key constraint 't2_ibfk_1' are incompatible.")
+```
+
+MySQL modify column problem: https://www.percona.com/blog/2019/06/04/ddl-queries-foreign-key-columns-MySQL-pxc/
+
+What if the user really needs to modify column type, such as from `INT` to `BIGINT`. Maybe we can offer a variable such as `alter-foreign-keys-method=auto`,
+then when the user modifies the column type, TiDB will auto-modify the related foreign key column's type. For easy implementation and to reduce risk, maybe only support modifying column type which doesn't need to reorg table row data.
+
+## DML Technical Design
+
+### DML On Child Table
+
+On Child Table Insert Or Update, need to Find FK column value that exist in the reference table:
+
+1. Get reference table info by table name.
+2. Get the related fk index of the reference table.
+3. tiny optimize, check fk column value exists in reference table cache(map[string(index_key)]struct).
+3. Get related row in reference.
+- Construct index key and then use snapshot `Iter` and `Seek` API to scan. If the index is unique and only contain
+ foreign key columns, use the snapshot `Get` API.
+ - `Iter` default scan batch size is 256, need to set 2 to avoid read unnecessary data.
+4. compact column value to make sure exist.
+5. If relate row exist in reference table, also need to add lock in the related row.
+6. put column value into reference fk column value cache.
+
+#### Lock
+
+Let's see an example in MySQL first:
+
+prepare:
+```sql
+create table t1 (id int key,a int, b int, unique index(a, b, id));
+create table t2 (id int key,a int, b int, index (a,b,id), foreign key fk(a, b) references t1(a, b));
+insert into t1 values (-1, 1, 1);
+```
+
+Then, execute the following SQL in 2 sessions:
+
+| Session 1 | Session 2 |
+| -------------------------------- | ------------------------------------------- |
+| Begin; | |
+| insert into t2 values (1, 1, 1); | |
+| | delete from t1; -- Blocked by wait lock |
+| Commit | |
+| | ERROR: Cannot delete or update a parent row |
+
+So we need to add lock in the reference table when insert/update child table.
+
+##### In Pessimistic Transaction
+
+When TiDB add pessimistic locks, if related row exists in the reference table, also needs to add lock in the related row.
+
+##### In Optimistic Transaction
+
+Just like `SELECT FOR UPDATE` statement, need to use `doLockKeys` to lock the related row in the reference table.
+
+##### Issue
+
+TiDB currently only support `lock for update`(aka write-lock, such as `select for update`), and doesn't support `lock for share`(aka read-lock, such as `select for share`).
+
+So far we have to add `lock for update` in the reference table when insert/update child table, then the performance will be poor. After TiDB support `lock for share`, we should use `lock for share` instead.
+
+#### DML Load data
+
+Load data should also do foreign key check, but report a warning instead error:
+
+```sql
+create table t1 (id int key,a int, index(a));
+create table t2 (id int key,a int, foreign key fk(a) references t1(id) ON DELETE CASCADE);
+
+test> load data local infile 'data.csv' into table t2 FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n';
+Query OK, 0 rows affected
+test> show warnings;
++---------+------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| Level | Code | Message |
++---------+------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| Warning | 1452 | Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`id`) ON DELETE CASCADE) |
+| Warning | 1452 | Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`id`) ON DELETE CASCADE) |
++---------+------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+```
+
+`data.csv` is:
+```
+1,1
+2,2
+```
+
+### DML On reference Table
+
+On reference Table Delete Or Update:
+
+1. modify related child table row by referential action:
+- `CASCADE`: update/delete related child table row.
+- `SET NULL`: set related child row's foreign key columns value to NULL.
+- `RESTRICT`, `NO ACTION`: If related row exist in child table, reject update/delete the reference table.
+- `SET DEFAULT`: just like `RESTRICT`.
+
+modify related child table rows by the following step:
+1. get child table info by name(in reference table info).
+2. get the child table fk index's column info.
+3. build update executor to update child table rows.
+
+### Issue need to be discussed
+
+#### Affect Row
+
+related article: https://www.percona.com/blog/hidden-cost-of-foreign-key-constraints-in-MySQL/
+
+Here is a MySQL example:
+
+prepare:
+```sql
+create table t1 (id int key,a int, index(a));
+create table t2 (id int key,a int, foreign key fk(a) references t1(id) ON DELETE CASCADE);
+insert into t1 values (1, 1);
+insert into t2 values (1, 1);
+```
+
+Then delete on reference table:
+```sql
+> delete from t1 where id=1;
+Query OK, 1 row affected
+```
+
+As you can see, in the query result, the affected row is 1, but actually should be 2, since the related row in t2 is also been deleted.
+
+This is a MySQL behaviour, We can be compatible with it.
+
+#### DML Execution plan
+
+Here is a MySQL example:
+
+prepare:
+```sql
+create table t1 (id int key,a int, index(a));
+create table t2 (id int key,a int, foreign key fk(a) references t1(id) ON DELETE CASCADE);
+insert into t1 values (1, 1);
+insert into t2 values (1, 1);
+```
+
+```sql
+> explain delete from t1 where id = 1;
++----+-------------+-------+------------+-------+---------------+---------+---------+-------+------+----------+-------------+
+| id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
++----+-------------+-------+------------+-------+---------------+---------+---------+-------+------+----------+-------------+
+| 1 | DELETE | t1 | | range | PRIMARY | PRIMARY | 4 | const | 1 | 100.0 | Using where |
++----+-------------+-------+------------+-------+---------------+---------+---------+-------+------+----------+-------------+
+```
+
+From the plan, you can't see any information about the foreign key constraint which needs to delete the related row in child table `t2`.
+
+I think this is a MySQL issue, should we make TiDB plan better, at least when we meet some slow query, we can know maybe it is caused by modify related rows in the child table.
+
+Here is an example plan with foreign key:
+
+```sql
+> explain delete from t1 where id = 1;
++-------------------------+---------+------+---------------+-----------------------------------+
+| id | estRows | task | access object | operator info |
++-------------------------+---------+------+---------------+-----------------------------------+
+| Delete_2 | N/A | root | | N/A |
+| ├─Point_Get_1 | 1.00 | root | table:t1 | handle:1 |
+| └─Foreign_Key_Cascade_3 | 0.00 | root | table:t2 | foreign_key:fk, on_delete:CASCADE |
++-------------------------+---------+------+---------------+-----------------------------------+
+```
+
+And the `explain analyze` will show the foreign key cascade child plan:
+
+```sql
+> explain analyze delete from t1 where id = 1;
++-------------------------+---------+---------+------+---------------+----------------------------------------------------------+-----------------------------------+-----------+------+
+| id | estRows | actRows | task | access object | execution info | operator info | memory | disk |
++-------------------------+---------+---------+------+---------------+----------------------------------------------------------+-----------------------------------+-----------+------+
+| Delete_2 | N/A | 0 | root | | time:109.5µs, loops:1 | N/A | 380 Bytes | N/A |
+| ├─Point_Get_1 | 1.00 | 1 | root | table:t1 | time:62.7µs, loops:2, Get:{num_rpc:1, total_time:26.4µs} | handle:1 | N/A | N/A |
+| └─Foreign_Key_Cascade_3 | 0.00 | 0 | root | table:t2 | total:322.1µs, foreign_keys:1 | foreign_key:fk, on_delete:CASCADE | N/A | N/A |
+| └─Delete_7 | N/A | 0 | root | | time:23.5µs, loops:1 | N/A | 129 Bytes | N/A |
+| └─Point_Get_9 | 1.00 | 1 | root | table:t2 | time:12.6µs, loops:2, Get:{num_rpc:1, total_time:4.21µs} | handle:1 | N/A | N/A |
++-------------------------+---------+---------+------+---------------+----------------------------------------------------------+-----------------------------------+-----------+------+
+```
+
+##### CockroachDB DML Execution Plan
+
+```sql
+CREATE TABLE customers_2 (
+ id INT PRIMARY KEY
+);
+CREATE TABLE orders_2 (
+ id INT PRIMARY KEY,
+ customer_id INT REFERENCES customers_2(id) ON UPDATE CASCADE ON DELETE CASCADE
+);
+INSERT INTO customers_2 VALUES (1), (2), (3);
+INSERT INTO orders_2 VALUES (100,1), (101,2), (102,3), (103,1);
+```
+
+```sql
+> explain analyze UPDATE customers_2 SET id = 23 WHERE id = 1;
+ info
+--------------------------------------------------
+ planning time: 494µs
+ execution time: 5ms
+ distribution: local
+ vectorized: true
+ rows read from KV: 6 (170 B)
+ cumulative time spent in KV: 978µs
+ maximum memory usage: 100 KiB
+ network usage: 0 B (0 messages)
+ regions: us-east1
+
+ • root
+ │
+ ├── • update
+ │ │ nodes: n1
+ │ │ regions: us-east1
+ │ │ actual row count: 1
+ │ │ table: customers_2
+ │ │ set: id
+ │ │
+ │ └── • buffer
+ │ │ label: buffer 1
+ │ │
+ │ └── • render
+ │ │ nodes: n1
+ │ │ regions: us-east1
+ │ │ actual row count: 1
+ │ │ KV rows read: 1
+ │ │ KV bytes read: 27 B
+ │ │
+ │ └── • scan
+ │ nodes: n1
+ │ regions: us-east1
+ │ actual row count: 1
+ │ KV rows read: 1
+ │ KV bytes read: 27 B
+ │ missing stats
+ │ table: customers_2@primary
+ │ spans: [/1 - /1]
+ │ locking strength: for update
+ │
+ └── • fk-cascade
+ fk: fk_customer_id_ref_customers_2
+ input: buffer 1
+```
+
+##### PostgreSQL DML Execution Plan
+
+```sql
+postgres=# explain analyze UPDATE customers_2 SET id = 20 WHERE id = 23;
+ QUERY PLAN
+-------------------------------------------------------------------------------------------------------------------------------------
+ Update on customers_2 (cost=0.15..8.17 rows=1 width=10) (actual time=0.039..0.039 rows=0 loops=1)
+ -> Index Scan using customers_2_pkey on customers_2 (cost=0.15..8.17 rows=1 width=10) (actual time=0.016..0.016 rows=1 loops=1)
+ Index Cond: (id = 23)
+ Planning Time: 0.057 ms
+ Trigger for constraint orders_2_customer_id_fkey on customers_2: time=0.045 calls=1
+ Trigger for constraint orders_2_customer_id_fkey on orders_2: time=0.023 calls=2
+ Execution Time: 0.129 ms
+```
+
+## Compatibility
+
+Since the old version TiDB already support foreign key syntax, but doesn't support it, after upgrade TiDB to latest version, the foreign key created before won't take effect. Only foreign keys created in the new version actually take effect.
+
+You can use `SHOW CREATE TABLE` to see whethere a foreign key is take effect, the old version foreign key will have a comment `/* FOREIGN KEY INVALID */` to indicate it is invalid, the new version foreign key doesn't have this comment.
+
+```sql
+> show create table t2\G
+ ***************************[ 1. row ]***************************
+ Table | t2
+Create Table | CREATE TABLE `t2` (
+ `id` int(11) NOT NULL,
+ PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,
+ CONSTRAINT `fk` FOREIGN KEY (`id`) REFERENCES `test`.`t1` (`id`) ON DELETE CASCADE ON UPDATE CASCADE /* FOREIGN KEY INVALID */
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin
+```
+
+## Impact
+
+### Impact of data replication
+
+#### TiCDC
+
+When syncing table data to downstream TiDB, TiCDC should `set @@foreign_key_checks=0` in downstream TiDB.
+
+#### DM
+
+When doing full synchronization, DM should `set @@foreign_key_checks=0` in downstream TiDB.
+
+When doing incremental synchronization, DM should set `foreign_key_checks` session variable according to MySQL binlog.
+
+#### BR
+
+When restore data into TiDB, BR should `set @@foreign_key_checks=0` in TiDB.
+
+## Test Case
+
+Cascade modification test case:
+
+```sql
+drop table if exists t3,t2,t1;
+create table t1 (id int key,a int, index(a));
+create table t2 (id int key,a int, foreign key fk(a) references t1(id) ON DELETE CASCADE);
+create table t3 (id int key,a int, foreign key fk(a) references t2(id) ON DELETE CASCADE);
+insert into t1 values (1,1);
+insert into t2 values (2,1);
+insert into t3 values (3,2);
+delete from t1 where id = 1; -- both t1, t2, t3 rows are deleted.
+```
+
+Following is a MySQL test case about `SET DEFAULT`, as you can see, MySQL actualy doesn't support `SET DEFAULT`, the behaviour is just like `RESTRICT`
+
+```sql
+MySQL>create table t1 (a int,b int, index(a,b)) ;
+Query OK, 0 rows affected
+Time: 0.022s
+MySQL>create table t (a int, b int, foreign key fk_a(a) references test.t1(a) ON DELETE SET DEFAULT);
+Query OK, 0 rows affected
+Time: 0.019s
+MySQL>insert into t1 values (1,1);
+Query OK, 1 row affected
+Time: 0.003s
+MySQL>insert into t values (1,1);
+Query OK, 1 row affected
+Time: 0.006s
+MySQL>delete from t1 where a=1;
+(1451, 'Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t`, CONSTRAINT `t_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`a`))')
+MySQL>select version();
++-----------+
+| version() |
++-----------+
+| 8.0.29 |
++-----------+
+```
+
+### Self-Referencing Tables
+
+For example, table `employee` has a column `manager_id` references to `employee.id`.
+
+```sql
+create table employee (id int key,manager_id int, foreign key fk(manager_id) references employee(id) ON DELETE CASCADE);
+insert into employee values (1,1);
+insert into employee values (2,1);
+
+test> delete from employee where id=1;
+Query OK, 1 row affected
+test> select * from employee;
++----+---+
+| id | a |
++----+---+
+0 rows in set
+```
+
+A case of self-reference and cyclical dependencies:
+
+```sql
+test> create table t (id int key,a int, foreign key fk_a(a) references t(id) ON DELETE CASCADE, foreign key fk_id(id) references t(a) ON DELETE CASCADE);
+Query OK, 0 rows affected
+Time: 0.045s
+test> insert into t values (1,1);
+(1452, 'Cannot add or update a child row: a foreign key constraint fails (`test`.`t`, CONSTRAINT `t_ibfk_2` FOREIGN KEY (`id`) REFERENCES `t` (`a`) ON DELETE CASCADE)')
+```
+
+### Cyclical Dependencies
+
+```sql
+create table t1 (id int key,a int, index(a));
+create table t2 (id int key,a int, foreign key fk(a) references t1(id) ON DELETE CASCADE);
+insert into t1 values (1,1);
+ALTER TABLE t1 ADD foreign key fk(a) references t2(id) ON DELETE CASCADE;
+(1452, 'Cannot add or update a child row: a foreign key constraint fails (`test`.`#sql-298_8`, CONSTRAINT `t1_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t2` (`id`) ON DELETE CASCADE)')
+```
+
+```sql
+set @@foreign_key_checks=0;
+create table t1 (id int key,a int, foreign key fk(a) references t2(id) ON DELETE CASCADE);
+create table t2 (id int key,a int, foreign key fk(a) references t1(id) ON DELETE CASCADE);
+insert into t1 values (1, 2);
+insert into t2 values (2, 1);
+set @@foreign_key_checks=1; -- add test case without this.
+delete from t1 where id=1;
+test> select * from t2;
++----+---+
+| id | a |
++----+---+
+0 rows in set
+Time: 0.004s
+test> select * from t1;
++----+---+
+| id | a |
++----+---+
+0 rows in set
+```
+
+### Check order
+
+check order should check unique/primary key constrain first:
+
+```sql
+test> create table t1 (id int key,a int, index(a));
+test> create table t2 (id int key,a int, foreign key fk(a) references t1(id) ON DELETE CASCADE);
+test> insert into t1 values (1, 1);
+test> insert into t2 values (1, 1);
+test> insert into t2 values (1, 2);
+(1062, "Duplicate entry '1' for key 't2.PRIMARY'")
+test> insert ignore into t2 values (1, 2);
+Query OK, 0 rows affected
+```
+
+### MATCH FULL or MATCH SIMPLE
+
+This definition is from [CRDB](https://www.cockroachlabs.com/docs/v22.1/foreign-key.html#match-composite-foreign-keys-with-match-simple-and-match-full). MySQL doesn't mention it, here is a MySQL test case:
+
+Here is an MySQL example:
+
+```sql
+create table t1 (i int, a int,b int, index(a,b)) ;
+create table t (a int, b int, foreign key fk_a(a,b) references test.t1(a,b));
+
+test> insert into t values (null,1);
+Query OK, 1 row affected
+test> insert into t values (null,null);
+Query OK, 1 row affected
+test> insert into t values (1,null);
+Query OK, 1 row affected
+```
+
+## reference
+
+- [MySQL FOREIGN KEY Constraints Document](https://dev.mysql.com/doc/refman/8.0/en/create-table-foreign-keys.html#foreign-key-adding)
+- [3 Common Foreign Key Mistakes (And How to Avoid Them)](https://www.cockroachlabs.com/blog/common-foreign-key-mistakes/)
+- [Hidden Cost of Foreign Key Constraints in MySQL](https://www.percona.com/blog/hidden-cost-of-foreign-key-constraints-in-MySQL/)
+- [DDL Queries on Foreign Key Columns in MySQL/PXC](https://www.percona.com/blog/2019/06/04/ddl-queries-foreign-key-columns-mysql-pxc/)
diff --git a/docs/design/2022-09-28-flashback-to-timestamp.md b/docs/design/2022-09-28-flashback-to-timestamp.md
index 9f3c2e2ef7fab..d71ff39404179 100644
--- a/docs/design/2022-09-28-flashback-to-timestamp.md
+++ b/docs/design/2022-09-28-flashback-to-timestamp.md
@@ -1,5 +1,6 @@
# Proposal: Flashback To Timestamp
-- Author(s): [Defined2014](https://github.com/Defined2014) and [JmPotato](https://github.com/JmPotato)
+- Author(s): [Defined2014](https://github.com/Defined2014) and [JmPotato](https://github.com/JmPotato)
+- Last updated: 2022-12-19
- Tracking Issues: https://github.com/pingcap/tidb/issues/37197 and https://github.com/tikv/tikv/issues/13303
## Abstract
@@ -61,19 +62,19 @@ Then a `Flashback To Timestamp` DDL job can be simply divided into the following
* Pre-checks. After all checks are passed, TiDB will disable GC and closed PD schedule for the cluster. The specific checks are as follows:
* The FlashbackTS is after `tikv_gc_safe_point`.
- * The FlashbackTS is before the minimal store resolved TS.
- * No related DDL history in flashback time range.
+ * The FlashbackTS is before the minimal store resolved timestamp.
+ * No DDL job was executing at FlashbackTS.
* No running related DDL jobs.
* TiDB get flashback key ranges and splits them into separate regions to avoid locking unrelated key ranges. Then TiDB send `PrepareFlashbackToVersion` RPC requests to lock regions in TiKV. Once locked, no more read, write and scheduling operations are allowed for those regions.
-* After locked all relevant key ranges, the DDL Owner will update schema version and synchronize it to other TiDBs. When other TiDB applies the `SchemaDiff` of type `Flashback To Timestamp`, it will disconnect all relevant links.
+* After locked all relevant key ranges, the DDL owner will update schema version and synchronize it to other TiDBs. When other TiDB applies the `SchemaDiff` of type `Flashback To Timestamp`, it will disconnect all relevant links.
* Send `FlashbackToVersion` RPC requests to all relevant key ranges with same `commit_ts`. Each region handles its own flashback progress independently.
* Read the old MVCC data and write it again with the given `commit_ts` to pretend it's a new transaction commit.
* Release the Raft proposing lock and resume the lease read.
-* TiDB checks whether all the requests returned successfully, and retries those that failed with same `commit_ts` until the whole flashback is done.
+* TiDB checks whether all the requests returned successfully, and retries those failed requests with the same `commit_ts`. After all requests finished, the DDL owner update the schema version with new type named `ReloadSchemaMap`, so that all TiDB servers regenerate the schema map from TiKV.
* After `Flashback To Timestamp` is finished, TiDB will restore all changed global variables and restart PD schedule. At the same time, notify `Stats Handle` to reload statistics from TiKV.
@@ -101,17 +102,7 @@ FLASHBACK TABLE [table1], [table2] TO TIMESTAMP '2022-08-10 08:00:00';
### Limitations and future Work
-1. DDL history exists for the flashback time period is not currently supported, the error message is shown below:
-
-```sql
-mysql> ALTER TABLE t ADD INDEX i(a);
-Query OK, 0 rows affected (2.99 sec)
-
-mysql> FLASHBACK CLUSTER TO TIMESTAMP '2022-10-10 11:53:30';
-ERROR 1105 (HY000): Had ddl history during [2022-10-10 11:53:30 +0800 CST, now), can't do flashback
-```
-
-2. Compare with the other DDL jobs, `Flashback To Timestamp` job cannot be rollbacked after some regions failure and also needs to resend rpc to all regions when ddl owner crashed. In the future, we will improve those two issues with a new TiKV interface and new distributed processing ddl framework.
+1. Compare with the other DDL jobs, `Flashback To Timestamp` job cannot be rollbacked after some regions failure and also needs to resend rpc to all regions when ddl owner crashed. In the future, we will improve those two issues with a new TiKV interface and new distributed processing ddl framework.
### Alternative Solutions
diff --git a/docs/design/2022-09-29-reorganize-partition.md b/docs/design/2022-09-29-reorganize-partition.md
new file mode 100644
index 0000000000000..56e380826efa7
--- /dev/null
+++ b/docs/design/2022-09-29-reorganize-partition.md
@@ -0,0 +1,180 @@
+# TiDB Design Documents
+
+- Author(s): [Mattias Jonsson](http://github.com/mjonss)
+- Discussion PR: https://github.com/pingcap/tidb/issues/38535
+- Tracking Issue: https://github.com/pingcap/tidb/issues/15000
+
+## Table of Contents
+
+* [Introduction](#introduction)
+* [Motivation or Background](#motivation-or-background)
+* [Detailed Design](#detailed-design)
+ * [Schema change states for REORGANIZE PARTITION](#schema-change-states-for-reorganize-partition)
+ * [Error Handling](#error-handling)
+ * [Notes](#notes)
+* [Test Design](#test-design)
+ * [Benchmark Tests](#benchmark-tests)
+* [Impacts & Risks](#impacts--risks)
+
+## Introduction
+
+Support ALTER TABLE t REORGANIZE PARTITION p1,p2 INTO (partition pNew1 values...)
+
+## Motivation or Background
+
+TiDB is currently lacking the support of changing the partitions of a partitioned table, it only supports adding and dropping LIST/RANGE partitions.
+Supporting REORGANIZE PARTITIONs will allow RANGE partitioned tables to have a MAXVALUE partition to catch all values and split it into new ranges. Similar with LIST partitions where one can split or merge different partitions.
+
+When this is implemented, it will also allow future PRs transforming a non-partitioned table into a partitioned table as well as remove partitioning and make a partitioned table a normal non-partitioned table, as well as COALESCE PARTITION and ADD PARTITION for HASH partitioned tables, which is different ALTER statements but can use the same implementation as REORGANIZE PARTITION
+
+The operation should be online, and must handle multiple partitions as well as large data sets.
+
+Possible usage scenarios:
+- Full table copy
+ - merging all partitions to a single table (ALTER TABLE t REMOVE PARTITIONING)
+ - splitting data from many to many partitions, like change the number of HASH partitions
+ - splitting a table to many partitions (ALTER TABLE t PARTITION BY ...)
+- Partial table copy (not full table/all partitions)
+ - split one or more partitions
+ - merge two or more partitions
+
+These different use cases can have different optimizations, but the generic form must still be solved:
+- N partitions, where each partition has M indexes
+
+First implementation should be based on the merge-txn (row-by-row batch read, update record key with new Physical Table ID, write) transactional batches and then create the indexes in batches index by index, partition by partition.
+Later we can implement the ingest (lightning way) optimization, since DDL module are on the way of evolution to do reorg tasks more efficiency.
+
+## Detailed Design
+
+There are two parts of the design:
+- Schema change states throughout the operation
+- Reorganization implementation, which will be handled in the StateWriteReorganization state.
+
+Where the schema change states will clarify which different steps that will be done in which schema state transitions.
+
+### Schema change states for REORGANIZE PARTITION
+
+Since this operation will:
+- create new partitions
+- copy data from dropped partitions to new partitions and create their indexes
+- change the partition definitions
+- drop existing partitions
+
+It will use all these schema change stages:
+
+ // StateNone means this schema element is absent and can't be used.
+ StateNone SchemaState = iota
+ - Check if the table structure after the ALTER is valid
+ - Use the generate physical table ids to each new partition (that was generated already by the client sending the ALTER command).
+ - Update the meta data with the new partitions (AddingDefinitions) and which partitions to be dropped (DroppingDefinitions), so that new transactions can double write.
+ - Set placement rules
+ - Set TiFlash Replicas
+ - Set legacy Bundles (non-sql placement)
+ - Set the state to StateDeleteOnly
+
+ // StateDeleteOnly means we can only delete items for this schema element (the new partition).
+ StateDeleteOnly
+ - Set the state to StateWriteOnly
+
+ // StateWriteOnly means we can use any write operation on this schema element,
+ // but outer can't read the changed data.
+ StateWriteOnly
+ - Set the state to StateWriteReorganization
+
+ // StateWriteReorganization means we are re-organizing whole data after write only state.
+ StateWriteReorganization
+ - Copy the data from the partitions to be dropped (one at a time) and insert it into the new partitions. This needs a new backfillWorker implementation.
+ - Recreate the indexes one by one for the new partitions (one partition at a time) (create an element for each index and reuse the addIndexWorker). (Note: this can be optimized in the futute, either with the new fast add index implementation, based on lightning. Or by either writing the index entries at the same time as the records, in the previous step, or if the partitioning columns are included in the index or handle)
+ - Replace the old partitions with the new partitions in the metadata when the data copying is done
+ - Set the state to StateDeleteReorganization
+
+ // StateDeleteReorganization means we are re-organizing whole data after delete only state.
+ StateDeleteReorganization - we are using this state in a slightly different way than the comment above says.
+ This state is needed since we cannot directly move from StateWriteReorganization to StatePublic.
+ Imagine that the StateWriteReorganization is complete and we are updating the schema version, then if a transaction seeing the new schema version is writing to the new partitions, then those changes needs to be written to the old partitions as well, so new transactions in other nodes using the older schema version can still see the changes.
+ - Remove the notion of new partitions (AddingDefinitions) and which partitions to be dropped (DroppingDefinitions) and double writing will stop when it goes to StatePublic.
+ - Register the range delete of the old partition data (in finishJob / deleteRange).
+ - Set the state to StatePublic
+
+ // StatePublic means this schema element is ok for all write and read operations.
+ StatePublic
+ - Table structure is now complete and the table is ready to use with its new partitioning scheme
+ - Note that there is a background job for the GCWorker to do in its deleteRange function.
+
+During the reorganization happens in the background the normal write path needs to check if there are any new partitions in the metadata and also check if the updated/deleted/inserted row would match a new partition, and if so, also do the same operation in the new partition, just like during adding index or modify column operations currently does. (To be implemented in `(*partitionedTable) AddRecord/UpdateRecord/RemoveRecord`)
+
+Example of why an extra state between StateWriteReorganize and StatePublic is needed:
+
+```sql
+-- table:
+CREATE TABLE t (a int) PARTITION BY LIST (a) (PARTITION p0 VALUES IN (1,2,3,4,5), PARTITION p1 VALUES IN (6,7,8,9,10));
+-- during alter operation:
+ALTER TABLE t REORGANIZE PARTITION p0 INTO (PARTITION p0a VALUES IN (1,2,3), PARTITION p0b VALUES IN (4,5));
+```
+
+Partition within parentheses `(p0a [1] p0b [0])` is hidden or to be deleted by GC/DeleteRange. Values in the brackets after the partition `p0a [2]`.
+
+If we go directly from StateWriteReorganize to StatePublic, then clients one schema version behind will not see changes to the new partitions:
+
+| Data (TiKV/Unistore) | TiDB client 1 | TiDB client 2 |
+| --------------------------------------- | ------------------------------------ | ------------------------------------------------------------ |
+| p0 [] p1 [] StateWriteReorganize | | |
+| p0 [] p1 [] (p0a [] p0b []) | | |
+| (p0 []) p1 [] p0a [] p0b [] StatePublic | | |
+| (p0 []) p1 [] p0a [2] p0b [] | StatePublic INSERT INTO T VALUES (2) | |
+| (p0 []) p1 [] p0a [2] p0b [] | | StateWriteReorganize SELECT * FROM t => [] (only sees p0,p1) |
+
+
+But if we add a state between StateWriteReorganize and StatePublic and double write to the old partitions during that state it works:
+
+
+| Data (TiKV/Unistore) | TiDB client 1 | TiDB client 2 |
+| ------------------------------------------------- | ---------------------------------------------- | -------------------------------------------------------------------- |
+| p0 [] p1 [] (p0a [] p0b []) StateWriteReorganize | | |
+| (p0 []) p1 [] p0a [] p0b [] StateDeleteReorganize | | |
+| (p0 [2]) p1 [] p0a [2] p0b [] | StateDeleteReorganize INSERT INTO T VALUES (2) | |
+| (p0 [2]) p1 [] p0a [2] p0b [] | | StateWriteReorganize SELECT * FROM t => [2] (only sees p0,p1) |
+| (p0 [2]) p1 [] p0a [2] p0b [] StatePublic | | |
+| (p0 [2]) p1 [] p0a [2] p0b [4] | StatePublic INSERT INTO T VALUES (4) | |
+| (p0 [2]) p1 [] p0a [2] p0b [4] | | StateDeleteReorganize SELECT * FROM t => [2,4] (sees p0a,p0b,p1) |
+
+
+### Error handling
+
+If any non-retryable error occurs, we will call onDropTablePartition and adjust the logic in that function to also handle the roll back of reorganize partition, in a similar way as it does with model.ActionAddTablePartition.
+
+### Notes
+
+Note that parser support already exists.
+There should be no issues with upgrading, and downgrade will not be supported during the DDL.
+
+Notes:
+- statistics should be removed from the old partitions.
+- statistics will not be generated for the new partitions (future optimization possible, to get statistics during the data copying?)
+- the global statistics (table level) will remain the same, since the data has not changed.
+- this DDL will be online, while MySQL is blocking on MDL.
+
+## Test Design
+
+Re-use tests from other DDLs like Modify column, but adjust them for Reorganize partition.
+A separate test plan will be created and a test report will be written and signed off when the tests are completed.
+
+### Benchmark Tests
+
+Correctness and functionality is higher priority than performance.
+
+## Impacts & Risks
+
+Impacts:
+- better usability of partitioned tables
+- online alter in TiDB, where MySQL is blocking
+- all affected data needs to be read (CPU/IO/Network load on TiDB/PD/TiKV), even multiple times in case of indexes.
+- all data needs to be writted (duplicated, both row-data and indexes), including transaction logs (more disk space on TiKV, CPU/IO/Network load on TiDB/PD/TiKV and TiFlash if configured on the table).
+
+Risks:
+- introduction of bugs
+ - in the DDL code
+ - in the write path (double writing the changes for transactions running during the DDL)
+- out of disk space
+- out of memory
+- general resource usage, resulting in lower performance of the cluster
diff --git a/docs/design/2022-11-22-view-hint.md b/docs/design/2022-11-22-view-hint.md
new file mode 100644
index 0000000000000..b780208a13f45
--- /dev/null
+++ b/docs/design/2022-11-22-view-hint.md
@@ -0,0 +1,134 @@
+# View Hint Design Doc
+- Author: Reminiscent
+- Tracking Issue: https://github.com/pingcap/tidb/issues/37887
+
+## Background
+Hints that specify a table generally refer to tables in the DELETE, SELECT, or UPDATE query block in which the hint occurs, not to tables inside any views referenced by the statement. So we introduce the view hint to specify the table in view instead of embedding the hint in the view.
+
+In Oracle, there are three ways to use the global hint. (Node: the `{}` part is only used for explanation)
+```SQL
+CREATE OR REPLACE VIEW v AS
+SELECT {SEL$2} * from e1 join (select {SEL$3} * from e3) e2 on e1.a = e2.a;
+
+SELECT {SEL$1} * FROM v;
+
+A. SELECT /*+ INDEX(v.e2.e3 idx) */ * FROM v; // /*+ INDEX(@SEL$1 v.e2.e3 idx) */
+
+B. SELECT /*+ INDEX(@SEL$2 e2.e3 idx) */ * FROM v;
+
+C. SELECT /*+ INDEX(@SEL$3 e3 idx) */ * FROM v;
+```
+
+Compared with TiDB, Oracle has two differences:
+1. Oracle can use `alias.table` to represent in subquery, such as e2.e3. Besides, TiDB can use `db.table` to represent a table.
+2. The count for query block number treats view like a subquery, which means the select parts in view are counted.
+
+Based on the difference, there are some reasons why TiDB can not just use the grammar from Oracle:
+1. Compatibility
+ 1. Grammar.
+ 1. We can not use the `alias.table` to represent in subquery, such as e2.e3.
+ 2. We can use `db.table` to represent a table. So if we want to use the `view.table` to represent a table in view, we should change the grammar or it will conflict with db.table.
+ 2. The count for the query block.
+ 1. Previously, the query block in view would not be counted. But now, if we take the view into consideration, it will change the origin count. For example, in the following part. The count of the query block for the `select a from t1` will be changed from `@SEL_2` to `@SEL_3`. So if we use the query block related hints for this part, it will be invalid or represent the content in the view.
+
+```SQL
+CREATE OR REPLACE VIEW v AS
+SELECT {SEL$2} * FROM t;
+
+SELECT {SEL$1} * FROM v JOIN (select {SEL$3} a from t1) t2 on v.a = t2.a;
+```
+
+So based on the above reasons, we should introduce another way to let hint take effect in the view.
+
+## Detailed Design
+### How does origin hint framework work?
+1. Parser: parse the sql text and get the basic information about the hint. Handle hint syntax error in this phase.
+2. Optimizer:
+ 1. Divide and mark the query block. Besides, group the hints in the same query blocks.
+ 2. In the plan builder phase, when we try to build select. We will handle the hints in the current query block. Including doing some simple checks and building the hints structure which can be used by planner.
+ 3. When we build some logical operators, we will use the hints which belongs to the current query block. And tt will use the table in hint to match the table in the plan node. For example, when we build the `DataSource` operator, it will generate the possible access path based on the index hints. When we build the `Aggregation` operator, it will set the aggregation algorithm based on the agg hints. And for the `Join` operator, it will store the hint in the join node and use the hint information in the physical optimization phase. The warning about which table is not used in the hint will be recorded in this phase.
+ 4. Use the hint information in the physical optimization phase to determine which physical algorithm should be used. And if the hint can not take effect, it will report warning. For example, if the join can not use the index join, but we set the index join hint in the sql text. It will report related warnings.
+
+### View Hint Design
+Based on the goal and current infrastructure for hint. I extend the current usage of the qb_name hint to a bigger scope to support the view hint.
+
+An example to show the usage of the current `qb_name` hint.
+```SQL
+select /*+ stream_agg(@qb) merge_join(t1@qb)*/ * from (select /*+ qb_name(qb) */ count(*) from t1 join t2 on t1.a = t2.a) tt;
+```
+1. First, we define the name for some query blocks.
+2. Then we can use the query block name to represent the query block.
+
+Based on the meaning of `qb_name` hint now, we can expand it to support the view. The basic idea is the same here. We define the query block name in the view first. And then we can use the query block name to represent the contents in the view. Now the grammar is expanded from
+`qb_name(name)` in the query block which you want to rename
+To
+`qb_name(name, viewName@queryBlockNum . {viewName}@queryBlockNum . ...)` in the first query block to represent any query block. Besides, we will reset the count for query block in every view. It means, for every view, it always counts from 1 and it will not effect the outer part.
+For example:
+```SQL
+create table t(a int, b int);
+create table t1(a int, b int);
+create table t2(a int, b int);
+
+create view v as select {@SEL_1}{5} t.a, t.b from t join (select {@SEL_2}{6} t1.a from t1 join t2 on t1.b=t2.b) tt on t.a = tt.a;
+
+create view v1 as select {@SEL_1}{3} t.a, t.b from t join (select {@SEL_2}{4} from t1 join v on t1.b=v.b) tt on t.a = tt.a;
+
+create view v2 as select {@SEL_1}{1} t.a, t.b from t join (select {@SEL_2}{2} t1.a from t1 join v1 join v3 on t1.b=v1.b) tt on t.a = tt.a;
+
+select {@SEL_1} * from v2;
+
+/* We can use the following part to represent the {1} - {6} */
+1: qb_name(v2_sel1, v2@sel_1 . @sel_1)
+2: qb_name(v2_sel2, v2@sel_1 . @sel_2)
+3: qb_name(v1_sel1, v2@sel_1 . v1@sel_2 . @sel_1)
+4: qb_name(v1_sel2, v2@sel_1 . v1@sel_2 . @sel_2)
+5: qb_name(v_sel1, v2@sel_1 . v1@sel_2 . v@sel_2 . @sel_1)
+6: qb_name(v_sel2, v2@sel_1 . v1@sel_2 . v@sel_2 . @sel_2)
+```
+Take the previous as example:
+```SQL
+CREATE OR REPLACE VIEW v AS
+SELECT * from e1 join (select count(*) from e3) e2 on e1.a = e2.a;
+
+
+/* In Oracle */
+A1. SELECT /*+ INDEX(v.e2.e3 idx) */ * FROM v;
+
+A2. SELECT /*+ INDEX(@SEL$1 v.e2.e3 idx) */ * FROM v;
+
+B. SELECT /*+ INDEX(@SEL$2 e2.e3 idx) */ * FROM v;
+
+C. SELECT /*+ INDEX(@SEL$3 e3 idx) */ * FROM v;
+
+/* In TiDB */
+SELECT /*+ qb_name(viewSub, v@sel_1 . @sel_2) use_index(e3@viewSub, idx) hash_agg(viewSub) */ * FROM v;
+```
+
+### Implementation
+Parser part is easy to implement. Just to expand the origin `qb_name` hint grammar. The only problem maybe is how to express the nested view(use dot or blank or something else).
+
+For the planner part:
+1. At the beginning of the optimization, we should handle the query block name hint for view and the other method hints for view. And group these hints based on the query block name.
+2. When we try to build the data source from the view, we have to traverse all of the query blocks for views. Check whether the view name in hint can match the data source or not. If there are some hints that can match, we pass it to the `buildDataSourceFromView`.
+3. When we try to build the view plan, we first handle the hints which are passed by the caller. Distinguish which hints belong to the current view and which belongs to the nested view. If the hint belongs to the current view, we transform the hint to the normal hint. If the hints belong to the nested view. Then we will do the same thing, like step2.
+
+Besides the planner part, we need support to show the query block for a sql to increase usability. The user can copy the result and use it in hint directly.
+
+### Support Scope
+1. We can support almost all physical algorithm's hints. Like join hints/ agg hints/ index etc.
+2. Do not support the leading hints which may be across the view. But we can support the leading hint in the same view.
+
+### Pros and Cons
+Pros:
+1. No compatibility problems. Just expand the usage of the existing hint.
+2. It is easier to implement. It can use the origin hints' infrastructure as much as possible.
+3. It can support almost all the hints which can take effect in the query block. Oracle can only support the join order, join method and access path hints.
+
+Cons:
+1. It may not be easy to write the query block name hint for a view.
+2. The user should define the query block name hint first.
+
+## Reference
+[Oracle Global Hint](https://docs.oracle.com/cd/E18283_01/server.112/e16638/hintsref.htm#i27644)
+
+
diff --git a/docs/logo_with_text.png b/docs/logo_with_text.png
deleted file mode 100644
index 722bbf8f8c53a..0000000000000
Binary files a/docs/logo_with_text.png and /dev/null differ
diff --git a/docs/tidb-architecture.png b/docs/tidb-architecture.png
new file mode 100644
index 0000000000000..e3360c45258dd
Binary files /dev/null and b/docs/tidb-architecture.png differ
diff --git a/docs/tidb-logo-with-text.png b/docs/tidb-logo-with-text.png
new file mode 100644
index 0000000000000..111465b9fc842
Binary files /dev/null and b/docs/tidb-logo-with-text.png differ
diff --git a/domain/BUILD.bazel b/domain/BUILD.bazel
index f7ef9baba6907..6e1618129990e 100644
--- a/domain/BUILD.bazel
+++ b/domain/BUILD.bazel
@@ -6,6 +6,7 @@ go_library(
"domain.go",
"domain_sysvars.go",
"domainctx.go",
+ "historical_stats.go",
"optimize_trace.go",
"plan_replayer.go",
"plan_replayer_dump.go",
@@ -31,10 +32,12 @@ go_library(
"//errno",
"//infoschema",
"//infoschema/perfschema",
+ "//keyspace",
"//kv",
"//meta",
"//metrics",
"//owner",
+ "//parser",
"//parser/ast",
"//parser/model",
"//parser/mysql",
@@ -45,25 +48,31 @@ go_library(
"//sessionctx/variable",
"//statistics/handle",
"//telemetry",
+ "//ttl/ttlworker",
"//types",
"//util",
"//util/chunk",
"//util/dbterror",
"//util/domainutil",
"//util/engine",
+ "//util/etcd",
"//util/execdetails",
"//util/expensivequery",
+ "//util/intest",
"//util/logutil",
"//util/memory",
"//util/memoryusagealarm",
"//util/printer",
+ "//util/replayer",
"//util/servermemorylimit",
"//util/sqlexec",
+ "//util/syncutil",
"@com_github_burntsushi_toml//:toml",
"@com_github_ngaut_pools//:pools",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/metapb",
+ "@com_github_pingcap_kvproto//pkg/pdpb",
"@com_github_pingcap_log//:log",
"@com_github_stretchr_testify//require",
"@com_github_tikv_client_go_v2//oracle",
@@ -105,6 +114,7 @@ go_test(
"//ddl",
"//domain/infosync",
"//errno",
+ "//keyspace",
"//kv",
"//metrics",
"//parser/ast",
@@ -118,6 +128,7 @@ go_test(
"//testkit/testsetup",
"//util",
"//util/mock",
+ "//util/replayer",
"@com_github_ngaut_pools//:pools",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
diff --git a/domain/db_test.go b/domain/db_test.go
index ff6e6b625788a..02f716a27b3a8 100644
--- a/domain/db_test.go
+++ b/domain/db_test.go
@@ -23,6 +23,7 @@ import (
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/domain/infosync"
+ "github.com/pingcap/tidb/keyspace"
"github.com/pingcap/tidb/server"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/store/mockstore"
@@ -73,7 +74,7 @@ func TestNormalSessionPool(t *testing.T) {
domain, err := session.BootstrapSession(store)
require.NoError(t, err)
defer domain.Close()
- info, err1 := infosync.GlobalInfoSyncerInit(context.Background(), "t", func() uint64 { return 1 }, nil, true)
+ info, err1 := infosync.GlobalInfoSyncerInit(context.Background(), "t", func() uint64 { return 1 }, nil, nil, nil, keyspace.CodecV1, true)
require.NoError(t, err1)
conf := config.GetGlobalConfig()
conf.Socket = ""
@@ -107,7 +108,7 @@ func TestAbnormalSessionPool(t *testing.T) {
domain, err := session.BootstrapSession(store)
require.NoError(t, err)
defer domain.Close()
- info, err1 := infosync.GlobalInfoSyncerInit(context.Background(), "t", func() uint64 { return 1 }, nil, true)
+ info, err1 := infosync.GlobalInfoSyncerInit(context.Background(), "t", func() uint64 { return 1 }, nil, nil, nil, keyspace.CodecV1, true)
require.NoError(t, err1)
conf := config.GetGlobalConfig()
conf.Socket = ""
diff --git a/domain/domain.go b/domain/domain.go
index 1016d5ba9b5cb..bd5d0f63c5698 100644
--- a/domain/domain.go
+++ b/domain/domain.go
@@ -17,7 +17,9 @@ package domain
import (
"context"
"fmt"
+ "math"
"math/rand"
+ "sort"
"strconv"
"strings"
"sync"
@@ -29,6 +31,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/metapb"
+ "github.com/pingcap/kvproto/pkg/pdpb"
"github.com/pingcap/log"
"github.com/pingcap/tidb/bindinfo"
"github.com/pingcap/tidb/br/pkg/streamhelper"
@@ -43,6 +46,7 @@ import (
"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/infoschema/perfschema"
+ "github.com/pingcap/tidb/keyspace"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/metrics"
@@ -57,17 +61,22 @@ import (
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/statistics/handle"
"github.com/pingcap/tidb/telemetry"
+ "github.com/pingcap/tidb/ttl/ttlworker"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/domainutil"
"github.com/pingcap/tidb/util/engine"
+ "github.com/pingcap/tidb/util/etcd"
"github.com/pingcap/tidb/util/expensivequery"
+ "github.com/pingcap/tidb/util/intest"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/memoryusagealarm"
+ "github.com/pingcap/tidb/util/replayer"
"github.com/pingcap/tidb/util/servermemorylimit"
"github.com/pingcap/tidb/util/sqlexec"
+ "github.com/tikv/client-go/v2/tikv"
"github.com/tikv/client-go/v2/txnkv/transaction"
pd "github.com/tikv/pd/client"
clientv3 "go.etcd.io/etcd/client/v3"
@@ -90,33 +99,42 @@ func NewMockDomain() *Domain {
// Domain represents a storage space. Different domains can use the same database name.
// Multiple domains can be used in parallel without synchronization.
type Domain struct {
- store kv.Storage
- infoCache *infoschema.InfoCache
- privHandle *privileges.Handle
- bindHandle atomic.Pointer[bindinfo.BindHandle]
- statsHandle unsafe.Pointer
- statsLease time.Duration
- ddl ddl.DDL
- info *infosync.InfoSyncer
- globalCfgSyncer *globalconfigsync.GlobalConfigSyncer
- m sync.Mutex
- SchemaValidator SchemaValidator
- sysSessionPool *sessionPool
- exit chan struct{}
- etcdClient *clientv3.Client
+ store kv.Storage
+ infoCache *infoschema.InfoCache
+ privHandle *privileges.Handle
+ bindHandle atomic.Pointer[bindinfo.BindHandle]
+ statsHandle unsafe.Pointer
+ statsLease time.Duration
+ ddl ddl.DDL
+ info *infosync.InfoSyncer
+ globalCfgSyncer *globalconfigsync.GlobalConfigSyncer
+ m sync.Mutex
+ SchemaValidator SchemaValidator
+ sysSessionPool *sessionPool
+ exit chan struct{}
+ // `etcdClient` must be used when keyspace is not set, or when the logic to each etcd path needs to be separated by keyspace.
+ etcdClient *clientv3.Client
+ // `unprefixedEtcdCli` will never set the etcd namespace prefix by keyspace.
+ // It is only used in storeMinStartTS and RemoveMinStartTS now.
+ // It must be used when the etcd path isn't needed to separate by keyspace.
+ // See keyspace RFC: https://github.com/pingcap/tidb/pull/39685
+ unprefixedEtcdCli *clientv3.Client
sysVarCache sysVarCache // replaces GlobalVariableCache
slowQuery *topNSlowQueries
expensiveQueryHandle *expensivequery.Handle
memoryUsageAlarmHandle *memoryusagealarm.Handle
serverMemoryLimitHandle *servermemorylimit.Handle
- wg util.WaitGroupWrapper
- statsUpdating atomicutil.Int32
- cancel context.CancelFunc
- indexUsageSyncLease time.Duration
- dumpFileGcChecker *dumpFileGcChecker
- planReplayerHandle *planReplayerHandle
- expiredTimeStamp4PC types.Time
- logBackupAdvancer *daemon.OwnerDaemon
+ // TODO: use Run for each process in future pr
+ wg *util.WaitGroupEnhancedWrapper
+ statsUpdating atomicutil.Int32
+ cancel context.CancelFunc
+ indexUsageSyncLease time.Duration
+ dumpFileGcChecker *dumpFileGcChecker
+ planReplayerHandle *planReplayerHandle
+ expiredTimeStamp4PC types.Time
+ logBackupAdvancer *daemon.OwnerDaemon
+ historicalStatsWorker *HistoricalStatsWorker
+ ttlJobManager atomic.Pointer[ttlworker.JobManager]
serverID uint64
serverIDSession *concurrency.Session
@@ -181,6 +199,7 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
// 1. Not first time bootstrap loading, which needs a full load.
// 2. It is newer than the current one, so it will be "the current one" after this function call.
// 3. There are less 100 diffs.
+ // 4. No regenrated schema diff.
startTime := time.Now()
if currentSchemaVersion != 0 && neededSchemaVersion > currentSchemaVersion && neededSchemaVersion-currentSchemaVersion < 100 {
is, relatedChanges, err := do.tryLoadSchemaDiffs(m, currentSchemaVersion, neededSchemaVersion)
@@ -190,6 +209,7 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
zap.Int64("currentSchemaVersion", currentSchemaVersion),
zap.Int64("neededSchemaVersion", neededSchemaVersion),
zap.Duration("start time", time.Since(startTime)),
+ zap.Int64("gotSchemaVersion", is.SchemaMetaVersion()),
zap.Int64s("phyTblIDs", relatedChanges.PhyTblIDS),
zap.Uint64s("actionTypes", relatedChanges.ActionTypes))
return is, false, currentSchemaVersion, relatedChanges, nil
@@ -208,7 +228,12 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
return nil, false, currentSchemaVersion, nil, err
}
- newISBuilder, err := infoschema.NewBuilder(do.Store(), do.sysFacHack).InitWithDBInfos(schemas, policies, neededSchemaVersion)
+ resourceGroups, err := do.fetchResourceGroups(m)
+ if err != nil {
+ return nil, false, currentSchemaVersion, nil, err
+ }
+
+ newISBuilder, err := infoschema.NewBuilder(do.Store(), do.sysFacHack).InitWithDBInfos(schemas, policies, resourceGroups, neededSchemaVersion)
if err != nil {
return nil, false, currentSchemaVersion, nil, err
}
@@ -240,6 +265,14 @@ func (do *Domain) fetchPolicies(m *meta.Meta) ([]*model.PolicyInfo, error) {
return allPolicies, nil
}
+func (do *Domain) fetchResourceGroups(m *meta.Meta) ([]*model.ResourceGroupInfo, error) {
+ allResourceGroups, err := m.ListResourceGroups()
+ if err != nil {
+ return nil, err
+ }
+ return allResourceGroups, nil
+}
+
func (do *Domain) fetchAllSchemasWithTables(m *meta.Meta) ([]*model.DBInfo, error) {
allSchemas, err := m.ListDatabases()
if err != nil {
@@ -340,12 +373,15 @@ func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64
if err != nil {
return nil, nil, err
}
+ if diff.RegenerateSchemaMap {
+ return nil, nil, errors.Errorf("Meets a schema diff with RegenerateSchemaMap flag")
+ }
if canSkipSchemaCheckerDDL(diff.Type) {
continue
}
phyTblIDs = append(phyTblIDs, IDs...)
for i := 0; i < len(IDs); i++ {
- actions = append(actions, uint64(1< 0 {
- do.wg.Add(1)
// Local store needs to get the change information for every DDL state in each session.
- go do.loadSchemaInLoop(ctx, ddlLease)
- }
- do.wg.Run(do.mdlCheckLoop)
- do.wg.Add(3)
- go do.topNSlowQueryLoop()
- go do.infoSyncerKeeper()
- go do.globalConfigSyncerKeeper()
+ do.wg.Run(func() {
+ do.loadSchemaInLoop(ctx, ddlLease)
+ }, "loadSchemaInLoop")
+ }
+ do.wg.Run(do.mdlCheckLoop, "mdlCheckLoop")
+ do.wg.Run(do.topNSlowQueryLoop, "topNSlowQueryLoop")
+ do.wg.Run(do.infoSyncerKeeper, "infoSyncerKeeper")
+ do.wg.Run(do.globalConfigSyncerKeeper, "globalConfigSyncerKeeper")
if !skipRegisterToDashboard {
- do.wg.Add(1)
- go do.topologySyncerKeeper()
+ do.wg.Run(do.topologySyncerKeeper, "topologySyncerKeeper")
}
- if pdClient != nil {
- do.wg.Add(1)
- go do.closestReplicaReadCheckLoop(ctx, pdClient)
+ if pdCli != nil {
+ do.wg.Run(func() {
+ do.closestReplicaReadCheckLoop(ctx, pdCli)
+ }, "closestReplicaReadCheckLoop")
}
- err = do.initLogBackup(ctx, pdClient)
+ err = do.initLogBackup(ctx, pdCli)
if err != nil {
return err
}
@@ -1058,6 +1134,11 @@ func (do *Domain) Init(
return nil
}
+// SetOnClose used to set do.onClose func.
+func (do *Domain) SetOnClose(onClose func()) {
+ do.onClose = onClose
+}
+
func (do *Domain) initLogBackup(ctx context.Context, pdClient pd.Client) error {
cfg := config.GetGlobalConfig()
if pdClient == nil || do.etcdClient == nil {
@@ -1074,7 +1155,7 @@ func (do *Domain) initLogBackup(ctx context.Context, pdClient pd.Client) error {
if err != nil {
return err
}
- do.wg.Run(loop)
+ do.wg.Run(loop, "logBackupAdvancer")
return nil
}
@@ -1091,7 +1172,6 @@ func (do *Domain) closestReplicaReadCheckLoop(ctx context.Context, pdClient pd.C
ticker := time.NewTicker(time.Minute)
defer func() {
ticker.Stop()
- do.wg.Done()
logutil.BgLogger().Info("closestReplicaReadCheckLoop exited.")
}()
for {
@@ -1106,8 +1186,12 @@ func (do *Domain) closestReplicaReadCheckLoop(ctx context.Context, pdClient pd.C
}
}
+// Periodically check and update the replica-read status when `tidb_replica_read` is set to "closest-adaptive"
+// We disable "closest-adaptive" in following conditions to ensure the read traffic is evenly distributed across
+// all AZs:
+// - There are no TiKV servers in the AZ of this tidb instance
+// - The AZ if this tidb contains more tidb than other AZ and this tidb's id is the bigger one.
func (do *Domain) checkReplicaRead(ctx context.Context, pdClient pd.Client) error {
- // fast path
do.sysVarCache.RLock()
replicaRead := do.sysVarCache.global[variable.TiDBReplicaRead]
do.sysVarCache.RUnlock()
@@ -1116,6 +1200,24 @@ func (do *Domain) checkReplicaRead(ctx context.Context, pdClient pd.Client) erro
logutil.BgLogger().Debug("closest replica read is not enabled, skip check!", zap.String("mode", replicaRead))
return nil
}
+
+ serverInfo, err := infosync.GetServerInfo()
+ if err != nil {
+ return err
+ }
+ zone := ""
+ for k, v := range serverInfo.Labels {
+ if k == placement.DCLabelKey && v != "" {
+ zone = v
+ break
+ }
+ }
+ if zone == "" {
+ logutil.BgLogger().Debug("server contains no 'zone' label, disable closest replica read", zap.Any("labels", serverInfo.Labels))
+ variable.SetEnableAdaptiveReplicaRead(false)
+ return nil
+ }
+
stores, err := pdClient.GetAllStores(ctx, pd.WithExcludeTombstone())
if err != nil {
return err
@@ -1135,32 +1237,48 @@ func (do *Domain) checkReplicaRead(ctx context.Context, pdClient pd.Client) erro
}
}
- enabled := false
- // if stores don't have zone labels or are distribued in 1 zone, just disable cloeset replica read.
- if len(storeZones) > 1 {
- enabled = true
- servers, err := infosync.GetAllServerInfo(ctx)
- if err != nil {
- return err
- }
- for _, s := range servers {
- if v, ok := s.Labels[placement.DCLabelKey]; ok && v != "" {
- if _, ok := storeZones[v]; !ok {
- enabled = false
- break
- }
+ // no stores in this AZ
+ if _, ok := storeZones[zone]; !ok {
+ variable.SetEnableAdaptiveReplicaRead(false)
+ return nil
+ }
+
+ servers, err := infosync.GetAllServerInfo(ctx)
+ if err != nil {
+ return err
+ }
+ svrIdsInThisZone := make([]string, 0)
+ for _, s := range servers {
+ if v, ok := s.Labels[placement.DCLabelKey]; ok && v != "" {
+ if _, ok := storeZones[v]; ok {
storeZones[v] += 1
- }
- }
- if enabled {
- for _, count := range storeZones {
- if count == 0 {
- enabled = false
- break
+ if v == zone {
+ svrIdsInThisZone = append(svrIdsInThisZone, s.ID)
}
}
}
}
+ enabledCount := math.MaxInt
+ for _, count := range storeZones {
+ if count < enabledCount {
+ enabledCount = count
+ }
+ }
+ // sort tidb in the same AZ by ID and disable the tidb with bigger ID
+ // because ID is unchangeable, so this is a simple and stable algorithm to select
+ // some instances across all tidb servers.
+ if enabledCount < len(svrIdsInThisZone) {
+ sort.Slice(svrIdsInThisZone, func(i, j int) bool {
+ return strings.Compare(svrIdsInThisZone[i], svrIdsInThisZone[j]) < 0
+ })
+ }
+ enabled := true
+ for _, s := range svrIdsInThisZone[enabledCount:] {
+ if s == serverInfo.ID {
+ enabled = false
+ break
+ }
+ }
if variable.SetEnableAdaptiveReplicaRead(enabled) {
logutil.BgLogger().Info("tidb server adaptive closest replica read is changed", zap.Bool("enable", enabled))
@@ -1253,6 +1371,14 @@ func (do *Domain) GetEtcdClient() *clientv3.Client {
return do.etcdClient
}
+// GetPDClient returns the PD client.
+func (do *Domain) GetPDClient() pd.Client {
+ if store, ok := do.store.(kv.StorageWithPD); ok {
+ return store.GetPDClient()
+ }
+ return nil
+}
+
// LoadPrivilegeLoop create a goroutine loads privilege tables in a loop, it
// should be called only once in BootstrapSession.
func (do *Domain) LoadPrivilegeLoop(sctx sessionctx.Context) error {
@@ -1275,10 +1401,8 @@ func (do *Domain) LoadPrivilegeLoop(sctx sessionctx.Context) error {
duration = 10 * time.Minute
}
- do.wg.Add(1)
- go func() {
+ do.wg.Run(func() {
defer func() {
- do.wg.Done()
logutil.BgLogger().Info("loadPrivilegeInLoop exited.")
util.Recover(metrics.LabelDomain, "loadPrivilegeInLoop", nil, false)
}()
@@ -1308,7 +1432,7 @@ func (do *Domain) LoadPrivilegeLoop(sctx sessionctx.Context) error {
logutil.BgLogger().Error("load privilege failed", zap.Error(err))
}
}
- }()
+ }, "loadPrivilegeInLoop")
return nil
}
@@ -1325,10 +1449,9 @@ func (do *Domain) LoadSysVarCacheLoop(ctx sessionctx.Context) error {
if do.etcdClient != nil {
watchCh = do.etcdClient.Watch(context.Background(), sysVarCacheKey)
}
- do.wg.Add(1)
- go func() {
+
+ do.wg.Run(func() {
defer func() {
- do.wg.Done()
logutil.BgLogger().Info("LoadSysVarCacheLoop exited.")
util.Recover(metrics.LabelDomain, "LoadSysVarCacheLoop", nil, false)
}()
@@ -1371,7 +1494,63 @@ func (do *Domain) LoadSysVarCacheLoop(ctx sessionctx.Context) error {
logutil.BgLogger().Error("LoadSysVarCacheLoop failed", zap.Error(err))
}
}
- }()
+ }, "LoadSysVarCacheLoop")
+ return nil
+}
+
+// WatchTiFlashComputeNodeChange create a routine to watch if the topology of tiflash_compute node is changed.
+// TODO: tiflashComputeNodeKey is not put to etcd yet(finish this when AutoScaler is done)
+//
+// store cache will only be invalidated every n seconds.
+func (do *Domain) WatchTiFlashComputeNodeChange() error {
+ var watchCh clientv3.WatchChan
+ if do.etcdClient != nil {
+ watchCh = do.etcdClient.Watch(context.Background(), tiflashComputeNodeKey)
+ }
+ duration := 10 * time.Second
+ do.wg.Run(func() {
+ defer func() {
+ logutil.BgLogger().Info("WatchTiFlashComputeNodeChange exit")
+ util.Recover(metrics.LabelDomain, "WatchTiFlashComputeNodeChange", nil, false)
+ }()
+
+ var count int
+ var logCount int
+ for {
+ ok := true
+ var watched bool
+ select {
+ case <-do.exit:
+ return
+ case _, ok = <-watchCh:
+ watched = true
+ case <-time.After(duration):
+ }
+ if !ok {
+ logutil.BgLogger().Error("WatchTiFlashComputeNodeChange watch channel closed")
+ watchCh = do.etcdClient.Watch(context.Background(), tiflashComputeNodeKey)
+ count++
+ if count > 10 {
+ time.Sleep(time.Duration(count) * time.Second)
+ }
+ continue
+ }
+ count = 0
+ switch s := do.store.(type) {
+ case tikv.Storage:
+ logCount++
+ s.GetRegionCache().InvalidateTiFlashComputeStores()
+ if logCount == 6 {
+ // Print log every 6*duration seconds.
+ logutil.BgLogger().Debug("tiflash_compute store cache invalied, will update next query", zap.Bool("watched", watched))
+ logCount = 0
+ }
+ default:
+ logutil.BgLogger().Debug("No need to watch tiflash_compute store cache for non-tikv store")
+ return
+ }
+ }
+ }, "WatchTiFlashComputeNodeChange")
return nil
}
@@ -1406,10 +1585,8 @@ func (do *Domain) LoadBindInfoLoop(ctxForHandle sessionctx.Context, ctxForEvolve
}
func (do *Domain) globalBindHandleWorkerLoop(owner owner.Manager) {
- do.wg.Add(1)
- go func() {
+ do.wg.Run(func() {
defer func() {
- do.wg.Done()
logutil.BgLogger().Info("globalBindHandleWorkerLoop exited.")
util.Recover(metrics.LabelDomain, "globalBindHandleWorkerLoop", nil, false)
}()
@@ -1446,14 +1623,12 @@ func (do *Domain) globalBindHandleWorkerLoop(owner owner.Manager) {
}
}
}
- }()
+ }, "globalBindHandleWorkerLoop")
}
func (do *Domain) handleEvolvePlanTasksLoop(ctx sessionctx.Context, owner owner.Manager) {
- do.wg.Add(1)
- go func() {
+ do.wg.Run(func() {
defer func() {
- do.wg.Done()
logutil.BgLogger().Info("handleEvolvePlanTasksLoop exited.")
util.Recover(metrics.LabelDomain, "handleEvolvePlanTasksLoop", nil, false)
}()
@@ -1471,7 +1646,7 @@ func (do *Domain) handleEvolvePlanTasksLoop(ctx sessionctx.Context, owner owner.
}
}
}
- }()
+ }, "handleEvolvePlanTasksLoop")
}
// TelemetryReportLoop create a goroutine that reports usage data in a loop, it should be called only once
@@ -1483,10 +1658,8 @@ func (do *Domain) TelemetryReportLoop(ctx sessionctx.Context) {
logutil.BgLogger().Warn("Initial telemetry run failed", zap.Error(err))
}
- do.wg.Add(1)
- go func() {
+ do.wg.Run(func() {
defer func() {
- do.wg.Done()
logutil.BgLogger().Info("TelemetryReportLoop exited.")
util.Recover(metrics.LabelDomain, "TelemetryReportLoop", nil, false)
}()
@@ -1507,16 +1680,14 @@ func (do *Domain) TelemetryReportLoop(ctx sessionctx.Context) {
}
}
}
- }()
+ }, "TelemetryReportLoop")
}
// TelemetryRotateSubWindowLoop create a goroutine that rotates the telemetry window regularly.
func (do *Domain) TelemetryRotateSubWindowLoop(ctx sessionctx.Context) {
ctx.GetSessionVars().InRestrictedSQL = true
- do.wg.Add(1)
- go func() {
+ do.wg.Run(func() {
defer func() {
- do.wg.Done()
logutil.BgLogger().Info("TelemetryRotateSubWindowLoop exited.")
util.Recover(metrics.LabelDomain, "TelemetryRotateSubWindowLoop", nil, false)
}()
@@ -1528,56 +1699,108 @@ func (do *Domain) TelemetryRotateSubWindowLoop(ctx sessionctx.Context) {
telemetry.RotateSubWindow()
}
}
- }()
+ }, "TelemetryRotateSubWindowLoop")
}
// SetupPlanReplayerHandle setup plan replayer handle
-func (do *Domain) SetupPlanReplayerHandle(ctx sessionctx.Context) {
- do.planReplayerHandle = &planReplayerHandle{
- planReplayerTaskCollectorHandle: &planReplayerTaskCollectorHandle{
- sctx: ctx,
- },
+func (do *Domain) SetupPlanReplayerHandle(collectorSctx sessionctx.Context, workersSctxs []sessionctx.Context) {
+ ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnStats)
+ do.planReplayerHandle = &planReplayerHandle{}
+ do.planReplayerHandle.planReplayerTaskCollectorHandle = &planReplayerTaskCollectorHandle{
+ ctx: ctx,
+ sctx: collectorSctx,
+ }
+ taskCH := make(chan *PlanReplayerDumpTask, 16)
+ taskStatus := &planReplayerDumpTaskStatus{}
+ taskStatus.finishedTaskMu.finishedTask = map[replayer.PlanReplayerTaskKey]struct{}{}
+ taskStatus.runningTaskMu.runningTasks = map[replayer.PlanReplayerTaskKey]struct{}{}
+
+ do.planReplayerHandle.planReplayerTaskDumpHandle = &planReplayerTaskDumpHandle{
+ taskCH: taskCH,
+ status: taskStatus,
+ }
+ do.planReplayerHandle.planReplayerTaskDumpHandle.workers = make([]*planReplayerTaskDumpWorker, 0)
+ for i := 0; i < len(workersSctxs); i++ {
+ worker := &planReplayerTaskDumpWorker{
+ ctx: ctx,
+ sctx: workersSctxs[i],
+ taskCH: taskCH,
+ status: taskStatus,
+ }
+ do.planReplayerHandle.planReplayerTaskDumpHandle.workers = append(do.planReplayerHandle.planReplayerTaskDumpHandle.workers, worker)
+ }
+}
+
+// SetupHistoricalStatsWorker setups worker
+func (do *Domain) SetupHistoricalStatsWorker(ctx sessionctx.Context) {
+ do.historicalStatsWorker = &HistoricalStatsWorker{
+ tblCH: make(chan int64, 16),
+ sctx: ctx,
}
}
// SetupDumpFileGCChecker setup sctx
func (do *Domain) SetupDumpFileGCChecker(ctx sessionctx.Context) {
do.dumpFileGcChecker.setupSctx(ctx)
+ do.dumpFileGcChecker.planReplayerTaskStatus = do.planReplayerHandle.status
}
-var planReplayerHandleLease = 10 * time.Second
+var planReplayerHandleLease atomic.Uint64
+
+func init() {
+ planReplayerHandleLease.Store(uint64(10 * time.Second))
+ enableDumpHistoricalStats.Store(true)
+}
// DisablePlanReplayerBackgroundJob4Test disable plan replayer handle for test
func DisablePlanReplayerBackgroundJob4Test() {
- planReplayerHandleLease = 0
+ planReplayerHandleLease.Store(0)
+}
+
+// DisableDumpHistoricalStats4Test disable historical dump worker for test
+func DisableDumpHistoricalStats4Test() {
+ enableDumpHistoricalStats.Store(false)
}
// StartPlanReplayerHandle start plan replayer handle job
func (do *Domain) StartPlanReplayerHandle() {
- if planReplayerHandleLease < 1 {
+ lease := planReplayerHandleLease.Load()
+ if lease < 1 {
return
}
- do.wg.Add(1)
- go func() {
- tikcer := time.NewTicker(planReplayerHandleLease)
+ do.wg.Run(func() {
+ logutil.BgLogger().Info("PlanReplayerTaskCollectHandle started")
+ tikcer := time.NewTicker(time.Duration(lease))
defer func() {
tikcer.Stop()
- do.wg.Done()
- logutil.BgLogger().Info("PlanReplayerHandle exited.")
- util.Recover(metrics.LabelDomain, "PlanReplayerHandle", nil, false)
+ util.Recover(metrics.LabelDomain, "PlanReplayerTaskCollectHandle", nil, false)
+ logutil.BgLogger().Info("PlanReplayerTaskCollectHandle exited.")
}()
for {
select {
case <-do.exit:
return
case <-tikcer.C:
- err := do.planReplayerHandle.CollectPlanReplayerTask(context.Background())
+ err := do.planReplayerHandle.CollectPlanReplayerTask()
if err != nil {
logutil.BgLogger().Warn("plan replayer handle collect tasks failed", zap.Error(err))
}
}
}
- }()
+ }, "PlanReplayerTaskCollectHandle")
+
+ do.wg.Run(func() {
+ logutil.BgLogger().Info("PlanReplayerTaskDumpHandle started")
+ defer func() {
+ util.Recover(metrics.LabelDomain, "PlanReplayerTaskDumpHandle", nil, false)
+ logutil.BgLogger().Info("PlanReplayerTaskDumpHandle exited.")
+ }()
+ for _, worker := range do.planReplayerHandle.planReplayerTaskDumpHandle.workers {
+ go worker.run()
+ }
+ <-do.exit
+ do.planReplayerHandle.planReplayerTaskDumpHandle.Close()
+ }, "PlanReplayerTaskDumpHandle")
}
// GetPlanReplayerHandle returns plan replayer handle
@@ -1587,14 +1810,13 @@ func (do *Domain) GetPlanReplayerHandle() *planReplayerHandle {
// DumpFileGcCheckerLoop creates a goroutine that handles `exit` and `gc`.
func (do *Domain) DumpFileGcCheckerLoop() {
- do.wg.Add(1)
- go func() {
+ do.wg.Run(func() {
+ logutil.BgLogger().Info("dumpFileGcChecker started")
gcTicker := time.NewTicker(do.dumpFileGcChecker.gcLease)
defer func() {
gcTicker.Stop()
- do.wg.Done()
- logutil.BgLogger().Info("dumpFileGcChecker exited.")
util.Recover(metrics.LabelDomain, "dumpFileGcCheckerLoop", nil, false)
+ logutil.BgLogger().Info("dumpFileGcChecker exited.")
}()
for {
select {
@@ -1604,7 +1826,44 @@ func (do *Domain) DumpFileGcCheckerLoop() {
do.dumpFileGcChecker.gcDumpFiles(time.Hour)
}
}
- }()
+ }, "dumpFileGcChecker")
+}
+
+// GetHistoricalStatsWorker gets historical workers
+func (do *Domain) GetHistoricalStatsWorker() *HistoricalStatsWorker {
+ return do.historicalStatsWorker
+}
+
+// EnableDumpHistoricalStats used to control whether enbale dump stats for unit test
+var enableDumpHistoricalStats atomic.Bool
+
+// StartHistoricalStatsWorker start historical workers running
+func (do *Domain) StartHistoricalStatsWorker() {
+ if !enableDumpHistoricalStats.Load() {
+ return
+ }
+ do.wg.Run(func() {
+ logutil.BgLogger().Info("HistoricalStatsWorker started")
+ defer func() {
+ util.Recover(metrics.LabelDomain, "HistoricalStatsWorkerLoop", nil, false)
+ logutil.BgLogger().Info("HistoricalStatsWorker exited.")
+ }()
+ for {
+ select {
+ case <-do.exit:
+ close(do.historicalStatsWorker.tblCH)
+ return
+ case tblID, ok := <-do.historicalStatsWorker.tblCH:
+ if !ok {
+ return
+ }
+ err := do.historicalStatsWorker.DumpHistoricalStats(tblID, do.StatsHandle())
+ if err != nil {
+ logutil.BgLogger().Warn("dump historical stats failed", zap.Error(err), zap.Int64("tableID", tblID))
+ }
+ }
+ }
+ }, "HistoricalStatsWorker")
}
// StatsHandle returns the statistic handle.
@@ -1613,8 +1872,8 @@ func (do *Domain) StatsHandle() *handle.Handle {
}
// CreateStatsHandle is used only for test.
-func (do *Domain) CreateStatsHandle(ctx sessionctx.Context) error {
- h, err := handle.NewHandle(ctx, do.statsLease, do.sysSessionPool, &do.sysProcesses, do.ServerID)
+func (do *Domain) CreateStatsHandle(ctx, initStatsCtx sessionctx.Context) error {
+ h, err := handle.NewHandle(ctx, initStatsCtx, do.statsLease, do.sysSessionPool, &do.sysProcesses, do.ServerID)
if err != nil {
return err
}
@@ -1677,8 +1936,8 @@ func (do *Domain) SetupAnalyzeExec(ctxs []sessionctx.Context) {
}
// LoadAndUpdateStatsLoop loads and updates stats info.
-func (do *Domain) LoadAndUpdateStatsLoop(ctxs []sessionctx.Context) error {
- if err := do.UpdateTableStatsLoop(ctxs[0]); err != nil {
+func (do *Domain) LoadAndUpdateStatsLoop(ctxs []sessionctx.Context, initStatsCtx sessionctx.Context) error {
+ if err := do.UpdateTableStatsLoop(ctxs[0], initStatsCtx); err != nil {
return err
}
do.StartLoadStatsSubWorkers(ctxs[1:])
@@ -1688,9 +1947,9 @@ func (do *Domain) LoadAndUpdateStatsLoop(ctxs []sessionctx.Context) error {
// UpdateTableStatsLoop creates a goroutine loads stats info and updates stats info in a loop.
// It will also start a goroutine to analyze tables automatically.
// It should be called only once in BootstrapSession.
-func (do *Domain) UpdateTableStatsLoop(ctx sessionctx.Context) error {
+func (do *Domain) UpdateTableStatsLoop(ctx, initStatsCtx sessionctx.Context) error {
ctx.GetSessionVars().InRestrictedSQL = true
- statsHandle, err := handle.NewHandle(ctx, do.statsLease, do.sysSessionPool, &do.sysProcesses, do.ServerID)
+ statsHandle, err := handle.NewHandle(ctx, initStatsCtx, do.statsLease, do.sysSessionPool, &do.sysProcesses, do.ServerID)
if err != nil {
return err
}
@@ -1698,20 +1957,21 @@ func (do *Domain) UpdateTableStatsLoop(ctx sessionctx.Context) error {
do.ddl.RegisterStatsHandle(statsHandle)
// Negative stats lease indicates that it is in test, it does not need update.
if do.statsLease >= 0 {
- do.wg.Run(do.loadStatsWorker)
+ do.wg.Run(do.loadStatsWorker, "loadStatsWorker")
}
owner := do.newOwnerManager(handle.StatsPrompt, handle.StatsOwnerKey)
if do.indexUsageSyncLease > 0 {
- do.wg.Add(1)
- go do.syncIndexUsageWorker(owner)
+ do.wg.Run(func() {
+ do.syncIndexUsageWorker(owner)
+ }, "syncIndexUsageWorker")
}
if do.statsLease <= 0 {
return nil
}
do.SetStatsUpdating(true)
- do.wg.Run(func() { do.updateStatsWorker(ctx, owner) })
- do.wg.Run(func() { do.autoAnalyzeWorker(owner) })
- do.wg.Run(func() { do.gcAnalyzeHistory(owner) })
+ do.wg.Run(func() { do.updateStatsWorker(ctx, owner) }, "updateStatsWorker")
+ do.wg.Run(func() { do.autoAnalyzeWorker(owner) }, "autoAnalyzeWorker")
+ do.wg.Run(func() { do.gcAnalyzeHistory(owner) }, "gcAnalyzeHistory")
return nil
}
@@ -1721,7 +1981,7 @@ func (do *Domain) StartLoadStatsSubWorkers(ctxList []sessionctx.Context) {
for i, ctx := range ctxList {
statsHandle.StatsLoad.SubCtxs[i] = ctx
do.wg.Add(1)
- go statsHandle.SubLoadWorker(ctx, do.exit, &do.wg)
+ go statsHandle.SubLoadWorker(ctx, do.exit, do.wg)
}
}
@@ -1748,15 +2008,17 @@ func (do *Domain) loadStatsWorker() {
lease = 3 * time.Second
}
loadTicker := time.NewTicker(lease)
+ updStatsHealthyTicker := time.NewTicker(20 * lease)
defer func() {
loadTicker.Stop()
+ updStatsHealthyTicker.Stop()
logutil.BgLogger().Info("loadStatsWorker exited.")
}()
statsHandle := do.StatsHandle()
t := time.Now()
err := statsHandle.InitStats(do.InfoSchema())
if err != nil {
- logutil.BgLogger().Debug("init stats info failed", zap.Error(err))
+ logutil.BgLogger().Error("init stats info failed", zap.Duration("take time", time.Since(t)), zap.Error(err))
} else {
logutil.BgLogger().Info("init stats info time", zap.Duration("take time", time.Since(t)))
}
@@ -1775,6 +2037,8 @@ func (do *Domain) loadStatsWorker() {
if err != nil {
logutil.BgLogger().Debug("load histograms failed", zap.Error(err))
}
+ case <-updStatsHealthyTicker.C:
+ statsHandle.UpdateStatsHealthyMetrics()
case <-do.exit:
return
}
@@ -1788,7 +2052,6 @@ func (do *Domain) syncIndexUsageWorker(owner owner.Manager) {
handle := do.StatsHandle()
defer func() {
idxUsageSyncTicker.Stop()
- do.wg.Done()
logutil.BgLogger().Info("syncIndexUsageWorker exited.")
}()
for {
@@ -1811,8 +2074,30 @@ func (do *Domain) syncIndexUsageWorker(owner owner.Manager) {
}
}
+func (do *Domain) updateStatsWorkerExitPreprocessing(statsHandle *handle.Handle, owner owner.Manager) {
+ ch := make(chan struct{}, 1)
+ timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ go func() {
+ logutil.BgLogger().Info("updateStatsWorker is going to exit, start to flush stats")
+ statsHandle.FlushStats()
+ logutil.BgLogger().Info("updateStatsWorker ready to release owner")
+ owner.Cancel()
+ ch <- struct{}{}
+ }()
+ select {
+ case <-ch:
+ logutil.BgLogger().Info("updateStatsWorker exit preprocessing finished")
+ return
+ case <-timeout.Done():
+ logutil.BgLogger().Warn("updateStatsWorker exit preprocessing timeout, force exiting")
+ return
+ }
+}
+
func (do *Domain) updateStatsWorker(ctx sessionctx.Context, owner owner.Manager) {
defer util.Recover(metrics.LabelDomain, "updateStatsWorker", nil, false)
+ logutil.BgLogger().Info("updateStatsWorker started.")
lease := do.statsLease
deltaUpdateTicker := time.NewTicker(20 * lease)
gcStatsTicker := time.NewTicker(100 * lease)
@@ -1830,19 +2115,19 @@ func (do *Domain) updateStatsWorker(ctx sessionctx.Context, owner owner.Manager)
deltaUpdateTicker.Stop()
readMemTricker.Stop()
do.SetStatsUpdating(false)
+ util.Recover(metrics.LabelDomain, "updateStatsWorker", nil, false)
logutil.BgLogger().Info("updateStatsWorker exited.")
}()
for {
select {
case <-do.exit:
- statsHandle.FlushStats()
- owner.Cancel()
+ do.updateStatsWorkerExitPreprocessing(statsHandle, owner)
return
// This channel is sent only by ddl owner.
case t := <-statsHandle.DDLEventCh():
err := statsHandle.HandleDDLEvent(t)
if err != nil {
- logutil.BgLogger().Debug("handle ddl event failed", zap.Error(err))
+ logutil.BgLogger().Error("handle ddl event failed", zap.String("event", t.String()), zap.Error(err))
}
case <-deltaUpdateTicker.C:
err := statsHandle.DumpStatsDeltaToKV(handle.DumpDelta)
@@ -1951,8 +2236,9 @@ func (do *Domain) ServerMemoryLimitHandle() *servermemorylimit.Handle {
}
const (
- privilegeKey = "/tidb/privilege"
- sysVarCacheKey = "/tidb/sysvars"
+ privilegeKey = "/tidb/privilege"
+ sysVarCacheKey = "/tidb/sysvars"
+ tiflashComputeNodeKey = "/tiflash/new_tiflash_compute_nodes"
)
// NotifyUpdatePrivilege updates privilege key in etcd, TiDB client that watches
@@ -1989,7 +2275,7 @@ func (do *Domain) NotifyUpdatePrivilege() error {
// NotifyUpdateSysVarCache updates the sysvar cache key in etcd, which other TiDB
// clients are subscribed to for updates. For the caller, the cache is also built
// synchronously so that the effect is immediate.
-func (do *Domain) NotifyUpdateSysVarCache() {
+func (do *Domain) NotifyUpdateSysVarCache(updateLocal bool) {
if do.etcdClient != nil {
row := do.etcdClient.KV
_, err := row.Put(context.Background(), sysVarCacheKey, "")
@@ -1998,8 +2284,10 @@ func (do *Domain) NotifyUpdateSysVarCache() {
}
}
// update locally
- if err := do.rebuildSysVarCache(nil); err != nil {
- logutil.BgLogger().Error("rebuilding sysvar cache failed", zap.Error(err))
+ if updateLocal {
+ if err := do.rebuildSysVarCache(nil); err != nil {
+ logutil.BgLogger().Error("rebuilding sysvar cache failed", zap.Error(err))
+ }
}
}
@@ -2008,10 +2296,8 @@ func (do *Domain) LoadSigningCertLoop(signingCert, signingKey string) {
sessionstates.SetCertPath(signingCert)
sessionstates.SetKeyPath(signingKey)
- do.wg.Add(1)
- go func() {
+ do.wg.Run(func() {
defer func() {
- do.wg.Done()
logutil.BgLogger().Debug("loadSigningCertLoop exited.")
util.Recover(metrics.LabelDomain, "LoadSigningCertLoop", nil, false)
}()
@@ -2023,7 +2309,7 @@ func (do *Domain) LoadSigningCertLoop(signingCert, signingKey string) {
return
}
}
- }()
+ }, "loadSigningCertLoop")
}
// ServerID gets serverID.
@@ -2259,6 +2545,26 @@ func (do *Domain) serverIDKeeper() {
}
}
+// StartTTLJobManager creates and starts the ttl job manager
+func (do *Domain) StartTTLJobManager() {
+ do.wg.Run(func() {
+ defer func() {
+ logutil.BgLogger().Info("ttlJobManager exited.")
+ }()
+
+ ttlJobManager := ttlworker.NewJobManager(do.ddl.GetID(), do.sysSessionPool, do.store, do.etcdClient)
+ do.ttlJobManager.Store(ttlJobManager)
+ ttlJobManager.Start()
+
+ <-do.exit
+ }, "ttlJobManager")
+}
+
+// TTLJobManager returns the ttl job manager on this domain
+func (do *Domain) TTLJobManager() *ttlworker.JobManager {
+ return do.ttlJobManager.Load()
+}
+
func init() {
initByLDFlagsForGlobalKill()
telemetry.GetDomainInfoSchema = func(ctx sessionctx.Context) infoschema.InfoSchema {
diff --git a/domain/domain_sysvars.go b/domain/domain_sysvars.go
index da2b3aeac3fd3..3d102687454d2 100644
--- a/domain/domain_sysvars.go
+++ b/domain/domain_sysvars.go
@@ -37,6 +37,9 @@ func (do *Domain) initDomainSysVars() {
variable.SetExternalTimestamp = do.setExternalTimestamp
variable.GetExternalTimestamp = do.getExternalTimestamp
+
+ setGlobalResourceControlFunc := do.setGlobalResourceControl
+ variable.SetGlobalResourceControl.Store(&setGlobalResourceControlFunc)
}
// setStatsCacheCapacity sets statsCache cap
@@ -67,6 +70,14 @@ func (do *Domain) setPDClientDynamicOption(name, sVal string) {
}
}
+func (do *Domain) setGlobalResourceControl(enable bool) {
+ if enable {
+ variable.EnableGlobalResourceControlFunc()
+ } else {
+ variable.DisableGlobalResourceControlFunc()
+ }
+}
+
// updatePDClient is used to set the dynamic option into the PD client.
func (do *Domain) updatePDClient(option pd.DynamicOption, val interface{}) error {
store, ok := do.store.(interface{ GetPDClient() pd.Client })
diff --git a/domain/domain_test.go b/domain/domain_test.go
index 621f0fb2c431f..bd9287fe730ec 100644
--- a/domain/domain_test.go
+++ b/domain/domain_test.go
@@ -17,6 +17,8 @@ package domain
import (
"context"
"crypto/tls"
+ "encoding/json"
+ "fmt"
"net"
"runtime"
"testing"
@@ -66,7 +68,7 @@ func TestInfo(t *testing.T) {
Storage: s,
pdAddrs: []string{cluster.Members[0].GRPCURL()}}
ddlLease := 80 * time.Millisecond
- dom := NewDomain(mockStore, ddlLease, 0, 0, 0, mockFactory, nil)
+ dom := NewDomain(mockStore, ddlLease, 0, 0, 0, mockFactory)
defer func() {
dom.Close()
err := s.Close()
@@ -169,7 +171,7 @@ func TestStatWorkRecoverFromPanic(t *testing.T) {
require.NoError(t, err)
ddlLease := 80 * time.Millisecond
- dom := NewDomain(store, ddlLease, 0, 0, 0, mockFactory, nil)
+ dom := NewDomain(store, ddlLease, 0, 0, 0, mockFactory)
metrics.PanicCounter.Reset()
// Since the stats lease is 0 now, so create a new ticker will panic.
@@ -236,7 +238,7 @@ func TestClosestReplicaReadChecker(t *testing.T) {
require.NoError(t, err)
ddlLease := 80 * time.Millisecond
- dom := NewDomain(store, ddlLease, 0, 0, 0, mockFactory, nil)
+ dom := NewDomain(store, ddlLease, 0, 0, 0, mockFactory)
defer func() {
dom.Close()
require.Nil(t, store.Close())
@@ -247,7 +249,29 @@ func TestClosestReplicaReadChecker(t *testing.T) {
}
dom.sysVarCache.Unlock()
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/domain/infosync/mockGetAllServerInfo", `return("")`))
+ makeFailpointRes := func(v interface{}) string {
+ bytes, err := json.Marshal(v)
+ require.NoError(t, err)
+ return fmt.Sprintf("return(`%s`)", string(bytes))
+ }
+
+ mockedAllServerInfos := map[string]*infosync.ServerInfo{
+ "s1": {
+ ID: "s1",
+ Labels: map[string]string{
+ "zone": "zone1",
+ },
+ },
+ "s2": {
+ ID: "s2",
+ Labels: map[string]string{
+ "zone": "zone2",
+ },
+ },
+ }
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/domain/infosync/mockGetAllServerInfo", makeFailpointRes(mockedAllServerInfos)))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/domain/infosync/mockGetServerInfo", makeFailpointRes(mockedAllServerInfos["s2"])))
stores := []*metapb.Store{
{
@@ -304,8 +328,77 @@ func TestClosestReplicaReadChecker(t *testing.T) {
require.False(t, variable.IsAdaptiveReplicaReadEnabled())
}
+ // partial matches
+ mockedAllServerInfos = map[string]*infosync.ServerInfo{
+ "s1": {
+ ID: "s1",
+ Labels: map[string]string{
+ "zone": "zone1",
+ },
+ },
+ "s2": {
+ ID: "s2",
+ Labels: map[string]string{
+ "zone": "zone2",
+ },
+ },
+ "s22": {
+ ID: "s22",
+ Labels: map[string]string{
+ "zone": "zone2",
+ },
+ },
+ "s3": {
+ ID: "s3",
+ Labels: map[string]string{
+ "zone": "zone3",
+ },
+ },
+ "s4": {
+ ID: "s4",
+ Labels: map[string]string{
+ "zone": "zone4",
+ },
+ },
+ }
+ pdClient.stores = stores
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/domain/infosync/mockGetAllServerInfo", makeFailpointRes(mockedAllServerInfos)))
+ cases := []struct {
+ id string
+ matches bool
+ }{
+ {
+ id: "s1",
+ matches: true,
+ },
+ {
+ id: "s2",
+ matches: true,
+ },
+ {
+ id: "s22",
+ matches: false,
+ },
+ {
+ id: "s3",
+ matches: true,
+ },
+ {
+ id: "s4",
+ matches: false,
+ },
+ }
+ for _, c := range cases {
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/domain/infosync/mockGetServerInfo", makeFailpointRes(mockedAllServerInfos[c.id])))
+ variable.SetEnableAdaptiveReplicaRead(!c.matches)
+ err = dom.checkReplicaRead(ctx, pdClient)
+ require.Nil(t, err)
+ require.Equal(t, c.matches, variable.IsAdaptiveReplicaReadEnabled())
+ }
+
variable.SetEnableAdaptiveReplicaRead(true)
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/domain/infosync/mockGetAllServerInfo"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/domain/infosync/mockGetServerInfo"))
}
type mockInfoPdClient struct {
diff --git a/domain/globalconfigsync/globalconfig.go b/domain/globalconfigsync/globalconfig.go
index 5bbb6a260e3c8..35b4447189884 100644
--- a/domain/globalconfigsync/globalconfig.go
+++ b/domain/globalconfigsync/globalconfig.go
@@ -1,4 +1,4 @@
-// Copyright 2021 PingCAP, Inc.
+// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -41,7 +41,7 @@ func (s *GlobalConfigSyncer) StoreGlobalConfig(ctx context.Context, item pd.Glob
if s.pd == nil {
return nil
}
- err := s.pd.StoreGlobalConfig(ctx, []pd.GlobalConfigItem{item})
+ err := s.pd.StoreGlobalConfig(ctx, "", []pd.GlobalConfigItem{item})
if err != nil {
return err
}
diff --git a/domain/globalconfigsync/globalconfig_test.go b/domain/globalconfigsync/globalconfig_test.go
index c7dab0064dedb..2d97b7c9e2b9f 100644
--- a/domain/globalconfigsync/globalconfig_test.go
+++ b/domain/globalconfigsync/globalconfig_test.go
@@ -1,4 +1,4 @@
-// Copyright 2021 PingCAP, Inc.
+// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -58,11 +58,12 @@ func TestGlobalConfigSyncer(t *testing.T) {
syncer.Notify(pd.GlobalConfigItem{Name: "a", Value: "b"})
err = syncer.StoreGlobalConfig(context.Background(), <-syncer.NotifyCh)
require.NoError(t, err)
- items, err := client.LoadGlobalConfig(context.Background(), []string{"a"})
+ items, revision, err := client.LoadGlobalConfig(context.Background(), []string{"a"}, "")
require.NoError(t, err)
- require.Equal(t, len(items), 1)
- require.Equal(t, items[0].Name, "/global/config/a")
- require.Equal(t, items[0].Value, "b")
+ require.Equal(t, 1, len(items))
+ require.Equal(t, "/global/config/a", items[0].Name)
+ require.Equal(t, int64(0), revision)
+ require.Equal(t, "b", items[0].Value)
}
func TestStoreGlobalConfig(t *testing.T) {
@@ -88,19 +89,23 @@ func TestStoreGlobalConfig(t *testing.T) {
_, err = se.Execute(context.Background(), "set @@global.tidb_enable_top_sql=1;")
require.NoError(t, err)
+ _, err = se.Execute(context.Background(), "set @@global.tidb_source_id=2;")
+ require.NoError(t, err)
for i := 0; i < 20; i++ {
time.Sleep(100 * time.Millisecond)
client :=
store.(kv.StorageWithPD).GetPDClient()
// enable top sql will be translated to enable_resource_metering
- items, err := client.LoadGlobalConfig(context.Background(), []string{"enable_resource_metering"})
+ items, _, err := client.LoadGlobalConfig(context.Background(), []string{"enable_resource_metering", "source_id"}, "")
require.NoError(t, err)
- if len(items) == 1 && items[0].Value == "" {
+ if len(items) == 2 && items[0].Value == "" {
continue
}
- require.Len(t, items, 1)
+ require.Len(t, items, 2)
require.Equal(t, items[0].Name, "/global/config/enable_resource_metering")
require.Equal(t, items[0].Value, "true")
+ require.Equal(t, items[1].Name, "/global/config/source_id")
+ require.Equal(t, items[1].Value, "2")
return
}
require.Fail(t, "timeout for waiting global config synced")
diff --git a/domain/historical_stats.go b/domain/historical_stats.go
new file mode 100644
index 0000000000000..6d4125b75f5d7
--- /dev/null
+++ b/domain/historical_stats.go
@@ -0,0 +1,103 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package domain
+
+import (
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/metrics"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/statistics/handle"
+ "github.com/pingcap/tidb/util/logutil"
+ "go.uber.org/zap"
+)
+
+var (
+ generateHistoricalStatsSuccessCounter = metrics.HistoricalStatsCounter.WithLabelValues("generate", "success")
+ generateHistoricalStatsFailedCounter = metrics.HistoricalStatsCounter.WithLabelValues("generate", "fail")
+)
+
+// HistoricalStatsWorker indicates for dump historical stats
+type HistoricalStatsWorker struct {
+ tblCH chan int64
+ sctx sessionctx.Context
+}
+
+// SendTblToDumpHistoricalStats send tableID to worker to dump historical stats
+func (w *HistoricalStatsWorker) SendTblToDumpHistoricalStats(tableID int64) {
+ send := enableDumpHistoricalStats.Load()
+ failpoint.Inject("sendHistoricalStats", func(val failpoint.Value) {
+ if val.(bool) {
+ send = true
+ }
+ })
+ if !send {
+ return
+ }
+ select {
+ case w.tblCH <- tableID:
+ return
+ default:
+ logutil.BgLogger().Warn("discard dump historical stats task", zap.Int64("table-id", tableID))
+ }
+}
+
+// DumpHistoricalStats dump stats by given tableID
+func (w *HistoricalStatsWorker) DumpHistoricalStats(tableID int64, statsHandle *handle.Handle) error {
+ historicalStatsEnabled, err := statsHandle.CheckHistoricalStatsEnable()
+ if err != nil {
+ return errors.Errorf("check tidb_enable_historical_stats failed: %v", err)
+ }
+ if !historicalStatsEnabled {
+ return nil
+ }
+ sctx := w.sctx
+ is := GetDomain(sctx).InfoSchema()
+ isPartition := false
+ var tblInfo *model.TableInfo
+ tbl, existed := is.TableByID(tableID)
+ if !existed {
+ tbl, db, p := is.FindTableByPartitionID(tableID)
+ if tbl != nil && db != nil && p != nil {
+ isPartition = true
+ tblInfo = tbl.Meta()
+ } else {
+ return errors.Errorf("cannot get table by id %d", tableID)
+ }
+ } else {
+ tblInfo = tbl.Meta()
+ }
+ dbInfo, existed := is.SchemaByTable(tblInfo)
+ if !existed {
+ return errors.Errorf("cannot get DBInfo by TableID %d", tableID)
+ }
+ if _, err := statsHandle.RecordHistoricalStatsToStorage(dbInfo.Name.O, tblInfo, tableID, isPartition); err != nil {
+ generateHistoricalStatsFailedCounter.Inc()
+ return errors.Errorf("record table %s.%s's historical stats failed, err:%v", dbInfo.Name.O, tblInfo.Name.O, err)
+ }
+ generateHistoricalStatsSuccessCounter.Inc()
+ return nil
+}
+
+// GetOneHistoricalStatsTable gets one tableID from channel, only used for test
+func (w *HistoricalStatsWorker) GetOneHistoricalStatsTable() int64 {
+ select {
+ case tblID := <-w.tblCH:
+ return tblID
+ default:
+ return -1
+ }
+}
diff --git a/domain/infosync/BUILD.bazel b/domain/infosync/BUILD.bazel
index c0936390ff016..f050873ee128f 100644
--- a/domain/infosync/BUILD.bazel
+++ b/domain/infosync/BUILD.bazel
@@ -8,6 +8,7 @@ go_library(
"label_manager.go",
"placement_manager.go",
"region.go",
+ "resource_group_manager.go",
"schedule_manager.go",
"tiflash_manager.go",
],
@@ -40,8 +41,12 @@ go_library(
"@com_github_gorilla_mux//:mux",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_pingcap_kvproto//pkg/kvrpcpb",
"@com_github_pingcap_kvproto//pkg/metapb",
+ "@com_github_pingcap_kvproto//pkg/resource_manager",
+ "@com_github_pingcap_log//:log",
"@com_github_tikv_client_go_v2//oracle",
+ "@com_github_tikv_client_go_v2//tikv",
"@com_github_tikv_pd_client//:client",
"@io_etcd_go_etcd_client_v3//:client",
"@io_etcd_go_etcd_client_v3//concurrency",
@@ -59,6 +64,7 @@ go_test(
deps = [
"//ddl/placement",
"//ddl/util",
+ "//keyspace",
"//parser/model",
"//testkit/testsetup",
"//util",
diff --git a/domain/infosync/info.go b/domain/infosync/info.go
index a2af6c5dfa58f..1e82ce308ac9b 100644
--- a/domain/infosync/info.go
+++ b/domain/infosync/info.go
@@ -33,6 +33,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/metapb"
+ rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/ddl/label"
"github.com/pingcap/tidb/ddl/placement"
@@ -54,6 +55,7 @@ import (
"github.com/pingcap/tidb/util/pdapi"
"github.com/pingcap/tidb/util/versioninfo"
"github.com/tikv/client-go/v2/oracle"
+ "github.com/tikv/client-go/v2/tikv"
pd "github.com/tikv/pd/client"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency"
@@ -94,12 +96,18 @@ var ErrPrometheusAddrIsNotSet = dbterror.ClassDomain.NewStd(errno.ErrPrometheusA
// InfoSyncer stores server info to etcd when the tidb-server starts and delete when tidb-server shuts down.
type InfoSyncer struct {
- etcdCli *clientv3.Client
- info *ServerInfo
- serverInfoPath string
- minStartTS uint64
- minStartTSPath string
- managerMu struct {
+ // `etcdClient` must be used when keyspace is not set, or when the logic to each etcd path needs to be separated by keyspace.
+ etcdCli *clientv3.Client
+ // `unprefixedEtcdCli` will never set the etcd namespace prefix by keyspace.
+ // It is only used in storeMinStartTS and RemoveMinStartTS now.
+ // It must be used when the etcd path isn't needed to separate by keyspace.
+ // See keyspace RFC: https://github.com/pingcap/tidb/pull/39685
+ unprefixedEtcdCli *clientv3.Client
+ info *ServerInfo
+ serverInfoPath string
+ minStartTS uint64
+ minStartTSPath string
+ managerMu struct {
mu sync.RWMutex
util2.SessionManager
}
@@ -111,6 +119,7 @@ type InfoSyncer struct {
placementManager PlacementManager
scheduleManager ScheduleManager
tiflashReplicaManager TiFlashReplicaManager
+ resourceGroupManager pd.ResourceManagerClient
}
// ServerInfo is server static information.
@@ -178,12 +187,21 @@ func setGlobalInfoSyncer(is *InfoSyncer) {
}
// GlobalInfoSyncerInit return a new InfoSyncer. It is exported for testing.
-func GlobalInfoSyncerInit(ctx context.Context, id string, serverIDGetter func() uint64, etcdCli *clientv3.Client, skipRegisterToDashBoard bool) (*InfoSyncer, error) {
+func GlobalInfoSyncerInit(
+ ctx context.Context,
+ id string,
+ serverIDGetter func() uint64,
+ etcdCli, unprefixedEtcdCli *clientv3.Client,
+ pdCli pd.Client,
+ codec tikv.Codec,
+ skipRegisterToDashBoard bool,
+) (*InfoSyncer, error) {
is := &InfoSyncer{
- etcdCli: etcdCli,
- info: getServerInfo(id, serverIDGetter),
- serverInfoPath: fmt.Sprintf("%s/%s", ServerInformationPath, id),
- minStartTSPath: fmt.Sprintf("%s/%s", ServerMinStartTSPath, id),
+ etcdCli: etcdCli,
+ unprefixedEtcdCli: unprefixedEtcdCli,
+ info: getServerInfo(id, serverIDGetter),
+ serverInfoPath: fmt.Sprintf("%s/%s", ServerInformationPath, id),
+ minStartTSPath: fmt.Sprintf("%s/%s", ServerMinStartTSPath, id),
}
err := is.init(ctx, skipRegisterToDashBoard)
if err != nil {
@@ -192,7 +210,8 @@ func GlobalInfoSyncerInit(ctx context.Context, id string, serverIDGetter func()
is.labelRuleManager = initLabelRuleManager(etcdCli)
is.placementManager = initPlacementManager(etcdCli)
is.scheduleManager = initScheduleManager(etcdCli)
- is.tiflashReplicaManager = initTiFlashReplicaManager(etcdCli)
+ is.tiflashReplicaManager = initTiFlashReplicaManager(etcdCli, codec)
+ is.resourceGroupManager = initResourceGroupManager(pdCli)
setGlobalInfoSyncer(is)
return is, nil
}
@@ -237,13 +256,20 @@ func initPlacementManager(etcdCli *clientv3.Client) PlacementManager {
return &PDPlacementManager{etcdCli: etcdCli}
}
-func initTiFlashReplicaManager(etcdCli *clientv3.Client) TiFlashReplicaManager {
+func initResourceGroupManager(pdCli pd.Client) pd.ResourceManagerClient {
+ if pdCli == nil {
+ return &mockResourceGroupManager{groups: make(map[string]*rmpb.ResourceGroup)}
+ }
+ return pdCli
+}
+
+func initTiFlashReplicaManager(etcdCli *clientv3.Client, codec tikv.Codec) TiFlashReplicaManager {
if etcdCli == nil {
m := mockTiFlashReplicaManagerCtx{tiflashProgressCache: make(map[int64]float64)}
return &m
}
logutil.BgLogger().Warn("init TiFlashReplicaManager", zap.Strings("pd addrs", etcdCli.Endpoints()))
- return &TiFlashReplicaManagerCtx{etcdCli: etcdCli, tiflashProgressCache: make(map[int64]float64)}
+ return &TiFlashReplicaManagerCtx{etcdCli: etcdCli, tiflashProgressCache: make(map[int64]float64), codec: codec}
}
func initScheduleManager(etcdCli *clientv3.Client) ScheduleManager {
@@ -282,6 +308,11 @@ func SetMockTiFlash(tiflash *MockTiFlash) {
// GetServerInfo gets self server static information.
func GetServerInfo() (*ServerInfo, error) {
+ failpoint.Inject("mockGetServerInfo", func(v failpoint.Value) {
+ var res ServerInfo
+ err := json.Unmarshal([]byte(v.(string)), &res)
+ failpoint.Return(&res, err)
+ })
is, err := getGlobalInfoSyncer()
if err != nil {
return nil, err
@@ -316,20 +347,10 @@ func (is *InfoSyncer) getServerInfoByID(ctx context.Context, id string) (*Server
// GetAllServerInfo gets all servers static information from etcd.
func GetAllServerInfo(ctx context.Context) (map[string]*ServerInfo, error) {
- failpoint.Inject("mockGetAllServerInfo", func() {
- res := map[string]*ServerInfo{
- "fa598405-a08e-4e74-83ff-75c30b1daedc": {
- Labels: map[string]string{
- "zone": "zone1",
- },
- },
- "ad84dbbd-5a50-4742-a73c-4f674d41d4bd": {
- Labels: map[string]string{
- "zone": "zone2",
- },
- },
- }
- failpoint.Return(res, nil)
+ failpoint.Inject("mockGetAllServerInfo", func(val failpoint.Value) {
+ res := make(map[string]*ServerInfo)
+ err := json.Unmarshal([]byte(val.(string)), &res)
+ failpoint.Return(res, err)
})
is, err := getGlobalInfoSyncer()
if err != nil {
@@ -508,7 +529,7 @@ func doRequestWithFailpoint(req *http.Request) (resp *http.Response, err error)
return util2.InternalHTTPClient().Do(req)
}
-// GetAllRuleBundles is used to get all rule bundles from PD. It is used to load full rules from PD while fullload infoschema.
+// GetAllRuleBundles is used to get all rule bundles from PD It is used to load full rules from PD while fullload infoschema.
func GetAllRuleBundles(ctx context.Context) ([]*placement.Bundle, error) {
is, err := getGlobalInfoSyncer()
if err != nil {
@@ -568,6 +589,56 @@ func PutRuleBundlesWithRetry(ctx context.Context, bundles []*placement.Bundle, m
return
}
+// GetResourceGroup is used to get one specific resource group from resource manager.
+func GetResourceGroup(ctx context.Context, name string) (*rmpb.ResourceGroup, error) {
+ is, err := getGlobalInfoSyncer()
+ if err != nil {
+ return nil, err
+ }
+
+ return is.resourceGroupManager.GetResourceGroup(ctx, name)
+}
+
+// ListResourceGroups is used to get all resource groups from resource manager.
+func ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) {
+ is, err := getGlobalInfoSyncer()
+ if err != nil {
+ return nil, err
+ }
+
+ return is.resourceGroupManager.ListResourceGroups(ctx)
+}
+
+// AddResourceGroup is used to create one specific resource group to resource manager.
+func AddResourceGroup(ctx context.Context, group *rmpb.ResourceGroup) error {
+ is, err := getGlobalInfoSyncer()
+ if err != nil {
+ return err
+ }
+ _, err = is.resourceGroupManager.AddResourceGroup(ctx, group)
+ return err
+}
+
+// ModifyResourceGroup is used to modify one specific resource group to resource manager.
+func ModifyResourceGroup(ctx context.Context, group *rmpb.ResourceGroup) error {
+ is, err := getGlobalInfoSyncer()
+ if err != nil {
+ return err
+ }
+ _, err = is.resourceGroupManager.ModifyResourceGroup(ctx, group)
+ return err
+}
+
+// DeleteResourceGroup is used to delete one specific resource group from resource manager.
+func DeleteResourceGroup(ctx context.Context, name string) error {
+ is, err := getGlobalInfoSyncer()
+ if err != nil {
+ return err
+ }
+ _, err = is.resourceGroupManager.DeleteResourceGroup(ctx, name)
+ return err
+}
+
// PutRuleBundlesWithDefaultRetry will retry for default times
func PutRuleBundlesWithDefaultRetry(ctx context.Context, bundles []*placement.Bundle) (err error) {
return PutRuleBundlesWithRetry(ctx, bundles, SyncBundlesMaxRetry, RequestRetryInterval)
@@ -669,20 +740,20 @@ func (is *InfoSyncer) GetMinStartTS() uint64 {
// storeMinStartTS stores self server min start timestamp to etcd.
func (is *InfoSyncer) storeMinStartTS(ctx context.Context) error {
- if is.etcdCli == nil {
+ if is.unprefixedEtcdCli == nil {
return nil
}
- return util.PutKVToEtcd(ctx, is.etcdCli, keyOpDefaultRetryCnt, is.minStartTSPath,
+ return util.PutKVToEtcd(ctx, is.unprefixedEtcdCli, keyOpDefaultRetryCnt, is.minStartTSPath,
strconv.FormatUint(is.minStartTS, 10),
clientv3.WithLease(is.session.Lease()))
}
// RemoveMinStartTS removes self server min start timestamp from etcd.
func (is *InfoSyncer) RemoveMinStartTS() {
- if is.etcdCli == nil {
+ if is.unprefixedEtcdCli == nil {
return
}
- err := util.DeleteKeyFromEtcd(is.minStartTSPath, is.etcdCli, keyOpDefaultRetryCnt, keyOpDefaultTimeout)
+ err := util.DeleteKeyFromEtcd(is.minStartTSPath, is.unprefixedEtcdCli, keyOpDefaultRetryCnt, keyOpDefaultTimeout)
if err != nil {
logutil.BgLogger().Error("remove minStartTS failed", zap.Error(err))
}
@@ -694,8 +765,6 @@ func (is *InfoSyncer) ReportMinStartTS(store kv.Storage) {
if sm == nil {
return
}
- pl := sm.ShowProcessList()
- innerSessionStartTSList := sm.GetInternalSessionStartTSList()
// Calculate the lower limit of the start timestamp to avoid extremely old transaction delaying GC.
currentVer, err := store.CurrentVersion(kv.GlobalTxnScope)
@@ -709,18 +778,8 @@ func (is *InfoSyncer) ReportMinStartTS(store kv.Storage) {
minStartTS := oracle.GoTimeToTS(now)
logutil.BgLogger().Debug("ReportMinStartTS", zap.Uint64("initial minStartTS", minStartTS),
zap.Uint64("StartTSLowerLimit", startTSLowerLimit))
- for _, info := range pl {
- if info.CurTxnStartTS > startTSLowerLimit && info.CurTxnStartTS < minStartTS {
- minStartTS = info.CurTxnStartTS
- }
- }
-
- for _, innerTS := range innerSessionStartTSList {
- logutil.BgLogger().Debug("ReportMinStartTS", zap.Uint64("Internal Session Transaction StartTS", innerTS))
- kv.PrintLongTimeInternalTxn(now, innerTS, false)
- if innerTS > startTSLowerLimit && innerTS < minStartTS {
- minStartTS = innerTS
- }
+ if ts := sm.GetMinStartTS(startTSLowerLimit); ts > startTSLowerLimit && ts < minStartTS {
+ minStartTS = ts
}
is.minStartTS = kv.GetMinInnerTxnStartTS(now, startTSLowerLimit, minStartTS)
diff --git a/domain/infosync/info_test.go b/domain/infosync/info_test.go
index 90a30d8f1f161..ee97406eef01a 100644
--- a/domain/infosync/info_test.go
+++ b/domain/infosync/info_test.go
@@ -28,6 +28,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl/placement"
"github.com/pingcap/tidb/ddl/util"
+ "github.com/pingcap/tidb/keyspace"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/testkit/testsetup"
util2 "github.com/pingcap/tidb/util"
@@ -67,7 +68,7 @@ func TestTopology(t *testing.T) {
require.NoError(t, err)
}()
- info, err := GlobalInfoSyncerInit(ctx, currentID, func() uint64 { return 1 }, client, false)
+ info, err := GlobalInfoSyncerInit(ctx, currentID, func() uint64 { return 1 }, client, client, nil, keyspace.CodecV1, false)
require.NoError(t, err)
err = info.newTopologySessionAndStoreServerInfo(ctx, util2.NewSessionDefaultRetryCnt)
@@ -152,7 +153,7 @@ func (is *InfoSyncer) ttlKeyExists(ctx context.Context) (bool, error) {
}
func TestPutBundlesRetry(t *testing.T) {
- _, err := GlobalInfoSyncerInit(context.TODO(), "test", func() uint64 { return 1 }, nil, false)
+ _, err := GlobalInfoSyncerInit(context.TODO(), "test", func() uint64 { return 1 }, nil, nil, nil, keyspace.CodecV1, false)
require.NoError(t, err)
bundle, err := placement.NewBundleFromOptions(&model.PlacementSettings{PrimaryRegion: "r1", Regions: "r1,r2"})
@@ -216,7 +217,7 @@ func TestPutBundlesRetry(t *testing.T) {
func TestTiFlashManager(t *testing.T) {
ctx := context.Background()
- _, err := GlobalInfoSyncerInit(ctx, "test", func() uint64 { return 1 }, nil, false)
+ _, err := GlobalInfoSyncerInit(ctx, "test", func() uint64 { return 1 }, nil, nil, nil, keyspace.CodecV1, false)
tiflash := NewMockTiFlash()
SetMockTiFlash(tiflash)
diff --git a/domain/infosync/placement_manager.go b/domain/infosync/placement_manager.go
index 5ae047a80cf2c..223dafd0467a3 100644
--- a/domain/infosync/placement_manager.go
+++ b/domain/infosync/placement_manager.go
@@ -21,9 +21,11 @@ import (
"path"
"sync"
+ "github.com/pingcap/log"
"github.com/pingcap/tidb/ddl/placement"
"github.com/pingcap/tidb/util/pdapi"
clientv3 "go.etcd.io/etcd/client/v3"
+ "go.uber.org/zap"
)
// PlacementManager manages placement settings
@@ -72,6 +74,7 @@ func (m *PDPlacementManager) PutRuleBundles(ctx context.Context, bundles []*plac
return err
}
+ log.Debug("Put placement rule bundles", zap.String("rules", string(b)))
_, err = doRequest(ctx, "PutPlacementRules", m.etcdCli.Endpoints(), path.Join(pdapi.Config, "placement-rule")+"?partial=true", "POST", bytes.NewReader(b))
return err
}
diff --git a/domain/infosync/resource_group_manager.go b/domain/infosync/resource_group_manager.go
new file mode 100644
index 0000000000000..6b8f876f9bb11
--- /dev/null
+++ b/domain/infosync/resource_group_manager.go
@@ -0,0 +1,79 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package infosync
+
+import (
+ "context"
+ "sync"
+
+ rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
+ pd "github.com/tikv/pd/client"
+)
+
+type mockResourceGroupManager struct {
+ sync.RWMutex
+ groups map[string]*rmpb.ResourceGroup
+}
+
+var _ pd.ResourceManagerClient = (*mockResourceGroupManager)(nil)
+
+func (m *mockResourceGroupManager) ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) {
+ m.RLock()
+ defer m.RUnlock()
+ groups := make([]*rmpb.ResourceGroup, 0, len(m.groups))
+ for _, group := range m.groups {
+ groups = append(groups, group)
+ }
+ return groups, nil
+}
+
+func (m *mockResourceGroupManager) GetResourceGroup(ctx context.Context, name string) (*rmpb.ResourceGroup, error) {
+ m.RLock()
+ defer m.RUnlock()
+ group, ok := m.groups[name]
+ if !ok {
+ return nil, nil
+ }
+ return group, nil
+}
+
+func (m *mockResourceGroupManager) AddResourceGroup(ctx context.Context, group *rmpb.ResourceGroup) (string, error) {
+ m.Lock()
+ defer m.Unlock()
+ m.groups[group.Name] = group
+ return "Success!", nil
+}
+
+func (m *mockResourceGroupManager) ModifyResourceGroup(ctx context.Context, group *rmpb.ResourceGroup) (string, error) {
+ m.Lock()
+ defer m.Unlock()
+ m.groups[group.Name] = group
+ return "Success!", nil
+}
+
+func (m *mockResourceGroupManager) DeleteResourceGroup(ctx context.Context, name string) (string, error) {
+ m.Lock()
+ defer m.Unlock()
+ delete(m.groups, name)
+ return "Success!", nil
+}
+
+func (m *mockResourceGroupManager) AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) {
+ return nil, nil
+}
+
+func (m *mockResourceGroupManager) WatchResourceGroup(ctx context.Context, revision int64) (chan []*rmpb.ResourceGroup, error) {
+ return nil, nil
+}
diff --git a/domain/infosync/tiflash_manager.go b/domain/infosync/tiflash_manager.go
index 4d01c64de002d..6e47d881215d4 100644
--- a/domain/infosync/tiflash_manager.go
+++ b/domain/infosync/tiflash_manager.go
@@ -31,12 +31,16 @@ import (
"github.com/gorilla/mux"
"github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/tidb/ddl/placement"
"github.com/pingcap/tidb/store/helper"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/util/codec"
+ "github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/pdapi"
+ "github.com/tikv/client-go/v2/tikv"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
)
@@ -76,6 +80,7 @@ type TiFlashReplicaManagerCtx struct {
etcdCli *clientv3.Client
sync.RWMutex // protect tiflashProgressCache
tiflashProgressCache map[int64]float64
+ codec tikv.Codec
}
// Close is called to close TiFlashReplicaManagerCtx.
@@ -89,10 +94,19 @@ func getTiFlashPeerWithoutLagCount(tiFlashStores map[int64]helper.StoreStat, tab
for _, store := range tiFlashStores {
regionReplica := make(map[int64]int)
err := helper.CollectTiFlashStatus(store.Store.StatusAddress, tableID, ®ionReplica)
+ failpoint.Inject("OneTiFlashStoreDown", func() {
+ if store.Store.StateName == "Down" {
+ err = errors.New("mock TiFlasah down")
+ }
+ })
if err != nil {
logutil.BgLogger().Error("Fail to get peer status from TiFlash.",
zap.Int64("tableID", tableID))
- return 0, err
+ // Just skip down or offline or tomestone stores, because PD will migrate regions from these stores.
+ if store.Store.StateName == "Up" || store.Store.StateName == "Disconnected" {
+ return 0, err
+ }
+ continue
}
flashPeerCount += len(regionReplica)
}
@@ -220,6 +234,11 @@ func (m *TiFlashReplicaManagerCtx) SetTiFlashGroupConfig(ctx context.Context) er
// SetPlacementRule is a helper function to set placement rule.
func (m *TiFlashReplicaManagerCtx) SetPlacementRule(ctx context.Context, rule placement.TiFlashRule) error {
+ // TiDB 6.6 doesn't support tiflash multi-tenancy yet.
+ // TODO(iosmanthus): remove this check after TiDB supports tiflash multi-tenancy.
+ if m.codec.GetAPIVersion() == kvrpcpb.APIVersion_V2 {
+ return errors.Trace(dbterror.ErrNotSupportedYet.GenWithStackByArgs("set TiFlash replica count while enabling API V2"))
+ }
if err := m.SetTiFlashGroupConfig(ctx); err != nil {
return err
}
diff --git a/domain/plan_replayer.go b/domain/plan_replayer.go
index efc2e8ad21429..bb36048f02b26 100644
--- a/domain/plan_replayer.go
+++ b/domain/plan_replayer.go
@@ -27,16 +27,18 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/bindinfo"
- "github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/domain/infosync"
"github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/metrics"
+ "github.com/pingcap/tidb/parser"
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/variable"
- "github.com/pingcap/tidb/statistics/handle"
+ "github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/replayer"
"github.com/pingcap/tidb/util/sqlexec"
"go.uber.org/zap"
)
@@ -45,16 +47,10 @@ import (
// For now it is used by `plan replayer` and `trace plan` statement
type dumpFileGcChecker struct {
sync.Mutex
- gcLease time.Duration
- paths []string
- sctx sessionctx.Context
-}
-
-// GetPlanReplayerDirName returns plan replayer directory path.
-// The path is related to the process id.
-func GetPlanReplayerDirName() string {
- tidbLogDir := filepath.Dir(config.GetGlobalConfig().Log.File.Filename)
- return filepath.Join(tidbLogDir, "replayer")
+ gcLease time.Duration
+ paths []string
+ sctx sessionctx.Context
+ planReplayerTaskStatus *planReplayerDumpTaskStatus
}
func parseType(s string) string {
@@ -105,7 +101,7 @@ func (p *dumpFileGcChecker) gcDumpFilesByPath(path string, t time.Duration) {
logutil.BgLogger().Error("[dumpFileGcChecker] parseTime failed", zap.Error(err), zap.String("filename", fileName))
continue
}
- isPlanReplayer := parseType(fileName) == "replayer"
+ isPlanReplayer := strings.Contains(fileName, "replayer")
if !createTime.After(gcTime) {
err := os.Remove(filepath.Join(path, f.Name()))
if err != nil {
@@ -115,23 +111,12 @@ func (p *dumpFileGcChecker) gcDumpFilesByPath(path string, t time.Duration) {
logutil.BgLogger().Info("dumpFileGcChecker successful", zap.String("filename", fileName))
if isPlanReplayer && p.sctx != nil {
deletePlanReplayerStatus(context.Background(), p.sctx, fileName)
+ p.planReplayerTaskStatus.clearFinishedTask()
}
}
}
}
-type planReplayerHandle struct {
- *planReplayerTaskCollectorHandle
-}
-
-type planReplayerTaskCollectorHandle struct {
- taskMu struct {
- sync.RWMutex
- tasks map[PlanReplayerTaskKey]struct{}
- }
- sctx sessionctx.Context
-}
-
func deletePlanReplayerStatus(ctx context.Context, sctx sessionctx.Context, token string) {
ctx1 := kv.WithInternalSourceType(ctx, kv.InternalTxnStats)
exec := sctx.(sqlexec.SQLExecutor)
@@ -153,62 +138,116 @@ func insertPlanReplayerStatus(ctx context.Context, sctx sessionctx.Context, reco
instance = fmt.Sprintf("%s:%d", serverInfo.IP, serverInfo.Port)
}
for _, record := range records {
- if !record.Internal {
- if len(record.FailedReason) > 0 {
- insertExternalPlanReplayerErrorStatusRecord(ctx1, sctx, instance, record)
- } else {
- insertExternalPlanReplayerSuccessStatusRecord(ctx1, sctx, instance, record)
- }
+ if len(record.FailedReason) > 0 {
+ insertPlanReplayerErrorStatusRecord(ctx1, sctx, instance, record)
+ } else {
+ insertPlanReplayerSuccessStatusRecord(ctx1, sctx, instance, record)
}
}
}
-func insertExternalPlanReplayerErrorStatusRecord(ctx context.Context, sctx sessionctx.Context, instance string, record PlanReplayerStatusRecord) {
- exec := sctx.(sqlexec.SQLExecutor)
- _, err := exec.ExecuteInternal(ctx, fmt.Sprintf(
- "insert into mysql.plan_replayer_status (origin_sql, fail_reason, instance) values ('%s','%s','%s')",
- record.OriginSQL, record.FailedReason, instance))
+func insertPlanReplayerErrorStatusRecord(ctx context.Context, sctx sessionctx.Context, instance string, record PlanReplayerStatusRecord) {
+ exec := sctx.(sqlexec.RestrictedSQLExecutor)
+ _, _, err := exec.ExecRestrictedSQL(ctx, nil, fmt.Sprintf(
+ "insert into mysql.plan_replayer_status (sql_digest, plan_digest, origin_sql, fail_reason, instance) values ('%s','%s','%s','%s','%s')",
+ record.SQLDigest, record.PlanDigest, record.OriginSQL, record.FailedReason, instance))
if err != nil {
logutil.BgLogger().Warn("insert mysql.plan_replayer_status record failed",
zap.Error(err))
}
}
-func insertExternalPlanReplayerSuccessStatusRecord(ctx context.Context, sctx sessionctx.Context, instance string, record PlanReplayerStatusRecord) {
- exec := sctx.(sqlexec.SQLExecutor)
- _, err := exec.ExecuteInternal(ctx, fmt.Sprintf(
- "insert into mysql.plan_replayer_status (origin_sql, token, instance) values ('%s','%s','%s')",
- record.OriginSQL, record.Token, instance))
+func insertPlanReplayerSuccessStatusRecord(ctx context.Context, sctx sessionctx.Context, instance string, record PlanReplayerStatusRecord) {
+ exec := sctx.(sqlexec.RestrictedSQLExecutor)
+ _, _, err := exec.ExecRestrictedSQL(ctx, nil, fmt.Sprintf(
+ "insert into mysql.plan_replayer_status (sql_digest, plan_digest, origin_sql, token, instance) values ('%s','%s','%s','%s','%s')",
+ record.SQLDigest, record.PlanDigest, record.OriginSQL, record.Token, instance))
if err != nil {
logutil.BgLogger().Warn("insert mysql.plan_replayer_status record failed",
+ zap.String("sql", record.OriginSQL),
zap.Error(err))
+ // try insert record without original sql
+ _, _, err = exec.ExecRestrictedSQL(ctx, nil, fmt.Sprintf(
+ "insert into mysql.plan_replayer_status (sql_digest, plan_digest, token, instance) values ('%s','%s','%s','%s')",
+ record.SQLDigest, record.PlanDigest, record.Token, instance))
+ if err != nil {
+ logutil.BgLogger().Warn("insert mysql.plan_replayer_status record failed",
+ zap.String("sqlDigest", record.SQLDigest),
+ zap.String("planDigest", record.PlanDigest),
+ zap.Error(err))
+ }
+ }
+}
+
+var (
+ planReplayerCaptureTaskSendCounter = metrics.PlanReplayerTaskCounter.WithLabelValues("capture", "send")
+ planReplayerCaptureTaskDiscardCounter = metrics.PlanReplayerTaskCounter.WithLabelValues("capture", "discard")
+
+ planReplayerRegisterTaskGauge = metrics.PlanReplayerRegisterTaskGauge
+)
+
+type planReplayerHandle struct {
+ *planReplayerTaskCollectorHandle
+ *planReplayerTaskDumpHandle
+}
+
+// SendTask send dumpTask in background task handler
+func (h *planReplayerHandle) SendTask(task *PlanReplayerDumpTask) bool {
+ select {
+ case h.planReplayerTaskDumpHandle.taskCH <- task:
+ // we directly remove the task key if we put task in channel successfully, if the task was failed to dump,
+ // the task handle will re-add the task in next loop
+ if !task.IsContinuesCapture {
+ h.planReplayerTaskCollectorHandle.removeTask(task.PlanReplayerTaskKey)
+ }
+ planReplayerCaptureTaskSendCounter.Inc()
+ return true
+ default:
+ planReplayerCaptureTaskDiscardCounter.Inc()
+ // directly discard the task if the task channel is full in order not to block the query process
+ logutil.BgLogger().Warn("discard one plan replayer dump task",
+ zap.String("sql-digest", task.SQLDigest), zap.String("plan-digest", task.PlanDigest))
+ return false
}
}
+type planReplayerTaskCollectorHandle struct {
+ taskMu struct {
+ sync.RWMutex
+ tasks map[replayer.PlanReplayerTaskKey]struct{}
+ }
+ ctx context.Context
+ sctx sessionctx.Context
+}
+
// CollectPlanReplayerTask collects all unhandled plan replayer task
-func (h *planReplayerTaskCollectorHandle) CollectPlanReplayerTask(ctx context.Context) error {
- ctx1 := kv.WithInternalSourceType(ctx, kv.InternalTxnStats)
- allKeys, err := h.collectAllPlanReplayerTask(ctx1)
+func (h *planReplayerTaskCollectorHandle) CollectPlanReplayerTask() error {
+ allKeys, err := h.collectAllPlanReplayerTask(h.ctx)
if err != nil {
return err
}
- tasks := make([]PlanReplayerTaskKey, 0)
+ tasks := make([]replayer.PlanReplayerTaskKey, 0)
for _, key := range allKeys {
- unhandled, err := checkUnHandledReplayerTask(ctx1, h.sctx, key)
+ unhandled, err := checkUnHandledReplayerTask(h.ctx, h.sctx, key)
if err != nil {
+ logutil.BgLogger().Warn("[plan-replayer-task] collect plan replayer task failed", zap.Error(err))
return err
}
if unhandled {
+ logutil.BgLogger().Debug("[plan-replayer-task] collect plan replayer task success",
+ zap.String("sql-digest", key.SQLDigest),
+ zap.String("plan-digest", key.PlanDigest))
tasks = append(tasks, key)
}
}
h.setupTasks(tasks)
+ planReplayerRegisterTaskGauge.Set(float64(len(tasks)))
return nil
}
// GetTasks get all tasks
-func (h *planReplayerTaskCollectorHandle) GetTasks() []PlanReplayerTaskKey {
- tasks := make([]PlanReplayerTaskKey, 0)
+func (h *planReplayerTaskCollectorHandle) GetTasks() []replayer.PlanReplayerTaskKey {
+ tasks := make([]replayer.PlanReplayerTaskKey, 0)
h.taskMu.RLock()
defer h.taskMu.RUnlock()
for taskKey := range h.taskMu.tasks {
@@ -217,8 +256,8 @@ func (h *planReplayerTaskCollectorHandle) GetTasks() []PlanReplayerTaskKey {
return tasks
}
-func (h *planReplayerTaskCollectorHandle) setupTasks(tasks []PlanReplayerTaskKey) {
- r := make(map[PlanReplayerTaskKey]struct{})
+func (h *planReplayerTaskCollectorHandle) setupTasks(tasks []replayer.PlanReplayerTaskKey) {
+ r := make(map[replayer.PlanReplayerTaskKey]struct{})
for _, task := range tasks {
r[task] = struct{}{}
}
@@ -227,7 +266,13 @@ func (h *planReplayerTaskCollectorHandle) setupTasks(tasks []PlanReplayerTaskKey
h.taskMu.tasks = r
}
-func (h *planReplayerTaskCollectorHandle) collectAllPlanReplayerTask(ctx context.Context) ([]PlanReplayerTaskKey, error) {
+func (h *planReplayerTaskCollectorHandle) removeTask(taskKey replayer.PlanReplayerTaskKey) {
+ h.taskMu.Lock()
+ defer h.taskMu.Unlock()
+ delete(h.taskMu.tasks, taskKey)
+}
+
+func (h *planReplayerTaskCollectorHandle) collectAllPlanReplayerTask(ctx context.Context) ([]replayer.PlanReplayerTaskKey, error) {
exec := h.sctx.(sqlexec.SQLExecutor)
rs, err := exec.ExecuteInternal(ctx, "select sql_digest, plan_digest from mysql.plan_replayer_task")
if err != nil {
@@ -241,20 +286,216 @@ func (h *planReplayerTaskCollectorHandle) collectAllPlanReplayerTask(ctx context
if rows, err = sqlexec.DrainRecordSet(ctx, rs, 8); err != nil {
return nil, errors.Trace(err)
}
- allKeys := make([]PlanReplayerTaskKey, 0, len(rows))
+ allKeys := make([]replayer.PlanReplayerTaskKey, 0, len(rows))
for _, row := range rows {
sqlDigest, planDigest := row.GetString(0), row.GetString(1)
- allKeys = append(allKeys, PlanReplayerTaskKey{
- sqlDigest: sqlDigest,
- planDigest: planDigest,
+ allKeys = append(allKeys, replayer.PlanReplayerTaskKey{
+ SQLDigest: sqlDigest,
+ PlanDigest: planDigest,
})
}
return allKeys, nil
}
-func checkUnHandledReplayerTask(ctx context.Context, sctx sessionctx.Context, task PlanReplayerTaskKey) (bool, error) {
+type planReplayerDumpTaskStatus struct {
+ // running task records the task running by all workers in order to avoid multi workers running the same task key
+ runningTaskMu struct {
+ sync.RWMutex
+ runningTasks map[replayer.PlanReplayerTaskKey]struct{}
+ }
+
+ // finished task records the finished task in order to avoid running finished task key
+ finishedTaskMu struct {
+ sync.RWMutex
+ finishedTask map[replayer.PlanReplayerTaskKey]struct{}
+ }
+}
+
+// GetRunningTaskStatusLen used for unit test
+func (r *planReplayerDumpTaskStatus) GetRunningTaskStatusLen() int {
+ r.runningTaskMu.RLock()
+ defer r.runningTaskMu.RUnlock()
+ return len(r.runningTaskMu.runningTasks)
+}
+
+// CleanFinishedTaskStatus clean then finished tasks, only used for unit test
+func (r *planReplayerDumpTaskStatus) CleanFinishedTaskStatus() {
+ r.finishedTaskMu.Lock()
+ defer r.finishedTaskMu.Unlock()
+ r.finishedTaskMu.finishedTask = map[replayer.PlanReplayerTaskKey]struct{}{}
+}
+
+// GetFinishedTaskStatusLen used for unit test
+func (r *planReplayerDumpTaskStatus) GetFinishedTaskStatusLen() int {
+ r.finishedTaskMu.RLock()
+ defer r.finishedTaskMu.RUnlock()
+ return len(r.finishedTaskMu.finishedTask)
+}
+
+func (r *planReplayerDumpTaskStatus) occupyRunningTaskKey(task *PlanReplayerDumpTask) bool {
+ r.runningTaskMu.Lock()
+ defer r.runningTaskMu.Unlock()
+ _, ok := r.runningTaskMu.runningTasks[task.PlanReplayerTaskKey]
+ if ok {
+ return false
+ }
+ r.runningTaskMu.runningTasks[task.PlanReplayerTaskKey] = struct{}{}
+ return true
+}
+
+func (r *planReplayerDumpTaskStatus) releaseRunningTaskKey(task *PlanReplayerDumpTask) {
+ r.runningTaskMu.Lock()
+ defer r.runningTaskMu.Unlock()
+ delete(r.runningTaskMu.runningTasks, task.PlanReplayerTaskKey)
+}
+
+func (r *planReplayerDumpTaskStatus) checkTaskKeyFinishedBefore(task *PlanReplayerDumpTask) bool {
+ r.finishedTaskMu.RLock()
+ defer r.finishedTaskMu.RUnlock()
+ _, ok := r.finishedTaskMu.finishedTask[task.PlanReplayerTaskKey]
+ return ok
+}
+
+func (r *planReplayerDumpTaskStatus) setTaskFinished(task *PlanReplayerDumpTask) {
+ r.finishedTaskMu.Lock()
+ defer r.finishedTaskMu.Unlock()
+ r.finishedTaskMu.finishedTask[task.PlanReplayerTaskKey] = struct{}{}
+}
+
+func (r *planReplayerDumpTaskStatus) clearFinishedTask() {
+ r.finishedTaskMu.Lock()
+ defer r.finishedTaskMu.Unlock()
+ r.finishedTaskMu.finishedTask = map[replayer.PlanReplayerTaskKey]struct{}{}
+}
+
+type planReplayerTaskDumpWorker struct {
+ ctx context.Context
+ sctx sessionctx.Context
+ taskCH <-chan *PlanReplayerDumpTask
+ status *planReplayerDumpTaskStatus
+}
+
+func (w *planReplayerTaskDumpWorker) run() {
+ logutil.BgLogger().Info("planReplayerTaskDumpWorker started.")
+ for task := range w.taskCH {
+ w.handleTask(task)
+ }
+ logutil.BgLogger().Info("planReplayerTaskDumpWorker exited.")
+}
+
+func (w *planReplayerTaskDumpWorker) handleTask(task *PlanReplayerDumpTask) {
+ sqlDigest := task.SQLDigest
+ planDigest := task.PlanDigest
+ check := true
+ occupy := true
+ handleTask := true
+ defer func() {
+ util.Recover(metrics.LabelDomain, "PlanReplayerTaskDumpWorker", nil, false)
+ logutil.BgLogger().Debug("[plan-replayer-capture] handle task",
+ zap.String("sql-digest", sqlDigest),
+ zap.String("plan-digest", planDigest),
+ zap.Bool("check", check),
+ zap.Bool("occupy", occupy),
+ zap.Bool("handle", handleTask))
+ }()
+ if task.IsContinuesCapture {
+ if w.status.checkTaskKeyFinishedBefore(task) {
+ check = false
+ return
+ }
+ }
+ occupy = w.status.occupyRunningTaskKey(task)
+ if !occupy {
+ return
+ }
+ handleTask = w.HandleTask(task)
+ w.status.releaseRunningTaskKey(task)
+}
+
+// HandleTask handled task
+func (w *planReplayerTaskDumpWorker) HandleTask(task *PlanReplayerDumpTask) (success bool) {
+ defer func() {
+ if success && task.IsContinuesCapture {
+ w.status.setTaskFinished(task)
+ }
+ }()
+ taskKey := task.PlanReplayerTaskKey
+ unhandled, err := checkUnHandledReplayerTask(w.ctx, w.sctx, taskKey)
+ if err != nil {
+ logutil.BgLogger().Warn("[plan-replayer-capture] check task failed",
+ zap.String("sqlDigest", taskKey.SQLDigest),
+ zap.String("planDigest", taskKey.PlanDigest),
+ zap.Error(err))
+ return false
+ }
+ // the task is processed, thus we directly skip it.
+ if !unhandled {
+ return true
+ }
+
+ file, fileName, err := replayer.GeneratePlanReplayerFile(task.IsCapture, task.IsContinuesCapture, variable.EnableHistoricalStatsForCapture.Load())
+ if err != nil {
+ logutil.BgLogger().Warn("[plan-replayer-capture] generate task file failed",
+ zap.String("sqlDigest", taskKey.SQLDigest),
+ zap.String("planDigest", taskKey.PlanDigest),
+ zap.Error(err))
+ return false
+ }
+ task.Zf = file
+ task.FileName = fileName
+ if task.InExecute && len(task.NormalizedSQL) > 0 {
+ p := parser.New()
+ stmts, _, err := p.ParseSQL(task.NormalizedSQL)
+ if err != nil {
+ logutil.BgLogger().Warn("[plan-replayer-capture] parse normalized sql failed",
+ zap.String("sql", task.NormalizedSQL),
+ zap.String("sqlDigest", taskKey.SQLDigest),
+ zap.String("planDigest", taskKey.PlanDigest),
+ zap.Error(err))
+ return false
+ }
+ task.ExecStmts = stmts
+ }
+ err = DumpPlanReplayerInfo(w.ctx, w.sctx, task)
+ if err != nil {
+ logutil.BgLogger().Warn("[plan-replayer-capture] dump task result failed",
+ zap.String("sqlDigest", taskKey.SQLDigest),
+ zap.String("planDigest", taskKey.PlanDigest),
+ zap.Error(err))
+ return false
+ }
+ return true
+}
+
+type planReplayerTaskDumpHandle struct {
+ taskCH chan *PlanReplayerDumpTask
+ status *planReplayerDumpTaskStatus
+ workers []*planReplayerTaskDumpWorker
+}
+
+// GetTaskStatus used for test
+func (h *planReplayerTaskDumpHandle) GetTaskStatus() *planReplayerDumpTaskStatus {
+ return h.status
+}
+
+// GetWorker used for test
+func (h *planReplayerTaskDumpHandle) GetWorker() *planReplayerTaskDumpWorker {
+ return h.workers[0]
+}
+
+// Close make finished flag ture
+func (h *planReplayerTaskDumpHandle) Close() {
+ close(h.taskCH)
+}
+
+// DrainTask drain a task for unit test
+func (h *planReplayerTaskDumpHandle) DrainTask() *PlanReplayerDumpTask {
+ return <-h.taskCH
+}
+
+func checkUnHandledReplayerTask(ctx context.Context, sctx sessionctx.Context, task replayer.PlanReplayerTaskKey) (bool, error) {
exec := sctx.(sqlexec.SQLExecutor)
- rs, err := exec.ExecuteInternal(ctx, fmt.Sprintf("select * from mysql.plan_replayer_status where sql_digest = '%v' and plan_digest = '%v' and fail_reason is null", task.sqlDigest, task.planDigest))
+ rs, err := exec.ExecuteInternal(ctx, fmt.Sprintf("select * from mysql.plan_replayer_status where sql_digest = '%v' and plan_digest = '%v' and fail_reason is null", task.SQLDigest, task.PlanDigest))
if err != nil {
return false, err
}
@@ -272,28 +513,59 @@ func checkUnHandledReplayerTask(ctx context.Context, sctx sessionctx.Context, ta
return true, nil
}
+// CheckPlanReplayerTaskExists checks whether plan replayer capture task exists already
+func CheckPlanReplayerTaskExists(ctx context.Context, sctx sessionctx.Context, sqlDigest, planDigest string) (bool, error) {
+ exec := sctx.(sqlexec.SQLExecutor)
+ rs, err := exec.ExecuteInternal(ctx, fmt.Sprintf("select * from mysql.plan_replayer_task where sql_digest = '%v' and plan_digest = '%v'",
+ sqlDigest, planDigest))
+ if err != nil {
+ return false, err
+ }
+ if rs == nil {
+ return false, nil
+ }
+ var rows []chunk.Row
+ defer terror.Call(rs.Close)
+ if rows, err = sqlexec.DrainRecordSet(ctx, rs, 8); err != nil {
+ return false, errors.Trace(err)
+ }
+ if len(rows) > 0 {
+ return true, nil
+ }
+ return false, nil
+}
+
// PlanReplayerStatusRecord indicates record in mysql.plan_replayer_status
type PlanReplayerStatusRecord struct {
- Internal bool
+ SQLDigest string
+ PlanDigest string
OriginSQL string
Token string
FailedReason string
}
-// PlanReplayerTaskKey indicates key of a plan replayer task
-type PlanReplayerTaskKey struct {
- sqlDigest string
- planDigest string
-}
-
// PlanReplayerDumpTask wrap the params for plan replayer dump
type PlanReplayerDumpTask struct {
+ replayer.PlanReplayerTaskKey
+
+ // tmp variables stored during the query
+ TblStats map[int64]interface{}
+ InExecute bool
+ NormalizedSQL string
+
+ // variables used to dump the plan
+ StartTS uint64
SessionBindings []*bindinfo.BindRecord
EncodedPlan string
- FileName string
- Zf *os.File
SessionVars *variable.SessionVars
- TblStats map[int64]*handle.JSONTable
ExecStmts []ast.StmtNode
Analyze bool
+
+ FileName string
+ Zf *os.File
+
+ // IsCapture indicates whether the task is from capture
+ IsCapture bool
+ // IsContinuesCapture indicates whether the task is from continues capture
+ IsContinuesCapture bool
}
diff --git a/domain/plan_replayer_dump.go b/domain/plan_replayer_dump.go
index 93d0278a4ba3d..c656cb11d56e1 100644
--- a/domain/plan_replayer_dump.go
+++ b/domain/plan_replayer_dump.go
@@ -28,6 +28,7 @@ import (
"github.com/pingcap/tidb/bindinfo"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/infoschema"
+ "github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/sessionctx"
@@ -41,6 +42,8 @@ import (
)
const (
+ // PlanReplayerSQLMetaFile indicates sql meta path for plan replayer
+ PlanReplayerSQLMetaFile = "sql_meta.toml"
// PlanReplayerConfigFile indicates config file path for plan replayer
PlanReplayerConfigFile = "config.toml"
// PlanReplayerMetaFile meta file path for plan replayer
@@ -53,6 +56,23 @@ const (
PlanReplayerSessionBindingFile = "session_bindings.sql"
// PlanReplayerGlobalBindingFile indicates global binding file path for plan replayer
PlanReplayerGlobalBindingFile = "global_bindings.sql"
+ // PlanReplayerSchemaMetaFile indicates the schema meta
+ PlanReplayerSchemaMetaFile = "schema_meta.txt"
+)
+
+const (
+ // PlanReplayerSQLMetaStartTS indicates the startTS in plan replayer sql meta
+ PlanReplayerSQLMetaStartTS = "startTS"
+ // PlanReplayerTaskMetaIsCapture indicates whether this task is capture task
+ PlanReplayerTaskMetaIsCapture = "isCapture"
+ // PlanReplayerTaskMetaIsContinues indicates whether this task is continues task
+ PlanReplayerTaskMetaIsContinues = "isContinues"
+ // PlanReplayerTaskMetaSQLDigest indicates the sql digest of this task
+ PlanReplayerTaskMetaSQLDigest = "sqlDigest"
+ // PlanReplayerTaskMetaPlanDigest indicates the plan digest of this task
+ PlanReplayerTaskMetaPlanDigest = "planDigest"
+ // PlanReplayerTaskEnableHistoricalStats indicates whether the task is using historical stats
+ PlanReplayerTaskEnableHistoricalStats = "enableHistoricalStats"
)
type tableNamePair struct {
@@ -88,12 +108,14 @@ func (tne *tableNameExtractor) Leave(in ast.Node) (ast.Node, bool) {
tne.err = err
return in, true
}
- tp := tableNamePair{DBName: t.Schema.L, TableName: t.Name.L, IsView: isView}
- if tp.DBName == "" {
- tp.DBName = tne.curDB.L
- }
- if _, ok := tne.names[tp]; !ok {
- tne.names[tp] = struct{}{}
+ if t.TableInfo != nil {
+ tp := tableNamePair{DBName: t.Schema.L, TableName: t.Name.L, IsView: isView}
+ if tp.DBName == "" {
+ tp.DBName = tne.curDB.L
+ }
+ if _, ok := tne.names[tp]; !ok {
+ tne.names[tp] = struct{}{}
+ }
}
} else if s, ok := in.(*ast.SelectStmt); ok {
if s.With != nil && len(s.With.CTEs) > 0 {
@@ -128,11 +150,18 @@ func (tne *tableNameExtractor) handleIsView(t *ast.TableName) (bool, error) {
return true, nil
}
+var (
+ planReplayerDumpTaskSuccess = metrics.PlanReplayerTaskCounter.WithLabelValues("dump", "success")
+ planReplayerDumpTaskFailed = metrics.PlanReplayerTaskCounter.WithLabelValues("dump", "fail")
+)
+
// DumpPlanReplayerInfo will dump the information about sqls.
// The files will be organized into the following format:
/*
+ |-sql_meta.toml
|-meta.txt
|-schema
+ | |-schema_meta.txt
| |-db1.table1.schema.txt
| |-db2.table2.schema.txt
| |-....
@@ -144,6 +173,10 @@ func (tne *tableNameExtractor) handleIsView(t *ast.TableName) (bool, error) {
| |-stats1.json
| |-stats2.json
| |-....
+ |-statsMem
+ | |-stats1.txt
+ | |-stats2.txt
+ | |-....
|-config.toml
|-table_tiflash_replica.txt
|-variables.toml
@@ -164,22 +197,62 @@ func DumpPlanReplayerInfo(ctx context.Context, sctx sessionctx.Context,
sessionVars := task.SessionVars
execStmts := task.ExecStmts
zw := zip.NewWriter(zf)
- records := generateRecords(task)
+ var records []PlanReplayerStatusRecord
+ sqls := make([]string, 0)
+ for _, execStmt := range task.ExecStmts {
+ sqls = append(sqls, execStmt.Text())
+ }
+ if task.IsCapture {
+ logutil.BgLogger().Info("[plan-replayer-dump] start to dump plan replayer result",
+ zap.String("sql-digest", task.SQLDigest),
+ zap.String("plan-digest", task.PlanDigest),
+ zap.Strings("sql", sqls),
+ zap.Bool("isContinues", task.IsContinuesCapture))
+ } else {
+ logutil.BgLogger().Info("[plan-replayer-dump] start to dump plan replayer result",
+ zap.Strings("sqls", sqls))
+ }
defer func() {
- err = zw.Close()
+ errMsg := ""
if err != nil {
- logutil.BgLogger().Error("Closing zip writer failed", zap.Error(err), zap.String("filename", fileName))
+ if task.IsCapture {
+ logutil.BgLogger().Info("[plan-replayer-dump] dump file failed",
+ zap.String("sql-digest", task.SQLDigest),
+ zap.String("plan-digest", task.PlanDigest),
+ zap.Strings("sql", sqls),
+ zap.Bool("isContinues", task.IsContinuesCapture))
+ } else {
+ logutil.BgLogger().Info("[plan-replayer-dump] start to dump plan replayer result",
+ zap.Strings("sqls", sqls))
+ }
+ errMsg = err.Error()
+ planReplayerDumpTaskFailed.Inc()
+ } else {
+ planReplayerDumpTaskSuccess.Inc()
}
- err = zf.Close()
- if err != nil {
- logutil.BgLogger().Error("Closing zip file failed", zap.Error(err), zap.String("filename", fileName))
+ err1 := zw.Close()
+ if err1 != nil {
+ logutil.BgLogger().Error("[plan-replayer-dump] Closing zip writer failed", zap.Error(err), zap.String("filename", fileName))
+ errMsg = errMsg + "," + err1.Error()
+ }
+ err2 := zf.Close()
+ if err2 != nil {
+ logutil.BgLogger().Error("[plan-replayer-dump] Closing zip file failed", zap.Error(err), zap.String("filename", fileName))
+ errMsg = errMsg + "," + err2.Error()
+ }
+ if len(errMsg) > 0 {
for i, record := range records {
- record.FailedReason = err.Error()
+ record.FailedReason = errMsg
records[i] = record
}
}
insertPlanReplayerStatus(ctx, sctx, records)
}()
+ // Dump SQLMeta
+ if err = dumpSQLMeta(zw, task); err != nil {
+ return err
+ }
+
// Dump config
if err = dumpConfig(zw); err != nil {
return err
@@ -209,8 +282,23 @@ func DumpPlanReplayerInfo(ctx context.Context, sctx sessionctx.Context,
return err
}
- // Dump stats
- if err = dumpStats(zw, pairs, task.TblStats, do); err != nil {
+ // For capture task, we dump stats in storage only if EnableHistoricalStatsForCapture is disabled.
+ // For manual plan replayer dump command, we directly dump stats in storage
+ if task.IsCapture {
+ if !task.IsContinuesCapture && variable.EnableHistoricalStatsForCapture.Load() {
+ // Dump stats
+ if err = dumpStats(zw, pairs, do); err != nil {
+ return err
+ }
+ }
+ } else {
+ // Dump stats
+ if err = dumpStats(zw, pairs, do); err != nil {
+ return err
+ }
+ }
+
+ if err = dumpStatsMemStatus(zw, pairs, do); err != nil {
return err
}
@@ -241,10 +329,11 @@ func DumpPlanReplayerInfo(ctx context.Context, sctx sessionctx.Context,
}
if len(task.EncodedPlan) > 0 {
+ records = generateRecords(task)
return dumpEncodedPlan(sctx, zw, task.EncodedPlan)
}
// Dump explain
- return dumpExplain(sctx, zw, execStmts, task.Analyze)
+ return dumpExplain(sctx, zw, task, &records)
}
func generateRecords(task *PlanReplayerDumpTask) []PlanReplayerStatusRecord {
@@ -252,15 +341,34 @@ func generateRecords(task *PlanReplayerDumpTask) []PlanReplayerStatusRecord {
if len(task.ExecStmts) > 0 {
for _, execStmt := range task.ExecStmts {
records = append(records, PlanReplayerStatusRecord{
- OriginSQL: execStmt.Text(),
- Token: task.FileName,
- Internal: false,
+ SQLDigest: task.SQLDigest,
+ PlanDigest: task.PlanDigest,
+ OriginSQL: execStmt.Text(),
+ Token: task.FileName,
})
}
}
return records
}
+func dumpSQLMeta(zw *zip.Writer, task *PlanReplayerDumpTask) error {
+ cf, err := zw.Create(PlanReplayerSQLMetaFile)
+ if err != nil {
+ return errors.AddStack(err)
+ }
+ varMap := make(map[string]string)
+ varMap[PlanReplayerSQLMetaStartTS] = strconv.FormatUint(task.StartTS, 10)
+ varMap[PlanReplayerTaskMetaIsCapture] = strconv.FormatBool(task.IsCapture)
+ varMap[PlanReplayerTaskMetaIsContinues] = strconv.FormatBool(task.IsContinuesCapture)
+ varMap[PlanReplayerTaskMetaSQLDigest] = task.SQLDigest
+ varMap[PlanReplayerTaskMetaPlanDigest] = task.PlanDigest
+ varMap[PlanReplayerTaskEnableHistoricalStats] = strconv.FormatBool(variable.EnableHistoricalStatsForCapture.Load())
+ if err := toml.NewEncoder(cf).Encode(varMap); err != nil {
+ return errors.AddStack(err)
+ }
+ return nil
+}
+
func dumpConfig(zw *zip.Writer) error {
cf, err := zw.Create(PlanReplayerConfigFile)
if err != nil {
@@ -310,21 +418,70 @@ func dumpTiFlashReplica(ctx sessionctx.Context, zw *zip.Writer, pairs map[tableN
}
func dumpSchemas(ctx sessionctx.Context, zw *zip.Writer, pairs map[tableNamePair]struct{}) error {
+ tables := make(map[tableNamePair]struct{})
for pair := range pairs {
err := getShowCreateTable(pair, zw, ctx)
if err != nil {
return err
}
+ if !pair.IsView {
+ tables[pair] = struct{}{}
+ }
+ }
+ return dumpSchemaMeta(zw, tables)
+}
+
+func dumpSchemaMeta(zw *zip.Writer, tables map[tableNamePair]struct{}) error {
+ zf, err := zw.Create(fmt.Sprintf("schema/%v", PlanReplayerSchemaMetaFile))
+ if err != nil {
+ return err
+ }
+ for table := range tables {
+ _, err := fmt.Fprintf(zf, "%s;%s", table.DBName, table.TableName)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func dumpStatsMemStatus(zw *zip.Writer, pairs map[tableNamePair]struct{}, do *Domain) error {
+ statsHandle := do.StatsHandle()
+ is := do.InfoSchema()
+ for pair := range pairs {
+ if pair.IsView {
+ continue
+ }
+ tbl, err := is.TableByName(model.NewCIStr(pair.DBName), model.NewCIStr(pair.TableName))
+ if err != nil {
+ return err
+ }
+ tblStats := statsHandle.GetTableStats(tbl.Meta())
+ if tblStats == nil {
+ continue
+ }
+ statsMemFw, err := zw.Create(fmt.Sprintf("statsMem/%v.%v.txt", pair.DBName, pair.TableName))
+ if err != nil {
+ return errors.AddStack(err)
+ }
+ fmt.Fprintf(statsMemFw, "[INDEX]\n")
+ for _, indice := range tblStats.Indices {
+ fmt.Fprintf(statsMemFw, "%s\n", fmt.Sprintf("%s=%s", indice.Info.Name.String(), indice.StatusToString()))
+ }
+ fmt.Fprintf(statsMemFw, "[COLUMN]\n")
+ for _, col := range tblStats.Columns {
+ fmt.Fprintf(statsMemFw, "%s\n", fmt.Sprintf("%s=%s", col.Info.Name.String(), col.StatusToString()))
+ }
}
return nil
}
-func dumpStats(zw *zip.Writer, pairs map[tableNamePair]struct{}, tblJSONStats map[int64]*handle.JSONTable, do *Domain) error {
+func dumpStats(zw *zip.Writer, pairs map[tableNamePair]struct{}, do *Domain) error {
for pair := range pairs {
if pair.IsView {
continue
}
- jsonTbl, err := getStatsForTable(do, tblJSONStats, pair)
+ jsonTbl, err := getStatsForTable(do, pair)
if err != nil {
return err
}
@@ -484,12 +641,12 @@ func dumpEncodedPlan(ctx sessionctx.Context, zw *zip.Writer, encodedPlan string)
return nil
}
-func dumpExplain(ctx sessionctx.Context, zw *zip.Writer, execStmts []ast.StmtNode, isAnalyze bool) error {
- for i, stmtExec := range execStmts {
+func dumpExplain(ctx sessionctx.Context, zw *zip.Writer, task *PlanReplayerDumpTask, records *[]PlanReplayerStatusRecord) error {
+ for i, stmtExec := range task.ExecStmts {
sql := stmtExec.Text()
var recordSets []sqlexec.RecordSet
var err error
- if isAnalyze {
+ if task.Analyze {
// Explain analyze
recordSets, err = ctx.(sqlexec.SQLExecutor).Execute(context.Background(), fmt.Sprintf("explain analyze %s", sql))
if err != nil {
@@ -518,6 +675,10 @@ func dumpExplain(ctx sessionctx.Context, zw *zip.Writer, execStmts []ast.StmtNod
return err
}
}
+ *records = append(*records, PlanReplayerStatusRecord{
+ OriginSQL: sql,
+ Token: task.FileName,
+ })
}
return nil
}
@@ -553,19 +714,14 @@ func extractTableNames(ctx context.Context, sctx sessionctx.Context,
return r, nil
}
-func getStatsForTable(do *Domain, tblJSONStats map[int64]*handle.JSONTable, pair tableNamePair) (*handle.JSONTable, error) {
+func getStatsForTable(do *Domain, pair tableNamePair) (*handle.JSONTable, error) {
is := do.InfoSchema()
h := do.StatsHandle()
tbl, err := is.TableByName(model.NewCIStr(pair.DBName), model.NewCIStr(pair.TableName))
if err != nil {
return nil, err
}
- js, ok := tblJSONStats[tbl.Meta().ID]
- if ok && js != nil {
- return js, nil
- }
- js, err = h.DumpStatsToJSON(pair.DBName, tbl.Meta(), nil, true)
- return js, err
+ return h.DumpStatsToJSON(pair.DBName, tbl.Meta(), nil, true)
}
func getShowCreateTable(pair tableNamePair, zw *zip.Writer, ctx sessionctx.Context) error {
diff --git a/domain/plan_replayer_handle_test.go b/domain/plan_replayer_handle_test.go
index 2c25f56e15045..cb5d0bd5126bb 100644
--- a/domain/plan_replayer_handle_test.go
+++ b/domain/plan_replayer_handle_test.go
@@ -15,7 +15,7 @@
package domain_test
import (
- "context"
+ "fmt"
"testing"
"github.com/pingcap/tidb/testkit"
@@ -31,14 +31,14 @@ func TestPlanReplayerHandleCollectTask(t *testing.T) {
tk.MustExec("delete from mysql.plan_replayer_task")
tk.MustExec("delete from mysql.plan_replayer_status")
tk.MustExec("insert into mysql.plan_replayer_task (sql_digest, plan_digest) values ('123','123');")
- err := prHandle.CollectPlanReplayerTask(context.Background())
+ err := prHandle.CollectPlanReplayerTask()
require.NoError(t, err)
require.Len(t, prHandle.GetTasks(), 1)
// assert no task
tk.MustExec("delete from mysql.plan_replayer_task")
tk.MustExec("delete from mysql.plan_replayer_status")
- err = prHandle.CollectPlanReplayerTask(context.Background())
+ err = prHandle.CollectPlanReplayerTask()
require.NoError(t, err)
require.Len(t, prHandle.GetTasks(), 0)
@@ -48,7 +48,7 @@ func TestPlanReplayerHandleCollectTask(t *testing.T) {
tk.MustExec("insert into mysql.plan_replayer_task (sql_digest, plan_digest) values ('123','123');")
tk.MustExec("insert into mysql.plan_replayer_task (sql_digest, plan_digest) values ('345','345');")
tk.MustExec("insert into mysql.plan_replayer_status(sql_digest, plan_digest, token, instance) values ('123','123','123','123')")
- err = prHandle.CollectPlanReplayerTask(context.Background())
+ err = prHandle.CollectPlanReplayerTask()
require.NoError(t, err)
require.Len(t, prHandle.GetTasks(), 1)
@@ -58,7 +58,64 @@ func TestPlanReplayerHandleCollectTask(t *testing.T) {
tk.MustExec("insert into mysql.plan_replayer_task (sql_digest, plan_digest) values ('123','123');")
tk.MustExec("insert into mysql.plan_replayer_task (sql_digest, plan_digest) values ('345','345');")
tk.MustExec("insert into mysql.plan_replayer_status(sql_digest, plan_digest, fail_reason, instance) values ('123','123','123','123')")
- err = prHandle.CollectPlanReplayerTask(context.Background())
+ err = prHandle.CollectPlanReplayerTask()
require.NoError(t, err)
require.Len(t, prHandle.GetTasks(), 2)
}
+
+func TestPlanReplayerHandleDumpTask(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ prHandle := dom.GetPlanReplayerHandle()
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int)")
+ tk.MustQuery("select * from t;")
+ _, d := tk.Session().GetSessionVars().StmtCtx.SQLDigest()
+ _, pd := tk.Session().GetSessionVars().StmtCtx.GetPlanDigest()
+ sqlDigest := d.String()
+ planDigest := pd.String()
+
+ // register task
+ tk.MustExec("delete from mysql.plan_replayer_task")
+ tk.MustExec("delete from mysql.plan_replayer_status")
+ tk.MustExec(fmt.Sprintf("insert into mysql.plan_replayer_task (sql_digest, plan_digest) values ('%v','%v');", sqlDigest, planDigest))
+ err := prHandle.CollectPlanReplayerTask()
+ require.NoError(t, err)
+ require.Len(t, prHandle.GetTasks(), 1)
+
+ tk.MustExec("SET @@tidb_enable_plan_replayer_capture = ON;")
+
+ // capture task and dump
+ tk.MustQuery("select * from t;")
+ task := prHandle.DrainTask()
+ require.NotNil(t, task)
+ worker := prHandle.GetWorker()
+ success := worker.HandleTask(task)
+ require.True(t, success)
+ require.Equal(t, prHandle.GetTaskStatus().GetRunningTaskStatusLen(), 0)
+ // assert memory task consumed
+ require.Len(t, prHandle.GetTasks(), 0)
+
+ // assert collect task again and no more memory task
+ err = prHandle.CollectPlanReplayerTask()
+ require.NoError(t, err)
+ require.Len(t, prHandle.GetTasks(), 0)
+
+ // clean the task and register task
+ prHandle.GetTaskStatus().CleanFinishedTaskStatus()
+ tk.MustExec("delete from mysql.plan_replayer_task")
+ tk.MustExec("delete from mysql.plan_replayer_status")
+ tk.MustExec(fmt.Sprintf("insert into mysql.plan_replayer_task (sql_digest, plan_digest) values ('%v','%v');", sqlDigest, "*"))
+ err = prHandle.CollectPlanReplayerTask()
+ require.NoError(t, err)
+ require.Len(t, prHandle.GetTasks(), 1)
+ tk.MustQuery("select * from t;")
+ task = prHandle.DrainTask()
+ require.NotNil(t, task)
+ worker = prHandle.GetWorker()
+ success = worker.HandleTask(task)
+ require.True(t, success)
+ require.Equal(t, prHandle.GetTaskStatus().GetRunningTaskStatusLen(), 0)
+ // assert capture * task still remained
+ require.Len(t, prHandle.GetTasks(), 1)
+}
diff --git a/domain/plan_replayer_test.go b/domain/plan_replayer_test.go
index 7b44db9b8d239..5e0912b86e66c 100644
--- a/domain/plan_replayer_test.go
+++ b/domain/plan_replayer_test.go
@@ -21,6 +21,7 @@ import (
"testing"
"time"
+ "github.com/pingcap/tidb/util/replayer"
"github.com/stretchr/testify/require"
)
@@ -28,15 +29,15 @@ func TestPlanReplayerGC(t *testing.T) {
startTime := time.Now()
time := startTime.UnixNano()
fileName := fmt.Sprintf("replayer_single_xxxxxx_%v.zip", time)
- err := os.MkdirAll(GetPlanReplayerDirName(), os.ModePerm)
+ err := os.MkdirAll(replayer.GetPlanReplayerDirName(), os.ModePerm)
require.NoError(t, err)
- path := filepath.Join(GetPlanReplayerDirName(), fileName)
+ path := filepath.Join(replayer.GetPlanReplayerDirName(), fileName)
zf, err := os.Create(path)
require.NoError(t, err)
zf.Close()
handler := &dumpFileGcChecker{
- paths: []string{GetPlanReplayerDirName()},
+ paths: []string{replayer.GetPlanReplayerDirName()},
}
handler.gcDumpFiles(0)
diff --git a/domain/schema_validator.go b/domain/schema_validator.go
index 6f028fdca2e70..20018e9c82b19 100644
--- a/domain/schema_validator.go
+++ b/domain/schema_validator.go
@@ -149,12 +149,13 @@ func (s *schemaValidator) Update(leaseGrantTS uint64, oldVer, currVer int64, cha
actionTypes = change.ActionTypes
}
for idx, ac := range actionTypes {
- // NOTE: ac is not an action type, it is (1 << action type).
- if ac == 1<= 64, the value of left shift equals 0, and it will not impact amend txn
+ changedTblMap[tblID] |= 1 << item.relatedActions[i]
affected = true
}
}
@@ -198,22 +199,15 @@ func (s *schemaValidator) isRelatedTablesChanged(currVer int64, tableIDs []int64
}
if len(changedTblMap) > 0 {
tblIds := make([]int64, 0, len(changedTblMap))
- actionTypes := make([]uint64, 0, len(changedTblMap))
for id := range changedTblMap {
tblIds = append(tblIds, id)
}
slices.Sort(tblIds)
- for _, tblID := range tblIds {
- actionTypes = append(actionTypes, changedTblMap[tblID])
- }
- res.PhyTblIDS = tblIds
- res.ActionTypes = actionTypes
- res.Amendable = true
logutil.BgLogger().Info("schema of tables in the transaction are changed", zap.Int64s("conflicted table IDs", tblIds),
zap.Int64("transaction schema", currVer), zap.Int64s("schema versions that changed the tables", changedSchemaVers))
- return res, true
+ return true
}
- return res, false
+ return false
}
func (s *schemaValidator) findNewerDeltas(currVer int64) []deltaSchemaInfo {
@@ -251,12 +245,8 @@ func (s *schemaValidator) Check(txnTS uint64, schemaVer int64, relatedPhysicalTa
// When disabling MDL -> enabling MDL, the old transaction's needCheckSchema is true, we need to check it.
// When enabling MDL -> disabling MDL, the old transaction's needCheckSchema is false, so still need to check it, and variable EnableMDL is false now.
if needCheckSchema || !variable.EnableMDL.Load() {
- relatedChanges, changed := s.isRelatedTablesChanged(schemaVer, relatedPhysicalTableIDs)
+ changed := s.isRelatedTablesChanged(schemaVer, relatedPhysicalTableIDs)
if changed {
- if relatedChanges.Amendable {
- relatedChanges.LatestInfoSchema = s.latestInfoSchema
- return &relatedChanges, ResultFail
- }
return nil, ResultFail
}
}
diff --git a/domain/schema_validator_test.go b/domain/schema_validator_test.go
index a18fbcb4a435a..ddcc57634ab60 100644
--- a/domain/schema_validator_test.go
+++ b/domain/schema_validator_test.go
@@ -61,7 +61,7 @@ func subTestSchemaValidatorGeneral(t *testing.T) {
// Stop the validator, validator's items value is nil.
validator.Stop()
require.False(t, validator.IsStarted())
- _, isTablesChanged := validator.isRelatedTablesChanged(item.schemaVer, []int64{10})
+ isTablesChanged := validator.isRelatedTablesChanged(item.schemaVer, []int64{10})
require.True(t, isTablesChanged)
_, valid = validator.Check(item.leaseGrantTS, item.schemaVer, []int64{10}, true)
require.Equal(t, ResultUnknown, valid)
@@ -91,12 +91,12 @@ func subTestSchemaValidatorGeneral(t *testing.T) {
validator.Update(ts, currVer, newItem.schemaVer, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1, 2, 3}, ActionTypes: []uint64{1, 2, 3}})
// Make sure the updated table IDs don't be covered with the same schema version.
validator.Update(ts, newItem.schemaVer, newItem.schemaVer, nil)
- _, isTablesChanged = validator.isRelatedTablesChanged(currVer, nil)
+ isTablesChanged = validator.isRelatedTablesChanged(currVer, nil)
require.False(t, isTablesChanged)
- _, isTablesChanged = validator.isRelatedTablesChanged(currVer, []int64{2})
+ isTablesChanged = validator.isRelatedTablesChanged(currVer, []int64{2})
require.Truef(t, isTablesChanged, "currVer %d, newItem %v", currVer, newItem)
// The current schema version is older than the oldest schema version.
- _, isTablesChanged = validator.isRelatedTablesChanged(-1, nil)
+ isTablesChanged = validator.isRelatedTablesChanged(-1, nil)
require.Truef(t, isTablesChanged, "currVer %d, newItem %v", currVer, newItem)
// All schema versions is expired.
@@ -214,10 +214,8 @@ func subTestEnqueueActionType(t *testing.T) {
// Check the flag set by schema diff, note tableID = 3 has been set flag 0x3 in schema version 9, and flag 0x4
// in schema version 10, so the resActions for tableID = 3 should be 0x3 & 0x4 = 0x7.
- relatedChanges, isTablesChanged := validator.isRelatedTablesChanged(5, []int64{1, 2, 3, 4})
+ isTablesChanged := validator.isRelatedTablesChanged(5, []int64{1, 2, 3, 4})
require.True(t, isTablesChanged)
- require.Equal(t, []int64{1, 2, 3, 4}, relatedChanges.PhyTblIDS)
- require.Equal(t, []uint64{15, 2, 7, 4}, relatedChanges.ActionTypes)
}
type leaseGrantItem struct {
diff --git a/domain/sysvar_cache.go b/domain/sysvar_cache.go
index 370260a67c02a..1611231d42ad5 100644
--- a/domain/sysvar_cache.go
+++ b/domain/sysvar_cache.go
@@ -17,13 +17,13 @@ package domain
import (
"context"
"fmt"
- "sync"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/sqlexec"
+ "github.com/pingcap/tidb/util/syncutil"
"go.uber.org/zap"
"golang.org/x/exp/maps"
)
@@ -36,10 +36,10 @@ import (
// sysVarCache represents the cache of system variables broken up into session and global scope.
type sysVarCache struct {
- sync.RWMutex // protects global and session maps
- global map[string]string
- session map[string]string
- rebuildLock sync.Mutex // protects concurrent rebuild
+ syncutil.RWMutex // protects global and session maps
+ global map[string]string
+ session map[string]string
+ rebuildLock syncutil.Mutex // protects concurrent rebuild
}
func (do *Domain) rebuildSysVarCacheIfNeeded() (err error) {
diff --git a/domain/test_helper.go b/domain/test_helper.go
index e63e8dee7389b..e8c106c29d23b 100644
--- a/domain/test_helper.go
+++ b/domain/test_helper.go
@@ -26,7 +26,7 @@ import (
// MockInfoCacheAndLoadInfoSchema only used in unit tests.
func (do *Domain) MockInfoCacheAndLoadInfoSchema(is infoschema.InfoSchema) {
- do.infoCache = infoschema.NewCache(16)
+ do.infoCache.Reset(16)
do.infoCache.Insert(is, 0)
}
diff --git a/dumpling/export/BUILD.bazel b/dumpling/export/BUILD.bazel
index 08cc7fe2e664e..f47a49249adee 100644
--- a/dumpling/export/BUILD.bazel
+++ b/dumpling/export/BUILD.bazel
@@ -103,9 +103,11 @@ go_test(
"//util/filter",
"//util/promutil",
"//util/table-filter",
+ "@com_github_coreos_go_semver//semver",
"@com_github_data_dog_go_sqlmock//:go-sqlmock",
"@com_github_go_sql_driver_mysql//:mysql",
"@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
"@com_github_prometheus_client_golang//prometheus/collectors",
"@com_github_stretchr_testify//require",
"@org_golang_x_sync//errgroup",
diff --git a/dumpling/export/config.go b/dumpling/export/config.go
index 57504e4e20882..36af37b30e924 100644
--- a/dumpling/export/config.go
+++ b/dumpling/export/config.go
@@ -4,6 +4,7 @@ package export
import (
"context"
+ "crypto/tls"
"encoding/json"
"fmt"
"net"
@@ -15,7 +16,6 @@ import (
"github.com/coreos/go-semver/semver"
"github.com/docker/go-units"
"github.com/go-sql-driver/mysql"
- "github.com/google/uuid"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/br/pkg/storage"
@@ -25,6 +25,7 @@ import (
filter "github.com/pingcap/tidb/util/table-filter"
"github.com/prometheus/client_golang/prometheus"
"github.com/spf13/pflag"
+ "go.uber.org/atomic"
"go.uber.org/zap"
)
@@ -103,7 +104,7 @@ type Config struct {
User string
Password string `json:"-"`
Security struct {
- DriveTLSName string `json:"-"`
+ TLS *tls.Config `json:"-"`
CAPath string
CertPath string
KeyPath string
@@ -144,6 +145,9 @@ type Config struct {
PromFactory promutil.Factory `json:"-"`
PromRegistry promutil.Registry `json:"-"`
ExtStorage storage.ExternalStorage `json:"-"`
+
+ IOTotalBytes *atomic.Uint64
+ Net string
}
// ServerInfoUnknown is the unknown database type to dumpling
@@ -212,6 +216,9 @@ func (conf *Config) GetDriverConfig(db string) *mysql.Config {
driverCfg.User = conf.User
driverCfg.Passwd = conf.Password
driverCfg.Net = "tcp"
+ if conf.Net != "" {
+ driverCfg.Net = conf.Net
+ }
driverCfg.Addr = hostPort
driverCfg.DBName = db
driverCfg.Collation = "utf8mb4_general_ci"
@@ -219,8 +226,17 @@ func (conf *Config) GetDriverConfig(db string) *mysql.Config {
driverCfg.WriteTimeout = 30 * time.Second
driverCfg.InterpolateParams = true
driverCfg.MaxAllowedPacket = 0
- if conf.Security.DriveTLSName != "" {
- driverCfg.TLSConfig = conf.Security.DriveTLSName
+ if conf.Security.TLS != nil {
+ driverCfg.TLS = conf.Security.TLS
+ } else {
+ // Use TLS first.
+ driverCfg.AllowFallbackToPlaintext = true
+ /* #nosec G402 */
+ driverCfg.TLS = &tls.Config{
+ InsecureSkipVerify: true,
+ MinVersion: tls.VersionTLS10,
+ NextProtos: []string{"h2", "http/1.1"}, // specify `h2` to let Go use HTTP/2.
+ }
}
if conf.AllowCleartextPasswords {
driverCfg.AllowCleartextPasswords = true
@@ -653,7 +669,7 @@ func adjustConfig(conf *Config, fns ...func(*Config) error) error {
return nil
}
-func registerTLSConfig(conf *Config) error {
+func buildTLSConfig(conf *Config) error {
tlsConfig, err := util.NewTLSConfig(
util.WithCAPath(conf.Security.CAPath),
util.WithCertAndKeyPath(conf.Security.CertPath, conf.Security.KeyPath),
@@ -663,14 +679,8 @@ func registerTLSConfig(conf *Config) error {
if err != nil {
return errors.Trace(err)
}
-
- if tlsConfig == nil {
- return nil
- }
-
- conf.Security.DriveTLSName = "dumpling" + uuid.NewString()
- err = mysql.RegisterTLSConfig(conf.Security.DriveTLSName, tlsConfig)
- return errors.Trace(err)
+ conf.Security.TLS = tlsConfig
+ return nil
}
func validateSpecifiedSQL(conf *Config) error {
diff --git a/dumpling/export/dump.go b/dumpling/export/dump.go
index 7e5a81e0f3ce1..82e093a213b6b 100644
--- a/dumpling/export/dump.go
+++ b/dumpling/export/dump.go
@@ -10,13 +10,16 @@ import (
"encoding/hex"
"fmt"
"math/big"
+ "net"
"strconv"
"strings"
"sync/atomic"
"time"
+ "github.com/coreos/go-semver/semver"
// import mysql driver
"github.com/go-sql-driver/mysql"
+ "github.com/google/uuid"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
pclog "github.com/pingcap/log"
@@ -31,8 +34,10 @@ import (
"github.com/pingcap/tidb/parser/format"
"github.com/pingcap/tidb/store/helper"
"github.com/pingcap/tidb/tablecodec"
+ "github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/codec"
pd "github.com/tikv/pd/client"
+ gatomic "go.uber.org/atomic"
"go.uber.org/zap"
"golang.org/x/exp/slices"
"golang.org/x/sync/errgroup"
@@ -42,6 +47,10 @@ var openDBFunc = openDB
var errEmptyHandleVals = errors.New("empty handleVals for TiDB table")
+// After TiDB v6.2.0 we always enable tidb_enable_paging by default.
+// see https://docs.pingcap.com/zh/tidb/dev/system-variables#tidb_enable_paging-%E4%BB%8E-v540-%E7%89%88%E6%9C%AC%E5%BC%80%E5%A7%8B%E5%BC%95%E5%85%A5
+var enablePagingVersion = semver.New("6.2.0")
+
// Dumper is the dump progress structure
type Dumper struct {
tctx *tcontext.Context
@@ -95,12 +104,23 @@ func NewDumper(ctx context.Context, conf *Config) (*Dumper, error) {
}()
err = adjustConfig(conf,
- registerTLSConfig,
+ buildTLSConfig,
validateSpecifiedSQL,
adjustFileFormat)
if err != nil {
return nil, err
}
+ failpoint.Inject("SetIOTotalBytes", func(_ failpoint.Value) {
+ d.conf.IOTotalBytes = gatomic.NewUint64(0)
+ d.conf.Net = uuid.New().String()
+ go func() {
+ for {
+ time.Sleep(10 * time.Millisecond)
+ d.tctx.L().Logger.Info("IOTotalBytes", zap.Uint64("IOTotalBytes", d.conf.IOTotalBytes.Load()))
+ }
+ }()
+ })
+
err = runSteps(d,
initLogger,
createExternalStore,
@@ -1199,9 +1219,7 @@ func dumpTableMeta(tctx *tcontext.Context, conf *Config, conn *BaseConn, db stri
selectedField: selectField,
selectedLen: selectLen,
hasImplicitRowID: hasImplicitRowID,
- specCmts: []string{
- "/*!40101 SET NAMES binary*/;",
- },
+ specCmts: getSpecialComments(conf.ServerInfo.ServerType),
}
if conf.NoSchemas {
@@ -1264,9 +1282,6 @@ func (d *Dumper) Close() error {
if d.dbHandle != nil {
return d.dbHandle.Close()
}
- if d.conf.Security.DriveTLSName != "" {
- mysql.DeregisterTLSConfig(d.conf.Security.DriveTLSName)
- }
return nil
}
@@ -1333,6 +1348,22 @@ func startHTTPService(d *Dumper) error {
// openSQLDB is an initialization step of Dumper.
func openSQLDB(d *Dumper) error {
+ if d.conf.IOTotalBytes != nil {
+ mysql.RegisterDialContext(d.conf.Net, func(ctx context.Context, addr string) (net.Conn, error) {
+ dial := &net.Dialer{}
+ conn, err := dial.DialContext(ctx, "tcp", addr)
+ if err != nil {
+ return nil, err
+ }
+ tcpConn := conn.(*net.TCPConn)
+ // try https://github.com/go-sql-driver/mysql/blob/bcc459a906419e2890a50fc2c99ea6dd927a88f2/connector.go#L56-L64
+ err = tcpConn.SetKeepAlive(true)
+ if err != nil {
+ d.tctx.L().Logger.Warn("fail to keep alive", zap.Error(err))
+ }
+ return util.NewTCPConnWithIOCounter(tcpConn, d.conf.IOTotalBytes), nil
+ })
+ }
conf := d.conf
c, err := mysql.NewConnector(conf.GetDriverConfig(""))
if err != nil {
@@ -1511,6 +1542,19 @@ func updateServiceSafePoint(tctx *tcontext.Context, pdClient pd.Client, ttl int6
}
}
+// setDefaultSessionParams is a step to set default params for session params.
+func setDefaultSessionParams(si version.ServerInfo, sessionParams map[string]interface{}) {
+ defaultSessionParams := map[string]interface{}{}
+ if si.ServerType == version.ServerTypeTiDB && si.HasTiKV && si.ServerVersion.Compare(*enablePagingVersion) >= 0 {
+ defaultSessionParams["tidb_enable_paging"] = "ON"
+ }
+ for k, v := range defaultSessionParams {
+ if _, ok := sessionParams[k]; !ok {
+ sessionParams[k] = v
+ }
+ }
+}
+
// setSessionParam is an initialization step of Dumper.
func setSessionParam(d *Dumper) error {
conf, pool := d.conf, d.dbHandle
@@ -1531,7 +1575,7 @@ func setSessionParam(d *Dumper) error {
d.L().Info("cannot check whether TiDB has TiKV, will apply tidb_snapshot by default. This won't affect dump process", log.ShortError(err))
}
if conf.ServerInfo.HasTiKV {
- sessionParam["tidb_snapshot"] = snapshot
+ sessionParam[snapshotVar] = snapshot
}
}
}
diff --git a/dumpling/export/dump_test.go b/dumpling/export/dump_test.go
index c9a40bba28d6f..f77171b1cfa8c 100644
--- a/dumpling/export/dump_test.go
+++ b/dumpling/export/dump_test.go
@@ -9,7 +9,10 @@ import (
"time"
"github.com/DATA-DOG/go-sqlmock"
+ "github.com/coreos/go-semver/semver"
+ "github.com/go-sql-driver/mysql"
"github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/br/pkg/version"
tcontext "github.com/pingcap/tidb/dumpling/context"
"github.com/pingcap/tidb/parser"
@@ -224,3 +227,128 @@ func TestUnregisterMetrics(t *testing.T) {
// should not panic
require.Error(t, err)
}
+
+func TestSetDefaultSessionParams(t *testing.T) {
+ testCases := []struct {
+ si version.ServerInfo
+ sessionParams map[string]interface{}
+ expectedParams map[string]interface{}
+ }{
+ {
+ si: version.ServerInfo{
+ ServerType: version.ServerTypeTiDB,
+ HasTiKV: true,
+ ServerVersion: semver.New("6.1.0"),
+ },
+ sessionParams: map[string]interface{}{
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ expectedParams: map[string]interface{}{
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ },
+ {
+ si: version.ServerInfo{
+ ServerType: version.ServerTypeTiDB,
+ HasTiKV: true,
+ ServerVersion: semver.New("6.2.0"),
+ },
+ sessionParams: map[string]interface{}{
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ expectedParams: map[string]interface{}{
+ "tidb_enable_paging": "ON",
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ },
+ {
+ si: version.ServerInfo{
+ ServerType: version.ServerTypeTiDB,
+ HasTiKV: true,
+ ServerVersion: semver.New("6.2.0"),
+ },
+ sessionParams: map[string]interface{}{
+ "tidb_enable_paging": "OFF",
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ expectedParams: map[string]interface{}{
+ "tidb_enable_paging": "OFF",
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ },
+ {
+ si: version.ServerInfo{
+ ServerType: version.ServerTypeMySQL,
+ ServerVersion: semver.New("8.0.32"),
+ },
+ sessionParams: map[string]interface{}{},
+ expectedParams: map[string]interface{}{},
+ },
+ }
+
+ for _, testCase := range testCases {
+ setDefaultSessionParams(testCase.si, testCase.sessionParams)
+ require.Equal(t, testCase.expectedParams, testCase.sessionParams)
+ }
+}
+
+func TestSetSessionParams(t *testing.T) {
+ // case 1: fail to set tidb_snapshot, should return error with hint
+ db, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ defer func() {
+ require.NoError(t, db.Close())
+ }()
+
+ mock.ExpectQuery("SELECT @@tidb_config").
+ WillReturnError(errors.New("mock error"))
+ mock.ExpectQuery("SELECT COUNT\\(1\\) as c FROM MYSQL.TiDB WHERE VARIABLE_NAME='tikv_gc_safe_point'").
+ WillReturnError(errors.New("mock error"))
+ tikvErr := &mysql.MySQLError{
+ Number: 1105,
+ Message: "can not get 'tikv_gc_safe_point'",
+ }
+ mock.ExpectExec("SET SESSION tidb_snapshot").
+ WillReturnError(tikvErr)
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/dumpling/export/SkipResetDB", "return(true)"))
+ defer failpoint.Disable("github.com/pingcap/tidb/dumpling/export/SkipResetDB=return(true)")
+
+ tctx, cancel := tcontext.Background().WithLogger(appLogger).WithCancel()
+ defer cancel()
+
+ conf := DefaultConfig()
+ conf.ServerInfo = version.ServerInfo{
+ ServerType: version.ServerTypeTiDB,
+ HasTiKV: false,
+ }
+ conf.Snapshot = "439153276059648000"
+ conf.Consistency = ConsistencyTypeSnapshot
+ d := &Dumper{
+ tctx: tctx,
+ conf: conf,
+ cancelCtx: cancel,
+ dbHandle: db,
+ }
+ err = setSessionParam(d)
+ require.ErrorContains(t, err, "consistency=none")
+
+ // case 2: fail to set other
+ conf.ServerInfo = version.ServerInfo{
+ ServerType: version.ServerTypeMySQL,
+ HasTiKV: false,
+ }
+ conf.Snapshot = ""
+ conf.Consistency = ConsistencyTypeFlush
+ conf.SessionParams = map[string]interface{}{
+ "mock": "UTC",
+ }
+ d.dbHandle = db
+ mock.ExpectExec("SET SESSION mock").
+ WillReturnError(errors.New("Unknown system variable mock"))
+ mock.ExpectClose()
+ mock.ExpectClose()
+
+ err = setSessionParam(d)
+ require.NoError(t, err)
+}
diff --git a/dumpling/export/ir.go b/dumpling/export/ir.go
index 4b98019605e9c..aa1b2070591ab 100644
--- a/dumpling/export/ir.go
+++ b/dumpling/export/ir.go
@@ -8,6 +8,7 @@ import (
"strings"
"github.com/pingcap/errors"
+ "github.com/pingcap/tidb/br/pkg/version"
tcontext "github.com/pingcap/tidb/dumpling/context"
)
@@ -85,7 +86,7 @@ type MetaIR interface {
MetaSQL() string
}
-func setTableMetaFromRows(rows *sql.Rows) (TableMeta, error) {
+func setTableMetaFromRows(serverType version.ServerType, rows *sql.Rows) (TableMeta, error) {
tps, err := rows.ColumnTypes()
if err != nil {
return nil, errors.Trace(err)
@@ -101,6 +102,6 @@ func setTableMetaFromRows(rows *sql.Rows) (TableMeta, error) {
colTypes: tps,
selectedField: strings.Join(nms, ","),
selectedLen: len(nms),
- specCmts: []string{"/*!40101 SET NAMES binary*/;"},
+ specCmts: getSpecialComments(serverType),
}, nil
}
diff --git a/dumpling/export/ir_impl.go b/dumpling/export/ir_impl.go
index bca821f612623..d1efa75db3365 100644
--- a/dumpling/export/ir_impl.go
+++ b/dumpling/export/ir_impl.go
@@ -7,6 +7,7 @@ import (
"strings"
"github.com/pingcap/errors"
+ "github.com/pingcap/tidb/br/pkg/version"
tcontext "github.com/pingcap/tidb/dumpling/context"
"go.uber.org/zap"
)
@@ -370,3 +371,22 @@ func (td *multiQueriesChunk) Close() error {
func (*multiQueriesChunk) RawRows() *sql.Rows {
return nil
}
+
+var serverSpecialComments = map[version.ServerType][]string{
+ version.ServerTypeMySQL: {
+ "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;",
+ "/*!40101 SET NAMES binary*/;",
+ },
+ version.ServerTypeTiDB: {
+ "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;",
+ "/*!40101 SET NAMES binary*/;",
+ },
+ version.ServerTypeMariaDB: {
+ "/*!40101 SET NAMES binary*/;",
+ "SET FOREIGN_KEY_CHECKS=0;",
+ },
+}
+
+func getSpecialComments(serverType version.ServerType) []string {
+ return serverSpecialComments[serverType]
+}
diff --git a/dumpling/export/sql.go b/dumpling/export/sql.go
index 837bec568b9a7..60d14ac49e14c 100644
--- a/dumpling/export/sql.go
+++ b/dumpling/export/sql.go
@@ -29,6 +29,7 @@ import (
const (
orderByTiDBRowID = "ORDER BY `_tidb_rowid`"
+ snapshotVar = "tidb_snapshot"
)
type listTableType int
@@ -851,7 +852,9 @@ func resetDBWithSessionParams(tctx *tcontext.Context, db *sql.DB, cfg *mysql.Con
s := fmt.Sprintf("SET SESSION %s = ?", k)
_, err := db.ExecContext(tctx, s, pv)
if err != nil {
- if isUnknownSystemVariableErr(err) {
+ if k == snapshotVar {
+ err = errors.Annotate(err, "fail to set snapshot for tidb, please set --consistency=none/--consistency=lock or fix snapshot problem")
+ } else if isUnknownSystemVariableErr(err) {
tctx.L().Info("session variable is not supported by db", zap.String("variable", k), zap.Reflect("value", v))
continue
}
@@ -876,6 +879,9 @@ func resetDBWithSessionParams(tctx *tcontext.Context, db *sql.DB, cfg *mysql.Con
}
cfg.Params[k] = s
}
+ failpoint.Inject("SkipResetDB", func(_ failpoint.Value) {
+ failpoint.Return(db, nil)
+ })
db.Close()
c, err := mysql.NewConnector(cfg)
diff --git a/dumpling/export/sql_test.go b/dumpling/export/sql_test.go
index 04615637be8f1..5ae4c7278efa4 100644
--- a/dumpling/export/sql_test.go
+++ b/dumpling/export/sql_test.go
@@ -1308,6 +1308,8 @@ func buildMockNewRows(mock sqlmock.Sqlmock, columns []string, driverValues [][]d
}
func readRegionCsvDriverValues(t *testing.T) [][]driver.Value {
+ t.Helper()
+
csvFilename := "region_results.csv"
file, err := os.Open(csvFilename)
require.NoError(t, err)
diff --git a/dumpling/export/status_test.go b/dumpling/export/status_test.go
index 14c08c7c36ba7..7c340b06dcf83 100644
--- a/dumpling/export/status_test.go
+++ b/dumpling/export/status_test.go
@@ -48,7 +48,7 @@ func TestSpeedRecorder(t *testing.T) {
for _, tc := range testCases {
time.Sleep(time.Duration(tc.spentTime) * time.Second)
recentSpeed := speedRecorder.GetSpeed(tc.finished)
- if math.Abs(tc.expected-recentSpeed) > 1 {
+ if math.Abs(tc.expected-recentSpeed)/tc.expected > 0.1 {
require.FailNow(t, "speed is unexpected", "expected: %5.2f, recent: %5.2f", tc.expected, recentSpeed)
}
}
diff --git a/dumpling/export/writer.go b/dumpling/export/writer.go
index e64c067c807b7..ae037041596ae 100644
--- a/dumpling/export/writer.go
+++ b/dumpling/export/writer.go
@@ -133,7 +133,7 @@ func (w *Writer) WritePolicyMeta(policy, createSQL string) error {
if err != nil {
return err
}
- return writeMetaToFile(tctx, "placement-policy", createSQL, w.extStorage, fileName+".sql", conf.CompressType)
+ return w.writeMetaToFile(tctx, "placement-policy", createSQL, fileName+".sql")
}
// WriteDatabaseMeta writes database meta to a file
@@ -143,7 +143,7 @@ func (w *Writer) WriteDatabaseMeta(db, createSQL string) error {
if err != nil {
return err
}
- return writeMetaToFile(tctx, db, createSQL, w.extStorage, fileName+".sql", conf.CompressType)
+ return w.writeMetaToFile(tctx, db, createSQL, fileName+".sql")
}
// WriteTableMeta writes table meta to a file
@@ -153,7 +153,7 @@ func (w *Writer) WriteTableMeta(db, table, createSQL string) error {
if err != nil {
return err
}
- return writeMetaToFile(tctx, db, createSQL, w.extStorage, fileName+".sql", conf.CompressType)
+ return w.writeMetaToFile(tctx, db, createSQL, fileName+".sql")
}
// WriteViewMeta writes view meta to a file
@@ -167,11 +167,11 @@ func (w *Writer) WriteViewMeta(db, view, createTableSQL, createViewSQL string) e
if err != nil {
return err
}
- err = writeMetaToFile(tctx, db, createTableSQL, w.extStorage, fileNameTable+".sql", conf.CompressType)
+ err = w.writeMetaToFile(tctx, db, createTableSQL, fileNameTable+".sql")
if err != nil {
return err
}
- return writeMetaToFile(tctx, db, createViewSQL, w.extStorage, fileNameView+".sql", conf.CompressType)
+ return w.writeMetaToFile(tctx, db, createViewSQL, fileNameView+".sql")
}
// WriteSequenceMeta writes sequence meta to a file
@@ -181,7 +181,7 @@ func (w *Writer) WriteSequenceMeta(db, sequence, createSQL string) error {
if err != nil {
return err
}
- return writeMetaToFile(tctx, db, createSQL, w.extStorage, fileName+".sql", conf.CompressType)
+ return w.writeMetaToFile(tctx, db, createSQL, fileName+".sql")
}
// WriteTableData writes table data to a file with retry
@@ -213,10 +213,14 @@ func (w *Writer) WriteTableData(meta TableMeta, ir TableDataIR, currentChunk int
return
}
if conf.SQL != "" {
- meta, err = setTableMetaFromRows(ir.RawRows())
+ rows := ir.RawRows()
+ meta, err = setTableMetaFromRows(w.conf.ServerInfo.ServerType, rows)
if err != nil {
return err
}
+ if err = rows.Err(); err != nil {
+ return errors.Trace(err)
+ }
}
defer func() {
_ = ir.Close()
@@ -270,19 +274,17 @@ func (w *Writer) tryToWriteTableData(tctx *tcontext.Context, meta TableMeta, ir
return nil
}
-func writeMetaToFile(tctx *tcontext.Context, target, metaSQL string, s storage.ExternalStorage, path string, compressType storage.CompressType) error {
- fileWriter, tearDown, err := buildFileWriter(tctx, s, path, compressType)
+func (w *Writer) writeMetaToFile(tctx *tcontext.Context, target, metaSQL string, path string) error {
+ fileWriter, tearDown, err := buildFileWriter(tctx, w.extStorage, path, w.conf.CompressType)
if err != nil {
return errors.Trace(err)
}
defer tearDown(tctx)
return WriteMeta(tctx, &metaData{
- target: target,
- metaSQL: metaSQL,
- specCmts: []string{
- "/*!40101 SET NAMES binary*/;",
- },
+ target: target,
+ metaSQL: metaSQL,
+ specCmts: getSpecialComments(w.conf.ServerInfo.ServerType),
}, fileWriter)
}
diff --git a/dumpling/export/writer_test.go b/dumpling/export/writer_test.go
index b5f54f3debcb9..4f07d25b7b224 100644
--- a/dumpling/export/writer_test.go
+++ b/dumpling/export/writer_test.go
@@ -5,13 +5,13 @@ package export
import (
"context"
"database/sql/driver"
- "io/ioutil"
"os"
"path"
"sync"
"testing"
"github.com/DATA-DOG/go-sqlmock"
+ "github.com/pingcap/tidb/br/pkg/version"
tcontext "github.com/pingcap/tidb/dumpling/context"
"github.com/pingcap/tidb/util/promutil"
"github.com/stretchr/testify/require"
@@ -31,9 +31,9 @@ func TestWriteDatabaseMeta(t *testing.T) {
_, err = os.Stat(p)
require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
require.NoError(t, err)
- require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE DATABASE `test`;\n", string(bytes))
+ require.Equal(t, "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;\n/*!40101 SET NAMES binary*/;\nCREATE DATABASE `test`;\n", string(bytes))
}
func TestWritePolicyMeta(t *testing.T) {
@@ -50,9 +50,9 @@ func TestWritePolicyMeta(t *testing.T) {
_, err = os.Stat(p)
require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
require.NoError(t, err)
- require.Equal(t, "/*!40101 SET NAMES binary*/;\ncreate placement policy `y` followers=2;\n", string(bytes))
+ require.Equal(t, "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;\n/*!40101 SET NAMES binary*/;\ncreate placement policy `y` followers=2;\n", string(bytes))
}
func TestWriteTableMeta(t *testing.T) {
@@ -68,9 +68,9 @@ func TestWriteTableMeta(t *testing.T) {
p := path.Join(dir, "test.t-schema.sql")
_, err = os.Stat(p)
require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
require.NoError(t, err)
- require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes))
+ require.Equal(t, "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;\n/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes))
}
func TestWriteViewMeta(t *testing.T) {
@@ -80,7 +80,7 @@ func TestWriteViewMeta(t *testing.T) {
writer := createTestWriter(config, t)
- specCmt := "/*!40101 SET NAMES binary*/;\n"
+ specCmt := "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;\n/*!40101 SET NAMES binary*/;\n"
createTableSQL := "CREATE TABLE `v`(\n`a` int\n)ENGINE=MyISAM;\n"
createViewSQL := "DROP TABLE IF EXISTS `v`;\nDROP VIEW IF EXISTS `v`;\nSET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT;\nSET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS;\nSET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION;\nSET character_set_client = utf8;\nSET character_set_results = utf8;\nSET collation_connection = utf8_general_ci;\nCREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v` (`a`) AS SELECT `t`.`a` AS `a` FROM `test`.`t`;\nSET character_set_client = @PREV_CHARACTER_SET_CLIENT;\nSET character_set_results = @PREV_CHARACTER_SET_RESULTS;\nSET collation_connection = @PREV_COLLATION_CONNECTION;\n"
err := writer.WriteViewMeta("test", "v", createTableSQL, createViewSQL)
@@ -89,14 +89,14 @@ func TestWriteViewMeta(t *testing.T) {
p := path.Join(dir, "test.v-schema.sql")
_, err = os.Stat(p)
require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, specCmt+createTableSQL, string(bytes))
p = path.Join(dir, "test.v-schema-view.sql")
_, err = os.Stat(p)
require.NoError(t, err)
- bytes, err = ioutil.ReadFile(p)
+ bytes, err = os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, specCmt+createViewSQL, string(bytes))
}
@@ -126,7 +126,7 @@ func TestWriteTableData(t *testing.T) {
p := path.Join(dir, "test.employee.000000000.sql")
_, err = os.Stat(p)
require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
require.NoError(t, err)
expected := "/*!40101 SET NAMES binary*/;\n" +
@@ -182,7 +182,7 @@ func TestWriteTableDataWithFileSize(t *testing.T) {
p = path.Join(dir, p)
_, err := os.Stat(p)
require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, expected, string(bytes))
}
@@ -232,7 +232,7 @@ func TestWriteTableDataWithFileSizeAndRows(t *testing.T) {
p = path.Join(dir, p)
_, err = os.Stat(p)
require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, expected, string(bytes))
}
@@ -281,7 +281,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
p = path.Join(config.OutputDirPath, p)
_, err = os.Stat(p)
require.NoError(t, err)
- bytes, err1 := ioutil.ReadFile(p)
+ bytes, err1 := os.ReadFile(p)
require.NoError(t, err1)
require.Equal(t, expected, string(bytes))
}
@@ -297,7 +297,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
require.NoError(t, err)
err = os.RemoveAll(config.OutputDirPath)
require.NoError(t, err)
- config.OutputDirPath, err = ioutil.TempDir("", "dumpling")
+ config.OutputDirPath, err = os.MkdirTemp("", "dumpling")
writer = createTestWriter(config, t)
@@ -322,7 +322,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
p = path.Join(config.OutputDirPath, p)
_, err = os.Stat(p)
require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, expected, string(bytes))
}
@@ -331,6 +331,9 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
var mu sync.Mutex
func createTestWriter(conf *Config, t *testing.T) *Writer {
+ t.Helper()
+ conf.ServerInfo.ServerType = version.ServerTypeMySQL
+
mu.Lock()
extStore, err := conf.createExternalStorage(context.Background())
mu.Unlock()
diff --git a/dumpling/tests/basic/run.sh b/dumpling/tests/basic/run.sh
index 11a738805275b..5b549cf857e79 100644
--- a/dumpling/tests/basic/run.sh
+++ b/dumpling/tests/basic/run.sh
@@ -114,8 +114,8 @@ echo "expected panic 0, actual ${actual}"
[ "$actual" = 0 ]
# check stdout, should contain mysql error log
-actual=$(grep -w "Error 1064: You have an error in your SQL syntax" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
-echo "expect contain Error 1064, actual ${actual}"
+actual=$(grep -w "You have an error in your SQL syntax" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
+echo "expect contain error in SQL syntax, actual ${actual}"
[ "$actual" -ge 1 ]
# Test for consistency lock with empty database.
@@ -128,3 +128,14 @@ run_dumpling --consistency lock -B "$DB_NAME" -L ${DUMPLING_OUTPUT_DIR}/dumpling
cnt=$(grep -w "$DB_NAME" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}-schema-create.sql|wc -l)
echo "records count is ${cnt}"
[ "$cnt" = 1 ]
+
+# Test for recording network usage
+run_sql "drop database if exists test_db;"
+run_sql "create database test_db;"
+run_sql "create table test_db.test_table (a int primary key);"
+run_sql "insert into test_db.test_table values (1),(2),(3),(4),(5),(6),(7),(8);"
+
+export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/SetIOTotalBytes=return(1)"
+run_dumpling -B "test_db" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log
+cnt=$(grep "IOTotalBytes=" ${DUMPLING_OUTPUT_DIR}/dumpling.log | grep -v "IOTotalBytes=0" | wc -l)
+[ "$cnt" -ge 1 ]
\ No newline at end of file
diff --git a/dumpling/tests/e2e/run.sh b/dumpling/tests/e2e/run.sh
index f5da32acc33e0..73b580ca594d9 100644
--- a/dumpling/tests/e2e/run.sh
+++ b/dumpling/tests/e2e/run.sh
@@ -37,4 +37,24 @@ run_lightning $cur/conf/lightning.toml
# check mysql and tidb data
check_sync_diff $cur/conf/diff_config.toml
+# test e2e with compress option again
+
+# drop database on tidb
+export DUMPLING_TEST_PORT=4000
+run_sql "drop database if exists $DB_NAME;"
+
+export DUMPLING_TEST_PORT=3306
+
+# dumping
+export DUMPLING_TEST_DATABASE=$DB_NAME
+rm -rf $DUMPLING_OUTPUT_DIR
+run_dumpling --compress "snappy"
+
+cat "$cur/conf/lightning.toml"
+# use lightning import data to tidb
+run_lightning $cur/conf/lightning.toml
+
+# check mysql and tidb data
+check_sync_diff $cur/conf/diff_config.toml
+
diff --git a/dumpling/tests/e2e_csv/run.sh b/dumpling/tests/e2e_csv/run.sh
index d80e321d9294a..9c5afaca469d7 100644
--- a/dumpling/tests/e2e_csv/run.sh
+++ b/dumpling/tests/e2e_csv/run.sh
@@ -24,6 +24,7 @@ run_sql_file "$DUMPLING_TEST_DIR/data/e2e_csv.t.sql"
run() {
echo "*** running subtest case ***"
+ echo "compress is $compress"
echo "escape_backslash is $escape_backslash"
echo "csv_delimiter is $csv_delimiter"
echo "csv_separator is $csv_separator"
@@ -36,7 +37,11 @@ run() {
# dumping
export DUMPLING_TEST_PORT=3306
export DUMPLING_TEST_DATABASE=$DB_NAME
- run_dumpling --filetype="csv" --escape-backslash=$escape_backslash --csv-delimiter="$csv_delimiter" --csv-separator="$csv_separator"
+ rm -rf "$DUMPLING_OUTPUT_DIR"
+ if [ $compress = "space" ]; then
+ compress=""
+ fi
+ run_dumpling --filetype="csv" --escape-backslash=$escape_backslash --csv-delimiter="$csv_delimiter" --csv-separator="$csv_separator" --compress="$compress"
# construct lightning configuration
mkdir -p $DUMPLING_TEST_DIR/conf
@@ -67,18 +72,22 @@ run() {
escape_backslash_arr="true false"
csv_delimiter_arr="\" '"
csv_separator_arr=', a aa |*|'
+compress_arr='space gzip snappy zstd'
-for escape_backslash in $escape_backslash_arr
+for compress in $compress_arr
do
- for csv_separator in $csv_separator_arr
+ for escape_backslash in $escape_backslash_arr
do
- for csv_delimiter in $csv_delimiter_arr
+ for csv_separator in $csv_separator_arr
do
- run
+ for csv_delimiter in $csv_delimiter_arr
+ do
+ run
+ done
+ if [ "$escape_backslash" = "true" ]; then
+ csv_delimiter=""
+ run
+ fi
done
- if [ "$escape_backslash" = "true" ]; then
- csv_delimiter=""
- run
- fi
done
done
diff --git a/dumpling/tests/e2e_foreign_key/conf/diff_config.toml b/dumpling/tests/e2e_foreign_key/conf/diff_config.toml
new file mode 100644
index 0000000000000..634c8416891eb
--- /dev/null
+++ b/dumpling/tests/e2e_foreign_key/conf/diff_config.toml
@@ -0,0 +1,29 @@
+# diff Configuration.
+
+check-thread-count = 4
+
+export-fix-sql = true
+
+check-struct-only = false
+
+[task]
+ output-dir = "./output"
+
+ source-instances = ["mysql1"]
+
+ target-instance = "tidb0"
+
+ target-check-tables = ["e2e_foreign_key.parent", "e2e_foreign_key.child"]
+
+[data-sources]
+[data-sources.mysql1]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = ""
+
+[data-sources.tidb0]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+password = ""
diff --git a/dumpling/tests/e2e_foreign_key/conf/lightning.toml b/dumpling/tests/e2e_foreign_key/conf/lightning.toml
new file mode 100644
index 0000000000000..e96fdaaf0daeb
--- /dev/null
+++ b/dumpling/tests/e2e_foreign_key/conf/lightning.toml
@@ -0,0 +1,20 @@
+### tidb-lightning config
+
+[lightning]
+server-mode = false
+level = "error"
+check-requirements = false
+
+[tikv-importer]
+backend = "tidb"
+on-duplicate = "error"
+
+[mydumper]
+data-source-dir = "/tmp/dumpling_test_result/sql_res.e2e_foreign_key"
+
+[tidb]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+password = ""
+status-port = 10080
diff --git a/dumpling/tests/e2e_foreign_key/data/e2e_foreign_key.sql b/dumpling/tests/e2e_foreign_key/data/e2e_foreign_key.sql
new file mode 100644
index 0000000000000..4b8ff0b1df203
--- /dev/null
+++ b/dumpling/tests/e2e_foreign_key/data/e2e_foreign_key.sql
@@ -0,0 +1,8 @@
+create database e2e_foreign_key DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+use e2e_foreign_key;
+create table parent (id int key);
+create table child (id int key, pid int, constraint fk_1 foreign key (pid) references parent(id));
+insert into parent values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
+insert into child values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10);
+set foreign_key_checks=0;
+insert into child values (100,100);
diff --git a/dumpling/tests/e2e_foreign_key/run.sh b/dumpling/tests/e2e_foreign_key/run.sh
new file mode 100644
index 0000000000000..99285d627fdf7
--- /dev/null
+++ b/dumpling/tests/e2e_foreign_key/run.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
+
+set -eu
+cur=$(cd `dirname $0`; pwd)
+
+DB_NAME="e2e_foreign_key"
+
+# drop database on tidb
+export DUMPLING_TEST_PORT=4000
+run_sql "drop database if exists $DB_NAME;"
+
+# drop database on mysql
+export DUMPLING_TEST_PORT=3306
+run_sql "drop database if exists $DB_NAME;"
+
+# build data on mysql
+run_sql_file "$DUMPLING_BASE_NAME/data/e2e_foreign_key.sql"
+
+# dumping
+export DUMPLING_TEST_DATABASE=$DB_NAME
+run_dumpling
+
+cat "$cur/conf/lightning.toml"
+# use lightning import data to tidb
+run_lightning $cur/conf/lightning.toml
+
+# check mysql and tidb data
+check_sync_diff $cur/conf/diff_config.toml
diff --git a/dumpling/tests/file_size/run.sh b/dumpling/tests/file_size/run.sh
index 6b0577dc2e514..174b1e0def008 100644
--- a/dumpling/tests/file_size/run.sh
+++ b/dumpling/tests/file_size/run.sh
@@ -14,8 +14,8 @@ chars_20="1111_0000_1111_0000_"
# insert 100 records, each occupies 20 bytes
run_sql "insert into t values $(seq -s, 100 | sed 's/,*$//g' | sed "s/[0-9]*/('$chars_20')/g");"
-# dumping with file size = 311 bytes, actually 10 rows
-run_dumpling -F 311B
+# dumping with file size = 348 bytes, actually 10 rows
+run_dumpling -F 348B
# the dumping result is expected to be:
# 10 files for insertion(each conatins 10 records / 200 bytes)
diff --git a/dumpling/tests/naughty_strings/data/naughty_strings.escape.sql b/dumpling/tests/naughty_strings/data/naughty_strings.escape.sql
index 1fa1d0d223013..d73ee94aee1a7 100644
--- a/dumpling/tests/naughty_strings/data/naughty_strings.escape.sql
+++ b/dumpling/tests/naughty_strings/data/naughty_strings.escape.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
INSERT INTO `escape` VALUES
('''', '"'),
diff --git a/dumpling/tests/naughty_strings/data/naughty_strings.t.sql b/dumpling/tests/naughty_strings/data/naughty_strings.t.sql
index fa5d74f15f934..88128d75dfd4c 100644
--- a/dumpling/tests/naughty_strings/data/naughty_strings.t.sql
+++ b/dumpling/tests/naughty_strings/data/naughty_strings.t.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
INSERT INTO `t` VALUES
(''),
diff --git a/dumpling/tests/naughty_strings/expect/naughty_strings.escape.sql b/dumpling/tests/naughty_strings/expect/naughty_strings.escape.sql
index bc578c659f4c4..607aa53e0acef 100644
--- a/dumpling/tests/naughty_strings/expect/naughty_strings.escape.sql
+++ b/dumpling/tests/naughty_strings/expect/naughty_strings.escape.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
INSERT INTO `escape` VALUES
('\'','\"'),
diff --git a/dumpling/tests/naughty_strings/expect/naughty_strings.t.sql b/dumpling/tests/naughty_strings/expect/naughty_strings.t.sql
index 1dd0c5ae209a3..5f3468aeb108c 100644
--- a/dumpling/tests/naughty_strings/expect/naughty_strings.t.sql
+++ b/dumpling/tests/naughty_strings/expect/naughty_strings.t.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
INSERT INTO `t` VALUES
(''),
diff --git a/dumpling/tests/non_exist_tables/run.sh b/dumpling/tests/non_exist_tables/run.sh
index 78003069cce9d..180db04407f3a 100644
--- a/dumpling/tests/non_exist_tables/run.sh
+++ b/dumpling/tests/non_exist_tables/run.sh
@@ -21,6 +21,6 @@ rm -rf "$DUMPLING_OUTPUT_DIR"
run_dumpling --consistency=lock --tables-list "$DB_NAME.$TABLE_NAME2" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log
set -e
-actual=$(grep -w "Error 1146: Table 'non_exist_tables.t2' doesn't exist" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
+actual=$(grep -w "Table 'non_exist_tables.t2' doesn't exist" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
echo "expected 1 return error when specifying --tables-list with non-existing tables, actual ${actual}"
[ $actual = 1 ]
diff --git a/dumpling/tests/placement_policy/result/x-placement-policy-create.sql b/dumpling/tests/placement_policy/result/x-placement-policy-create.sql
index 8a41e6c2a3522..9a7dac22244b0 100644
--- a/dumpling/tests/placement_policy/result/x-placement-policy-create.sql
+++ b/dumpling/tests/placement_policy/result/x-placement-policy-create.sql
@@ -1,2 +1,3 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
/*T![placement] CREATE PLACEMENT POLICY `x` PRIMARY_REGION="cn-east-1" REGIONS="cn-east-1,cn-east" */;
diff --git a/dumpling/tests/placement_policy/result/x1-placement-policy-create.sql b/dumpling/tests/placement_policy/result/x1-placement-policy-create.sql
index d5e7f8ad8dda5..ebed679154ef5 100644
--- a/dumpling/tests/placement_policy/result/x1-placement-policy-create.sql
+++ b/dumpling/tests/placement_policy/result/x1-placement-policy-create.sql
@@ -1,2 +1,3 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
/*T![placement] CREATE PLACEMENT POLICY `x1` FOLLOWERS=4 */;
diff --git a/dumpling/tests/primary_key/result/pk_case_0.sql b/dumpling/tests/primary_key/result/pk_case_0.sql
index 6c8c7fb9e6538..0168395142ef6 100644
--- a/dumpling/tests/primary_key/result/pk_case_0.sql
+++ b/dumpling/tests/primary_key/result/pk_case_0.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
INSERT INTO `pk_case_0` VALUES
(0,10),
diff --git a/dumpling/tests/primary_key/result/pk_case_1.sql b/dumpling/tests/primary_key/result/pk_case_1.sql
index 4a671cc64f247..741c36cd64e4a 100644
--- a/dumpling/tests/primary_key/result/pk_case_1.sql
+++ b/dumpling/tests/primary_key/result/pk_case_1.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
INSERT INTO `pk_case_1` VALUES
(0,10),
diff --git a/dumpling/tests/primary_key/result/pk_case_2.sql b/dumpling/tests/primary_key/result/pk_case_2.sql
index af9494d00645d..0605e11e5572c 100644
--- a/dumpling/tests/primary_key/result/pk_case_2.sql
+++ b/dumpling/tests/primary_key/result/pk_case_2.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
INSERT INTO `pk_case_2` VALUES
(0,10),
diff --git a/dumpling/tests/primary_key/result/pk_case_3.sql b/dumpling/tests/primary_key/result/pk_case_3.sql
index a29e568dd1340..2da84a1a1b91c 100644
--- a/dumpling/tests/primary_key/result/pk_case_3.sql
+++ b/dumpling/tests/primary_key/result/pk_case_3.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
INSERT INTO `pk_case_3` VALUES
(6,4,x'000000000101000000000000000000f03f000000000000f03f'),
diff --git a/dumpling/tests/quote/data/quote-database-schema-create-mysql57.sql b/dumpling/tests/quote/data/quote-database-schema-create-mysql57.sql
index a5df1c26b6c8b..8cc30bc18ca25 100755
--- a/dumpling/tests/quote/data/quote-database-schema-create-mysql57.sql
+++ b/dumpling/tests/quote/data/quote-database-schema-create-mysql57.sql
@@ -1,2 +1,3 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
CREATE DATABASE `quo``te/database` /*!40100 DEFAULT CHARACTER SET latin1 */;
diff --git a/dumpling/tests/quote/data/quote-database-schema-create.sql b/dumpling/tests/quote/data/quote-database-schema-create.sql
index d895b4678b912..4a5bfe7c297fc 100755
--- a/dumpling/tests/quote/data/quote-database-schema-create.sql
+++ b/dumpling/tests/quote/data/quote-database-schema-create.sql
@@ -1,2 +1,3 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
CREATE DATABASE `quo``te/database` /*!40100 DEFAULT CHARACTER SET latin1 */ /*!80016 DEFAULT ENCRYPTION='N' */;
diff --git a/dumpling/tests/quote/data/quote-database.quote-table-schema-mysql57.sql b/dumpling/tests/quote/data/quote-database.quote-table-schema-mysql57.sql
index b3c55dee26330..4b0c6e85c30eb 100755
--- a/dumpling/tests/quote/data/quote-database.quote-table-schema-mysql57.sql
+++ b/dumpling/tests/quote/data/quote-database.quote-table-schema-mysql57.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
CREATE TABLE `quo``te/table` (
`quo``te/col` int(11) NOT NULL,
diff --git a/dumpling/tests/quote/data/quote-database.quote-table-schema.sql b/dumpling/tests/quote/data/quote-database.quote-table-schema.sql
index 61bfcc50d113f..cf357c7603836 100755
--- a/dumpling/tests/quote/data/quote-database.quote-table-schema.sql
+++ b/dumpling/tests/quote/data/quote-database.quote-table-schema.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
CREATE TABLE `quo``te/table` (
`quo``te/col` int NOT NULL,
diff --git a/dumpling/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql b/dumpling/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql
index 5cee6b7b4a67d..3f36d8fc8a099 100755
--- a/dumpling/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql
+++ b/dumpling/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
INSERT INTO `quo``te/table` (`quo``te/col`,`a`) VALUES
(0,10),
diff --git a/dumpling/tests/quote/data/quote-database.quote-table.000000000.sql b/dumpling/tests/quote/data/quote-database.quote-table.000000000.sql
index 5cee6b7b4a67d..3f36d8fc8a099 100755
--- a/dumpling/tests/quote/data/quote-database.quote-table.000000000.sql
+++ b/dumpling/tests/quote/data/quote-database.quote-table.000000000.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
INSERT INTO `quo``te/table` (`quo``te/col`,`a`) VALUES
(0,10),
diff --git a/dumpling/tests/rows/run.sh b/dumpling/tests/rows/run.sh
index 84175b025b9b2..41124fd267a0d 100644
--- a/dumpling/tests/rows/run.sh
+++ b/dumpling/tests/rows/run.sh
@@ -46,7 +46,7 @@ check_sync_diff $cur/conf/diff_config.toml
# test dumpling with both rows and filesize
rm -rf "$DUMPLING_OUTPUT_DIR"
-run_dumpling --rows 10 --filesize 100B --loglevel debug
+run_dumpling --rows 10 --filesize 140B --loglevel debug
# the dumping result is expected to be:
# 50 files for insertion
file_num=$(find "$DUMPLING_OUTPUT_DIR" -maxdepth 1 -iname "$DB_NAME.$TABLE_NAME.*.sql" | wc -l)
diff --git a/dumpling/tests/sequences/data/sequences.s-schema-sequence-expect.sql b/dumpling/tests/sequences/data/sequences.s-schema-sequence-expect.sql
index 0fb6daa6fc5ad..0f33902517ff1 100644
--- a/dumpling/tests/sequences/data/sequences.s-schema-sequence-expect.sql
+++ b/dumpling/tests/sequences/data/sequences.s-schema-sequence-expect.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
CREATE SEQUENCE `s` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=InnoDB;
SELECT SETVAL(`s`,1001);
diff --git a/dumpling/tests/views/data/views-schema-create.sql b/dumpling/tests/views/data/views-schema-create.sql
index af5ca6d166d0e..88ce12d5c3ccd 100644
--- a/dumpling/tests/views/data/views-schema-create.sql
+++ b/dumpling/tests/views/data/views-schema-create.sql
@@ -1,2 +1,3 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
CREATE DATABASE `views` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */ /*!80016 DEFAULT ENCRYPTION='N' */;
diff --git a/dumpling/tests/views/data/views.v-schema-view.sql b/dumpling/tests/views/data/views.v-schema-view.sql
index 06d73e73fe664..3063ec02815cd 100644
--- a/dumpling/tests/views/data/views.v-schema-view.sql
+++ b/dumpling/tests/views/data/views.v-schema-view.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
DROP TABLE IF EXISTS `v`;
DROP VIEW IF EXISTS `v`;
diff --git a/dumpling/tests/views/data/views.v-schema.sql b/dumpling/tests/views/data/views.v-schema.sql
index 8e6bf5d441640..0d2bddbdffc13 100644
--- a/dumpling/tests/views/data/views.v-schema.sql
+++ b/dumpling/tests/views/data/views.v-schema.sql
@@ -1,3 +1,4 @@
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
/*!40101 SET NAMES binary*/;
CREATE TABLE `v`(
`a` int,
diff --git a/errno/errcode.go b/errno/errcode.go
index 60257be0b76c9..0501bc0020e07 100644
--- a/errno/errcode.go
+++ b/errno/errcode.go
@@ -825,6 +825,7 @@ const (
ErrMaxExecTimeExceeded = 1907
ErrForeignKeyCascadeDepthExceeded = 3008
ErrInvalidFieldSize = 3013
+ ErrPasswordExpireAnonymousUser = 3016
ErrInvalidArgumentForLogarithm = 3020
ErrAggregateOrderNonAggQuery = 3029
ErrUserLockWrongName = 3057
@@ -892,6 +893,7 @@ const (
ErrIllegalPrivilegeLevel = 3619
ErrCTEMaxRecursionDepth = 3636
ErrNotHintUpdatable = 3637
+ ErrExistsInHistoryPassword = 3638
ErrForeignKeyCannotDropParent = 3730
ErrForeignKeyCannotUseVirtualColumn = 3733
ErrForeignKeyNoColumnInParent = 3734
@@ -913,11 +915,14 @@ const (
ErrFunctionalIndexRowValueIsNotAllowed = 3800
ErrDependentByFunctionalIndex = 3837
ErrCannotConvertString = 3854
+ ErrDependentByPartitionFunctional = 3855
ErrInvalidJSONValueForFuncIndex = 3903
ErrJSONValueOutOfRangeForFuncIndex = 3904
ErrFunctionalIndexDataIsTooLong = 3907
ErrFunctionalIndexNotApplicable = 3909
ErrDynamicPrivilegeNotRegistered = 3929
+ ErUserAccessDeniedForUserAccountBlockedByPasswordLock = 3955
+ ErrJSONInBooleanContext = 3986
ErrTableWithoutPrimaryKey = 3750
// MariaDB errors.
ErrOnlyOneDefaultPartionAllowed = 4030
@@ -1034,6 +1039,12 @@ const (
ErrGettingNoopVariable = 8145
ErrCannotMigrateSession = 8146
ErrLazyUniquenessCheckFailure = 8147
+ ErrUnsupportedColumnInTTLConfig = 8148
+ ErrTTLColumnCannotDrop = 8149
+ ErrSetTTLOptionForNonTTLTable = 8150
+ ErrTempTableNotAllowedWithTTL = 8151
+ ErrUnsupportedTTLReferencedByFK = 8152
+ ErrUnsupportedPrimaryKeyTypeWithTTL = 8153
// Error codes used by TiDB ddl package
ErrUnsupportedDDLOperation = 8200
@@ -1082,6 +1093,10 @@ const (
ErrPartitionColumnStatsMissing = 8244
ErrColumnInChange = 8245
ErrDDLSetting = 8246
+ ErrIngestFailed = 8247
+ ErrResourceGroupExists = 8248
+ ErrResourceGroupNotExists = 8249
+ ErrResourceGroupSupportDisabled = 8250
// TiKV/PD/TiFlash errors.
ErrPDServerTimeout = 9001
diff --git a/errno/errname.go b/errno/errname.go
index 8f5fd8542bd1a..0f6c3db0e1645 100644
--- a/errno/errname.go
+++ b/errno/errname.go
@@ -92,7 +92,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrMultiplePriKey: mysql.Message("Multiple primary key defined", nil),
ErrTooManyKeys: mysql.Message("Too many keys specified; max %d keys allowed", nil),
ErrTooManyKeyParts: mysql.Message("Too many key parts specified; max %d parts allowed", nil),
- ErrTooLongKey: mysql.Message("Specified key was too long; max key length is %d bytes", nil),
+ ErrTooLongKey: mysql.Message("Specified key was too long (%d bytes); max key length is %d bytes", nil),
ErrKeyColumnDoesNotExits: mysql.Message("Key column '%-.192s' doesn't exist in table", nil),
ErrBlobUsedAsKey: mysql.Message("BLOB column '%-.192s' can't be used in key specification with the used table type", nil),
ErrTooBigFieldlength: mysql.Message("Column length too big for column '%-.192s' (max = %d); use BLOB or TEXT instead", nil),
@@ -783,7 +783,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrInnodbIndexCorrupt: mysql.Message("Index corrupt: %s", nil),
ErrInvalidYearColumnLength: mysql.Message("Supports only YEAR or YEAR(4) column", nil),
ErrNotValidPassword: mysql.Message("Your password does not satisfy the current policy requirements", nil),
- ErrMustChangePassword: mysql.Message("You must SET PASSWORD before executing this statement", nil),
+ ErrMustChangePassword: mysql.Message("You must reset your password using ALTER USER statement before executing this statement", nil),
ErrFkNoIndexChild: mysql.Message("Failed to add the foreign key constraint. Missing index for constraint '%s' in the foreign table '%s'", nil),
ErrForeignKeyNoIndexInParent: mysql.Message("Failed to add the foreign key constraint. Missing index for constraint '%s' in the referenced table '%s'", nil),
ErrFkFailAddSystem: mysql.Message("Failed to add the foreign key constraint '%s' to system tables", nil),
@@ -836,10 +836,12 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrDependentByGeneratedColumn: mysql.Message("Column '%s' has a generated column dependency.", nil),
ErrGeneratedColumnRefAutoInc: mysql.Message("Generated column '%s' cannot refer to auto-increment column.", nil),
ErrAccountHasBeenLocked: mysql.Message("Access denied for user '%s'@'%s'. Account is locked.", nil),
+ ErUserAccessDeniedForUserAccountBlockedByPasswordLock: mysql.Message("Access denied for user '%s'@'%s'. Account is blocked for %s day(s) (%s day(s) remaining) due to %d consecutive failed logins.", nil),
ErrWarnConflictingHint: mysql.Message("Hint %s is ignored as conflicting/duplicated.", nil),
ErrUnresolvedHintName: mysql.Message("Unresolved name '%s' for %s hint", nil),
ErrForeignKeyCascadeDepthExceeded: mysql.Message("Foreign key cascade delete/update exceeds max depth of %v.", nil),
ErrInvalidFieldSize: mysql.Message("Invalid size for column '%s'.", nil),
+ ErrPasswordExpireAnonymousUser: mysql.Message("The password for anonymous user cannot be expired.", nil),
ErrInvalidArgumentForLogarithm: mysql.Message("Invalid argument for logarithm", nil),
ErrAggregateOrderNonAggQuery: mysql.Message("Expression #%d of ORDER BY contains aggregate function and applies to the result of a non-aggregated query", nil),
ErrIncorrectType: mysql.Message("Incorrect type for argument %s in function %s.", nil),
@@ -887,6 +889,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrMaxExecTimeExceeded: mysql.Message("Query execution was interrupted, max_execution_time exceeded.", nil),
ErrLockAcquireFailAndNoWaitSet: mysql.Message("Statement aborted because lock(s) could not be acquired immediately and NOWAIT is set.", nil),
ErrNotHintUpdatable: mysql.Message("Variable '%s' cannot be set using SET_VAR hint.", nil),
+ ErrExistsInHistoryPassword: mysql.Message("Cannot use these credentials for '%s@%s' because they contradict the password history policy.", nil),
ErrForeignKeyCannotDropParent: mysql.Message("Cannot drop table '%s' referenced by a foreign key constraint '%s' on table '%s'.", nil),
ErrForeignKeyCannotUseVirtualColumn: mysql.Message("Foreign key '%s' uses virtual column '%s' which is not supported.", nil),
ErrForeignKeyNoColumnInParent: mysql.Message("Failed to add the foreign key constraint. Missing column '%s' for constraint '%s' in the referenced table '%s'", nil),
@@ -905,6 +908,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrFKIncompatibleColumns: mysql.Message("Referencing column '%s' and referenced column '%s' in foreign key constraint '%s' are incompatible.", nil),
ErrFunctionalIndexRowValueIsNotAllowed: mysql.Message("Expression of expression index '%s' cannot refer to a row value", nil),
ErrDependentByFunctionalIndex: mysql.Message("Column '%s' has an expression index dependency and cannot be dropped or renamed", nil),
+ ErrDependentByPartitionFunctional: mysql.Message("Column '%s' has a partitioning function dependency and cannot be dropped or renamed", nil),
ErrCannotConvertString: mysql.Message("Cannot convert string '%.64s' from %s to %s", nil),
ErrInvalidJSONValueForFuncIndex: mysql.Message("Invalid JSON value for CAST for expression index '%s'", nil),
ErrJSONValueOutOfRangeForFuncIndex: mysql.Message("Out of range JSON value for CAST for expression index '%s'", nil),
@@ -920,6 +924,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrInvalidRequiresSingleReference: mysql.Message("In recursive query block of Recursive Common Table Expression '%s', the recursive table must be referenced only once, and not in any subquery", nil),
ErrCTEMaxRecursionDepth: mysql.Message("Recursive query aborted after %d iterations. Try increasing @@cte_max_recursion_depth to a larger value", nil),
ErrTableWithoutPrimaryKey: mysql.Message("Unable to create or change a table without a primary key, when the system variable 'sql_require_primary_key' is set. Add a primary key to the table or unset this variable to avoid this message. Note that tables without a primary key can cause performance problems in row-based replication, so please consult your DBA before changing this setting.", nil),
+ ErrJSONInBooleanContext: mysql.Message("Evaluating a JSON value in SQL boolean context does an implicit comparison against JSON integer 0; if this is not what you want, consider converting JSON to a SQL numeric type with JSON_VALUE RETURNING", nil),
// MariaDB errors.
ErrOnlyOneDefaultPartionAllowed: mysql.Message("Only one DEFAULT partition allowed", nil),
ErrWrongPartitionTypeExpectedSystemTime: mysql.Message("Wrong partitioning type, expected type: `SYSTEM_TIME`", nil),
@@ -933,102 +938,108 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrSequenceInvalidTableStructure: mysql.Message("Sequence '%-.64s.%-.64s' table structure is invalid (%s)", nil),
// TiDB errors.
- ErrMemExceedThreshold: mysql.Message("%s holds %dB memory, exceeds threshold %dB.%s", nil),
- ErrForUpdateCantRetry: mysql.Message("[%d] can not retry select for update statement", nil),
- ErrAdminCheckTable: mysql.Message("TiDB admin check table failed.", nil),
- ErrOptOnTemporaryTable: mysql.Message("`%s` is unsupported on temporary tables.", nil),
- ErrDropTableOnTemporaryTable: mysql.Message("`drop global temporary table` can only drop global temporary table", nil),
- ErrTxnTooLarge: mysql.Message("Transaction is too large, size: %d", nil),
- ErrWriteConflictInTiDB: mysql.Message("Write conflict, txnStartTS %d is stale", nil),
- ErrInvalidPluginID: mysql.Message("Wrong plugin id: %s, valid plugin id is [name]-[version], both name and version should not contain '-'", nil),
- ErrInvalidPluginManifest: mysql.Message("Cannot read plugin %s's manifest", nil),
- ErrInvalidPluginName: mysql.Message("Plugin load with %s but got wrong name %s", nil),
- ErrInvalidPluginVersion: mysql.Message("Plugin load with %s but got %s", nil),
- ErrDuplicatePlugin: mysql.Message("Plugin [%s] is redeclared", nil),
- ErrInvalidPluginSysVarName: mysql.Message("Plugin %s's sysVar %s must start with its plugin name %s", nil),
- ErrRequireVersionCheckFail: mysql.Message("Plugin %s require %s be %v but got %v", nil),
- ErrUnsupportedReloadPlugin: mysql.Message("Plugin %s isn't loaded so cannot be reloaded", nil),
- ErrUnsupportedReloadPluginVar: mysql.Message("Reload plugin with different sysVar is unsupported %v", nil),
- ErrTableLocked: mysql.Message("Table '%s' was locked in %s by %v", nil),
- ErrNotExist: mysql.Message("Error: key not exist", nil),
- ErrTxnRetryable: mysql.Message("Error: KV error safe to retry %s ", []int{0}),
- ErrCannotSetNilValue: mysql.Message("can not set nil value", nil),
- ErrInvalidTxn: mysql.Message("invalid transaction", nil),
- ErrEntryTooLarge: mysql.Message("entry too large, the max entry size is %d, the size of data is %d", nil),
- ErrNotImplemented: mysql.Message("not implemented", nil),
- ErrInfoSchemaExpired: mysql.Message("Information schema is out of date: schema failed to update in 1 lease, please make sure TiDB can connect to TiKV", nil),
- ErrInfoSchemaChanged: mysql.Message("Information schema is changed during the execution of the statement(for example, table definition may be updated by other DDL ran in parallel). If you see this error often, try increasing `tidb_max_delta_schema_count`", nil),
- ErrBadNumber: mysql.Message("Bad Number", nil),
- ErrCastAsSignedOverflow: mysql.Message("Cast to signed converted positive out-of-range integer to it's negative complement", nil),
- ErrCastNegIntAsUnsigned: mysql.Message("Cast to unsigned converted negative integer to it's positive complement", nil),
- ErrInvalidYearFormat: mysql.Message("invalid year format", nil),
- ErrInvalidYear: mysql.Message("invalid year", nil),
- ErrIncorrectDatetimeValue: mysql.Message("Incorrect datetime value: '%s'", []int{0}),
- ErrInvalidTimeFormat: mysql.Message("invalid time format: '%v'", []int{0}),
- ErrInvalidWeekModeFormat: mysql.Message("invalid week mode format: '%v'", nil),
- ErrFieldGetDefaultFailed: mysql.Message("Field '%s' get default value fail", nil),
- ErrIndexOutBound: mysql.Message("Index column %s offset out of bound, offset: %d, row: %v", []int{2}),
- ErrUnsupportedOp: mysql.Message("operation not supported", nil),
- ErrRowNotFound: mysql.Message("can not find the row: %s", []int{0}),
- ErrTableStateCantNone: mysql.Message("table %s can't be in none state", nil),
- ErrColumnStateCantNone: mysql.Message("column %s can't be in none state", nil),
- ErrColumnStateNonPublic: mysql.Message("can not use non-public column", nil),
- ErrIndexStateCantNone: mysql.Message("index %s can't be in none state", nil),
- ErrInvalidRecordKey: mysql.Message("invalid record key", nil),
- ErrUnsupportedValueForVar: mysql.Message("variable '%s' does not yet support value: %s", nil),
- ErrUnsupportedIsolationLevel: mysql.Message("The isolation level '%s' is not supported. Set tidb_skip_isolation_level_check=1 to skip this error", nil),
- ErrInvalidDDLWorker: mysql.Message("Invalid DDL worker", nil),
- ErrUnsupportedDDLOperation: mysql.Message("Unsupported %s", nil),
- ErrNotOwner: mysql.Message("TiDB server is not a DDL owner", nil),
- ErrCantDecodeRecord: mysql.Message("Cannot decode %s value, because %v", nil),
- ErrInvalidDDLJob: mysql.Message("Invalid DDL job", nil),
- ErrInvalidDDLJobFlag: mysql.Message("Invalid DDL job flag", nil),
- ErrWaitReorgTimeout: mysql.Message("Timeout waiting for data reorganization", nil),
- ErrInvalidStoreVersion: mysql.Message("Invalid storage current version: %d", nil),
- ErrUnknownTypeLength: mysql.Message("Unknown length for type %d", nil),
- ErrUnknownFractionLength: mysql.Message("Unknown length for type %d and fraction %d", nil),
- ErrInvalidDDLJobVersion: mysql.Message("Version %d of DDL job is greater than current one: %d", nil),
- ErrInvalidSplitRegionRanges: mysql.Message("Failed to split region ranges: %s", nil),
- ErrReorgPanic: mysql.Message("Reorg worker panic", nil),
- ErrInvalidDDLState: mysql.Message("Invalid %s state: %v", nil),
- ErrCancelledDDLJob: mysql.Message("Cancelled DDL job", nil),
- ErrRepairTable: mysql.Message("Failed to repair table: %s", nil),
- ErrLoadPrivilege: mysql.Message("Load privilege table fail: %s", nil),
- ErrInvalidPrivilegeType: mysql.Message("unknown privilege type %s", nil),
- ErrUnknownFieldType: mysql.Message("unknown field type", nil),
- ErrInvalidSequence: mysql.Message("invalid sequence", nil),
- ErrInvalidType: mysql.Message("invalid type", nil),
- ErrCantGetValidID: mysql.Message("Cannot get a valid auto-ID when retrying the statement", nil),
- ErrCantSetToNull: mysql.Message("cannot set variable to null", nil),
- ErrSnapshotTooOld: mysql.Message("snapshot is older than GC safe point %s", nil),
- ErrInvalidTableID: mysql.Message("invalid TableID", nil),
- ErrInvalidAutoRandom: mysql.Message("Invalid auto random: %s", nil),
- ErrInvalidHashKeyFlag: mysql.Message("invalid encoded hash key flag", nil),
- ErrInvalidListIndex: mysql.Message("invalid list index", nil),
- ErrInvalidListMetaData: mysql.Message("invalid list meta data", nil),
- ErrWriteOnSnapshot: mysql.Message("write on snapshot", nil),
- ErrInvalidKey: mysql.Message("invalid key", nil),
- ErrInvalidIndexKey: mysql.Message("invalid index key", nil),
- ErrDataInconsistent: mysql.Message("data inconsistency in table: %s, index: %s, handle: %s, index-values:%#v != record-values:%#v", []int{2, 3, 4}),
- ErrDDLReorgElementNotExist: mysql.Message("DDL reorg element does not exist", nil),
- ErrDDLJobNotFound: mysql.Message("DDL Job:%v not found", nil),
- ErrCancelFinishedDDLJob: mysql.Message("This job:%v is finished, so can't be cancelled", nil),
- ErrCannotCancelDDLJob: mysql.Message("This job:%v is almost finished, can't be cancelled now", nil),
- ErrUnknownAllocatorType: mysql.Message("Invalid allocator type", nil),
- ErrAutoRandReadFailed: mysql.Message("Failed to read auto-random value from storage engine", nil),
- ErrInvalidIncrementAndOffset: mysql.Message("Invalid auto_increment settings: auto_increment_increment: %d, auto_increment_offset: %d, both of them must be in range [1..65535]", nil),
- ErrDataInconsistentMismatchCount: mysql.Message("data inconsistency in table: %s, index: %s, index-count:%d != record-count:%d", nil),
- ErrDataInconsistentMismatchIndex: mysql.Message("data inconsistency in table: %s, index: %s, col: %s, handle: %#v, index-values:%#v != record-values:%#v, compare err:%#v", []int{3, 4, 5, 6}),
- ErrInconsistentRowValue: mysql.Message("writing inconsistent data in table: %s, expected-values:{%s} != record-values:{%s}", []int{1, 2}),
- ErrInconsistentHandle: mysql.Message("writing inconsistent data in table: %s, index: %s, index-handle:%#v != record-handle:%#v, index: %#v, record: %#v", []int{2, 3, 4, 5}),
- ErrInconsistentIndexedValue: mysql.Message("writing inconsistent data in table: %s, index: %s, col: %s, indexed-value:{%s} != record-value:{%s}", []int{3, 4}),
- ErrAssertionFailed: mysql.Message("assertion failed: key: %s, assertion: %s, start_ts: %v, existing start ts: %v, existing commit ts: %v", []int{0}),
- ErrInstanceScope: mysql.Message("modifying %s will require SET GLOBAL in a future version of TiDB", nil),
- ErrNonTransactionalJobFailure: mysql.Message("non-transactional job failed, job id: %d, total jobs: %d. job range: [%s, %s], job sql: %s, err: %v", []int{2, 3, 4}),
- ErrSettingNoopVariable: mysql.Message("setting %s has no effect in TiDB", nil),
- ErrGettingNoopVariable: mysql.Message("variable %s has no effect in TiDB", nil),
- ErrCannotMigrateSession: mysql.Message("cannot migrate the current session: %s", nil),
- ErrLazyUniquenessCheckFailure: mysql.Message("transaction aborted because lazy uniqueness check is enabled and an error occurred: %s", nil),
+ ErrMemExceedThreshold: mysql.Message("%s holds %dB memory, exceeds threshold %dB.%s", nil),
+ ErrForUpdateCantRetry: mysql.Message("[%d] can not retry select for update statement", nil),
+ ErrAdminCheckTable: mysql.Message("TiDB admin check table failed.", nil),
+ ErrOptOnTemporaryTable: mysql.Message("`%s` is unsupported on temporary tables.", nil),
+ ErrDropTableOnTemporaryTable: mysql.Message("`drop global temporary table` can only drop global temporary table", nil),
+ ErrTxnTooLarge: mysql.Message("Transaction is too large, size: %d", nil),
+ ErrWriteConflictInTiDB: mysql.Message("Write conflict, txnStartTS %d is stale", nil),
+ ErrInvalidPluginID: mysql.Message("Wrong plugin id: %s, valid plugin id is [name]-[version], both name and version should not contain '-'", nil),
+ ErrInvalidPluginManifest: mysql.Message("Cannot read plugin %s's manifest", nil),
+ ErrInvalidPluginName: mysql.Message("Plugin load with %s but got wrong name %s", nil),
+ ErrInvalidPluginVersion: mysql.Message("Plugin load with %s but got %s", nil),
+ ErrDuplicatePlugin: mysql.Message("Plugin [%s] is redeclared", nil),
+ ErrInvalidPluginSysVarName: mysql.Message("Plugin %s's sysVar %s must start with its plugin name %s", nil),
+ ErrRequireVersionCheckFail: mysql.Message("Plugin %s require %s be %v but got %v", nil),
+ ErrUnsupportedReloadPlugin: mysql.Message("Plugin %s isn't loaded so cannot be reloaded", nil),
+ ErrUnsupportedReloadPluginVar: mysql.Message("Reload plugin with different sysVar is unsupported %v", nil),
+ ErrTableLocked: mysql.Message("Table '%s' was locked in %s by %v", nil),
+ ErrNotExist: mysql.Message("Error: key not exist", nil),
+ ErrTxnRetryable: mysql.Message("Error: KV error safe to retry %s ", []int{0}),
+ ErrCannotSetNilValue: mysql.Message("can not set nil value", nil),
+ ErrInvalidTxn: mysql.Message("invalid transaction", nil),
+ ErrEntryTooLarge: mysql.Message("entry too large, the max entry size is %d, the size of data is %d", nil),
+ ErrNotImplemented: mysql.Message("not implemented", nil),
+ ErrInfoSchemaExpired: mysql.Message("Information schema is out of date: schema failed to update in 1 lease, please make sure TiDB can connect to TiKV", nil),
+ ErrInfoSchemaChanged: mysql.Message("Information schema is changed during the execution of the statement(for example, table definition may be updated by other DDL ran in parallel). If you see this error often, try increasing `tidb_max_delta_schema_count`", nil),
+ ErrBadNumber: mysql.Message("Bad Number", nil),
+ ErrCastAsSignedOverflow: mysql.Message("Cast to signed converted positive out-of-range integer to it's negative complement", nil),
+ ErrCastNegIntAsUnsigned: mysql.Message("Cast to unsigned converted negative integer to it's positive complement", nil),
+ ErrInvalidYearFormat: mysql.Message("invalid year format", nil),
+ ErrInvalidYear: mysql.Message("invalid year", nil),
+ ErrIncorrectDatetimeValue: mysql.Message("Incorrect datetime value: '%s'", []int{0}),
+ ErrInvalidTimeFormat: mysql.Message("invalid time format: '%v'", []int{0}),
+ ErrInvalidWeekModeFormat: mysql.Message("invalid week mode format: '%v'", nil),
+ ErrFieldGetDefaultFailed: mysql.Message("Field '%s' get default value fail", nil),
+ ErrIndexOutBound: mysql.Message("Index column %s offset out of bound, offset: %d, row: %v", []int{2}),
+ ErrUnsupportedOp: mysql.Message("operation not supported", nil),
+ ErrRowNotFound: mysql.Message("can not find the row: %s", []int{0}),
+ ErrTableStateCantNone: mysql.Message("table %s can't be in none state", nil),
+ ErrColumnStateCantNone: mysql.Message("column %s can't be in none state", nil),
+ ErrColumnStateNonPublic: mysql.Message("can not use non-public column", nil),
+ ErrIndexStateCantNone: mysql.Message("index %s can't be in none state", nil),
+ ErrInvalidRecordKey: mysql.Message("invalid record key", nil),
+ ErrUnsupportedValueForVar: mysql.Message("variable '%s' does not yet support value: %s", nil),
+ ErrUnsupportedIsolationLevel: mysql.Message("The isolation level '%s' is not supported. Set tidb_skip_isolation_level_check=1 to skip this error", nil),
+ ErrInvalidDDLWorker: mysql.Message("Invalid DDL worker", nil),
+ ErrUnsupportedDDLOperation: mysql.Message("Unsupported %s", nil),
+ ErrNotOwner: mysql.Message("TiDB server is not a DDL owner", nil),
+ ErrCantDecodeRecord: mysql.Message("Cannot decode %s value, because %v", nil),
+ ErrInvalidDDLJob: mysql.Message("Invalid DDL job", nil),
+ ErrInvalidDDLJobFlag: mysql.Message("Invalid DDL job flag", nil),
+ ErrWaitReorgTimeout: mysql.Message("Timeout waiting for data reorganization", nil),
+ ErrInvalidStoreVersion: mysql.Message("Invalid storage current version: %d", nil),
+ ErrUnknownTypeLength: mysql.Message("Unknown length for type %d", nil),
+ ErrUnknownFractionLength: mysql.Message("Unknown length for type %d and fraction %d", nil),
+ ErrInvalidDDLJobVersion: mysql.Message("Version %d of DDL job is greater than current one: %d", nil),
+ ErrInvalidSplitRegionRanges: mysql.Message("Failed to split region ranges: %s", nil),
+ ErrReorgPanic: mysql.Message("Reorg worker panic", nil),
+ ErrInvalidDDLState: mysql.Message("Invalid %s state: %v", nil),
+ ErrCancelledDDLJob: mysql.Message("Cancelled DDL job", nil),
+ ErrRepairTable: mysql.Message("Failed to repair table: %s", nil),
+ ErrLoadPrivilege: mysql.Message("Load privilege table fail: %s", nil),
+ ErrInvalidPrivilegeType: mysql.Message("unknown privilege type %s", nil),
+ ErrUnknownFieldType: mysql.Message("unknown field type", nil),
+ ErrInvalidSequence: mysql.Message("invalid sequence", nil),
+ ErrInvalidType: mysql.Message("invalid type", nil),
+ ErrCantGetValidID: mysql.Message("Cannot get a valid auto-ID when retrying the statement", nil),
+ ErrCantSetToNull: mysql.Message("cannot set variable to null", nil),
+ ErrSnapshotTooOld: mysql.Message("snapshot is older than GC safe point %s", nil),
+ ErrInvalidTableID: mysql.Message("invalid TableID", nil),
+ ErrInvalidAutoRandom: mysql.Message("Invalid auto random: %s", nil),
+ ErrInvalidHashKeyFlag: mysql.Message("invalid encoded hash key flag", nil),
+ ErrInvalidListIndex: mysql.Message("invalid list index", nil),
+ ErrInvalidListMetaData: mysql.Message("invalid list meta data", nil),
+ ErrWriteOnSnapshot: mysql.Message("write on snapshot", nil),
+ ErrInvalidKey: mysql.Message("invalid key", nil),
+ ErrInvalidIndexKey: mysql.Message("invalid index key", nil),
+ ErrDataInconsistent: mysql.Message("data inconsistency in table: %s, index: %s, handle: %s, index-values:%#v != record-values:%#v", []int{2, 3, 4}),
+ ErrDDLReorgElementNotExist: mysql.Message("DDL reorg element does not exist", nil),
+ ErrDDLJobNotFound: mysql.Message("DDL Job:%v not found", nil),
+ ErrCancelFinishedDDLJob: mysql.Message("This job:%v is finished, so can't be cancelled", nil),
+ ErrCannotCancelDDLJob: mysql.Message("This job:%v is almost finished, can't be cancelled now", nil),
+ ErrUnknownAllocatorType: mysql.Message("Invalid allocator type", nil),
+ ErrAutoRandReadFailed: mysql.Message("Failed to read auto-random value from storage engine", nil),
+ ErrInvalidIncrementAndOffset: mysql.Message("Invalid auto_increment settings: auto_increment_increment: %d, auto_increment_offset: %d, both of them must be in range [1..65535]", nil),
+ ErrDataInconsistentMismatchCount: mysql.Message("data inconsistency in table: %s, index: %s, index-count:%d != record-count:%d", nil),
+ ErrDataInconsistentMismatchIndex: mysql.Message("data inconsistency in table: %s, index: %s, col: %s, handle: %#v, index-values:%#v != record-values:%#v, compare err:%#v", []int{3, 4, 5, 6}),
+ ErrInconsistentRowValue: mysql.Message("writing inconsistent data in table: %s, expected-values:{%s} != record-values:{%s}", []int{1, 2}),
+ ErrInconsistentHandle: mysql.Message("writing inconsistent data in table: %s, index: %s, index-handle:%#v != record-handle:%#v, index: %#v, record: %#v", []int{2, 3, 4, 5}),
+ ErrInconsistentIndexedValue: mysql.Message("writing inconsistent data in table: %s, index: %s, col: %s, indexed-value:{%s} != record-value:{%s}", []int{3, 4}),
+ ErrAssertionFailed: mysql.Message("assertion failed: key: %s, assertion: %s, start_ts: %v, existing start ts: %v, existing commit ts: %v", []int{0}),
+ ErrInstanceScope: mysql.Message("modifying %s will require SET GLOBAL in a future version of TiDB", nil),
+ ErrNonTransactionalJobFailure: mysql.Message("non-transactional job failed, job id: %d, total jobs: %d. job range: [%s, %s], job sql: %s, err: %v", []int{2, 3, 4}),
+ ErrSettingNoopVariable: mysql.Message("setting %s has no effect in TiDB", nil),
+ ErrGettingNoopVariable: mysql.Message("variable %s has no effect in TiDB", nil),
+ ErrCannotMigrateSession: mysql.Message("cannot migrate the current session: %s", nil),
+ ErrLazyUniquenessCheckFailure: mysql.Message("transaction aborted because lazy uniqueness check is enabled and an error occurred: %s", nil),
+ ErrUnsupportedColumnInTTLConfig: mysql.Message("Field '%-.192s' is of a not supported type for TTL config, expect DATETIME, DATE or TIMESTAMP", nil),
+ ErrTTLColumnCannotDrop: mysql.Message("Cannot drop column '%-.192s': needed in TTL config", nil),
+ ErrSetTTLOptionForNonTTLTable: mysql.Message("Cannot set %s on a table without TTL config", nil),
+ ErrTempTableNotAllowedWithTTL: mysql.Message("Set TTL for temporary table is not allowed", nil),
+ ErrUnsupportedTTLReferencedByFK: mysql.Message("Set TTL for a table referenced by foreign key is not allowed", nil),
+ ErrUnsupportedPrimaryKeyTypeWithTTL: mysql.Message("Unsupported clustered primary key type FLOAT/DOUBLE for TTL", nil),
ErrWarnOptimizerHintInvalidInteger: mysql.Message("integer value is out of range in '%s'", nil),
ErrWarnOptimizerHintUnsupportedHint: mysql.Message("Optimizer hint %s is not supported by TiDB and is ignored", nil),
@@ -1072,9 +1083,10 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrInvalidTableSample: mysql.Message("Invalid TABLESAMPLE: %s", nil),
ErrJSONObjectKeyTooLong: mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil),
- ErrPartitionStatsMissing: mysql.Message("Build table: %s global-level stats failed due to missing partition-level stats", nil),
- ErrPartitionColumnStatsMissing: mysql.Message("Build table: %s global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", nil),
+ ErrPartitionStatsMissing: mysql.Message("Build global-level stats failed due to missing partition-level stats: %s", nil),
+ ErrPartitionColumnStatsMissing: mysql.Message("Build global-level stats failed due to missing partition-level column stats: %s, please run analyze table to refresh columns of all partitions", nil),
ErrDDLSetting: mysql.Message("Error happened when enable/disable DDL: %s", nil),
+ ErrIngestFailed: mysql.Message("Ingest failed: %s", nil),
ErrNotSupportedWithSem: mysql.Message("Feature '%s' is not supported when security enhanced mode is enabled", nil),
ErrPlacementPolicyCheck: mysql.Message("Placement policy didn't meet the constraint, reason: %s", nil),
@@ -1087,10 +1099,14 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrPlacementPolicyWithDirectOption: mysql.Message("Placement policy '%s' can't co-exist with direct placement options", nil),
ErrPlacementPolicyInUse: mysql.Message("Placement policy '%-.192s' is still in use", nil),
ErrOptOnCacheTable: mysql.Message("'%s' is unsupported on cache tables.", nil),
+ ErrResourceGroupExists: mysql.Message("Resource group '%-.192s' already exists", nil),
+ ErrResourceGroupNotExists: mysql.Message("Unknown resource group '%-.192s'", nil),
+
+ ErrColumnInChange: mysql.Message("column %s id %d does not exist, this column may have been updated by other DDL ran in parallel", nil),
+ ErrResourceGroupSupportDisabled: mysql.Message("Resource control feature is disabled. Run `SET GLOBAL tidb_enable_resource_control='on'` to enable the feature", nil),
- ErrColumnInChange: mysql.Message("column %s id %d does not exist, this column may have been updated by other DDL ran in parallel", nil),
// TiKV/PD errors.
- ErrPDServerTimeout: mysql.Message("PD server timeout", nil),
+ ErrPDServerTimeout: mysql.Message("PD server timeout: %s", nil),
ErrTiKVServerTimeout: mysql.Message("TiKV server timeout", nil),
ErrTiKVServerBusy: mysql.Message("TiKV server is busy", nil),
ErrTiFlashServerTimeout: mysql.Message("TiFlash server timeout", nil),
diff --git a/errors.toml b/errors.toml
index 31a56ef6b1d17..9b503c113a836 100644
--- a/errors.toml
+++ b/errors.toml
@@ -66,6 +66,11 @@ error = '''
restore met a invalid peer
'''
+["BR:EBS:ErrRestoreRegionWithoutPeer"]
+error = '''
+restore met a region without any peer
+'''
+
["BR:EBS:ErrRestoreTotalKVMismatch"]
error = '''
restore total tikvs mismatch
@@ -456,6 +461,11 @@ error = '''
update pd error
'''
+["Lightning:PreCheck:ErrCheckCDCPiTR"]
+error = '''
+check TiCDC/PiTR task error
+'''
+
["Lightning:PreCheck:ErrCheckCSVHeader"]
error = '''
check csv header error
@@ -516,6 +526,11 @@ error = '''
encode kv error in file %s at offset %d
'''
+["Lightning:Restore:ErrFoundDuplicateKey"]
+error = '''
+found duplicate key '%s', value '%s'
+'''
+
["Lightning:Restore:ErrInvalidMetaStatus"]
error = '''
invalid meta status: '%s'
@@ -653,7 +668,7 @@ Too many keys specified; max %d keys allowed
["ddl:1071"]
error = '''
-Specified key was too long; max key length is %d bytes
+Specified key was too long (%d bytes); max key length is %d bytes
'''
["ddl:1072"]
@@ -756,6 +771,11 @@ error = '''
Incorrect usage of %s and %s
'''
+["ddl:1235"]
+error = '''
+This version of TiDB doesn't yet support '%s'
+'''
+
["ddl:1246"]
error = '''
Converting column '%s' from %s to %s
@@ -1161,6 +1181,11 @@ error = '''
Column '%s' has an expression index dependency and cannot be dropped or renamed
'''
+["ddl:3855"]
+error = '''
+Column '%s' has a partitioning function dependency and cannot be dropped or renamed
+'''
+
["ddl:4135"]
error = '''
Sequence '%-.64s.%-.64s' has run out
@@ -1191,6 +1216,36 @@ error = '''
`%s` is unsupported on temporary tables.
'''
+["ddl:8148"]
+error = '''
+Field '%-.192s' is of a not supported type for TTL config, expect DATETIME, DATE or TIMESTAMP
+'''
+
+["ddl:8149"]
+error = '''
+Cannot drop column '%-.192s': needed in TTL config
+'''
+
+["ddl:8150"]
+error = '''
+Cannot set %s on a table without TTL config
+'''
+
+["ddl:8151"]
+error = '''
+Set TTL for temporary table is not allowed
+'''
+
+["ddl:8152"]
+error = '''
+Set TTL for a table referenced by foreign key is not allowed
+'''
+
+["ddl:8153"]
+error = '''
+Unsupported clustered primary key type FLOAT/DOUBLE for TTL
+'''
+
["ddl:8200"]
error = '''
Unsupported shard_row_id_bits for table with primary key as row id
@@ -1336,6 +1391,11 @@ error = '''
Error happened when enable/disable DDL: %s
'''
+["ddl:8247"]
+error = '''
+Ingest failed: %s
+'''
+
["domain:8027"]
error = '''
Information schema is out of date: schema failed to update in 1 lease, please make sure TiDB can connect to TiKV
@@ -1451,6 +1511,11 @@ error = '''
SET PASSWORD has no significance for user '%-.48s'@'%-.255s' as authentication plugin does not support it.
'''
+["executor:1819"]
+error = '''
+Your password does not satisfy the current policy requirements
+'''
+
["executor:1827"]
error = '''
The password hash doesn't have the expected format. Check if the correct password algorithm is being used with the PASSWORD() function.
@@ -1461,6 +1526,11 @@ error = '''
Foreign key cascade delete/update exceeds max depth of %v.
'''
+["executor:3016"]
+error = '''
+The password for anonymous user cannot be expired.
+'''
+
["executor:3523"]
error = '''
Unknown authorization ID %.256s
@@ -1476,6 +1546,11 @@ error = '''
Recursive query aborted after %d iterations. Try increasing @@cte_max_recursion_depth to a larger value
'''
+["executor:3638"]
+error = '''
+Cannot use these credentials for '%s@%s' because they contradict the password history policy.
+'''
+
["executor:3929"]
error = '''
Dynamic privilege '%s' is not registered with the server.
@@ -1636,6 +1711,21 @@ error = '''
Invalid data type for JSON data in argument %d to function %s; a JSON string or JSON type is required.
'''
+["expression:3752"]
+error = '''
+Value is out of range for expression index '%s' at row %d
+'''
+
+["expression:3903"]
+error = '''
+Invalid JSON value for CAST for expression index '%s'
+'''
+
+["expression:3907"]
+error = '''
+Data too long for expression index '%s'
+'''
+
["expression:8128"]
error = '''
Invalid TABLESAMPLE: %s
@@ -1781,6 +1871,16 @@ error = '''
Unknown placement policy '%-.192s'
'''
+["meta:8248"]
+error = '''
+Resource group '%-.192s' already exists
+'''
+
+["meta:8249"]
+error = '''
+Unknown resource group '%-.192s'
+'''
+
["planner:1044"]
error = '''
Access denied for user '%-.48s'@'%-.255s' to database '%-.192s'
@@ -2231,6 +2331,11 @@ error = '''
There is no such grant defined for user '%-.48s' on host '%-.255s'
'''
+["privilege:1862"]
+error = '''
+Your password has expired. To log in you must change it using a client that supports expired passwords.
+'''
+
["privilege:3530"]
error = '''
%s is not granted to %s
@@ -2346,6 +2451,11 @@ error = '''
Changing schema from '%-.192s' to '%-.192s' is not allowed.
'''
+["schema:1506"]
+error = '''
+Foreign key clause is not yet supported in conjunction with partitioning
+'''
+
["schema:1822"]
error = '''
Failed to add the foreign key constraint. Missing index for constraint '%s' in the referenced table '%s'
@@ -2416,6 +2526,21 @@ error = '''
Unknown placement policy '%-.192s'
'''
+["schema:8248"]
+error = '''
+Resource group '%-.192s' already exists
+'''
+
+["schema:8249"]
+error = '''
+Unknown resource group '%-.192s'
+'''
+
+["schema:8250"]
+error = '''
+Resource control feature is disabled. Run `SET GLOBAL tidb_enable_resource_control='on'` to enable the feature
+'''
+
["session:8002"]
error = '''
[%d] can not retry select for update statement
@@ -2588,7 +2713,7 @@ TTL manager has timed out, pessimistic locks may expire, please commit or rollba
["tikv:9001"]
error = '''
-PD server timeout
+PD server timeout: %s
'''
["tikv:9002"]
@@ -2726,6 +2851,11 @@ error = '''
Datetime function: %-.32s field overflow
'''
+["types:1525"]
+error = '''
+Incorrect %-.32s value: '%-.128s'
+'''
+
["types:1690"]
error = '''
%s value is out of range in '%s'
@@ -2773,12 +2903,12 @@ TiDB does not yet support JSON objects with the key length >= 65536
["types:8131"]
error = '''
-Build table: %s global-level stats failed due to missing partition-level stats
+Build global-level stats failed due to missing partition-level stats: %s
'''
["types:8244"]
error = '''
-Build table: %s global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions
+Build global-level stats failed due to missing partition-level column stats: %s, please run analyze table to refresh columns of all partitions
'''
["variable:1193"]
diff --git a/executor/BUILD.bazel b/executor/BUILD.bazel
index 6a300dbeaf654..aa288e1180843 100644
--- a/executor/BUILD.bazel
+++ b/executor/BUILD.bazel
@@ -85,6 +85,7 @@ go_library(
"slow_query.go",
"sort.go",
"split.go",
+ "stmtsummary.go",
"table_reader.go",
"trace.go",
"union_scan.go",
@@ -121,9 +122,11 @@ go_library(
"//parser/ast",
"//parser/auth",
"//parser/charset",
+ "//parser/format",
"//parser/model",
"//parser/mysql",
"//parser/terror",
+ "//parser/tidb",
"//parser/types",
"//planner",
"//planner/core",
@@ -177,10 +180,12 @@ go_library(
"//util/mathutil",
"//util/memory",
"//util/mvmap",
+ "//util/password-validation",
"//util/pdapi",
"//util/plancodec",
"//util/printer",
"//util/ranger",
+ "//util/replayer",
"//util/resourcegrouptag",
"//util/rowDecoder",
"//util/rowcodec",
@@ -190,12 +195,14 @@ go_library(
"//util/size",
"//util/sqlexec",
"//util/stmtsummary",
+ "//util/stmtsummary/v2:stmtsummary",
"//util/stringutil",
"//util/table-filter",
"//util/timeutil",
"//util/tls",
"//util/topsql",
"//util/topsql/state",
+ "//util/tracing",
"@com_github_burntsushi_toml//:toml",
"@com_github_gogo_protobuf//proto",
"@com_github_ngaut_pools//:pools",
@@ -210,6 +217,7 @@ go_library(
"@com_github_pingcap_kvproto//pkg/encryptionpb",
"@com_github_pingcap_kvproto//pkg/kvrpcpb",
"@com_github_pingcap_kvproto//pkg/metapb",
+ "@com_github_pingcap_kvproto//pkg/resource_manager",
"@com_github_pingcap_kvproto//pkg/tikvpb",
"@com_github_pingcap_log//:log",
"@com_github_pingcap_sysutil//:sysutil",
@@ -234,6 +242,7 @@ go_library(
"@org_golang_google_grpc//:grpc",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//credentials",
+ "@org_golang_google_grpc//credentials/insecure",
"@org_golang_google_grpc//status",
"@org_golang_x_exp//slices",
"@org_golang_x_sync//errgroup",
@@ -245,7 +254,7 @@ go_library(
go_test(
name = "executor_test",
- timeout = "moderate",
+ timeout = "short",
srcs = [
"adapter_test.go",
"admin_test.go",
@@ -267,7 +276,6 @@ go_test(
"delete_test.go",
"distsql_test.go",
"executor_failpoint_test.go",
- "executor_issue_test.go",
"executor_pkg_test.go",
"executor_required_rows_test.go",
"executor_test.go",
@@ -277,6 +285,7 @@ go_test(
"explainfor_test.go",
"grant_test.go",
"hash_table_test.go",
+ "historical_stats_test.go",
"hot_regions_history_table_test.go",
"index_advise_test.go",
"index_lookup_join_test.go",
@@ -319,6 +328,7 @@ go_test(
"split_test.go",
"stale_txn_test.go",
"statement_context_test.go",
+ "stmtsummary_test.go",
"table_readers_required_rows_test.go",
"temporary_table_test.go",
"tikv_regions_peers_table_test.go",
@@ -328,7 +338,6 @@ go_test(
"utils_test.go",
"window_test.go",
"write_concurrent_test.go",
- "write_test.go",
],
data = glob(["testdata/**"]),
embed = [":executor"],
@@ -356,7 +365,6 @@ go_test(
"//parser",
"//parser/ast",
"//parser/auth",
- "//parser/charset",
"//parser/model",
"//parser/mysql",
"//parser/terror",
@@ -370,6 +378,7 @@ go_test(
"//sessionctx/binloginfo",
"//sessionctx/stmtctx",
"//sessionctx/variable",
+ "//sessionctx/variable/featuretag/distributereorg",
"//sessiontxn",
"//sessiontxn/staleread",
"//statistics",
@@ -409,9 +418,11 @@ go_test(
"//util/pdapi",
"//util/plancodec",
"//util/ranger",
+ "//util/replayer",
"//util/rowcodec",
"//util/set",
"//util/sqlexec",
+ "//util/stmtsummary/v2:stmtsummary",
"//util/stringutil",
"//util/tableutil",
"//util/timeutil",
diff --git a/executor/adapter.go b/executor/adapter.go
index db9fbbaa929e0..7702f2e05875a 100644
--- a/executor/adapter.go
+++ b/executor/adapter.go
@@ -18,16 +18,17 @@ import (
"bytes"
"context"
"fmt"
+ "math"
"runtime/trace"
"strconv"
"strings"
"sync/atomic"
"time"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/log"
+ "github.com/pingcap/tidb/bindinfo"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/ddl/placement"
"github.com/pingcap/tidb/domain"
@@ -57,11 +58,14 @@ import (
"github.com/pingcap/tidb/util/mathutil"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/plancodec"
+ "github.com/pingcap/tidb/util/replayer"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/pingcap/tidb/util/stmtsummary"
+ stmtsummaryv2 "github.com/pingcap/tidb/util/stmtsummary/v2"
"github.com/pingcap/tidb/util/stringutil"
"github.com/pingcap/tidb/util/topsql"
topsqlstate "github.com/pingcap/tidb/util/topsql/state"
+ "github.com/pingcap/tidb/util/tracing"
"github.com/prometheus/client_golang/prometheus"
tikverr "github.com/tikv/client-go/v2/error"
"github.com/tikv/client-go/v2/oracle"
@@ -78,6 +82,22 @@ var (
totalQueryProcHistogramInternal = metrics.TotalQueryProcHistogram.WithLabelValues(metrics.LblInternal)
totalCopProcHistogramInternal = metrics.TotalCopProcHistogram.WithLabelValues(metrics.LblInternal)
totalCopWaitHistogramInternal = metrics.TotalCopWaitHistogram.WithLabelValues(metrics.LblInternal)
+
+ selectForUpdateFirstAttemptDuration = metrics.PessimisticDMLDurationByAttempt.WithLabelValues("select-for-update", "first-attempt")
+ selectForUpdateRetryDuration = metrics.PessimisticDMLDurationByAttempt.WithLabelValues("select-for-update", "retry")
+ dmlFirstAttemptDuration = metrics.PessimisticDMLDurationByAttempt.WithLabelValues("dml", "first-attempt")
+ dmlRetryDuration = metrics.PessimisticDMLDurationByAttempt.WithLabelValues("dml", "retry")
+
+ // aggressiveLockingTxnUsedCount counts transactions where at least one statement has aggressive locking enabled.
+ aggressiveLockingTxnUsedCount = metrics.AggressiveLockingUsageCount.WithLabelValues(metrics.LblAggressiveLockingTxnUsed)
+ // aggressiveLockingStmtUsedCount counts statements that have aggressive locking enabled.
+ aggressiveLockingStmtUsedCount = metrics.AggressiveLockingUsageCount.WithLabelValues(metrics.LblAggressiveLockingStmtUsed)
+ // aggressiveLockingTxnUsedCount counts transactions where at least one statement has aggressive locking enabled,
+ // and it takes effect (which is determined according to whether lock-with-conflict has occurred during execution).
+ aggressiveLockingTxnEffectiveCount = metrics.AggressiveLockingUsageCount.WithLabelValues(metrics.LblAggressiveLockingTxnEffective)
+ // aggressiveLockingTxnUsedCount counts statements where at least one statement has aggressive locking enabled,
+ // and it takes effect (which is determined according to whether lock-with-conflict has occurred during execution).
+ aggressiveLockingStmtEffectiveCount = metrics.AggressiveLockingUsageCount.WithLabelValues(metrics.LblAggressiveLockingStmtEffective)
)
// processinfoSetter is the interface use to set current running process info.
@@ -194,10 +214,12 @@ type TelemetryInfo struct {
UseNonRecursive bool
UseRecursive bool
UseMultiSchemaChange bool
- UesExchangePartition bool
+ UseExchangePartition bool
UseFlashbackToCluster bool
PartitionTelemetry *PartitionTelemetryInfo
AccountLockTelemetry *AccountLockTelemetryInfo
+ UseIndexMerge bool
+ UseTableLookUp bool
}
// PartitionTelemetryInfo records table partition telemetry information during execution.
@@ -215,6 +237,8 @@ type PartitionTelemetryInfo struct {
UseCreateIntervalPartition bool
UseAddIntervalPartition bool
UseDropIntervalPartition bool
+ UseCompactTablePartition bool
+ UseReorganizePartition bool
}
// AccountLockTelemetryInfo records account lock/unlock information during execution
@@ -273,12 +297,12 @@ func (a *ExecStmt) GetStmtNode() ast.StmtNode {
// PointGet short path for point exec directly from plan, keep only necessary steps
func (a *ExecStmt) PointGet(ctx context.Context) (*recordSet, error) {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("ExecStmt.PointGet", opentracing.ChildOf(span.Context()))
- span1.LogKV("sql", a.OriginText())
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
+ r, ctx := tracing.StartRegionEx(ctx, "ExecStmt.PointGet")
+ defer r.End()
+ if r.Span != nil {
+ r.Span.LogKV("sql", a.OriginText())
}
+
failpoint.Inject("assertTxnManagerInShortPointGetPlan", func() {
sessiontxn.RecordAssert(a.Ctx, "assertTxnManagerInShortPointGetPlan", true)
// stale read should not reach here
@@ -293,8 +317,12 @@ func (a *ExecStmt) PointGet(ctx context.Context) (*recordSet, error) {
}
a.Ctx.GetSessionVars().StmtCtx.Priority = kv.PriorityHigh
+ var pointExecutor *PointGetExecutor
+ useMaxTS := startTs == math.MaxUint64
+
// try to reuse point get executor
- if a.PsStmt.Executor != nil {
+ // We should only use the cached the executor when the startTS is MaxUint64
+ if a.PsStmt.Executor != nil && useMaxTS {
exec, ok := a.PsStmt.Executor.(*PointGetExecutor)
if !ok {
logutil.Logger(ctx).Error("invalid executor type, not PointGetExecutor for point get path")
@@ -304,17 +332,21 @@ func (a *ExecStmt) PointGet(ctx context.Context) (*recordSet, error) {
pointGetPlan := a.PsStmt.PreparedAst.CachedPlan.(*plannercore.PointGetPlan)
exec.Init(pointGetPlan)
a.PsStmt.Executor = exec
+ pointExecutor = exec
}
}
- if a.PsStmt.Executor == nil {
+
+ if pointExecutor == nil {
b := newExecutorBuilder(a.Ctx, a.InfoSchema, a.Ti)
- newExecutor := b.build(a.Plan)
+ pointExecutor = b.build(a.Plan).(*PointGetExecutor)
if b.err != nil {
return nil, b.err
}
- a.PsStmt.Executor = newExecutor
+
+ if useMaxTS {
+ a.PsStmt.Executor = pointExecutor
+ }
}
- pointExecutor := a.PsStmt.Executor.(*PointGetExecutor)
if err = pointExecutor.Open(ctx); err != nil {
terror.Call(pointExecutor.Close)
@@ -432,6 +464,18 @@ func (a *ExecStmt) Exec(ctx context.Context) (_ sqlexec.RecordSet, err error) {
if lockKeysCnt > 0 {
metrics.StatementLockKeysCount.Observe(float64(lockKeysCnt))
}
+
+ execDetails := a.Ctx.GetSessionVars().StmtCtx.GetExecDetails()
+ if err == nil && execDetails.LockKeysDetail != nil &&
+ (execDetails.LockKeysDetail.AggressiveLockNewCount > 0 || execDetails.LockKeysDetail.AggressiveLockDerivedCount > 0) {
+ a.Ctx.GetSessionVars().TxnCtx.AggressiveLockingUsed = true
+ // If this statement is finished when some of the keys are locked with conflict in the last retry, or
+ // some of the keys are derived from the previous retry, we consider the optimization of aggressive locking
+ // takes effect on this statement.
+ if execDetails.LockKeysDetail.LockedWithConflictCount > 0 || execDetails.LockKeysDetail.AggressiveLockDerivedCount > 0 {
+ a.Ctx.GetSessionVars().TxnCtx.AggressiveLockingEffective = true
+ }
+ }
return
}
if str, ok := r.(string); !ok || !strings.Contains(str, memory.PanicMemoryExceed) {
@@ -467,8 +511,20 @@ func (a *ExecStmt) Exec(ctx context.Context) (_ sqlexec.RecordSet, err error) {
if !ok {
oriIso = "REPEATABLE-READ"
}
- terror.Log(sctx.GetSessionVars().SetSystemVar(variable.TiDBBuildStatsConcurrency, "1"))
- sctx.GetSessionVars().SetDistSQLScanConcurrency(1)
+ autoConcurrency, err1 := sctx.GetSessionVars().GetSessionOrGlobalSystemVar(ctx, variable.TiDBAutoBuildStatsConcurrency)
+ terror.Log(err1)
+ if err1 == nil {
+ terror.Log(sctx.GetSessionVars().SetSystemVar(variable.TiDBBuildStatsConcurrency, autoConcurrency))
+ }
+ sVal, err2 := sctx.GetSessionVars().GetSessionOrGlobalSystemVar(ctx, variable.TiDBSysProcScanConcurrency)
+ terror.Log(err2)
+ if err2 == nil {
+ concurrency, err3 := strconv.ParseInt(sVal, 10, 64)
+ terror.Log(err3)
+ if err3 == nil {
+ sctx.GetSessionVars().SetDistSQLScanConcurrency(int(concurrency))
+ }
+ }
sctx.GetSessionVars().SetIndexSerialScanConcurrency(1)
terror.Log(sctx.GetSessionVars().SetSystemVar(variable.TxnIsolation, ast.ReadCommitted))
defer func() {
@@ -568,7 +624,7 @@ func (a *ExecStmt) handleStmtForeignKeyTrigger(ctx context.Context, e Executor)
// change first.
// Since `UnionScanExec` use `SnapshotIter` and `SnapshotGetter` to read txn mem-buffer, if we don't do `StmtCommit`,
// then the fk cascade executor can't read the mem-buffer changed by the ExecStmt.
- a.Ctx.StmtCommit()
+ a.Ctx.StmtCommit(ctx)
}
err := a.handleForeignKeyTrigger(ctx, e, 1)
if err != nil {
@@ -591,10 +647,6 @@ func (a *ExecStmt) handleForeignKeyTrigger(ctx context.Context, e Executor, dept
if !ok {
return nil
}
- a.Ctx.GetSessionVars().StmtCtx.InHandleForeignKeyTrigger = true
- defer func() {
- a.Ctx.GetSessionVars().StmtCtx.InHandleForeignKeyTrigger = false
- }()
fkChecks := exec.GetFKChecks()
for _, fkCheck := range fkChecks {
err := fkCheck.doCheck(ctx)
@@ -624,12 +676,26 @@ func (a *ExecStmt) handleForeignKeyTrigger(ctx context.Context, e Executor, dept
// 4. `StmtCommit` to commit the kv change to transaction mem-buffer.
// 5. If the foreign key cascade behaviour has more fk value need to be cascaded, go to step 1.
func (a *ExecStmt) handleForeignKeyCascade(ctx context.Context, fkc *FKCascadeExec, depth int) error {
+ if a.Ctx.GetSessionVars().StmtCtx.RuntimeStatsColl != nil {
+ fkc.stats = &FKCascadeRuntimeStats{}
+ defer a.Ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(fkc.plan.ID(), fkc.stats)
+ }
if len(fkc.fkValues) == 0 && len(fkc.fkUpdatedValuesMap) == 0 {
return nil
}
if depth > maxForeignKeyCascadeDepth {
return ErrForeignKeyCascadeDepthExceeded.GenWithStackByArgs(maxForeignKeyCascadeDepth)
}
+ a.Ctx.GetSessionVars().StmtCtx.InHandleForeignKeyTrigger = true
+ defer func() {
+ a.Ctx.GetSessionVars().StmtCtx.InHandleForeignKeyTrigger = false
+ }()
+ if fkc.stats != nil {
+ start := time.Now()
+ defer func() {
+ fkc.stats.Total += time.Since(start)
+ }()
+ }
for {
e, err := fkc.buildExecutor(ctx)
if err != nil || e == nil {
@@ -649,7 +715,7 @@ func (a *ExecStmt) handleForeignKeyCascade(ctx context.Context, fkc *FKCascadeEx
}
// Call `StmtCommit` uses to flush the fk cascade executor change into txn mem-buffer,
// then the later fk cascade executors can see the mem-buffer changes.
- a.Ctx.StmtCommit()
+ a.Ctx.StmtCommit(ctx)
err = a.handleForeignKeyTrigger(ctx, e, depth+1)
if err != nil {
return err
@@ -823,8 +889,25 @@ func (a *ExecStmt) handlePessimisticSelectForUpdate(ctx context.Context, e Execu
return nil, errors.New("can not execute write statement when 'tidb_snapshot' is set")
}
+ txnManager := sessiontxn.GetTxnManager(a.Ctx)
+ err := txnManager.OnHandlePessimisticStmtStart(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ isFirstAttempt := true
+
for {
+ startTime := time.Now()
rs, err := a.runPessimisticSelectForUpdate(ctx, e)
+
+ if isFirstAttempt {
+ selectForUpdateFirstAttemptDuration.Observe(time.Since(startTime).Seconds())
+ isFirstAttempt = false
+ } else {
+ selectForUpdateRetryDuration.Observe(time.Since(startTime).Seconds())
+ }
+
e, err = a.handlePessimisticLockError(ctx, err)
if err != nil {
return nil, err
@@ -832,6 +915,8 @@ func (a *ExecStmt) handlePessimisticSelectForUpdate(ctx context.Context, e Execu
if e == nil {
return rs, nil
}
+
+ failpoint.Inject("pessimisticSelectForUpdateRetry", nil)
}
}
@@ -862,11 +947,8 @@ func (a *ExecStmt) runPessimisticSelectForUpdate(ctx context.Context, e Executor
func (a *ExecStmt) handleNoDelayExecutor(ctx context.Context, e Executor) (sqlexec.RecordSet, error) {
sctx := a.Ctx
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("executor.handleNoDelayExecutor", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "executor.handleNoDelayExecutor")
+ defer r.End()
var err error
defer func() {
@@ -927,18 +1009,39 @@ func (a *ExecStmt) handlePessimisticDML(ctx context.Context, e Executor) (err er
err = ErrLazyUniquenessCheckFailure.GenWithStackByArgs(err.Error())
}
}()
+
+ txnManager := sessiontxn.GetTxnManager(a.Ctx)
+ err = txnManager.OnHandlePessimisticStmtStart(ctx)
+ if err != nil {
+ return err
+ }
+
+ isFirstAttempt := true
+
for {
- startPointGetLocking := time.Now()
+ if !isFirstAttempt {
+ failpoint.Inject("pessimisticDMLRetry", nil)
+ }
+
+ startTime := time.Now()
_, err = a.handleNoDelayExecutor(ctx, e)
if !txn.Valid() {
return err
}
+
+ if isFirstAttempt {
+ dmlFirstAttemptDuration.Observe(time.Since(startTime).Seconds())
+ isFirstAttempt = false
+ } else {
+ dmlRetryDuration.Observe(time.Since(startTime).Seconds())
+ }
+
if err != nil {
// It is possible the DML has point get plan that locks the key.
e, err = a.handlePessimisticLockError(ctx, err)
if err != nil {
if ErrDeadlock.Equal(err) {
- metrics.StatementDeadlockDetectDuration.Observe(time.Since(startPointGetLocking).Seconds())
+ metrics.StatementDeadlockDetectDuration.Observe(time.Since(startTime).Seconds())
}
return err
}
@@ -1036,7 +1139,7 @@ func (a *ExecStmt) handlePessimisticLockError(ctx context.Context, lockErr error
return nil, err
}
// Rollback the statement change before retry it.
- a.Ctx.StmtRollback()
+ a.Ctx.StmtRollback(ctx, true)
a.Ctx.GetSessionVars().StmtCtx.ResetForRetry()
a.Ctx.GetSessionVars().RetryInfo.ResetOffset()
@@ -1332,6 +1435,8 @@ func (a *ExecStmt) observePhaseDurations(internal bool, commitDetails *util.Comm
// 4. update the `PrevStmt` in session variable.
// 5. reset `DurationParse` in session variable.
func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, err error, hasMoreResults bool) {
+ a.checkPlanReplayerCapture(txnTS)
+
sessVars := a.Ctx.GetSessionVars()
execDetail := sessVars.StmtCtx.GetExecDetails()
// Attach commit/lockKeys runtime stats to executor runtime stats.
@@ -1383,10 +1488,53 @@ func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, err error, hasMoreResults boo
sessVars.DurationParse = 0
// Clean the stale read flag when statement execution finish
sessVars.StmtCtx.IsStaleness = false
+ // Clean the MPP query info
+ sessVars.StmtCtx.MPPQueryInfo.QueryID.Store(0)
+ sessVars.StmtCtx.MPPQueryInfo.QueryTS.Store(0)
+ sessVars.StmtCtx.MPPQueryInfo.AllocatedMPPTaskID.Store(0)
if sessVars.StmtCtx.ReadFromTableCache {
metrics.ReadFromTableCacheCounter.Inc()
}
+
+ // Update aggressive locking related counters by stmt
+ if execDetail.LockKeysDetail != nil {
+ if execDetail.LockKeysDetail.AggressiveLockNewCount > 0 || execDetail.LockKeysDetail.AggressiveLockDerivedCount > 0 {
+ aggressiveLockingStmtUsedCount.Inc()
+ // If this statement is finished when some of the keys are locked with conflict in the last retry, or
+ // some of the keys are derived from the previous retry, we consider the optimization of aggressive locking
+ // takes effect on this statement.
+ if execDetail.LockKeysDetail.LockedWithConflictCount > 0 || execDetail.LockKeysDetail.AggressiveLockDerivedCount > 0 {
+ aggressiveLockingStmtEffectiveCount.Inc()
+ }
+ }
+ }
+ // If the transaction is committed, update aggressive locking related counters by txn
+ if execDetail.CommitDetail != nil {
+ if sessVars.TxnCtx.AggressiveLockingUsed {
+ aggressiveLockingTxnUsedCount.Inc()
+ }
+ if sessVars.TxnCtx.AggressiveLockingEffective {
+ aggressiveLockingTxnEffectiveCount.Inc()
+ }
+ }
+}
+
+func (a *ExecStmt) checkPlanReplayerCapture(txnTS uint64) {
+ if kv.GetInternalSourceType(a.GoCtx) == kv.InternalTxnStats {
+ return
+ }
+ se := a.Ctx
+ if !se.GetSessionVars().InRestrictedSQL && se.GetSessionVars().IsPlanReplayerCaptureEnabled() {
+ stmtNode := a.GetStmtNode()
+ if se.GetSessionVars().EnablePlanReplayedContinuesCapture {
+ if checkPlanReplayerContinuesCaptureValidStmt(stmtNode) {
+ checkPlanReplayerContinuesCapture(se, stmtNode, txnTS)
+ }
+ } else {
+ checkPlanReplayerCaptureTask(se, stmtNode, txnTS)
+ }
+ }
}
// CloseRecordSet will finish the execution of current statement and do some record work
@@ -1503,6 +1651,7 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) {
IsWriteCacheTable: stmtCtx.WaitLockLeaseTime > 0,
StatsLoadStatus: convertStatusIntoString(a.Ctx, stmtCtx.StatsLoadStatus),
IsSyncStatsFailed: stmtCtx.IsSyncStatsFailed,
+ Warnings: collectWarningsForSlowLog(stmtCtx),
}
failpoint.Inject("assertSyncStatsFailed", func(val failpoint.Value) {
if val.(bool) {
@@ -1514,7 +1663,7 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) {
if a.retryCount > 0 {
slowItems.ExecRetryTime = costTime - sessVars.DurationParse - sessVars.DurationCompile - time.Since(a.retryStartTime)
}
- if _, ok := a.StmtNode.(*ast.CommitStmt); ok {
+ if _, ok := a.StmtNode.(*ast.CommitStmt); ok && sessVars.PrevStmt != nil {
slowItems.PrevStmt = sessVars.PrevStmt.String()
}
slowLog := sessVars.SlowLogFormat(slowItems)
@@ -1558,6 +1707,33 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) {
}
}
+func extractMsgFromSQLWarn(SQLWarn *stmtctx.SQLWarn) string {
+ // Currently, this function is only used in collectWarningsForSlowLog.
+ // collectWarningsForSlowLog can make sure SQLWarn is not nil so no need to add a nil check here.
+ warn := errors.Cause(SQLWarn.Err)
+ if x, ok := warn.(*terror.Error); ok && x != nil {
+ sqlErr := terror.ToSQLError(x)
+ return sqlErr.Message
+ }
+ return warn.Error()
+}
+
+func collectWarningsForSlowLog(stmtCtx *stmtctx.StatementContext) []variable.JSONSQLWarnForSlowLog {
+ warnings := stmtCtx.GetWarnings()
+ extraWarnings := stmtCtx.GetExtraWarnings()
+ res := make([]variable.JSONSQLWarnForSlowLog, len(warnings)+len(extraWarnings))
+ for i := range warnings {
+ res[i].Level = warnings[i].Level
+ res[i].Message = extractMsgFromSQLWarn(&warnings[i])
+ }
+ for i := range extraWarnings {
+ res[len(warnings)+i].Level = extraWarnings[i].Level
+ res[len(warnings)+i].Message = extractMsgFromSQLWarn(&extraWarnings[i])
+ res[len(warnings)+i].IsExtra = true
+ }
+ return res
+}
+
// GetResultRowsCount gets the count of the statement result rows.
func GetResultRowsCount(stmtCtx *stmtctx.StatementContext, p plannercore.Plan) int64 {
runtimeStatsColl := stmtCtx.RuntimeStatsColl
@@ -1629,6 +1805,11 @@ func getPlanDigest(stmtCtx *stmtctx.StatementContext) (string, *parser.Digest) {
return normalized, planDigest
}
+// GetEncodedPlan returned same as getEncodedPlan
+func GetEncodedPlan(stmtCtx *stmtctx.StatementContext, genHint bool) (encodedPlan, hintStr string) {
+ return getEncodedPlan(stmtCtx, genHint)
+}
+
// getEncodedPlan gets the encoded plan, and generates the hint string if indicated.
func getEncodedPlan(stmtCtx *stmtctx.StatementContext, genHint bool) (encodedPlan, hintStr string) {
var hintSet bool
@@ -1670,7 +1851,7 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
}
// Internal SQLs must also be recorded to keep the consistency of `PrevStmt` and `PrevStmtDigest`.
- if !stmtsummary.StmtSummaryByDigestMap.Enabled() || ((sessVars.InRestrictedSQL || len(userString) == 0) && !stmtsummary.StmtSummaryByDigestMap.EnabledInternal()) {
+ if !stmtsummaryv2.Enabled() || ((sessVars.InRestrictedSQL || len(userString) == 0) && !stmtsummaryv2.EnabledInternal()) {
sessVars.SetPrevStmtDigest("")
return
}
@@ -1787,7 +1968,7 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
if a.retryCount > 0 {
stmtExecInfo.ExecRetryTime = costTime - sessVars.DurationParse - sessVars.DurationCompile - time.Since(a.retryStartTime)
}
- stmtsummary.StmtSummaryByDigestMap.AddStatement(stmtExecInfo)
+ stmtsummaryv2.Add(stmtExecInfo)
}
// GetTextToLog return the query text to log.
@@ -1873,7 +2054,6 @@ func (a *ExecStmt) getSQLPlanDigest() ([]byte, []byte) {
}
return sqlDigest, planDigest
}
-
func convertStatusIntoString(sctx sessionctx.Context, statsLoadStatus map[model.TableItemID]string) map[string]map[string]string {
if len(statsLoadStatus) < 1 {
return nil
@@ -1911,3 +2091,92 @@ func convertStatusIntoString(sctx sessionctx.Context, statsLoadStatus map[model.
}
return r
}
+
+// only allow select/delete/update/insert/execute stmt captured by continues capture
+func checkPlanReplayerContinuesCaptureValidStmt(stmtNode ast.StmtNode) bool {
+ switch stmtNode.(type) {
+ case *ast.SelectStmt, *ast.DeleteStmt, *ast.UpdateStmt, *ast.InsertStmt, *ast.ExecuteStmt:
+ return true
+ default:
+ return false
+ }
+}
+
+func checkPlanReplayerCaptureTask(sctx sessionctx.Context, stmtNode ast.StmtNode, startTS uint64) {
+ dom := domain.GetDomain(sctx)
+ if dom == nil {
+ return
+ }
+ handle := dom.GetPlanReplayerHandle()
+ if handle == nil {
+ return
+ }
+ tasks := handle.GetTasks()
+ if len(tasks) == 0 {
+ return
+ }
+ _, sqlDigest := sctx.GetSessionVars().StmtCtx.SQLDigest()
+ _, planDigest := sctx.GetSessionVars().StmtCtx.GetPlanDigest()
+ if sqlDigest == nil || planDigest == nil {
+ return
+ }
+ key := replayer.PlanReplayerTaskKey{
+ SQLDigest: sqlDigest.String(),
+ PlanDigest: planDigest.String(),
+ }
+ for _, task := range tasks {
+ if task.SQLDigest == sqlDigest.String() {
+ if task.PlanDigest == "*" || task.PlanDigest == planDigest.String() {
+ sendPlanReplayerDumpTask(key, sctx, stmtNode, startTS, false)
+ return
+ }
+ }
+ }
+}
+
+func checkPlanReplayerContinuesCapture(sctx sessionctx.Context, stmtNode ast.StmtNode, startTS uint64) {
+ dom := domain.GetDomain(sctx)
+ if dom == nil {
+ return
+ }
+ handle := dom.GetPlanReplayerHandle()
+ if handle == nil {
+ return
+ }
+ _, sqlDigest := sctx.GetSessionVars().StmtCtx.SQLDigest()
+ _, planDigest := sctx.GetSessionVars().StmtCtx.GetPlanDigest()
+ key := replayer.PlanReplayerTaskKey{
+ SQLDigest: sqlDigest.String(),
+ PlanDigest: planDigest.String(),
+ }
+ existed := sctx.GetSessionVars().CheckPlanReplayerFinishedTaskKey(key)
+ if existed {
+ return
+ }
+ sendPlanReplayerDumpTask(key, sctx, stmtNode, startTS, true)
+ sctx.GetSessionVars().AddPlanReplayerFinishedTaskKey(key)
+}
+
+func sendPlanReplayerDumpTask(key replayer.PlanReplayerTaskKey, sctx sessionctx.Context, stmtNode ast.StmtNode,
+ startTS uint64, isContinuesCapture bool) {
+ stmtCtx := sctx.GetSessionVars().StmtCtx
+ handle := sctx.Value(bindinfo.SessionBindInfoKeyType).(*bindinfo.SessionHandle)
+ dumpTask := &domain.PlanReplayerDumpTask{
+ PlanReplayerTaskKey: key,
+ StartTS: startTS,
+ TblStats: stmtCtx.TableStats,
+ SessionBindings: handle.GetAllBindRecord(),
+ SessionVars: sctx.GetSessionVars(),
+ ExecStmts: []ast.StmtNode{stmtNode},
+ Analyze: false,
+ IsCapture: true,
+ IsContinuesCapture: isContinuesCapture,
+ }
+ dumpTask.EncodedPlan, _ = GetEncodedPlan(stmtCtx, false)
+ if _, ok := stmtNode.(*ast.ExecuteStmt); ok {
+ nsql, _ := sctx.GetSessionVars().StmtCtx.SQLDigest()
+ dumpTask.InExecute = true
+ dumpTask.NormalizedSQL = nsql
+ }
+ domain.GetDomain(sctx).GetPlanReplayerHandle().SendTask(dumpTask)
+}
diff --git a/executor/admin.go b/executor/admin.go
index 1a0f5579281cc..33088c234d7e2 100644
--- a/executor/admin.go
+++ b/executor/admin.go
@@ -21,6 +21,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/tidb/distsql"
+ "github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/ast"
@@ -163,7 +164,7 @@ func (e *CheckIndexRangeExec) constructIndexScanPB() *tipb.Executor {
idxExec := &tipb.IndexScan{
TableId: e.table.ID,
IndexId: e.index.ID,
- Columns: util.ColumnsToProto(e.cols, e.table.PKIsHandle),
+ Columns: util.ColumnsToProto(e.cols, e.table.PKIsHandle, true),
}
return &tipb.Executor{Tp: tipb.ExecType_TypeIndexScan, IdxScan: idxExec}
}
@@ -191,6 +192,9 @@ type RecoverIndexExec struct {
srcChunk *chunk.Chunk
handleCols plannercore.HandleCols
+ containsGenedCol bool
+ cols []*expression.Column
+
// below buf is used to reduce allocations.
recoverRows []recoverRows
idxValsBufs [][]types.Datum
@@ -265,10 +269,11 @@ func (e *RecoverIndexExec) buildTableScan(ctx context.Context, txn kv.Transactio
return nil, err
}
var builder distsql.RequestBuilder
- builder.KeyRanges, err = buildRecoverIndexKeyRanges(e.ctx.GetSessionVars().StmtCtx, e.physicalID, startHandle)
+ keyRanges, err := buildRecoverIndexKeyRanges(e.ctx.GetSessionVars().StmtCtx, e.physicalID, startHandle)
if err != nil {
return nil, err
}
+ builder.KeyRanges = kv.NewNonParitionedKeyRanges(keyRanges)
kvReq, err := builder.
SetDAGRequest(dagPB).
SetStartTS(txn.StartTS()).
@@ -378,10 +383,13 @@ func (e *RecoverIndexExec) fetchRecoverRows(ctx context.Context, srcResult dists
if err != nil {
return nil, err
}
- idxVals := extractIdxVals(row, e.idxValsBufs[result.scanRowCount], e.colFieldTypes, idxValLen)
+ idxVals, err := e.buildIndexedValues(row, e.idxValsBufs[result.scanRowCount], e.colFieldTypes, idxValLen)
+ if err != nil {
+ return nil, err
+ }
e.idxValsBufs[result.scanRowCount] = idxVals
rsData := tables.TryGetHandleRestoredDataWrapper(e.table.Meta(), plannercore.GetCommonHandleDatum(e.handleCols, row), nil, e.index.Meta())
- e.recoverRows = append(e.recoverRows, recoverRows{handle: handle, idxVals: idxVals, rsData: rsData, skip: false})
+ e.recoverRows = append(e.recoverRows, recoverRows{handle: handle, idxVals: idxVals, rsData: rsData, skip: true})
result.scanRowCount++
result.currentHandle = handle
}
@@ -390,22 +398,71 @@ func (e *RecoverIndexExec) fetchRecoverRows(ctx context.Context, srcResult dists
return e.recoverRows, nil
}
+func (e *RecoverIndexExec) buildIndexedValues(row chunk.Row, idxVals []types.Datum, fieldTypes []*types.FieldType, idxValLen int) ([]types.Datum, error) {
+ if !e.containsGenedCol {
+ return extractIdxVals(row, idxVals, fieldTypes, idxValLen), nil
+ }
+
+ if e.cols == nil {
+ columns, _, err := expression.ColumnInfos2ColumnsAndNames(e.ctx, model.NewCIStr("mock"), e.table.Meta().Name, e.table.Meta().Columns, e.table.Meta())
+ if err != nil {
+ return nil, err
+ }
+ e.cols = columns
+ }
+
+ if cap(idxVals) < idxValLen {
+ idxVals = make([]types.Datum, idxValLen)
+ } else {
+ idxVals = idxVals[:idxValLen]
+ }
+
+ for i, col := range e.index.Meta().Columns {
+ if e.table.Meta().Columns[col.Offset].IsGenerated() {
+ val, err := e.cols[col.Offset].EvalVirtualColumn(row)
+ if err != nil {
+ return nil, err
+ }
+ val.Copy(&idxVals[i])
+ } else {
+ val := row.GetDatum(col.Offset, &(e.table.Meta().Columns[col.Offset].FieldType))
+ val.Copy(&idxVals[i])
+ }
+ }
+ return idxVals, nil
+}
+
func (e *RecoverIndexExec) batchMarkDup(txn kv.Transaction, rows []recoverRows) error {
if len(rows) == 0 {
return nil
}
e.batchKeys = e.batchKeys[:0]
sc := e.ctx.GetSessionVars().StmtCtx
- distinctFlags := make([]bool, len(rows))
+ distinctFlags := make([]bool, 0, len(rows))
+ rowIdx := make([]int, 0, len(rows))
+ cnt := 0
for i, row := range rows {
- idxKey, distinct, err := e.index.GenIndexKey(sc, row.idxVals, row.handle, e.idxKeyBufs[i])
- if err != nil {
- return err
- }
- e.idxKeyBufs[i] = idxKey
+ iter := e.index.GenIndexKVIter(sc, row.idxVals, row.handle, nil)
+ for iter.Valid() {
+ var buf []byte
+ if cnt < len(e.idxKeyBufs) {
+ buf = e.idxKeyBufs[cnt]
+ }
+ key, _, distinct, err := iter.Next(buf)
+ if err != nil {
+ return err
+ }
+ if cnt < len(e.idxKeyBufs) {
+ e.idxKeyBufs[cnt] = key
+ } else {
+ e.idxKeyBufs = append(e.idxKeyBufs, key)
+ }
- e.batchKeys = append(e.batchKeys, idxKey)
- distinctFlags[i] = distinct
+ cnt++
+ e.batchKeys = append(e.batchKeys, key)
+ distinctFlags = append(distinctFlags, distinct)
+ rowIdx = append(rowIdx, i)
+ }
}
values, err := txn.BatchGet(context.Background(), e.batchKeys)
@@ -418,21 +475,22 @@ func (e *RecoverIndexExec) batchMarkDup(txn kv.Transaction, rows []recoverRows)
// 3. non-unique-key is duplicate, skip it.
isCommonHandle := e.table.Meta().IsCommonHandle
for i, key := range e.batchKeys {
- if val, found := values[string(key)]; found {
+ val, found := values[string(key)]
+ if found {
if distinctFlags[i] {
handle, err1 := tablecodec.DecodeHandleInUniqueIndexValue(val, isCommonHandle)
if err1 != nil {
return err1
}
- if handle.Compare(rows[i].handle) != 0 {
+ if handle.Compare(rows[rowIdx[i]].handle) != 0 {
logutil.BgLogger().Warn("recover index: the constraint of unique index is broken, handle in index is not equal to handle in table",
zap.String("index", e.index.Meta().Name.O), zap.ByteString("indexKey", key),
- zap.Stringer("handleInTable", rows[i].handle), zap.Stringer("handleInIndex", handle))
+ zap.Stringer("handleInTable", rows[rowIdx[i]].handle), zap.Stringer("handleInIndex", handle))
}
}
- rows[i].skip = true
}
+ rows[rowIdx[i]].skip = found && rows[rowIdx[i]].skip
}
return nil
}
@@ -550,7 +608,7 @@ func (e *CleanupIndexExec) getIdxColTypes() []*types.FieldType {
}
e.idxColFieldTypes = make([]*types.FieldType, 0, len(e.columns))
for _, col := range e.columns {
- e.idxColFieldTypes = append(e.idxColFieldTypes, &col.FieldType)
+ e.idxColFieldTypes = append(e.idxColFieldTypes, col.FieldType.ArrayType())
}
return e.idxColFieldTypes
}
@@ -737,7 +795,16 @@ func (e *CleanupIndexExec) buildIndexScan(ctx context.Context, txn kv.Transactio
sc := e.ctx.GetSessionVars().StmtCtx
var builder distsql.RequestBuilder
ranges := ranger.FullRange()
- kvReq, err := builder.SetIndexRanges(sc, e.physicalID, e.index.Meta().ID, ranges).
+ keyRanges, err := distsql.IndexRangesToKVRanges(sc, e.physicalID, e.index.Meta().ID, ranges, nil)
+ if err != nil {
+ return nil, err
+ }
+ err = keyRanges.SetToNonPartitioned()
+ if err != nil {
+ return nil, err
+ }
+ keyRanges.FirstPartitionRange()[0].StartKey = kv.Key(e.lastIdxKey).PrefixNext()
+ kvReq, err := builder.SetWrappedKeyRanges(keyRanges).
SetDAGRequest(dagPB).
SetStartTS(txn.StartTS()).
SetKeepOrder(true).
@@ -748,7 +815,6 @@ func (e *CleanupIndexExec) buildIndexScan(ctx context.Context, txn kv.Transactio
return nil, err
}
- kvReq.KeyRanges[0].StartKey = kv.Key(e.lastIdxKey).PrefixNext()
kvReq.Concurrency = 1
result, err := distsql.Select(ctx, e.ctx, kvReq, e.getIdxColTypes(), statistics.NewQueryFeedback(0, nil, 0, false))
if err != nil {
@@ -805,7 +871,7 @@ func (e *CleanupIndexExec) constructIndexScanPB() *tipb.Executor {
idxExec := &tipb.IndexScan{
TableId: e.physicalID,
IndexId: e.index.Meta().ID,
- Columns: util.ColumnsToProto(e.columns, e.table.Meta().PKIsHandle),
+ Columns: util.ColumnsToProto(e.columns, e.table.Meta().PKIsHandle, true),
PrimaryColumnIds: tables.TryGetCommonPkColumnIds(e.table.Meta()),
}
return &tipb.Executor{Tp: tipb.ExecType_TypeIndexScan, IdxScan: idxExec}
diff --git a/executor/admin_test.go b/executor/admin_test.go
index 23b57e9c316b6..b60c2d44ca2f1 100644
--- a/executor/admin_test.go
+++ b/executor/admin_test.go
@@ -133,7 +133,7 @@ func TestAdminCheckIndexInLocalTemporaryMode(t *testing.T) {
tk.MustExec("drop table if exists local_temporary_admin_test;")
tk.MustExec("create temporary table local_temporary_admin_test (c1 int, c2 int, c3 int default 1, primary key (c1), index (c1), unique key(c2))")
tk.MustExec("insert local_temporary_admin_test (c1, c2) values (1,1), (2,2), (3,3);")
- _, err := tk.Exec("admin check table local_temporary_admin_test;")
+ err := tk.ExecToErr("admin check table local_temporary_admin_test;")
require.EqualError(t, err, core.ErrOptOnTemporaryTable.GenWithStackByArgs("admin check table").Error())
tk.MustExec("drop table if exists temporary_admin_test;")
@@ -302,6 +302,131 @@ func TestAdminRecoverIndex(t *testing.T) {
tk.MustExec("admin check index admin_test c2")
tk.MustExec("admin check table admin_test")
+
+ tk.MustExec("drop table if exists admin_test")
+ tk.MustExec("create table admin_test (c1 int, c2 int, c3 int default 1, primary key(c1), unique key i1((c2+1)))")
+ tk.MustExec("insert admin_test (c1, c2) values (1, 1), (2, 2), (3, 3), (10, 10), (20, 20)")
+ r = tk.MustQuery("admin recover index admin_test i1")
+ r.Check(testkit.Rows("0 5"))
+ tk.MustExec("admin check table admin_test")
+ ctx = mock.NewContext()
+ ctx.Store = store
+ is = domain.InfoSchema()
+ dbName = model.NewCIStr("test")
+ tblName = model.NewCIStr("admin_test")
+ tbl, err = is.TableByName(dbName, tblName)
+ require.NoError(t, err)
+
+ tblInfo = tbl.Meta()
+ idxInfo = tblInfo.FindIndexByName("i1")
+ indexOpr = tables.NewIndex(tblInfo.ID, tblInfo, idxInfo)
+ sc = ctx.GetSessionVars().StmtCtx
+ txn, err = store.Begin()
+ require.NoError(t, err)
+ err = indexOpr.Delete(sc, txn, types.MakeDatums(2), kv.IntHandle(1))
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(i1)")
+ r.Check(testkit.Rows("4"))
+ err = tk.ExecToErr("admin check table admin_test")
+ require.Error(t, err)
+ r = tk.MustQuery("admin recover index admin_test i1")
+ r.Check(testkit.Rows("1 5"))
+ tk.MustExec("admin check table admin_test")
+
+ tk.MustExec("drop table if exists admin_test")
+ tk.MustExec("create table admin_test (c1 int, c2 int, c3 int default 1, primary key(c1), unique key i1(c1, c2));")
+ tk.MustExec("insert admin_test (c1, c2) values (1, 1), (2, 2), (3, 3), (10, 10), (20, 20);")
+ tk.MustExec("admin recover index admin_test i1;")
+}
+
+func TestAdminRecoverMVIndex(t *testing.T) {
+ store, domain := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(pk int primary key, a json, index idx((cast(a as signed array))))")
+ tk.MustExec("insert into t values (0, '[0,1,2]')")
+ tk.MustExec("insert into t values (1, '[1,2,3]')")
+ tk.MustExec("insert into t values (2, '[2,3,4]')")
+ tk.MustExec("insert into t values (3, '[3,4,5]')")
+ tk.MustExec("insert into t values (4, '[4,5,6]')")
+ tk.MustExec("admin check table t")
+
+ ctx := mock.NewContext()
+ ctx.Store = store
+ is := domain.InfoSchema()
+ dbName := model.NewCIStr("test")
+ tblName := model.NewCIStr("t")
+ tbl, err := is.TableByName(dbName, tblName)
+ require.NoError(t, err)
+ tblInfo := tbl.Meta()
+ idxInfo := tblInfo.Indices[0]
+ tk.Session().GetSessionVars().IndexLookupSize = 3
+ tk.Session().GetSessionVars().MaxChunkSize = 3
+
+ cpIdx := idxInfo.Clone()
+ cpIdx.MVIndex = false
+ indexOpr := tables.NewIndex(tblInfo.ID, tblInfo, cpIdx)
+
+ txn, err := store.Begin()
+ require.NoError(t, err)
+ err = indexOpr.Delete(ctx.GetSessionVars().StmtCtx, txn, types.MakeDatums(2), kv.IntHandle(1))
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table t")
+ require.Error(t, err)
+ r := tk.MustQuery("admin recover index t idx")
+ r.Check(testkit.Rows("1 5"))
+ tk.MustExec("admin check table t")
+}
+
+func TestAdminCleanupMVIndex(t *testing.T) {
+ store, domain := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(pk int primary key, a json, index idx((cast(a as signed array))))")
+ tk.MustExec("insert into t values (0, '[0,1,2]')")
+ tk.MustExec("insert into t values (1, '[1,2,3]')")
+ tk.MustExec("insert into t values (2, '[2,3,4]')")
+ tk.MustExec("insert into t values (3, '[3,4,5]')")
+ tk.MustExec("insert into t values (4, '[4,5,6]')")
+ tk.MustExec("admin check table t")
+
+ // Make some corrupted index. Build the index information.
+ ctx := mock.NewContext()
+ ctx.Store = store
+ is := domain.InfoSchema()
+ dbName := model.NewCIStr("test")
+ tblName := model.NewCIStr("t")
+ tbl, err := is.TableByName(dbName, tblName)
+ require.NoError(t, err)
+ tblInfo := tbl.Meta()
+ idxInfo := tblInfo.Indices[0]
+ tk.Session().GetSessionVars().IndexLookupSize = 3
+ tk.Session().GetSessionVars().MaxChunkSize = 3
+
+ cpIdx := idxInfo.Clone()
+ cpIdx.MVIndex = false
+ indexOpr := tables.NewIndex(tblInfo.ID, tblInfo, cpIdx)
+
+ txn, err := store.Begin()
+ require.NoError(t, err)
+ _, err = indexOpr.Create(ctx, txn, types.MakeDatums(9), kv.IntHandle(9), nil)
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table t")
+ require.Error(t, err)
+
+ r := tk.MustQuery("admin cleanup index t idx")
+ r.Check(testkit.Rows("1"))
+ tk.MustExec("admin check table t")
}
func TestClusteredIndexAdminRecoverIndex(t *testing.T) {
@@ -843,6 +968,65 @@ func TestClusteredAdminCleanupIndex(t *testing.T) {
tk.MustExec("admin check table admin_test")
}
+func TestAdminCheckTableWithMultiValuedIndex(t *testing.T) {
+ store, domain := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(pk int primary key, a json, index idx((cast(a as signed array))))")
+ tk.MustExec("insert into t values (0, '[0,1,2]')")
+ tk.MustExec("insert into t values (1, '[1,2,3]')")
+ tk.MustExec("insert into t values (2, '[2,3,4]')")
+ tk.MustExec("insert into t values (3, '[3,4,5]')")
+ tk.MustExec("insert into t values (4, '[4,5,6]')")
+ tk.MustExec("admin check table t")
+
+ // Make some corrupted index. Build the index information.
+ ctx := mock.NewContext()
+ ctx.Store = store
+ is := domain.InfoSchema()
+ dbName := model.NewCIStr("test")
+ tblName := model.NewCIStr("t")
+ tbl, err := is.TableByName(dbName, tblName)
+ require.NoError(t, err)
+ tblInfo := tbl.Meta()
+ idxInfo := tblInfo.Indices[0]
+ sc := ctx.GetSessionVars().StmtCtx
+ tk.Session().GetSessionVars().IndexLookupSize = 3
+ tk.Session().GetSessionVars().MaxChunkSize = 3
+
+ cpIdx := idxInfo.Clone()
+ cpIdx.MVIndex = false
+ indexOpr := tables.NewIndex(tblInfo.ID, tblInfo, cpIdx)
+ txn, err := store.Begin()
+ require.NoError(t, err)
+ err = indexOpr.Delete(sc, txn, types.MakeDatums(0), kv.IntHandle(0))
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table t")
+ require.Error(t, err)
+ require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
+
+ txn, err = store.Begin()
+ require.NoError(t, err)
+ _, err = indexOpr.Create(ctx, txn, types.MakeDatums(0), kv.IntHandle(0), nil)
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ tk.MustExec("admin check table t")
+
+ txn, err = store.Begin()
+ require.NoError(t, err)
+ _, err = indexOpr.Create(ctx, txn, types.MakeDatums(9), kv.IntHandle(9), nil)
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table t")
+ require.Error(t, err)
+}
+
func TestAdminCheckPartitionTableFailed(t *testing.T) {
store, domain := testkit.CreateMockStoreAndDomain(t)
@@ -1095,9 +1279,7 @@ func TestCheckFailReport(t *testing.T) {
require.NoError(t, txn.Commit(tk.ctx))
ctx, hook := withLogHook(tk.ctx, t, "inconsistency")
- _, err = tk.Exec(ctx, "admin check table admin_test")
- require.Error(t, err)
- require.Equal(t, "[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 1, index-values:\"\" != record-values:\"handle: 1, values: [KindInt64 1]\"", err.Error())
+ tk.MustGetErrMsg(ctx, "admin check table admin_test", "[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 1, index-values:\"\" != record-values:\"handle: 1, values: [KindInt64 1]\"")
hook.checkLogCount(t, 1)
hook.logs[0].checkMsg(t, "admin check found data inconsistency")
hook.logs[0].checkField(t,
@@ -1119,9 +1301,7 @@ func TestCheckFailReport(t *testing.T) {
require.NoError(t, txn.Commit(tk.ctx))
ctx, hook := withLogHook(tk.ctx, t, "inconsistency")
- _, err = tk.Exec(ctx, "admin check table admin_test")
- require.Error(t, err)
- require.Equal(t, "[admin:8223]data inconsistency in table: admin_test, index: k2, handle: 1, index-values:\"\" != record-values:\"handle: 1, values: [KindString 10]\"", err.Error())
+ tk.MustGetErrMsg(ctx, "admin check table admin_test", "[admin:8223]data inconsistency in table: admin_test, index: k2, handle: 1, index-values:\"\" != record-values:\"handle: 1, values: [KindString 10]\"")
hook.checkLogCount(t, 1)
hook.logs[0].checkMsg(t, "admin check found data inconsistency")
hook.logs[0].checkField(t,
@@ -1143,9 +1323,8 @@ func TestCheckFailReport(t *testing.T) {
require.NoError(t, txn.Commit(tk.ctx))
ctx, hook := withLogHook(tk.ctx, t, "inconsistency")
- _, err = tk.Exec(ctx, "admin check table admin_test")
- require.Error(t, err)
- require.Equal(t, "[admin:8223]data inconsistency in table: admin_test, index: k2, handle: 1, index-values:\"handle: 1, values: [KindString 100 KindInt64 1]\" != record-values:\"\"", err.Error())
+ tk.MustGetErrMsg(ctx, "admin check table admin_test",
+ "[admin:8223]data inconsistency in table: admin_test, index: k2, handle: 1, index-values:\"handle: 1, values: [KindString 100 KindInt64 1]\" != record-values:\"\"")
hook.checkLogCount(t, 1)
logEntry := hook.logs[0]
logEntry.checkMsg(t, "admin check found data inconsistency")
@@ -1188,9 +1367,8 @@ func TestCheckFailReport(t *testing.T) {
require.NoError(t, txn.Commit(tk.ctx))
ctx, hook := withLogHook(tk.ctx, t, "inconsistency")
- _, err = tk.Exec(ctx, "admin check table admin_test")
- require.Error(t, err)
- require.Equal(t, "[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 1, index-values:\"handle: 1, values: [KindInt64 10 KindInt64 1]\" != record-values:\"\"", err.Error())
+ tk.MustGetErrMsg(ctx, "admin check table admin_test",
+ "[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 1, index-values:\"handle: 1, values: [KindInt64 10 KindInt64 1]\" != record-values:\"\"")
hook.checkLogCount(t, 1)
logEntry := hook.logs[0]
logEntry.checkMsg(t, "admin check found data inconsistency")
@@ -1233,9 +1411,8 @@ func TestCheckFailReport(t *testing.T) {
require.NoError(t, err)
require.NoError(t, txn.Commit(tk.ctx))
ctx, hook := withLogHook(tk.ctx, t, "inconsistency")
- _, err = tk.Exec(ctx, "admin check table admin_test")
- require.Error(t, err)
- require.Equal(t, "[executor:8134]data inconsistency in table: admin_test, index: uk1, col: c2, handle: \"1\", index-values:\"KindInt64 20\" != record-values:\"KindInt64 10\", compare err:", err.Error())
+ tk.MustGetErrMsg(ctx, "admin check table admin_test",
+ "[executor:8134]data inconsistency in table: admin_test, index: uk1, col: c2, handle: \"1\", index-values:\"KindInt64 20\" != record-values:\"KindInt64 10\", compare err:")
hook.checkLogCount(t, 1)
logEntry := hook.logs[0]
logEntry.checkMsg(t, "admin check found data inconsistency")
@@ -1261,9 +1438,8 @@ func TestCheckFailReport(t *testing.T) {
require.NoError(t, err)
require.NoError(t, txn.Commit(tk.ctx))
ctx, hook := withLogHook(tk.ctx, t, "inconsistency")
- _, err = tk.Exec(ctx, "admin check table admin_test")
- require.Error(t, err)
- require.Equal(t, "[executor:8134]data inconsistency in table: admin_test, index: k2, col: c3, handle: \"1\", index-values:\"KindString 200\" != record-values:\"KindString 100\", compare err:", err.Error())
+ tk.MustGetErrMsg(ctx, "admin check table admin_test",
+ "[executor:8134]data inconsistency in table: admin_test, index: k2, col: c3, handle: \"1\", index-values:\"KindString 200\" != record-values:\"KindString 100\", compare err:")
hook.checkLogCount(t, 1)
logEntry := hook.logs[0]
logEntry.checkMsg(t, "admin check found data inconsistency")
@@ -1301,12 +1477,10 @@ func TestCheckFailReport(t *testing.T) {
// TODO(tiancaiamao): admin check doesn't support the chunk protocol.
// Remove this after https://github.com/pingcap/tidb/issues/35156
- _, err = tk.Exec(ctx, "set @@tidb_enable_chunk_rpc = off")
- require.NoError(t, err)
+ tk.MustExec(ctx, "set @@tidb_enable_chunk_rpc = off")
- _, err = tk.Exec(ctx, "admin check table admin_test")
- require.Error(t, err)
- require.Equal(t, `[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 282574488403969, index-values:"handle: 282574488403969, values: [KindInt64 282578800083201 KindInt64 282574488403969]" != record-values:""`, err.Error())
+ tk.MustGetErrMsg(ctx, "admin check table admin_test",
+ `[admin:8223]data inconsistency in table: admin_test, index: uk1, handle: 282574488403969, index-values:"handle: 282574488403969, values: [KindInt64 282578800083201 KindInt64 282574488403969]" != record-values:""`)
hook.checkLogCount(t, 1)
logEntry := hook.logs[0]
logEntry.checkMsg(t, "admin check found data inconsistency")
diff --git a/executor/aggfuncs/BUILD.bazel b/executor/aggfuncs/BUILD.bazel
index 5c01950eef836..ce1aa103869f0 100644
--- a/executor/aggfuncs/BUILD.bazel
+++ b/executor/aggfuncs/BUILD.bazel
@@ -58,7 +58,7 @@ go_library(
go_test(
name = "aggfuncs_test",
- timeout = "short",
+ timeout = "moderate",
srcs = [
"aggfunc_test.go",
"export_test.go",
@@ -89,7 +89,7 @@ go_test(
embed = [":aggfuncs"],
flaky = True,
race = "on",
- shard_count = 10,
+ shard_count = 40,
deps = [
"//expression",
"//expression/aggregation",
diff --git a/executor/aggregate.go b/executor/aggregate.go
index 771d928c9bbad..30b86164ec371 100644
--- a/executor/aggregate.go
+++ b/executor/aggregate.go
@@ -244,6 +244,9 @@ func (d *HashAggIntermData) getPartialResultBatch(_ *stmtctx.StatementContext, p
// Close implements the Executor Close interface.
func (e *HashAggExec) Close() error {
+ if e.stats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
+ }
if e.isUnparallelExec {
var firstErr error
e.childResult = nil
@@ -1131,7 +1134,6 @@ func (e *HashAggExec) initRuntimeStats() {
stats.PartialStats = make([]*AggWorkerStat, 0, stats.PartialConcurrency)
stats.FinalStats = make([]*AggWorkerStat, 0, stats.FinalConcurrency)
e.stats = stats
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
}
diff --git a/executor/aggregate_test.go b/executor/aggregate_test.go
index 1868802d65d8d..bd6366d6d115d 100644
--- a/executor/aggregate_test.go
+++ b/executor/aggregate_test.go
@@ -1088,6 +1088,14 @@ func TestIssue10608(t *testing.T) {
tk.MustExec("insert into t values(508931), (508932)")
tk.MustQuery("select (select /*+ stream_agg() */ group_concat(concat(123,'-')) from t where t.a = s.b group by t.a) as t from s;").Check(testkit.Rows("123-", "123-"))
tk.MustQuery("select (select /*+ hash_agg() */ group_concat(concat(123,'-')) from t where t.a = s.b group by t.a) as t from s;").Check(testkit.Rows("123-", "123-"))
+
+ tk.MustExec("CREATE TABLE `t49`(`c0` char(1) DEFAULT '1', `c2` char(1) DEFAULT NULL, UNIQUE KEY `c2` (`c2`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;")
+ tk.MustExec("INSERT INTO `t49` VALUES ('0','0'),('0','1');")
+ tk.MustExec("CREATE TABLE `t0` (`c0` blob DEFAULT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;")
+ tk.MustExec("INSERT INTO `t0` VALUES (_binary ']'),(_binary '777926278'),(_binary '0.2136404982804636'),(_binary '1901362489'),(_binary '1558203848'),(''),(_binary '1830406335'),(''),(_binary '0'),(NULL),(_binary '601930250'),(_binary '1558203848'),(_binary '-122008948'),(_binary '-2053608489'),(_binary 'hb/vt <7'),(_binary 'RC&2*'),(_binary '1'),(_binary '-1722334316'),(_binary '1830406335'),(_binary '1372126029'),(_binary '882291196'),(NULL),(_binary '-399693596');")
+ tk.MustExec("CREATE ALGORITHM=TEMPTABLE DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v0` (`c0`, `c1`, `c2`) AS SELECT NULL AS `NULL`,`t49`.`c2` AS `c2`,(((CASE _UTF8MB4'I되EkfIO퀶' WHEN NULL THEN `t49`.`c0` WHEN `t49`.`c2` THEN `t0`.`c0` ELSE (CASE `t49`.`c0` WHEN _UTF8MB4'%' THEN 1035293362 ELSE _UTF8MB4',' END) END))<<(`t49`.`c0`)) AS `(((CASE 'I되EkfIO퀶' WHEN NULL THEN t49.c0 WHEN t49.c2 THEN t0.c0 ELSE (CASE t49.c0 WHEN '%' THEN 1035293362 ELSE ',' END ) END ))<<(t49.c0))` FROM (`t0`) JOIN `t49` WHERE TRUE;")
+ tk.MustQuery("SELECT /*+ STREAM_AGG()*/v0.c0 FROM t49, v0 LEFT OUTER JOIN t0 ON ('Iw') GROUP BY true;").
+ Check(testkit.Rows(""))
}
func TestIssue12759HashAggCalledByApply(t *testing.T) {
diff --git a/executor/analyze.go b/executor/analyze.go
index da74d8248a90c..705e6eed6c590 100644
--- a/executor/analyze.go
+++ b/executor/analyze.go
@@ -267,20 +267,8 @@ func recordHistoricalStats(sctx sessionctx.Context, tableID int64) error {
if !historicalStatsEnabled {
return nil
}
-
- is := domain.GetDomain(sctx).InfoSchema()
- tbl, existed := is.TableByID(tableID)
- if !existed {
- return errors.Errorf("cannot get table by id %d", tableID)
- }
- tblInfo := tbl.Meta()
- dbInfo, existed := is.SchemaByTable(tblInfo)
- if !existed {
- return errors.Errorf("cannot get DBInfo by TableID %d", tableID)
- }
- if _, err := statsHandle.RecordHistoricalStatsToStorage(dbInfo.Name.O, tblInfo); err != nil {
- return errors.Errorf("record table %s.%s's historical stats failed", dbInfo.Name.O, tblInfo.Name.O)
- }
+ historicalStatsWorker := domain.GetDomain(sctx).GetHistoricalStatsWorker()
+ historicalStatsWorker.SendTblToDumpHistoricalStats(tableID)
return nil
}
@@ -303,6 +291,8 @@ func (e *AnalyzeExec) handleResultsError(ctx context.Context, concurrency int, n
}
}
+ tableIDs := map[int64]struct{}{}
+
// save analyze results in single-thread.
statsHandle := domain.GetDomain(e.ctx).StatsHandle()
panicCnt := 0
@@ -323,17 +313,15 @@ func (e *AnalyzeExec) handleResultsError(ctx context.Context, concurrency int, n
continue
}
handleGlobalStats(needGlobalStats, globalStatsMap, results)
+ tableIDs[results.TableID.GetStatisticsID()] = struct{}{}
- if err1 := statsHandle.SaveTableStatsToStorage(results, e.ctx.GetSessionVars().EnableAnalyzeSnapshot); err1 != nil {
+ if err1 := statsHandle.SaveTableStatsToStorage(results, e.ctx.GetSessionVars().EnableAnalyzeSnapshot, handle.StatsMetaHistorySourceAnalyze); err1 != nil {
+ tableID := results.TableID.TableID
err = err1
- logutil.Logger(ctx).Error("save table stats to storage failed", zap.Error(err))
+ logutil.Logger(ctx).Error("save table stats to storage failed", zap.Error(err), zap.Int64("tableID", tableID))
finishJobWithLog(e.ctx, results.Job, err)
} else {
finishJobWithLog(e.ctx, results.Job, nil)
- // Dump stats to historical storage.
- if err := recordHistoricalStats(e.ctx, results.TableID.TableID); err != nil {
- logutil.BgLogger().Error("record historical stats failed", zap.Error(err))
- }
}
invalidInfoSchemaStatCache(results.TableID.GetStatisticsID())
if atomic.LoadUint32(&e.ctx.GetSessionVars().Killed) == 1 {
@@ -341,6 +329,13 @@ func (e *AnalyzeExec) handleResultsError(ctx context.Context, concurrency int, n
return errors.Trace(ErrQueryInterrupted)
}
}
+ // Dump stats to historical storage.
+ for tableID := range tableIDs {
+ if err := recordHistoricalStats(e.ctx, tableID); err != nil {
+ logutil.BgLogger().Error("record historical stats failed", zap.Error(err))
+ }
+ }
+
return err
}
@@ -359,6 +354,7 @@ func (e *AnalyzeExec) handleResultsErrorWithConcurrency(ctx context.Context, sta
worker.run(ctx1, e.ctx.GetSessionVars().EnableAnalyzeSnapshot)
})
}
+ tableIDs := map[int64]struct{}{}
panicCnt := 0
var err error
for panicCnt < statsConcurrency {
@@ -381,6 +377,7 @@ func (e *AnalyzeExec) handleResultsErrorWithConcurrency(ctx context.Context, sta
continue
}
handleGlobalStats(needGlobalStats, globalStatsMap, results)
+ tableIDs[results.TableID.GetStatisticsID()] = struct{}{}
saveResultsCh <- results
}
close(saveResultsCh)
@@ -393,6 +390,12 @@ func (e *AnalyzeExec) handleResultsErrorWithConcurrency(ctx context.Context, sta
}
err = errors.New(strings.Join(errMsg, ","))
}
+ for tableID := range tableIDs {
+ // Dump stats to historical storage.
+ if err := recordHistoricalStats(e.ctx, tableID); err != nil {
+ logutil.BgLogger().Error("record historical stats failed", zap.Error(err))
+ }
+ }
return err
}
diff --git a/executor/analyze_col.go b/executor/analyze_col.go
index a846816b18428..dc5194e9f8fa9 100644
--- a/executor/analyze_col.go
+++ b/executor/analyze_col.go
@@ -123,6 +123,7 @@ func (e *AnalyzeColumnsExec) buildResp(ranges []*ranger.Range) (distsql.SelectRe
SetKeepOrder(true).
SetConcurrency(e.concurrency).
SetMemTracker(e.memTracker).
+ SetResourceGroupName(e.ctx.GetSessionVars().ResourceGroupName).
Build()
if err != nil {
return nil, err
diff --git a/executor/analyze_col_v2.go b/executor/analyze_col_v2.go
index 68a02485c0048..1d9913d5f23e3 100644
--- a/executor/analyze_col_v2.go
+++ b/executor/analyze_col_v2.go
@@ -32,6 +32,7 @@ import (
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/statistics"
+ "github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util"
@@ -187,7 +188,7 @@ func (e *AnalyzeColumnsExecV2) decodeSampleDataWithVirtualColumn(
}
}
}
- err := FillVirtualColumnValue(fieldTps, virtualColIdx, schema, e.colsInfo, e.ctx, chk)
+ err := table.FillVirtualColumnValue(fieldTps, virtualColIdx, schema.Columns, e.colsInfo, e.ctx, chk)
if err != nil {
return err
}
@@ -199,6 +200,27 @@ func (e *AnalyzeColumnsExecV2) decodeSampleDataWithVirtualColumn(
return nil
}
+func printAnalyzeMergeCollectorLog(oldRootCount, newRootCount, subCount, tableID, partitionID int64, isPartition bool, info string, index int) {
+ if index < 0 {
+ logutil.BgLogger().Debug(info,
+ zap.Int64("tableID", tableID),
+ zap.Int64("partitionID", partitionID),
+ zap.Bool("isPartitionTable", isPartition),
+ zap.Int64("oldRootCount", oldRootCount),
+ zap.Int64("newRootCount", newRootCount),
+ zap.Int64("subCount", subCount))
+ } else {
+ logutil.BgLogger().Debug(info,
+ zap.Int64("tableID", tableID),
+ zap.Int64("partitionID", partitionID),
+ zap.Bool("isPartitionTable", isPartition),
+ zap.Int64("oldRootCount", oldRootCount),
+ zap.Int64("newRootCount", newRootCount),
+ zap.Int64("subCount", subCount),
+ zap.Int("subCollectorIndex", index))
+ }
+}
+
func (e *AnalyzeColumnsExecV2) buildSamplingStats(
ranges []*ranger.Range,
needExtStats bool,
@@ -235,7 +257,7 @@ func (e *AnalyzeColumnsExecV2) buildSamplingStats(
e.samplingMergeWg = &util.WaitGroupWrapper{}
e.samplingMergeWg.Add(statsConcurrency)
for i := 0; i < statsConcurrency; i++ {
- go e.subMergeWorker(mergeResultCh, mergeTaskCh, l, i == 0)
+ go e.subMergeWorker(mergeResultCh, mergeTaskCh, l, i)
}
if err = readDataAndSendTask(e.ctx, e.resultHandler, mergeTaskCh, e.memTracker); err != nil {
return 0, nil, nil, nil, nil, getAnalyzePanicErr(err)
@@ -255,7 +277,12 @@ func (e *AnalyzeColumnsExecV2) buildSamplingStats(
continue
}
oldRootCollectorSize := rootRowCollector.Base().MemSize
+ oldRootCollectorCount := rootRowCollector.Base().Count
rootRowCollector.MergeCollector(mergeResult.collector)
+ newRootCollectorCount := rootRowCollector.Base().Count
+ printAnalyzeMergeCollectorLog(oldRootCollectorCount, newRootCollectorCount,
+ mergeResult.collector.Base().Count, e.tableID.TableID, e.tableID.PartitionID, e.tableID.IsPartitionTable(),
+ "merge subMergeWorker in AnalyzeColumnsExecV2", -1)
e.memTracker.Consume(rootRowCollector.Base().MemSize - oldRootCollectorSize - mergeResult.collector.Base().MemSize)
}
defer e.memTracker.Release(rootRowCollector.Base().MemSize)
@@ -544,7 +571,8 @@ func (e *AnalyzeColumnsExecV2) buildSubIndexJobForSpecialIndex(indexInfos []*mod
return tasks
}
-func (e *AnalyzeColumnsExecV2) subMergeWorker(resultCh chan<- *samplingMergeResult, taskCh <-chan []byte, l int, isClosedChanThread bool) {
+func (e *AnalyzeColumnsExecV2) subMergeWorker(resultCh chan<- *samplingMergeResult, taskCh <-chan []byte, l int, index int) {
+ isClosedChanThread := index == 0
defer func() {
if r := recover(); r != nil {
logutil.BgLogger().Error("analyze worker panicked", zap.Any("recover", r), zap.Stack("stack"))
@@ -567,6 +595,13 @@ func (e *AnalyzeColumnsExecV2) subMergeWorker(resultCh chan<- *samplingMergeResu
failpoint.Inject("mockAnalyzeSamplingMergeWorkerPanic", func() {
panic("failpoint triggered")
})
+ failpoint.Inject("mockAnalyzeMergeWorkerSlowConsume", func(val failpoint.Value) {
+ times := val.(int)
+ for i := 0; i < times; i++ {
+ e.memTracker.Consume(5 << 20)
+ time.Sleep(100 * time.Millisecond)
+ }
+ })
retCollector := statistics.NewRowSampleCollector(int(e.analyzePB.ColReq.SampleSize), e.analyzePB.ColReq.GetSampleRate(), l)
for i := 0; i < l; i++ {
retCollector.Base().FMSketches = append(retCollector.Base().FMSketches, statistics.NewFMSketch(maxSketchSize))
@@ -589,7 +624,12 @@ func (e *AnalyzeColumnsExecV2) subMergeWorker(resultCh chan<- *samplingMergeResu
subCollector.Base().FromProto(colResp.RowCollector, e.memTracker)
UpdateAnalyzeJob(e.ctx, e.job, subCollector.Base().Count)
oldRetCollectorSize := retCollector.Base().MemSize
+ oldRetCollectorCount := retCollector.Base().Count
retCollector.MergeCollector(subCollector)
+ newRetCollectorCount := retCollector.Base().Count
+ printAnalyzeMergeCollectorLog(oldRetCollectorCount, newRetCollectorCount, subCollector.Base().Count,
+ e.tableID.TableID, e.tableID.PartitionID, e.TableID.IsPartitionTable(),
+ "merge subCollector in concurrency in AnalyzeColumnsExecV2", index)
newRetCollectorSize := retCollector.Base().MemSize
subCollectorSize := subCollector.Base().MemSize
e.memTracker.Consume(newRetCollectorSize - oldRetCollectorSize - subCollectorSize)
diff --git a/executor/analyze_fast.go b/executor/analyze_fast.go
index 5917a5b336ae0..b9dcc5d55ee97 100644
--- a/executor/analyze_fast.go
+++ b/executor/analyze_fast.go
@@ -404,6 +404,7 @@ func (e *AnalyzeFastExec) handleScanTasks(bo *tikv.Backoffer) (keysSize int, err
snapshot.SetOption(kv.ReplicaRead, kv.ReplicaReadFollower)
}
setOptionForTopSQL(e.ctx.GetSessionVars().StmtCtx, snapshot)
+ snapshot.SetOption(kv.ResourceGroupName, e.ctx.GetSessionVars().ResourceGroupName)
for _, t := range e.scanTasks {
iter, err := snapshot.Iter(kv.Key(t.StartKey), kv.Key(t.EndKey))
if err != nil {
@@ -430,6 +431,7 @@ func (e *AnalyzeFastExec) handleSampTasks(workID int, step uint32, err *error) {
}
snapshot.SetOption(kv.NotFillCache, true)
snapshot.SetOption(kv.Priority, kv.PriorityLow)
+ snapshot.SetOption(kv.ResourceGroupName, e.ctx.GetSessionVars().ResourceGroupName)
setOptionForTopSQL(e.ctx.GetSessionVars().StmtCtx, snapshot)
readReplicaType := e.ctx.GetSessionVars().GetReplicaRead()
if readReplicaType.IsFollowerRead() {
diff --git a/executor/analyze_global_stats.go b/executor/analyze_global_stats.go
index 82c2678953285..e8f8d53b8adbf 100644
--- a/executor/analyze_global_stats.go
+++ b/executor/analyze_global_stats.go
@@ -21,6 +21,7 @@ import (
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/statistics"
+ "github.com/pingcap/tidb/statistics/handle"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/logutil"
"go.uber.org/zap"
@@ -53,7 +54,9 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, needGlobalStats boo
globalStatsTableIDs[globalStatsID.tableID] = struct{}{}
}
statsHandle := domain.GetDomain(e.ctx).StatsHandle()
+ tableIDs := map[int64]struct{}{}
for tableID := range globalStatsTableIDs {
+ tableIDs[tableID] = struct{}{}
tableAllPartitionStats := make(map[int64]*statistics.Table)
for globalStatsID, info := range globalStatsMap {
if globalStatsID.tableID != tableID {
@@ -73,10 +76,11 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, needGlobalStats boo
globalStatsID.tableID, info.isIndex, info.histIDs,
tableAllPartitionStats)
if err != nil {
+ logutil.BgLogger().Error("merge global stats failed",
+ zap.String("info", job.JobInfo), zap.Error(err), zap.Int64("tableID", tableID))
if types.ErrPartitionStatsMissing.Equal(err) || types.ErrPartitionColumnStatsMissing.Equal(err) {
// When we find some partition-level stats are missing, we need to report warning.
e.ctx.GetSessionVars().StmtCtx.AppendWarning(err)
- return nil
}
return err
}
@@ -93,20 +97,24 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, needGlobalStats boo
info.statsVersion,
1,
true,
+ handle.StatsMetaHistorySourceAnalyze,
)
if err != nil {
- logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.Error(err))
- }
- // Dump stats to historical storage.
- if err := recordHistoricalStats(e.ctx, globalStatsID.tableID); err != nil {
- logutil.BgLogger().Error("record historical stats failed", zap.Error(err))
+ logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.String("info", job.JobInfo),
+ zap.Int64("histID", hg.ID), zap.Error(err), zap.Int64("tableID", tableID))
}
}
- return nil
+ return err
}()
FinishAnalyzeMergeJob(e.ctx, job, mergeStatsErr)
}
}
+ for tableID := range tableIDs {
+ // Dump stats to historical storage.
+ if err := recordHistoricalStats(e.ctx, tableID); err != nil {
+ logutil.BgLogger().Error("record historical stats failed", zap.Error(err))
+ }
+ }
return nil
}
diff --git a/executor/analyze_idx.go b/executor/analyze_idx.go
index 2ac3bce77c139..f89d804788edd 100644
--- a/executor/analyze_idx.go
+++ b/executor/analyze_idx.go
@@ -155,6 +155,7 @@ func (e *AnalyzeIndexExec) fetchAnalyzeResult(ranges []*ranger.Range, isNullRang
SetStartTS(startTS).
SetKeepOrder(true).
SetConcurrency(e.concurrency).
+ SetResourceGroupName(e.ctx.GetSessionVars().ResourceGroupName).
Build()
if err != nil {
return err
diff --git a/executor/analyze_test.go b/executor/analyze_test.go
index bbe3f5b8d1b1e..a6cdea833df50 100644
--- a/executor/analyze_test.go
+++ b/executor/analyze_test.go
@@ -417,3 +417,24 @@ func TestAnalyzePartitionTableByConcurrencyInDynamic(t *testing.T) {
tk.MustQuery("show stats_topn where partition_name = 'global' and table_name = 't'").CheckAt([]int{5, 6}, expected)
}
}
+
+func TestMergeGlobalStatsWithUnAnalyzedPartition(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("set tidb_partition_prune_mode=dynamic;")
+ tk.MustExec("CREATE TABLE `t` ( `id` int(11) DEFAULT NULL, `a` int(11) DEFAULT NULL, `b` int(11) DEFAULT NULL, `c` int(11) DEFAULT NULL ) PARTITION BY RANGE (`id`) (PARTITION `p0` VALUES LESS THAN (3), PARTITION `p1` VALUES LESS THAN (7), PARTITION `p2` VALUES LESS THAN (11));")
+ tk.MustExec("insert into t values (1,1,1,1),(2,2,2,2),(4,4,4,4),(5,5,5,5),(6,6,6,6),(8,8,8,8),(9,9,9,9);")
+ tk.MustExec("create index idxa on t (a);")
+ tk.MustExec("create index idxb on t (b);")
+ tk.MustExec("create index idxc on t (c);")
+ tk.MustExec("analyze table t partition p0 index idxa;")
+ tk.MustExec("analyze table t partition p1 index idxb;")
+ tk.MustExec("analyze table t partition p2 index idxc;")
+ tk.MustQuery("show warnings").Check(testkit.Rows(
+ "Warning 1105 The version 2 would collect all statistics not only the selected indexes",
+ "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p2"))
+ tk.MustExec("analyze table t partition p0;")
+ tk.MustQuery("show warnings").Check(testkit.Rows(
+ "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0"))
+}
diff --git a/executor/analyze_utils.go b/executor/analyze_utils.go
index c6d4886d79b7c..cdf47373d29f0 100644
--- a/executor/analyze_utils.go
+++ b/executor/analyze_utils.go
@@ -17,6 +17,7 @@ package executor
import (
"context"
"strconv"
+ "strings"
"sync"
"github.com/pingcap/errors"
@@ -24,6 +25,7 @@ import (
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/statistics"
+ "github.com/pingcap/tidb/util/memory"
"go.uber.org/atomic"
)
@@ -45,8 +47,13 @@ func isAnalyzeWorkerPanic(err error) bool {
}
func getAnalyzePanicErr(r interface{}) error {
- if msg, ok := r.(string); ok && msg == globalPanicAnalyzeMemoryExceed {
- return errAnalyzeOOM
+ if msg, ok := r.(string); ok {
+ if msg == globalPanicAnalyzeMemoryExceed {
+ return errAnalyzeOOM
+ }
+ if strings.Contains(msg, memory.PanicMemoryExceed) {
+ return errors.Errorf(msg, errAnalyzeOOM)
+ }
}
if err, ok := r.(error); ok {
if err.Error() == globalPanicAnalyzeMemoryExceed {
diff --git a/executor/analyze_worker.go b/executor/analyze_worker.go
index 68c2f6a1f6ec8..688f89f5a120d 100644
--- a/executor/analyze_worker.go
+++ b/executor/analyze_worker.go
@@ -59,17 +59,13 @@ func (worker *analyzeSaveStatsWorker) run(ctx context.Context, analyzeSnapshot b
worker.errCh <- errors.Trace(ErrQueryInterrupted)
return
}
- err := handle.SaveTableStatsToStorage(worker.sctx, results, analyzeSnapshot)
+ err := handle.SaveTableStatsToStorage(worker.sctx, results, analyzeSnapshot, handle.StatsMetaHistorySourceAnalyze)
if err != nil {
logutil.Logger(ctx).Error("save table stats to storage failed", zap.Error(err))
finishJobWithLog(worker.sctx, results.Job, err)
worker.errCh <- err
} else {
finishJobWithLog(worker.sctx, results.Job, nil)
- // Dump stats to historical storage.
- if err := recordHistoricalStats(worker.sctx, results.TableID.TableID); err != nil {
- logutil.BgLogger().Error("record historical stats failed", zap.Error(err))
- }
}
invalidInfoSchemaStatCache(results.TableID.GetStatisticsID())
if err != nil {
diff --git a/executor/analyzetest/BUILD.bazel b/executor/analyzetest/BUILD.bazel
index 53126213363a5..3112abe57c00f 100644
--- a/executor/analyzetest/BUILD.bazel
+++ b/executor/analyzetest/BUILD.bazel
@@ -8,7 +8,6 @@ go_test(
"main_test.go",
],
flaky = True,
- race = "on",
shard_count = 50,
deps = [
"//domain",
@@ -30,6 +29,7 @@ go_test(
"//tablecodec",
"//testkit",
"//types",
+ "//util",
"//util/codec",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
diff --git a/executor/analyzetest/analyze_test.go b/executor/analyzetest/analyze_test.go
index c5043935e1650..843200fea6cf9 100644
--- a/executor/analyzetest/analyze_test.go
+++ b/executor/analyzetest/analyze_test.go
@@ -16,8 +16,8 @@ package analyzetest
import (
"context"
- "encoding/json"
"fmt"
+ "runtime"
"strconv"
"strings"
"testing"
@@ -44,6 +44,7 @@ import (
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/types"
+ "github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/codec"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/testutils"
@@ -2163,102 +2164,6 @@ func TestAnalyzeColumnsErrorAndWarning(t *testing.T) {
}
}
-func TestRecordHistoryStatsAfterAnalyze(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
-
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@tidb_analyze_version = 2")
- tk.MustExec("set global tidb_enable_historical_stats = 0")
- tk.MustExec("use test")
- tk.MustExec("drop table if exists t")
- tk.MustExec("create table t(a int, b varchar(10))")
-
- h := dom.StatsHandle()
- is := dom.InfoSchema()
- tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
- require.NoError(t, err)
-
- // 1. switch off the tidb_enable_historical_stats, and there is no records in table `mysql.stats_history`
- rows := tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", tableInfo.Meta().ID)).Rows()
- num, _ := strconv.Atoi(rows[0][0].(string))
- require.Equal(t, num, 0)
-
- tk.MustExec("analyze table t with 2 topn")
- rows = tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", tableInfo.Meta().ID)).Rows()
- num, _ = strconv.Atoi(rows[0][0].(string))
- require.Equal(t, num, 0)
-
- // 2. switch on the tidb_enable_historical_stats and do analyze
- tk.MustExec("set global tidb_enable_historical_stats = 1")
- defer tk.MustExec("set global tidb_enable_historical_stats = 0")
- tk.MustExec("analyze table t with 2 topn")
- rows = tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", tableInfo.Meta().ID)).Rows()
- num, _ = strconv.Atoi(rows[0][0].(string))
- require.GreaterOrEqual(t, num, 1)
-
- // 3. dump current stats json
- dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil, true)
- require.NoError(t, err)
- jsOrigin, _ := json.Marshal(dumpJSONTable)
-
- // 4. get the historical stats json
- rows = tk.MustQuery(fmt.Sprintf("select * from mysql.stats_history where table_id = '%d' and create_time = ("+
- "select create_time from mysql.stats_history where table_id = '%d' order by create_time desc limit 1) "+
- "order by seq_no", tableInfo.Meta().ID, tableInfo.Meta().ID)).Rows()
- num = len(rows)
- require.GreaterOrEqual(t, num, 1)
- data := make([][]byte, num)
- for i, row := range rows {
- data[i] = []byte(row[1].(string))
- }
- jsonTbl, err := handle.BlocksToJSONTable(data)
- require.NoError(t, err)
- jsCur, err := json.Marshal(jsonTbl)
- require.NoError(t, err)
- // 5. historical stats must be equal to the current stats
- require.JSONEq(t, string(jsOrigin), string(jsCur))
-}
-
-func TestRecordHistoryStatsMetaAfterAnalyze(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
-
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("set @@tidb_analyze_version = 2")
- tk.MustExec("set global tidb_enable_historical_stats = 0")
- tk.MustExec("use test")
- tk.MustExec("drop table if exists t")
- tk.MustExec("create table t(a int, b int)")
- tk.MustExec("analyze table test.t")
-
- h := dom.StatsHandle()
- is := dom.InfoSchema()
- tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
- require.NoError(t, err)
-
- // 1. switch off the tidb_enable_historical_stats, and there is no record in table `mysql.stats_meta_history`
- tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d'", tableInfo.Meta().ID)).Check(testkit.Rows("0"))
- // insert demo tuples, and there is no record either.
- insertNums := 5
- for i := 0; i < insertNums; i++ {
- tk.MustExec("insert into test.t (a,b) values (1,1), (2,2), (3,3)")
- err := h.DumpStatsDeltaToKV(handle.DumpDelta)
- require.NoError(t, err)
- }
- tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d'", tableInfo.Meta().ID)).Check(testkit.Rows("0"))
-
- // 2. switch on the tidb_enable_historical_stats and insert tuples to produce count/modifyCount delta change.
- tk.MustExec("set global tidb_enable_historical_stats = 1")
- defer tk.MustExec("set global tidb_enable_historical_stats = 0")
-
- for i := 0; i < insertNums; i++ {
- tk.MustExec("insert into test.t (a,b) values (1,1), (2,2), (3,3)")
- err := h.DumpStatsDeltaToKV(handle.DumpDelta)
- require.NoError(t, err)
- }
- tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta_history where table_id = '%d' order by create_time", tableInfo.Meta().ID)).Sort().Check(
- testkit.Rows("18 18", "21 21", "24 24", "27 27", "30 30"))
-}
-
func checkAnalyzeStatus(t *testing.T, tk *testkit.TestKit, jobInfo, status, failReason, comment string, timeLimit int64) {
rows := tk.MustQuery("show analyze status where table_schema = 'test' and table_name = 't' and partition_name = ''").Rows()
require.Equal(t, 1, len(rows), comment)
@@ -2836,8 +2741,8 @@ PARTITION BY RANGE ( a ) (
tk.MustQuery("show warnings").Sort().Check(testkit.Rows(
"Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0",
"Warning 1105 Ignore columns and options when analyze partition in dynamic mode",
- "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats",
- "Warning 8131 Build table: `t` index: `idx` global-level stats failed due to missing partition-level stats",
+ "Warning 8131 Build global-level stats failed due to missing partition-level stats: table `t` partition `p1`",
+ "Warning 8131 Build global-level stats failed due to missing partition-level stats: table `t` partition `p1`",
))
tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1")
require.NoError(t, h.LoadNeededHistograms())
@@ -2849,8 +2754,8 @@ PARTITION BY RANGE ( a ) (
tk.MustExec("analyze table t partition p0")
tk.MustQuery("show warnings").Sort().Check(testkit.Rows(
"Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0",
- "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats",
- "Warning 8131 Build table: `t` index: `idx` global-level stats failed due to missing partition-level stats",
+ "Warning 8131 Build global-level stats failed due to missing partition-level stats: table `t` partition `p1`",
+ "Warning 8131 Build global-level stats failed due to missing partition-level stats: table `t` partition `p1`",
))
tbl = h.GetTableStats(tableInfo)
require.Equal(t, tbl.Version, lastVersion) // global stats not updated
@@ -2904,7 +2809,7 @@ PARTITION BY RANGE ( a ) (
tk.MustExec("analyze table t partition p1 columns a,b,d with 1 topn, 3 buckets")
tk.MustQuery("show warnings").Sort().Check(testkit.Rows(
"Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1",
- "Warning 8244 Build table: `t` column: `d` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions",
+ "Warning 8244 Build global-level stats failed due to missing partition-level column stats: table `t` partition `p0` column `d`, please run analyze table to refresh columns of all partitions",
))
// analyze partition with existing table-level options and existing partition stats under dynamic
@@ -2914,7 +2819,7 @@ PARTITION BY RANGE ( a ) (
tk.MustQuery("show warnings").Sort().Check(testkit.Rows(
"Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1",
"Warning 1105 Ignore columns and options when analyze partition in dynamic mode",
- "Warning 8244 Build table: `t` column: `d` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions",
+ "Warning 8244 Build global-level stats failed due to missing partition-level column stats: table `t` partition `p0` column `d`, please run analyze table to refresh columns of all partitions",
))
// analyze partition with existing table-level & partition-level options and existing partition stats under dynamic
@@ -2923,18 +2828,19 @@ PARTITION BY RANGE ( a ) (
tk.MustQuery("show warnings").Sort().Check(testkit.Rows(
"Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1",
"Warning 1105 Ignore columns and options when analyze partition in dynamic mode",
- "Warning 8244 Build table: `t` column: `d` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions",
+ "Warning 8244 Build global-level stats failed due to missing partition-level column stats: table `t` partition `p0` column `d`, please run analyze table to refresh columns of all partitions",
))
- tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1")
- require.NoError(t, h.LoadNeededHistograms())
- tbl := h.GetTableStats(tableInfo)
- require.Equal(t, 4, len(tbl.Columns))
+ // flaky test, fix it later
+ //tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1")
+ //require.NoError(t, h.LoadNeededHistograms())
+ //tbl := h.GetTableStats(tableInfo)
+ //require.Equal(t, 0, len(tbl.Columns))
// ignore both p0's 3 buckets, persisted-partition-options' 1 bucket, just use table-level 2 buckets
tk.MustExec("analyze table t partition p0")
tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1")
require.NoError(t, h.LoadNeededHistograms())
- tbl = h.GetTableStats(tableInfo)
+ tbl := h.GetTableStats(tableInfo)
require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets))
}
@@ -2974,8 +2880,8 @@ PARTITION BY RANGE ( a ) (
// analyze partition with index and with options are allowed under dynamic V1
tk.MustExec("analyze table t partition p0 with 1 topn, 3 buckets")
tk.MustQuery("show warnings").Sort().Check(testkit.Rows(
- "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats",
- "Warning 8131 Build table: `t` index: `idx` global-level stats failed due to missing partition-level stats",
+ "Warning 8131 Build global-level stats failed due to missing partition-level stats: table `t` partition `p1`",
+ "Warning 8131 Build global-level stats failed due to missing partition-level stats: table `t` partition `p1`",
))
tk.MustExec("analyze table t partition p1 with 1 topn, 3 buckets")
tk.MustQuery("show warnings").Sort().Check(testkit.Rows())
@@ -3157,3 +3063,115 @@ func TestAutoAnalyzeAwareGlobalVariableChange(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/injectBaseCount"))
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/injectBaseModifyCount"))
}
+
+func TestGlobalMemoryControlForAnalyze(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk0 := testkit.NewTestKit(t, store)
+ tk0.MustExec("set global tidb_mem_oom_action = 'cancel'")
+ tk0.MustExec("set global tidb_server_memory_limit = 512MB")
+ tk0.MustExec("set global tidb_server_memory_limit_sess_min_size = 128")
+
+ sm := &testkit.MockSessionManager{
+ PS: []*util.ProcessInfo{tk0.Session().ShowProcess()},
+ }
+ dom.ServerMemoryLimitHandle().SetSessionManager(sm)
+ go dom.ServerMemoryLimitHandle().Run()
+
+ tk0.MustExec("use test")
+ tk0.MustExec("create table t(a int)")
+ tk0.MustExec("insert into t select 1")
+ for i := 1; i <= 8; i++ {
+ tk0.MustExec("insert into t select * from t") // 256 Lines
+ }
+ sql := "analyze table t with 1.0 samplerate;" // Need about 100MB
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/memory/ReadMemStats", `return(536870912)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/mockAnalyzeMergeWorkerSlowConsume", `return(100)`))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/memory/ReadMemStats"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/mockAnalyzeMergeWorkerSlowConsume"))
+ }()
+ _, err := tk0.Exec(sql)
+ require.True(t, strings.Contains(err.Error(), "Out Of Memory Quota!"))
+ runtime.GC()
+}
+
+func TestGlobalMemoryControlForAutoAnalyze(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ originalVal1 := tk.MustQuery("select @@global.tidb_mem_oom_action").Rows()[0][0].(string)
+ tk.MustExec("set global tidb_mem_oom_action = 'cancel'")
+ //originalVal2 := tk.MustQuery("select @@global.tidb_server_memory_limit").Rows()[0][0].(string)
+ tk.MustExec("set global tidb_server_memory_limit = 512MB")
+ originalVal3 := tk.MustQuery("select @@global.tidb_server_memory_limit_sess_min_size").Rows()[0][0].(string)
+ tk.MustExec("set global tidb_server_memory_limit_sess_min_size = 128")
+ defer func() {
+ tk.MustExec(fmt.Sprintf("set global tidb_mem_oom_action = %v", originalVal1))
+ //tk.MustExec(fmt.Sprintf("set global tidb_server_memory_limit = %v", originalVal2))
+ tk.MustExec(fmt.Sprintf("set global tidb_server_memory_limit_sess_min_size = %v", originalVal3))
+ }()
+
+ // clean child trackers
+ oldChildTrackers := executor.GlobalAnalyzeMemoryTracker.GetChildrenForTest()
+ for _, tracker := range oldChildTrackers {
+ tracker.Detach()
+ }
+ defer func() {
+ for _, tracker := range oldChildTrackers {
+ tracker.AttachTo(executor.GlobalAnalyzeMemoryTracker)
+ }
+ }()
+ childTrackers := executor.GlobalAnalyzeMemoryTracker.GetChildrenForTest()
+ require.Len(t, childTrackers, 0)
+
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int)")
+ tk.MustExec("insert into t select 1")
+ for i := 1; i <= 8; i++ {
+ tk.MustExec("insert into t select * from t") // 256 Lines
+ }
+ _, err0 := tk.Exec("analyze table t with 1.0 samplerate;")
+ require.NoError(t, err0)
+ rs0 := tk.MustQuery("select fail_reason from mysql.analyze_jobs where table_name=? and state=? limit 1", "t", "failed")
+ require.Len(t, rs0.Rows(), 0)
+
+ h := dom.StatsHandle()
+ originalVal4 := handle.AutoAnalyzeMinCnt
+ originalVal5 := tk.MustQuery("select @@global.tidb_auto_analyze_ratio").Rows()[0][0].(string)
+ handle.AutoAnalyzeMinCnt = 0
+ tk.MustExec("set global tidb_auto_analyze_ratio = 0.001")
+ defer func() {
+ handle.AutoAnalyzeMinCnt = originalVal4
+ tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal5))
+ }()
+
+ sm := &testkit.MockSessionManager{
+ Dom: dom,
+ PS: []*util.ProcessInfo{tk.Session().ShowProcess()},
+ }
+ dom.ServerMemoryLimitHandle().SetSessionManager(sm)
+ go dom.ServerMemoryLimitHandle().Run()
+
+ tk.MustExec("insert into t values(4),(5),(6)")
+ require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll))
+ err := h.Update(dom.InfoSchema())
+ require.NoError(t, err)
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/memory/ReadMemStats", `return(536870912)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/mockAnalyzeMergeWorkerSlowConsume", `return(100)`))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/memory/ReadMemStats"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/mockAnalyzeMergeWorkerSlowConsume"))
+ }()
+ tk.MustQuery("select 1")
+ childTrackers = executor.GlobalAnalyzeMemoryTracker.GetChildrenForTest()
+ require.Len(t, childTrackers, 0)
+
+ h.HandleAutoAnalyze(dom.InfoSchema())
+ rs := tk.MustQuery("select fail_reason from mysql.analyze_jobs where table_name=? and state=? limit 1", "t", "failed")
+ failReason := rs.Rows()[0][0].(string)
+ require.True(t, strings.Contains(failReason, "Out Of Memory Quota!"))
+
+ childTrackers = executor.GlobalAnalyzeMemoryTracker.GetChildrenForTest()
+ require.Len(t, childTrackers, 0)
+}
diff --git a/executor/autoidtest/BUILD.bazel b/executor/autoidtest/BUILD.bazel
new file mode 100644
index 0000000000000..28aad8e63fa8c
--- /dev/null
+++ b/executor/autoidtest/BUILD.bazel
@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+ name = "autoidtest_test",
+ timeout = "short",
+ srcs = [
+ "autoid_test.go",
+ "main_test.go",
+ ],
+ flaky = True,
+ race = "on",
+ shard_count = 5,
+ deps = [
+ "//autoid_service",
+ "//config",
+ "//ddl/testutil",
+ "//meta/autoid",
+ "//parser/mysql",
+ "//session",
+ "//sessionctx/variable",
+ "//testkit",
+ "//testkit/testutil",
+ "@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_stretchr_testify//require",
+ "@com_github_tikv_client_go_v2//tikv",
+ "@org_uber_go_goleak//:goleak",
+ ],
+)
diff --git a/executor/autoidtest/autoid_test.go b/executor/autoidtest/autoid_test.go
new file mode 100644
index 0000000000000..eb8cc3f874159
--- /dev/null
+++ b/executor/autoidtest/autoid_test.go
@@ -0,0 +1,769 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package autoid_test
+
+import (
+ "context"
+ "fmt"
+ "strconv"
+ "strings"
+ "testing"
+
+ "github.com/pingcap/failpoint"
+ _ "github.com/pingcap/tidb/autoid_service"
+ ddltestutil "github.com/pingcap/tidb/ddl/testutil"
+ "github.com/pingcap/tidb/parser/mysql"
+ "github.com/pingcap/tidb/session"
+ "github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/pingcap/tidb/testkit/testutil"
+ "github.com/stretchr/testify/require"
+)
+
+// Test filter different kind of allocators.
+// In special ddl type, for example:
+// 1: ActionRenameTable : it will abandon all the old allocators.
+// 2: ActionRebaseAutoID : it will drop row-id-type allocator.
+// 3: ActionModifyTableAutoIdCache : it will drop row-id-type allocator.
+// 3: ActionRebaseAutoRandomBase : it will drop auto-rand-type allocator.
+func TestFilterDifferentAllocators(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("drop table if exists t1")
+
+ for _, str := range []string{"", " AUTO_ID_CACHE 1"} {
+ tk.MustExec("create table t(a bigint auto_random(5) key, b int auto_increment unique)" + str)
+ tk.MustExec("insert into t values()")
+ tk.MustQuery("select b from t").Check(testkit.Rows("1"))
+ allHandles, err := ddltestutil.ExtractAllTableHandles(tk.Session(), "test", "t")
+ require.NoError(t, err)
+ require.Equal(t, 1, len(allHandles))
+ orderedHandles := testutil.MaskSortHandles(allHandles, 5, mysql.TypeLonglong)
+ require.Equal(t, int64(1), orderedHandles[0])
+ tk.MustExec("delete from t")
+
+ // Test rebase auto_increment.
+ tk.MustExec("alter table t auto_increment 3000000")
+ tk.MustExec("insert into t values()")
+ tk.MustQuery("select b from t").Check(testkit.Rows("3000000"))
+ allHandles, err = ddltestutil.ExtractAllTableHandles(tk.Session(), "test", "t")
+ require.NoError(t, err)
+ require.Equal(t, 1, len(allHandles))
+ orderedHandles = testutil.MaskSortHandles(allHandles, 5, mysql.TypeLonglong)
+ require.Equal(t, int64(2), orderedHandles[0])
+ tk.MustExec("delete from t")
+
+ // Test rebase auto_random.
+ tk.MustExec("alter table t auto_random_base 3000000")
+ tk.MustExec("insert into t values()")
+ tk.MustQuery("select b from t").Check(testkit.Rows("3000001"))
+ allHandles, err = ddltestutil.ExtractAllTableHandles(tk.Session(), "test", "t")
+ require.NoError(t, err)
+ require.Equal(t, 1, len(allHandles))
+ orderedHandles = testutil.MaskSortHandles(allHandles, 5, mysql.TypeLonglong)
+ require.Equal(t, int64(3000000), orderedHandles[0])
+ tk.MustExec("delete from t")
+
+ // Test rename table.
+ tk.MustExec("rename table t to t1")
+ tk.MustExec("insert into t1 values()")
+ res := tk.MustQuery("select b from t1")
+ strInt64, err := strconv.ParseInt(res.Rows()[0][0].(string), 10, 64)
+ require.NoError(t, err)
+ require.GreaterOrEqual(t, strInt64, int64(3000002))
+ allHandles, err = ddltestutil.ExtractAllTableHandles(tk.Session(), "test", "t1")
+ require.NoError(t, err)
+ require.Equal(t, 1, len(allHandles))
+ orderedHandles = testutil.MaskSortHandles(allHandles, 5, mysql.TypeLonglong)
+ require.Greater(t, orderedHandles[0], int64(3000001))
+
+ tk.MustExec("drop table t1")
+ }
+}
+
+func TestAutoIncrementInsertMinMax(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ cases := []struct {
+ t string
+ s string
+ vals []int64
+ expect [][]interface{}
+ }{
+ {"tinyint", "signed", []int64{-128, 0, 127}, testkit.Rows("-128", "1", "2", "3", "127")},
+ {"tinyint", "unsigned", []int64{0, 127, 255}, testkit.Rows("1", "2", "127", "128", "255")},
+ {"smallint", "signed", []int64{-32768, 0, 32767}, testkit.Rows("-32768", "1", "2", "3", "32767")},
+ {"smallint", "unsigned", []int64{0, 32767, 65535}, testkit.Rows("1", "2", "32767", "32768", "65535")},
+ {"mediumint", "signed", []int64{-8388608, 0, 8388607}, testkit.Rows("-8388608", "1", "2", "3", "8388607")},
+ {"mediumint", "unsigned", []int64{0, 8388607, 16777215}, testkit.Rows("1", "2", "8388607", "8388608", "16777215")},
+ {"integer", "signed", []int64{-2147483648, 0, 2147483647}, testkit.Rows("-2147483648", "1", "2", "3", "2147483647")},
+ {"integer", "unsigned", []int64{0, 2147483647, 4294967295}, testkit.Rows("1", "2", "2147483647", "2147483648", "4294967295")},
+ {"bigint", "signed", []int64{-9223372036854775808, 0, 9223372036854775807}, testkit.Rows("-9223372036854775808", "1", "2", "3", "9223372036854775807")},
+ {"bigint", "unsigned", []int64{0, 9223372036854775807}, testkit.Rows("1", "2", "9223372036854775807", "9223372036854775808")},
+ }
+
+ for _, option := range []string{"", "auto_id_cache 1", "auto_id_cache 100"} {
+ for idx, c := range cases {
+ sql := fmt.Sprintf("create table t%d (a %s %s key auto_increment) %s", idx, c.t, c.s, option)
+ tk.MustExec(sql)
+
+ for _, val := range c.vals {
+ tk.MustExec(fmt.Sprintf("insert into t%d values (%d)", idx, val))
+ tk.Exec(fmt.Sprintf("insert into t%d values ()", idx)) // ignore error
+ }
+
+ tk.MustQuery(fmt.Sprintf("select * from t%d order by a", idx)).Check(c.expect)
+
+ tk.MustExec(fmt.Sprintf("drop table t%d", idx))
+ }
+ }
+
+ tk.MustExec("create table t10 (a integer key auto_increment) auto_id_cache 1")
+ err := tk.ExecToErr("insert into t10 values (2147483648)")
+ require.Error(t, err)
+ err = tk.ExecToErr("insert into t10 values (-2147483649)")
+ require.Error(t, err)
+}
+
+func TestInsertWithAutoidSchema(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec(`use test`)
+ tk.MustExec(`create table t1(id int primary key auto_increment, n int);`)
+ tk.MustExec(`create table t2(id int unsigned primary key auto_increment, n int);`)
+ tk.MustExec(`create table t3(id tinyint primary key auto_increment, n int);`)
+ tk.MustExec(`create table t4(id int primary key, n float auto_increment, key I_n(n));`)
+ tk.MustExec(`create table t5(id int primary key, n float unsigned auto_increment, key I_n(n));`)
+ tk.MustExec(`create table t6(id int primary key, n double auto_increment, key I_n(n));`)
+ tk.MustExec(`create table t7(id int primary key, n double unsigned auto_increment, key I_n(n));`)
+ // test for inserting multiple values
+ tk.MustExec(`create table t8(id int primary key auto_increment, n int);`)
+
+ testInsertWithAutoidSchema(t, tk)
+}
+
+func TestInsertWithAutoidSchemaCache(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec(`use test`)
+ tk.MustExec(`create table t1(id int primary key auto_increment, n int) AUTO_ID_CACHE 1;`)
+ tk.MustExec(`create table t2(id int unsigned primary key auto_increment, n int) AUTO_ID_CACHE 1;`)
+ tk.MustExec(`create table t3(id tinyint primary key auto_increment, n int) AUTO_ID_CACHE 1;`)
+ tk.MustExec(`create table t4(id int primary key, n float auto_increment, key I_n(n)) AUTO_ID_CACHE 1;`)
+ tk.MustExec(`create table t5(id int primary key, n float unsigned auto_increment, key I_n(n)) AUTO_ID_CACHE 1;`)
+ tk.MustExec(`create table t6(id int primary key, n double auto_increment, key I_n(n)) AUTO_ID_CACHE 1;`)
+ tk.MustExec(`create table t7(id int primary key, n double unsigned auto_increment, key I_n(n)) AUTO_ID_CACHE 1;`)
+ // test for inserting multiple values
+ tk.MustExec(`create table t8(id int primary key auto_increment, n int);`)
+
+ testInsertWithAutoidSchema(t, tk)
+}
+
+func testInsertWithAutoidSchema(t *testing.T, tk *testkit.TestKit) {
+ tests := []struct {
+ insert string
+ query string
+ result [][]interface{}
+ }{
+ {
+ `insert into t1(id, n) values(1, 1)`,
+ `select * from t1 where id = 1`,
+ testkit.Rows(`1 1`),
+ },
+ {
+ `insert into t1(n) values(2)`,
+ `select * from t1 where id = 2`,
+ testkit.Rows(`2 2`),
+ },
+ {
+ `insert into t1(n) values(3)`,
+ `select * from t1 where id = 3`,
+ testkit.Rows(`3 3`),
+ },
+ {
+ `insert into t1(id, n) values(-1, 4)`,
+ `select * from t1 where id = -1`,
+ testkit.Rows(`-1 4`),
+ },
+ {
+ `insert into t1(n) values(5)`,
+ `select * from t1 where id = 4`,
+ testkit.Rows(`4 5`),
+ },
+ {
+ `insert into t1(id, n) values('5', 6)`,
+ `select * from t1 where id = 5`,
+ testkit.Rows(`5 6`),
+ },
+ {
+ `insert into t1(n) values(7)`,
+ `select * from t1 where id = 6`,
+ testkit.Rows(`6 7`),
+ },
+ {
+ `insert into t1(id, n) values(7.4, 8)`,
+ `select * from t1 where id = 7`,
+ testkit.Rows(`7 8`),
+ },
+ {
+ `insert into t1(id, n) values(7.5, 9)`,
+ `select * from t1 where id = 8`,
+ testkit.Rows(`8 9`),
+ },
+ {
+ `insert into t1(n) values(9)`,
+ `select * from t1 where id = 9`,
+ testkit.Rows(`9 9`),
+ },
+ // test last insert id
+ {
+ `insert into t1 values(3000, -1), (null, -2)`,
+ `select * from t1 where id = 3000`,
+ testkit.Rows(`3000 -1`),
+ },
+ {
+ `;`,
+ `select * from t1 where id = 3001`,
+ testkit.Rows(`3001 -2`),
+ },
+ {
+ `;`,
+ `select last_insert_id()`,
+ testkit.Rows(`3001`),
+ },
+ {
+ `insert into t2(id, n) values(1, 1)`,
+ `select * from t2 where id = 1`,
+ testkit.Rows(`1 1`),
+ },
+ {
+ `insert into t2(n) values(2)`,
+ `select * from t2 where id = 2`,
+ testkit.Rows(`2 2`),
+ },
+ {
+ `insert into t2(n) values(3)`,
+ `select * from t2 where id = 3`,
+ testkit.Rows(`3 3`),
+ },
+ {
+ `insert into t3(id, n) values(1, 1)`,
+ `select * from t3 where id = 1`,
+ testkit.Rows(`1 1`),
+ },
+ {
+ `insert into t3(n) values(2)`,
+ `select * from t3 where id = 2`,
+ testkit.Rows(`2 2`),
+ },
+ {
+ `insert into t3(n) values(3)`,
+ `select * from t3 where id = 3`,
+ testkit.Rows(`3 3`),
+ },
+ {
+ `insert into t3(id, n) values(-1, 4)`,
+ `select * from t3 where id = -1`,
+ testkit.Rows(`-1 4`),
+ },
+ {
+ `insert into t3(n) values(5)`,
+ `select * from t3 where id = 4`,
+ testkit.Rows(`4 5`),
+ },
+ {
+ `insert into t4(id, n) values(1, 1)`,
+ `select * from t4 where id = 1`,
+ testkit.Rows(`1 1`),
+ },
+ {
+ `insert into t4(id) values(2)`,
+ `select * from t4 where id = 2`,
+ testkit.Rows(`2 2`),
+ },
+ {
+ `insert into t4(id, n) values(3, -1)`,
+ `select * from t4 where id = 3`,
+ testkit.Rows(`3 -1`),
+ },
+ {
+ `insert into t4(id) values(4)`,
+ `select * from t4 where id = 4`,
+ testkit.Rows(`4 3`),
+ },
+ {
+ `insert into t4(id, n) values(5, 5.5)`,
+ `select * from t4 where id = 5`,
+ testkit.Rows(`5 5.5`),
+ },
+ {
+ `insert into t4(id) values(6)`,
+ `select * from t4 where id = 6`,
+ testkit.Rows(`6 7`),
+ },
+ {
+ `insert into t4(id, n) values(7, '7.7')`,
+ `select * from t4 where id = 7`,
+ testkit.Rows(`7 7.7`),
+ },
+ {
+ `insert into t4(id) values(8)`,
+ `select * from t4 where id = 8`,
+ testkit.Rows(`8 9`),
+ },
+ {
+ `insert into t4(id, n) values(9, 10.4)`,
+ `select * from t4 where id = 9`,
+ testkit.Rows(`9 10.4`),
+ },
+ {
+ `insert into t4(id) values(10)`,
+ `select * from t4 where id = 10`,
+ testkit.Rows(`10 11`),
+ },
+ {
+ `insert into t5(id, n) values(1, 1)`,
+ `select * from t5 where id = 1`,
+ testkit.Rows(`1 1`),
+ },
+ {
+ `insert into t5(id) values(2)`,
+ `select * from t5 where id = 2`,
+ testkit.Rows(`2 2`),
+ },
+ {
+ `insert into t5(id) values(3)`,
+ `select * from t5 where id = 3`,
+ testkit.Rows(`3 3`),
+ },
+ {
+ `insert into t6(id, n) values(1, 1)`,
+ `select * from t6 where id = 1`,
+ testkit.Rows(`1 1`),
+ },
+ {
+ `insert into t6(id) values(2)`,
+ `select * from t6 where id = 2`,
+ testkit.Rows(`2 2`),
+ },
+ {
+ `insert into t6(id, n) values(3, -1)`,
+ `select * from t6 where id = 3`,
+ testkit.Rows(`3 -1`),
+ },
+ {
+ `insert into t6(id) values(4)`,
+ `select * from t6 where id = 4`,
+ testkit.Rows(`4 3`),
+ },
+ {
+ `insert into t6(id, n) values(5, 5.5)`,
+ `select * from t6 where id = 5`,
+ testkit.Rows(`5 5.5`),
+ },
+ {
+ `insert into t6(id) values(6)`,
+ `select * from t6 where id = 6`,
+ testkit.Rows(`6 7`),
+ },
+ {
+ `insert into t6(id, n) values(7, '7.7')`,
+ `select * from t4 where id = 7`,
+ testkit.Rows(`7 7.7`),
+ },
+ {
+ `insert into t6(id) values(8)`,
+ `select * from t4 where id = 8`,
+ testkit.Rows(`8 9`),
+ },
+ {
+ `insert into t6(id, n) values(9, 10.4)`,
+ `select * from t6 where id = 9`,
+ testkit.Rows(`9 10.4`),
+ },
+ {
+ `insert into t6(id) values(10)`,
+ `select * from t6 where id = 10`,
+ testkit.Rows(`10 11`),
+ },
+ {
+ `insert into t7(id, n) values(1, 1)`,
+ `select * from t7 where id = 1`,
+ testkit.Rows(`1 1`),
+ },
+ {
+ `insert into t7(id) values(2)`,
+ `select * from t7 where id = 2`,
+ testkit.Rows(`2 2`),
+ },
+ {
+ `insert into t7(id) values(3)`,
+ `select * from t7 where id = 3`,
+ testkit.Rows(`3 3`),
+ },
+
+ // the following is test for insert multiple values.
+ {
+ `insert into t8(n) values(1),(2)`,
+ `select * from t8 where id = 1`,
+ testkit.Rows(`1 1`),
+ },
+ {
+ `;`,
+ `select * from t8 where id = 2`,
+ testkit.Rows(`2 2`),
+ },
+ {
+ `;`,
+ `select last_insert_id();`,
+ testkit.Rows(`1`),
+ },
+ // test user rebase and auto alloc mixture.
+ {
+ `insert into t8 values(null, 3),(-1, -1),(null,4),(null, 5)`,
+ `select * from t8 where id = 3`,
+ testkit.Rows(`3 3`),
+ },
+ // -1 won't rebase allocator here cause -1 < base.
+ {
+ `;`,
+ `select * from t8 where id = -1`,
+ testkit.Rows(`-1 -1`),
+ },
+ {
+ `;`,
+ `select * from t8 where id = 4`,
+ testkit.Rows(`4 4`),
+ },
+ {
+ `;`,
+ `select * from t8 where id = 5`,
+ testkit.Rows(`5 5`),
+ },
+ {
+ `;`,
+ `select last_insert_id();`,
+ testkit.Rows(`3`),
+ },
+ {
+ `insert into t8 values(null, 6),(10, 7),(null, 8)`,
+ `select * from t8 where id = 6`,
+ testkit.Rows(`6 6`),
+ },
+ // 10 will rebase allocator here.
+ {
+ `;`,
+ `select * from t8 where id = 10`,
+ testkit.Rows(`10 7`),
+ },
+ {
+ `;`,
+ `select * from t8 where id = 11`,
+ testkit.Rows(`11 8`),
+ },
+ {
+ `;`,
+ `select last_insert_id()`,
+ testkit.Rows(`6`),
+ },
+ // fix bug for last_insert_id should be first allocated id in insert rows (skip the rebase id).
+ {
+ `insert into t8 values(100, 9),(null,10),(null,11)`,
+ `select * from t8 where id = 100`,
+ testkit.Rows(`100 9`),
+ },
+ {
+ `;`,
+ `select * from t8 where id = 101`,
+ testkit.Rows(`101 10`),
+ },
+ {
+ `;`,
+ `select * from t8 where id = 102`,
+ testkit.Rows(`102 11`),
+ },
+ {
+ `;`,
+ `select last_insert_id()`,
+ testkit.Rows(`101`),
+ },
+ // test with sql_mode: NO_AUTO_VALUE_ON_ZERO.
+ {
+ `;`,
+ `select @@sql_mode`,
+ testkit.Rows(`ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION`),
+ },
+ {
+ `;`,
+ "set session sql_mode = `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO`",
+ nil,
+ },
+ {
+ `insert into t8 values (0, 12), (null, 13)`,
+ `select * from t8 where id = 0`,
+ testkit.Rows(`0 12`),
+ },
+ {
+ `;`,
+ `select * from t8 where id = 103`,
+ testkit.Rows(`103 13`),
+ },
+ {
+ `;`,
+ `select last_insert_id()`,
+ testkit.Rows(`103`),
+ },
+ // test without sql_mode: NO_AUTO_VALUE_ON_ZERO.
+ {
+ `;`,
+ "set session sql_mode = `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION`",
+ nil,
+ },
+ // value 0 will be substitute by autoid.
+ {
+ `insert into t8 values (0, 14), (null, 15)`,
+ `select * from t8 where id = 104`,
+ testkit.Rows(`104 14`),
+ },
+ {
+ `;`,
+ `select * from t8 where id = 105`,
+ testkit.Rows(`105 15`),
+ },
+ {
+ `;`,
+ `select last_insert_id()`,
+ testkit.Rows(`104`),
+ },
+ // last test : auto increment allocation can find in retryInfo.
+ {
+ `retry : insert into t8 values (null, 16), (null, 17)`,
+ `select * from t8 where id = 1000`,
+ testkit.Rows(`1000 16`),
+ },
+ {
+ `;`,
+ `select * from t8 where id = 1001`,
+ testkit.Rows(`1001 17`),
+ },
+ {
+ `;`,
+ `select last_insert_id()`,
+ // this insert doesn't has the last_insert_id, should be same as the last insert case.
+ testkit.Rows(`104`),
+ },
+ }
+
+ for _, tt := range tests {
+ if strings.HasPrefix(tt.insert, "retry : ") {
+ // it's the last retry insert case, change the sessionVars.
+ retryInfo := &variable.RetryInfo{Retrying: true}
+ retryInfo.AddAutoIncrementID(1000)
+ retryInfo.AddAutoIncrementID(1001)
+ tk.Session().GetSessionVars().RetryInfo = retryInfo
+ tk.MustExec(tt.insert[8:])
+ tk.Session().GetSessionVars().RetryInfo = &variable.RetryInfo{}
+ } else {
+ tk.MustExec(tt.insert)
+ }
+ if tt.query == "set session sql_mode = `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO`" ||
+ tt.query == "set session sql_mode = `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION`" {
+ tk.MustExec(tt.query)
+ } else {
+ tk.MustQuery(tt.query).Check(tt.result)
+ }
+ }
+}
+
+// TestAutoIDIncrementAndOffset There is a potential issue in MySQL: when the value of auto_increment_offset is greater
+// than that of auto_increment_increment, the value of auto_increment_offset is ignored
+// (https://dev.mysql.com/doc/refman/8.0/en/replication-options-master.html#sysvar_auto_increment_increment),
+// This issue is a flaw of the implementation of MySQL and it doesn't exist in TiDB.
+func TestAutoIDIncrementAndOffset(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec(`use test`)
+ // Test for offset is larger than increment.
+ tk.Session().GetSessionVars().AutoIncrementIncrement = 5
+ tk.Session().GetSessionVars().AutoIncrementOffset = 10
+
+ for _, str := range []string{"", " AUTO_ID_CACHE 1"} {
+ tk.MustExec(`create table io (a int key auto_increment)` + str)
+ tk.MustExec(`insert into io values (null),(null),(null)`)
+ tk.MustQuery(`select * from io`).Check(testkit.Rows("10", "15", "20"))
+ tk.MustExec(`drop table io`)
+ }
+
+ // Test handle is PK.
+ for _, str := range []string{"", " AUTO_ID_CACHE 1"} {
+ tk.MustExec(`create table io (a int key auto_increment)` + str)
+ tk.Session().GetSessionVars().AutoIncrementOffset = 10
+ tk.Session().GetSessionVars().AutoIncrementIncrement = 2
+ tk.MustExec(`insert into io values (),(),()`)
+ tk.MustQuery(`select * from io`).Check(testkit.Rows("10", "12", "14"))
+ tk.MustExec(`delete from io`)
+
+ // Test reset the increment.
+ tk.Session().GetSessionVars().AutoIncrementIncrement = 5
+ tk.MustExec(`insert into io values (),(),()`)
+ tk.MustQuery(`select * from io`).Check(testkit.Rows("15", "20", "25"))
+ tk.MustExec(`delete from io`)
+
+ tk.Session().GetSessionVars().AutoIncrementIncrement = 10
+ tk.MustExec(`insert into io values (),(),()`)
+ tk.MustQuery(`select * from io`).Check(testkit.Rows("30", "40", "50"))
+ tk.MustExec(`delete from io`)
+
+ tk.Session().GetSessionVars().AutoIncrementIncrement = 5
+ tk.MustExec(`insert into io values (),(),()`)
+ tk.MustQuery(`select * from io`).Check(testkit.Rows("55", "60", "65"))
+ tk.MustExec(`drop table io`)
+ }
+
+ // Test handle is not PK.
+ for _, str := range []string{"", " AUTO_ID_CACHE 1"} {
+ tk.Session().GetSessionVars().AutoIncrementIncrement = 2
+ tk.Session().GetSessionVars().AutoIncrementOffset = 10
+ tk.MustExec(`create table io (a int, b int auto_increment, key(b))` + str)
+ tk.MustExec(`insert into io(b) values (null),(null),(null)`)
+ // AutoID allocation will take increment and offset into consideration.
+ tk.MustQuery(`select b from io`).Check(testkit.Rows("10", "12", "14"))
+ if str == "" {
+ // HandleID allocation will ignore the increment and offset.
+ tk.MustQuery(`select _tidb_rowid from io`).Check(testkit.Rows("15", "16", "17"))
+ } else {
+ // Separate row id and auto inc id, increment and offset works on auto inc id
+ tk.MustQuery(`select _tidb_rowid from io`).Check(testkit.Rows("1", "2", "3"))
+ }
+ tk.MustExec(`delete from io`)
+
+ tk.Session().GetSessionVars().AutoIncrementIncrement = 10
+ tk.MustExec(`insert into io(b) values (null),(null),(null)`)
+ tk.MustQuery(`select b from io`).Check(testkit.Rows("20", "30", "40"))
+ if str == "" {
+ tk.MustQuery(`select _tidb_rowid from io`).Check(testkit.Rows("41", "42", "43"))
+ } else {
+ tk.MustQuery(`select _tidb_rowid from io`).Check(testkit.Rows("4", "5", "6"))
+ }
+
+ // Test invalid value.
+ tk.Session().GetSessionVars().AutoIncrementIncrement = -1
+ tk.Session().GetSessionVars().AutoIncrementOffset = -2
+ tk.MustGetErrMsg(`insert into io(b) values (null),(null),(null)`,
+ "[autoid:8060]Invalid auto_increment settings: auto_increment_increment: -1, auto_increment_offset: -2, both of them must be in range [1..65535]")
+ tk.MustExec(`delete from io`)
+
+ tk.Session().GetSessionVars().AutoIncrementIncrement = 65536
+ tk.Session().GetSessionVars().AutoIncrementOffset = 65536
+ tk.MustGetErrMsg(`insert into io(b) values (null),(null),(null)`,
+ "[autoid:8060]Invalid auto_increment settings: auto_increment_increment: 65536, auto_increment_offset: 65536, both of them must be in range [1..65535]")
+
+ tk.MustExec(`drop table io`)
+ }
+}
+
+func TestRenameTableForAutoIncrement(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("USE test;")
+ tk.MustExec("drop table if exists t1, t2, t3;")
+ tk.MustExec("create table t1 (id int key auto_increment);")
+ tk.MustExec("insert into t1 values ()")
+ tk.MustExec("rename table t1 to t11")
+ tk.MustExec("insert into t11 values ()")
+ // TODO(tiancaiamao): fix bug and uncomment here, rename table should not discard the cached AUTO_ID.
+ // tk.MustQuery("select * from t11").Check(testkit.Rows("1", "2"))
+
+ // auto_id_cache 1 use another implementation and do not have such bug.
+ tk.MustExec("create table t2 (id int key auto_increment) auto_id_cache 1;")
+ tk.MustExec("insert into t2 values ()")
+ tk.MustExec("rename table t2 to t22")
+ tk.MustExec("insert into t22 values ()")
+ tk.MustQuery("select * from t22").Check(testkit.Rows("1", "2"))
+
+ tk.MustExec("create table t3 (id int key auto_increment) auto_id_cache 100;")
+ tk.MustExec("insert into t3 values ()")
+ tk.MustExec("rename table t3 to t33")
+ tk.MustExec("insert into t33 values ()")
+ // TODO(tiancaiamao): fix bug and uncomment here, rename table should not discard the cached AUTO_ID.
+ // tk.MustQuery("select * from t33").Check(testkit.Rows("1", "2"))
+}
+
+func TestAlterTableAutoIDCache(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("USE test;")
+ tk.MustExec("drop table if exists t_473;")
+ tk.MustExec("create table t_473 (id int key auto_increment)")
+ tk.MustExec("insert into t_473 values ()")
+ tk.MustQuery("select * from t_473").Check(testkit.Rows("1"))
+ rs, err := tk.Exec("show table t_473 next_row_id")
+ require.NoError(t, err)
+ rows, err1 := session.ResultSetToStringSlice(context.Background(), tk.Session(), rs)
+ require.NoError(t, err1)
+ // "test t_473 id 1013608 AUTO_INCREMENT"
+ val, err2 := strconv.ParseUint(rows[0][3], 10, 64)
+ require.NoError(t, err2)
+
+ tk.MustExec("alter table t_473 auto_id_cache = 100")
+ tk.MustQuery("show table t_473 next_row_id").Check(testkit.Rows(
+ fmt.Sprintf("test t_473 id %d _TIDB_ROWID", val),
+ "test t_473 id 1 AUTO_INCREMENT",
+ ))
+ tk.MustExec("insert into t_473 values ()")
+ tk.MustQuery("select * from t_473").Check(testkit.Rows("1", fmt.Sprintf("%d", val)))
+ tk.MustQuery("show table t_473 next_row_id").Check(testkit.Rows(
+ fmt.Sprintf("test t_473 id %d _TIDB_ROWID", val+100),
+ "test t_473 id 1 AUTO_INCREMENT",
+ ))
+
+ // Note that auto_id_cache=1 use a different implementation, switch between them is not allowed.
+ // TODO: relax this restriction and update the test case.
+ tk.MustExecToErr("alter table t_473 auto_id_cache = 1")
+}
+
+func TestMockAutoIDServiceError(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("USE test;")
+ tk.MustExec("create table t_mock_err (id int key auto_increment) auto_id_cache 1")
+
+ failpoint.Enable("github.com/pingcap/tidb/autoid_service/mockErr", `return(true)`)
+ defer failpoint.Disable("github.com/pingcap/tidb/autoid_service/mockErr")
+ // Cover a bug that the autoid client retry non-retryable errors forever cause dead loop.
+ tk.MustExecToErr("insert into t_mock_err values (),()") // mock error, instead of dead loop
+}
+
+func TestIssue39528(t *testing.T) {
+ // When AUTO_ID_CACHE is 1, it should not affect row id setting when autoid and rowid are separated.
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test;")
+ tk.MustExec("create table issue39528 (id int unsigned key nonclustered auto_increment) shard_row_id_bits=4 auto_id_cache 1;")
+ tk.MustExec("insert into issue39528 values ()")
+ tk.MustExec("insert into issue39528 values ()")
+
+ ctx := context.Background()
+ var codeRun bool
+ ctx = context.WithValue(ctx, "testIssue39528", &codeRun)
+ _, err := tk.ExecWithContext(ctx, "insert into issue39528 values ()")
+ require.NoError(t, err)
+ // Make sure the code does not visit tikv on allocate path.
+ require.False(t, codeRun)
+}
diff --git a/executor/autoidtest/main_test.go b/executor/autoidtest/main_test.go
new file mode 100644
index 0000000000000..f87db4afe1371
--- /dev/null
+++ b/executor/autoidtest/main_test.go
@@ -0,0 +1,44 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package autoid_test
+
+import (
+ "testing"
+
+ "github.com/pingcap/tidb/config"
+ "github.com/pingcap/tidb/meta/autoid"
+ "github.com/tikv/client-go/v2/tikv"
+ "go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+ autoid.SetStep(5000)
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.Log.SlowThreshold = 30000 // 30s
+ conf.TiKVClient.AsyncCommit.SafeWindow = 0
+ conf.TiKVClient.AsyncCommit.AllowedClockDrift = 0
+ conf.Experimental.AllowsExpressionIndex = true
+ })
+ tikv.EnableFailpoints()
+
+ opts := []goleak.Option{
+ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
+ goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
+ goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
+ goleak.IgnoreTopFunction("gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun"),
+ goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"),
+ }
+ goleak.VerifyTestMain(m, opts...)
+}
diff --git a/executor/batch_checker.go b/executor/batch_checker.go
index d3820ecb0d08c..838c6af7bace0 100644
--- a/executor/batch_checker.go
+++ b/executor/batch_checker.go
@@ -146,8 +146,33 @@ func getKeysNeedCheckOneRow(ctx sessionctx.Context, t table.Table, row []types.D
}
}
- // addChangingColTimes is used to fetch values while processing "modify/change column" operation.
- addChangingColTimes := 0
+ // extraColumns is used to fetch values while processing "add/drop/modify/change column" operation.
+ extraColumns := 0
+ for _, col := range t.WritableCols() {
+ // if there is a changing column, append the dependency column for index fetch values
+ if col.ChangeStateInfo != nil && col.State != model.StatePublic {
+ value, err := table.CastValue(ctx, row[col.DependencyColumnOffset], col.ColumnInfo, false, false)
+ if err != nil {
+ return nil, err
+ }
+ row = append(row, value)
+ extraColumns++
+ continue
+ }
+
+ if col.State != model.StatePublic {
+ // only append origin default value for index fetch values
+ if col.Offset >= len(row) {
+ value, err := table.GetColOriginDefaultValue(ctx, col.ToInfo())
+ if err != nil {
+ return nil, err
+ }
+
+ row = append(row, value)
+ extraColumns++
+ }
+ }
+ }
// append unique keys and errors
for _, v := range t.Indices() {
if !tables.IsIndexWritable(v) {
@@ -159,40 +184,38 @@ func getKeysNeedCheckOneRow(ctx sessionctx.Context, t table.Table, row []types.D
if t.Meta().IsCommonHandle && v.Meta().Primary {
continue
}
- if len(row) < len(t.WritableCols()) && addChangingColTimes == 0 {
- if col := tables.FindChangingCol(t.WritableCols(), v.Meta()); col != nil {
- row = append(row, row[col.DependencyColumnOffset])
- addChangingColTimes++
- }
- }
colVals, err1 := v.FetchValues(row, nil)
if err1 != nil {
return nil, err1
}
// Pass handle = 0 to GenIndexKey,
// due to we only care about distinct key.
- key, distinct, err1 := v.GenIndexKey(ctx.GetSessionVars().StmtCtx,
- colVals, kv.IntHandle(0), nil)
- if err1 != nil {
- return nil, err1
- }
- // Skip the non-distinct keys.
- if !distinct {
- continue
- }
- colValStr, err1 := formatDataForDupError(colVals)
- if err1 != nil {
- return nil, err1
+ iter := v.GenIndexKVIter(ctx.GetSessionVars().StmtCtx, colVals, kv.IntHandle(0), nil)
+ for iter.Valid() {
+ key, _, distinct, err1 := iter.Next(nil)
+ if err1 != nil {
+ return nil, err1
+ }
+ // Skip the non-distinct keys.
+ if !distinct {
+ continue
+ }
+ // If index is used ingest ways, then we should check key from temp index.
+ if v.Meta().State != model.StatePublic && v.Meta().BackfillState != model.BackfillStateInapplicable {
+ _, key, _ = tables.GenTempIdxKeyByState(v.Meta(), key)
+ }
+ colValStr, err1 := formatDataForDupError(colVals)
+ if err1 != nil {
+ return nil, err1
+ }
+ uniqueKeys = append(uniqueKeys, &keyValueWithDupInfo{
+ newKey: key,
+ dupErr: kv.ErrKeyExists.FastGenByArgs(colValStr, fmt.Sprintf("%s.%s", v.TableMeta().Name.String(), v.Meta().Name.String())),
+ commonHandle: t.Meta().IsCommonHandle,
+ })
}
- uniqueKeys = append(uniqueKeys, &keyValueWithDupInfo{
- newKey: key,
- dupErr: kv.ErrKeyExists.FastGenByArgs(colValStr, fmt.Sprintf("%s.%s", v.TableMeta().Name.String(), v.Meta().Name.String())),
- commonHandle: t.Meta().IsCommonHandle,
- })
- }
- if addChangingColTimes == 1 {
- row = row[:len(row)-1]
}
+ row = row[:len(row)-extraColumns]
result = append(result, toBeCheckedRow{
row: row,
handleKey: handleKey,
diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go
index 1af256ade8c31..ee9808700aaec 100644
--- a/executor/batch_point_get.go
+++ b/executor/batch_point_get.go
@@ -29,6 +29,7 @@ import (
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/variable"
driver "github.com/pingcap/tidb/store/driver/txn"
+ "github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
@@ -158,6 +159,9 @@ func MockNewCacheTableSnapShot(snapshot kv.Snapshot, memBuffer kv.MemBuffer) *ca
// Close implements the Executor interface.
func (e *BatchPointGetExec) Close() error {
+ if e.runtimeStats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
+ }
if e.runtimeStats != nil && e.snapshot != nil {
e.snapshot.SetOption(kv.CollectRuntimeStats, nil)
}
@@ -190,7 +194,7 @@ func (e *BatchPointGetExec) Next(ctx context.Context, req *chunk.Chunk) error {
e.index++
}
- err := FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, e.schema, e.columns, e.ctx, req)
+ err := table.FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, e.schema.Columns, e.columns, e.ctx, req)
if err != nil {
return err
}
diff --git a/executor/benchmark_test.go b/executor/benchmark_test.go
index 99bb6ceec9103..3f64164332ce7 100644
--- a/executor/benchmark_test.go
+++ b/executor/benchmark_test.go
@@ -906,37 +906,44 @@ func prepare4HashJoin(testCase *hashJoinTestCase, innerExec, outerExec Executor)
joinSchema.Append(cols1...)
}
- joinKeys := make([]*expression.Column, 0, len(testCase.keyIdx))
- for _, keyIdx := range testCase.keyIdx {
- joinKeys = append(joinKeys, cols0[keyIdx])
- }
- probeKeys := make([]*expression.Column, 0, len(testCase.keyIdx))
- for _, keyIdx := range testCase.keyIdx {
- probeKeys = append(probeKeys, cols1[keyIdx])
- }
+ joinKeysColIdx := make([]int, 0, len(testCase.keyIdx))
+ joinKeysColIdx = append(joinKeysColIdx, testCase.keyIdx...)
+ probeKeysColIdx := make([]int, 0, len(testCase.keyIdx))
+ probeKeysColIdx = append(probeKeysColIdx, testCase.keyIdx...)
e := &HashJoinExec{
baseExecutor: newBaseExecutor(testCase.ctx, joinSchema, 5, innerExec, outerExec),
- probeSideTupleFetcher: probeSideTupleFetcher{
+ hashJoinCtx: &hashJoinCtx{
+ sessCtx: testCase.ctx,
+ joinType: testCase.joinType, // 0 for InnerJoin, 1 for LeftOutersJoin, 2 for RightOuterJoin
+ isOuterJoin: false,
+ useOuterToBuild: testCase.useOuterToBuild,
+ concurrency: uint(testCase.concurrency),
+ probeTypes: retTypes(outerExec),
+ buildTypes: retTypes(innerExec),
+ },
+ probeSideTupleFetcher: &probeSideTupleFetcher{
probeSideExec: outerExec,
},
- probeWorkers: make([]probeWorker, testCase.concurrency),
- concurrency: uint(testCase.concurrency),
- joinType: testCase.joinType, // 0 for InnerJoin, 1 for LeftOutersJoin, 2 for RightOuterJoin
- isOuterJoin: false,
- buildKeys: joinKeys,
- probeKeys: probeKeys,
- buildSideExec: innerExec,
- buildSideEstCount: float64(testCase.rows),
- useOuterToBuild: testCase.useOuterToBuild,
+ probeWorkers: make([]*probeWorker, testCase.concurrency),
+ buildWorker: &buildWorker{
+ buildKeyColIdx: joinKeysColIdx,
+ buildSideExec: innerExec,
+ },
}
childrenUsedSchema := markChildrenUsedCols(e.Schema(), e.children[0].Schema(), e.children[1].Schema())
- defaultValues := make([]types.Datum, e.buildSideExec.Schema().Len())
+ defaultValues := make([]types.Datum, e.buildWorker.buildSideExec.Schema().Len())
lhsTypes, rhsTypes := retTypes(innerExec), retTypes(outerExec)
for i := uint(0); i < e.concurrency; i++ {
- e.probeWorkers[i].joiner = newJoiner(testCase.ctx, e.joinType, true, defaultValues,
- nil, lhsTypes, rhsTypes, childrenUsedSchema, false)
+ e.probeWorkers[i] = &probeWorker{
+ workerID: i,
+ hashJoinCtx: e.hashJoinCtx,
+ joiner: newJoiner(testCase.ctx, e.joinType, true, defaultValues,
+ nil, lhsTypes, rhsTypes, childrenUsedSchema, false),
+ probeKeyColIdx: probeKeysColIdx,
+ }
}
+ e.buildWorker.hashJoinCtx = e.hashJoinCtx
memLimit := int64(-1)
if testCase.disk {
memLimit = 1
@@ -1194,7 +1201,7 @@ func benchmarkBuildHashTable(b *testing.B, casTest *hashJoinTestCase, dataSource
close(innerResultCh)
b.StartTimer()
- if err := exec.buildHashTableForList(innerResultCh); err != nil {
+ if err := exec.buildWorker.buildHashTableForList(innerResultCh); err != nil {
b.Fatal(err)
}
diff --git a/executor/bind.go b/executor/bind.go
index cf337968d4130..90272e6878620 100644
--- a/executor/bind.go
+++ b/executor/bind.go
@@ -38,6 +38,9 @@ type SQLBindExec struct {
isGlobal bool
bindAst ast.StmtNode
newStatus string
+ source string // by manual or from history, only in create stmt
+ sqlDigest string
+ planDigest string
}
// Next implements the Executor Next interface.
@@ -48,6 +51,8 @@ func (e *SQLBindExec) Next(ctx context.Context, req *chunk.Chunk) error {
return e.createSQLBind()
case plannercore.OpSQLBindDrop:
return e.dropSQLBind()
+ case plannercore.OpSQLBindDropByDigest:
+ return e.dropSQLBindByDigest()
case plannercore.OpFlushBindings:
return e.flushBindings()
case plannercore.OpCaptureBindings:
@@ -58,6 +63,8 @@ func (e *SQLBindExec) Next(ctx context.Context, req *chunk.Chunk) error {
return e.reloadBindings()
case plannercore.OpSetBindingStatus:
return e.setBindingStatus()
+ case plannercore.OpSetBindingStatusByDigest:
+ return e.setBindingStatusByDigest()
default:
return errors.Errorf("unsupported SQL bind operation: %v", e.sqlBindOp)
}
@@ -83,6 +90,20 @@ func (e *SQLBindExec) dropSQLBind() error {
return err
}
+func (e *SQLBindExec) dropSQLBindByDigest() error {
+ if e.sqlDigest == "" {
+ return errors.New("sql digest is empty")
+ }
+ if !e.isGlobal {
+ handle := e.ctx.Value(bindinfo.SessionBindInfoKeyType).(*bindinfo.SessionHandle)
+ err := handle.DropBindRecordByDigest(e.sqlDigest)
+ return err
+ }
+ affectedRows, err := domain.GetDomain(e.ctx).BindHandle().DropBindRecordByDigest(e.sqlDigest)
+ e.ctx.GetSessionVars().StmtCtx.AddAffectedRows(affectedRows)
+ return err
+}
+
func (e *SQLBindExec) setBindingStatus() error {
var bindInfo *bindinfo.Binding
if e.bindSQL != "" {
@@ -100,6 +121,15 @@ func (e *SQLBindExec) setBindingStatus() error {
return err
}
+func (e *SQLBindExec) setBindingStatusByDigest() error {
+ ok, err := domain.GetDomain(e.ctx).BindHandle().SetBindRecordStatusByDigest(e.newStatus, e.sqlDigest)
+ if err == nil && !ok {
+ warningMess := errors.New("There are no bindings can be set the status. Please check the SQL text")
+ e.ctx.GetSessionVars().StmtCtx.AppendWarning(warningMess)
+ }
+ return err
+}
+
func (e *SQLBindExec) createSQLBind() error {
// For audit log, SQLBindExec execute "explain" statement internally, save and recover stmtctx
// is necessary to avoid 'create binding' been recorded as 'explain'.
@@ -109,11 +139,13 @@ func (e *SQLBindExec) createSQLBind() error {
}()
bindInfo := bindinfo.Binding{
- BindSQL: e.bindSQL,
- Charset: e.charset,
- Collation: e.collation,
- Status: bindinfo.Enabled,
- Source: bindinfo.Manual,
+ BindSQL: e.bindSQL,
+ Charset: e.charset,
+ Collation: e.collation,
+ Status: bindinfo.Enabled,
+ Source: e.source,
+ SQLDigest: e.sqlDigest,
+ PlanDigest: e.planDigest,
}
record := &bindinfo.BindRecord{
OriginalSQL: e.normdOrigSQL,
diff --git a/executor/brie.go b/executor/brie.go
index 608cfd6336b52..b71826ff7df13 100644
--- a/executor/brie.go
+++ b/executor/brie.go
@@ -199,7 +199,7 @@ func (bq *brieQueue) clearTask(sc *stmtctx.StatementContext) {
func (b *executorBuilder) parseTSString(ts string) (uint64, error) {
sc := &stmtctx.StatementContext{TimeZone: b.ctx.GetSessionVars().Location()}
- t, err := types.ParseTime(sc, ts, mysql.TypeTimestamp, types.MaxFsp)
+ t, err := types.ParseTime(sc, ts, mysql.TypeTimestamp, types.MaxFsp, nil)
if err != nil {
return 0, err
}
@@ -524,6 +524,8 @@ func (gs *tidbGlueSession) CreateTable(ctx context.Context, dbName model.CIStr,
return err
}
gs.se.SetValue(sessionctx.QueryString, result.String())
+ // Disable foreign key check when batch create tables.
+ gs.se.GetSessionVars().ForeignKeyChecks = false
// Clone() does not clone partitions yet :(
table = table.Clone()
diff --git a/executor/builder.go b/executor/builder.go
index dfd44d549eff1..132ae66272c69 100644
--- a/executor/builder.go
+++ b/executor/builder.go
@@ -433,7 +433,8 @@ func buildIndexLookUpChecker(b *executorBuilder, p *plannercore.PhysicalIndexLoo
tps := make([]*types.FieldType, 0, fullColLen)
for _, col := range is.Columns {
- tps = append(tps, &(col.FieldType))
+ // tps is used to decode the index, we should use the element type of the array if any.
+ tps = append(tps, col.FieldType.ArrayType())
}
if !e.isCommonHandle() {
@@ -480,24 +481,35 @@ func (b *executorBuilder) buildCheckTable(v *plannercore.CheckTable) Executor {
return e
}
-func buildIdxColsConcatHandleCols(tblInfo *model.TableInfo, indexInfo *model.IndexInfo) []*model.ColumnInfo {
- handleLen := 1
+func buildIdxColsConcatHandleCols(tblInfo *model.TableInfo, indexInfo *model.IndexInfo, hasGenedCol bool) []*model.ColumnInfo {
var pkCols []*model.IndexColumn
if tblInfo.IsCommonHandle {
pkIdx := tables.FindPrimaryIndex(tblInfo)
pkCols = pkIdx.Columns
- handleLen = len(pkIdx.Columns)
}
- columns := make([]*model.ColumnInfo, 0, len(indexInfo.Columns)+handleLen)
- for _, idxCol := range indexInfo.Columns {
- columns = append(columns, tblInfo.Columns[idxCol.Offset])
+
+ columns := make([]*model.ColumnInfo, 0, len(indexInfo.Columns)+len(pkCols))
+ if hasGenedCol {
+ columns = tblInfo.Columns
+ } else {
+ for _, idxCol := range indexInfo.Columns {
+ if tblInfo.PKIsHandle && tblInfo.GetPkColInfo().Offset == idxCol.Offset {
+ continue
+ }
+ columns = append(columns, tblInfo.Columns[idxCol.Offset])
+ }
}
+
if tblInfo.IsCommonHandle {
for _, c := range pkCols {
columns = append(columns, tblInfo.Columns[c.Offset])
}
return columns
}
+ if tblInfo.PKIsHandle {
+ columns = append(columns, tblInfo.Columns[tblInfo.GetPkColInfo().Offset])
+ return columns
+ }
handleOffset := len(columns)
handleColsInfo := &model.ColumnInfo{
ID: model.ExtraHandleID,
@@ -522,12 +534,20 @@ func (b *executorBuilder) buildRecoverIndex(v *plannercore.RecoverIndex) Executo
b.err = errors.Errorf("secondary index `%v` is not found in table `%v`", v.IndexName, v.Table.Name.O)
return nil
}
+ var hasGenedCol bool
+ for _, iCol := range index.Meta().Columns {
+ if tblInfo.Columns[iCol.Offset].IsGenerated() {
+ hasGenedCol = true
+ }
+ }
+ cols := buildIdxColsConcatHandleCols(tblInfo, index.Meta(), hasGenedCol)
e := &RecoverIndexExec{
- baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()),
- columns: buildIdxColsConcatHandleCols(tblInfo, index.Meta()),
- index: index,
- table: t,
- physicalID: t.Meta().ID,
+ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()),
+ columns: cols,
+ containsGenedCol: hasGenedCol,
+ index: index,
+ table: t,
+ physicalID: t.Meta().ID,
}
sessCtx := e.ctx.GetSessionVars().StmtCtx
e.handleCols = buildHandleColsForExec(sessCtx, tblInfo, index.Meta(), e.columns)
@@ -584,7 +604,7 @@ func (b *executorBuilder) buildCleanupIndex(v *plannercore.CleanupIndex) Executo
}
e := &CleanupIndexExec{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()),
- columns: buildIdxColsConcatHandleCols(tblInfo, index.Meta()),
+ columns: buildIdxColsConcatHandleCols(tblInfo, index.Meta(), false),
index: index,
table: t,
physicalID: t.Meta().ID,
@@ -777,6 +797,7 @@ func (b *executorBuilder) buildShow(v *plannercore.PhysicalShow) Executor {
Partition: v.Partition,
Column: v.Column,
IndexName: v.IndexName,
+ ResourceGroupName: model.NewCIStr(v.ResourceGroupName),
Flag: v.Flag,
Roles: v.Roles,
User: v.User,
@@ -935,6 +956,7 @@ func (b *executorBuilder) buildLoadData(v *plannercore.LoadData) Executor {
IgnoreLines: v.IgnoreLines,
ColumnAssignments: v.ColumnAssignments,
ColumnsAndUserVars: v.ColumnsAndUserVars,
+ OnDuplicate: v.OnDuplicate,
Ctx: b.ctx,
}
columnNames := loadDataInfo.initFieldMappings()
@@ -945,7 +967,7 @@ func (b *executorBuilder) buildLoadData(v *plannercore.LoadData) Executor {
}
loadDataExec := &LoadDataExec{
baseExecutor: newBaseExecutor(b.ctx, nil, v.ID()),
- IsLocal: v.IsLocal,
+ FileLocRef: v.FileLocRef,
OnDuplicate: v.OnDuplicate,
loadDataInfo: loadDataInfo,
}
@@ -1003,6 +1025,28 @@ func (b *executorBuilder) buildPlanReplayer(v *plannercore.PlanReplayer) Executo
}
return e
}
+ if v.Capture {
+ e := &PlanReplayerExec{
+ baseExecutor: newBaseExecutor(b.ctx, nil, v.ID()),
+ CaptureInfo: &PlanReplayerCaptureInfo{
+ SQLDigest: v.SQLDigest,
+ PlanDigest: v.PlanDigest,
+ },
+ }
+ return e
+ }
+ if v.Remove {
+ e := &PlanReplayerExec{
+ baseExecutor: newBaseExecutor(b.ctx, nil, v.ID()),
+ CaptureInfo: &PlanReplayerCaptureInfo{
+ SQLDigest: v.SQLDigest,
+ PlanDigest: v.PlanDigest,
+ Remove: true,
+ },
+ }
+ return e
+ }
+
e := &PlanReplayerExec{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()),
DumpInfo: &PlanReplayerDumpInfo{
@@ -1075,7 +1119,12 @@ func (b *executorBuilder) setTelemetryInfo(v *plannercore.DDL) {
}
b.Ti.PartitionTelemetry.UseAddIntervalPartition = true
case ast.AlterTableExchangePartition:
- b.Ti.UesExchangePartition = true
+ b.Ti.UseExchangePartition = true
+ case ast.AlterTableReorganizePartition:
+ if b.Ti.PartitionTelemetry == nil {
+ b.Ti.PartitionTelemetry = &PartitionTelemetryInfo{}
+ }
+ b.Ti.PartitionTelemetry.UseReorganizePartition = true
}
}
case *ast.CreateTableStmt:
@@ -1389,17 +1438,6 @@ func (b *executorBuilder) buildMergeJoin(v *plannercore.PhysicalMergeJoin) Execu
return e
}
-func (b *executorBuilder) buildSideEstCount(v *plannercore.PhysicalHashJoin) float64 {
- buildSide := v.Children()[v.InnerChildIdx]
- if v.UseOuterToBuild {
- buildSide = v.Children()[1-v.InnerChildIdx]
- }
- if buildSide.Stats().HistColl == nil || buildSide.Stats().HistColl.Pseudo {
- return 0.0
- }
- return buildSide.StatsCount()
-}
-
func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) Executor {
leftExec := b.build(v.Children()[0])
if b.err != nil {
@@ -1412,12 +1450,19 @@ func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) Executo
}
e := &HashJoinExec{
- baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID(), leftExec, rightExec),
- concurrency: v.Concurrency,
- joinType: v.JoinType,
- isOuterJoin: v.JoinType.IsOuterJoin(),
- useOuterToBuild: v.UseOuterToBuild,
+ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID(), leftExec, rightExec),
+ probeSideTupleFetcher: &probeSideTupleFetcher{},
+ probeWorkers: make([]*probeWorker, v.Concurrency),
+ buildWorker: &buildWorker{},
+ hashJoinCtx: &hashJoinCtx{
+ sessCtx: b.ctx,
+ isOuterJoin: v.JoinType.IsOuterJoin(),
+ useOuterToBuild: v.UseOuterToBuild,
+ joinType: v.JoinType,
+ concurrency: v.Concurrency,
+ },
}
+ e.hashJoinCtx.allocPool = e.AllocPool
defaultValues := v.DefaultValues
lhsTypes, rhsTypes := retTypes(leftExec), retTypes(rightExec)
if v.InnerChildIdx == 1 {
@@ -1435,15 +1480,17 @@ func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) Executo
leftIsBuildSide := true
e.isNullEQ = v.IsNullEQ
+ var probeKeys, probeNAKeys, buildKeys, buildNAKeys []*expression.Column
+ var buildSideExec Executor
if v.UseOuterToBuild {
// update the buildSideEstCount due to changing the build side
if v.InnerChildIdx == 1 {
- e.buildSideExec, e.buildKeys, e.buildNAKeys = leftExec, v.LeftJoinKeys, v.LeftNAJoinKeys
- e.probeSideTupleFetcher.probeSideExec, e.probeKeys, e.probeNAKeys = rightExec, v.RightJoinKeys, v.RightNAJoinKeys
+ buildSideExec, buildKeys, buildNAKeys = leftExec, v.LeftJoinKeys, v.LeftNAJoinKeys
+ e.probeSideTupleFetcher.probeSideExec, probeKeys, probeNAKeys = rightExec, v.RightJoinKeys, v.RightNAJoinKeys
e.outerFilter = v.LeftConditions
} else {
- e.buildSideExec, e.buildKeys, e.buildNAKeys = rightExec, v.RightJoinKeys, v.RightNAJoinKeys
- e.probeSideTupleFetcher.probeSideExec, e.probeKeys, e.probeNAKeys = leftExec, v.LeftJoinKeys, v.LeftNAJoinKeys
+ buildSideExec, buildKeys, buildNAKeys = rightExec, v.RightJoinKeys, v.RightNAJoinKeys
+ e.probeSideTupleFetcher.probeSideExec, probeKeys, probeNAKeys = leftExec, v.LeftJoinKeys, v.LeftNAJoinKeys
e.outerFilter = v.RightConditions
leftIsBuildSide = false
}
@@ -1452,27 +1499,48 @@ func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) Executo
}
} else {
if v.InnerChildIdx == 0 {
- e.buildSideExec, e.buildKeys, e.buildNAKeys = leftExec, v.LeftJoinKeys, v.LeftNAJoinKeys
- e.probeSideTupleFetcher.probeSideExec, e.probeKeys, e.probeNAKeys = rightExec, v.RightJoinKeys, v.RightNAJoinKeys
+ buildSideExec, buildKeys, buildNAKeys = leftExec, v.LeftJoinKeys, v.LeftNAJoinKeys
+ e.probeSideTupleFetcher.probeSideExec, probeKeys, probeNAKeys = rightExec, v.RightJoinKeys, v.RightNAJoinKeys
e.outerFilter = v.RightConditions
} else {
- e.buildSideExec, e.buildKeys, e.buildNAKeys = rightExec, v.RightJoinKeys, v.RightNAJoinKeys
- e.probeSideTupleFetcher.probeSideExec, e.probeKeys, e.probeNAKeys = leftExec, v.LeftJoinKeys, v.LeftNAJoinKeys
+ buildSideExec, buildKeys, buildNAKeys = rightExec, v.RightJoinKeys, v.RightNAJoinKeys
+ e.probeSideTupleFetcher.probeSideExec, probeKeys, probeNAKeys = leftExec, v.LeftJoinKeys, v.LeftNAJoinKeys
e.outerFilter = v.LeftConditions
leftIsBuildSide = false
}
if defaultValues == nil {
- defaultValues = make([]types.Datum, e.buildSideExec.Schema().Len())
+ defaultValues = make([]types.Datum, buildSideExec.Schema().Len())
}
}
+ probeKeyColIdx := make([]int, len(probeKeys))
+ probeNAKeColIdx := make([]int, len(probeNAKeys))
+ buildKeyColIdx := make([]int, len(buildKeys))
+ buildNAKeyColIdx := make([]int, len(buildNAKeys))
+ for i := range buildKeys {
+ buildKeyColIdx[i] = buildKeys[i].Index
+ }
+ for i := range buildNAKeys {
+ buildNAKeyColIdx[i] = buildNAKeys[i].Index
+ }
+ for i := range probeKeys {
+ probeKeyColIdx[i] = probeKeys[i].Index
+ }
+ for i := range probeNAKeys {
+ probeNAKeColIdx[i] = probeNAKeys[i].Index
+ }
isNAJoin := len(v.LeftNAJoinKeys) > 0
- e.buildSideEstCount = b.buildSideEstCount(v)
childrenUsedSchema := markChildrenUsedCols(v.Schema(), v.Children()[0].Schema(), v.Children()[1].Schema())
- e.probeWorkers = make([]probeWorker, e.concurrency)
for i := uint(0); i < e.concurrency; i++ {
- e.probeWorkers[i].joiner = newJoiner(b.ctx, v.JoinType, v.InnerChildIdx == 0, defaultValues,
- v.OtherConditions, lhsTypes, rhsTypes, childrenUsedSchema, isNAJoin)
+ e.probeWorkers[i] = &probeWorker{
+ hashJoinCtx: e.hashJoinCtx,
+ workerID: i,
+ joiner: newJoiner(b.ctx, v.JoinType, v.InnerChildIdx == 0, defaultValues, v.OtherConditions, lhsTypes, rhsTypes, childrenUsedSchema, isNAJoin),
+ probeKeyColIdx: probeKeyColIdx,
+ probeNAKeyColIdx: probeNAKeColIdx,
+ }
}
+ e.buildWorker.buildKeyColIdx, e.buildWorker.buildNAKeyColIdx, e.buildWorker.buildSideExec, e.buildWorker.hashJoinCtx = buildKeyColIdx, buildNAKeyColIdx, buildSideExec, e.hashJoinCtx
+ e.hashJoinCtx.isNullAware = isNAJoin
executorCountHashJoinExec.Inc()
// We should use JoinKey to construct the type information using by hashing, instead of using the child's schema directly.
@@ -1694,7 +1762,7 @@ func (b *executorBuilder) buildTableDual(v *plannercore.PhysicalTableDual) Execu
// `getSnapshotTS` returns for-update-ts if in insert/update/delete/lock statement otherwise the isolation read ts
// Please notice that in RC isolation, the above two ts are the same
-func (b *executorBuilder) getSnapshotTS() (uint64, error) {
+func (b *executorBuilder) getSnapshotTS() (ts uint64, err error) {
if b.forDataReaderBuilder {
return b.dataReaderTS, nil
}
@@ -1885,8 +1953,6 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo
strings.ToLower(infoschema.TableTiFlashReplica),
strings.ToLower(infoschema.TableTiDBServersInfo),
strings.ToLower(infoschema.TableTiKVStoreStatus),
- strings.ToLower(infoschema.TableStatementsSummaryEvicted),
- strings.ToLower(infoschema.ClusterTableStatementsSummaryEvicted),
strings.ToLower(infoschema.TableClientErrorsSummaryGlobal),
strings.ToLower(infoschema.TableClientErrorsSummaryByUser),
strings.ToLower(infoschema.TableClientErrorsSummaryByHost),
@@ -1899,7 +1965,8 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo
strings.ToLower(infoschema.TableMemoryUsage),
strings.ToLower(infoschema.TableMemoryUsageOpsHistory),
strings.ToLower(infoschema.ClusterTableMemoryUsage),
- strings.ToLower(infoschema.ClusterTableMemoryUsageOpsHistory):
+ strings.ToLower(infoschema.ClusterTableMemoryUsageOpsHistory),
+ strings.ToLower(infoschema.TableResourceGroups):
return &MemTableReaderExec{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()),
table: v.Table,
@@ -1940,16 +2007,18 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo
}
case strings.ToLower(infoschema.TableStatementsSummary),
strings.ToLower(infoschema.TableStatementsSummaryHistory),
+ strings.ToLower(infoschema.TableStatementsSummaryEvicted),
+ strings.ToLower(infoschema.ClusterTableStatementsSummary),
strings.ToLower(infoschema.ClusterTableStatementsSummaryHistory),
- strings.ToLower(infoschema.ClusterTableStatementsSummary):
+ strings.ToLower(infoschema.ClusterTableStatementsSummaryEvicted):
+ var extractor *plannercore.StatementsSummaryExtractor
+ if v.Extractor != nil {
+ extractor = v.Extractor.(*plannercore.StatementsSummaryExtractor)
+ }
return &MemTableReaderExec{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()),
table: v.Table,
- retriever: &stmtSummaryTableRetriever{
- table: v.Table,
- columns: v.Columns,
- extractor: v.Extractor.(*plannercore.StatementsSummaryExtractor),
- },
+ retriever: buildStmtSummaryRetriever(b.ctx, v.Table, v.Columns, extractor),
}
case strings.ToLower(infoschema.TableColumns):
return &MemTableReaderExec{
@@ -1963,7 +2032,6 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo
viewOutputNamesMap: make(map[int64]types.NameSlice),
},
}
-
case strings.ToLower(infoschema.TableSlowQuery), strings.ToLower(infoschema.ClusterTableSlowLog):
memTracker := memory.NewTracker(v.ID(), -1)
memTracker.AttachTo(b.ctx.GetSessionVars().StmtCtx.MemTracker)
@@ -2550,7 +2618,7 @@ func (b *executorBuilder) buildAnalyzeSamplingPushdown(task plannercore.AnalyzeC
SampleSize: int64(opts[ast.AnalyzeOptNumSamples]),
SampleRate: sampleRate,
SketchSize: maxSketchSize,
- ColumnsInfo: util.ColumnsToProto(task.ColsInfo, task.TblInfo.PKIsHandle),
+ ColumnsInfo: util.ColumnsToProto(task.ColsInfo, task.TblInfo.PKIsHandle, false),
ColumnGroups: colGroups,
}
if task.TblInfo != nil {
@@ -2711,7 +2779,7 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeCo
BucketSize: int64(opts[ast.AnalyzeOptNumBuckets]),
SampleSize: MaxRegionSampleSize,
SketchSize: maxSketchSize,
- ColumnsInfo: util.ColumnsToProto(cols, task.HandleCols != nil && task.HandleCols.IsInt()),
+ ColumnsInfo: util.ColumnsToProto(cols, task.HandleCols != nil && task.HandleCols.IsInt(), false),
CmsketchDepth: &depth,
CmsketchWidth: &width,
}
@@ -3255,7 +3323,11 @@ func (b *executorBuilder) buildIndexLookUpMergeJoin(v *plannercore.PhysicalIndex
}
func (b *executorBuilder) buildIndexNestedLoopHashJoin(v *plannercore.PhysicalIndexHashJoin) Executor {
- e := b.buildIndexLookUpJoin(&(v.PhysicalIndexJoin)).(*IndexLookUpJoin)
+ join := b.buildIndexLookUpJoin(&(v.PhysicalIndexJoin))
+ if b.err != nil {
+ return nil
+ }
+ e := join.(*IndexLookUpJoin)
idxHash := &IndexNestedLoopHashJoin{
IndexLookUpJoin: *e,
keepOuterOrder: v.KeepOuterOrder,
@@ -3363,22 +3435,22 @@ func (b *executorBuilder) buildMPPGather(v *plannercore.PhysicalTableReader) Exe
b.err = err
return nil
}
+
gather := &MPPGather{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()),
is: b.is,
originalPlan: v.GetTablePlan(),
startTS: startTs,
+ mppQueryID: kv.MPPQueryID{QueryTs: getMPPQueryTS(b.ctx), LocalQueryID: getMPPQueryID(b.ctx), ServerID: domain.GetDomain(b.ctx).ServerID()},
+ memTracker: memory.NewTracker(v.ID(), -1),
}
+ gather.memTracker.AttachTo(b.ctx.GetSessionVars().StmtCtx.MemTracker)
return gather
}
// buildTableReader builds a table reader executor. It first build a no range table reader,
// and then update it ranges from table scan plan.
func (b *executorBuilder) buildTableReader(v *plannercore.PhysicalTableReader) Executor {
- if v.StoreType != kv.TiKV && b.isStaleness {
- b.err = errors.New("stale requests require tikv backend")
- return nil
- }
failpoint.Inject("checkUseMPP", func(val failpoint.Value) {
if !b.ctx.GetSessionVars().InRestrictedSQL && val.(bool) != useMPPExecution(b.ctx, v) {
if val.(bool) {
@@ -3477,17 +3549,43 @@ func buildIndexRangeForEachPartition(ctx sessionctx.Context, usedPartitions []ta
return nextRange, nil
}
-func keyColumnsIncludeAllPartitionColumns(keyColumns []int, pe *tables.PartitionExpr) bool {
- tmp := make(map[int]struct{}, len(keyColumns))
- for _, offset := range keyColumns {
- tmp[offset] = struct{}{}
+func getPartitionKeyColOffsets(keyColIDs []int64, pt table.PartitionedTable) []int {
+ keyColOffsets := make([]int, len(keyColIDs))
+ for i, colID := range keyColIDs {
+ offset := -1
+ for j, col := range pt.Cols() {
+ if colID == col.ID {
+ offset = j
+ break
+ }
+ }
+ if offset == -1 {
+ return nil
+ }
+ keyColOffsets[i] = offset
+ }
+
+ t, ok := pt.(interface {
+ PartitionExpr() *tables.PartitionExpr
+ })
+ if !ok {
+ return nil
+ }
+ pe := t.PartitionExpr()
+ if pe == nil {
+ return nil
+ }
+
+ offsetMap := make(map[int]struct{})
+ for _, offset := range keyColOffsets {
+ offsetMap[offset] = struct{}{}
}
for _, offset := range pe.ColumnOffset {
- if _, ok := tmp[offset]; !ok {
- return false
+ if _, ok := offsetMap[offset]; !ok {
+ return nil
}
}
- return true
+ return keyColOffsets
}
func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table, schema *expression.Schema, partitionInfo *plannercore.PartitionInfo,
@@ -3502,15 +3600,6 @@ func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table
return nil, false, nil, err
}
- // check whether can runtime prune.
- type partitionExpr interface {
- PartitionExpr() (*tables.PartitionExpr, error)
- }
- pe, err := tbl.(partitionExpr).PartitionExpr()
- if err != nil {
- return nil, false, nil, err
- }
-
// recalculate key column offsets
if len(lookUpContent) == 0 {
return nil, false, nil, nil
@@ -3518,37 +3607,17 @@ func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table
if lookUpContent[0].keyColIDs == nil {
return nil, false, nil, plannercore.ErrInternal.GenWithStack("cannot get column IDs when dynamic pruning")
}
- keyColOffsets := make([]int, len(lookUpContent[0].keyColIDs))
- for i, colID := range lookUpContent[0].keyColIDs {
- offset := -1
- for j, col := range partitionTbl.Cols() {
- if colID == col.ID {
- offset = j
- break
- }
- }
- if offset == -1 {
- return nil, false, nil, plannercore.ErrInternal.GenWithStack("invalid column offset when dynamic pruning")
- }
- keyColOffsets[i] = offset
- }
-
- offsetMap := make(map[int]bool)
- for _, offset := range keyColOffsets {
- offsetMap[offset] = true
- }
- for _, offset := range pe.ColumnOffset {
- if _, ok := offsetMap[offset]; !ok {
- return condPruneResult, false, nil, nil
- }
+ keyColOffsets := getPartitionKeyColOffsets(lookUpContent[0].keyColIDs, partitionTbl)
+ if len(keyColOffsets) == 0 {
+ return condPruneResult, false, nil, nil
}
locateKey := make([]types.Datum, len(partitionTbl.Cols()))
partitions := make(map[int64]table.PhysicalTable)
contentPos = make([]int64, len(lookUpContent))
for idx, content := range lookUpContent {
- for i, date := range content.keys {
- locateKey[keyColOffsets[i]] = date
+ for i, data := range content.keys {
+ locateKey[keyColOffsets[i]] = data
}
p, err := partitionTbl.GetPartitionByRow(builder.ctx, locateKey)
if err != nil {
@@ -3813,6 +3882,9 @@ func buildNoRangeIndexLookUpReader(b *executorBuilder, v *plannercore.PhysicalIn
}
func (b *executorBuilder) buildIndexLookUpReader(v *plannercore.PhysicalIndexLookUpReader) Executor {
+ if b.Ti != nil {
+ b.Ti.UseTableLookUp = true
+ }
is := v.IndexPlans[0].(*plannercore.PhysicalIndexScan)
if err := b.validCanReadTemporaryOrCacheTable(is.Table); err != nil {
b.err = err
@@ -3940,6 +4012,7 @@ func buildNoRangeIndexMergeReader(b *executorBuilder, v *plannercore.PhysicalInd
isCorColInPartialFilters: isCorColInPartialFilters,
isCorColInTableFilter: isCorColInTableFilter,
isCorColInPartialAccess: isCorColInPartialAccess,
+ isIntersection: v.IsIntersectionType,
}
collectTable := false
e.tableRequest.CollectRangeCounts = &collectTable
@@ -3947,6 +4020,10 @@ func buildNoRangeIndexMergeReader(b *executorBuilder, v *plannercore.PhysicalInd
}
func (b *executorBuilder) buildIndexMergeReader(v *plannercore.PhysicalIndexMergeReader) Executor {
+ if b.Ti != nil {
+ b.Ti.UseIndexMerge = true
+ b.Ti.UseTableLookUp = true
+ }
ts := v.TablePlans[0].(*plannercore.PhysicalTableScan)
if err := b.validCanReadTemporaryOrCacheTable(ts.Table); err != nil {
b.err = err
@@ -4111,12 +4188,6 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
}
tbl, _ := builder.is.TableByID(tbInfo.ID)
pt := tbl.(table.PartitionedTable)
- pe, err := tbl.(interface {
- PartitionExpr() (*tables.PartitionExpr, error)
- }).PartitionExpr()
- if err != nil {
- return nil, err
- }
partitionInfo := &v.PartitionInfo
usedPartitionList, err := builder.partitionPruning(pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames)
if err != nil {
@@ -4127,15 +4198,19 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
usedPartitions[p.GetPhysicalID()] = p
}
var kvRanges []kv.KeyRange
+ var keyColOffsets []int
+ if len(lookUpContents) > 0 {
+ keyColOffsets = getPartitionKeyColOffsets(lookUpContents[0].keyColIDs, pt)
+ }
if v.IsCommonHandle {
- if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) {
- locateKey := make([]types.Datum, e.Schema().Len())
+ if len(keyColOffsets) > 0 {
+ locateKey := make([]types.Datum, len(pt.Cols()))
kvRanges = make([]kv.KeyRange, 0, len(lookUpContents))
// lookUpContentsByPID groups lookUpContents by pid(partition) so that kv ranges for same partition can be merged.
lookUpContentsByPID := make(map[int64][]*indexJoinLookUpContent)
for _, content := range lookUpContents {
- for i, date := range content.keys {
- locateKey[content.keyCols[i]] = date
+ for i, data := range content.keys {
+ locateKey[keyColOffsets[i]] = data
}
p, err := pt.GetPartitionByRow(e.ctx, locateKey)
if err != nil {
@@ -4174,12 +4249,12 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
handles, lookUpContents := dedupHandles(lookUpContents)
- if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) {
- locateKey := make([]types.Datum, e.Schema().Len())
+ if len(keyColOffsets) > 0 {
+ locateKey := make([]types.Datum, len(pt.Cols()))
kvRanges = make([]kv.KeyRange, 0, len(lookUpContents))
for _, content := range lookUpContents {
- for i, date := range content.keys {
- locateKey[content.keyCols[i]] = date
+ for i, data := range content.keys {
+ locateKey[keyColOffsets[i]] = data
}
p, err := pt.GetPartitionByRow(e.ctx, locateKey)
if err != nil {
@@ -4190,13 +4265,13 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
continue
}
handle := kv.IntHandle(content.keys[0].GetInt64())
- tmp, _ := distsql.TableHandlesToKVRanges(pid, []kv.Handle{handle})
- kvRanges = append(kvRanges, tmp...)
+ ranges, _ := distsql.TableHandlesToKVRanges(pid, []kv.Handle{handle})
+ kvRanges = append(kvRanges, ranges...)
}
} else {
for _, p := range usedPartitionList {
- tmp, _ := distsql.TableHandlesToKVRanges(p.GetPhysicalID(), handles)
- kvRanges = append(kvRanges, tmp...)
+ ranges, _ := distsql.TableHandlesToKVRanges(p.GetPhysicalID(), handles)
+ kvRanges = append(kvRanges, ranges...)
}
}
@@ -4233,32 +4308,37 @@ type kvRangeBuilderFromRangeAndPartition struct {
}
func (h kvRangeBuilderFromRangeAndPartition) buildKeyRangeSeparately(ranges []*ranger.Range) ([]int64, [][]kv.KeyRange, error) {
- ret := make([][]kv.KeyRange, 0, len(h.partitions))
+ ret := make([][]kv.KeyRange, len(h.partitions))
pids := make([]int64, 0, len(h.partitions))
- for _, p := range h.partitions {
+ for i, p := range h.partitions {
pid := p.GetPhysicalID()
+ pids = append(pids, pid)
meta := p.Meta()
+ if len(ranges) == 0 {
+ continue
+ }
kvRange, err := distsql.TableHandleRangesToKVRanges(h.sctx.GetSessionVars().StmtCtx, []int64{pid}, meta != nil && meta.IsCommonHandle, ranges, nil)
if err != nil {
return nil, nil, err
}
- pids = append(pids, pid)
- ret = append(ret, kvRange)
+ ret[i] = kvRange.AppendSelfTo(ret[i])
}
return pids, ret, nil
}
-func (h kvRangeBuilderFromRangeAndPartition) buildKeyRange(ranges []*ranger.Range) ([]kv.KeyRange, error) {
- //nolint: prealloc
- var ret []kv.KeyRange
- for _, p := range h.partitions {
+func (h kvRangeBuilderFromRangeAndPartition) buildKeyRange(ranges []*ranger.Range) ([][]kv.KeyRange, error) {
+ ret := make([][]kv.KeyRange, len(h.partitions))
+ if len(ranges) == 0 {
+ return ret, nil
+ }
+ for i, p := range h.partitions {
pid := p.GetPhysicalID()
meta := p.Meta()
kvRange, err := distsql.TableHandleRangesToKVRanges(h.sctx.GetSessionVars().StmtCtx, []int64{pid}, meta != nil && meta.IsCommonHandle, ranges, nil)
if err != nil {
return nil, err
}
- ret = append(ret, kvRange...)
+ ret[i] = kvRange.AppendSelfTo(ret[i])
}
return ret, nil
}
@@ -4305,7 +4385,7 @@ func (builder *dataReaderBuilder) buildTableReaderBase(ctx context.Context, e *T
if err != nil {
return nil, err
}
- e.kvRanges = append(e.kvRanges, kvReq.KeyRanges...)
+ e.kvRanges = kvReq.KeyRanges.AppendSelfTo(e.kvRanges)
e.resultHandler = &tableResultHandler{}
result, err := builder.SelectResult(ctx, builder.ctx, kvReq, retTypes(e), e.feedback, getPhysicalPlanIDs(e.plans), e.id)
if err != nil {
@@ -4328,6 +4408,8 @@ func (builder *dataReaderBuilder) buildTableReaderFromHandles(ctx context.Contex
} else {
b.SetTableHandles(getPhysicalTableID(e.table), handles)
}
+ } else {
+ b.SetKeyRanges(nil)
}
return builder.buildTableReaderBase(ctx, e, b)
}
@@ -4386,6 +4468,9 @@ func (builder *dataReaderBuilder) buildIndexReaderForIndexJoin(ctx context.Conte
func (builder *dataReaderBuilder) buildIndexLookUpReaderForIndexJoin(ctx context.Context, v *plannercore.PhysicalIndexLookUpReader,
lookUpContents []*indexJoinLookUpContent, indexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager, memTracker *memory.Tracker, interruptSignal *atomic.Value) (Executor, error) {
+ if builder.Ti != nil {
+ builder.Ti.UseTableLookUp = true
+ }
e, err := buildNoRangeIndexLookUpReader(builder.executorBuilder, v)
if err != nil {
return nil, err
@@ -4516,6 +4601,9 @@ func buildRangesForIndexJoin(ctx sessionctx.Context, lookUpContents []*indexJoin
func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, lookUpContents []*indexJoinLookUpContent,
ranges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager, memTracker *memory.Tracker, interruptSignal *atomic.Value) (_ []kv.KeyRange, err error) {
kvRanges := make([]kv.KeyRange, 0, len(ranges)*len(lookUpContents))
+ if len(ranges) == 0 {
+ return []kv.KeyRange{}, nil
+ }
lastPos := len(ranges[0].LowVal) - 1
sc := ctx.GetSessionVars().StmtCtx
tmpDatumRanges := make([]*ranger.Range, 0, len(lookUpContents))
@@ -4528,7 +4616,7 @@ func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, l
}
if cwc == nil {
// Index id is -1 means it's a common handle.
- var tmpKvRanges []kv.KeyRange
+ var tmpKvRanges *kv.KeyRanges
var err error
if indexID == -1 {
tmpKvRanges, err = distsql.CommonHandleRangesToKVRanges(sc, []int64{tableID}, ranges)
@@ -4538,7 +4626,7 @@ func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, l
if err != nil {
return nil, err
}
- kvRanges = append(kvRanges, tmpKvRanges...)
+ kvRanges = tmpKvRanges.AppendSelfTo(kvRanges)
continue
}
nextColRanges, err := cwc.BuildRangesByRow(ctx, content.row)
@@ -4575,9 +4663,11 @@ func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, l
}
// Index id is -1 means it's a common handle.
if indexID == -1 {
- return distsql.CommonHandleRangesToKVRanges(ctx.GetSessionVars().StmtCtx, []int64{tableID}, tmpDatumRanges)
+ tmpKeyRanges, err := distsql.CommonHandleRangesToKVRanges(ctx.GetSessionVars().StmtCtx, []int64{tableID}, tmpDatumRanges)
+ return tmpKeyRanges.FirstPartitionRange(), err
}
- return distsql.IndexRangesToKVRangesWithInterruptSignal(ctx.GetSessionVars().StmtCtx, tableID, indexID, tmpDatumRanges, nil, memTracker, interruptSignal)
+ tmpKeyRanges, err := distsql.IndexRangesToKVRangesWithInterruptSignal(ctx.GetSessionVars().StmtCtx, tableID, indexID, tmpDatumRanges, nil, memTracker, interruptSignal)
+ return tmpKeyRanges.FirstPartitionRange(), err
}
func (b *executorBuilder) buildWindow(v *plannercore.PhysicalWindow) Executor {
@@ -4770,6 +4860,9 @@ func (b *executorBuilder) buildSQLBindExec(v *plannercore.SQLBindPlan) Executor
isGlobal: v.IsGlobal,
bindAst: v.BindStmt,
newStatus: v.NewStatus,
+ source: v.Source,
+ sqlDigest: v.SQLDigest,
+ planDigest: v.PlanDigest,
}
return e
}
@@ -4859,13 +4952,13 @@ func (b *executorBuilder) buildBatchPointGet(plan *plannercore.BatchPointGetPlan
if e.ctx.GetSessionVars().IsReplicaReadClosestAdaptive() {
e.snapshot.SetOption(kv.ReplicaReadAdjuster, newReplicaReadAdjuster(e.ctx, plan.GetAvgRowSize()))
}
+ e.snapshot.SetOption(kv.ResourceGroupName, b.ctx.GetSessionVars().ResourceGroupName)
if e.runtimeStats != nil {
snapshotStats := &txnsnapshot.SnapshotRuntimeStats{}
e.stats = &runtimeStatsWithSnapshot{
SnapshotRuntimeStats: snapshotStats,
}
e.snapshot.SetOption(kv.CollectRuntimeStats, snapshotStats)
- b.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
if plan.IndexInfo != nil {
@@ -5261,6 +5354,10 @@ func (b *executorBuilder) buildCompactTable(v *plannercore.CompactTable) Executo
}
partitionIDs = append(partitionIDs, partitionID)
}
+ if b.Ti.PartitionTelemetry == nil {
+ b.Ti.PartitionTelemetry = &PartitionTelemetryInfo{}
+ }
+ b.Ti.PartitionTelemetry.UseCompactTablePartition = true
}
return &CompactTableTiFlashExec{
diff --git a/executor/checksum.go b/executor/checksum.go
index fa2d02338ec3f..845c1b85d4c66 100644
--- a/executor/checksum.go
+++ b/executor/checksum.go
@@ -246,6 +246,7 @@ func (c *checksumContext) buildTableRequest(ctx sessionctx.Context, tableID int6
SetChecksumRequest(checksum).
SetStartTS(c.StartTs).
SetConcurrency(ctx.GetSessionVars().DistSQLScanConcurrency()).
+ SetResourceGroupName(ctx.GetSessionVars().ResourceGroupName).
Build()
}
@@ -263,6 +264,7 @@ func (c *checksumContext) buildIndexRequest(ctx sessionctx.Context, tableID int6
SetChecksumRequest(checksum).
SetStartTS(c.StartTs).
SetConcurrency(ctx.GetSessionVars().DistSQLScanConcurrency()).
+ SetResourceGroupName(ctx.GetSessionVars().ResourceGroupName).
Build()
}
diff --git a/executor/compiler.go b/executor/compiler.go
index 241b15874e1e2..29c024a9991e6 100644
--- a/executor/compiler.go
+++ b/executor/compiler.go
@@ -18,7 +18,6 @@ import (
"context"
"strings"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/config"
@@ -32,6 +31,7 @@ import (
"github.com/pingcap/tidb/sessiontxn/staleread"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/memory"
+ "github.com/pingcap/tidb/util/tracing"
"go.uber.org/zap"
)
@@ -56,11 +56,9 @@ type Compiler struct {
// Compile compiles an ast.StmtNode to a physical plan.
func (c *Compiler) Compile(ctx context.Context, stmtNode ast.StmtNode) (_ *ExecStmt, err error) {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("executor.Compile", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "executor.Compile")
+ defer r.End()
+
defer func() {
r := recover()
if r == nil {
@@ -154,6 +152,9 @@ func (c *Compiler) Compile(ctx context.Context, stmtNode ast.StmtNode) (_ *ExecS
}
}
}
+ if err = sessiontxn.OptimizeWithPlanAndThenWarmUp(c.Ctx, stmt.Plan); err != nil {
+ return nil, err
+ }
return stmt, nil
}
diff --git a/executor/ddl.go b/executor/ddl.go
index 85918bb7d4f54..28365e2495a74 100644
--- a/executor/ddl.go
+++ b/executor/ddl.go
@@ -204,6 +204,12 @@ func (e *DDLExec) Next(ctx context.Context, req *chunk.Chunk) (err error) {
err = e.executeDropPlacementPolicy(x)
case *ast.AlterPlacementPolicyStmt:
err = e.executeAlterPlacementPolicy(x)
+ case *ast.CreateResourceGroupStmt:
+ err = e.executeCreateResourceGroup(x)
+ case *ast.DropResourceGroupStmt:
+ err = e.executeDropResourceGroup(x)
+ case *ast.AlterResourceGroupStmt:
+ err = e.executeAlterResourceGroup(x)
}
if err != nil {
// If the owner return ErrTableNotExists error when running this DDL, it may be caused by schema changed,
@@ -735,3 +741,24 @@ func (e *DDLExec) executeDropPlacementPolicy(s *ast.DropPlacementPolicyStmt) err
func (e *DDLExec) executeAlterPlacementPolicy(s *ast.AlterPlacementPolicyStmt) error {
return domain.GetDomain(e.ctx).DDL().AlterPlacementPolicy(e.ctx, s)
}
+
+func (e *DDLExec) executeCreateResourceGroup(s *ast.CreateResourceGroupStmt) error {
+ if !variable.EnableResourceControl.Load() {
+ return infoschema.ErrResourceGroupSupportDisabled
+ }
+ return domain.GetDomain(e.ctx).DDL().AddResourceGroup(e.ctx, s)
+}
+
+func (e *DDLExec) executeAlterResourceGroup(s *ast.AlterResourceGroupStmt) error {
+ if !variable.EnableResourceControl.Load() {
+ return infoschema.ErrResourceGroupSupportDisabled
+ }
+ return domain.GetDomain(e.ctx).DDL().AlterResourceGroup(e.ctx, s)
+}
+
+func (e *DDLExec) executeDropResourceGroup(s *ast.DropResourceGroupStmt) error {
+ if !variable.EnableResourceControl.Load() {
+ return infoschema.ErrResourceGroupSupportDisabled
+ }
+ return domain.GetDomain(e.ctx).DDL().DropResourceGroup(e.ctx, s)
+}
diff --git a/executor/ddl_test.go b/executor/ddl_test.go
index d4d0d59d4ab5b..9c19483f0d94c 100644
--- a/executor/ddl_test.go
+++ b/executor/ddl_test.go
@@ -39,6 +39,7 @@ import (
"github.com/pingcap/tidb/parser/terror"
plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/sessionctx/variable/featuretag/distributereorg"
"github.com/pingcap/tidb/sessiontxn"
"github.com/pingcap/tidb/store/mockstore"
"github.com/pingcap/tidb/table"
@@ -79,10 +80,8 @@ func TestInTxnExecDDLFail(t *testing.T) {
tk.MustExec("insert into t values (1);")
tk.MustExec("begin;")
tk.MustExec("insert into t values (1);")
- _, err := tk.Exec("truncate table t;")
- require.EqualError(t, err, "[kv:1062]Duplicate entry '1' for key 't.PRIMARY'")
- result := tk.MustQuery("select count(*) from t")
- result.Check(testkit.Rows("1"))
+ tk.MustGetErrMsg("truncate table t;", "[kv:1062]Duplicate entry '1' for key 't.PRIMARY'")
+ tk.MustQuery("select count(*) from t").Check(testkit.Rows("1"))
}
func TestInTxnExecDDLInvalid(t *testing.T) {
@@ -212,11 +211,9 @@ func TestCreateView(t *testing.T) {
// test create a exist view
tk.MustExec("CREATE VIEW view_t AS select id , name from source_table")
defer tk.MustExec("DROP VIEW IF EXISTS view_t")
- _, err := tk.Exec("CREATE VIEW view_t AS select id , name from source_table")
- require.EqualError(t, err, "[schema:1050]Table 'test.view_t' already exists")
+ tk.MustGetErrMsg("CREATE VIEW view_t AS select id , name from source_table", "[schema:1050]Table 'test.view_t' already exists")
// create view on nonexistent table
- _, err = tk.Exec("create view v1 (c,d) as select a,b from t1")
- require.EqualError(t, err, "[schema:1146]Table 'test.t1' doesn't exist")
+ tk.MustGetErrMsg("create view v1 (c,d) as select a,b from t1", "[schema:1146]Table 'test.t1' doesn't exist")
// simple view
tk.MustExec("create table t1 (a int ,b int)")
tk.MustExec("insert into t1 values (1,2), (1,3), (2,4), (2,5), (3,10)")
@@ -231,26 +228,22 @@ func TestCreateView(t *testing.T) {
// view with select wild card
tk.MustExec("create view v5 as select * from t1")
tk.MustExec("create view v6 (c,d) as select * from t1")
- _, err = tk.Exec("create view v7 (c,d,e) as select * from t1")
- require.Equal(t, dbterror.ErrViewWrongList.Error(), err.Error())
+ tk.MustGetErrCode("create view v7 (c,d,e) as select * from t1", errno.ErrViewWrongList)
// drop multiple views in a statement
tk.MustExec("drop view v1,v2,v3,v4,v5,v6")
// view with variable
tk.MustExec("create view v1 (c,d) as select a,b+@@global.max_user_connections from t1")
- _, err = tk.Exec("create view v1 (c,d) as select a,b from t1 where a = @@global.max_user_connections")
- require.EqualError(t, err, "[schema:1050]Table 'test.v1' already exists")
+ tk.MustGetErrMsg("create view v1 (c,d) as select a,b from t1 where a = @@global.max_user_connections", "[schema:1050]Table 'test.v1' already exists")
tk.MustExec("drop view v1")
// view with different col counts
- _, err = tk.Exec("create view v1 (c,d,e) as select a,b from t1 ")
- require.Equal(t, dbterror.ErrViewWrongList.Error(), err.Error())
- _, err = tk.Exec("create view v1 (c) as select a,b from t1 ")
- require.Equal(t, dbterror.ErrViewWrongList.Error(), err.Error())
+ tk.MustGetErrCode("create view v1 (c,d,e) as select a,b from t1 ", errno.ErrViewWrongList)
+ tk.MustGetErrCode("create view v1 (c) as select a,b from t1 ", errno.ErrViewWrongList)
// view with or_replace flag
tk.MustExec("drop view if exists v1")
tk.MustExec("create view v1 (c,d) as select a,b from t1")
tk.MustExec("create or replace view v1 (c,d) as select a,b from t1 ")
tk.MustExec("create table if not exists t1 (a int ,b int)")
- _, err = tk.Exec("create or replace view t1 as select * from t1")
+ err := tk.ExecToErr("create or replace view t1 as select * from t1")
require.Equal(t, dbterror.ErrWrongObject.GenWithStackByArgs("test", "t1", "VIEW").Error(), err.Error())
// create view using prepare
tk.MustExec(`prepare stmt from "create view v10 (x) as select 1";`)
@@ -259,8 +252,7 @@ func TestCreateView(t *testing.T) {
// create view on union
tk.MustExec("drop table if exists t1, t2")
tk.MustExec("drop view if exists v")
- _, err = tk.Exec("create view v as select * from t1 union select * from t2")
- require.True(t, terror.ErrorEqual(err, infoschema.ErrTableNotExists))
+ tk.MustGetDBError("create view v as select * from t1 union select * from t2", infoschema.ErrTableNotExists)
tk.MustExec("create table t1(a int, b int)")
tk.MustExec("create table t2(a int, b int)")
tk.MustExec("insert into t1 values(1,2), (1,1), (1,2)")
@@ -268,14 +260,12 @@ func TestCreateView(t *testing.T) {
tk.MustExec("create definer='root'@'localhost' view v as select * from t1 union select * from t2")
tk.MustQuery("select * from v").Sort().Check(testkit.Rows("1 1", "1 2", "1 3"))
tk.MustExec("alter table t1 drop column a")
- _, err = tk.Exec("select * from v")
- require.True(t, terror.ErrorEqual(err, plannercore.ErrViewInvalid))
+ tk.MustGetDBError("select * from v", plannercore.ErrViewInvalid)
tk.MustExec("alter table t1 add column a int")
tk.MustQuery("select * from v").Sort().Check(testkit.Rows("1 1", "1 3", " 1", " 2"))
tk.MustExec("alter table t1 drop column a")
tk.MustExec("alter table t2 drop column b")
- _, err = tk.Exec("select * from v")
- require.True(t, terror.ErrorEqual(err, plannercore.ErrViewInvalid))
+ tk.MustGetDBError("select * from v", plannercore.ErrViewInvalid)
tk.MustExec("drop view v")
tk.MustExec("create view v as (select * from t1)")
@@ -294,8 +284,7 @@ func TestCreateView(t *testing.T) {
tk.MustExec("create table test_v_nested(a int)")
tk.MustExec("create definer='root'@'localhost' view v_nested as select * from test_v_nested")
tk.MustExec("create definer='root'@'localhost' view v_nested2 as select * from v_nested")
- _, err = tk.Exec("create or replace definer='root'@'localhost' view v_nested as select * from v_nested2")
- require.True(t, terror.ErrorEqual(err, plannercore.ErrNoSuchTable))
+ tk.MustGetDBError("create or replace definer='root'@'localhost' view v_nested as select * from v_nested2", plannercore.ErrNoSuchTable)
tk.MustExec("drop table test_v_nested")
tk.MustExec("drop view v_nested, v_nested2")
@@ -322,8 +311,7 @@ func TestViewRecursion(t *testing.T) {
tk.MustExec("create definer='root'@'localhost' view recursive_view2 as select * from recursive_view1")
tk.MustExec("drop table t")
tk.MustExec("rename table recursive_view2 to t")
- _, err := tk.Exec("select * from recursive_view1")
- require.True(t, terror.ErrorEqual(err, plannercore.ErrViewRecursive))
+ tk.MustGetDBError("select * from recursive_view1", plannercore.ErrViewRecursive)
tk.MustExec("drop view recursive_view1, t")
}
@@ -333,8 +321,8 @@ func TestIssue16250(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("create table if not exists t(a int)")
tk.MustExec("create view view_issue16250 as select * from t")
- _, err := tk.Exec("truncate table view_issue16250")
- require.EqualError(t, err, "[schema:1146]Table 'test.view_issue16250' doesn't exist")
+ tk.MustGetErrMsg("truncate table view_issue16250",
+ "[schema:1146]Table 'test.view_issue16250' doesn't exist")
}
func TestIssue24771(t *testing.T) {
@@ -564,12 +552,23 @@ func TestAlterTableAddColumn(t *testing.T) {
tk.MustExec("alter table alter_test add column c3 varchar(50) default 'CURRENT_TIMESTAMP'")
tk.MustQuery("select c3 from alter_test").Check(testkit.Rows("CURRENT_TIMESTAMP"))
tk.MustExec("create or replace view alter_view as select c1,c2 from alter_test")
- _, err = tk.Exec("alter table alter_view add column c4 varchar(50)")
+ err = tk.ExecToErr("alter table alter_view add column c4 varchar(50)")
require.Equal(t, dbterror.ErrWrongObject.GenWithStackByArgs("test", "alter_view", "BASE TABLE").Error(), err.Error())
tk.MustExec("drop view alter_view")
tk.MustExec("create sequence alter_seq")
- _, err = tk.Exec("alter table alter_seq add column c int")
+ err = tk.ExecToErr("alter table alter_seq add column c int")
require.Equal(t, dbterror.ErrWrongObject.GenWithStackByArgs("test", "alter_seq", "BASE TABLE").Error(), err.Error())
+ tk.MustExec("alter table alter_test add column c4 date default current_date")
+ now = time.Now().Format(types.DateFormat)
+ r, err = tk.Exec("select c4 from alter_test")
+ require.NoError(t, err)
+ req = r.NewChunk(nil)
+ err = r.Next(context.Background(), req)
+ require.NoError(t, err)
+ row = req.GetRow(0)
+ require.Equal(t, 1, row.Len())
+ require.GreaterOrEqual(t, now, row.GetTime(0).String())
+ require.Nil(t, r.Close())
tk.MustExec("drop sequence alter_seq")
}
@@ -591,11 +590,11 @@ func TestAlterTableAddColumns(t *testing.T) {
require.Nil(t, r.Close())
tk.MustQuery("select c3 from alter_test").Check(testkit.Rows("CURRENT_TIMESTAMP"))
tk.MustExec("create or replace view alter_view as select c1,c2 from alter_test")
- _, err = tk.Exec("alter table alter_view add column (c4 varchar(50), c5 varchar(50))")
+ err = tk.ExecToErr("alter table alter_view add column (c4 varchar(50), c5 varchar(50))")
require.Equal(t, dbterror.ErrWrongObject.GenWithStackByArgs("test", "alter_view", "BASE TABLE").Error(), err.Error())
tk.MustExec("drop view alter_view")
tk.MustExec("create sequence alter_seq")
- _, err = tk.Exec("alter table alter_seq add column (c1 int, c2 varchar(10))")
+ err = tk.ExecToErr("alter table alter_seq add column (c1 int, c2 varchar(10))")
require.Equal(t, dbterror.ErrWrongObject.GenWithStackByArgs("test", "alter_seq", "BASE TABLE").Error(), err.Error())
tk.MustExec("drop sequence alter_seq")
}
@@ -662,8 +661,7 @@ func TestAlterTableModifyColumn(t *testing.T) {
tk.MustExec("drop table if exists modify_column_multiple_collate;")
tk.MustExec("create table modify_column_multiple_collate (a char(1) collate utf8_bin collate utf8_general_ci) charset utf8mb4 collate utf8mb4_bin")
- _, err = tk.Exec("alter table modify_column_multiple_collate modify column a char(1) charset utf8mb4 collate utf8mb4_bin;")
- require.NoError(t, err)
+ tk.MustExec("alter table modify_column_multiple_collate modify column a char(1) charset utf8mb4 collate utf8mb4_bin;")
tt, err = domain.GetDomain(tk.Session()).InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("modify_column_multiple_collate"))
require.NoError(t, err)
require.Equal(t, "utf8mb4", tt.Cols()[0].GetCharset())
@@ -919,10 +917,8 @@ func TestShardRowIDBits(t *testing.T) {
tk.MustExec("insert into t1 values(1)")
// continue inserting will fail.
- _, err = tk.Exec("insert into t1 values(2)")
- require.Truef(t, autoid.ErrAutoincReadFailed.Equal(err), "err:%v", err)
- _, err = tk.Exec("insert into t1 values(3)")
- require.Truef(t, autoid.ErrAutoincReadFailed.Equal(err), "err:%v", err)
+ tk.MustGetDBError("insert into t1 values(2)", autoid.ErrAutoincReadFailed)
+ tk.MustGetDBError("insert into t1 values(3)", autoid.ErrAutoincReadFailed)
}
func TestAutoRandomBitsData(t *testing.T) {
@@ -1131,65 +1127,6 @@ func TestAutoRandomClusteredPrimaryKey(t *testing.T) {
tk.MustQuery("select a from t;").Check(testkit.Rows("1"))
}
-// Test filter different kind of allocators.
-// In special ddl type, for example:
-// 1: ActionRenameTable : it will abandon all the old allocators.
-// 2: ActionRebaseAutoID : it will drop row-id-type allocator.
-// 3: ActionModifyTableAutoIdCache : it will drop row-id-type allocator.
-// 3: ActionRebaseAutoRandomBase : it will drop auto-rand-type allocator.
-func TestFilterDifferentAllocators(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk.MustExec("drop table if exists t")
- tk.MustExec("drop table if exists t1")
-
- tk.MustExec("create table t(a bigint auto_random(5) key, b int auto_increment unique)")
- tk.MustExec("insert into t values()")
- tk.MustQuery("select b from t").Check(testkit.Rows("1"))
- allHandles, err := ddltestutil.ExtractAllTableHandles(tk.Session(), "test", "t")
- require.NoError(t, err)
- require.Equal(t, 1, len(allHandles))
- orderedHandles := testutil.MaskSortHandles(allHandles, 5, mysql.TypeLonglong)
- require.Equal(t, int64(1), orderedHandles[0])
- tk.MustExec("delete from t")
-
- // Test rebase auto_increment.
- tk.MustExec("alter table t auto_increment 3000000")
- tk.MustExec("insert into t values()")
- tk.MustQuery("select b from t").Check(testkit.Rows("3000000"))
- allHandles, err = ddltestutil.ExtractAllTableHandles(tk.Session(), "test", "t")
- require.NoError(t, err)
- require.Equal(t, 1, len(allHandles))
- orderedHandles = testutil.MaskSortHandles(allHandles, 5, mysql.TypeLonglong)
- require.Equal(t, int64(2), orderedHandles[0])
- tk.MustExec("delete from t")
-
- // Test rebase auto_random.
- tk.MustExec("alter table t auto_random_base 3000000")
- tk.MustExec("insert into t values()")
- tk.MustQuery("select b from t").Check(testkit.Rows("3000001"))
- allHandles, err = ddltestutil.ExtractAllTableHandles(tk.Session(), "test", "t")
- require.NoError(t, err)
- require.Equal(t, 1, len(allHandles))
- orderedHandles = testutil.MaskSortHandles(allHandles, 5, mysql.TypeLonglong)
- require.Equal(t, int64(3000000), orderedHandles[0])
- tk.MustExec("delete from t")
-
- // Test rename table.
- tk.MustExec("rename table t to t1")
- tk.MustExec("insert into t1 values()")
- res := tk.MustQuery("select b from t1")
- strInt64, err := strconv.ParseInt(res.Rows()[0][0].(string), 10, 64)
- require.NoError(t, err)
- require.Greater(t, strInt64, int64(3000002))
- allHandles, err = ddltestutil.ExtractAllTableHandles(tk.Session(), "test", "t1")
- require.NoError(t, err)
- require.Equal(t, 1, len(allHandles))
- orderedHandles = testutil.MaskSortHandles(allHandles, 5, mysql.TypeLonglong)
- require.Greater(t, orderedHandles[0], int64(3000001))
-}
-
func TestMaxHandleAddIndex(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -1223,8 +1160,7 @@ func TestSetDDLReorgWorkerCnt(t *testing.T) {
err = ddlutil.LoadDDLReorgVars(context.Background(), tk.Session())
require.NoError(t, err)
require.Equal(t, int32(100), variable.GetDDLReorgWorkerCounter())
- _, err = tk.Exec("set @@global.tidb_ddl_reorg_worker_cnt = invalid_val")
- require.Truef(t, terror.ErrorEqual(err, variable.ErrWrongTypeForVar), "err %v", err)
+ tk.MustGetDBError("set @@global.tidb_ddl_reorg_worker_cnt = invalid_val", variable.ErrWrongTypeForVar)
tk.MustExec("set @@global.tidb_ddl_reorg_worker_cnt = 100")
err = ddlutil.LoadDDLReorgVars(context.Background(), tk.Session())
require.NoError(t, err)
@@ -1266,8 +1202,7 @@ func TestSetDDLReorgBatchSize(t *testing.T) {
err = ddlutil.LoadDDLReorgVars(context.Background(), tk.Session())
require.NoError(t, err)
require.Equal(t, variable.MaxDDLReorgBatchSize, variable.GetDDLReorgBatchSize())
- _, err = tk.Exec("set @@global.tidb_ddl_reorg_batch_size = invalid_val")
- require.True(t, terror.ErrorEqual(err, variable.ErrWrongTypeForVar), "err %v", err)
+ tk.MustGetDBError("set @@global.tidb_ddl_reorg_batch_size = invalid_val", variable.ErrWrongTypeForVar)
tk.MustExec("set @@global.tidb_ddl_reorg_batch_size = 100")
err = ddlutil.LoadDDLReorgVars(context.Background(), tk.Session())
require.NoError(t, err)
@@ -1374,8 +1309,7 @@ func TestSetDDLErrorCountLimit(t *testing.T) {
err = ddlutil.LoadDDLVars(tk.Session())
require.NoError(t, err)
require.Equal(t, int64(math.MaxInt64), variable.GetDDLErrorCountLimit())
- _, err = tk.Exec("set @@global.tidb_ddl_error_count_limit = invalid_val")
- require.True(t, terror.ErrorEqual(err, variable.ErrWrongTypeForVar), "err %v", err)
+ tk.MustGetDBError("set @@global.tidb_ddl_error_count_limit = invalid_val", variable.ErrWrongTypeForVar)
tk.MustExec("set @@global.tidb_ddl_error_count_limit = 100")
err = ddlutil.LoadDDLVars(tk.Session())
require.NoError(t, err)
@@ -1384,6 +1318,20 @@ func TestSetDDLErrorCountLimit(t *testing.T) {
res.Check(testkit.Rows("100"))
}
+func TestLoadDDLDistributeVars(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ require.Equal(t, variable.DefTiDBDDLEnableDistributeReorg, distributereorg.TiDBEnableDistributeReorg)
+
+ tk.MustGetDBError("set @@global.tidb_ddl_distribute_reorg = invalid_val", variable.ErrWrongValueForVar)
+ require.Equal(t, distributereorg.TiDBEnableDistributeReorg, variable.DDLEnableDistributeReorg.Load())
+ tk.MustExec("set @@global.tidb_ddl_distribute_reorg = 'on'")
+ require.Equal(t, true, variable.DDLEnableDistributeReorg.Load())
+ tk.MustExec(fmt.Sprintf("set @@global.tidb_ddl_distribute_reorg = %v", distributereorg.TiDBEnableDistributeReorg))
+ require.Equal(t, distributereorg.TiDBEnableDistributeReorg, variable.DDLEnableDistributeReorg.Load())
+}
+
// Test issue #9205, fix the precision problem for time type default values
// See https://github.com/pingcap/tidb/issues/9205 for details
func TestIssue9205(t *testing.T) {
@@ -1432,39 +1380,21 @@ func TestCheckDefaultFsp(t *testing.T) {
tk.MustExec("use test")
tk.MustExec(`drop table if exists t;`)
- _, err := tk.Exec("create table t ( tt timestamp default now(1));")
- require.EqualError(t, err, "[ddl:1067]Invalid default value for 'tt'")
-
- _, err = tk.Exec("create table t ( tt timestamp(1) default current_timestamp);")
- require.EqualError(t, err, "[ddl:1067]Invalid default value for 'tt'")
-
- _, err = tk.Exec("create table t ( tt timestamp(1) default now(2));")
- require.EqualError(t, err, "[ddl:1067]Invalid default value for 'tt'")
+ tk.MustGetErrMsg("create table t ( tt timestamp default now(1));", "[ddl:1067]Invalid default value for 'tt'")
+ tk.MustGetErrMsg("create table t ( tt timestamp(1) default current_timestamp);", "[ddl:1067]Invalid default value for 'tt'")
+ tk.MustGetErrMsg("create table t ( tt timestamp(1) default now(2));", "[ddl:1067]Invalid default value for 'tt'")
tk.MustExec("create table t ( tt timestamp(1) default now(1));")
tk.MustExec("create table t2 ( tt timestamp default current_timestamp());")
tk.MustExec("create table t3 ( tt timestamp default current_timestamp(0));")
- _, err = tk.Exec("alter table t add column ttt timestamp default now(2);")
- require.EqualError(t, err, "[ddl:1067]Invalid default value for 'ttt'")
-
- _, err = tk.Exec("alter table t add column ttt timestamp(5) default current_timestamp;")
- require.EqualError(t, err, "[ddl:1067]Invalid default value for 'ttt'")
-
- _, err = tk.Exec("alter table t add column ttt timestamp(5) default now(2);")
- require.EqualError(t, err, "[ddl:1067]Invalid default value for 'ttt'")
-
- _, err = tk.Exec("alter table t modify column tt timestamp(1) default now();")
- require.EqualError(t, err, "[ddl:1067]Invalid default value for 'tt'")
-
- _, err = tk.Exec("alter table t modify column tt timestamp(4) default now(5);")
- require.EqualError(t, err, "[ddl:1067]Invalid default value for 'tt'")
-
- _, err = tk.Exec("alter table t change column tt tttt timestamp(4) default now(5);")
- require.EqualError(t, err, "[ddl:1067]Invalid default value for 'tttt'")
-
- _, err = tk.Exec("alter table t change column tt tttt timestamp(1) default now();")
- require.EqualError(t, err, "[ddl:1067]Invalid default value for 'tttt'")
+ tk.MustGetErrMsg("alter table t add column ttt timestamp default now(2);", "[ddl:1067]Invalid default value for 'ttt'")
+ tk.MustGetErrMsg("alter table t add column ttt timestamp(5) default current_timestamp;", "[ddl:1067]Invalid default value for 'ttt'")
+ tk.MustGetErrMsg("alter table t add column ttt timestamp(5) default now(2);", "[ddl:1067]Invalid default value for 'ttt'")
+ tk.MustGetErrMsg("alter table t modify column tt timestamp(1) default now();", "[ddl:1067]Invalid default value for 'tt'")
+ tk.MustGetErrMsg("alter table t modify column tt timestamp(4) default now(5);", "[ddl:1067]Invalid default value for 'tt'")
+ tk.MustGetErrMsg("alter table t change column tt tttt timestamp(4) default now(5);", "[ddl:1067]Invalid default value for 'tttt'")
+ tk.MustGetErrMsg("alter table t change column tt tttt timestamp(1) default now();", "[ddl:1067]Invalid default value for 'tttt'")
}
func TestTimestampMinDefaultValue(t *testing.T) {
@@ -1618,3 +1548,138 @@ func TestRenameMultiTables(t *testing.T) {
tk.MustExec("drop database rename2")
tk.MustExec("drop database rename3")
}
+
+func TestCreateTableWithTTL(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk.MustExec("CREATE TABLE t (created_at datetime) TTL = `created_at` + INTERVAL 5 DAY")
+ tk.MustQuery("SHOW CREATE TABLE t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`created_at` + INTERVAL 5 DAY */ /*T![ttl] TTL_ENABLE='ON' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+ tk.MustExec("DROP TABLE t")
+
+ tk.MustGetErrMsg("CREATE TABLE t (id int) TTL = `id` + INTERVAL 5 DAY", "[ddl:8148]Field 'id' is of a not supported type for TTL config, expect DATETIME, DATE or TIMESTAMP")
+
+ tk.MustGetErrMsg("CREATE TABLE t (id int) TTL_ENABLE = 'ON'", "[ddl:8150]Cannot set TTL_ENABLE on a table without TTL config")
+
+ tk.MustGetErrMsg("CREATE TABLE t (id int) TTL_JOB_INTERVAL = '1h'", "[ddl:8150]Cannot set TTL_JOB_INTERVAL on a table without TTL config")
+
+ tk.MustExec("CREATE TABLE t (created_at datetime) TTL_ENABLE = 'ON' TTL = `created_at` + INTERVAL 1 DAY TTL_ENABLE = 'OFF' TTL_JOB_INTERVAL = '1d'")
+ tk.MustQuery("SHOW CREATE TABLE t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`created_at` + INTERVAL 1 DAY */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1d' */"))
+ tk.MustExec("DROP TABLE t")
+
+ // when multiple ttl and ttl_enable configs are submitted, only the last one will be handled
+ tk.MustExec("CREATE TABLE t (created_at datetime) TTL_ENABLE = 'ON' TTL = `created_at` + INTERVAL 1 DAY TTL = `created_at` + INTERVAL 2 DAY TTL = `created_at` + INTERVAL 3 DAY TTL_ENABLE = 'OFF'")
+ tk.MustQuery("SHOW CREATE TABLE t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`created_at` + INTERVAL 3 DAY */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+ tk.MustExec("DROP TABLE t")
+}
+
+func TestAlterTTLInfo(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk.MustExec("CREATE TABLE t (created_at datetime, updated_at datetime, wrong_type int) TTL = `created_at` + INTERVAL 5 DAY")
+ tk.MustExec("ALTER TABLE t TTL = `updated_at` + INTERVAL 2 YEAR")
+ tk.MustQuery("SHOW CREATE TABLE t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL,\n `updated_at` datetime DEFAULT NULL,\n `wrong_type` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`updated_at` + INTERVAL 2 YEAR */ /*T![ttl] TTL_ENABLE='ON' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+
+ tk.MustExec("ALTER TABLE t TTL_ENABLE = 'OFF'")
+ tk.MustQuery("SHOW CREATE TABLE t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL,\n `updated_at` datetime DEFAULT NULL,\n `wrong_type` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`updated_at` + INTERVAL 2 YEAR */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+
+ tk.MustExec("ALTER TABLE t TTL_JOB_INTERVAL = '1d'")
+ tk.MustQuery("SHOW CREATE TABLE t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL,\n `updated_at` datetime DEFAULT NULL,\n `wrong_type` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`updated_at` + INTERVAL 2 YEAR */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1d' */"))
+
+ tk.MustGetErrMsg("ALTER TABLE t TTL = `not_exist` + INTERVAL 2 YEAR", "[ddl:1054]Unknown column 'not_exist' in 'TTL config'")
+
+ tk.MustGetErrMsg("ALTER TABLE t TTL = `wrong_type` + INTERVAL 2 YEAR", "[ddl:8148]Field 'wrong_type' is of a not supported type for TTL config, expect DATETIME, DATE or TIMESTAMP")
+
+ tk.MustGetErrMsg("ALTER TABLE t DROP COLUMN updated_at", "[ddl:8149]Cannot drop column 'updated_at': needed in TTL config")
+ tk.MustGetErrMsg("ALTER TABLE t CHANGE updated_at updated_at_new INT", "[ddl:8148]Field 'updated_at_new' is of a not supported type for TTL config, expect DATETIME, DATE or TIMESTAMP")
+
+ tk.MustExec("ALTER TABLE t RENAME COLUMN `updated_at` TO `updated_at_2`")
+ tk.MustQuery("SHOW CREATE TABLE t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL,\n `updated_at_2` datetime DEFAULT NULL,\n `wrong_type` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`updated_at_2` + INTERVAL 2 YEAR */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1d' */"))
+
+ tk.MustExec("ALTER TABLE t CHANGE `updated_at_2` `updated_at_3` date")
+ tk.MustQuery("SHOW CREATE TABLE t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL,\n `updated_at_3` date DEFAULT NULL,\n `wrong_type` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`updated_at_3` + INTERVAL 2 YEAR */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1d' */"))
+
+ tk.MustExec("ALTER TABLE t TTL = `updated_at_3` + INTERVAL 3 YEAR")
+ tk.MustQuery("SHOW CREATE TABLE t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL,\n `updated_at_3` date DEFAULT NULL,\n `wrong_type` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`updated_at_3` + INTERVAL 3 YEAR */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1d' */"))
+
+ tk.MustGetErrMsg("ALTER TABLE t TTL_ENABLE = 'OFF' REMOVE TTL", "[ddl:8200]Unsupported multi schema change for alter table ttl")
+
+ tk.MustExec("ALTER TABLE t REMOVE TTL")
+ tk.MustQuery("SHOW CREATE TABLE t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL,\n `updated_at_3` date DEFAULT NULL,\n `wrong_type` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
+
+ tk.MustGetErrMsg("ALTER TABLE t TTL_ENABLE = 'OFF'", "[ddl:8150]Cannot set TTL_ENABLE on a table without TTL config")
+
+ tk.MustGetErrMsg("ALTER TABLE t TTL_JOB_INTERVAL = '1h'", "[ddl:8150]Cannot set TTL_JOB_INTERVAL on a table without TTL config")
+}
+
+func TestDisableTTLForTempTable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk.MustGetDBError("CREATE TEMPORARY TABLE t (created_at datetime) TTL = `created_at` + INTERVAL 5 DAY", dbterror.ErrTempTableNotAllowedWithTTL)
+}
+
+func TestDisableTTLForFKParentTable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ // alter ttl for a FK parent table is not allowed
+ tk.MustExec("set global tidb_enable_foreign_key='ON'")
+ tk.MustExec("CREATE TABLE t (id int primary key, created_at datetime)")
+ tk.MustExec("CREATE TABLE t_1 (t_id int, foreign key fk_t_id(t_id) references t(id))")
+ tk.MustGetDBError("ALTER TABLE t TTL = created_at + INTERVAL 5 YEAR", dbterror.ErrUnsupportedTTLReferencedByFK)
+ tk.MustExec("drop table t,t_1")
+
+ // refuse to reference TTL key when create table
+ tk.MustExec("CREATE TABLE t (id int primary key, created_at datetime) TTL = created_at + INTERVAL 5 YEAR")
+ tk.MustGetDBError("CREATE TABLE t_1 (t_id int, foreign key fk_t_id(t_id) references t(id))", dbterror.ErrUnsupportedTTLReferencedByFK)
+ tk.MustExec("drop table t")
+
+ // refuse to add foreign key reference TTL table
+ tk.MustExec("CREATE TABLE t (id int primary key, created_at datetime) TTL = created_at + INTERVAL 5 YEAR")
+ tk.MustExec("CREATE TABLE t_1 (t_id int)")
+ tk.MustGetDBError("ALTER TABLE t_1 ADD FOREIGN KEY fk_t_id(t_id) references t(id)", dbterror.ErrUnsupportedTTLReferencedByFK)
+ tk.MustExec("drop table t,t_1")
+}
+
+func TestCheckPrimaryKeyForTTLTable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ // create table should fail when pk contains double/float
+ tk.MustGetDBError("create table t1(id float primary key, t timestamp) TTL=`t`+INTERVAL 1 DAY", dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL)
+ tk.MustGetDBError("create table t1(id float(10,2) primary key, t timestamp) TTL=`t`+INTERVAL 1 DAY", dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL)
+ tk.MustGetDBError("create table t1(id double primary key, t timestamp) TTL=`t`+INTERVAL 1 DAY", dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL)
+ tk.MustGetDBError("create table t1(id float(10,2) primary key, t timestamp) TTL=`t`+INTERVAL 1 DAY", dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL)
+ tk.MustGetDBError("create table t1(id1 int, id2 float, t timestamp, primary key(id1, id2)) TTL=`t`+INTERVAL 1 DAY", dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL)
+ tk.MustGetDBError("create table t1(id1 int, id2 double, t timestamp, primary key(id1, id2)) TTL=`t`+INTERVAL 1 DAY", dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL)
+
+ // alter table should fail when pk contains double/float
+ tk.MustExec("create table t1(id float primary key, t timestamp)")
+ tk.MustExec("create table t2(id double primary key, t timestamp)")
+ tk.MustExec("create table t3(id1 int, id2 float, primary key(id1, id2), t timestamp)")
+ tk.MustExec("create table t4(id1 int, id2 double, primary key(id1, id2), t timestamp)")
+ tk.MustGetDBError("alter table t1 TTL=`t`+INTERVAL 1 DAY", dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL)
+ tk.MustGetDBError("alter table t2 TTL=`t`+INTERVAL 1 DAY", dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL)
+ tk.MustGetDBError("alter table t3 TTL=`t`+INTERVAL 1 DAY", dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL)
+ tk.MustGetDBError("alter table t4 TTL=`t`+INTERVAL 1 DAY", dbterror.ErrUnsupportedPrimaryKeyTypeWithTTL)
+
+ // create table should not fail when the pk is not clustered
+ tk.MustExec("create table t11(id float primary key nonclustered, t timestamp) TTL=`t`+INTERVAL 1 DAY")
+ tk.MustExec("create table t12(id double primary key nonclustered, t timestamp) TTL=`t`+INTERVAL 1 DAY")
+ tk.MustExec("create table t13(id1 int, id2 float, t timestamp, primary key(id1, id2) nonclustered) TTL=`t`+INTERVAL 1 DAY")
+
+ // alter table should not fail when the pk is not clustered
+ tk.MustExec("create table t21(id float primary key nonclustered, t timestamp)")
+ tk.MustExec("create table t22(id double primary key nonclustered, t timestamp)")
+ tk.MustExec("create table t23(id1 int, id2 float, t timestamp, primary key(id1, id2) nonclustered)")
+ tk.MustExec("alter table t21 TTL=`t`+INTERVAL 1 DAY")
+ tk.MustExec("alter table t22 TTL=`t`+INTERVAL 1 DAY")
+ tk.MustExec("alter table t23 TTL=`t`+INTERVAL 1 DAY")
+}
diff --git a/executor/delete.go b/executor/delete.go
index 979db825dfb66..3fedb55806364 100644
--- a/executor/delete.go
+++ b/executor/delete.go
@@ -153,7 +153,7 @@ func (e *DeleteExec) doBatchDelete(ctx context.Context) error {
return ErrBatchInsertFail.GenWithStack("BatchDelete failed with error: %v", err)
}
e.memTracker.Consume(-int64(txn.Size()))
- e.ctx.StmtCommit()
+ e.ctx.StmtCommit(ctx)
if err := sessiontxn.NewTxnInStmt(ctx, e.ctx); err != nil {
// We should return a special error for batch insert.
return ErrBatchInsertFail.GenWithStack("BatchDelete failed with error: %v", err)
@@ -175,8 +175,14 @@ func (e *DeleteExec) composeTblRowMap(tblRowMap tableRowMapType, colPosInfos []p
return err
}
// tblRowMap[info.TblID][handle] hold the row datas binding to this table and this handle.
- _, exist := tblRowMap[info.TblID].Get(handle)
- memDelta := tblRowMap[info.TblID].Set(handle, joinedRow[info.Start:info.End])
+ row, exist := tblRowMap[info.TblID].Get(handle)
+ if !exist {
+ row = make([]types.Datum, info.End-info.Start)
+ }
+ for i, d := range joinedRow[info.Start:info.End] {
+ d.Copy(&row[i])
+ }
+ memDelta := tblRowMap[info.TblID].Set(handle, row)
if !exist {
memDelta += types.EstimatedMemUsage(joinedRow, 1)
memDelta += int64(handle.ExtraMemSize())
@@ -192,6 +198,7 @@ func (e *DeleteExec) deleteMultiTablesByChunk(ctx context.Context) error {
fields := retTypes(e.children[0])
chk := tryNewCacheChunk(e.children[0])
memUsageOfChk := int64(0)
+ joinedDatumRowBuffer := make([]types.Datum, len(fields))
for {
e.memTracker.Consume(-memUsageOfChk)
iter := chunk.NewIterator4Chunk(chk)
@@ -206,13 +213,13 @@ func (e *DeleteExec) deleteMultiTablesByChunk(ctx context.Context) error {
e.memTracker.Consume(memUsageOfChk)
for joinedChunkRow := iter.Begin(); joinedChunkRow != iter.End(); joinedChunkRow = iter.Next() {
- joinedDatumRow := joinedChunkRow.GetDatumRow(fields)
- err := e.composeTblRowMap(tblRowMap, colPosInfos, joinedDatumRow)
+ joinedDatumRowBuffer = joinedChunkRow.GetDatumRowWithBuffer(fields, joinedDatumRowBuffer)
+ err := e.composeTblRowMap(tblRowMap, colPosInfos, joinedDatumRowBuffer)
if err != nil {
return err
}
}
- chk = chunk.Renew(chk, e.maxChunkSize)
+ chk = tryNewCacheChunk(e.children[0])
}
return e.removeRowsInTblRowMap(tblRowMap)
@@ -234,26 +241,20 @@ func (e *DeleteExec) removeRowsInTblRowMap(tblRowMap tableRowMapType) error {
}
func (e *DeleteExec) removeRow(ctx sessionctx.Context, t table.Table, h kv.Handle, data []types.Datum) error {
- txnState, err := e.ctx.Txn(false)
- if err != nil {
- return err
- }
- memUsageOfTxnState := txnState.Size()
- err = t.RemoveRecord(ctx, h, data)
+ err := t.RemoveRecord(ctx, h, data)
if err != nil {
return err
}
- err = e.onRemoveRowForFK(ctx, t, data)
+ tid := t.Meta().ID
+ err = onRemoveRowForFK(ctx, data, e.fkChecks[tid], e.fkCascades[tid])
if err != nil {
return err
}
- e.memTracker.Consume(int64(txnState.Size() - memUsageOfTxnState))
ctx.GetSessionVars().StmtCtx.AddAffectedRows(1)
return nil
}
-func (e *DeleteExec) onRemoveRowForFK(ctx sessionctx.Context, t table.Table, data []types.Datum) error {
- fkChecks := e.fkChecks[t.Meta().ID]
+func onRemoveRowForFK(ctx sessionctx.Context, data []types.Datum, fkChecks []*FKCheckExec, fkCascades []*FKCascadeExec) error {
sc := ctx.GetSessionVars().StmtCtx
for _, fkc := range fkChecks {
err := fkc.deleteRowNeedToCheck(sc, data)
@@ -261,7 +262,6 @@ func (e *DeleteExec) onRemoveRowForFK(ctx sessionctx.Context, t table.Table, dat
return err
}
}
- fkCascades := e.fkCascades[t.Meta().ID]
for _, fkc := range fkCascades {
err := fkc.onDeleteRow(sc, data)
if err != nil {
diff --git a/executor/distsql.go b/executor/distsql.go
index 0cef7e66d441e..a96954b9fba6f 100644
--- a/executor/distsql.go
+++ b/executor/distsql.go
@@ -39,6 +39,7 @@ import (
"github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/statistics"
"github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util"
@@ -75,7 +76,9 @@ type lookupTableTask struct {
idxRows *chunk.Chunk
cursor int
- doneCh chan error
+ // after the cop task is built, buildDone will be set to the current instant, for Next wait duration statistic.
+ buildDoneTime time.Time
+ doneCh chan error
// indexOrder map is used to save the original index order for the handles.
// Without this map, the original index order might be lost.
@@ -243,11 +246,18 @@ func (e *IndexReaderExecutor) Next(ctx context.Context, req *chunk.Chunk) error
return err
}
+// TODO: cleanup this method.
func (e *IndexReaderExecutor) buildKeyRanges(sc *stmtctx.StatementContext, ranges []*ranger.Range, physicalID int64) ([]kv.KeyRange, error) {
+ var (
+ rRanges *kv.KeyRanges
+ err error
+ )
if e.index.ID == -1 {
- return distsql.CommonHandleRangesToKVRanges(sc, []int64{physicalID}, ranges)
+ rRanges, err = distsql.CommonHandleRangesToKVRanges(sc, []int64{physicalID}, ranges)
+ } else {
+ rRanges, err = distsql.IndexRangesToKVRanges(sc, physicalID, e.index.ID, ranges, e.feedback)
}
- return distsql.IndexRangesToKVRanges(sc, physicalID, e.index.ID, ranges, e.feedback)
+ return rRanges.FirstPartitionRange(), err
}
// Open implements the Executor Open interface.
@@ -458,9 +468,6 @@ func (e *IndexLookUpExecutor) Open(ctx context.Context) error {
func (e *IndexLookUpExecutor) buildTableKeyRanges() (err error) {
sc := e.ctx.GetSessionVars().StmtCtx
if e.partitionTableMode {
- if e.keepOrder { // this case should be prevented by the optimizer
- return errors.New("invalid execution plan: cannot keep order when accessing a partition table by IndexLookUpReader")
- }
e.feedback.Invalidate() // feedback for partition tables is not ready
e.partitionKVRanges = make([][]kv.KeyRange, 0, len(e.prunedPartitions))
for _, p := range e.prunedPartitions {
@@ -472,7 +479,7 @@ func (e *IndexLookUpExecutor) buildTableKeyRanges() (err error) {
if e.partitionRangeMap != nil && e.partitionRangeMap[physicalID] != nil {
ranges = e.partitionRangeMap[physicalID]
}
- var kvRange []kv.KeyRange
+ var kvRange *kv.KeyRanges
if e.index.ID == -1 {
kvRange, err = distsql.CommonHandleRangesToKVRanges(sc, []int64{physicalID}, ranges)
} else {
@@ -481,15 +488,17 @@ func (e *IndexLookUpExecutor) buildTableKeyRanges() (err error) {
if err != nil {
return err
}
- e.partitionKVRanges = append(e.partitionKVRanges, kvRange)
+ e.partitionKVRanges = append(e.partitionKVRanges, kvRange.FirstPartitionRange())
}
} else {
physicalID := getPhysicalTableID(e.table)
+ var kvRanges *kv.KeyRanges
if e.index.ID == -1 {
- e.kvRanges, err = distsql.CommonHandleRangesToKVRanges(sc, []int64{physicalID}, e.ranges)
+ kvRanges, err = distsql.CommonHandleRangesToKVRanges(sc, []int64{physicalID}, e.ranges)
} else {
- e.kvRanges, err = distsql.IndexRangesToKVRanges(sc, physicalID, e.index.ID, e.ranges, e.feedback)
+ kvRanges, err = distsql.IndexRangesToKVRanges(sc, physicalID, e.index.ID, e.ranges, e.feedback)
}
+ e.kvRanges = kvRanges.FirstPartitionRange()
}
return err
}
@@ -718,6 +727,9 @@ func (e *IndexLookUpExecutor) buildTableReader(ctx context.Context, task *lookup
// Close implements Exec Close interface.
func (e *IndexLookUpExecutor) Close() error {
+ if e.stats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
+ }
e.kvRanges = e.kvRanges[:0]
if e.dummy {
return nil
@@ -780,13 +792,32 @@ func (e *IndexLookUpExecutor) getResultTask() (*lookupTableTask, error) {
if e.resultCurr != nil && e.resultCurr.cursor < len(e.resultCurr.rows) {
return e.resultCurr, nil
}
+ var (
+ enableStats = e.stats != nil
+ start time.Time
+ indexFetchedInstant time.Time
+ )
+ if enableStats {
+ start = time.Now()
+ }
task, ok := <-e.resultCh
if !ok {
return nil, nil
}
+ if enableStats {
+ indexFetchedInstant = time.Now()
+ }
if err := <-task.doneCh; err != nil {
return nil, err
}
+ if enableStats {
+ e.stats.NextWaitIndexScan += indexFetchedInstant.Sub(start)
+ if task.buildDoneTime.After(indexFetchedInstant) {
+ e.stats.NextWaitTableLookUpBuild += task.buildDoneTime.Sub(indexFetchedInstant)
+ indexFetchedInstant = task.buildDoneTime
+ }
+ e.stats.NextWaitTableLookUpResp += time.Since(indexFetchedInstant)
+ }
// Release the memory usage of last task before we handle a new task.
if e.resultCurr != nil {
@@ -802,7 +833,6 @@ func (e *IndexLookUpExecutor) initRuntimeStats() {
indexScanBasicStats: &execdetails.BasicRuntimeStats{},
Concurrency: e.ctx.GetSessionVars().IndexLookupConcurrency(),
}
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
}
@@ -870,7 +900,7 @@ func (w *indexWorker) fetchHandles(ctx context.Context, result distsql.SelectRes
idxID := w.idxLookup.getIndexPlanRootID()
if w.idxLookup.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl != nil {
if idxID != w.idxLookup.id && w.idxLookup.stats != nil {
- w.idxLookup.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(idxID, w.idxLookup.stats.indexScanBasicStats)
+ w.idxLookup.stats.indexScanBasicStats = w.idxLookup.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetBasicRuntimeStats(idxID)
}
}
for {
@@ -1110,6 +1140,10 @@ type IndexLookUpRunTimeStats struct {
TableRowScan int64
TableTaskNum int64
Concurrency int
+ // Record the `Next` call affected wait duration details.
+ NextWaitIndexScan time.Duration
+ NextWaitTableLookUpBuild time.Duration
+ NextWaitTableLookUpResp time.Duration
}
func (e *IndexLookUpRunTimeStats) String() string {
@@ -1133,6 +1167,15 @@ func (e *IndexLookUpRunTimeStats) String() string {
}
buf.WriteString(fmt.Sprintf(" table_task: {total_time: %v, num: %d, concurrency: %d}", execdetails.FormatDuration(time.Duration(tableScan)), tableTaskNum, concurrency))
}
+ if e.NextWaitIndexScan > 0 || e.NextWaitTableLookUpBuild > 0 || e.NextWaitTableLookUpResp > 0 {
+ if buf.Len() > 0 {
+ buf.WriteByte(',')
+ fmt.Fprintf(&buf, " next: {wait_index: %s, wait_table_lookup_build: %s, wait_table_lookup_resp: %s}",
+ execdetails.FormatDuration(e.NextWaitIndexScan),
+ execdetails.FormatDuration(e.NextWaitTableLookUpBuild),
+ execdetails.FormatDuration(e.NextWaitTableLookUpResp))
+ }
+ }
return buf.String()
}
@@ -1153,6 +1196,9 @@ func (e *IndexLookUpRunTimeStats) Merge(other execdetails.RuntimeStats) {
e.TaskWait += tmp.TaskWait
e.TableRowScan += tmp.TableRowScan
e.TableTaskNum += tmp.TableTaskNum
+ e.NextWaitIndexScan += tmp.NextWaitIndexScan
+ e.NextWaitTableLookUpBuild += tmp.NextWaitTableLookUpBuild
+ e.NextWaitTableLookUpResp += tmp.NextWaitTableLookUpResp
}
// Tp implements the RuntimeStats interface.
@@ -1246,36 +1292,27 @@ func (w *tableWorker) compareData(ctx context.Context, task *lookupTableTask, ta
sctx := w.idxLookup.ctx.GetSessionVars().StmtCtx
for i := range vals {
col := w.idxTblCols[i]
- tp := &col.FieldType
- idxVal := idxRow.GetDatum(i, tp)
+ idxVal := idxRow.GetDatum(i, w.idxColTps[i])
tablecodec.TruncateIndexValue(&idxVal, w.idxLookup.index.Columns[i], col.ColumnInfo)
- cmpRes, err := idxVal.Compare(sctx, &vals[i], collators[i])
+ cmpRes, err := tables.CompareIndexAndVal(sctx, vals[i], idxVal, collators[i], col.FieldType.IsArray() && vals[i].Kind() == types.KindMysqlJSON)
if err != nil {
- fts := make([]*types.FieldType, 0, len(w.idxTblCols))
- for _, c := range w.idxTblCols {
- fts = append(fts, &c.FieldType)
- }
return ir().ReportAdminCheckInconsistentWithColInfo(ctx,
handle,
col.Name.O,
- idxRow.GetDatum(i, tp),
+ idxVal,
vals[i],
err,
- &consistency.RecordData{Handle: handle, Values: getDatumRow(&idxRow, fts)},
+ &consistency.RecordData{Handle: handle, Values: getDatumRow(&idxRow, w.idxColTps)},
)
}
if cmpRes != 0 {
- fts := make([]*types.FieldType, 0, len(w.idxTblCols))
- for _, c := range w.idxTblCols {
- fts = append(fts, &c.FieldType)
- }
return ir().ReportAdminCheckInconsistentWithColInfo(ctx,
handle,
col.Name.O,
- idxRow.GetDatum(i, tp),
+ idxRow.GetDatum(i, w.idxColTps[i]),
vals[i],
err,
- &consistency.RecordData{Handle: handle, Values: getDatumRow(&idxRow, fts)},
+ &consistency.RecordData{Handle: handle, Values: getDatumRow(&idxRow, w.idxColTps)},
)
}
}
@@ -1300,6 +1337,7 @@ func getDatumRow(r *chunk.Row, fields []*types.FieldType) []types.Datum {
// Then we hold the returning rows and finish this task.
func (w *tableWorker) executeTask(ctx context.Context, task *lookupTableTask) error {
tableReader, err := w.idxLookup.buildTableReader(ctx, task)
+ task.buildDoneTime = time.Now()
if err != nil {
logutil.Logger(ctx).Error("build table reader failed", zap.Error(err))
return err
diff --git a/executor/distsql_test.go b/executor/distsql_test.go
index 59b3aecc2bb6a..50c4a311a1eb9 100644
--- a/executor/distsql_test.go
+++ b/executor/distsql_test.go
@@ -316,9 +316,8 @@ func TestPartitionTableIndexLookUpReader(t *testing.T) {
tk.MustQuery("select * from t where a>=1 and a<15 order by a").Check(testkit.Rows("1 1", "2 2", "11 11", "12 12"))
tk.MustQuery("select * from t where a>=1 and a<15 order by a limit 1").Check(testkit.Rows("1 1"))
tk.MustQuery("select * from t where a>=1 and a<15 order by a limit 3").Check(testkit.Rows("1 1", "2 2", "11 11"))
- tk.MustQuery("select * from t where a>=1 and a<15 limit 3").Check(testkit.Rows("1 1", "2 2", "11 11"))
- tk.MustQuery("select * from t where a between 1 and 15 limit 3").Check(testkit.Rows("1 1", "2 2", "11 11"))
- tk.MustQuery("select * from t where a between 1 and 15 limit 3 offset 1").Check(testkit.Rows("2 2", "11 11", "12 12"))
+ tk.MustQuery("select * from t where a between 1 and 15 order by a limit 3").Check(testkit.Rows("1 1", "2 2", "11 11"))
+ tk.MustQuery("select * from t where a between 1 and 15 order by a limit 3 offset 1").Check(testkit.Rows("2 2", "11 11", "12 12"))
}
func TestPartitionTableRandomlyIndexLookUpReader(t *testing.T) {
@@ -359,17 +358,24 @@ func TestPartitionTableRandomlyIndexLookUpReader(t *testing.T) {
func TestIndexLookUpStats(t *testing.T) {
stats := &executor.IndexLookUpRunTimeStats{
- FetchHandleTotal: int64(5 * time.Second),
- FetchHandle: int64(2 * time.Second),
- TaskWait: int64(2 * time.Second),
- TableRowScan: int64(2 * time.Second),
- TableTaskNum: 2,
- Concurrency: 1,
+ FetchHandleTotal: int64(5 * time.Second),
+ FetchHandle: int64(2 * time.Second),
+ TaskWait: int64(2 * time.Second),
+ TableRowScan: int64(2 * time.Second),
+ TableTaskNum: 2,
+ Concurrency: 1,
+ NextWaitIndexScan: time.Second,
+ NextWaitTableLookUpBuild: 2 * time.Second,
+ NextWaitTableLookUpResp: 3 * time.Second,
}
- require.Equal(t, "index_task: {total_time: 5s, fetch_handle: 2s, build: 1s, wait: 2s}, table_task: {total_time: 2s, num: 2, concurrency: 1}", stats.String())
+ require.Equal(t, "index_task: {total_time: 5s, fetch_handle: 2s, build: 1s, wait: 2s}"+
+ ", table_task: {total_time: 2s, num: 2, concurrency: 1}"+
+ ", next: {wait_index: 1s, wait_table_lookup_build: 2s, wait_table_lookup_resp: 3s}", stats.String())
require.Equal(t, stats.Clone().String(), stats.String())
stats.Merge(stats.Clone())
- require.Equal(t, "index_task: {total_time: 10s, fetch_handle: 4s, build: 2s, wait: 4s}, table_task: {total_time: 4s, num: 4, concurrency: 1}", stats.String())
+ require.Equal(t, "index_task: {total_time: 10s, fetch_handle: 4s, build: 2s, wait: 4s}"+
+ ", table_task: {total_time: 4s, num: 4, concurrency: 1}"+
+ ", next: {wait_index: 2s, wait_table_lookup_build: 4s, wait_table_lookup_resp: 6s}", stats.String())
}
func TestIndexLookUpGetResultChunk(t *testing.T) {
@@ -634,3 +640,61 @@ func TestCoprocessorPagingReqKeyRangeSorted(t *testing.T) {
tk.MustExec(`set @a=0x61219F79C90D3541F70E, @b=5501707547099269248, @c=0xEC43EFD30131DEA2CB8B, @d="呣丼蒢咿卻鹻铴础湜僂頃dž縍套衞陀碵碼幓9", @e="鹹楞睕堚尛鉌翡佾搁紟精廬姆燵藝潐楻翇慸嵊";`)
tk.MustExec(`execute stmt using @a,@b,@c,@d,@e;`)
}
+
+func TestCoprocessorBatchByStore(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t, t1")
+ tk.MustExec("create table t(id int primary key, c1 int, c2 int, key i(c1))")
+ tk.MustExec(`create table t1(id int primary key, c1 int, c2 int, key i(c1)) partition by range(id) (
+ partition p0 values less than(10000),
+ partition p1 values less than (50000),
+ partition p2 values less than (100000))`)
+ for i := 0; i < 10; i++ {
+ tk.MustExec("insert into t values(?, ?, ?)", i*10000, i*10000, i%2)
+ tk.MustExec("insert into t1 values(?, ?, ?)", i*10000, i*10000, i%2)
+ }
+ tk.MustQuery("split table t between (0) and (100000) regions 20").Check(testkit.Rows("20 1"))
+ tk.MustQuery("split table t1 between (0) and (100000) regions 20").Check(testkit.Rows("60 1"))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/store/copr/setRangesPerTask", "return(1)"))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/store/copr/setRangesPerTask"))
+ }()
+ ranges := []string{
+ "(c1 >= 0 and c1 < 5000)",
+ "(c1 >= 10000 and c1 < 15000)",
+ "(c1 >= 20000 and c1 < 25000)",
+ "(c1 >= 30000 and c1 < 35000)",
+ "(c1 >= 40000 and c1 < 45000)",
+ "(c1 >= 50000 and c1 < 55000)",
+ "(c1 >= 60000 and c1 < 65000)",
+ "(c1 >= 70000 and c1 < 75000)",
+ "(c1 >= 80000 and c1 < 85000)",
+ "(c1 >= 90000 and c1 < 95000)",
+ }
+ evenRows := testkit.Rows("0 0 0", "20000 20000 0", "40000 40000 0", "60000 60000 0", "80000 80000 0")
+ oddRows := testkit.Rows("10000 10000 1", "30000 30000 1", "50000 50000 1", "70000 70000 1", "90000 90000 1")
+ reverseOddRows := testkit.Rows("90000 90000 1", "70000 70000 1", "50000 50000 1", "30000 30000 1", "10000 10000 1")
+ for _, table := range []string{"t", "t1"} {
+ baseSQL := fmt.Sprintf("select * from %s force index(i) where id < 100000 and (%s)", table, strings.Join(ranges, " or "))
+ for _, paging := range []string{"on", "off"} {
+ tk.MustExec("set session tidb_enable_paging=?", paging)
+ for size := 0; size < 10; size++ {
+ tk.MustExec("set session tidb_store_batch_size=?", size)
+ tk.MustQuery(baseSQL + " and c2 = 0").Sort().Check(evenRows)
+ tk.MustQuery(baseSQL + " and c2 = 1").Sort().Check(oddRows)
+ tk.MustQuery(baseSQL + " and c2 = 0 order by c1 asc").Check(evenRows)
+ tk.MustQuery(baseSQL + " and c2 = 1 order by c1 desc").Check(reverseOddRows)
+ // every batched task will get region error and fallback.
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/store/copr/batchCopRegionError", "return"))
+ tk.MustQuery(baseSQL + " and c2 = 0").Sort().Check(evenRows)
+ tk.MustQuery(baseSQL + " and c2 = 1").Sort().Check(oddRows)
+ tk.MustQuery(baseSQL + " and c2 = 0 order by c1 asc").Check(evenRows)
+ tk.MustQuery(baseSQL + " and c2 = 1 order by c1 desc").Check(reverseOddRows)
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/store/copr/batchCopRegionError"))
+ }
+ }
+ }
+}
diff --git a/executor/errors.go b/executor/errors.go
index 4a0c7f9215875..565a712d1c7d9 100644
--- a/executor/errors.go
+++ b/executor/errors.go
@@ -69,8 +69,11 @@ var (
ErrFuncNotEnabled = dbterror.ClassExecutor.NewStdErr(mysql.ErrNotSupportedYet, parser_mysql.Message("%-.32s is not supported. To enable this experimental feature, set '%-.32s' in the configuration file.", nil))
errSavepointNotExists = dbterror.ClassExecutor.NewStd(mysql.ErrSpDoesNotExist)
ErrForeignKeyCascadeDepthExceeded = dbterror.ClassExecutor.NewStd(mysql.ErrForeignKeyCascadeDepthExceeded)
+ ErrPasswordExpireAnonymousUser = dbterror.ClassExecutor.NewStd(mysql.ErrPasswordExpireAnonymousUser)
+ errMustChangePassword = dbterror.ClassExecutor.NewStd(mysql.ErrMustChangePassword)
ErrWrongStringLength = dbterror.ClassDDL.NewStd(mysql.ErrWrongStringLength)
errUnsupportedFlashbackTmpTable = dbterror.ClassDDL.NewStdErr(mysql.ErrUnsupportedDDLOperation, parser_mysql.Message("Recover/flashback table is not supported on temporary tables", nil))
errTruncateWrongInsertValue = dbterror.ClassTable.NewStdErr(mysql.ErrTruncatedWrongValue, parser_mysql.Message("Incorrect %-.32s value: '%-.128s' for column '%.192s' at row %d", nil))
+ ErrExistsInHistoryPassword = dbterror.ClassExecutor.NewStd(mysql.ErrExistsInHistoryPassword)
)
diff --git a/executor/executor.go b/executor/executor.go
index 6401d41e7e77e..e02abeed16d75 100644
--- a/executor/executor.go
+++ b/executor/executor.go
@@ -19,7 +19,6 @@ import (
"fmt"
"math"
"runtime/pprof"
- "runtime/trace"
"strconv"
"strings"
"sync"
@@ -68,6 +67,7 @@ import (
"github.com/pingcap/tidb/util/resourcegrouptag"
"github.com/pingcap/tidb/util/topsql"
topsqlstate "github.com/pingcap/tidb/util/topsql/state"
+ "github.com/pingcap/tidb/util/tracing"
tikverr "github.com/tikv/client-go/v2/error"
tikvstore "github.com/tikv/client-go/v2/kv"
tikvutil "github.com/tikv/client-go/v2/util"
@@ -272,8 +272,7 @@ func newBaseExecutor(ctx sessionctx.Context, schema *expression.Schema, id int,
}
if ctx.GetSessionVars().StmtCtx.RuntimeStatsColl != nil {
if e.id > 0 {
- e.runtimeStats = &execdetails.BasicRuntimeStats{}
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(id, e.runtimeStats)
+ e.runtimeStats = e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetBasicRuntimeStats(id)
}
}
if schema != nil {
@@ -315,15 +314,11 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) error {
if atomic.LoadUint32(&sessVars.Killed) == 1 {
return ErrQueryInterrupted
}
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan(fmt.Sprintf("%T.Next", e), opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
- if trace.IsEnabled() {
- defer trace.StartRegion(ctx, fmt.Sprintf("%T.Next", e)).End()
- }
- if topsqlstate.TopSQLEnabled() && sessVars.StmtCtx.IsSQLAndPlanRegistered.CAS(false, true) {
+
+ r, ctx := tracing.StartRegionEx(ctx, fmt.Sprintf("%T.Next", e))
+ defer r.End()
+
+ if topsqlstate.TopSQLEnabled() && sessVars.StmtCtx.IsSQLAndPlanRegistered.CompareAndSwap(false, true) {
registerSQLAndPlanInExecForTopSQL(sessVars)
}
err := e.Next(ctx, req)
@@ -354,7 +349,7 @@ func (e *CancelDDLJobsExec) Open(ctx context.Context) error {
if err != nil {
return err
}
- e.errs, err = ddl.CancelJobs(newSess, e.ctx.GetStore(), e.jobIDs)
+ e.errs, err = ddl.CancelJobs(newSess, e.jobIDs)
e.releaseSysSession(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), newSess)
return err
}
@@ -399,7 +394,7 @@ func (e *ShowNextRowIDExec) Next(ctx context.Context, req *chunk.Chunk) error {
tblMeta := tbl.Meta()
allocators := tbl.Allocators(e.ctx)
- for _, alloc := range allocators {
+ for _, alloc := range allocators.Allocs {
nextGlobalID, err := alloc.NextGlobalAutoID()
if err != nil {
return err
@@ -407,7 +402,16 @@ func (e *ShowNextRowIDExec) Next(ctx context.Context, req *chunk.Chunk) error {
var colName, idType string
switch alloc.GetType() {
- case autoid.RowIDAllocType, autoid.AutoIncrementType:
+ case autoid.RowIDAllocType:
+ idType = "_TIDB_ROWID"
+ if tblMeta.PKIsHandle {
+ if col := tblMeta.GetAutoIncrementColInfo(); col != nil {
+ colName = col.Name.O
+ }
+ } else {
+ colName = model.ExtraHandleName.O
+ }
+ case autoid.AutoIncrementType:
idType = "AUTO_INCREMENT"
if tblMeta.PKIsHandle {
if col := tblMeta.GetAutoIncrementColInfo(); col != nil {
@@ -587,7 +591,7 @@ func (e *DDLJobRetriever) appendJobToChunk(req *chunk.Chunk, job *model.Job, che
req.AppendInt64(0, job.ID)
req.AppendString(1, schemaName)
req.AppendString(2, tableName)
- req.AppendString(3, subJob.Type.String()+" /* subjob */")
+ req.AppendString(3, subJob.Type.String()+" /* subjob */"+showAddIdxReorgTpInSubJob(subJob))
req.AppendString(4, subJob.SchemaState.String())
req.AppendInt64(5, job.SchemaID)
req.AppendInt64(6, job.TableID)
@@ -612,6 +616,16 @@ func showAddIdxReorgTp(job *model.Job) string {
return ""
}
+func showAddIdxReorgTpInSubJob(subJob *model.SubJob) string {
+ if subJob.Type == model.ActionAddIndex || subJob.Type == model.ActionAddPrimaryKey {
+ tp := subJob.ReorgTp.String()
+ if len(tp) > 0 {
+ return " /* " + tp + " */"
+ }
+ }
+ return ""
+}
+
func ts2Time(timestamp uint64, loc *time.Location) types.Time {
duration := time.Duration(math.Pow10(9-types.DefaultFsp)) * time.Nanosecond
t := model.TSConvert2Time(timestamp)
@@ -666,8 +680,21 @@ func (e *ShowDDLJobQueriesExec) Open(ctx context.Context) error {
return err
}
- e.jobs = append(e.jobs, jobs...)
- e.jobs = append(e.jobs, historyJobs...)
+ appendedJobID := make(map[int64]struct{})
+ // deduplicate job results
+ // for situations when this operation happens at the same time with new DDLs being executed
+ for _, job := range jobs {
+ if _, ok := appendedJobID[job.ID]; !ok {
+ appendedJobID[job.ID] = struct{}{}
+ e.jobs = append(e.jobs, job)
+ }
+ }
+ for _, historyJob := range historyJobs {
+ if _, ok := appendedJobID[historyJob.ID]; !ok {
+ appendedJobID[historyJob.ID] = struct{}{}
+ e.jobs = append(e.jobs, historyJob)
+ }
+ }
return nil
}
@@ -741,8 +768,25 @@ func (e *ShowDDLJobQueriesWithRangeExec) Open(ctx context.Context) error {
return err
}
- e.jobs = append(e.jobs, jobs...)
- e.jobs = append(e.jobs, historyJobs...)
+ appendedJobID := make(map[int64]struct{})
+ // deduplicate job results
+ // for situations when this operation happens at the same time with new DDLs being executed
+ for _, job := range jobs {
+ if _, ok := appendedJobID[job.ID]; !ok {
+ appendedJobID[job.ID] = struct{}{}
+ e.jobs = append(e.jobs, job)
+ }
+ }
+ for _, historyJob := range historyJobs {
+ if _, ok := appendedJobID[historyJob.ID]; !ok {
+ appendedJobID[historyJob.ID] = struct{}{}
+ e.jobs = append(e.jobs, historyJob)
+ }
+ }
+
+ if e.cursor < int(e.offset) {
+ e.cursor = int(e.offset)
+ }
return nil
}
@@ -758,9 +802,12 @@ func (e *ShowDDLJobQueriesWithRangeExec) Next(ctx context.Context, req *chunk.Ch
}
numCurBatch := mathutil.Min(req.Capacity(), len(e.jobs)-e.cursor)
for i := e.cursor; i < e.cursor+numCurBatch; i++ {
- if i >= int(e.offset) && i < int(e.offset+e.limit) {
+ // i is make true to be >= int(e.offset)
+ if i < int(e.offset+e.limit) {
req.AppendString(0, strconv.FormatInt(e.jobs[i].ID, 10))
req.AppendString(1, e.jobs[i].Query)
+ } else {
+ break
}
}
e.cursor += numCurBatch
@@ -951,6 +998,9 @@ func (e *CheckTableExec) Next(ctx context.Context, req *chunk.Chunk) error {
idxNames := make([]string, 0, len(e.indexInfos))
for _, idx := range e.indexInfos {
+ if idx.MVIndex {
+ continue
+ }
idxNames = append(idxNames, idx.Name.O)
}
greater, idxOffset, err := admin.CheckIndicesCount(e.ctx, e.dbName, e.table.Meta().Name.O, idxNames)
@@ -970,7 +1020,13 @@ func (e *CheckTableExec) Next(ctx context.Context, req *chunk.Chunk) error {
// The number of table rows is equal to the number of index rows.
// TODO: Make the value of concurrency adjustable. And we can consider the number of records.
if len(e.srcs) == 1 {
- return e.checkIndexHandle(ctx, e.srcs[0])
+ err = e.checkIndexHandle(ctx, e.srcs[0])
+ if err == nil && e.srcs[0].index.MVIndex {
+ err = e.checkTableRecord(ctx, 0)
+ }
+ if err != nil {
+ return err
+ }
}
taskCh := make(chan *IndexLookUpExecutor, len(e.srcs))
failure := atomicutil.NewBool(false)
@@ -989,6 +1045,14 @@ func (e *CheckTableExec) Next(ctx context.Context, req *chunk.Chunk) error {
select {
case src := <-taskCh:
err1 := e.checkIndexHandle(ctx, src)
+ if err1 == nil && src.index.MVIndex {
+ for offset, idx := range e.indexInfos {
+ if idx.ID == src.index.ID {
+ err1 = e.checkTableRecord(ctx, offset)
+ break
+ }
+ }
+ }
if err1 != nil {
failure.Store(true)
logutil.Logger(ctx).Info("check index handle failed", zap.Error(err1))
@@ -1325,6 +1389,9 @@ type LimitExec struct {
// columnIdxsUsedByChild keep column indexes of child executor used for inline projection
columnIdxsUsedByChild []int
+
+ // Log the close time when opentracing is enabled.
+ span opentracing.Span
}
// Next implements the Executor Next interface.
@@ -1402,13 +1469,29 @@ func (e *LimitExec) Open(ctx context.Context) error {
e.childResult = tryNewCacheChunk(e.children[0])
e.cursor = 0
e.meetFirstBatch = e.begin == 0
+ if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
+ e.span = span
+ }
return nil
}
// Close implements the Executor Close interface.
func (e *LimitExec) Close() error {
+ start := time.Now()
+
e.childResult = nil
- return e.baseExecutor.Close()
+ err := e.baseExecutor.Close()
+
+ elapsed := time.Since(start)
+ if elapsed > time.Millisecond {
+ logutil.BgLogger().Info("limit executor close takes a long time",
+ zap.Duration("elapsed", elapsed))
+ if e.span != nil {
+ span1 := e.span.Tracer().StartSpan("limitExec.Close", opentracing.ChildOf(e.span.Context()), opentracing.StartTime(start))
+ defer span1.Finish()
+ }
+ }
+ return err
}
func (e *LimitExec) adjustRequiredRows(chk *chunk.Chunk) *chunk.Chunk {
@@ -1440,11 +1523,8 @@ func init() {
s.RewritePhaseInfo.DurationPreprocessSubQuery += time.Since(begin)
}(time.Now())
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("executor.EvalSubQuery", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "executor.EvalSubQuery")
+ defer r.End()
e := newExecutorBuilder(sctx, is, nil)
exec := e.build(p)
@@ -1925,7 +2005,7 @@ func (e *UnionExec) Close() error {
func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) {
vars := ctx.GetSessionVars()
var sc *stmtctx.StatementContext
- if vars.TxnCtx.CouldRetry {
+ if vars.TxnCtx.CouldRetry || mysql.HasCursorExistsFlag(vars.Status) {
// Must construct new statement context object, the retry history need context for every statement.
// TODO: Maybe one day we can get rid of transaction retry, then this logic can be deleted.
sc = &stmtctx.StatementContext{}
@@ -1951,8 +2031,13 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) {
sc.UseDynamicPruneMode = false
}
+ sc.StatsLoad.Timeout = 0
+ sc.StatsLoad.NeededItems = nil
+ sc.StatsLoad.ResultCh = nil
+
sc.SysdateIsNow = ctx.GetSessionVars().SysdateIsNow
+ vars.MemTracker.Detach()
vars.MemTracker.UnbindActions()
vars.MemTracker.SetBytesLimit(vars.MemQuotaQuery)
vars.MemTracker.ResetMaxConsumed()
@@ -1963,21 +2048,22 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) {
if _, ok := s.(*ast.AnalyzeTableStmt); ok {
sc.InitMemTracker(memory.LabelForAnalyzeMemory, -1)
vars.MemTracker.SetBytesLimit(-1)
+ vars.MemTracker.AttachTo(GlobalAnalyzeMemoryTracker)
} else {
sc.InitMemTracker(memory.LabelForSQLText, -1)
- logOnQueryExceedMemQuota := domain.GetDomain(ctx).ExpensiveQueryHandle().LogOnQueryExceedMemQuota
- switch variable.OOMAction.Load() {
- case variable.OOMActionCancel:
- action := &memory.PanicOnExceed{ConnID: vars.ConnectionID}
- action.SetLogHook(logOnQueryExceedMemQuota)
- vars.MemTracker.SetActionOnExceed(action)
- case variable.OOMActionLog:
- fallthrough
- default:
- action := &memory.LogOnExceed{ConnID: vars.ConnectionID}
- action.SetLogHook(logOnQueryExceedMemQuota)
- vars.MemTracker.SetActionOnExceed(action)
- }
+ }
+ logOnQueryExceedMemQuota := domain.GetDomain(ctx).ExpensiveQueryHandle().LogOnQueryExceedMemQuota
+ switch variable.OOMAction.Load() {
+ case variable.OOMActionCancel:
+ action := &memory.PanicOnExceed{ConnID: vars.ConnectionID}
+ action.SetLogHook(logOnQueryExceedMemQuota)
+ vars.MemTracker.SetActionOnExceed(action)
+ case variable.OOMActionLog:
+ fallthrough
+ default:
+ action := &memory.LogOnExceed{ConnID: vars.ConnectionID}
+ action.SetLogHook(logOnQueryExceedMemQuota)
+ vars.MemTracker.SetActionOnExceed(action)
}
sc.MemTracker.SessionID = vars.ConnectionID
sc.MemTracker.AttachTo(vars.MemTracker)
@@ -2154,6 +2240,9 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) {
vars.ClearStmtVars()
vars.PrevFoundInBinding = vars.FoundInBinding
vars.FoundInBinding = false
+ vars.DurationWaitTS = 0
+ vars.CurrInsertBatchExtraCols = nil
+ vars.CurrInsertValues = chunk.Row{}
return
}
@@ -2182,39 +2271,6 @@ func ResetUpdateStmtCtx(sc *stmtctx.StatementContext, stmt *ast.UpdateStmt, vars
sc.IgnoreNoPartition = stmt.IgnoreErr
}
-// FillVirtualColumnValue will calculate the virtual column value by evaluating generated
-// expression using rows from a chunk, and then fill this value into the chunk
-func FillVirtualColumnValue(virtualRetTypes []*types.FieldType, virtualColumnIndex []int,
- schema *expression.Schema, columns []*model.ColumnInfo, sctx sessionctx.Context, req *chunk.Chunk) error {
- if len(virtualColumnIndex) == 0 {
- return nil
- }
-
- virCols := chunk.NewChunkWithCapacity(virtualRetTypes, req.Capacity())
- iter := chunk.NewIterator4Chunk(req)
- for i, idx := range virtualColumnIndex {
- for row := iter.Begin(); row != iter.End(); row = iter.Next() {
- datum, err := schema.Columns[idx].EvalVirtualColumn(row)
- if err != nil {
- return err
- }
- // Because the expression might return different type from
- // the generated column, we should wrap a CAST on the result.
- castDatum, err := table.CastValue(sctx, datum, columns[idx], false, true)
- if err != nil {
- return err
- }
- // Handle the bad null error.
- if (mysql.HasNotNullFlag(columns[idx].GetFlag()) || mysql.HasPreventNullInsertFlag(columns[idx].GetFlag())) && castDatum.IsNull() {
- castDatum = table.GetZeroValue(columns[idx])
- }
- virCols.AppendDatum(i, &castDatum)
- }
- req.SetCol(idx, virCols.Column(i))
- }
- return nil
-}
-
func setOptionForTopSQL(sc *stmtctx.StatementContext, snapshot kv.Snapshot) {
if snapshot == nil {
return
diff --git a/executor/executor_required_rows_test.go b/executor/executor_required_rows_test.go
index cbca9914b5bc2..c3ac762050d24 100644
--- a/executor/executor_required_rows_test.go
+++ b/executor/executor_required_rows_test.go
@@ -22,6 +22,7 @@ import (
"testing"
"time"
+ "github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/expression/aggregation"
"github.com/pingcap/tidb/parser/ast"
@@ -211,6 +212,7 @@ func defaultCtx() sessionctx.Context {
ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(-1, ctx.GetSessionVars().MemQuotaQuery)
ctx.GetSessionVars().StmtCtx.DiskTracker = disk.NewTracker(-1, -1)
ctx.GetSessionVars().SnapshotTS = uint64(1)
+ domain.BindDomain(ctx, domain.NewMockDomain())
return ctx
}
diff --git a/executor/executor_test.go b/executor/executor_test.go
index 641f56817cd60..5e170213c4461 100644
--- a/executor/executor_test.go
+++ b/executor/executor_test.go
@@ -66,6 +66,7 @@ import (
"github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/mock"
+ "github.com/pingcap/tidb/util/replayer"
"github.com/pingcap/tidb/util/rowcodec"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/pingcap/tidb/util/timeutil"
@@ -81,12 +82,15 @@ func checkFileName(s string) bool {
"meta.txt",
"stats/test.t_dump_single.json",
"schema/test.t_dump_single.schema.txt",
+ "schema/schema_meta.txt",
"table_tiflash_replica.txt",
"variables.toml",
"session_bindings.sql",
"global_bindings.sql",
"sql/sql0.sql",
"explain/sql0.txt",
+ "statsMem/test.t_dump_single.txt",
+ "sql_meta.toml",
}
for _, f := range files {
if strings.Compare(f, s) == 0 {
@@ -174,6 +178,70 @@ func TestPlanReplayer(t *testing.T) {
require.Len(t, rows, 1)
}
+func TestPlanReplayerCaptureSEM(t *testing.T) {
+ originSEM := config.GetGlobalConfig().Security.EnableSEM
+ defer func() {
+ config.GetGlobalConfig().Security.EnableSEM = originSEM
+ }()
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("plan replayer capture '123' '123';")
+ tk.MustExec("create table t(id int)")
+ tk.MustQuery("plan replayer dump explain select * from t")
+ tk.MustQuery("select count(*) from mysql.plan_replayer_status").Check(testkit.Rows("1"))
+}
+
+func TestPlanReplayerCapture(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("plan replayer capture '123' '123';")
+ tk.MustQuery("select sql_digest, plan_digest from mysql.plan_replayer_task;").Check(testkit.Rows("123 123"))
+ tk.MustGetErrMsg("plan replayer capture '123' '123';", "plan replayer capture task already exists")
+ tk.MustExec("plan replayer capture remove '123' '123'")
+ tk.MustQuery("select count(*) from mysql.plan_replayer_task;").Check(testkit.Rows("0"))
+ tk.MustExec("create table t(id int)")
+ tk.MustExec("prepare stmt from 'update t set id = ? where id = ? + 1';")
+ tk.MustExec("SET @number = 5;")
+ tk.MustExec("execute stmt using @number,@number")
+ _, sqlDigest := tk.Session().GetSessionVars().StmtCtx.SQLDigest()
+ _, planDigest := tk.Session().GetSessionVars().StmtCtx.GetPlanDigest()
+ tk.MustExec("SET @@tidb_enable_plan_replayer_capture = ON;")
+ tk.MustExec(fmt.Sprintf("plan replayer capture '%v' '%v'", sqlDigest.String(), planDigest.String()))
+ err := dom.GetPlanReplayerHandle().CollectPlanReplayerTask()
+ require.NoError(t, err)
+ tk.MustExec("execute stmt using @number,@number")
+ task := dom.GetPlanReplayerHandle().DrainTask()
+ require.NotNil(t, task)
+}
+
+func TestPlanReplayerContinuesCapture(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("set @@global.tidb_enable_historical_stats='OFF'")
+ _, err := tk.Exec("set @@global.tidb_enable_plan_replayer_continues_capture='ON'")
+ require.Error(t, err)
+ require.Equal(t, err.Error(), "tidb_enable_historical_stats should be enabled before enabling tidb_enable_plan_replayer_continues_capture")
+
+ tk.MustExec("set @@global.tidb_enable_historical_stats='ON'")
+ tk.MustExec("set @@global.tidb_enable_plan_replayer_continues_capture='ON'")
+
+ prHandle := dom.GetPlanReplayerHandle()
+ tk.MustExec("delete from mysql.plan_replayer_status;")
+ tk.MustExec("use test")
+ tk.MustExec("create table t(id int);")
+ tk.MustExec("set @@tidb_enable_plan_replayer_continues_capture = 'ON'")
+ tk.MustQuery("select * from t;")
+ task := prHandle.DrainTask()
+ require.NotNil(t, task)
+ worker := prHandle.GetWorker()
+ success := worker.HandleTask(task)
+ require.True(t, success)
+ tk.MustQuery("select count(*) from mysql.plan_replayer_status").Check(testkit.Rows("1"))
+}
+
func TestShow(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -190,13 +258,16 @@ func TestShow(t *testing.T) {
tk.MustQuery("show create database test_show").Check(testkit.Rows("test_show CREATE DATABASE `test_show` /*!40100 DEFAULT CHARACTER SET utf8mb4 */"))
tk.MustQuery("show privileges").Check(testkit.Rows("Alter Tables To alter the table",
"Alter routine Functions,Procedures To alter or drop stored functions/procedures",
+ "Config Server Admin To use SHOW CONFIG and SET CONFIG statements",
"Create Databases,Tables,Indexes To create new databases and tables",
"Create routine Databases To use CREATE FUNCTION/PROCEDURE",
+ "Create role Server Admin To create new roles",
"Create temporary tables Databases To use CREATE TEMPORARY TABLE",
"Create view Tables To create new views",
"Create user Server Admin To create new users",
"Delete Tables To delete existing rows",
"Drop Databases,Tables To drop databases, tables, and views",
+ "Drop role Server Admin To drop roles",
"Event Server Admin To create, alter, drop and execute events",
"Execute Functions,Procedures To execute stored routines",
"File File access on server To read and write files on the server",
@@ -233,6 +304,7 @@ func TestShow(t *testing.T) {
"RESTRICTED_USER_ADMIN Server Admin ",
"RESTRICTED_CONNECTION_ADMIN Server Admin ",
"RESTRICTED_REPLICA_WRITER_ADMIN Server Admin ",
+ "RESOURCE_GROUP_ADMIN Server Admin ",
))
require.Len(t, tk.MustQuery("show table status").Rows(), 1)
}
@@ -1477,6 +1549,21 @@ func TestSetOperation(t *testing.T) {
tk.MustQuery("explain " + tt).Check(testkit.Rows(output[i].Plan...))
tk.MustQuery(tt).Sort().Check(testkit.Rows(output[i].Res...))
}
+
+ // from https://github.com/pingcap/tidb/issues/40279
+ tk.MustExec("CREATE TABLE `issue40279` (`a` char(155) NOT NULL DEFAULT 'on1unvbxp5sko6mbetn3ku26tuiyju7w3wc0olzto9ew7gsrx',`b` mediumint(9) NOT NULL DEFAULT '2525518',PRIMARY KEY (`b`,`a`) /*T![clustered_index] CLUSTERED */);")
+ tk.MustExec("insert into `issue40279` values ();")
+ tk.MustQuery("( select `issue40279`.`b` as r0 , from_base64( `issue40279`.`a` ) as r1 from `issue40279` ) " +
+ "except ( " +
+ "select `issue40279`.`a` as r0 , elt(2, `issue40279`.`a` , `issue40279`.`a` ) as r1 from `issue40279`);").
+ Check(testkit.Rows("2525518 "))
+ tk.MustExec("drop table if exists t2")
+
+ tk.MustExec("CREATE TABLE `t2` ( `a` varchar(20) CHARACTER SET gbk COLLATE gbk_chinese_ci DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")
+ tk.MustExec("insert into t2 values(0xCED2)")
+ result := tk.MustQuery("(select elt(2,t2.a,t2.a) from t2) except (select 0xCED2 from t2)")
+ rows := result.Rows()
+ require.Len(t, rows, 0)
}
func TestSetOperationOnDiffColType(t *testing.T) {
@@ -1560,7 +1647,7 @@ func TestPlanReplayerDumpSingle(t *testing.T) {
res := tk.MustQuery("plan replayer dump explain select * from t_dump_single")
path := testdata.ConvertRowsToStrings(res.Rows())
- reader, err := zip.OpenReader(filepath.Join(domain.GetPlanReplayerDirName(), path[0]))
+ reader, err := zip.OpenReader(filepath.Join(replayer.GetPlanReplayerDirName(), path[0]))
require.NoError(t, err)
defer func() { require.NoError(t, reader.Close()) }()
for _, file := range reader.File {
@@ -1913,7 +2000,7 @@ func TestCheckIndex(t *testing.T) {
tbInfo := tbl.Meta()
alloc := autoid.NewAllocator(store, dbInfo.ID, tbInfo.ID, false, autoid.RowIDAllocType)
- tb, err := tables.TableFromMeta(autoid.NewAllocators(alloc), tbInfo)
+ tb, err := tables.TableFromMeta(autoid.NewAllocators(false, alloc), tbInfo)
require.NoError(t, err)
_, err = se.Execute(context.Background(), "admin check index t c")
@@ -2042,6 +2129,8 @@ func TestIncorrectLimitArg(t *testing.T) {
tk.MustGetErrMsg(`execute stmt1 using @a;`, `[planner:1210]Incorrect arguments to LIMIT`)
tk.MustGetErrMsg(`execute stmt2 using @b, @a;`, `[planner:1210]Incorrect arguments to LIMIT`)
+ tk.MustGetErrMsg(`execute stmt2 using @a, @b;`, `[planner:1210]Incorrect arguments to LIMIT`)
+ tk.MustGetErrMsg(`execute stmt2 using @a, @a;`, `[planner:1210]Incorrect arguments to LIMIT`)
}
func TestExecutorLimit(t *testing.T) {
@@ -2415,7 +2504,7 @@ func TestTimestampDefaultValueTimeZone(t *testing.T) {
tk.MustExec(`set time_zone = '+00:00'`)
timeIn0 := tk.MustQuery("select b from t").Rows()[0][0]
require.NotEqual(t, timeIn8, timeIn0)
- datumTimeIn8, err := expression.GetTimeValue(tk.Session(), timeIn8, mysql.TypeTimestamp, 0)
+ datumTimeIn8, err := expression.GetTimeValue(tk.Session(), timeIn8, mysql.TypeTimestamp, 0, nil)
require.NoError(t, err)
tIn8To0 := datumTimeIn8.GetMysqlTime()
timeZoneIn8, err := time.LoadLocation("Asia/Shanghai")
@@ -3578,10 +3667,10 @@ func TestPointGetPreparedPlan(t *testing.T) {
pspk1Id, _, _, err := tk.Session().PrepareStmt("select * from t where a = ?")
require.NoError(t, err)
- tk.Session().GetSessionVars().PreparedStmts[pspk1Id].(*plannercore.PlanCacheStmt).PreparedAst.UseCache = false
+ tk.Session().GetSessionVars().PreparedStmts[pspk1Id].(*plannercore.PlanCacheStmt).StmtCacheable = false
pspk2Id, _, _, err := tk.Session().PrepareStmt("select * from t where ? = a ")
require.NoError(t, err)
- tk.Session().GetSessionVars().PreparedStmts[pspk2Id].(*plannercore.PlanCacheStmt).PreparedAst.UseCache = false
+ tk.Session().GetSessionVars().PreparedStmts[pspk2Id].(*plannercore.PlanCacheStmt).StmtCacheable = false
ctx := context.Background()
// first time plan generated
@@ -3621,7 +3710,7 @@ func TestPointGetPreparedPlan(t *testing.T) {
// unique index
psuk1Id, _, _, err := tk.Session().PrepareStmt("select * from t where b = ? ")
require.NoError(t, err)
- tk.Session().GetSessionVars().PreparedStmts[psuk1Id].(*plannercore.PlanCacheStmt).PreparedAst.UseCache = false
+ tk.Session().GetSessionVars().PreparedStmts[psuk1Id].(*plannercore.PlanCacheStmt).StmtCacheable = false
rs, err = tk.Session().ExecutePreparedStmt(ctx, psuk1Id, expression.Args2Expressions4Test(1))
require.NoError(t, err)
@@ -3739,7 +3828,7 @@ func TestPointGetPreparedPlanWithCommitMode(t *testing.T) {
pspk1Id, _, _, err := tk1.Session().PrepareStmt("select * from t where a = ?")
require.NoError(t, err)
- tk1.Session().GetSessionVars().PreparedStmts[pspk1Id].(*plannercore.PlanCacheStmt).PreparedAst.UseCache = false
+ tk1.Session().GetSessionVars().PreparedStmts[pspk1Id].(*plannercore.PlanCacheStmt).StmtCacheable = false
ctx := context.Background()
// first time plan generated
@@ -3805,11 +3894,11 @@ func TestPointUpdatePreparedPlan(t *testing.T) {
updateID1, pc, _, err := tk.Session().PrepareStmt(`update t set c = c + 1 where a = ?`)
require.NoError(t, err)
- tk.Session().GetSessionVars().PreparedStmts[updateID1].(*plannercore.PlanCacheStmt).PreparedAst.UseCache = false
+ tk.Session().GetSessionVars().PreparedStmts[updateID1].(*plannercore.PlanCacheStmt).StmtCacheable = false
require.Equal(t, 1, pc)
updateID2, pc, _, err := tk.Session().PrepareStmt(`update t set c = c + 2 where ? = a`)
require.NoError(t, err)
- tk.Session().GetSessionVars().PreparedStmts[updateID2].(*plannercore.PlanCacheStmt).PreparedAst.UseCache = false
+ tk.Session().GetSessionVars().PreparedStmts[updateID2].(*plannercore.PlanCacheStmt).StmtCacheable = false
require.Equal(t, 1, pc)
ctx := context.Background()
@@ -3844,7 +3933,7 @@ func TestPointUpdatePreparedPlan(t *testing.T) {
// unique index
updUkID1, _, _, err := tk.Session().PrepareStmt(`update t set c = c + 10 where b = ?`)
require.NoError(t, err)
- tk.Session().GetSessionVars().PreparedStmts[updUkID1].(*plannercore.PlanCacheStmt).PreparedAst.UseCache = false
+ tk.Session().GetSessionVars().PreparedStmts[updUkID1].(*plannercore.PlanCacheStmt).StmtCacheable = false
rs, err = tk.Session().ExecutePreparedStmt(ctx, updUkID1, expression.Args2Expressions4Test(3))
require.Nil(t, rs)
require.NoError(t, err)
@@ -3913,7 +4002,7 @@ func TestPointUpdatePreparedPlanWithCommitMode(t *testing.T) {
ctx := context.Background()
updateID1, _, _, err := tk1.Session().PrepareStmt(`update t set c = c + 1 where a = ?`)
- tk1.Session().GetSessionVars().PreparedStmts[updateID1].(*plannercore.PlanCacheStmt).PreparedAst.UseCache = false
+ tk1.Session().GetSessionVars().PreparedStmts[updateID1].(*plannercore.PlanCacheStmt).StmtCacheable = false
require.NoError(t, err)
// first time plan generated
@@ -4320,7 +4409,7 @@ func TestAdminShowDDLJobs(t *testing.T) {
require.NoError(t, err)
err = meta.NewMeta(txn).AddHistoryDDLJob(job, true)
require.NoError(t, err)
- tk.Session().StmtCommit()
+ tk.Session().StmtCommit(context.Background())
re = tk.MustQuery("admin show ddl jobs 1")
row = re.Rows()[0]
@@ -4594,13 +4683,10 @@ func TestUnion2(t *testing.T) {
terr = errors.Cause(err).(*terror.Error)
require.Equal(t, errors.ErrCode(mysql.ErrWrongUsage), terr.Code())
- _, err = tk.Exec("(select a from t order by a) union all select a from t limit 1 union all select a from t limit 1")
- require.Truef(t, terror.ErrorEqual(err, plannercore.ErrWrongUsage), "err %v", err)
+ tk.MustGetDBError("(select a from t order by a) union all select a from t limit 1 union all select a from t limit 1", plannercore.ErrWrongUsage)
- _, err = tk.Exec("(select a from t limit 1) union all select a from t limit 1")
- require.NoError(t, err)
- _, err = tk.Exec("(select a from t order by a) union all select a from t order by a")
- require.NoError(t, err)
+ tk.MustExec("(select a from t limit 1) union all select a from t limit 1")
+ tk.MustExec("(select a from t order by a) union all select a from t order by a")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int)")
@@ -4671,8 +4757,8 @@ func TestUnion2(t *testing.T) {
tk.MustExec("insert into t2 values(3,'c'),(4,'d'),(5,'f'),(6,'e')")
tk.MustExec("analyze table t1")
tk.MustExec("analyze table t2")
- _, err = tk.Exec("(select a,b from t1 limit 2) union all (select a,b from t2 order by a limit 1) order by t1.b")
- require.Equal(t, "[planner:1250]Table 't1' from one of the SELECTs cannot be used in global ORDER clause", err.Error())
+ tk.MustGetErrMsg("(select a,b from t1 limit 2) union all (select a,b from t2 order by a limit 1) order by t1.b",
+ "[planner:1250]Table 't1' from one of the SELECTs cannot be used in global ORDER clause")
// #issue 9900
tk.MustExec("drop table if exists t")
@@ -4826,15 +4912,11 @@ func TestSQLMode(t *testing.T) {
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (a tinyint not null)")
tk.MustExec("set sql_mode = 'STRICT_TRANS_TABLES'")
- _, err := tk.Exec("insert t values ()")
- require.Error(t, err)
-
- _, err = tk.Exec("insert t values ('1000')")
- require.Error(t, err)
+ tk.ExecToErr("insert t values ()")
+ tk.ExecToErr("insert t values ('1000')")
tk.MustExec("create table if not exists tdouble (a double(3,2))")
- _, err = tk.Exec("insert tdouble values (10.23)")
- require.Error(t, err)
+ tk.ExecToErr("insert tdouble values (10.23)")
tk.MustExec("set sql_mode = ''")
tk.MustExec("insert t values ()")
@@ -4862,8 +4944,7 @@ func TestSQLMode(t *testing.T) {
tk2.MustQuery("select * from t2").Check(testkit.Rows("abc"))
// session1 is still in strict mode.
- _, err = tk.Exec("insert t2 values ('abcd')")
- require.Error(t, err)
+ tk.ExecToErr("insert t2 values ('abcd')")
// Restore original global strict mode.
tk.MustExec("set @@global.sql_mode = 'STRICT_TRANS_TABLES'")
}
@@ -5502,6 +5583,8 @@ func TestAdmin(t *testing.T) {
}))
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
tk.MustExec("drop table if exists admin_test")
tk.MustExec("create table admin_test (c1 int, c2 int, c3 int default 1, index (c1))")
tk.MustExec("insert admin_test (c1) values (1),(2),(NULL)")
@@ -5611,6 +5694,68 @@ func TestAdmin(t *testing.T) {
result.Check(testkit.Rows(fmt.Sprintf("%d %s", historyJobs[2].ID, historyJobs[2].Query), fmt.Sprintf("%d %s", historyJobs[3].ID, historyJobs[3].Query), fmt.Sprintf("%d %s", historyJobs[4].ID, historyJobs[4].Query)))
require.NoError(t, err)
+ // check situations when `admin show ddl job 20` happens at the same time with new DDLs being executed
+ var wg sync.WaitGroup
+ wg.Add(2)
+ flag := true
+ go func() {
+ defer wg.Done()
+ for i := 0; i < 10; i++ {
+ tk.MustExec("drop table if exists admin_test9")
+ tk.MustExec("create table admin_test9 (c1 int, c2 int, c3 int default 1, index (c1))")
+ }
+ }()
+ go func() {
+ // check that the result set has no duplication
+ defer wg.Done()
+ for i := 0; i < 10; i++ {
+ result := tk2.MustQuery(`admin show ddl job queries 20`)
+ rows := result.Rows()
+ rowIDs := make(map[string]struct{})
+ for _, row := range rows {
+ rowID := fmt.Sprintf("%v", row[0])
+ if _, ok := rowIDs[rowID]; ok {
+ flag = false
+ return
+ }
+ rowIDs[rowID] = struct{}{}
+ }
+ }
+ }()
+ wg.Wait()
+ require.True(t, flag)
+
+ // check situations when `admin show ddl job queries limit 3 offset 2` happens at the same time with new DDLs being executed
+ var wg2 sync.WaitGroup
+ wg2.Add(2)
+ flag = true
+ go func() {
+ defer wg2.Done()
+ for i := 0; i < 10; i++ {
+ tk.MustExec("drop table if exists admin_test9")
+ tk.MustExec("create table admin_test9 (c1 int, c2 int, c3 int default 1, index (c1))")
+ }
+ }()
+ go func() {
+ // check that the result set has no duplication
+ defer wg2.Done()
+ for i := 0; i < 10; i++ {
+ result := tk2.MustQuery(`admin show ddl job queries limit 3 offset 2`)
+ rows := result.Rows()
+ rowIDs := make(map[string]struct{})
+ for _, row := range rows {
+ rowID := fmt.Sprintf("%v", row[0])
+ if _, ok := rowIDs[rowID]; ok {
+ flag = false
+ return
+ }
+ rowIDs[rowID] = struct{}{}
+ }
+ }
+ }()
+ wg2.Wait()
+ require.True(t, flag)
+
// check table test
tk.MustExec("create table admin_test1 (c1 int, c2 int default 1, index (c1))")
tk.MustExec("insert admin_test1 (c1) values (21),(22)")
@@ -5947,6 +6092,8 @@ func TestSummaryFailedUpdate(t *testing.T) {
tk.Session().SetSessionManager(sm)
dom.ExpensiveQueryHandle().SetSessionManager(sm)
defer tk.MustExec("SET GLOBAL tidb_mem_oom_action = DEFAULT")
+ tk.MustQuery("select variable_value from mysql.GLOBAL_VARIABLES where variable_name = 'tidb_mem_oom_action'").Check(testkit.Rows("LOG"))
+
tk.MustExec("SET GLOBAL tidb_mem_oom_action='CANCEL'")
require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
tk.MustExec("set @@tidb_mem_quota_query=1")
@@ -6226,8 +6373,7 @@ func TestSessionRootTrackerDetach(t *testing.T) {
tk.MustExec("create table t(a int, b int, index idx(a))")
tk.MustExec("create table t1(a int, c int, index idx(a))")
tk.MustExec("set tidb_mem_quota_query=10")
- err := tk.ExecToErr("select /*+hash_join(t1)*/ t.a, t1.a from t use index(idx), t1 use index(idx) where t.a = t1.a")
- require.Contains(t, err.Error(), "Out Of Memory Quota!")
+ tk.MustContainErrMsg("select /*+hash_join(t1)*/ t.a, t1.a from t use index(idx), t1 use index(idx) where t.a = t1.a", "Out Of Memory Quota!")
tk.MustExec("set tidb_mem_quota_query=1000")
rs, err := tk.Exec("select /*+hash_join(t1)*/ t.a, t1.a from t use index(idx), t1 use index(idx) where t.a = t1.a")
require.NoError(t, err)
@@ -6236,3 +6382,165 @@ func TestSessionRootTrackerDetach(t *testing.T) {
require.NoError(t, err)
require.Nil(t, tk.Session().GetSessionVars().MemTracker.GetFallbackForTest(false))
}
+
+func TestIssue39211(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t;")
+ tk.MustExec("drop table if exists s;")
+
+ tk.MustExec("CREATE TABLE `t` ( `a` int(11) DEFAULT NULL, `b` int(11) DEFAULT NULL);")
+ tk.MustExec("CREATE TABLE `s` ( `a` int(11) DEFAULT NULL, `b` int(11) DEFAULT NULL);")
+ tk.MustExec("insert into t values(1,1),(2,2);")
+ tk.MustExec("insert into t select * from t;")
+ tk.MustExec("insert into t select * from t;")
+ tk.MustExec("insert into t select * from t;")
+ tk.MustExec("insert into t select * from t;")
+ tk.MustExec("insert into t select * from t;")
+ tk.MustExec("insert into t select * from t;")
+ tk.MustExec("insert into t select * from t;")
+ tk.MustExec("insert into t select * from t;")
+
+ tk.MustExec("insert into s values(3,3),(4,4),(1,null),(2,null),(null,null);")
+ tk.MustExec("insert into s select * from s;")
+ tk.MustExec("insert into s select * from s;")
+ tk.MustExec("insert into s select * from s;")
+ tk.MustExec("insert into s select * from s;")
+ tk.MustExec("insert into s select * from s;")
+
+ tk.MustExec("set @@tidb_max_chunk_size=32;")
+ tk.MustExec("set @@tidb_enable_null_aware_anti_join=true;")
+ tk.MustQuery("select * from t where (a,b) not in (select a, b from s);").Check(testkit.Rows())
+}
+
+func TestPlanReplayerDumpTPCDS(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec(`create table catalog_sales
+(
+ cs_sold_date_sk int ,
+ cs_sold_time_sk int ,
+ cs_ship_date_sk int ,
+ cs_bill_customer_sk int ,
+ cs_bill_cdemo_sk int ,
+ cs_bill_hdemo_sk int ,
+ cs_bill_addr_sk int ,
+ cs_ship_customer_sk int ,
+ cs_ship_cdemo_sk int ,
+ cs_ship_hdemo_sk int ,
+ cs_ship_addr_sk int ,
+ cs_call_center_sk int ,
+ cs_catalog_page_sk int ,
+ cs_ship_mode_sk int ,
+ cs_warehouse_sk int ,
+ cs_item_sk int not null,
+ cs_promo_sk int ,
+ cs_order_number int not null,
+ cs_quantity int ,
+ cs_wholesale_cost decimal(7,2) ,
+ cs_list_price decimal(7,2) ,
+ cs_sales_price decimal(7,2) ,
+ cs_ext_discount_amt decimal(7,2) ,
+ cs_ext_sales_price decimal(7,2) ,
+ cs_ext_wholesale_cost decimal(7,2) ,
+ cs_ext_list_price decimal(7,2) ,
+ cs_ext_tax decimal(7,2) ,
+ cs_coupon_amt decimal(7,2) ,
+ cs_ext_ship_cost decimal(7,2) ,
+ cs_net_paid decimal(7,2) ,
+ cs_net_paid_inc_tax decimal(7,2) ,
+ cs_net_paid_inc_ship decimal(7,2) ,
+ cs_net_paid_inc_ship_tax decimal(7,2) ,
+ cs_net_profit decimal(7,2) ,
+ primary key (cs_item_sk, cs_order_number)
+);`)
+ tk.MustExec(`create table store_sales
+(
+ ss_sold_date_sk int ,
+ ss_sold_time_sk int ,
+ ss_item_sk int not null,
+ ss_customer_sk int ,
+ ss_cdemo_sk int ,
+ ss_hdemo_sk int ,
+ ss_addr_sk int ,
+ ss_store_sk int ,
+ ss_promo_sk int ,
+ ss_ticket_number int not null,
+ ss_quantity int ,
+ ss_wholesale_cost decimal(7,2) ,
+ ss_list_price decimal(7,2) ,
+ ss_sales_price decimal(7,2) ,
+ ss_ext_discount_amt decimal(7,2) ,
+ ss_ext_sales_price decimal(7,2) ,
+ ss_ext_wholesale_cost decimal(7,2) ,
+ ss_ext_list_price decimal(7,2) ,
+ ss_ext_tax decimal(7,2) ,
+ ss_coupon_amt decimal(7,2) ,
+ ss_net_paid decimal(7,2) ,
+ ss_net_paid_inc_tax decimal(7,2) ,
+ ss_net_profit decimal(7,2) ,
+ primary key (ss_item_sk, ss_ticket_number)
+);`)
+ tk.MustExec(`create table date_dim
+(
+ d_date_sk int not null,
+ d_date_id char(16) not null,
+ d_date date ,
+ d_month_seq int ,
+ d_week_seq int ,
+ d_quarter_seq int ,
+ d_year int ,
+ d_dow int ,
+ d_moy int ,
+ d_dom int ,
+ d_qoy int ,
+ d_fy_year int ,
+ d_fy_quarter_seq int ,
+ d_fy_week_seq int ,
+ d_day_name char(9) ,
+ d_quarter_name char(6) ,
+ d_holiday char(1) ,
+ d_weekend char(1) ,
+ d_following_holiday char(1) ,
+ d_first_dom int ,
+ d_last_dom int ,
+ d_same_day_ly int ,
+ d_same_day_lq int ,
+ d_current_day char(1) ,
+ d_current_week char(1) ,
+ d_current_month char(1) ,
+ d_current_quarter char(1) ,
+ d_current_year char(1) ,
+ primary key (d_date_sk)
+);`)
+ tk.MustQuery(`plan replayer dump explain with ssci as (
+select ss_customer_sk customer_sk
+ ,ss_item_sk item_sk
+from store_sales,date_dim
+where ss_sold_date_sk = d_date_sk
+ and d_month_seq between 1212 and 1212 + 11
+group by ss_customer_sk
+ ,ss_item_sk),
+csci as(
+ select cs_bill_customer_sk customer_sk
+ ,cs_item_sk item_sk
+from catalog_sales,date_dim
+where cs_sold_date_sk = d_date_sk
+ and d_month_seq between 1212 and 1212 + 11
+group by cs_bill_customer_sk
+ ,cs_item_sk)
+ select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only
+ ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only
+ ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog
+from ssci left join csci on (ssci.customer_sk=csci.customer_sk
+ and ssci.item_sk = csci.item_sk)
+UNION
+ select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only
+ ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only
+ ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog
+from ssci right join csci on (ssci.customer_sk=csci.customer_sk
+ and ssci.item_sk = csci.item_sk)
+limit 100;`)
+}
diff --git a/executor/explain_test.go b/executor/explain_test.go
index c67c61a67a909..28dbbbe12fcf6 100644
--- a/executor/explain_test.go
+++ b/executor/explain_test.go
@@ -16,6 +16,7 @@ package executor_test
import (
"bytes"
+ "encoding/json"
"fmt"
"regexp"
"strconv"
@@ -246,6 +247,7 @@ func TestExplainAnalyzeExecutionInfo(t *testing.T) {
checkExecutionInfo(t, tk, "explain analyze select * from t use index(k)")
checkExecutionInfo(t, tk, "explain analyze with recursive cte(a) as (select 1 union select a + 1 from cte where a < 1000) select * from cte;")
+ tk.MustExec("set @@foreign_key_checks=0")
tk.MustExec("CREATE TABLE IF NOT EXISTS nation ( N_NATIONKEY BIGINT NOT NULL,N_NAME CHAR(25) NOT NULL,N_REGIONKEY BIGINT NOT NULL,N_COMMENT VARCHAR(152),PRIMARY KEY (N_NATIONKEY));")
tk.MustExec("CREATE TABLE IF NOT EXISTS part ( P_PARTKEY BIGINT NOT NULL,P_NAME VARCHAR(55) NOT NULL,P_MFGR CHAR(25) NOT NULL,P_BRAND CHAR(10) NOT NULL,P_TYPE VARCHAR(25) NOT NULL,P_SIZE BIGINT NOT NULL,P_CONTAINER CHAR(10) NOT NULL,P_RETAILPRICE DECIMAL(15,2) NOT NULL,P_COMMENT VARCHAR(23) NOT NULL,PRIMARY KEY (P_PARTKEY));")
tk.MustExec("CREATE TABLE IF NOT EXISTS supplier ( S_SUPPKEY BIGINT NOT NULL,S_NAME CHAR(25) NOT NULL,S_ADDRESS VARCHAR(40) NOT NULL,S_NATIONKEY BIGINT NOT NULL,S_PHONE CHAR(15) NOT NULL,S_ACCTBAL DECIMAL(15,2) NOT NULL,S_COMMENT VARCHAR(101) NOT NULL,PRIMARY KEY (S_SUPPKEY),CONSTRAINT FOREIGN KEY SUPPLIER_FK1 (S_NATIONKEY) references nation(N_NATIONKEY));")
@@ -364,7 +366,7 @@ func TestCheckActRowsWithUnistore(t *testing.T) {
},
{
sql: "select count(*) from t_unistore_act_rows group by b",
- expected: []string{"2", "2", "2", "4"},
+ expected: []string{"2", "4", "4"},
},
{
sql: "with cte(a) as (select a from t_unistore_act_rows) select (select 1 from cte limit 1) from cte;",
@@ -515,3 +517,96 @@ func TestIssue35105(t *testing.T) {
require.Error(t, tk.ExecToErr("explain analyze insert into t values (1), (2), (3)"))
tk.MustQuery("select * from t").Check(testkit.Rows("2"))
}
+
+func flatJSONPlan(j *plannercore.ExplainInfoForEncode) (res []*plannercore.ExplainInfoForEncode) {
+ if j == nil {
+ return
+ }
+ res = append(res, j)
+ for _, child := range j.SubOperators {
+ res = append(res, flatJSONPlan(child)...)
+ }
+ return
+}
+
+func TestExplainJSON(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1, t2")
+ tk.MustExec("create table t1(id int, key(id))")
+ tk.MustExec("create table t2(id int, key(id))")
+ cases := []string{
+ "select * from t1",
+ "select count(*) from t2",
+ "select * from t1, t2 where t1.id = t2.id",
+ "select /*+ merge_join(t1, t2)*/ * from t1, t2 where t1.id = t2.id",
+ "with top10 as ( select * from t1 order by id desc limit 10 ) select * from top10 where id in (1,2)",
+ "insert into t1 values(1)",
+ "delete from t2 where t2.id > 10",
+ "update t2 set id = 1 where id =2",
+ "select * from t1 where t1.id < (select sum(t2.id) from t2 where t2.id = t1.id)",
+ }
+ // test syntax
+ tk.MustExec("explain format = 'tidb_json' select * from t1")
+ tk.MustExec("explain format = tidb_json select * from t1")
+ tk.MustExec("explain format = 'TIDB_JSON' select * from t1")
+ tk.MustExec("explain format = TIDB_JSON select * from t1")
+ tk.MustExec("explain analyze format = 'tidb_json' select * from t1")
+ tk.MustExec("explain analyze format = tidb_json select * from t1")
+ tk.MustExec("explain analyze format = 'TIDB_JSON' select * from t1")
+ tk.MustExec("explain analyze format = TIDB_JSON select * from t1")
+
+ // explain
+ for _, sql := range cases {
+ jsonForamt := "explain format = tidb_json " + sql
+ rowForamt := "explain format = row " + sql
+ resJSON := tk.MustQuery(jsonForamt).Rows()
+ resRow := tk.MustQuery(rowForamt).Rows()
+
+ j := new([]*plannercore.ExplainInfoForEncode)
+ require.NoError(t, json.Unmarshal([]byte(resJSON[0][0].(string)), j))
+ var flatJSONRows []*plannercore.ExplainInfoForEncode
+ for _, row := range *j {
+ flatJSONRows = append(flatJSONRows, flatJSONPlan(row)...)
+ }
+ require.Equal(t, len(flatJSONRows), len(resRow))
+
+ for i, row := range resRow {
+ require.Contains(t, row[0], flatJSONRows[i].ID)
+ require.Equal(t, flatJSONRows[i].EstRows, row[1])
+ require.Equal(t, flatJSONRows[i].TaskType, row[2])
+ require.Equal(t, flatJSONRows[i].AccessObject, row[3])
+ require.Equal(t, flatJSONRows[i].OperatorInfo, row[4])
+ }
+ }
+
+ // explain analyze
+ for _, sql := range cases {
+ jsonForamt := "explain analyze format = tidb_json " + sql
+ rowForamt := "explain analyze format = row " + sql
+ resJSON := tk.MustQuery(jsonForamt).Rows()
+ resRow := tk.MustQuery(rowForamt).Rows()
+
+ j := new([]*plannercore.ExplainInfoForEncode)
+ require.NoError(t, json.Unmarshal([]byte(resJSON[0][0].(string)), j))
+ var flatJSONRows []*plannercore.ExplainInfoForEncode
+ for _, row := range *j {
+ flatJSONRows = append(flatJSONRows, flatJSONPlan(row)...)
+ }
+ require.Equal(t, len(flatJSONRows), len(resRow))
+
+ for i, row := range resRow {
+ require.Contains(t, row[0], flatJSONRows[i].ID)
+ require.Equal(t, flatJSONRows[i].EstRows, row[1])
+ require.Equal(t, flatJSONRows[i].ActRows, row[2])
+ require.Equal(t, flatJSONRows[i].TaskType, row[3])
+ require.Equal(t, flatJSONRows[i].AccessObject, row[4])
+ require.Equal(t, flatJSONRows[i].OperatorInfo, row[6])
+ // executeInfo, memory, disk maybe vary in multi execution
+ require.NotEqual(t, flatJSONRows[i].ExecuteInfo, "")
+ require.NotEqual(t, flatJSONRows[i].MemoryInfo, "")
+ require.NotEqual(t, flatJSONRows[i].DiskInfo, "")
+ }
+ }
+}
diff --git a/executor/explainfor_test.go b/executor/explainfor_test.go
index 85652a5f04ddb..ddb0578338c6f 100644
--- a/executor/explainfor_test.go
+++ b/executor/explainfor_test.go
@@ -16,6 +16,7 @@ package executor_test
import (
"bytes"
+ "encoding/json"
"fmt"
"strconv"
"testing"
@@ -550,9 +551,9 @@ func TestIssue28259(t *testing.T) {
ps = []*util.ProcessInfo{tkProcess}
tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
res = tk.MustQuery("explain for connection " + strconv.FormatUint(tkProcess.ID, 10))
- require.Len(t, res.Rows(), 4)
- require.Regexp(t, ".*Selection.*", res.Rows()[0][0])
- require.Regexp(t, ".*IndexFullScan.*", res.Rows()[3][0])
+ require.Len(t, res.Rows(), 3)
+ require.Regexp(t, ".*Selection.*", res.Rows()[1][0])
+ require.Regexp(t, ".*IndexFullScan.*", res.Rows()[2][0])
res = tk.MustQuery("explain format = 'brief' select col1 from UK_GCOL_VIRTUAL_18588 use index(UK_COL1) " +
"where col1 between -1696020282760139948 and -2619168038882941276 or col1 < -4004648990067362699;")
@@ -588,11 +589,9 @@ func TestIssue28259(t *testing.T) {
ps = []*util.ProcessInfo{tkProcess}
tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
res = tk.MustQuery("explain for connection " + strconv.FormatUint(tkProcess.ID, 10))
- require.Len(t, res.Rows(), 5)
- require.Regexp(t, ".*Selection.*", res.Rows()[1][0])
- require.Equal(t, "lt(test.t.b, 1), or(and(ge(test.t.a, 2), le(test.t.a, 1)), lt(test.t.a, 1))", res.Rows()[1][4])
- require.Regexp(t, ".*IndexReader.*", res.Rows()[2][0])
- require.Regexp(t, ".*IndexRangeScan.*", res.Rows()[4][0])
+ require.Len(t, res.Rows(), 4)
+ require.Regexp(t, ".*Selection.*", res.Rows()[2][0])
+ require.Regexp(t, ".*IndexRangeScan.*", res.Rows()[3][0])
res = tk.MustQuery("explain format = 'brief' select a from t use index(idx) " +
"where (a between 0 and 2 or a < 2) and b < 1;")
@@ -635,12 +634,11 @@ func TestIssue28259(t *testing.T) {
ps = []*util.ProcessInfo{tkProcess}
tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
res = tk.MustQuery("explain for connection " + strconv.FormatUint(tkProcess.ID, 10))
- require.Len(t, res.Rows(), 6)
- require.Regexp(t, ".*Selection.*", res.Rows()[1][0])
- require.Regexp(t, ".*IndexLookUp.*", res.Rows()[2][0])
- require.Regexp(t, ".*IndexRangeScan.*", res.Rows()[3][0])
- require.Regexp(t, ".*Selection.*", res.Rows()[4][0])
- require.Regexp(t, ".*TableRowIDScan.*", res.Rows()[5][0])
+ require.Len(t, res.Rows(), 5)
+ require.Regexp(t, ".*IndexLookUp.*", res.Rows()[1][0])
+ require.Regexp(t, ".*IndexRangeScan.*", res.Rows()[2][0])
+ require.Regexp(t, ".*Selection.*", res.Rows()[3][0])
+ require.Regexp(t, ".*TableRowIDScan.*", res.Rows()[4][0])
res = tk.MustQuery("explain format = 'brief' select /*+ USE_INDEX(t, idx) */ a from t use index(idx) " +
"where (a between 0 and 2 or a < 2) and b < 1;")
@@ -859,7 +857,7 @@ func TestIndexMerge4PlanCache(t *testing.T) {
tk.MustExec("prepare stmt from 'select /*+ use_index_merge(t1) */ * from t1 where c=? or (b=? and (a >= ? and a <= ?));';")
tk.MustQuery("execute stmt using @a, @a, @b, @a").Check(testkit.Rows("10 10 10"))
tk.MustQuery("execute stmt using @b, @b, @b, @b").Check(testkit.Rows("11 11 11"))
- tk.MustQuery("select @@last_plan_from_cache;").Check(testkit.Rows("1"))
+ tk.MustQuery("select @@last_plan_from_cache;").Check(testkit.Rows("0"))
tk.MustExec("prepare stmt from 'select /*+ use_index_merge(t1) */ * from t1 where c=10 or (a >=? and a <= ?);';")
tk.MustExec("set @a=9, @b=10, @c=11;")
@@ -1395,3 +1393,74 @@ func TestIssue28792(t *testing.T) {
r2 := tk.MustQuery("EXPLAIN SELECT t12.a, t12.b FROM t12 LEFT JOIN t97 use index () on t12.b = t97.b;").Rows()
require.Equal(t, r2, r1)
}
+
+func TestExplainForJSON(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk1 := testkit.NewTestKit(t, store)
+ tk2 := testkit.NewTestKit(t, store)
+
+ tk1.MustExec("use test")
+ tk1.MustExec("set @@tidb_enable_collect_execution_info=0;")
+ tk1.MustExec("drop table if exists t1")
+ tk1.MustExec("create table t1(id int);")
+ tk1.MustQuery("select * from t1;")
+ tk1RootProcess := tk1.Session().ShowProcess()
+ ps := []*util.ProcessInfo{tk1RootProcess}
+ tk1.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
+ tk2.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
+ resRow := tk2.MustQuery(fmt.Sprintf("explain format = 'row' for connection %d", tk1RootProcess.ID)).Rows()
+ resJSON := tk2.MustQuery(fmt.Sprintf("explain format = 'tidb_json' for connection %d", tk1RootProcess.ID)).Rows()
+
+ j := new([]*core.ExplainInfoForEncode)
+ require.NoError(t, json.Unmarshal([]byte(resJSON[0][0].(string)), j))
+ flatJSONRows := make([]*core.ExplainInfoForEncode, 0)
+ for _, row := range *j {
+ flatJSONRows = append(flatJSONRows, flatJSONPlan(row)...)
+ }
+ require.Equal(t, len(flatJSONRows), len(resRow))
+
+ for i, row := range resRow {
+ require.Contains(t, row[0], flatJSONRows[i].ID)
+ require.Equal(t, flatJSONRows[i].EstRows, row[1])
+ require.Equal(t, flatJSONRows[i].TaskType, row[2])
+ require.Equal(t, flatJSONRows[i].AccessObject, row[3])
+ require.Equal(t, flatJSONRows[i].OperatorInfo, row[4])
+ }
+
+ tk1.MustExec("set @@tidb_enable_collect_execution_info=1;")
+ tk1.MustExec("drop table if exists t2")
+ tk1.MustExec("create table t2(id int);")
+ tk1.MustQuery("select * from t2;")
+ tk1RootProcess = tk1.Session().ShowProcess()
+ ps = []*util.ProcessInfo{tk1RootProcess}
+ tk1.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
+ tk2.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
+ resRow = tk2.MustQuery(fmt.Sprintf("explain format = 'row' for connection %d", tk1RootProcess.ID)).Rows()
+ resJSON = tk2.MustQuery(fmt.Sprintf("explain format = 'tidb_json' for connection %d", tk1RootProcess.ID)).Rows()
+
+ j = new([]*core.ExplainInfoForEncode)
+ require.NoError(t, json.Unmarshal([]byte(resJSON[0][0].(string)), j))
+ flatJSONRows = []*core.ExplainInfoForEncode{}
+ for _, row := range *j {
+ flatJSONRows = append(flatJSONRows, flatJSONPlan(row)...)
+ }
+ require.Equal(t, len(flatJSONRows), len(resRow))
+
+ for i, row := range resRow {
+ require.Contains(t, row[0], flatJSONRows[i].ID)
+ require.Equal(t, flatJSONRows[i].EstRows, row[1])
+ require.Equal(t, flatJSONRows[i].ActRows, row[2])
+ require.Equal(t, flatJSONRows[i].TaskType, row[3])
+ require.Equal(t, flatJSONRows[i].AccessObject, row[4])
+ require.Equal(t, flatJSONRows[i].OperatorInfo, row[6])
+ // executeInfo, memory, disk maybe vary in multi execution
+ require.NotEqual(t, flatJSONRows[i].ExecuteInfo, "")
+ require.NotEqual(t, flatJSONRows[i].MemoryInfo, "")
+ require.NotEqual(t, flatJSONRows[i].DiskInfo, "")
+ }
+ // test syntax
+ tk2.MustExec(fmt.Sprintf("explain format = 'tidb_json' for connection %d", tk1RootProcess.ID))
+ tk2.MustExec(fmt.Sprintf("explain format = tidb_json for connection %d", tk1RootProcess.ID))
+ tk2.MustExec(fmt.Sprintf("explain format = 'TIDB_JSON' for connection %d", tk1RootProcess.ID))
+ tk2.MustExec(fmt.Sprintf("explain format = TIDB_JSON for connection %d", tk1RootProcess.ID))
+}
diff --git a/executor/fktest/BUILD.bazel b/executor/fktest/BUILD.bazel
index dbdae1843edaf..2c9f00dfa0624 100644
--- a/executor/fktest/BUILD.bazel
+++ b/executor/fktest/BUILD.bazel
@@ -8,12 +8,16 @@ go_test(
"main_test.go",
],
flaky = True,
+ shard_count = 20,
deps = [
"//config",
"//executor",
+ "//infoschema",
"//kv",
"//meta/autoid",
+ "//parser",
"//parser/ast",
+ "//parser/auth",
"//parser/format",
"//parser/model",
"//parser/mysql",
diff --git a/executor/fktest/foreign_key_test.go b/executor/fktest/foreign_key_test.go
index 558c033b89621..6e2e1d83662f1 100644
--- a/executor/fktest/foreign_key_test.go
+++ b/executor/fktest/foreign_key_test.go
@@ -21,12 +21,17 @@ import (
"strconv"
"strings"
"sync"
+ "sync/atomic"
"testing"
"time"
+ "github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/executor"
+ "github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/parser"
"github.com/pingcap/tidb/parser/ast"
+ "github.com/pingcap/tidb/parser/auth"
"github.com/pingcap/tidb/parser/format"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
@@ -545,7 +550,7 @@ func TestForeignKeyOnInsertOnDuplicateParentTableCheck(t *testing.T) {
tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 11 21 a"))
tk.MustExec("insert into t1 (id, a, b) values (1, 11, 21) on duplicate key update id=11")
- tk.MustGetDBError("insert into t1 (id, a, b) values (1, 11, 21) on duplicate key update a=a+10, b=b+20", plannercore.ErrRowIsReferenced2)
+ tk.MustGetDBError("insert into t1 (id, a, b) values (11, 11, 21) on duplicate key update a=a+10, b=b+20", plannercore.ErrRowIsReferenced2)
tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("2 1112 2222", "3 1013 2023", "4 14 24", "11 11 21"))
tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 11 21 a"))
}
@@ -567,6 +572,15 @@ func TestForeignKeyOnInsertOnDuplicateParentTableCheck(t *testing.T) {
tk.MustGetDBError("insert into t1 (id, a, b) values (1, 0, 0) on duplicate key update id=100+id", plannercore.ErrRowIsReferenced2)
tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 111 21", "4 14 24", "102 12 22", "103 13 23"))
tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("11 1 21 a"))
+
+ // Case-10: Test insert into parent table failed cause by foreign key check, see https://github.com/pingcap/tidb/issues/39200.
+ tk.MustExec("drop table if exists t1,t2;")
+ tk.MustExec("create table t1 (id int key);")
+ tk.MustExec("create table t2 (id int, foreign key fk(id) references t1(id));")
+ tk.MustExec("set @@foreign_key_checks=0")
+ tk.MustExec("insert into t2 values (1)")
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("insert into t1 values (1) on duplicate key update id=2")
}
func TestForeignKey(t *testing.T) {
@@ -1957,6 +1971,7 @@ func TestShowCreateTableWithForeignKey(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
+ tk.MustExec("set @@global.tidb_enable_foreign_key=0")
tk.MustExec("create table t1 (id int key, leader int, leader2 int, index(leader), index(leader2), constraint fk foreign key (leader) references t1(id) ON DELETE CASCADE ON UPDATE SET NULL);")
tk.MustQuery("show create table t1").Check(testkit.Rows("t1 CREATE TABLE `t1` (\n" +
" `id` int(11) NOT NULL,\n" +
@@ -1997,18 +2012,19 @@ func TestDMLExplainAnalyzeFKInfo(t *testing.T) {
tk.MustExec("insert into t1 values (1), (2)")
tk.MustExec("insert into t2 values (1)")
res := tk.MustQuery("explain analyze insert ignore into t3 values (1, 1, 1), (2, 1, 1), (3, 2, 1), (4, 1, 1), (5, 2, 1), (6, 2, 1)")
- getExplainResultFn := func(res *testkit.Result) string {
- resBuff := bytes.NewBufferString("")
- for _, row := range res.Rows() {
- _, _ = fmt.Fprintf(resBuff, "%s\t", row)
- }
- return resBuff.String()
- }
- explain := getExplainResultFn(res)
- require.Regexpf(t, "time:.* loops:.* prepare:.* check_insert: {total_time:.* mem_insert_time:.* prefetch:.* fk_check:.* fk_num: 3.*", explain, "")
+ explain := getExplainResult(res)
+ require.Regexpf(t, "time:.* loops:.* prepare:.* check_insert: {total_time:.* mem_insert_time:.* prefetch:.* fk_check:.*", explain, "")
res = tk.MustQuery("explain analyze insert ignore into t3 values (7, null, null), (8, null, null)")
- explain = getExplainResultFn(res)
- require.NotContains(t, explain, "fk_check", explain, "")
+ explain = getExplainResult(res)
+ require.Regexpf(t, "time:.* loops:.* prepare:.* check_insert: {total_time:.* mem_insert_time:.* prefetch:.* fk_check:.*", explain, "")
+}
+
+func getExplainResult(res *testkit.Result) string {
+ resBuff := bytes.NewBufferString("")
+ for _, row := range res.Rows() {
+ _, _ = fmt.Fprintf(resBuff, "%s\t", row)
+ }
+ return resBuff.String()
}
func TestForeignKeyCascadeOnDiffColumnType(t *testing.T) {
@@ -2072,3 +2088,770 @@ func TestForeignKeyOnInsertOnDuplicateUpdate(t *testing.T) {
tk.MustQuery("select * from t2").Check(testkit.Rows("1"))
tk.MustQuery("select * from t3").Check(testkit.Rows("1"))
}
+
+func TestForeignKeyIssue39419(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key);")
+ tk.MustExec("create table t2 (id int key, a int, b int, " +
+ "foreign key fk_1 (a) references t1(id) ON DELETE SET NULL ON UPDATE SET NULL, " +
+ "foreign key fk_2 (b) references t1(id) ON DELETE CASCADE ON UPDATE CASCADE);")
+ tk.MustExec("insert into t1 values (1), (2), (3);")
+ tk.MustExec("insert into t2 values (1, 1, 1), (2, 2, 2), (3, 3, 3);")
+ tk.MustExec("update t1 set id=id+10 where id in (1, 3);")
+ tk.MustQuery("select * from t1 order by id").Check(testkit.Rows("2", "11", "13"))
+ tk.MustQuery("select * from t2 order by id").Check(testkit.Rows("1 11", "2 2 2", "3 13"))
+ tk.MustExec("delete from t1 where id = 2;")
+ tk.MustQuery("select * from t1 order by id").Check(testkit.Rows("11", "13"))
+ tk.MustQuery("select * from t2 order by id").Check(testkit.Rows("1 11", "3 13"))
+
+ tk.MustExec("drop table t1,t2")
+ tk.MustExec("create table t1 (id int, b int, index(id), foreign key fk_2 (b) references t1(id) ON UPDATE CASCADE);")
+ tk.MustExec("insert into t1 values (1, 1), (2, 2), (3, 3);")
+ tk.MustExec("update t1 set id=id+10 where id > 1")
+ tk.MustQuery("select * from t1 order by id").Check(testkit.Rows("1 1", "12 12", "13 13"))
+}
+
+func TestExplainAnalyzeDMLWithFKInfo(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key);")
+ tk.MustExec("create table t2 (id int key, foreign key fk(id) references t1(id) ON UPDATE CASCADE ON DELETE CASCADE);")
+ tk.MustExec("create table t3 (id int, unique index idx(id));")
+ tk.MustExec("create table t4 (id int, index idx_id(id),foreign key fk(id) references t3(id));")
+ tk.MustExec("create table t5 (id int key, id2 int, id3 int, unique index idx2(id2), index idx3(id3));")
+ tk.MustExec("create table t6 (id int, id2 int, id3 int, index idx_id(id), index idx_id2(id2), " +
+ "foreign key fk_1 (id) references t5(id) ON UPDATE CASCADE ON DELETE SET NULL, " +
+ "foreign key fk_2 (id2) references t5(id2) ON UPDATE CASCADE, " +
+ "foreign key fk_3 (id3) references t5(id3) ON DELETE CASCADE);")
+ tk.MustExec("create table t7(id int primary key, pid int, index(pid), foreign key(pid) references t7(id) on delete cascade);")
+
+ cases := []struct {
+ prepare []string
+ sql string
+ plan string
+ }{
+ // Test foreign key use primary key.
+ {
+ prepare: []string{
+ "insert into t1 values (1),(2),(3),(4),(5)",
+ },
+ sql: "explain analyze insert into t2 values (1),(2),(3);",
+ plan: "Insert_. N/A 0 root time:.*, loops:1, prepare:.*, insert:.*" +
+ "└─Foreign_Key_Check_. 0.00 0 root table:t1 total:.*, check:.*, lock:.*, foreign_keys:3 foreign_key:fk, check_exist N/A N/A",
+ },
+ {
+ sql: "explain analyze insert ignore into t2 values (10),(11),(12);",
+ plan: "Insert_.* fk_check.*" +
+ "└─Foreign_Key_Check_.* 0 root table:t1 total:0s, foreign_keys:3 foreign_key:fk, check_exist N/A N/A",
+ },
+ {
+ sql: "explain analyze update t2 set id=id+2 where id >1",
+ plan: "Update_.* 0 root time:.*, loops:1.*" +
+ "├─TableReader_.*" +
+ "│ └─TableRangeScan.*" +
+ "└─Foreign_Key_Check_.* 0 root table:t1 total:.*, check:.*, lock:.*, foreign_keys:2 foreign_key:fk, check_exist N/A N/A",
+ },
+ {
+ sql: "explain analyze delete from t1 where id>1",
+ plan: "Delete_.*" +
+ "├─TableReader_.*" +
+ "│ └─TableRangeScan_.*" +
+ "└─Foreign_Key_Cascade_.* 0 root table:t2 total:.*, foreign_keys:4 foreign_key:fk, on_delete:CASCADE N/A N/A.*" +
+ " └─Delete_.*" +
+ " └─Batch_Point_Get_.*",
+ },
+ {
+ sql: "explain analyze update t1 set id=id+1 where id = 1",
+ plan: "Update_.*" +
+ "├─Point_Get_.*" +
+ "└─Foreign_Key_Cascade_.* 0 root table:t2 total:.*, foreign_keys:1 foreign_key:fk, on_update:CASCADE N/A N/A.*" +
+ " └─Update_.*" +
+ " ├─Point_Get_.*" +
+ " └─Foreign_Key_Check_.*",
+ },
+ {
+ sql: "explain analyze insert into t1 values (1) on duplicate key update id = 100",
+ plan: "Insert_.*" +
+ "└─Foreign_Key_Cascade_.* 0 root table:t2 total:0s foreign_key:fk, on_update:CASCADE N/A N/A",
+ },
+ {
+ sql: "explain analyze insert into t1 values (2) on duplicate key update id = 100",
+ plan: "Insert_.*" +
+ "└─Foreign_Key_Cascade_.* 0 root table:t2 total:.*, foreign_keys:1 foreign_key:fk, on_update:CASCADE N/A N/A.*" +
+ " └─Update_.*" +
+ " ├─Point_Get_.*" +
+ " └─Foreign_Key_Check_.* 0 root table:t1 total:.*, check:.*, lock:.*, foreign_keys:1 foreign_key:fk, check_exist N/A N/A",
+ },
+ // Test foreign key use index.
+ {
+ prepare: []string{
+ "insert into t3 values (1),(2),(3),(4),(5)",
+ },
+ sql: "explain analyze insert into t4 values (1),(2),(3);",
+ plan: "Insert_.*" +
+ "└─Foreign_Key_Check_.* 0 root table:t3, index:idx total:.*, check:.*, lock:.*, foreign_keys:3 foreign_key:fk, check_exist N/A N/A",
+ },
+ {
+ sql: "explain analyze update t4 set id=id+2 where id >1",
+ plan: "Update_.*" +
+ "├─IndexReader_.*" +
+ "│ └─IndexRangeScan_.*" +
+ "└─Foreign_Key_Check_.* 0 root table:t3, index:idx total:.*, check:.*, lock:.*, foreign_keys:2 foreign_key:fk, check_exist N/A N/A",
+ },
+ {
+ sql: "explain analyze delete from t3 where id in (2,3)",
+ plan: "Delete_.*" +
+ "├─Batch_Point_Get_.*" +
+ "└─Foreign_Key_Check_.* 0 root table:t4, index:idx_id total:.*, check:.*, foreign_keys:2 foreign_key:fk, check_not_exist N/A N/A",
+ },
+ {
+ prepare: []string{
+ "insert into t3 values (2)",
+ },
+ sql: "explain analyze update t3 set id=id+1 where id = 2",
+ plan: "Update_.*" +
+ "├─Point_Get_.*" +
+ "└─Foreign_Key_Check_.* 0 root table:t4, index:idx_id total:.*, check:.*, foreign_keys:1 foreign_key:fk, check_not_exist N/A N/A",
+ },
+
+ {
+ sql: "explain analyze insert into t3 values (2) on duplicate key update id = 100",
+ plan: "Insert_.*" +
+ "└─Foreign_Key_Check_.* 0 root table:t4, index:idx_id total:0s foreign_key:fk, check_not_exist N/A N/A",
+ },
+ {
+ sql: "explain analyze insert into t3 values (3) on duplicate key update id = 100",
+ plan: "Insert_.*" +
+ "└─Foreign_Key_Check_.* 0 root table:t4, index:idx_id total:.*, check:.*, foreign_keys:1 foreign_key:fk, check_not_exist N/A N/A",
+ },
+ // Test multi-foreign keys in on table.
+ {
+ prepare: []string{
+ "insert into t5 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5)",
+ },
+ sql: "explain analyze insert into t6 values (1,1,1)",
+ plan: "Insert_.*" +
+ "├─Foreign_Key_Check_.* 0 root table:t5 total:.*, check:.*, lock:.*, foreign_keys:1 foreign_key:fk_1, check_exist N/A N/A.*" +
+ "├─Foreign_Key_Check_.* 0 root table:t5, index:idx2 total:.*, check:.*, lock:.*, foreign_keys:1 foreign_key:fk_2, check_exist N/A N/A.*" +
+ "└─Foreign_Key_Check_.* 0 root table:t5, index:idx3 total:.*, check:.*, lock:.*, foreign_keys:1 foreign_key:fk_3, check_exist N/A N/A",
+ },
+ {
+ sql: "explain analyze insert ignore into t6 values (1,1,10)",
+ plan: "Insert_.* root time:.* loops:.* prepare:.* check_insert.* fk_check:.*" +
+ "├─Foreign_Key_Check.* 0 root table:t5 total:0s, foreign_keys:1 foreign_key:fk_1, check_exist N/A N/A.*" +
+ "├─Foreign_Key_Check.* 0 root table:t5, index:idx2 total:0s, foreign_keys:1 foreign_key:fk_2, check_exist N/A N/A.*" +
+ "└─Foreign_Key_Check.* 0 root table:t5, index:idx3 total:0s, foreign_keys:1 foreign_key:fk_3, check_exist N/A N/A",
+ },
+ {
+ sql: "explain analyze update t6 set id=id+1, id3=id2+1 where id = 1",
+ plan: "Update_.*" +
+ "├─IndexLookUp_.*" +
+ "│ ├─IndexRangeScan_.*" +
+ "│ └─TableRowIDScan_.*" +
+ "├─Foreign_Key_Check_.* 0 root table:t5 total:.*, check:.*, lock:.*, foreign_keys:1 foreign_key:fk_1, check_exist N/A N/A.*" +
+ "└─Foreign_Key_Check_.* 0 root table:t5, index:idx3 total:.*, check:.*, lock:.*, foreign_keys:1 foreign_key:fk_3, check_exist N/A N/A",
+ },
+ {
+ sql: "explain analyze delete from t5 where id in (4,5)",
+ plan: "Delete_.*" +
+ "├─Batch_Point_Get_.*" +
+ "├─Foreign_Key_Check_.* 0 root table:t6, index:idx_id2 total:.*, check:.*, foreign_keys:2 foreign_key:fk_2, check_not_exist N/A N/A.*" +
+ "├─Foreign_Key_Cascade_.* 0 root table:t6, index:idx_id total:.*, foreign_keys:2 foreign_key:fk_1, on_delete:SET NULL N/A N/A.*" +
+ "│ └─Update_.*" +
+ "│ │ ├─IndexRangeScan_.*" +
+ "│ │ └─TableRowIDScan_.*" +
+ "│ └─Foreign_Key_Check_.* 0 root table:t5 total:0s foreign_key:fk_1, check_exist N/A N/A.*" +
+ "└─Foreign_Key_Cascade_.* 0 root table:t6, index:fk_3 total:.*, foreign_keys:2 foreign_key:fk_3, on_delete:CASCADE N/A N/A.*" +
+ " └─Delete_.*" +
+ " └─IndexLookUp_.*" +
+ " ├─IndexRangeScan_.*" +
+ " └─TableRowIDScan_.*",
+ },
+ {
+ sql: "explain analyze update t5 set id=id+1, id2=id2+1 where id = 3",
+ plan: "Update_.*" +
+ "├─Point_Get_.*" +
+ "├─Foreign_Key_Cascade_.* 0 root table:t6, index:idx_id total:.*, foreign_keys:1 foreign_key:fk_1, on_update:CASCADE N/A N/A.*" +
+ "│ └─Update_.*" +
+ "│ ├─IndexLookUp_.*" +
+ "│ │ ├─IndexRangeScan_.*" +
+ "│ │ └─TableRowIDScan_.*" +
+ "│ └─Foreign_Key_Check_.* 0 root table:t5 total:0s foreign_key:fk_1, check_exist N/A N/A.*" +
+ "└─Foreign_Key_Cascade_.* 0 root table:t6, index:idx_id2 total:.*, foreign_keys:1 foreign_key:fk_2, on_update:CASCADE N/A N/A.*" +
+ " └─Update_.*" +
+ " ├─IndexLookUp_.*" +
+ " │ ├─IndexRangeScan_.*" +
+ " │ └─TableRowIDScan_.*" +
+ " └─Foreign_Key_Check_.* 0 root table:t5, index:idx2 total:0s foreign_key:fk_2, check_exist N/A N/A",
+ },
+ {
+ prepare: []string{
+ "insert into t5 values (10,10,10)",
+ },
+ sql: "explain analyze update t5 set id=id+1, id2=id2+1, id3=id3+1 where id = 10",
+ plan: "Update_.*" +
+ "├─Point_Get_.*" +
+ "├─Foreign_Key_Check_.* 0 root table:t6, index:fk_3 total:.*, check:.*, foreign_keys:1 foreign_key:.*, check_not_exist N/A N/A.*" +
+ "├─Foreign_Key_Cascade_.* 0 root table:t6, index:idx_id total:.*, foreign_keys:1 foreign_key:fk_1, on_update:CASCADE N/A N/A.*" +
+ "│ └─Update_.*" +
+ "│ ├─IndexLookUp_.*" +
+ "│ │ ├─IndexRangeScan_.*" +
+ "│ │ └─TableRowIDScan_.*" +
+ "│ └─Foreign_Key_Check_.*" +
+ "└─Foreign_Key_Cascade_.* 0 root table:t6, index:idx_id2 total:.*, foreign_keys:1 foreign_key:fk_2, on_update:CASCADE N/A N/A.*" +
+ " └─Update_.*" +
+ " ├─IndexLookUp_.*" +
+ " │ ├─IndexRangeScan_.*" +
+ " │ └─TableRowIDScan_.*" +
+ " └─Foreign_Key_Check_.* 0 root table:t5, index:idx2 total:0s foreign_key:fk_2, check_exist N/A N/A",
+ },
+ {
+ sql: "explain analyze insert into t5 values (1,1,1) on duplicate key update id = 100, id3=100",
+ plan: "Insert_.*" +
+ "├─Foreign_Key_Check_.* 0 root table:t6, index:fk_3 total:.*, check:.*, foreign_keys:1 foreign_key:fk_3, check_not_exist N/A N/A.*" +
+ "└─Foreign_Key_Cascade_.* 0 root table:t6, index:idx_id total:.*, foreign_keys:1 foreign_key:fk_1, on_update:CASCADE N/A N/A.*" +
+ " └─Update_.*" +
+ " ├─IndexLookUp_.*" +
+ " │ ├─IndexRangeScan_.*" +
+ " │ └─TableRowIDScan_.*" +
+ " └─Foreign_Key_Check_.* 0 root table:t5 total:0s foreign_key:fk_1, check_exist N/A N/A",
+ },
+ {
+ prepare: []string{
+ "insert into t7 values(0,0),(1,0),(2,1),(3,2),(4,3),(5,4),(6,5),(7,6),(8,7),(9,8),(10,9),(11,10),(12,11),(13,12),(14,13);",
+ },
+ sql: "explain analyze delete from t7 where id = 0;",
+ plan: "Delete_.*" +
+ "├─Point_Get_.*" +
+ "└─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.* foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.* foreign_keys:2 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:.*, foreign_keys:1 foreign_key:fk_1, on_delete:CASCADE.*" +
+ " └─Delete_.*" +
+ " ├─UnionScan_.*" +
+ " │ └─IndexReader_.*" +
+ " │ └─IndexRangeScan_.*" +
+ " └─Foreign_Key_Cascade_.* 0 root table:t7, index:pid total:0s foreign_key:fk_1, on_delete:CASCADE.*",
+ },
+ }
+ for _, ca := range cases {
+ for _, sql := range ca.prepare {
+ tk.MustExec(sql)
+ }
+ res := tk.MustQuery(ca.sql)
+ explain := getExplainResult(res)
+ require.Regexp(t, ca.plan, explain)
+ }
+}
+
+func TestForeignKeyRuntimeStats(t *testing.T) {
+ checkStats := executor.FKCheckRuntimeStats{
+ Total: time.Second * 3,
+ Check: time.Second * 2,
+ Lock: time.Second,
+ Keys: 10,
+ }
+ require.Equal(t, "total:3s, check:2s, lock:1s, foreign_keys:10", checkStats.String())
+ checkStats.Merge(checkStats.Clone())
+ require.Equal(t, "total:6s, check:4s, lock:2s, foreign_keys:20", checkStats.String())
+ cascadeStats := executor.FKCascadeRuntimeStats{
+ Total: time.Second,
+ Keys: 10,
+ }
+ require.Equal(t, "total:1s, foreign_keys:10", cascadeStats.String())
+ cascadeStats.Merge(cascadeStats.Clone())
+ require.Equal(t, "total:2s, foreign_keys:20", cascadeStats.String())
+}
+
+func TestPrivilegeCheckInForeignKeyCascade(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key);")
+ tk.MustExec("create table t2 (id int key, foreign key fk (id) references t1(id) ON DELETE CASCADE ON UPDATE CASCADE);")
+ tk.MustExec("insert into t1 values (1), (2), (3);")
+ cases := []struct {
+ prepares []string
+ sql string
+ err error
+ t1Rows []string
+ t2Rows []string
+ }{
+ {
+ prepares: []string{"grant insert on test.t2 to 'u1'@'%';"},
+ sql: "insert into t2 values (1), (2), (3);",
+ t1Rows: []string{"1", "2", "3"},
+ t2Rows: []string{"1", "2", "3"},
+ },
+ {
+ prepares: []string{"grant select, delete on test.t1 to 'u1'@'%';"},
+ sql: "delete from t1 where id=1;",
+ t1Rows: []string{"2", "3"},
+ t2Rows: []string{"2", "3"},
+ },
+ {
+ prepares: []string{"grant select, update on test.t1 to 'u1'@'%';"},
+ sql: "update t1 set id=id+10 where id=2;",
+ t1Rows: []string{"3", "12"},
+ t2Rows: []string{"3", "12"},
+ },
+ }
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk2.MustExec("set @@foreign_key_checks=1")
+ for _, ca := range cases {
+ tk.MustExec("drop user if exists 'u1'@'%'")
+ tk.MustExec("create user 'u1'@'%' identified by '';")
+ for _, sql := range ca.prepares {
+ tk.MustExec(sql)
+ }
+ err := tk2.Session().Auth(&auth.UserIdentity{Username: "u1", Hostname: "localhost", CurrentUser: true, AuthUsername: "u1", AuthHostname: "%"}, nil, []byte("012345678901234567890"))
+ require.NoError(t, err)
+ if ca.err == nil {
+ tk2.MustExec(ca.sql)
+ } else {
+ err = tk2.ExecToErr(ca.sql)
+ require.Error(t, err)
+ }
+ tk.MustQuery("select * from t1 order by id").Check(testkit.Rows(ca.t1Rows...))
+ tk.MustQuery("select * from t2 order by id").Check(testkit.Rows(ca.t2Rows...))
+ }
+}
+
+func TestTableLockInForeignKeyCascade(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk2.MustExec("set @@foreign_key_checks=1")
+ // enable table lock
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.EnableTableLock = true
+ })
+ defer func() {
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.EnableTableLock = false
+ })
+ }()
+ tk.MustExec("create table t1 (id int key);")
+ tk.MustExec("create table t2 (id int key, foreign key fk (id) references t1(id) ON DELETE CASCADE ON UPDATE CASCADE);")
+ tk.MustExec("insert into t1 values (1), (2), (3);")
+ tk.MustExec("insert into t2 values (1), (2), (3);")
+ tk.MustExec("lock table t2 read;")
+ tk2.MustGetDBError("delete from t1 where id = 1", infoschema.ErrTableLocked)
+ tk.MustExec("unlock tables;")
+ tk2.MustExec("delete from t1 where id = 1")
+ tk.MustQuery("select * from t1 order by id").Check(testkit.Rows("2", "3"))
+ tk.MustQuery("select * from t2 order by id").Check(testkit.Rows("2", "3"))
+}
+
+func TestForeignKeyIssue39732(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_stmt_summary=1")
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create user 'u1'@'%' identified by '';")
+ tk.MustExec("GRANT ALL PRIVILEGES ON *.* TO 'u1'@'%'")
+ err := tk.Session().Auth(&auth.UserIdentity{Username: "u1", Hostname: "localhost", CurrentUser: true, AuthUsername: "u1", AuthHostname: "%"}, nil, []byte("012345678901234567890"))
+ require.NoError(t, err)
+ tk.MustExec("create table t1 (id int key, leader int, index(leader), foreign key (leader) references t1(id) ON DELETE CASCADE);")
+ tk.MustExec("insert into t1 values (1, null), (10, 1), (11, 1), (20, 10)")
+ tk.MustExec(`prepare stmt1 from 'delete from t1 where id = ?';`)
+ tk.MustExec(`set @a = 1;`)
+ tk.MustExec("execute stmt1 using @a;")
+ tk.MustQuery("select * from t1 order by id").Check(testkit.Rows())
+}
+
+func TestForeignKeyOnReplaceIntoChildTable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t_data (id int, a int, b int)")
+ tk.MustExec("insert into t_data (id, a, b) values (1, 1, 1), (2, 2, 2);")
+ for _, ca := range foreignKeyTestCase1 {
+ tk.MustExec("drop table if exists t2;")
+ tk.MustExec("drop table if exists t1;")
+ for _, sql := range ca.prepareSQLs {
+ tk.MustExec(sql)
+ }
+ tk.MustExec("replace into t1 (id, a, b) values (1, 1, 1);")
+ tk.MustExec("replace into t2 (id, a, b) values (1, 1, 1)")
+ tk.MustGetDBError("replace into t1 (id, a, b) values (1, 2, 3);", plannercore.ErrRowIsReferenced2)
+ if !ca.notNull {
+ tk.MustExec("replace into t2 (id, a, b) values (2, null, 1)")
+ tk.MustExec("replace into t2 (id, a, b) values (3, 1, null)")
+ tk.MustExec("replace into t2 (id, a, b) values (4, null, null)")
+ }
+ tk.MustGetDBError("replace into t2 (id, a, b) values (5, 1, 0);", plannercore.ErrNoReferencedRow2)
+ tk.MustGetDBError("replace into t2 (id, a, b) values (6, 0, 1);", plannercore.ErrNoReferencedRow2)
+ tk.MustGetDBError("replace into t2 (id, a, b) values (7, 2, 2);", plannercore.ErrNoReferencedRow2)
+ // Test replace into from select.
+ tk.MustExec("delete from t2")
+ tk.MustExec("replace into t2 (id, a, b) select id, a, b from t_data where t_data.id=1")
+ tk.MustGetDBError("replace into t2 (id, a, b) select id, a, b from t_data where t_data.id=2", plannercore.ErrNoReferencedRow2)
+
+ // Test in txn
+ tk.MustExec("delete from t2")
+ tk.MustExec("begin")
+ tk.MustExec("delete from t1 where a=1")
+ tk.MustGetDBError("replace into t2 (id, a, b) values (1, 1, 1)", plannercore.ErrNoReferencedRow2)
+ tk.MustExec("replace into t1 (id, a, b) values (2, 2, 2)")
+ tk.MustExec("replace into t2 (id, a, b) values (2, 2, 2)")
+ tk.MustGetDBError("replace into t1 (id, a, b) values (2, 2, 3);", plannercore.ErrRowIsReferenced2)
+ tk.MustExec("rollback")
+ tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 1 1"))
+ tk.MustQuery("select id, a, b from t2 order by id").Check(testkit.Rows())
+ }
+
+ // Case-10: test primary key is handle and contain foreign key column, and foreign key column has default value.
+ tk.MustExec("drop table if exists t2;")
+ tk.MustExec("drop table if exists t1;")
+ tk.MustExec("set @@tidb_enable_clustered_index=0;")
+ tk.MustExec("create table t1 (id int,a int, primary key(id));")
+ tk.MustExec("create table t2 (id int key,a int not null default 0, index (a), foreign key fk(a) references t1(id));")
+ tk.MustExec("replace into t1 values (1, 1);")
+ tk.MustExec("replace into t2 values (1, 1);")
+ tk.MustGetDBError("replace into t2 (id) values (10);", plannercore.ErrNoReferencedRow2)
+ tk.MustGetDBError("replace into t2 values (3, 2);", plannercore.ErrNoReferencedRow2)
+
+ // Case-11: test primary key is handle and contain foreign key column, and foreign key column doesn't have default value.
+ tk.MustExec("drop table if exists t2;")
+ tk.MustExec("create table t2 (id int key,a int, index (a), foreign key fk(a) references t1(id));")
+ tk.MustExec("replace into t2 values (1, 1);")
+ tk.MustExec("replace into t2 (id) values (10);")
+ tk.MustGetDBError("replace into t2 values (3, 2);", plannercore.ErrNoReferencedRow2)
+}
+
+func TestForeignKeyOnReplaceInto(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key, a int, index (a));")
+ tk.MustExec("create table t2 (id int key, a int, index (a), constraint fk_1 foreign key (a) references t1(a));")
+ tk.MustExec("replace into t1 values (1, 1);")
+ tk.MustExec("replace into t2 values (1, 1);")
+ tk.MustExec("replace into t2 (id) values (2);")
+ tk.MustGetDBError("replace into t2 values (1, 2);", plannercore.ErrNoReferencedRow2)
+ // Test fk check on replace into parent table.
+ tk.MustGetDBError("replace into t1 values (1, 2);", plannercore.ErrRowIsReferenced2)
+ // Test fk cascade delete on replace into parent table.
+ tk.MustExec("alter table t2 drop foreign key fk_1")
+ tk.MustExec("alter table t2 add constraint fk_1 foreign key (a) references t1(a) on delete cascade")
+ tk.MustExec("replace into t1 values (1, 2);")
+ tk.MustQuery("select id, a from t1").Check(testkit.Rows("1 2"))
+ tk.MustQuery("select * from t2").Check(testkit.Rows("2 "))
+ // Test fk cascade delete on replace into parent table.
+ tk.MustExec("alter table t2 drop foreign key fk_1")
+ tk.MustExec("alter table t2 add constraint fk_1 foreign key (a) references t1(a) on delete set null")
+ tk.MustExec("delete from t2")
+ tk.MustExec("delete from t1")
+ tk.MustExec("replace into t1 values (1, 1);")
+ tk.MustExec("replace into t2 values (1, 1);")
+ tk.MustExec("replace into t1 values (1, 2);")
+ tk.MustQuery("select id, a from t1").Check(testkit.Rows("1 2"))
+ tk.MustQuery("select id, a from t2").Check(testkit.Rows("1 "))
+
+ // Test cascade delete in self table by replace into statement.
+ tk.MustExec("drop table t1,t2")
+ tk.MustExec("create table t1 (id int key, name varchar(10), leader int, index(leader), foreign key (leader) references t1(id) ON DELETE CASCADE);")
+ tk.MustExec("replace into t1 values (1, 'boss', null), (10, 'l1_a', 1), (11, 'l1_b', 1), (12, 'l1_c', 1)")
+ tk.MustExec("replace into t1 values (100, 'l2_a1', 10), (101, 'l2_a2', 10), (102, 'l2_a3', 10)")
+ tk.MustExec("replace into t1 values (110, 'l2_b1', 11), (111, 'l2_b2', 11), (112, 'l2_b3', 11)")
+ tk.MustExec("replace into t1 values (120, 'l2_c1', 12), (121, 'l2_c2', 12), (122, 'l2_c3', 12)")
+ tk.MustExec("replace into t1 values (1000,'l3_a1', 100)")
+ tk.MustExec("replace into t1 values (1, 'new-boss', null)")
+ tk.MustQuery("select id from t1 order by id").Check(testkit.Rows("1"))
+}
+
+func TestForeignKeyLargeTxnErr(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int auto_increment key, pid int, name varchar(200), index(pid));")
+ tk.MustExec("insert into t1 (name) values ('abcdefghijklmnopqrstuvwxyz1234567890abcdefghijklmnopqrstuvwxyz1234567890abcdefghijklmnopqrstuvwxyz1234567890abcdefghijklmnopqrstuvwxyz1234567890abcdefghijklmnopqrstuvwxyz1234567890');")
+ for i := 0; i < 8; i++ {
+ tk.MustExec("insert into t1 (name) select name from t1;")
+ }
+ tk.MustQuery("select count(*) from t1").Check(testkit.Rows("256"))
+ tk.MustExec("update t1 set pid=1 where id>1")
+ tk.MustExec("alter table t1 add foreign key (pid) references t1 (id) on update cascade")
+ originLimit := atomic.LoadUint64(&kv.TxnTotalSizeLimit)
+ defer func() {
+ atomic.StoreUint64(&kv.TxnTotalSizeLimit, originLimit)
+ }()
+ // Set the limitation to a small value, make it easier to reach the limitation.
+ atomic.StoreUint64(&kv.TxnTotalSizeLimit, 10240)
+ tk.MustQuery("select sum(id) from t1").Check(testkit.Rows("32896"))
+ // foreign key cascade behaviour will cause ErrTxnTooLarge.
+ tk.MustGetDBError("update t1 set id=id+100000 where id=1", kv.ErrTxnTooLarge)
+ tk.MustQuery("select sum(id) from t1").Check(testkit.Rows("32896"))
+ tk.MustGetDBError("update t1 set id=id+100000 where id=1", kv.ErrTxnTooLarge)
+ tk.MustQuery("select id,pid from t1 where id<3 order by id").Check(testkit.Rows("1 ", "2 1"))
+ tk.MustExec("set @@foreign_key_checks=0")
+ tk.MustExec("update t1 set id=id+100000 where id=1")
+ tk.MustQuery("select id,pid from t1 where id<3 or pid is null order by id").Check(testkit.Rows("2 1", "100001 "))
+}
+
+func TestForeignKeyAndLockView(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key)")
+ tk.MustExec("create table t2 (id int key, foreign key (id) references t1(id) ON DELETE CASCADE ON UPDATE CASCADE)")
+ tk.MustExec("insert into t1 values (1)")
+ tk.MustExec("insert into t2 values (1)")
+ tk.MustExec("begin pessimistic")
+ tk.MustExec("set @@foreign_key_checks=0")
+ tk.MustExec("update t2 set id=2")
+
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("set @@foreign_key_checks=1")
+ tk2.MustExec("use test")
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ tk2.MustExec("begin pessimistic")
+ tk2.MustExec("update t1 set id=2 where id=1")
+ tk2.MustExec("commit")
+ }()
+ time.Sleep(time.Millisecond * 200)
+ _, digest := parser.NormalizeDigest("update t1 set id=2 where id=1")
+ tk.MustQuery("select CURRENT_SQL_DIGEST from information_schema.tidb_trx where state='LockWaiting' and db='test'").Check(testkit.Rows(digest.String()))
+ tk.MustGetErrMsg("update t1 set id=2", "[executor:1213]Deadlock found when trying to get lock; try restarting transaction")
+ wg.Wait()
+}
+
+func TestForeignKeyAndMemoryTracker(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int auto_increment key, pid int, name varchar(200), index(pid));")
+ tk.MustExec("insert into t1 (name) values ('abcdefghijklmnopqrstuvwxyz1234567890abcdefghijklmnopqrstuvwxyz');")
+ for i := 0; i < 8; i++ {
+ tk.MustExec("insert into t1 (name) select name from t1;")
+ }
+ tk.MustQuery("select count(*) from t1").Check(testkit.Rows("256"))
+ tk.MustExec("update t1 set pid=1 where id>1")
+ tk.MustExec("alter table t1 add foreign key (pid) references t1 (id) on update cascade")
+ tk.MustQuery("select sum(id) from t1").Check(testkit.Rows("32896"))
+ defer tk.MustExec("SET GLOBAL tidb_mem_oom_action = DEFAULT")
+ tk.MustExec("SET GLOBAL tidb_mem_oom_action='CANCEL'")
+ tk.MustExec("set @@tidb_mem_quota_query=40960;")
+ // foreign key cascade behaviour will exceed memory quota.
+ err := tk.ExecToErr("update t1 set id=id+100000 where id=1")
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "Out Of Memory Quota!")
+ tk.MustQuery("select id,pid from t1 where id = 1").Check(testkit.Rows("1 "))
+ tk.MustExec("set @@foreign_key_checks=0")
+ // After disable foreign_key_checks, following DML will execute successful.
+ tk.MustExec("update t1 set id=id+100000 where id=1")
+ tk.MustQuery("select id,pid from t1 where id<3 or pid is null order by id").Check(testkit.Rows("2 1", "100001 "))
+}
+
+func TestForeignKeyMetaInKeyColumnUsage(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (a int, b int, index(a, b));")
+ tk.MustExec("create table t2 (a int, b int, index(a, b), constraint fk foreign key(a, b) references t1(a, b));")
+ tk.MustQuery("select CONSTRAINT_NAME, TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_SCHEMA, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME from " +
+ "INFORMATION_SCHEMA.KEY_COLUMN_USAGE where CONSTRAINT_SCHEMA='test' and TABLE_NAME='t2' and REFERENCED_TABLE_SCHEMA is not null and REFERENCED_COLUMN_NAME is not null;").
+ Check(testkit.Rows("fk test t2 a test t1 a", "fk test t2 b test t1 b"))
+}
+
+func TestForeignKeyAndGeneratedColumn(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ // Test foreign key with parent column is virtual generated column.
+ tk.MustExec("create table t1 (a int, b int as (a+1) virtual, index(b));")
+ tk.MustGetErrMsg("create table t2 (a int, b int, constraint fk foreign key(b) references t1(b));", "[schema:3733]Foreign key 'fk' uses virtual column 'b' which is not supported.")
+ // Test foreign key with child column is virtual generated column.
+ tk.MustExec("drop table t1")
+ tk.MustExec("create table t1 (a int key);")
+ tk.MustGetErrMsg("create table t2 (a int, c int as (a+1) virtual, constraint fk foreign key(c) references t1(a));", "[schema:3733]Foreign key 'fk' uses virtual column 'c' which is not supported.")
+ // Test foreign key with parent column is stored generated column.
+ tk.MustExec("drop table if exists t1,t2")
+ tk.MustExec("create table t1 (a int, b int as (a) stored, index(b));")
+ tk.MustExec("create table t2 (a int, b int, constraint fk foreign key(b) references t1(b) on delete cascade on update cascade);")
+ tk.MustExec("insert into t1 (a) values (1),(2)")
+ tk.MustExec("insert into t2 (a) values (1),(2)")
+ tk.MustExec("update t2 set b=a")
+ tk.MustExec("insert into t2 values (1,1),(2,2)")
+ tk.MustGetDBError("insert into t2 values (3,3)", plannercore.ErrNoReferencedRow2)
+ tk.MustQuery("select * from t2 order by a").Check(testkit.Rows("1 1", "1 1", "2 2", "2 2"))
+ tk.MustExec("update t1 set a=a+10 where a=1")
+ tk.MustQuery("select * from t1 order by a").Check(testkit.Rows("2 2", "11 11"))
+ tk.MustQuery("select * from t2 order by a").Check(testkit.Rows("1 11", "1 11", "2 2", "2 2"))
+ tk.MustExec("delete from t1 where a=2")
+ tk.MustQuery("select * from t1 order by a").Check(testkit.Rows("11 11"))
+ tk.MustQuery("select * from t2 order by a").Check(testkit.Rows("1 11", "1 11"))
+ // Test foreign key with parent and child column is stored generated column.
+ tk.MustExec("drop table if exists t1,t2")
+ tk.MustExec("create table t1 (a int, b int as (a) stored, index(b));")
+ tk.MustGetErrMsg("create table t2 (a int, b int as (a) stored, constraint fk foreign key(b) references t1(b) on update cascade);", "[ddl:3104]Cannot define foreign key with ON UPDATE CASCADE clause on a generated column.")
+ tk.MustGetErrMsg("create table t2 (a int, b int as (a) stored, constraint fk foreign key(b) references t1(b) on delete set null);", "[ddl:3104]Cannot define foreign key with ON DELETE SET NULL clause on a generated column.")
+ tk.MustExec("create table t2 (a int, b int as (a) stored, constraint fk foreign key(b) references t1(b));")
+ tk.MustExec("insert into t1 (a) values (1),(2)")
+ tk.MustExec("insert into t2 (a) values (1),(2)")
+ tk.MustGetDBError("insert into t2 (a) values (3)", plannercore.ErrNoReferencedRow2)
+ tk.MustQuery("select * from t2 order by a").Check(testkit.Rows("1 1", "2 2"))
+ tk.MustGetDBError("delete from t1 where b=1", plannercore.ErrRowIsReferenced2)
+ tk.MustGetDBError("update t1 set a=a+10 where a=1", plannercore.ErrRowIsReferenced2)
+ tk.MustExec("alter table t2 drop foreign key fk")
+ tk.MustExec("alter table t2 add foreign key fk (b) references t1(b) on delete cascade")
+ tk.MustExec("delete from t1 where a=1")
+ tk.MustQuery("select * from t1 order by a").Check(testkit.Rows("2 2"))
+ tk.MustQuery("select * from t2 order by a").Check(testkit.Rows("2 2"))
+}
+
+func TestForeignKeyAndExpressionIndex(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (a int, b int, index idx1 (b), index idx2 ((b*2)));")
+ tk.MustExec("create table t2 (a int, b int, index((b*2)), constraint fk foreign key(b) references t1(b));")
+ tk.MustExec("insert into t1 values (1,1),(2,2)")
+ tk.MustExec("insert into t2 values (1,1),(2,2)")
+ tk.MustGetDBError("insert into t2 values (3,3)", plannercore.ErrNoReferencedRow2)
+ tk.MustGetDBError("update t1 set b=b+10 where b=1", plannercore.ErrRowIsReferenced2)
+ tk.MustGetDBError("delete from t1 where b=1", plannercore.ErrRowIsReferenced2)
+ tk.MustGetErrMsg("alter table t1 drop index idx1", "[ddl:1553]Cannot drop index 'idx1': needed in a foreign key constraint")
+ tk.MustGetErrMsg("alter table t2 drop index fk", "[ddl:1553]Cannot drop index 'fk': needed in a foreign key constraint")
+ tk.MustExec("alter table t2 drop foreign key fk")
+ tk.MustExec("alter table t2 add foreign key fk (b) references t1(b) on delete set null on update cascade")
+ tk.MustExec("update t1 set b=b+10 where b=1")
+ tk.MustExec("delete from t1 where b=2")
+ tk.MustQuery("select * from t1 order by a").Check(testkit.Rows("1 11"))
+ tk.MustQuery("select * from t2 order by a").Check(testkit.Rows("1 11", "2 "))
+ tk.MustExec("admin check table t1")
+ tk.MustExec("admin check table t2")
+}
+
+func TestForeignKeyAndMultiValuedIndex(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int primary key, a json, b int generated always as (a->'$.id') stored, index idx1(b), index idx2((cast(a ->'$.data' as signed array))))")
+ tk.MustExec("create table t2 (id int, b int, constraint fk foreign key(b) references t1(b));")
+ tk.MustExec(`insert into t1 (id, a) values (1, '{"id": "1", "data": [1,11,111]}')`)
+ tk.MustExec(`insert into t1 (id, a) values (2, '{"id": "2", "data": [2,22,222]}')`)
+ tk.MustExec("insert into t2 values (1,1),(2,2)")
+ tk.MustGetDBError("insert into t2 values (3,3)", plannercore.ErrNoReferencedRow2)
+ tk.MustGetDBError(`update t1 set a='{"id": "10", "data": [1,11,111]}' where id=1`, plannercore.ErrRowIsReferenced2)
+ tk.MustGetDBError(`delete from t1 where id=1`, plannercore.ErrRowIsReferenced2)
+ tk.MustExec("alter table t2 drop foreign key fk")
+ tk.MustExec("alter table t2 add foreign key fk (b) references t1(b) on delete set null on update cascade")
+ tk.MustExec(`update t1 set a='{"id": "10", "data": [1,11,111]}' where id=1`)
+ tk.MustExec(`delete from t1 where id=2`)
+ tk.MustQuery("select id,b from t1 order by id").Check(testkit.Rows("1 10"))
+ tk.MustQuery("select id,b from t2 order by id").Check(testkit.Rows("1 10", "2 "))
+ tk.MustExec("admin check table t1")
+ tk.MustExec("admin check table t2")
+}
+
+func TestForeignKeyAndSessionVariable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (t timestamp, index(t));")
+ tk.MustExec("create table t2 (t timestamp, foreign key (t) references t1(t) on delete cascade);")
+ tk.MustExec("set @@time_zone='+8:00';")
+ tk.MustExec("insert into t1 values ('2023-01-28 10:29:16');")
+ tk.MustExec("insert into t2 values ('2023-01-28 10:29:16');")
+ tk.MustExec("set @@time_zone='+6:00';")
+ tk.MustExec("delete from t1;")
+ tk.MustQuery("select * from t1").Check(testkit.Rows())
+ tk.MustQuery("select * from t2").Check(testkit.Rows())
+}
diff --git a/executor/foreign_key.go b/executor/foreign_key.go
index 62dbb400537e2..9908a72fd4b04 100644
--- a/executor/foreign_key.go
+++ b/executor/foreign_key.go
@@ -15,8 +15,11 @@
package executor
import (
+ "bytes"
"context"
+ "strconv"
"sync/atomic"
+ "time"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/kv"
@@ -31,6 +34,7 @@ import (
"github.com/pingcap/tidb/types"
driver "github.com/pingcap/tidb/types/parser_driver"
"github.com/pingcap/tidb/util/codec"
+ "github.com/pingcap/tidb/util/execdetails"
"github.com/pingcap/tidb/util/set"
"github.com/tikv/client-go/v2/txnkv/txnsnapshot"
)
@@ -60,12 +64,16 @@ type FKCheckExec struct {
// FKCheckRuntimeStats contains the FKCheckExec runtime stats.
type FKCheckRuntimeStats struct {
- Keys int
+ Total time.Duration
+ Check time.Duration
+ Lock time.Duration
+ Keys int
}
// FKCascadeExec uses to execute foreign key cascade behaviour.
type FKCascadeExec struct {
*fkValueHelper
+ plan *plannercore.FKCascade
b *executorBuilder
tp plannercore.FKCascadeType
referredFK *model.ReferredFKInfo
@@ -78,6 +86,8 @@ type FKCascadeExec struct {
fkValues [][]types.Datum
// new-value-key => UpdatedValuesCouple
fkUpdatedValuesMap map[string]*UpdatedValuesCouple
+
+ stats *FKCascadeRuntimeStats
}
// UpdatedValuesCouple contains the updated new row the old rows, exporting for test.
@@ -86,6 +96,12 @@ type UpdatedValuesCouple struct {
OldValuesList [][]types.Datum
}
+// FKCascadeRuntimeStats contains the FKCascadeExec runtime stats.
+type FKCascadeRuntimeStats struct {
+ Total time.Duration
+ Keys int
+}
+
func buildTblID2FKCheckExecs(sctx sessionctx.Context, tblID2Table map[int64]table.Table, tblID2FKChecks map[int64][]*plannercore.FKCheck) (map[int64][]*FKCheckExec, error) {
fkChecksMap := make(map[int64][]*FKCheckExec)
for tid, tbl := range tblID2Table {
@@ -137,6 +153,10 @@ func buildFKCheckExec(sctx sessionctx.Context, tbl table.Table, fkCheck *planner
}
func (fkc *FKCheckExec) insertRowNeedToCheck(sc *stmtctx.StatementContext, row []types.Datum) error {
+ if fkc.ReferredFK != nil {
+ // Insert into parent table doesn't need to do foreign key check.
+ return nil
+ }
return fkc.addRowNeedToCheck(sc, row)
}
@@ -171,6 +191,20 @@ func (fkc *FKCheckExec) addRowNeedToCheck(sc *stmtctx.StatementContext, row []ty
}
func (fkc *FKCheckExec) doCheck(ctx context.Context) error {
+ if fkc.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl != nil {
+ fkc.stats = &FKCheckRuntimeStats{}
+ defer fkc.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(fkc.ID(), fkc.stats)
+ }
+ if len(fkc.toBeCheckedKeys) == 0 && len(fkc.toBeCheckedPrefixKeys) == 0 {
+ return nil
+ }
+ start := time.Now()
+ if fkc.stats != nil {
+ defer func() {
+ fkc.stats.Keys = len(fkc.toBeCheckedKeys) + len(fkc.toBeCheckedPrefixKeys)
+ fkc.stats.Total = time.Since(start)
+ }()
+ }
txn, err := fkc.ctx.Txn(false)
if err != nil {
return err
@@ -183,6 +217,9 @@ func (fkc *FKCheckExec) doCheck(ctx context.Context) error {
if err != nil {
return err
}
+ if fkc.stats != nil {
+ fkc.stats.Check = time.Since(start)
+ }
if len(fkc.toBeLockedKeys) == 0 {
return nil
}
@@ -198,6 +235,9 @@ func (fkc *FKCheckExec) doCheck(ctx context.Context) error {
// doLockKeys may set TxnCtx.ForUpdate to 1, then if the lock meet write conflict, TiDB can't retry for update.
// So reset TxnCtx.ForUpdate to 0 then can be retry if meet write conflict.
atomic.StoreUint32(&sessVars.TxnCtx.ForUpdate, forUpdate)
+ if fkc.stats != nil {
+ fkc.stats.Lock = time.Since(start) - fkc.stats.Check
+ }
return err
}
@@ -473,6 +513,10 @@ type fkCheckKey struct {
}
func (fkc FKCheckExec) checkRows(ctx context.Context, sc *stmtctx.StatementContext, txn kv.Transaction, rows []toBeCheckedRow) error {
+ if fkc.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl != nil {
+ fkc.stats = &FKCheckRuntimeStats{}
+ defer fkc.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(fkc.ID(), fkc.stats)
+ }
if len(rows) == 0 {
return nil
}
@@ -585,6 +629,7 @@ func (b *executorBuilder) buildFKCascadeExec(tbl table.Table, fkCascade *planner
return &FKCascadeExec{
b: b,
fkValueHelper: helper,
+ plan: fkCascade,
tp: fkCascade.Tp,
referredFK: fkCascade.ReferredFK,
childTable: fkCascade.ChildTable.Meta(),
@@ -637,6 +682,7 @@ func (fkc *FKCascadeExec) buildExecutor(ctx context.Context) (Executor, error) {
if err != nil || p == nil {
return nil, err
}
+ fkc.plan.CascadePlans = append(fkc.plan.CascadePlans, p)
e := fkc.b.build(p)
return e, fkc.b.err
}
@@ -668,6 +714,9 @@ func (fkc *FKCascadeExec) buildFKCascadePlan(ctx context.Context) (plannercore.P
case model.ReferOptionCascade:
couple := fkc.fetchUpdatedValuesCouple()
if couple != nil && len(couple.NewValues) != 0 {
+ if fkc.stats != nil {
+ fkc.stats.Keys += len(couple.OldValuesList)
+ }
stmtNode = GenCascadeUpdateAST(fkc.referredFK.ChildSchema, fkc.childTable.Name, indexName, fkc.fkCols, couple)
}
case model.ReferOptionSetNull:
@@ -683,7 +732,7 @@ func (fkc *FKCascadeExec) buildFKCascadePlan(ctx context.Context) (plannercore.P
if err != nil {
return nil, err
}
- finalPlan, _, err := planner.Optimize(ctx, sctx, stmtNode, fkc.b.is)
+ finalPlan, err := planner.OptimizeForForeignKeyCascade(ctx, sctx, stmtNode, fkc.b.is)
if err != nil {
return nil, err
}
@@ -699,6 +748,9 @@ func (fkc *FKCascadeExec) fetchOnDeleteOrUpdateFKValues() [][]types.Datum {
fkValues = fkc.fkValues[:maxHandleFKValueInOneCascade]
fkc.fkValues = fkc.fkValues[maxHandleFKValueInOneCascade:]
}
+ if fkc.stats != nil {
+ fkc.stats.Keys += len(fkValues)
+ }
return fkValues
}
@@ -811,3 +863,87 @@ func genWhereConditionAstForMultiColumn(cols []*model.ColumnInfo, fkValues [][]t
List: valueList,
}
}
+
+// String implements the RuntimeStats interface.
+func (s *FKCheckRuntimeStats) String() string {
+ buf := bytes.NewBuffer(make([]byte, 0, 32))
+ buf.WriteString("total:")
+ buf.WriteString(execdetails.FormatDuration(s.Total))
+ if s.Check > 0 {
+ buf.WriteString(", check:")
+ buf.WriteString(execdetails.FormatDuration(s.Check))
+ }
+ if s.Lock > 0 {
+ buf.WriteString(", lock:")
+ buf.WriteString(execdetails.FormatDuration(s.Lock))
+ }
+ if s.Keys > 0 {
+ buf.WriteString(", foreign_keys:")
+ buf.WriteString(strconv.Itoa(s.Keys))
+ }
+ return buf.String()
+}
+
+// Clone implements the RuntimeStats interface.
+func (s *FKCheckRuntimeStats) Clone() execdetails.RuntimeStats {
+ newRs := &FKCheckRuntimeStats{
+ Total: s.Total,
+ Check: s.Check,
+ Lock: s.Lock,
+ Keys: s.Keys,
+ }
+ return newRs
+}
+
+// Merge implements the RuntimeStats interface.
+func (s *FKCheckRuntimeStats) Merge(other execdetails.RuntimeStats) {
+ tmp, ok := other.(*FKCheckRuntimeStats)
+ if !ok {
+ return
+ }
+ s.Total += tmp.Total
+ s.Check += tmp.Check
+ s.Lock += tmp.Lock
+ s.Keys += tmp.Keys
+}
+
+// Tp implements the RuntimeStats interface.
+func (s *FKCheckRuntimeStats) Tp() int {
+ return execdetails.TpFKCheckRuntimeStats
+}
+
+// String implements the RuntimeStats interface.
+func (s *FKCascadeRuntimeStats) String() string {
+ buf := bytes.NewBuffer(make([]byte, 0, 32))
+ buf.WriteString("total:")
+ buf.WriteString(execdetails.FormatDuration(s.Total))
+ if s.Keys > 0 {
+ buf.WriteString(", foreign_keys:")
+ buf.WriteString(strconv.Itoa(s.Keys))
+ }
+ return buf.String()
+}
+
+// Clone implements the RuntimeStats interface.
+func (s *FKCascadeRuntimeStats) Clone() execdetails.RuntimeStats {
+ newRs := &FKCascadeRuntimeStats{
+ Total: s.Total,
+ Keys: s.Keys,
+ }
+ return newRs
+}
+
+// Merge implements the RuntimeStats interface.
+func (s *FKCascadeRuntimeStats) Merge(other execdetails.RuntimeStats) {
+ tmp, ok := other.(*FKCascadeRuntimeStats)
+ if !ok {
+ return
+ }
+ s.Total += tmp.Total
+ s.Keys += tmp.Keys
+}
+
+// Tp implements the RuntimeStats interface.
+func (s *FKCascadeRuntimeStats) Tp() int {
+ return execdetails.TpFKCascadeRuntimeStats
+}
diff --git a/executor/grant_test.go b/executor/grant_test.go
index 4c9cb867f792f..3a7720d620673 100644
--- a/executor/grant_test.go
+++ b/executor/grant_test.go
@@ -99,10 +99,10 @@ func TestGrantDBScope(t *testing.T) {
}
// Grant in wrong scope.
- _, err := tk.Exec(` grant create user on test.* to 'testDB1'@'localhost';`)
+ err := tk.ExecToErr(` grant create user on test.* to 'testDB1'@'localhost';`)
require.True(t, terror.ErrorEqual(err, executor.ErrWrongUsage.GenWithStackByArgs("DB GRANT", "GLOBAL PRIVILEGES")))
- _, err = tk.Exec("GRANT SUPER ON test.* TO 'testDB1'@'localhost';")
+ err = tk.ExecToErr("GRANT SUPER ON test.* TO 'testDB1'@'localhost';")
require.True(t, terror.ErrorEqual(err, executor.ErrWrongUsage.GenWithStackByArgs("DB GRANT", "NON-DB PRIVILEGES")))
}
@@ -168,8 +168,8 @@ func TestGrantTableScope(t *testing.T) {
require.Greater(t, strings.Index(p, mysql.Priv2SetStr[v]), -1)
}
- _, err := tk.Exec("GRANT SUPER ON test2 TO 'testTbl1'@'localhost';")
- require.EqualError(t, err, "[executor:1144]Illegal GRANT/REVOKE command; please consult the manual to see which privileges can be used")
+ tk.MustGetErrMsg("GRANT SUPER ON test2 TO 'testTbl1'@'localhost';",
+ "[executor:1144]Illegal GRANT/REVOKE command; please consult the manual to see which privileges can be used")
}
func TestGrantColumnScope(t *testing.T) {
@@ -213,8 +213,8 @@ func TestGrantColumnScope(t *testing.T) {
require.Greater(t, strings.Index(p, mysql.Priv2SetStr[v]), -1)
}
- _, err := tk.Exec("GRANT SUPER(c2) ON test3 TO 'testCol1'@'localhost';")
- require.EqualError(t, err, "[executor:1221]Incorrect usage of COLUMN GRANT and NON-COLUMN PRIVILEGES")
+ tk.MustGetErrMsg("GRANT SUPER(c2) ON test3 TO 'testCol1'@'localhost';",
+ "[executor:1221]Incorrect usage of COLUMN GRANT and NON-COLUMN PRIVILEGES")
}
func TestIssue2456(t *testing.T) {
@@ -394,7 +394,7 @@ func TestMaintainRequire(t *testing.T) {
// test show create user
tk.MustExec(`CREATE USER 'u3'@'%' require issuer '/CN=TiDB admin/OU=TiDB/O=PingCAP/L=San Francisco/ST=California/C=US' subject '/CN=tester1/OU=TiDB/O=PingCAP.Inc/L=Haidian/ST=Beijing/C=ZH' cipher 'AES128-GCM-SHA256'`)
- tk.MustQuery("show create user 'u3'").Check(testkit.Rows("CREATE USER 'u3'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE CIPHER 'AES128-GCM-SHA256' ISSUER '/CN=TiDB admin/OU=TiDB/O=PingCAP/L=San Francisco/ST=California/C=US' SUBJECT '/CN=tester1/OU=TiDB/O=PingCAP.Inc/L=Haidian/ST=Beijing/C=ZH' PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK"))
+ tk.MustQuery("show create user 'u3'").Check(testkit.Rows("CREATE USER 'u3'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE CIPHER 'AES128-GCM-SHA256' ISSUER '/CN=TiDB admin/OU=TiDB/O=PingCAP/L=San Francisco/ST=California/C=US' SUBJECT '/CN=tester1/OU=TiDB/O=PingCAP.Inc/L=Haidian/ST=Beijing/C=ZH' PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT"))
// check issuer/subject/cipher value
err := tk.ExecToErr(`CREATE USER 'u4'@'%' require issuer 'CN=TiDB,OU=PingCAP'`)
diff --git a/executor/hash_table.go b/executor/hash_table.go
index d2b294f52d9ad..50acc4447f4df 100644
--- a/executor/hash_table.go
+++ b/executor/hash_table.go
@@ -92,6 +92,10 @@ func (s *hashStatistic) String() string {
return fmt.Sprintf("probe_collision:%v, build:%v", s.probeCollision, execdetails.FormatDuration(s.buildTableElapse))
}
+type hashNANullBucket struct {
+ entries []*naEntry
+}
+
// hashRowContainer handles the rows and the hash map of a table.
// NOTE: a hashRowContainer may be shallow copied by the invoker, define all the
// member attributes as pointer type to avoid unexpected problems.
@@ -104,16 +108,17 @@ type hashRowContainer struct {
hashTable baseHashTable
// hashNANullBucket stores the rows with any null value in NAAJ join key columns.
// After build process, NANUllBucket is read only here for multi probe worker.
- hashNANullBucket []*naEntry
+ hashNANullBucket *hashNANullBucket
rowContainer *chunk.RowContainer
memTracker *memory.Tracker
// chkBuf buffer the data reads from the disk if rowContainer is spilled.
- chkBuf *chunk.Chunk
+ chkBuf *chunk.Chunk
+ chkBufSizeForOneProbe int64
}
-func newHashRowContainer(sCtx sessionctx.Context, estCount int, hCtx *hashContext, allTypes []*types.FieldType) *hashRowContainer {
+func newHashRowContainer(sCtx sessionctx.Context, hCtx *hashContext, allTypes []*types.FieldType) *hashRowContainer {
maxChunkSize := sCtx.GetSessionVars().MaxChunkSize
rc := chunk.NewRowContainer(allTypes, maxChunkSize)
c := &hashRowContainer{
@@ -124,6 +129,9 @@ func newHashRowContainer(sCtx sessionctx.Context, estCount int, hCtx *hashContex
rowContainer: rc,
memTracker: memory.NewTracker(memory.LabelForRowContainer, -1),
}
+ if isNAAJ := len(hCtx.naKeyColIdx) > 0; isNAAJ {
+ c.hashNANullBucket = &hashNANullBucket{}
+ }
rc.GetMemTracker().AttachTo(c.GetMemTracker())
return c
}
@@ -206,6 +214,15 @@ func (c *hashRowContainer) GetAllMatchedRows(probeHCtx *hashContext, probeSideRo
return matched, nil
}
+// signalCheckpointForJoinMask indicates the times of row probe that a signal detection will be triggered.
+const signalCheckpointForJoinMask int = 1<<14 - 1
+
+// rowSize is the size of Row.
+const rowSize = int64(unsafe.Sizeof(chunk.Row{}))
+
+// rowPtrSize is the size of RowPtr.
+const rowPtrSize = int64(unsafe.Sizeof(chunk.RowPtr{}))
+
// GetMatchedRowsAndPtrs get matched rows and Ptrs from probeRow. It can be called
// in multiple goroutines while each goroutine should keep its own
// h and buf.
@@ -218,7 +235,23 @@ func (c *hashRowContainer) GetMatchedRowsAndPtrs(probeKey uint64, probeRow chunk
matched = matched[:0]
var matchedRow chunk.Row
matchedPtrs = matchedPtrs[:0]
- for _, ptr := range innerPtrs {
+
+ // Some variables used for memTracker.
+ var (
+ matchedDataSize = int64(cap(matched))*rowSize + int64(cap(matchedPtrs))*rowPtrSize
+ lastChunkBufPointer *chunk.Chunk = nil
+ memDelta int64 = 0
+ needTrackMemUsage = cap(innerPtrs) > signalCheckpointForJoinMask
+ )
+ c.chkBuf = nil
+ c.memTracker.Consume(-c.chkBufSizeForOneProbe)
+ if needTrackMemUsage {
+ c.memTracker.Consume(int64(cap(innerPtrs)) * rowPtrSize)
+ defer c.memTracker.Consume(-int64(cap(innerPtrs))*rowPtrSize + memDelta)
+ }
+ c.chkBufSizeForOneProbe = 0
+
+ for i, ptr := range innerPtrs {
matchedRow, c.chkBuf, err = c.rowContainer.GetRowAndAppendToChunk(ptr, c.chkBuf)
if err != nil {
return nil, nil, err
@@ -228,6 +261,19 @@ func (c *hashRowContainer) GetMatchedRowsAndPtrs(probeKey uint64, probeRow chunk
if err != nil {
return nil, nil, err
}
+ if needTrackMemUsage && c.chkBuf != lastChunkBufPointer && lastChunkBufPointer != nil {
+ lastChunkSize := lastChunkBufPointer.MemoryUsage()
+ c.chkBufSizeForOneProbe += lastChunkSize
+ memDelta += lastChunkSize
+ }
+ lastChunkBufPointer = c.chkBuf
+ if needTrackMemUsage && (i&signalCheckpointForJoinMask == signalCheckpointForJoinMask) {
+ // Trigger Consume for checking the OOM Action signal
+ memDelta += int64(cap(matched))*rowSize + int64(cap(matchedPtrs))*rowPtrSize - matchedDataSize
+ matchedDataSize = int64(cap(matched))*rowSize + int64(cap(matchedPtrs))*rowPtrSize
+ c.memTracker.Consume(memDelta + 1)
+ memDelta = 0
+ }
if !ok {
atomic.AddInt64(&c.stat.probeCollision, 1)
continue
@@ -248,7 +294,7 @@ func (c *hashRowContainer) GetNullBucketRows(probeHCtx *hashContext, probeSideRo
mayMatchedRow chunk.Row
)
matched = matched[:0]
- for _, nullEntry := range c.hashNANullBucket {
+ for _, nullEntry := range c.hashNANullBucket.entries {
mayMatchedRow, c.chkBuf, err = c.rowContainer.GetRowAndAppendToChunk(nullEntry.ptr, c.chkBuf)
if err != nil {
return nil, err
@@ -394,7 +440,7 @@ func (c *hashRowContainer) PutChunkSelected(chk *chunk.Chunk, selected, ignoreNu
// collect the null rows to slice.
rowPtr := chunk.RowPtr{ChkIdx: chkIdx, RowIdx: uint32(i)}
// do not directly ref the null bits map here, because the bit map will be reset and reused in next batch of chunk data.
- c.hashNANullBucket = append(c.hashNANullBucket, &naEntry{rowPtr, c.hCtx.naColNullBitMap[i].Clone()})
+ c.hashNANullBucket.entries = append(c.hashNANullBucket.entries, &naEntry{rowPtr, c.hCtx.naColNullBitMap[i].Clone()})
} else {
// insert the not-null rows to hash table.
key := c.hCtx.hashVals[i].Sum64()
diff --git a/executor/hash_table_test.go b/executor/hash_table_test.go
index 3b4a4acee5284..0a387e0e7e5b6 100644
--- a/executor/hash_table_test.go
+++ b/executor/hash_table_test.go
@@ -127,7 +127,7 @@ func testHashRowContainer(t *testing.T, hashFunc func() hash.Hash64, spill bool)
for i := 0; i < numRows; i++ {
hCtx.hashVals = append(hCtx.hashVals, hashFunc())
}
- rowContainer := newHashRowContainer(sctx, 0, hCtx, colTypes)
+ rowContainer := newHashRowContainer(sctx, hCtx, colTypes)
copiedRC = rowContainer.ShallowCopy()
tracker := rowContainer.GetMemTracker()
tracker.SetLabel(memory.LabelForBuildSideResult)
diff --git a/executor/historical_stats_test.go b/executor/historical_stats_test.go
new file mode 100644
index 0000000000000..9ddd34655369f
--- /dev/null
+++ b/executor/historical_stats_test.go
@@ -0,0 +1,397 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package executor_test
+
+import (
+ "encoding/json"
+ "fmt"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/statistics/handle"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/stretchr/testify/require"
+ "github.com/tikv/client-go/v2/oracle"
+)
+
+func TestRecordHistoryStatsAfterAnalyze(t *testing.T) {
+ failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)")
+ defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats")
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@tidb_analyze_version = 2")
+ tk.MustExec("set global tidb_enable_historical_stats = 0")
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, b varchar(10))")
+
+ h := dom.StatsHandle()
+ is := dom.InfoSchema()
+ tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+ require.NoError(t, err)
+
+ // 1. switch off the tidb_enable_historical_stats, and there is no records in table `mysql.stats_history`
+ rows := tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", tableInfo.Meta().ID)).Rows()
+ num, _ := strconv.Atoi(rows[0][0].(string))
+ require.Equal(t, num, 0)
+
+ tk.MustExec("analyze table t with 2 topn")
+ rows = tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", tableInfo.Meta().ID)).Rows()
+ num, _ = strconv.Atoi(rows[0][0].(string))
+ require.Equal(t, num, 0)
+
+ // 2. switch on the tidb_enable_historical_stats and do analyze
+ tk.MustExec("set global tidb_enable_historical_stats = 1")
+ defer tk.MustExec("set global tidb_enable_historical_stats = 0")
+ tk.MustExec("analyze table t with 2 topn")
+ // dump historical stats
+ hsWorker := dom.GetHistoricalStatsWorker()
+ tblID := hsWorker.GetOneHistoricalStatsTable()
+ err = hsWorker.DumpHistoricalStats(tblID, h)
+ require.Nil(t, err)
+ rows = tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", tableInfo.Meta().ID)).Rows()
+ num, _ = strconv.Atoi(rows[0][0].(string))
+ require.GreaterOrEqual(t, num, 1)
+
+ // 3. dump current stats json
+ dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil, true)
+ require.NoError(t, err)
+ jsOrigin, _ := json.Marshal(dumpJSONTable)
+
+ // 4. get the historical stats json
+ rows = tk.MustQuery(fmt.Sprintf("select * from mysql.stats_history where table_id = '%d' and create_time = ("+
+ "select create_time from mysql.stats_history where table_id = '%d' order by create_time desc limit 1) "+
+ "order by seq_no", tableInfo.Meta().ID, tableInfo.Meta().ID)).Rows()
+ num = len(rows)
+ require.GreaterOrEqual(t, num, 1)
+ data := make([][]byte, num)
+ for i, row := range rows {
+ data[i] = []byte(row[1].(string))
+ }
+ jsonTbl, err := handle.BlocksToJSONTable(data)
+ require.NoError(t, err)
+ jsCur, err := json.Marshal(jsonTbl)
+ require.NoError(t, err)
+ // 5. historical stats must be equal to the current stats
+ require.JSONEq(t, string(jsOrigin), string(jsCur))
+}
+
+func TestRecordHistoryStatsMetaAfterAnalyze(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@tidb_analyze_version = 2")
+ tk.MustExec("set global tidb_enable_historical_stats = 0")
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, b int)")
+ tk.MustExec("analyze table test.t")
+
+ h := dom.StatsHandle()
+ is := dom.InfoSchema()
+ tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+ require.NoError(t, err)
+
+ // 1. switch off the tidb_enable_historical_stats, and there is no record in table `mysql.stats_meta_history`
+ tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d'", tableInfo.Meta().ID)).Check(testkit.Rows("0"))
+ // insert demo tuples, and there is no record either.
+ insertNums := 5
+ for i := 0; i < insertNums; i++ {
+ tk.MustExec("insert into test.t (a,b) values (1,1), (2,2), (3,3)")
+ err := h.DumpStatsDeltaToKV(handle.DumpDelta)
+ require.NoError(t, err)
+ }
+ tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d'", tableInfo.Meta().ID)).Check(testkit.Rows("0"))
+
+ // 2. switch on the tidb_enable_historical_stats and insert tuples to produce count/modifyCount delta change.
+ tk.MustExec("set global tidb_enable_historical_stats = 1")
+ defer tk.MustExec("set global tidb_enable_historical_stats = 0")
+
+ for i := 0; i < insertNums; i++ {
+ tk.MustExec("insert into test.t (a,b) values (1,1), (2,2), (3,3)")
+ err := h.DumpStatsDeltaToKV(handle.DumpDelta)
+ require.NoError(t, err)
+ }
+ tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta_history where table_id = '%d' order by create_time", tableInfo.Meta().ID)).Sort().Check(
+ testkit.Rows("18 18", "21 21", "24 24", "27 27", "30 30"))
+ tk.MustQuery(fmt.Sprintf("select distinct source from mysql.stats_meta_history where table_id = '%d'", tableInfo.Meta().ID)).Sort().Check(testkit.Rows("flush stats"))
+
+ // assert delete
+ tk.MustExec("delete from test.t where test.t.a = 1")
+ err = h.DumpStatsDeltaToKV(handle.DumpAll)
+ require.NoError(t, err)
+ tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta where table_id = '%d' order by create_time desc", tableInfo.Meta().ID)).Sort().Check(
+ testkit.Rows("40 20"))
+ tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta_history where table_id = '%d' order by create_time desc limit 1", tableInfo.Meta().ID)).Sort().Check(
+ testkit.Rows("40 20"))
+
+ // assert update
+ tk.MustExec("update test.t set test.t.b = 4 where test.t.a = 2")
+ err = h.DumpStatsDeltaToKV(handle.DumpAll)
+ require.NoError(t, err)
+ tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta where table_id = '%d' order by create_time desc", tableInfo.Meta().ID)).Sort().Check(
+ testkit.Rows("50 20"))
+ tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta_history where table_id = '%d' order by create_time desc limit 1", tableInfo.Meta().ID)).Sort().Check(
+ testkit.Rows("50 20"))
+}
+
+func TestGCHistoryStatsAfterDropTable(t *testing.T) {
+ failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)")
+ defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats")
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set global tidb_enable_historical_stats = 1")
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, b varchar(10))")
+ tk.MustExec("analyze table test.t")
+ is := dom.InfoSchema()
+ tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+ require.NoError(t, err)
+ // dump historical stats
+ h := dom.StatsHandle()
+ hsWorker := dom.GetHistoricalStatsWorker()
+ tblID := hsWorker.GetOneHistoricalStatsTable()
+ err = hsWorker.DumpHistoricalStats(tblID, h)
+ require.Nil(t, err)
+
+ // assert the records of history stats table
+ tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d' order by create_time",
+ tableInfo.Meta().ID)).Check(testkit.Rows("1"))
+ tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'",
+ tableInfo.Meta().ID)).Check(testkit.Rows("1"))
+ // drop the table and gc stats
+ tk.MustExec("drop table t")
+ is = dom.InfoSchema()
+ h.GCStats(is, 0)
+
+ // assert stats_history tables delete the record of dropped table
+ tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d' order by create_time",
+ tableInfo.Meta().ID)).Check(testkit.Rows("0"))
+ tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'",
+ tableInfo.Meta().ID)).Check(testkit.Rows("0"))
+}
+
+func TestAssertHistoricalStatsAfterAlterTable(t *testing.T) {
+ failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)")
+ defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats")
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set global tidb_enable_historical_stats = 1")
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, b varchar(10),c int, KEY `idx` (`c`))")
+ tk.MustExec("analyze table test.t")
+ is := dom.InfoSchema()
+ tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+ require.NoError(t, err)
+ // dump historical stats
+ h := dom.StatsHandle()
+ hsWorker := dom.GetHistoricalStatsWorker()
+ tblID := hsWorker.GetOneHistoricalStatsTable()
+ err = hsWorker.DumpHistoricalStats(tblID, h)
+ require.Nil(t, err)
+
+ time.Sleep(1 * time.Second)
+ snapshot := oracle.GoTimeToTS(time.Now())
+ jsTable, err := h.DumpHistoricalStatsBySnapshot("test", tableInfo.Meta(), snapshot)
+ require.NoError(t, err)
+ require.NotNil(t, jsTable)
+ require.NotEqual(t, jsTable.Version, uint64(0))
+ originVersion := jsTable.Version
+
+ // assert historical stats non-change after drop column
+ tk.MustExec("alter table t drop column b")
+ h.GCStats(is, 0)
+ snapshot = oracle.GoTimeToTS(time.Now())
+ jsTable, err = h.DumpHistoricalStatsBySnapshot("test", tableInfo.Meta(), snapshot)
+ require.NoError(t, err)
+ require.NotNil(t, jsTable)
+ require.Equal(t, jsTable.Version, originVersion)
+
+ // assert historical stats non-change after drop index
+ tk.MustExec("alter table t drop index idx")
+ h.GCStats(is, 0)
+ snapshot = oracle.GoTimeToTS(time.Now())
+ jsTable, err = h.DumpHistoricalStatsBySnapshot("test", tableInfo.Meta(), snapshot)
+ require.NoError(t, err)
+ require.NotNil(t, jsTable)
+ require.Equal(t, jsTable.Version, originVersion)
+}
+
+func TestGCOutdatedHistoryStats(t *testing.T) {
+ failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)")
+ defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats")
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set global tidb_enable_historical_stats = 1")
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, b varchar(10))")
+ tk.MustExec("analyze table test.t")
+ is := dom.InfoSchema()
+ tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+ require.NoError(t, err)
+ // dump historical stats
+ h := dom.StatsHandle()
+ hsWorker := dom.GetHistoricalStatsWorker()
+ tblID := hsWorker.GetOneHistoricalStatsTable()
+ err = hsWorker.DumpHistoricalStats(tblID, h)
+ require.Nil(t, err)
+
+ // assert the records of history stats table
+ tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d' order by create_time",
+ tableInfo.Meta().ID)).Check(testkit.Rows("1"))
+ tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'",
+ tableInfo.Meta().ID)).Check(testkit.Rows("1"))
+
+ variable.HistoricalStatsDuration.Store(1 * time.Second)
+ time.Sleep(2 * time.Second)
+ err = dom.StatsHandle().ClearOutdatedHistoryStats()
+ require.NoError(t, err)
+ // assert the records of history stats table
+ tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d' order by create_time",
+ tableInfo.Meta().ID)).Check(testkit.Rows("0"))
+ tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'",
+ tableInfo.Meta().ID)).Check(testkit.Rows("0"))
+}
+
+func TestPartitionTableHistoricalStats(t *testing.T) {
+ failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)")
+ defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats")
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set global tidb_enable_historical_stats = 1")
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec(`CREATE TABLE t (a int, b int, index idx(b))
+PARTITION BY RANGE ( a ) (
+PARTITION p0 VALUES LESS THAN (6)
+)`)
+ tk.MustExec("delete from mysql.stats_history")
+
+ tk.MustExec("analyze table test.t")
+ // dump historical stats
+ h := dom.StatsHandle()
+ hsWorker := dom.GetHistoricalStatsWorker()
+
+ // assert global table and partition table be dumped
+ tblID := hsWorker.GetOneHistoricalStatsTable()
+ err := hsWorker.DumpHistoricalStats(tblID, h)
+ require.NoError(t, err)
+ tblID = hsWorker.GetOneHistoricalStatsTable()
+ err = hsWorker.DumpHistoricalStats(tblID, h)
+ require.NoError(t, err)
+ tk.MustQuery("select count(*) from mysql.stats_history").Check(testkit.Rows("2"))
+}
+
+func TestDumpHistoricalStatsByTable(t *testing.T) {
+ failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)")
+ defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats")
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set global tidb_enable_historical_stats = 1")
+ tk.MustExec("set @@tidb_partition_prune_mode='static'")
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec(`CREATE TABLE t (a int, b int, index idx(b))
+PARTITION BY RANGE ( a ) (
+PARTITION p0 VALUES LESS THAN (6)
+)`)
+ // dump historical stats
+ h := dom.StatsHandle()
+
+ tk.MustExec("analyze table t")
+ is := dom.InfoSchema()
+ tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+ require.NoError(t, err)
+ require.NotNil(t, tbl)
+
+ // dump historical stats
+ hsWorker := dom.GetHistoricalStatsWorker()
+ // only partition p0 stats will be dumped in static mode
+ tblID := hsWorker.GetOneHistoricalStatsTable()
+ require.NotEqual(t, tblID, -1)
+ err = hsWorker.DumpHistoricalStats(tblID, h)
+ require.NoError(t, err)
+ tblID = hsWorker.GetOneHistoricalStatsTable()
+ require.Equal(t, tblID, int64(-1))
+
+ time.Sleep(1 * time.Second)
+ snapshot := oracle.GoTimeToTS(time.Now())
+ jsTable, err := h.DumpHistoricalStatsBySnapshot("test", tbl.Meta(), snapshot)
+ require.NoError(t, err)
+ require.NotNil(t, jsTable)
+ // only has p0 stats
+ require.NotNil(t, jsTable.Partitions["p0"])
+ require.Nil(t, jsTable.Partitions["global"])
+
+ // change static to dynamic then assert
+ tk.MustExec("set @@tidb_partition_prune_mode='dynamic'")
+ tk.MustExec("analyze table t")
+ require.NoError(t, err)
+ // global and p0's stats will be dumped
+ tblID = hsWorker.GetOneHistoricalStatsTable()
+ require.NotEqual(t, tblID, -1)
+ err = hsWorker.DumpHistoricalStats(tblID, h)
+ require.NoError(t, err)
+ tblID = hsWorker.GetOneHistoricalStatsTable()
+ require.NotEqual(t, tblID, -1)
+ err = hsWorker.DumpHistoricalStats(tblID, h)
+ require.NoError(t, err)
+ time.Sleep(1 * time.Second)
+ snapshot = oracle.GoTimeToTS(time.Now())
+ jsTable, err = h.DumpHistoricalStatsBySnapshot("test", tbl.Meta(), snapshot)
+ require.NoError(t, err)
+ require.NotNil(t, jsTable)
+ // has both global and p0 stats
+ require.NotNil(t, jsTable.Partitions["p0"])
+ require.NotNil(t, jsTable.Partitions["global"])
+}
+
+func TestDumpHistoricalStatsFallback(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set global tidb_enable_historical_stats = 0")
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec(`CREATE TABLE t (a int, b int, index idx(b))
+PARTITION BY RANGE ( a ) (
+PARTITION p0 VALUES LESS THAN (6)
+)`)
+ // dump historical stats
+ tk.MustExec("analyze table t")
+ is := dom.InfoSchema()
+ tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+ require.NoError(t, err)
+ require.NotNil(t, tbl)
+
+ // dump historical stats
+ hsWorker := dom.GetHistoricalStatsWorker()
+ tblID := hsWorker.GetOneHistoricalStatsTable()
+ // assert no historical stats task generated
+ require.Equal(t, tblID, int64(-1))
+ tk.MustExec("set global tidb_enable_historical_stats = 1")
+ h := dom.StatsHandle()
+ jt, err := h.DumpHistoricalStatsBySnapshot("test", tbl.Meta(), oracle.GoTimeToTS(time.Now()))
+ require.NoError(t, err)
+ require.NotNil(t, jt)
+ require.False(t, jt.IsHistoricalStats)
+}
diff --git a/executor/index_advise_test.go b/executor/index_advise_test.go
index 3415ffe83537b..2e133d25502ff 100644
--- a/executor/index_advise_test.go
+++ b/executor/index_advise_test.go
@@ -65,3 +65,134 @@ func TestIndexAdvise(t *testing.T) {
require.Equal(t, uint64(4), ia.MaxIndexNum.PerTable)
require.Equal(t, uint64(5), ia.MaxIndexNum.PerDB)
}
+
+func TestIndexJoinProjPattern(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec(`create table t1(
+pnbrn_cnaps varchar(5) not null,
+new_accno varchar(18) not null,
+primary key(pnbrn_cnaps,new_accno) nonclustered
+);`)
+ tk.MustExec(`create table t2(
+pnbrn_cnaps varchar(5) not null,
+txn_accno varchar(18) not null,
+txn_dt date not null,
+yn_frz varchar(1) default null
+);`)
+ tk.MustExec(`insert into t1(pnbrn_cnaps,new_accno) values ("40001","123")`)
+ tk.MustExec(`insert into t2(pnbrn_cnaps, txn_accno, txn_dt, yn_frz) values ("40001","123","20221201","0");`)
+
+ sql := `update
+/*+ inl_join(a) */
+t2 b,
+(
+select t1.pnbrn_cnaps,
+t1.new_accno
+from t1
+where t1.pnbrn_cnaps = '40001'
+) a
+set b.yn_frz = '1'
+where b.txn_dt = str_to_date('20221201', '%Y%m%d')
+and b.pnbrn_cnaps = a.pnbrn_cnaps
+and b.txn_accno = a.new_accno;`
+ rows := [][]interface{}{
+ {"Update_8"},
+ {"└─IndexJoin_14"},
+ {" ├─TableReader_25(Build)"},
+ {" │ └─Selection_24"},
+ {" │ └─TableFullScan_23"},
+ {" └─IndexReader_12(Probe)"},
+ {" └─Selection_11"},
+ {" └─IndexRangeScan_10"},
+ }
+ tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='ON'")
+ tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows)
+ rows = [][]interface{}{
+ {"Update_8"},
+ {"└─HashJoin_10"},
+ {" ├─IndexReader_17(Build)"},
+ {" │ └─IndexRangeScan_16"},
+ {" └─TableReader_14(Probe)"},
+ {" └─Selection_13"},
+ {" └─TableFullScan_12"},
+ }
+ tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='OFF'")
+ tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows)
+
+ tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='ON'")
+ tk.MustExec(sql)
+ tk.MustQuery("select yn_frz from t2").Check(testkit.Rows("1"))
+}
+
+func TestIndexJoinSelPattern(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec(` create table tbl_miss(
+id bigint(20) unsigned not null
+,txn_dt date default null
+,perip_sys_uuid varchar(32) not null
+,rvrs_idr varchar(1) not null
+,primary key(id) clustered
+,key idx1 (txn_dt, perip_sys_uuid, rvrs_idr)
+);
+`)
+ tk.MustExec(`insert into tbl_miss (id,txn_dt,perip_sys_uuid,rvrs_idr) values (1,"20221201","123","1");`)
+ tk.MustExec(`create table tbl_src(
+txn_dt date default null
+,uuid varchar(32) not null
+,rvrs_idr char(1)
+,expd_inf varchar(5000)
+,primary key(uuid,rvrs_idr) nonclustered
+);
+`)
+ tk.MustExec(`insert into tbl_src (txn_dt,uuid,rvrs_idr) values ("20221201","123","1");`)
+ sql := `select /*+ use_index(mis,) inl_join(src) */
+ *
+ from tbl_miss mis
+ ,tbl_src src
+ where src.txn_dt >= str_to_date('20221201', '%Y%m%d')
+ and mis.id between 1 and 10000
+ and mis.perip_sys_uuid = src.uuid
+ and mis.rvrs_idr = src.rvrs_idr
+ and mis.txn_dt = src.txn_dt
+ and (
+ case when isnull(src.expd_inf) = 1 then ''
+ else
+ substr(concat_ws('',src.expd_inf,'~~'),
+ instr(concat_ws('',src.expd_inf,'~~'),'~~a4') + 4,
+ instr(substr(concat_ws('',src.expd_inf,'~~'),
+ instr(concat_ws('',src.expd_inf,'~~'),'~~a4') + 4, length(concat_ws('',src.expd_inf,'~~'))),'~~') -1)
+ end
+ ) != '01';`
+ rows := [][]interface{}{
+ {"HashJoin_9"},
+ {"├─TableReader_12(Build)"},
+ {"│ └─Selection_11"},
+ {"│ └─TableRangeScan_10"},
+ {"└─Selection_13(Probe)"},
+ {" └─TableReader_16"},
+ {" └─Selection_15"},
+ {" └─TableFullScan_14"},
+ }
+ tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='OFF'")
+ tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows)
+ rows = [][]interface{}{
+ {"IndexJoin_13"},
+ {"├─TableReader_25(Build)"},
+ {"│ └─Selection_24"},
+ {"│ └─TableRangeScan_23"},
+ {"└─Selection_12(Probe)"},
+ {" └─IndexLookUp_11"},
+ {" ├─IndexRangeScan_8(Build)"},
+ {" └─Selection_10(Probe)"},
+ {" └─TableRowIDScan_9"},
+ }
+ tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='ON'")
+ tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows)
+ tk.MustQuery(sql).Check(testkit.Rows("1 2022-12-01 123 1 2022-12-01 123 1 "))
+ tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='OFF'")
+ tk.MustQuery(sql).Check(testkit.Rows("1 2022-12-01 123 1 2022-12-01 123 1 "))
+}
diff --git a/executor/index_lookup_hash_join.go b/executor/index_lookup_hash_join.go
index c54b60749601d..58bd84ff6e4d6 100644
--- a/executor/index_lookup_hash_join.go
+++ b/executor/index_lookup_hash_join.go
@@ -134,7 +134,6 @@ func (e *IndexNestedLoopHashJoin) Open(ctx context.Context) error {
e.innerPtrBytes = make([][]byte, 0, 8)
if e.runtimeStats != nil {
e.stats = &indexLookUpJoinRuntimeStats{}
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
e.finished.Store(false)
return nil
@@ -288,6 +287,9 @@ func (e *IndexNestedLoopHashJoin) isDryUpTasks(ctx context.Context) bool {
// Close implements the IndexNestedLoopHashJoin Executor interface.
func (e *IndexNestedLoopHashJoin) Close() error {
+ if e.stats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
+ }
if e.cancelFunc != nil {
e.cancelFunc()
}
diff --git a/executor/index_lookup_join.go b/executor/index_lookup_join.go
index 05cc337d3d7ee..187e83cc0f763 100644
--- a/executor/index_lookup_join.go
+++ b/executor/index_lookup_join.go
@@ -171,7 +171,6 @@ func (e *IndexLookUpJoin) Open(ctx context.Context) error {
e.finished.Store(false)
if e.runtimeStats != nil {
e.stats = &indexLookUpJoinRuntimeStats{}
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
e.cancelFunc = nil
return nil
@@ -765,6 +764,9 @@ func (iw *innerWorker) hasNullInJoinKey(row chunk.Row) bool {
// Close implements the Executor interface.
func (e *IndexLookUpJoin) Close() error {
+ if e.stats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
+ }
if e.cancelFunc != nil {
e.cancelFunc()
}
diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go
index 2f251761b71c2..600f052b1225e 100644
--- a/executor/index_lookup_join_test.go
+++ b/executor/index_lookup_join_test.go
@@ -401,6 +401,7 @@ func TestIssue27138(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
+ tk.MustExec("set tidb_cost_model_version=1")
tk.MustExec("drop table if exists t1,t2")
tk.MustExec("set @old_tidb_partition_prune_mode=@@tidb_partition_prune_mode")
@@ -427,17 +428,19 @@ PARTITIONS 1`)
// Why does the t2.prefiller need be at least 2^32 ? If smaller the bug will not appear!?!
tk.MustExec("insert into t2 values ( pow(2,32), 1, 1), ( pow(2,32)+1, 2, 0)")
+ tk.MustExec(`analyze table t1`)
+ tk.MustExec(`analyze table t2`)
// Why must it be = 1 and not 2?
- tk.MustQuery("explain select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows("" +
- "IndexJoin_15 10.00 root inner join, inner:TableReader_14, outer key:test.t2.prefiller, inner key:test.t1.id, equal cond:eq(test.t2.prefiller, test.t1.id)]\n" +
- "[├─HashAgg_25(Build) 8.00 root group by:test.t2.prefiller, funcs:firstrow(test.t2.prefiller)->test.t2.prefiller]\n" +
- "[│ └─TableReader_26 8.00 root data:HashAgg_20]\n" +
- "[│ └─HashAgg_20 8.00 cop[tikv] group by:test.t2.prefiller, ]\n" +
- "[│ └─Selection_24 10.00 cop[tikv] eq(test.t2.postfiller, 1)]\n" +
- "[│ └─TableFullScan_23 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo]\n" +
- "[└─TableReader_14(Probe) 8.00 root partition:all data:TableRangeScan_13]\n" +
- "[ └─TableRangeScan_13 8.00 cop[tikv] table:t1 range: decided by [eq(test.t1.id, test.t2.prefiller)], keep order:false, stats:pseudo"))
+ tk.MustQuery("explain format='brief' select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows(""+
+ `IndexJoin 1.25 root inner join, inner:TableReader, outer key:test.t2.prefiller, inner key:test.t1.id, equal cond:eq(test.t2.prefiller, test.t1.id)`,
+ `├─HashAgg(Build) 1.00 root group by:test.t2.prefiller, funcs:firstrow(test.t2.prefiller)->test.t2.prefiller`,
+ `│ └─TableReader 1.00 root data:HashAgg`,
+ `│ └─HashAgg 1.00 cop[tikv] group by:test.t2.prefiller, `,
+ `│ └─Selection 1.00 cop[tikv] eq(test.t2.postfiller, 1)`,
+ `│ └─TableFullScan 2.00 cop[tikv] table:t2 keep order:false`,
+ `└─TableReader(Probe) 1.00 root partition:all data:TableRangeScan`,
+ ` └─TableRangeScan 1.00 cop[tikv] table:t1 range: decided by [eq(test.t1.id, test.t2.prefiller)], keep order:false, stats:pseudo`))
tk.MustQuery("show warnings").Check(testkit.Rows())
// without fix it fails with: "runtime error: index out of range [0] with length 0"
tk.MustQuery("select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows())
diff --git a/executor/index_lookup_merge_join.go b/executor/index_lookup_merge_join.go
index 8bd379944c825..e0fb176fff589 100644
--- a/executor/index_lookup_merge_join.go
+++ b/executor/index_lookup_merge_join.go
@@ -715,6 +715,9 @@ func (imw *innerMergeWorker) fetchNextInnerResult(ctx context.Context, task *loo
// Close implements the Executor interface.
func (e *IndexLookUpMergeJoin) Close() error {
+ if e.runtimeStats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.runtimeStats)
+ }
if e.cancelFunc != nil {
e.cancelFunc()
e.cancelFunc = nil
diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 0e7eb394710fd..c4da6edc9f5cb 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -50,6 +50,14 @@ var (
_ Executor = &IndexMergeReaderExecutor{}
)
+const (
+ partialIndexWorkerType = "IndexMergePartialIndexWorker"
+ partialTableWorkerType = "IndexMergePartialTableWorker"
+ processWorkerType = "IndexMergeProcessWorker"
+ partTblIntersectionWorkerType = "IndexMergePartTblIntersectionWorker"
+ tableScanWorkerType = "IndexMergeTableScanWorker"
+)
+
// IndexMergeReaderExecutor accesses a table with multiple index/table scan.
// There are three types of workers:
// 1. partialTableWorker/partialIndexWorker, which are used to fetch the handles
@@ -86,16 +94,16 @@ type IndexMergeReaderExecutor struct {
// All fields above are immutable.
- tblWorkerWg sync.WaitGroup
- idxWorkerWg sync.WaitGroup
- processWokerWg sync.WaitGroup
- finished chan struct{}
+ tblWorkerWg sync.WaitGroup
+ idxWorkerWg sync.WaitGroup
+ processWorkerWg sync.WaitGroup
+ finished chan struct{}
workerStarted bool
keyRanges [][]kv.KeyRange
- resultCh chan *lookupTableTask
- resultCurr *lookupTableTask
+ resultCh chan *indexMergeTableTask
+ resultCurr *indexMergeTableTask
feedbacks []*statistics.QueryFeedback
// memTracker is used to track the memory usage of this executor.
@@ -118,6 +126,16 @@ type IndexMergeReaderExecutor struct {
isCorColInPartialFilters []bool
isCorColInTableFilter bool
isCorColInPartialAccess []bool
+
+ // Whether it's intersection or union.
+ isIntersection bool
+}
+
+type indexMergeTableTask struct {
+ lookupTableTask
+
+ // parTblIdx are only used in indexMergeProcessWorker.fetchLoopIntersection.
+ parTblIdx int
}
// Table implements the dataSourceExecutor interface.
@@ -129,7 +147,12 @@ func (e *IndexMergeReaderExecutor) Table() table.Table {
func (e *IndexMergeReaderExecutor) Open(ctx context.Context) (err error) {
e.keyRanges = make([][]kv.KeyRange, 0, len(e.partialPlans))
e.initRuntimeStats()
-
+ if e.isCorColInTableFilter {
+ e.tableRequest.Executors, err = constructDistExec(e.ctx, e.tblPlans)
+ if err != nil {
+ return err
+ }
+ }
if err = e.rebuildRangeForCorCol(); err != nil {
return err
}
@@ -150,7 +173,7 @@ func (e *IndexMergeReaderExecutor) Open(ctx context.Context) (err error) {
}
}
e.finished = make(chan struct{})
- e.resultCh = make(chan *lookupTableTask, atomic.LoadInt32(&LookupTableTaskChannelSize))
+ e.resultCh = make(chan *indexMergeTableTask, atomic.LoadInt32(&LookupTableTaskChannelSize))
e.memTracker = memory.NewTracker(e.id, -1)
e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker)
return nil
@@ -194,7 +217,7 @@ func (e *IndexMergeReaderExecutor) buildKeyRangesForTable(tbl table.Table) (rang
if err != nil {
return nil, err
}
- keyRanges := append(firstKeyRanges, secondKeyRanges...)
+ keyRanges := append(firstKeyRanges.FirstPartitionRange(), secondKeyRanges.FirstPartitionRange()...)
ranges = append(ranges, keyRanges)
continue
}
@@ -202,15 +225,15 @@ func (e *IndexMergeReaderExecutor) buildKeyRangesForTable(tbl table.Table) (rang
if err != nil {
return nil, err
}
- ranges = append(ranges, keyRange)
+ ranges = append(ranges, keyRange.FirstPartitionRange())
}
return ranges, nil
}
func (e *IndexMergeReaderExecutor) startWorkers(ctx context.Context) error {
exitCh := make(chan struct{})
- workCh := make(chan *lookupTableTask, 1)
- fetchCh := make(chan *lookupTableTask, len(e.keyRanges))
+ workCh := make(chan *indexMergeTableTask, 1)
+ fetchCh := make(chan *indexMergeTableTask, len(e.keyRanges))
e.startIndexMergeProcessWorker(ctx, workCh, fetchCh)
@@ -237,30 +260,40 @@ func (e *IndexMergeReaderExecutor) startWorkers(ctx context.Context) error {
return nil
}
-func (e *IndexMergeReaderExecutor) waitPartialWorkersAndCloseFetchChan(fetchCh chan *lookupTableTask) {
+func (e *IndexMergeReaderExecutor) waitPartialWorkersAndCloseFetchChan(fetchCh chan *indexMergeTableTask) {
e.idxWorkerWg.Wait()
close(fetchCh)
}
-func (e *IndexMergeReaderExecutor) startIndexMergeProcessWorker(ctx context.Context, workCh chan<- *lookupTableTask, fetch <-chan *lookupTableTask) {
+func (e *IndexMergeReaderExecutor) startIndexMergeProcessWorker(ctx context.Context, workCh chan<- *indexMergeTableTask, fetch <-chan *indexMergeTableTask) {
idxMergeProcessWorker := &indexMergeProcessWorker{
indexMerge: e,
stats: e.stats,
}
- e.processWokerWg.Add(1)
+ e.processWorkerWg.Add(1)
go func() {
defer trace.StartRegion(ctx, "IndexMergeProcessWorker").End()
util.WithRecovery(
func() {
- idxMergeProcessWorker.fetchLoop(ctx, fetch, workCh, e.resultCh, e.finished)
+ if e.isIntersection {
+ idxMergeProcessWorker.fetchLoopIntersection(ctx, fetch, workCh, e.resultCh, e.finished)
+ } else {
+ idxMergeProcessWorker.fetchLoopUnion(ctx, fetch, workCh, e.resultCh, e.finished)
+ }
},
- idxMergeProcessWorker.handleLoopFetcherPanic(ctx, e.resultCh),
+ handleWorkerPanic(ctx, e.finished, e.resultCh, nil, processWorkerType),
)
- e.processWokerWg.Done()
+ e.processWorkerWg.Done()
}()
}
-func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *lookupTableTask, workID int) error {
+func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *indexMergeTableTask, workID int) error {
+ failpoint.Inject("testIndexMergeResultChCloseEarly", func(_ failpoint.Value) {
+ // Wait for processWorker to close resultCh.
+ time.Sleep(2)
+ // Should use fetchCh instead of resultCh to send error.
+ syncErr(ctx, e.finished, fetchCh, errors.New("testIndexMergeResultChCloseEarly"))
+ })
if e.runtimeStats != nil {
collExec := true
e.dagPBs[workID].CollectExecutionSummaries = &collExec
@@ -284,6 +317,17 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
defer e.idxWorkerWg.Done()
util.WithRecovery(
func() {
+ failpoint.Inject("testIndexMergePanicPartialIndexWorker", nil)
+ failpoint.Inject("mockSleepBeforeStartTableReader", func(_ failpoint.Value) {
+ select {
+ case <-ctx.Done():
+ failpoint.Return()
+ case <-e.finished:
+ failpoint.Return()
+ case <-exitCh:
+ failpoint.Return()
+ }
+ })
worker := &partialIndexWorker{
stats: e.stats,
idxID: e.getPartitalPlanID(workID),
@@ -291,13 +335,14 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
batchSize: e.maxChunkSize,
maxBatchSize: e.ctx.GetSessionVars().IndexLookupSize,
maxChunkSize: e.maxChunkSize,
+ memTracker: e.memTracker,
}
if e.isCorColInPartialFilters[workID] {
// We got correlated column, so need to refresh Selection operator.
var err error
if e.dagPBs[workID].Executors, err = constructDistExec(e.ctx, e.partialPlans[workID]); err != nil {
- worker.syncErr(e.resultCh, err)
+ syncErr(ctx, e.finished, fetchCh, err)
return
}
}
@@ -331,12 +376,12 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
})
kvReq, err := builder.SetKeyRanges(keyRange).Build()
if err != nil {
- worker.syncErr(e.resultCh, err)
+ syncErr(ctx, e.finished, fetchCh, err)
return
}
result, err := distsql.SelectWithRuntimeStats(ctx, e.ctx, kvReq, e.handleCols.GetFieldsTypes(), e.feedbacks[workID], getPhysicalPlanIDs(e.partialPlans[workID]), e.getPartitalPlanID(workID))
if err != nil {
- worker.syncErr(e.resultCh, err)
+ syncErr(ctx, e.finished, fetchCh, err)
return
}
worker.batchSize = e.maxChunkSize
@@ -349,7 +394,7 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
// fetch all data from this partition
ctx1, cancel := context.WithCancel(ctx)
- _, fetchErr := worker.fetchHandles(ctx1, result, exitCh, fetchCh, e.resultCh, e.finished, e.handleCols)
+ _, fetchErr := worker.fetchHandles(ctx1, result, exitCh, fetchCh, e.finished, e.handleCols, parTblIdx)
if fetchErr != nil { // this error is synced in fetchHandles(), don't sync it again
e.feedbacks[workID].Invalidate()
}
@@ -363,14 +408,14 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
}
}
},
- e.handleHandlesFetcherPanic(ctx, e.resultCh, "partialIndexWorker"),
+ handleWorkerPanic(ctx, e.finished, fetchCh, nil, partialIndexWorkerType),
)
}()
return nil
}
-func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *lookupTableTask, workID int) error {
+func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *indexMergeTableTask, workID int) error {
ts := e.partialPlans[workID][0].(*plannercore.PhysicalTableScan)
tbls := make([]table.Table, 0, 1)
@@ -387,6 +432,17 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
defer e.idxWorkerWg.Done()
util.WithRecovery(
func() {
+ failpoint.Inject("testIndexMergePanicPartialTableWorker", nil)
+ failpoint.Inject("mockSleepBeforeStartTableReader", func(_ failpoint.Value) {
+ select {
+ case <-ctx.Done():
+ failpoint.Return()
+ case <-e.finished:
+ failpoint.Return()
+ case <-exitCh:
+ failpoint.Return()
+ }
+ })
var err error
partialTableReader := &TableReaderExecutor{
baseExecutor: newBaseExecutor(e.ctx, ts.Schema(), e.getPartitalPlanID(workID)),
@@ -408,17 +464,18 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
maxBatchSize: e.ctx.GetSessionVars().IndexLookupSize,
maxChunkSize: e.maxChunkSize,
tableReader: partialTableReader,
+ memTracker: e.memTracker,
}
if e.isCorColInPartialFilters[workID] {
if e.dagPBs[workID].Executors, err = constructDistExec(e.ctx, e.partialPlans[workID]); err != nil {
- worker.syncErr(e.resultCh, err)
+ syncErr(ctx, e.finished, fetchCh, err)
return
}
partialTableReader.dagPB = e.dagPBs[workID]
}
- for _, tbl := range tbls {
+ for parTblIdx, tbl := range tbls {
// check if this executor is closed
select {
case <-e.finished:
@@ -430,7 +487,7 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
partialTableReader.table = tbl
if err = partialTableReader.Open(ctx); err != nil {
logutil.Logger(ctx).Error("open Select result failed:", zap.Error(err))
- worker.syncErr(e.resultCh, err)
+ syncErr(ctx, e.finished, fetchCh, err)
break
}
worker.batchSize = e.maxChunkSize
@@ -443,7 +500,7 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
// fetch all handles from this table
ctx1, cancel := context.WithCancel(ctx)
- _, fetchErr := worker.fetchHandles(ctx1, exitCh, fetchCh, e.resultCh, e.finished, e.handleCols)
+ _, fetchErr := worker.fetchHandles(ctx1, exitCh, fetchCh, e.finished, e.handleCols, parTblIdx)
if fetchErr != nil { // this error is synced in fetchHandles, so don't sync it again
e.feedbacks[workID].Invalidate()
}
@@ -459,7 +516,7 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
}
}
},
- e.handleHandlesFetcherPanic(ctx, e.resultCh, "partialTableWorker"),
+ handleWorkerPanic(ctx, e.finished, fetchCh, nil, partialTableWorkerType),
)
}()
return nil
@@ -470,7 +527,6 @@ func (e *IndexMergeReaderExecutor) initRuntimeStats() {
e.stats = &IndexMergeRuntimeStat{
Concurrency: e.ctx.GetSessionVars().IndexLookupConcurrency(),
}
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
}
@@ -496,18 +552,11 @@ type partialTableWorker struct {
maxChunkSize int
tableReader Executor
partition table.PhysicalTable // it indicates if this worker is accessing a particular partition table
+ memTracker *memory.Tracker
}
-func (w *partialTableWorker) syncErr(resultCh chan<- *lookupTableTask, err error) {
- doneCh := make(chan error, 1)
- doneCh <- err
- resultCh <- &lookupTableTask{
- doneCh: doneCh,
- }
-}
-
-func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *lookupTableTask, resultCh chan<- *lookupTableTask,
- finished <-chan struct{}, handleCols plannercore.HandleCols) (count int64, err error) {
+func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *indexMergeTableTask,
+ finished <-chan struct{}, handleCols plannercore.HandleCols, parTblIdx int) (count int64, err error) {
chk := w.sc.GetSessionVars().GetNewChunkWithCapacity(retTypes(w.tableReader), w.maxChunkSize, w.maxChunkSize, w.tableReader.base().AllocPool)
var basic *execdetails.BasicRuntimeStats
if be := w.tableReader.base(); be != nil && be.runtimeStats != nil {
@@ -517,14 +566,14 @@ func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan str
start := time.Now()
handles, retChunk, err := w.extractTaskHandles(ctx, chk, handleCols)
if err != nil {
- w.syncErr(resultCh, err)
+ syncErr(ctx, finished, fetchCh, err)
return count, err
}
if len(handles) == 0 {
return count, nil
}
count += int64(len(handles))
- task := w.buildTableTask(handles, retChunk)
+ task := w.buildTableTask(handles, retChunk, parTblIdx)
if w.stats != nil {
atomic.AddInt64(&w.stats.FetchIdxTime, int64(time.Since(start)))
}
@@ -546,6 +595,8 @@ func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan str
func (w *partialTableWorker) extractTaskHandles(ctx context.Context, chk *chunk.Chunk, handleCols plannercore.HandleCols) (
handles []kv.Handle, retChk *chunk.Chunk, err error) {
handles = make([]kv.Handle, 0, w.batchSize)
+ var memUsage int64
+ defer w.memTracker.Consume(-memUsage)
for len(handles) < w.batchSize {
chk.SetRequiredRows(w.batchSize-len(handles), w.maxChunkSize)
err = errors.Trace(w.tableReader.Next(ctx, chk))
@@ -553,8 +604,14 @@ func (w *partialTableWorker) extractTaskHandles(ctx context.Context, chk *chunk.
return handles, nil, err
}
if chk.NumRows() == 0 {
+ failpoint.Inject("testIndexMergeErrorPartialTableWorker", func(v failpoint.Value) {
+ failpoint.Return(handles, nil, errors.New(v.(string)))
+ })
return handles, retChk, nil
}
+ memDelta := chk.MemoryUsage()
+ memUsage += memDelta
+ w.memTracker.Consume(memDelta)
for i := 0; i < chk.NumRows(); i++ {
handle, err := handleCols.BuildHandle(chk.GetRow(i))
if err != nil {
@@ -570,19 +627,22 @@ func (w *partialTableWorker) extractTaskHandles(ctx context.Context, chk *chunk.
return handles, retChk, nil
}
-func (w *partialTableWorker) buildTableTask(handles []kv.Handle, retChk *chunk.Chunk) *lookupTableTask {
- task := &lookupTableTask{
- handles: handles,
- idxRows: retChk,
+func (w *partialTableWorker) buildTableTask(handles []kv.Handle, retChk *chunk.Chunk, parTblIdx int) *indexMergeTableTask {
+ task := &indexMergeTableTask{
+ lookupTableTask: lookupTableTask{
+ handles: handles,
+ idxRows: retChk,
- partitionTable: w.partition,
+ partitionTable: w.partition,
+ },
+ parTblIdx: parTblIdx,
}
task.doneCh = make(chan error, 1)
return task
}
-func (e *IndexMergeReaderExecutor) startIndexMergeTableScanWorker(ctx context.Context, workCh <-chan *lookupTableTask) {
+func (e *IndexMergeReaderExecutor) startIndexMergeTableScanWorker(ctx context.Context, workCh <-chan *indexMergeTableTask) {
lookupConcurrencyLimit := e.ctx.GetSessionVars().IndexLookupConcurrency()
e.tblWorkerWg.Add(lookupConcurrencyLimit)
for i := 0; i < lookupConcurrencyLimit; i++ {
@@ -597,10 +657,15 @@ func (e *IndexMergeReaderExecutor) startIndexMergeTableScanWorker(ctx context.Co
ctx1, cancel := context.WithCancel(ctx)
go func() {
defer trace.StartRegion(ctx, "IndexMergeTableScanWorker").End()
- var task *lookupTableTask
+ var task *indexMergeTableTask
util.WithRecovery(
- func() { task = worker.pickAndExecTask(ctx1) },
- worker.handlePickAndExecTaskPanic(ctx1, task),
+ // Note we use the address of `task` as the argument of both `pickAndExecTask` and `handleTableScanWorkerPanic`
+ // because `task` is expected to be assigned in `pickAndExecTask`, and this assignment should also be visible
+ // in `handleTableScanWorkerPanic` since it will get `doneCh` from `task`. Golang always pass argument by value,
+ // so if we don't use the address of `task` as the argument, the assignment to `task` in `pickAndExecTask` is
+ // not visible in `handleTableScanWorkerPanic`
+ func() { worker.pickAndExecTask(ctx1, &task) },
+ worker.handleTableScanWorkerPanic(ctx1, e.finished, &task, tableScanWorkerType),
)
cancel()
e.tblWorkerWg.Done()
@@ -622,11 +687,6 @@ func (e *IndexMergeReaderExecutor) buildFinalTableReader(ctx context.Context, tb
plans: e.tblPlans,
netDataSize: e.dataAvgRowSize * float64(len(handles)),
}
- if e.isCorColInTableFilter {
- if tableReaderExec.dagPB.Executors, err = constructDistExec(e.ctx, e.tblPlans); err != nil {
- return nil, err
- }
- }
tableReaderExec.buildVirtualColumnInfo()
// Reorder handles because SplitKeyRangesByLocations() requires startKey of kvRanges is ordered.
// Also it's good for performance.
@@ -666,7 +726,7 @@ func (e *IndexMergeReaderExecutor) Next(ctx context.Context, req *chunk.Chunk) e
}
}
-func (e *IndexMergeReaderExecutor) getResultTask() (*lookupTableTask, error) {
+func (e *IndexMergeReaderExecutor) getResultTask() (*indexMergeTableTask, error) {
if e.resultCurr != nil && e.resultCurr.cursor < len(e.resultCurr.rows) {
return e.resultCurr, nil
}
@@ -686,31 +746,53 @@ func (e *IndexMergeReaderExecutor) getResultTask() (*lookupTableTask, error) {
return e.resultCurr, nil
}
-func (e *IndexMergeReaderExecutor) handleHandlesFetcherPanic(ctx context.Context, resultCh chan<- *lookupTableTask, worker string) func(r interface{}) {
+func handleWorkerPanic(ctx context.Context, finished <-chan struct{}, ch chan<- *indexMergeTableTask, extraNotifyCh chan bool, worker string) func(r interface{}) {
return func(r interface{}) {
+ if worker == processWorkerType {
+ // There is only one processWorker, so it's safe to close here.
+ // No need to worry about "close on closed channel" error.
+ defer close(ch)
+ }
if r == nil {
return
}
- err4Panic := errors.Errorf("panic in IndexMergeReaderExecutor %s: %v", worker, r)
+ if extraNotifyCh != nil {
+ extraNotifyCh <- true
+ }
+
+ err4Panic := errors.Errorf("%s: %v", worker, r)
logutil.Logger(ctx).Error(err4Panic.Error())
doneCh := make(chan error, 1)
doneCh <- err4Panic
- resultCh <- &lookupTableTask{
- doneCh: doneCh,
+ task := &indexMergeTableTask{
+ lookupTableTask: lookupTableTask{
+ doneCh: doneCh,
+ },
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case <-finished:
+ return
+ case ch <- task:
+ return
}
}
}
// Close implements Exec Close interface.
func (e *IndexMergeReaderExecutor) Close() error {
+ if e.stats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
+ }
if e.finished == nil {
return nil
}
close(e.finished)
- e.processWokerWg.Wait()
e.tblWorkerWg.Wait()
e.idxWorkerWg.Wait()
+ e.processWorkerWg.Wait()
e.finished = nil
e.workerStarted = false
// TODO: how to store e.feedbacks
@@ -722,19 +804,34 @@ type indexMergeProcessWorker struct {
stats *IndexMergeRuntimeStat
}
-func (w *indexMergeProcessWorker) fetchLoop(ctx context.Context, fetchCh <-chan *lookupTableTask,
- workCh chan<- *lookupTableTask, resultCh chan<- *lookupTableTask, finished <-chan struct{}) {
- defer func() {
- close(workCh)
- close(resultCh)
- }()
+func (w *indexMergeProcessWorker) fetchLoopUnion(ctx context.Context, fetchCh <-chan *indexMergeTableTask,
+ workCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask, finished <-chan struct{}) {
+ failpoint.Inject("testIndexMergeResultChCloseEarly", func(_ failpoint.Value) {
+ failpoint.Return()
+ })
+ memTracker := memory.NewTracker(w.indexMerge.id, -1)
+ memTracker.AttachTo(w.indexMerge.memTracker)
+ defer memTracker.Detach()
+ defer close(workCh)
+ failpoint.Inject("testIndexMergePanicProcessWorkerUnion", nil)
distinctHandles := make(map[int64]*kv.HandleMap)
for task := range fetchCh {
+ select {
+ case err := <-task.doneCh:
+ // If got error from partialIndexWorker/partialTableWorker, stop processing.
+ if err != nil {
+ syncErr(ctx, finished, resultCh, err)
+ return
+ }
+ default:
+ }
start := time.Now()
handles := task.handles
fhs := make([]kv.Handle, 0, 8)
+ memTracker.Consume(int64(cap(task.handles) * 8))
+
var tblID int64
if w.indexMerge.partitionTableMode {
tblID = getPhysicalTableID(task.partitionTable)
@@ -755,11 +852,13 @@ func (w *indexMergeProcessWorker) fetchLoop(ctx context.Context, fetchCh <-chan
if len(fhs) == 0 {
continue
}
- task := &lookupTableTask{
- handles: fhs,
- doneCh: make(chan error, 1),
+ task := &indexMergeTableTask{
+ lookupTableTask: lookupTableTask{
+ handles: fhs,
+ doneCh: make(chan error, 1),
- partitionTable: task.partitionTable,
+ partitionTable: task.partitionTable,
+ },
}
if w.stats != nil {
w.stats.IndexMergeProcess += time.Since(start)
@@ -775,20 +874,189 @@ func (w *indexMergeProcessWorker) fetchLoop(ctx context.Context, fetchCh <-chan
}
}
-func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, resultCh chan<- *lookupTableTask) func(r interface{}) {
- return func(r interface{}) {
- if r == nil {
+type intersectionProcessWorker struct {
+ // key: parTblIdx, val: HandleMap
+ // Value of MemAwareHandleMap is *int to avoid extra Get().
+ handleMapsPerWorker map[int]*kv.MemAwareHandleMap[*int]
+ workerID int
+ workerCh chan *indexMergeTableTask
+ indexMerge *IndexMergeReaderExecutor
+ memTracker *memory.Tracker
+ batchSize int
+
+ // When rowDelta == memConsumeBatchSize, Consume(memUsage)
+ rowDelta int64
+ mapUsageDelta int64
+}
+
+func (w *intersectionProcessWorker) consumeMemDelta() {
+ w.memTracker.Consume(w.mapUsageDelta + w.rowDelta*int64(unsafe.Sizeof(int(0))))
+ w.mapUsageDelta = 0
+ w.rowDelta = 0
+}
+
+func (w *intersectionProcessWorker) doIntersectionPerPartition(ctx context.Context, workCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask, finished <-chan struct{}) {
+ failpoint.Inject("testIndexMergePanicPartitionTableIntersectionWorker", nil)
+ defer w.memTracker.Detach()
+
+ for task := range w.workerCh {
+ var ok bool
+ var hMap *kv.MemAwareHandleMap[*int]
+ if hMap, ok = w.handleMapsPerWorker[task.parTblIdx]; !ok {
+ hMap = kv.NewMemAwareHandleMap[*int]()
+ w.handleMapsPerWorker[task.parTblIdx] = hMap
+ }
+ var mapDelta int64
+ var rowDelta int64
+ for _, h := range task.handles {
+ // Use *int to avoid Get() again.
+ if cntPtr, ok := hMap.Get(h); ok {
+ (*cntPtr)++
+ } else {
+ cnt := 1
+ mapDelta += hMap.Set(h, &cnt) + int64(h.ExtraMemSize())
+ rowDelta += 1
+ }
+ }
+
+ logutil.BgLogger().Debug("intersectionProcessWorker handle tasks", zap.Int("workerID", w.workerID),
+ zap.Int("task.handles", len(task.handles)), zap.Int64("rowDelta", rowDelta))
+
+ w.mapUsageDelta += mapDelta
+ w.rowDelta += rowDelta
+ if w.rowDelta >= int64(w.batchSize) {
+ w.consumeMemDelta()
+ }
+ failpoint.Inject("testIndexMergeIntersectionWorkerPanic", nil)
+ }
+ if w.rowDelta > 0 {
+ w.consumeMemDelta()
+ }
+
+ // We assume the result of intersection is small, so no need to track memory.
+ intersectedMap := make(map[int][]kv.Handle, len(w.handleMapsPerWorker))
+ for parTblIdx, hMap := range w.handleMapsPerWorker {
+ hMap.Range(func(h kv.Handle, val interface{}) bool {
+ if *(val.(*int)) == len(w.indexMerge.partialPlans) {
+ // Means all partial paths have this handle.
+ intersectedMap[parTblIdx] = append(intersectedMap[parTblIdx], h)
+ }
+ return true
+ })
+ }
+
+ tasks := make([]*indexMergeTableTask, 0, len(w.handleMapsPerWorker))
+ for parTblIdx, intersected := range intersectedMap {
+ // Split intersected[parTblIdx] to avoid task is too large.
+ for len(intersected) > 0 {
+ length := w.batchSize
+ if length > len(intersected) {
+ length = len(intersected)
+ }
+ task := &indexMergeTableTask{
+ lookupTableTask: lookupTableTask{
+ handles: intersected[:length],
+ doneCh: make(chan error, 1),
+ },
+ }
+ intersected = intersected[length:]
+ if w.indexMerge.partitionTableMode {
+ task.partitionTable = w.indexMerge.prunedPartitions[parTblIdx]
+ }
+ tasks = append(tasks, task)
+ logutil.BgLogger().Debug("intersectionProcessWorker build tasks",
+ zap.Int("parTblIdx", parTblIdx), zap.Int("task.handles", len(task.handles)))
+ }
+ }
+ for _, task := range tasks {
+ select {
+ case <-ctx.Done():
return
+ case <-finished:
+ return
+ case workCh <- task:
+ resultCh <- task
}
+ }
+}
- err4Panic := errors.Errorf("panic in IndexMergeReaderExecutor indexMergeTableWorker: %v", r)
- logutil.Logger(ctx).Error(err4Panic.Error())
- doneCh := make(chan error, 1)
- doneCh <- err4Panic
- resultCh <- &lookupTableTask{
- doneCh: doneCh,
+// For each partition(dynamic mode), a map is used to do intersection. Key of the map is handle, and value is the number of times it occurs.
+// If the value of handle equals the number of partial paths, it should be sent to final_table_scan_worker.
+// To avoid too many goroutines, each intersectionProcessWorker can handle multiple partitions.
+func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fetchCh <-chan *indexMergeTableTask,
+ workCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask, finished <-chan struct{}) {
+ defer close(workCh)
+
+ if w.stats != nil {
+ start := time.Now()
+ defer func() {
+ w.stats.IndexMergeProcess += time.Since(start)
+ }()
+ }
+
+ failpoint.Inject("testIndexMergePanicProcessWorkerIntersection", nil)
+
+ // One goroutine may handle one or multiple partitions.
+ // Max number of partition number is 8192, we use ExecutorConcurrency to avoid too many goroutines.
+ maxWorkerCnt := w.indexMerge.ctx.GetSessionVars().IndexMergeIntersectionConcurrency()
+ maxChannelSize := atomic.LoadInt32(&LookupTableTaskChannelSize)
+ batchSize := w.indexMerge.ctx.GetSessionVars().IndexLookupSize
+
+ partCnt := 1
+ if w.indexMerge.partitionTableMode {
+ partCnt = len(w.indexMerge.prunedPartitions)
+ }
+ workerCnt := mathutil.Min(partCnt, maxWorkerCnt)
+ failpoint.Inject("testIndexMergeIntersectionConcurrency", func(val failpoint.Value) {
+ con := val.(int)
+ if con != workerCnt {
+ panic(fmt.Sprintf("unexpected workerCnt, expect %d, got %d", con, workerCnt))
+ }
+ })
+
+ workers := make([]*intersectionProcessWorker, 0, workerCnt)
+ wg := util.WaitGroupWrapper{}
+ errCh := make(chan bool, workerCnt)
+ for i := 0; i < workerCnt; i++ {
+ tracker := memory.NewTracker(w.indexMerge.id, -1)
+ tracker.AttachTo(w.indexMerge.memTracker)
+ worker := &intersectionProcessWorker{
+ workerID: i,
+ handleMapsPerWorker: make(map[int]*kv.MemAwareHandleMap[*int]),
+ workerCh: make(chan *indexMergeTableTask, maxChannelSize),
+ indexMerge: w.indexMerge,
+ memTracker: tracker,
+ batchSize: batchSize,
+ }
+ wg.RunWithRecover(func() {
+ defer trace.StartRegion(ctx, "IndexMergeIntersectionProcessWorker").End()
+ worker.doIntersectionPerPartition(ctx, workCh, resultCh, finished)
+ }, handleWorkerPanic(ctx, finished, resultCh, errCh, partTblIntersectionWorkerType))
+ workers = append(workers, worker)
+ }
+loop:
+ for task := range fetchCh {
+ select {
+ case err := <-task.doneCh:
+ // If got error from partialIndexWorker/partialTableWorker, stop processing.
+ if err != nil {
+ syncErr(ctx, finished, resultCh, err)
+ break loop
+ }
+ default:
+ }
+
+ select {
+ case workers[task.parTblIdx%workerCnt].workerCh <- task:
+ case <-errCh:
+ // If got error from intersectionProcessWorker, stop processing.
+ break loop
}
}
+ for _, processWorker := range workers {
+ close(processWorker.workerCh)
+ }
+ wg.Wait()
}
type partialIndexWorker struct {
@@ -799,13 +1067,27 @@ type partialIndexWorker struct {
maxBatchSize int
maxChunkSize int
partition table.PhysicalTable // it indicates if this worker is accessing a particular partition table
+ memTracker *memory.Tracker
}
-func (w *partialIndexWorker) syncErr(resultCh chan<- *lookupTableTask, err error) {
+func syncErr(ctx context.Context, finished <-chan struct{}, errCh chan<- *indexMergeTableTask, err error) {
+ logutil.BgLogger().Error("IndexMergeReaderExecutor.syncErr", zap.Error(err))
doneCh := make(chan error, 1)
doneCh <- err
- resultCh <- &lookupTableTask{
- doneCh: doneCh,
+ task := &indexMergeTableTask{
+ lookupTableTask: lookupTableTask{
+ doneCh: doneCh,
+ },
+ }
+
+ // ctx.Done and finished is to avoid write channel is stuck.
+ select {
+ case <-ctx.Done():
+ return
+ case <-finished:
+ return
+ case errCh <- task:
+ return
}
}
@@ -813,23 +1095,22 @@ func (w *partialIndexWorker) fetchHandles(
ctx context.Context,
result distsql.SelectResult,
exitCh <-chan struct{},
- fetchCh chan<- *lookupTableTask,
- resultCh chan<- *lookupTableTask,
+ fetchCh chan<- *indexMergeTableTask,
finished <-chan struct{},
- handleCols plannercore.HandleCols) (count int64, err error) {
+ handleCols plannercore.HandleCols,
+ parTblIdx int) (count int64, err error) {
chk := chunk.NewChunkWithCapacity(handleCols.GetFieldsTypes(), w.maxChunkSize)
var basicStats *execdetails.BasicRuntimeStats
if w.stats != nil {
if w.idxID != 0 {
- basicStats = &execdetails.BasicRuntimeStats{}
- w.sc.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(w.idxID, basicStats)
+ basicStats = w.sc.GetSessionVars().StmtCtx.RuntimeStatsColl.GetBasicRuntimeStats(w.idxID)
}
}
for {
start := time.Now()
handles, retChunk, err := w.extractTaskHandles(ctx, chk, result, handleCols)
if err != nil {
- w.syncErr(resultCh, err)
+ syncErr(ctx, finished, fetchCh, err)
return count, err
}
if len(handles) == 0 {
@@ -839,7 +1120,7 @@ func (w *partialIndexWorker) fetchHandles(
return count, nil
}
count += int64(len(handles))
- task := w.buildTableTask(handles, retChunk)
+ task := w.buildTableTask(handles, retChunk, parTblIdx)
if w.stats != nil {
atomic.AddInt64(&w.stats.FetchIdxTime, int64(time.Since(start)))
}
@@ -861,6 +1142,8 @@ func (w *partialIndexWorker) fetchHandles(
func (w *partialIndexWorker) extractTaskHandles(ctx context.Context, chk *chunk.Chunk, idxResult distsql.SelectResult, handleCols plannercore.HandleCols) (
handles []kv.Handle, retChk *chunk.Chunk, err error) {
handles = make([]kv.Handle, 0, w.batchSize)
+ var memUsage int64
+ defer w.memTracker.Consume(-memUsage)
for len(handles) < w.batchSize {
chk.SetRequiredRows(w.batchSize-len(handles), w.maxChunkSize)
err = errors.Trace(idxResult.Next(ctx, chk))
@@ -868,8 +1151,14 @@ func (w *partialIndexWorker) extractTaskHandles(ctx context.Context, chk *chunk.
return handles, nil, err
}
if chk.NumRows() == 0 {
+ failpoint.Inject("testIndexMergeErrorPartialIndexWorker", func(v failpoint.Value) {
+ failpoint.Return(handles, nil, errors.New(v.(string)))
+ })
return handles, retChk, nil
}
+ memDelta := chk.MemoryUsage()
+ memUsage += memDelta
+ w.memTracker.Consume(memDelta)
for i := 0; i < chk.NumRows(); i++ {
handle, err := handleCols.BuildHandleFromIndexRow(chk.GetRow(i))
if err != nil {
@@ -885,12 +1174,15 @@ func (w *partialIndexWorker) extractTaskHandles(ctx context.Context, chk *chunk.
return handles, retChk, nil
}
-func (w *partialIndexWorker) buildTableTask(handles []kv.Handle, retChk *chunk.Chunk) *lookupTableTask {
- task := &lookupTableTask{
- handles: handles,
- idxRows: retChk,
+func (w *partialIndexWorker) buildTableTask(handles []kv.Handle, retChk *chunk.Chunk, parTblIdx int) *indexMergeTableTask {
+ task := &indexMergeTableTask{
+ lookupTableTask: lookupTableTask{
+ handles: handles,
+ idxRows: retChk,
- partitionTable: w.partition,
+ partitionTable: w.partition,
+ },
+ parTblIdx: parTblIdx,
}
task.doneCh = make(chan error, 1)
@@ -899,7 +1191,7 @@ func (w *partialIndexWorker) buildTableTask(handles []kv.Handle, retChk *chunk.C
type indexMergeTableScanWorker struct {
stats *IndexMergeRuntimeStat
- workCh <-chan *lookupTableTask
+ workCh <-chan *indexMergeTableTask
finished <-chan struct{}
indexMergeExec *IndexMergeReaderExecutor
tblPlans []plannercore.PhysicalPlan
@@ -908,42 +1200,63 @@ type indexMergeTableScanWorker struct {
memTracker *memory.Tracker
}
-func (w *indexMergeTableScanWorker) pickAndExecTask(ctx context.Context) (task *lookupTableTask) {
+func (w *indexMergeTableScanWorker) pickAndExecTask(ctx context.Context, task **indexMergeTableTask) {
var ok bool
for {
waitStart := time.Now()
select {
- case task, ok = <-w.workCh:
+ case *task, ok = <-w.workCh:
if !ok {
return
}
case <-w.finished:
return
}
+ // Make sure panic failpoint is after fetch task from workCh.
+ // Otherwise cannot send error to task.doneCh.
+ failpoint.Inject("testIndexMergePanicTableScanWorker", nil)
+ failpoint.Inject("mockSleepBeforeStartTableReader", func(_ failpoint.Value) {
+ select {
+ case <-ctx.Done():
+ failpoint.Return()
+ case <-w.finished:
+ failpoint.Return()
+ }
+ })
execStart := time.Now()
- err := w.executeTask(ctx, task)
+ err := w.executeTask(ctx, *task)
if w.stats != nil {
atomic.AddInt64(&w.stats.WaitTime, int64(execStart.Sub(waitStart)))
atomic.AddInt64(&w.stats.FetchRow, int64(time.Since(execStart)))
atomic.AddInt64(&w.stats.TableTaskNum, 1)
}
- task.doneCh <- err
+ failpoint.Inject("testIndexMergePickAndExecTaskPanic", nil)
+ (*task).doneCh <- err
}
}
-func (w *indexMergeTableScanWorker) handlePickAndExecTaskPanic(ctx context.Context, task *lookupTableTask) func(r interface{}) {
+func (w *indexMergeTableScanWorker) handleTableScanWorkerPanic(ctx context.Context, finished <-chan struct{}, task **indexMergeTableTask, worker string) func(r interface{}) {
return func(r interface{}) {
if r == nil {
return
}
- err4Panic := errors.Errorf("panic in IndexMergeReaderExecutor indexMergeTableWorker: %v", r)
+ err4Panic := errors.Errorf("%s: %v", worker, r)
logutil.Logger(ctx).Error(err4Panic.Error())
- task.doneCh <- err4Panic
+ if *task != nil {
+ select {
+ case <-ctx.Done():
+ return
+ case <-finished:
+ return
+ case (*task).doneCh <- err4Panic:
+ return
+ }
+ }
}
}
-func (w *indexMergeTableScanWorker) executeTask(ctx context.Context, task *lookupTableTask) error {
+func (w *indexMergeTableScanWorker) executeTask(ctx context.Context, task *indexMergeTableTask) error {
tbl := w.indexMergeExec.table
if w.indexMergeExec.partitionTableMode {
tbl = task.partitionTable
diff --git a/executor/index_merge_reader_test.go b/executor/index_merge_reader_test.go
index 58dfa71814f28..be1ff66a163ab 100644
--- a/executor/index_merge_reader_test.go
+++ b/executor/index_merge_reader_test.go
@@ -23,7 +23,9 @@ import (
"testing"
"time"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/testkit"
+ "github.com/pingcap/tidb/testkit/testutil"
"github.com/pingcap/tidb/util"
"github.com/stretchr/testify/require"
)
@@ -49,6 +51,25 @@ func TestSingleTableRead(t *testing.T) {
tk.MustQuery("select /*+ use_index_merge(t1, t1a, t1b) */ sum(a) from t1 where a < 2 or b > 4").Check(testkit.Rows("6"))
}
+func TestIndexMergePickAndExecTaskPanic(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1, t2")
+ tk.MustExec("create table t1(id int primary key, a int, b int, c int, d int)")
+ tk.MustExec("create index t1a on t1(a)")
+ tk.MustExec("create index t1b on t1(b)")
+ tk.MustExec("insert into t1 values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5)")
+ tk.MustQuery("select /*+ use_index_merge(t1, primary, t1a) */ * from t1 where id < 2 or a > 4 order by id").Check(testkit.Rows("1 1 1 1 1",
+ "5 5 5 5 5"))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergePickAndExecTaskPanic", "panic(\"pickAndExecTaskPanic\")"))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/testIndexMergePickAndExecTaskPanic"))
+ }()
+ err := tk.QueryToErr("select /*+ use_index_merge(t1, primary, t1a) */ * from t1 where id < 2 or a > 4 order by id")
+ require.Contains(t, err.Error(), "pickAndExecTaskPanic")
+}
+
func TestJoin(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -86,7 +107,7 @@ func TestIndexMergeReaderIssue25045(t *testing.T) {
tk.MustExec("create table t1(a int primary key, b int, c int, key(b), key(c));")
tk.MustExec("INSERT INTO t1 VALUES (10, 10, 10), (11, 11, 11)")
tk.MustQuery("explain format='brief' select /*+ use_index_merge(t1) */ * from t1 where c=10 or (b=10 and a=10);").Check(testkit.Rows(
- "IndexMerge 0.01 root ",
+ "IndexMerge 0.01 root type: union",
"├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c(c) range:[10,10], keep order:false, stats:pseudo",
"├─TableRangeScan(Build) 1.00 cop[tikv] table:t1 range:[10,10], keep order:false, stats:pseudo",
"└─Selection(Probe) 0.01 cop[tikv] or(eq(test.t1.c, 10), and(eq(test.t1.b, 10), eq(test.t1.a, 10)))",
@@ -230,44 +251,64 @@ func TestIndexMergeInTransaction(t *testing.T) {
tk.MustExec("begin;")
// Expect two IndexScan(c1, c2).
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and c3 < 10;").Check(testkit.Rows(
- "IndexMerge_9 1841.86 root ",
+ "IndexMerge_9 1841.86 root type: union",
"├─IndexRangeScan_5(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo",
"├─IndexRangeScan_6(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo",
"└─Selection_8(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)",
" └─TableRowIDScan_7 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo"))
// Expect one IndexScan(c2) and one TableScan(pk).
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where (pk < 10 or c2 < 10) and c3 < 10;").Check(testkit.Rows(
- "IndexMerge_9 1106.67 root ",
+ "IndexMerge_9 1106.67 root type: union",
"├─TableRangeScan_5(Build) 3333.33 cop[tikv] table:t1 range:[-inf,10), keep order:false, stats:pseudo",
"├─IndexRangeScan_6(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo",
"└─Selection_8(Probe) 1106.67 cop[tikv] lt(test.t1.c3, 10)",
" └─TableRowIDScan_7 3330.01 cop[tikv] table:t1 keep order:false, stats:pseudo"))
+ tk.MustQuery("explain select /*+ use_index_merge(t1, c1, c2, c3) */ * from t1 where c1 < 10 and c2 < 10 and c3 < 10;").Check(testkit.Rows(
+ "IndexMerge_9 367.05 root type: intersection",
+ "├─IndexRangeScan_5(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo",
+ "├─IndexRangeScan_6(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo",
+ "├─IndexRangeScan_7(Build) 3323.33 cop[tikv] table:t1, index:c3(c3) range:[-inf,10), keep order:false, stats:pseudo",
+ "└─TableRowIDScan_8(Probe) 367.05 cop[tikv] table:t1 keep order:false, stats:pseudo"))
// Test with normal key.
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < -1) and c3 < 10;").Check(testkit.Rows())
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (c1 < -1 or c2 < 10) and c3 < 10;").Check(testkit.Rows())
+ tk.MustQuery("select /*+ use_index_merge(t1, c1, c2, c3) */ * from t1 where (c1 < 10 and c2 < -1) and c3 < 10;").Check(testkit.Rows())
+ tk.MustQuery("select /*+ use_index_merge(t1, c1, c2, c3) */ * from t1 where (c1 < -1 and c2 < 10) and c3 < 10;").Check(testkit.Rows())
+
tk.MustExec("insert into t1 values(1, 1, 1, 1);")
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < -1) and c3 < 10;").Check(testkit.Rows("1 1 1 1"))
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (c1 < -1 or c2 < 10) and c3 < 10;").Check(testkit.Rows("1 1 1 1"))
+ tk.MustQuery("select /*+ use_index_merge(t1, c1, c2, c3) */ * from t1 where (c1 < 10 and c2 < 10) and c3 < 10;").Check(testkit.Rows("1 1 1 1"))
+ tk.MustQuery("select /*+ use_index_merge(t1, c1, c2, c3) */ * from t1 where (c1 < 10 and c2 < 10) and c3 > 10;").Check(testkit.Rows())
+
tk.MustExec("update t1 set c3 = 100 where c3 = 1;")
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < -1) and c3 < 10;").Check(testkit.Rows())
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (c1 < -1 or c2 < 10) and c3 < 10;").Check(testkit.Rows())
+ tk.MustQuery("select /*+ use_index_merge(t1, c1, c2, c3) */ * from t1 where (c1 < 10 and c2 < 10) and c3 > 10;").Check(testkit.Rows("1 1 100 1"))
+
tk.MustExec("delete from t1;")
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < -1) and c3 < 10;").Check(testkit.Rows())
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (c1 < -1 or c2 < 10) and c3 < 10;").Check(testkit.Rows())
+ tk.MustQuery("select /*+ use_index_merge(t1, c1, c2, c3) */ * from t1 where (c1 < 10 and c2 < 10) and c3 > 10;").Check(testkit.Rows())
// Test with primary key, so the partialPlan is TableScan.
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (pk < -1 or c2 < 10) and c3 < 10;").Check(testkit.Rows())
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (pk < 10 or c2 < -1) and c3 < 10;").Check(testkit.Rows())
+ tk.MustQuery("select /*+ use_index_merge(t1, c2, c3, primary) */ * from t1 where (pk < -1 and c2 < 10) and c3 < 10;").Check(testkit.Rows())
+ tk.MustQuery("select /*+ use_index_merge(t1, c2, c3, primary) */ * from t1 where (pk < 10 and c2 < -1) and c3 < 10;").Check(testkit.Rows())
tk.MustExec("insert into t1 values(1, 1, 1, 1);")
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (pk < -1 or c2 < 10) and c3 < 10;").Check(testkit.Rows("1 1 1 1"))
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (pk < 10 or c2 < -1) and c3 < 10;").Check(testkit.Rows("1 1 1 1"))
+ tk.MustQuery("select /*+ use_index_merge(t1, c2, c3, primary) */ * from t1 where (pk < 10 and c2 < 10) and c3 < 10;").Check(testkit.Rows("1 1 1 1"))
tk.MustExec("update t1 set c3 = 100 where c3 = 1;")
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (pk < -1 or c2 < 10) and c3 < 10;").Check(testkit.Rows())
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (pk < 10 or c2 < -1) and c3 < 10;").Check(testkit.Rows())
+ tk.MustQuery("select /*+ use_index_merge(t1, c2, c3, primary) */ * from t1 where (pk < 10 and c2 < 10) and c3 > 10;").Check(testkit.Rows("1 1 100 1"))
tk.MustExec("delete from t1;")
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (pk < -1 or c2 < 10) and c3 < 10;").Check(testkit.Rows())
tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where (pk < 10 or c2 < -1) and c3 < 10;").Check(testkit.Rows())
+ tk.MustQuery("select /*+ use_index_merge(t1, c2, c3, primary) */ * from t1 where (pk < 10 and c2 < 10) and c3 > 10;").Check(testkit.Rows())
tk.MustExec("commit;")
if i == 1 {
@@ -281,14 +322,14 @@ func TestIndexMergeInTransaction(t *testing.T) {
tk.MustExec("begin;")
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and c3 < 10 for update;").Check(testkit.Rows(
"SelectLock_6 1841.86 root for update 0",
- "└─IndexMerge_11 1841.86 root ",
+ "└─IndexMerge_11 1841.86 root type: union",
" ├─IndexRangeScan_7(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo",
" ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo",
" └─Selection_10(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)",
" └─TableRowIDScan_9 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo"))
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where (pk < 10 or c2 < 10) and c3 < 10 for update;").Check(testkit.Rows(
"SelectLock_6 1106.67 root for update 0",
- "└─IndexMerge_11 1106.67 root ",
+ "└─IndexMerge_11 1106.67 root type: union",
" ├─TableRangeScan_7(Build) 3333.33 cop[tikv] table:t1 range:[-inf,10), keep order:false, stats:pseudo",
" ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo",
" └─Selection_10(Probe) 1106.67 cop[tikv] lt(test.t1.c3, 10)",
@@ -403,7 +444,7 @@ func TestIndexMergeReaderInTransIssue30685(t *testing.T) {
tk.MustExec("insert into t1 values(1, 1, 1, 1);")
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < -1 or c3 < 10) and c4 < 10;").Check(testkit.Rows(
"UnionScan_6 1841.86 root lt(test.t1.c4, 10), or(lt(test.t1.c1, -1), lt(test.t1.c3, 10))",
- "└─IndexMerge_11 1841.86 root ",
+ "└─IndexMerge_11 1841.86 root type: union",
" ├─TableRangeScan_7(Build) 3323.33 cop[tikv] table:t1 range:[-inf,-1), keep order:false, stats:pseudo",
" ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c3(c3) range:[-inf,10), keep order:false, stats:pseudo",
" └─Selection_10(Probe) 1841.86 cop[tikv] lt(test.t1.c4, 10)",
@@ -422,7 +463,7 @@ func TestIndexMergeReaderInTransIssue30685(t *testing.T) {
tk.MustExec("insert into t1 values('b', 1, 1, 1);")
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 'a' or c3 < 10) and c4 < 10;").Check(testkit.Rows(
"UnionScan_6 1841.86 root lt(test.t1.c4, 10), or(lt(test.t1.c1, \"a\"), lt(test.t1.c3, 10))",
- "└─IndexMerge_11 1841.86 root ",
+ "└─IndexMerge_11 1841.86 root type: union",
" ├─TableRangeScan_7(Build) 3323.33 cop[tikv] table:t1 range:[-inf,\"a\"), keep order:false, stats:pseudo",
" ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c3(c3) range:[-inf,10), keep order:false, stats:pseudo",
" └─Selection_10(Probe) 1841.86 cop[tikv] lt(test.t1.c4, 10)",
@@ -524,19 +565,19 @@ func TestPessimisticLockOnPartitionForIndexMerge(t *testing.T) {
" ├─IndexReader(Build) 3.00 root index:IndexFullScan",
" │ └─IndexFullScan 3.00 cop[tikv] table:t2, index:c_datetime(c_datetime) keep order:false",
" └─PartitionUnion(Probe) 5545.21 root ",
- " ├─IndexMerge 5542.21 root ",
+ " ├─IndexMerge 5542.21 root type: union",
" │ ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, partition:p0, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo",
" │ ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, partition:p0, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo",
" │ └─TableRowIDScan(Probe) 5542.21 cop[tikv] table:t1, partition:p0 keep order:false, stats:pseudo",
- " ├─IndexMerge 1.00 root ",
+ " ├─IndexMerge 1.00 root type: union",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p1, index:c1(c1) range:[-inf,10), keep order:false",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p1, index:c2(c2) range:[-inf,10), keep order:false",
" │ └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p1 keep order:false",
- " ├─IndexMerge 1.00 root ",
+ " ├─IndexMerge 1.00 root type: union",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p2, index:c1(c1) range:[-inf,10), keep order:false",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p2, index:c2(c2) range:[-inf,10), keep order:false",
" │ └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p2 keep order:false",
- " └─IndexMerge 1.00 root ",
+ " └─IndexMerge 1.00 root type: union",
" ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p3, index:c1(c1) range:[-inf,10), keep order:false",
" ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p3, index:c2(c2) range:[-inf,10), keep order:false",
" └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p3 keep order:false",
@@ -566,3 +607,277 @@ func TestPessimisticLockOnPartitionForIndexMerge(t *testing.T) {
// TODO: add support for index merge reader in dynamic tidb_partition_prune_mode
}
+
+func TestIndexMergeIntersectionConcurrency(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1")
+ tk.MustExec("create table t1(c1 int, c2 bigint, c3 bigint, primary key(c1), key(c2), key(c3)) partition by hash(c1) partitions 10;")
+ tk.MustExec("insert into t1 values(1, 1, 3000), (2, 1, 1)")
+ tk.MustExec("analyze table t1;")
+ tk.MustExec("set tidb_partition_prune_mode = 'dynamic'")
+ res := tk.MustQuery("explain select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Rows()
+ require.Contains(t, res[1][0], "IndexMerge")
+
+ // Default is tidb_executor_concurrency.
+ res = tk.MustQuery("select @@tidb_executor_concurrency;").Sort().Rows()
+ defExecCon := res[0][0].(string)
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionConcurrency", fmt.Sprintf("return(%s)", defExecCon)))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionConcurrency"))
+ }()
+ tk.MustQuery("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Check(testkit.Rows("1"))
+
+ tk.MustExec("set tidb_executor_concurrency = 10")
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionConcurrency", "return(10)"))
+ tk.MustQuery("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Check(testkit.Rows("1"))
+ // workerCnt = min(part_num, concurrency)
+ tk.MustExec("set tidb_executor_concurrency = 20")
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionConcurrency", "return(10)"))
+ tk.MustQuery("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Check(testkit.Rows("1"))
+ tk.MustExec("set tidb_executor_concurrency = 2")
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionConcurrency", "return(2)"))
+ tk.MustQuery("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Check(testkit.Rows("1"))
+
+ tk.MustExec("set tidb_index_merge_intersection_concurrency = 9")
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionConcurrency", "return(9)"))
+ tk.MustQuery("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Check(testkit.Rows("1"))
+ tk.MustExec("set tidb_index_merge_intersection_concurrency = 21")
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionConcurrency", "return(10)"))
+ tk.MustQuery("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Check(testkit.Rows("1"))
+ tk.MustExec("set tidb_index_merge_intersection_concurrency = 3")
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionConcurrency", "return(3)"))
+ tk.MustQuery("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Check(testkit.Rows("1"))
+
+ // Concurrency only works for dynamic pruning partition table, so real concurrency is 1.
+ tk.MustExec("set tidb_partition_prune_mode = 'static'")
+ tk.MustExec("set tidb_index_merge_intersection_concurrency = 9")
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionConcurrency", "return(1)"))
+ tk.MustQuery("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Check(testkit.Rows("1"))
+
+ // Concurrency only works for dynamic pruning partition table. so real concurrency is 1.
+ tk.MustExec("drop table if exists t1")
+ tk.MustExec("create table t1(c1 int, c2 bigint, c3 bigint, primary key(c1), key(c2), key(c3));")
+ tk.MustExec("insert into t1 values(1, 1, 3000), (2, 1, 1)")
+ tk.MustExec("set tidb_index_merge_intersection_concurrency = 9")
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionConcurrency", "return(1)"))
+ tk.MustQuery("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Check(testkit.Rows("1"))
+}
+
+func TestIntersectionWithDifferentConcurrency(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ var execCon []int
+ tblSchemas := []string{
+ // partition table
+ "create table t1(c1 int, c2 bigint, c3 bigint, primary key(c1), key(c2), key(c3)) partition by hash(c1) partitions 10;",
+ // non-partition table
+ "create table t1(c1 int, c2 bigint, c3 bigint, primary key(c1), key(c2), key(c3));",
+ }
+
+ for tblIdx, tblSchema := range tblSchemas {
+ if tblIdx == 0 {
+ // Test different intersectionProcessWorker with partition table(10 partitions).
+ execCon = []int{1, 3, 10, 11, 20}
+ } else {
+ // Default concurrency.
+ execCon = []int{5}
+ }
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1;")
+ tk.MustExec(tblSchema)
+
+ const queryCnt int = 10
+ const rowCnt int = 1000
+ curRowCnt := 0
+ insertStr := "insert into t1 values"
+ for i := 0; i < rowCnt; i++ {
+ if i != 0 {
+ insertStr += ", "
+ }
+ insertStr += fmt.Sprintf("(%d, %d, %d)", i, rand.Int(), rand.Int())
+ curRowCnt++
+ }
+ tk.MustExec(insertStr)
+ tk.MustExec("analyze table t1")
+
+ for _, concurrency := range execCon {
+ tk.MustExec(fmt.Sprintf("set tidb_executor_concurrency = %d", concurrency))
+ for i := 0; i < 2; i++ {
+ if i == 0 {
+ // Dynamic mode.
+ tk.MustExec("set tidb_partition_prune_mode = 'dynamic'")
+ res := tk.MustQuery("explain select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024")
+ require.Contains(t, res.Rows()[1][0], "IndexMerge")
+ } else {
+ tk.MustExec("set tidb_partition_prune_mode = 'static'")
+ res := tk.MustQuery("explain select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024")
+ if tblIdx == 0 {
+ // partition table
+ require.Contains(t, res.Rows()[1][0], "PartitionUnion")
+ require.Contains(t, res.Rows()[2][0], "IndexMerge")
+ } else {
+ require.Contains(t, res.Rows()[1][0], "IndexMerge")
+ }
+ }
+ for i := 0; i < queryCnt; i++ {
+ c3 := rand.Intn(1024)
+ res := tk.MustQuery(fmt.Sprintf("select /*+ no_index_merge() */ c1 from t1 where c2 < 1024 and c3 > %d", c3)).Sort().Rows()
+ tk.MustQuery(fmt.Sprintf("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > %d", c3)).Sort().Check(res)
+ }
+
+ // In tranaction
+ for i := 0; i < queryCnt; i++ {
+ tk.MustExec("begin;")
+ r := rand.Intn(3)
+ if r == 0 {
+ tk.MustExec(fmt.Sprintf("update t1 set c3 = %d where c1 = %d", rand.Int(), rand.Intn(rowCnt)))
+ } else if r == 1 {
+ tk.MustExec(fmt.Sprintf("delete from t1 where c1 = %d", rand.Intn(rowCnt)))
+ } else if r == 2 {
+ tk.MustExec(fmt.Sprintf("insert into t1 values(%d, %d, %d)", curRowCnt, rand.Int(), rand.Int()))
+ curRowCnt++
+ }
+ c3 := rand.Intn(1024)
+ res := tk.MustQuery(fmt.Sprintf("select /*+ no_index_merge() */ c1 from t1 where c2 < 1024 and c3 > %d", c3)).Sort().Rows()
+ tk.MustQuery(fmt.Sprintf("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > %d", c3)).Sort().Check(res)
+ tk.MustExec("commit;")
+ }
+ }
+ }
+ tk.MustExec("drop table t1")
+ }
+}
+
+func TestIntersectionWorkerPanic(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1")
+ tk.MustExec("create table t1(c1 int, c2 bigint, c3 bigint, primary key(c1), key(c2), key(c3)) partition by hash(c1) partitions 10;")
+ tk.MustExec("insert into t1 values(1, 1, 3000), (2, 1, 1)")
+ tk.MustExec("analyze table t1;")
+ tk.MustExec("set tidb_partition_prune_mode = 'dynamic'")
+ res := tk.MustQuery("explain select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024").Rows()
+ require.Contains(t, res[1][0], "IndexMerge")
+
+ // Test panic in intersection.
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionWorkerPanic", `panic("testIndexMergeIntersectionWorkerPanic")`))
+ err := tk.QueryToErr("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024")
+ require.Contains(t, err.Error(), "testIndexMergeIntersectionWorkerPanic")
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/testIndexMergeIntersectionWorkerPanic"))
+}
+
+func TestIntersectionMemQuota(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1")
+ tk.MustExec("create table t1(pk varchar(100) primary key, c1 int, c2 int, index idx1(c1), index idx2(c2))")
+
+ insertStr := "insert into t1 values"
+ for i := 0; i < 20; i++ {
+ if i != 0 {
+ insertStr += ", "
+ }
+ insertStr += fmt.Sprintf("('%s', %d, %d)", testutil.RandStringRunes(100), 1, 1)
+ }
+ tk.MustExec(insertStr)
+ res := tk.MustQuery("explain select /*+ use_index_merge(t1, primary, idx1, idx2) */ c1 from t1 where c1 < 1024 and c2 < 1024").Rows()
+ require.Contains(t, res[1][0], "IndexMerge")
+
+ tk.MustExec("set global tidb_mem_oom_action='CANCEL'")
+ defer tk.MustExec("set global tidb_mem_oom_action = DEFAULT")
+ tk.MustExec("set @@tidb_mem_quota_query = 4000")
+ err := tk.QueryToErr("select /*+ use_index_merge(t1, primary, idx1, idx2) */ c1 from t1 where c1 < 1024 and c2 < 1024")
+ require.Contains(t, err.Error(), "Out Of Memory Quota!")
+}
+
+func TestIndexMergePanic(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1")
+ tk.MustExec("create table t1(c1 int, c2 bigint, c3 bigint, primary key(c1), key(c2), key(c3));")
+ tk.MustExec("insert into t1 values(1, 1, 1), (100, 100, 100)")
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/testIndexMergeResultChCloseEarly", "return(true)"))
+ tk.MustExec("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c1 < 100 or c2 < 100")
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/testIndexMergeResultChCloseEarly"))
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1")
+ tk.MustExec("create table t1(c1 int, c2 bigint, c3 bigint, primary key(c1), key(c2), key(c3)) partition by hash(c1) partitions 10;")
+ insertStr := "insert into t1 values(0, 0, 0)"
+ for i := 1; i < 1000; i++ {
+ insertStr += fmt.Sprintf(", (%d, %d, %d)", i, i, i)
+ }
+ tk.MustExec(insertStr)
+ tk.MustExec("analyze table t1;")
+ tk.MustExec("set tidb_partition_prune_mode = 'dynamic'")
+
+ minV := 200
+ maxV := 1000
+ runSQL := func(fp string) {
+ var sql string
+ v1 := rand.Intn(maxV-minV) + minV
+ v2 := rand.Intn(maxV-minV) + minV
+ if !strings.Contains(fp, "Intersection") {
+ sql = fmt.Sprintf("select /*+ use_index_merge(t1) */ c1 from t1 where c1 < %d or c2 < %d;", v1, v2)
+ } else {
+ sql = fmt.Sprintf("select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c3 < %d and c2 < %d", v1, v2)
+ }
+ res := tk.MustQuery("explain " + sql).Rows()
+ require.Contains(t, res[1][0], "IndexMerge")
+ err := tk.QueryToErr(sql)
+ require.Contains(t, err.Error(), fp)
+ }
+
+ packagePath := "github.com/pingcap/tidb/executor/"
+ panicFPPaths := []string{
+ packagePath + "testIndexMergePanicPartialIndexWorker",
+ packagePath + "testIndexMergePanicPartialTableWorker",
+
+ packagePath + "testIndexMergePanicProcessWorkerUnion",
+ packagePath + "testIndexMergePanicProcessWorkerIntersection",
+ packagePath + "testIndexMergePanicPartitionTableIntersectionWorker",
+
+ packagePath + "testIndexMergePanicTableScanWorker",
+ }
+ for _, fp := range panicFPPaths {
+ fmt.Println("handling failpoint: ", fp)
+ if !strings.Contains(fp, "testIndexMergePanicTableScanWorker") {
+ // When mockSleepBeforeStartTableReader is enabled, will not read real data. This is to avoid leaking goroutines in coprocessor.
+ // But should disable mockSleepBeforeStartTableReader for testIndexMergePanicTableScanWorker.
+ // Because finalTableScanWorker need task.doneCh to pass error, so need partialIndexWorker/partialTableWorker runs normally.
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/mockSleepBeforeStartTableReader", "return(1000)"))
+ }
+ for i := 0; i < 1000; i++ {
+ require.NoError(t, failpoint.Enable(fp, fmt.Sprintf(`panic("%s")`, fp)))
+ runSQL(fp)
+ require.NoError(t, failpoint.Disable(fp))
+ }
+ if !strings.Contains(fp, "testIndexMergePanicTableScanWorker") {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/mockSleepBeforeStartTableReader"))
+ }
+ }
+
+ errFPPaths := []string{
+ packagePath + "testIndexMergeErrorPartialIndexWorker",
+ packagePath + "testIndexMergeErrorPartialTableWorker",
+ }
+ for _, fp := range errFPPaths {
+ fmt.Println("handling failpoint: ", fp)
+ require.NoError(t, failpoint.Enable(fp, fmt.Sprintf(`return("%s")`, fp)))
+ for i := 0; i < 100; i++ {
+ runSQL(fp)
+ }
+ require.NoError(t, failpoint.Disable(fp))
+ }
+}
diff --git a/executor/infoschema_cluster_table_test.go b/executor/infoschema_cluster_table_test.go
index 012cd98178443..b1a6d4c57f4f8 100644
--- a/executor/infoschema_cluster_table_test.go
+++ b/executor/infoschema_cluster_table_test.go
@@ -290,7 +290,7 @@ func TestTableStorageStats(t *testing.T) {
"test 2",
))
rows := tk.MustQuery("select TABLE_NAME from information_schema.TABLE_STORAGE_STATS where TABLE_SCHEMA = 'mysql';").Rows()
- result := 40
+ result := 46
require.Len(t, rows, result)
// More tests about the privileges.
diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go
index b3b881c1f65fa..44e36d2a26ae1 100644
--- a/executor/infoschema_reader.go
+++ b/executor/infoschema_reader.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/deadlock"
+ rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
"github.com/pingcap/tidb/ddl/label"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/domain/infosync"
@@ -69,7 +70,6 @@ import (
"github.com/pingcap/tidb/util/servermemorylimit"
"github.com/pingcap/tidb/util/set"
"github.com/pingcap/tidb/util/sqlexec"
- "github.com/pingcap/tidb/util/stmtsummary"
"github.com/pingcap/tidb/util/stringutil"
"github.com/tikv/client-go/v2/txnkv/txnlock"
"go.uber.org/zap"
@@ -157,9 +157,6 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
e.dataForTableTiFlashReplica(sctx, dbs)
case infoschema.TableTiKVStoreStatus:
err = e.dataForTiKVStoreStatus(sctx)
- case infoschema.TableStatementsSummaryEvicted,
- infoschema.ClusterTableStatementsSummaryEvicted:
- err = e.setDataForStatementsSummaryEvicted(sctx)
case infoschema.TableClientErrorsSummaryGlobal,
infoschema.TableClientErrorsSummaryByUser,
infoschema.TableClientErrorsSummaryByHost:
@@ -184,6 +181,8 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
err = e.setDataForMemoryUsageOpsHistory(sctx)
case infoschema.ClusterTableMemoryUsageOpsHistory:
err = e.setDataForClusterMemoryUsageOpsHistory(sctx)
+ case infoschema.TableResourceGroups:
+ err = e.setDataFromResourceGroups()
}
if err != nil {
return nil, err
@@ -386,7 +385,7 @@ func getAutoIncrementID(ctx sessionctx.Context, schema *model.DBInfo, tblInfo *m
if err != nil {
return 0, err
}
- return tbl.Allocators(ctx).Get(autoid.RowIDAllocType).Base() + 1, nil
+ return tbl.Allocators(ctx).Get(autoid.AutoIncrementType).Base() + 1, nil
}
func hasPriv(ctx sessionctx.Context, priv mysql.PrivilegeType) bool {
@@ -1121,17 +1120,23 @@ func (e *memtableRetriever) setDataFromPartitions(ctx context.Context, sctx sess
partitionMethod := table.Partition.Type.String()
partitionExpr := table.Partition.Expr
- if table.Partition.Type == model.PartitionTypeRange && len(table.Partition.Columns) > 0 {
- partitionMethod = "RANGE COLUMNS"
- partitionExpr = table.Partition.Columns[0].String()
- } else if table.Partition.Type == model.PartitionTypeList && len(table.Partition.Columns) > 0 {
- partitionMethod = "LIST COLUMNS"
+ if len(table.Partition.Columns) > 0 {
+ switch table.Partition.Type {
+ case model.PartitionTypeRange:
+ partitionMethod = "RANGE COLUMNS"
+ case model.PartitionTypeList:
+ partitionMethod = "LIST COLUMNS"
+ default:
+ return fmt.Errorf("Inconsistent partition type, have type %v, but with COLUMNS > 0 (%d)", table.Partition.Type, len(table.Partition.Columns))
+ }
buf := bytes.NewBuffer(nil)
for i, col := range table.Partition.Columns {
if i > 0 {
buf.WriteString(",")
}
+ buf.WriteString("`")
buf.WriteString(col.String())
+ buf.WriteString("`")
}
partitionExpr = buf.String()
}
@@ -1677,11 +1682,11 @@ func keyColumnUsageInTable(schema *model.DBInfo, table *model.TableInfo) [][]typ
}
}
for _, fk := range table.ForeignKeys {
- fkRefCol := ""
- if len(fk.RefCols) > 0 {
- fkRefCol = fk.RefCols[0].O
- }
for i, key := range fk.Cols {
+ fkRefCol := ""
+ if len(fk.RefCols) > i {
+ fkRefCol = fk.RefCols[i].O
+ }
col := nameToCol[key.L]
record := types.MakeDatums(
infoschema.CatalogVal, // CONSTRAINT_CATALOG
@@ -2303,22 +2308,6 @@ func (e *memtableRetriever) dataForTableTiFlashReplica(ctx sessionctx.Context, s
e.rows = rows
}
-func (e *memtableRetriever) setDataForStatementsSummaryEvicted(ctx sessionctx.Context) error {
- if !hasPriv(ctx, mysql.ProcessPriv) {
- return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS")
- }
- e.rows = stmtsummary.StmtSummaryByDigestMap.ToEvictedCountDatum()
- switch e.table.Name.O {
- case infoschema.ClusterTableStatementsSummaryEvicted:
- rows, err := infoschema.AppendHostInfoToRows(ctx, e.rows)
- if err != nil {
- return err
- }
- e.rows = rows
- }
- return nil
-}
-
func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context, tableName string) error {
// Seeing client errors should require the PROCESS privilege, with the exception of errors for your own user.
// This is similar to information_schema.processlist, which is the closest comparison.
@@ -2473,50 +2462,6 @@ func (e *memtableRetriever) setDataForClusterMemoryUsageOpsHistory(ctx sessionct
return nil
}
-type stmtSummaryTableRetriever struct {
- dummyCloser
- table *model.TableInfo
- columns []*model.ColumnInfo
- retrieved bool
- extractor *plannercore.StatementsSummaryExtractor
-}
-
-// retrieve implements the infoschemaRetriever interface
-func (e *stmtSummaryTableRetriever) retrieve(ctx context.Context, sctx sessionctx.Context) ([][]types.Datum, error) {
- if e.extractor.SkipRequest || e.retrieved {
- return nil, nil
- }
- e.retrieved = true
-
- var err error
- var instanceAddr string
- switch e.table.Name.O {
- case infoschema.ClusterTableStatementsSummary,
- infoschema.ClusterTableStatementsSummaryHistory:
- instanceAddr, err = infoschema.GetInstanceAddr(sctx)
- if err != nil {
- return nil, err
- }
- }
- user := sctx.GetSessionVars().User
- reader := stmtsummary.NewStmtSummaryReader(user, hasPriv(sctx, mysql.ProcessPriv), e.columns, instanceAddr, sctx.GetSessionVars().StmtCtx.TimeZone)
- if e.extractor.Enable {
- checker := stmtsummary.NewStmtSummaryChecker(e.extractor.Digests)
- reader.SetChecker(checker)
- }
- var rows [][]types.Datum
- switch e.table.Name.O {
- case infoschema.TableStatementsSummary,
- infoschema.ClusterTableStatementsSummary:
- rows = reader.GetStmtSummaryCurrentRows()
- case infoschema.TableStatementsSummaryHistory,
- infoschema.ClusterTableStatementsSummaryHistory:
- rows = reader.GetStmtSummaryHistoryRows()
- }
-
- return rows, nil
-}
-
// tidbTrxTableRetriever is the memtable retriever for the TIDB_TRX and CLUSTER_TIDB_TRX table.
type tidbTrxTableRetriever struct {
dummyCloser
@@ -2607,7 +2552,17 @@ func (e *tidbTrxTableRetriever) retrieve(ctx context.Context, sctx sessionctx.Co
row = append(row, types.NewDatum(nil))
}
} else {
- row = append(row, e.txnInfo[i].ToDatum(c.Name.O))
+ switch c.Name.O {
+ case txninfo.MemBufferBytesStr:
+ memDBFootprint := sctx.GetSessionVars().MemDBFootprint
+ var bytesConsumed int64
+ if memDBFootprint != nil {
+ bytesConsumed = memDBFootprint.BytesConsumed()
+ }
+ row = append(row, types.NewDatum(bytesConsumed))
+ default:
+ row = append(row, e.txnInfo[i].ToDatum(c.Name.O))
+ }
}
}
res = append(res, row)
@@ -3005,9 +2960,6 @@ func (e *hugeMemTableRetriever) retrieve(ctx context.Context, sctx sessionctx.Co
}
func adjustColumns(input [][]types.Datum, outColumns []*model.ColumnInfo, table *model.TableInfo) [][]types.Datum {
- if table.Name.O == infoschema.TableStatementsSummary {
- return input
- }
if len(outColumns) == len(table.Columns) {
return input
}
@@ -3376,6 +3328,40 @@ func (e *memtableRetriever) setDataFromPlacementPolicies(sctx sessionctx.Context
return nil
}
+func (e *memtableRetriever) setDataFromResourceGroups() error {
+ resourceGroups, err := infosync.ListResourceGroups(context.TODO())
+ if err != nil {
+ return errors.Errorf("failed to access resource group manager, error message is %s", err.Error())
+ }
+ rows := make([][]types.Datum, 0, len(resourceGroups))
+ for _, group := range resourceGroups {
+ //mode := ""
+ burstable := "NO"
+ switch group.Mode {
+ case rmpb.GroupMode_RUMode:
+ if group.RUSettings.RU.Settings.BurstLimit < 0 {
+ burstable = "YES"
+ }
+ row := types.MakeDatums(
+ group.Name,
+ group.RUSettings.RU.Settings.FillRate,
+ burstable,
+ )
+ rows = append(rows, row)
+ default:
+ //mode = "UNKNOWN_MODE"
+ row := types.MakeDatums(
+ group.Name,
+ nil,
+ nil,
+ )
+ rows = append(rows, row)
+ }
+ }
+ e.rows = rows
+ return nil
+}
+
func checkRule(rule *label.Rule) (dbName, tableName string, partitionName string, err error) {
s := strings.Split(rule.ID, "/")
if len(s) < 3 {
diff --git a/executor/infoschema_reader_test.go b/executor/infoschema_reader_test.go
index ed6eed4fb4607..ae625e7af0f60 100644
--- a/executor/infoschema_reader_test.go
+++ b/executor/infoschema_reader_test.go
@@ -254,7 +254,7 @@ func TestDDLJobs(t *testing.T) {
tk.MustExec("create table tt (a int);")
tk.MustExec("alter table tt add index t(a), add column b int")
tk.MustQuery("select db_name, table_name, job_type from information_schema.DDL_JOBS limit 3").Check(
- testkit.Rows("test_ddl_jobs tt alter table multi-schema change", "test_ddl_jobs tt add index /* subjob */", "test_ddl_jobs tt add column /* subjob */"))
+ testkit.Rows("test_ddl_jobs tt alter table multi-schema change", "test_ddl_jobs tt add index /* subjob */ /* txn-merge */", "test_ddl_jobs tt add column /* subjob */"))
}
func TestKeyColumnUsage(t *testing.T) {
@@ -471,10 +471,21 @@ func TestPartitionsTable(t *testing.T) {
tk.MustExec("DROP TABLE IF EXISTS `test_partitions`;")
tk.MustExec(`CREATE TABLE test_partitions1 (id int, b int, c varchar(5), primary key(id), index idx(c)) PARTITION BY RANGE COLUMNS(id) (PARTITION p0 VALUES LESS THAN (6), PARTITION p1 VALUES LESS THAN (11), PARTITION p2 VALUES LESS THAN (16));`)
- tk.MustQuery("select PARTITION_NAME,PARTITION_METHOD,PARTITION_EXPRESSION from information_schema.partitions where table_name = 'test_partitions1';").Check(testkit.Rows("p0 RANGE COLUMNS id", "p1 RANGE COLUMNS id", "p2 RANGE COLUMNS id"))
+ tk.MustQuery("select PARTITION_NAME,PARTITION_METHOD,PARTITION_EXPRESSION from information_schema.partitions where table_name = 'test_partitions1';").Check(testkit.Rows("p0 RANGE COLUMNS `id`", "p1 RANGE COLUMNS `id`", "p2 RANGE COLUMNS `id`"))
tk.MustExec("DROP TABLE test_partitions1")
- tk.MustExec("set @@session.tidb_enable_list_partition = ON")
+ tk.MustExec(`CREATE TABLE test_partitions (id int, b int, c varchar(5), primary key(id,b), index idx(c)) PARTITION BY RANGE COLUMNS(id,b) (PARTITION p0 VALUES LESS THAN (6,1), PARTITION p1 VALUES LESS THAN (11,9), PARTITION p2 VALUES LESS THAN (16,MAXVALUE))`)
+ tk.MustQuery("select PARTITION_NAME,PARTITION_METHOD,PARTITION_EXPRESSION from information_schema.partitions where table_name = 'test_partitions';").Check(testkit.Rows("p0 RANGE COLUMNS `id`,`b`", "p1 RANGE COLUMNS `id`,`b`", "p2 RANGE COLUMNS `id`,`b`"))
+ tk.MustExec("DROP TABLE test_partitions")
+
+ tk.MustExec(`create table test_partitions (a varchar(255), b int, c datetime) partition by list columns (b,a) (partition p0 values in ((1,"1"), (3,"3")), partition p1 values in ((2, "2")))`)
+ tk.MustQuery("select PARTITION_NAME,PARTITION_METHOD,PARTITION_EXPRESSION from information_schema.partitions where table_name = 'test_partitions'").Check(testkit.Rows("p0 LIST COLUMNS `b`,`a`", "p1 LIST COLUMNS `b`,`a`"))
+ tk.MustExec("drop table test_partitions")
+
+ tk.MustExec("create table test_partitions (a varchar(3)) partition by list columns (a) (partition p0 values in ('1'), partition p1 values in ('2'))")
+ tk.MustQuery("select PARTITION_NAME,PARTITION_METHOD,PARTITION_EXPRESSION from information_schema.partitions where table_name = 'test_partitions'").Check(testkit.Rows("p0 LIST COLUMNS `a`", "p1 LIST COLUMNS `a`"))
+ tk.MustExec("drop table test_partitions")
+
tk.MustExec("create table test_partitions (a int) partition by list (a) (partition p0 values in (1), partition p1 values in (2));")
tk.MustQuery("select PARTITION_NAME,PARTITION_METHOD,PARTITION_EXPRESSION from information_schema.partitions where table_name = 'test_partitions';").Check(testkit.Rows("p0 LIST `a`", "p1 LIST `a`"))
tk.MustExec("drop table test_partitions")
@@ -484,7 +495,7 @@ func TestPartitionsTable(t *testing.T) {
tk.MustExec("drop table test_partitions")
tk.MustExec("create table test_partitions (a bigint, b date) partition by list columns (a,b) (partition p0 values in ((1,'2020-09-28'),(1,'2020-09-29')));")
- tk.MustQuery("select PARTITION_NAME,PARTITION_METHOD,PARTITION_EXPRESSION from information_schema.partitions where table_name = 'test_partitions';").Check(testkit.Rows("p0 LIST COLUMNS a,b"))
+ tk.MustQuery("select PARTITION_NAME,PARTITION_METHOD,PARTITION_EXPRESSION from information_schema.partitions where table_name = 'test_partitions';").Check(testkit.Rows("p0 LIST COLUMNS `a`,`b`"))
pid, err := strconv.Atoi(tk.MustQuery("select TIDB_PARTITION_ID from information_schema.partitions where table_name = 'test_partitions';").Rows()[0][0].(string))
require.NoError(t, err)
require.Greater(t, pid, 0)
diff --git a/executor/insert.go b/executor/insert.go
index 9b286297351b9..2b6e45fb018d7 100644
--- a/executor/insert.go
+++ b/executor/insert.go
@@ -21,7 +21,6 @@ import (
"runtime/trace"
"time"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/expression"
@@ -31,12 +30,14 @@ import (
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/stringutil"
+ "github.com/pingcap/tidb/util/tracing"
"go.uber.org/zap"
)
@@ -70,7 +71,6 @@ func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error {
return err
}
setOptionForTopSQL(sessVars.StmtCtx, txn)
- txnSize := txn.Size()
sessVars.StmtCtx.AddRecordRows(uint64(len(rows)))
// If you use the IGNORE keyword, duplicate-key error that occurs while executing the INSERT statement are ignored.
// For example, without IGNORE, a row that duplicates an existing UNIQUE index or PRIMARY KEY value in
@@ -113,16 +113,12 @@ func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error {
e.stats.CheckInsertTime += time.Since(start)
}
}
- e.memTracker.Consume(int64(txn.Size() - txnSize))
return nil
}
func prefetchUniqueIndices(ctx context.Context, txn kv.Transaction, rows []toBeCheckedRow) (map[string][]byte, error) {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("prefetchUniqueIndices", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "prefetchUniqueIndices")
+ defer r.End()
nKeys := 0
for _, r := range rows {
@@ -150,16 +146,19 @@ func prefetchUniqueIndices(ctx context.Context, txn kv.Transaction, rows []toBeC
}
func prefetchConflictedOldRows(ctx context.Context, txn kv.Transaction, rows []toBeCheckedRow, values map[string][]byte) error {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("prefetchConflictedOldRows", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "prefetchConflictedOldRows")
+ defer r.End()
batchKeys := make([]kv.Key, 0, len(rows))
for _, r := range rows {
for _, uk := range r.uniqueKeys {
if val, found := values[string(uk.newKey)]; found {
+ if isTemp, _ := tablecodec.CheckTempIndexKey(uk.newKey); isTemp {
+ // If it is a temp index, the value cannot be decoded by DecodeHandleInUniqueIndexValue.
+ // Since this function is an optimization, we can skip prefetching the rows referenced by
+ // temp indexes.
+ continue
+ }
handle, err := tablecodec.DecodeHandleInUniqueIndexValue(val, uk.commonHandle)
if err != nil {
return err
@@ -178,11 +177,7 @@ func (e *InsertValues) prefetchDataCache(ctx context.Context, txn kv.Transaction
return nil
}
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("prefetchDataCache", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ defer tracing.StartRegion(ctx, "prefetchDataCache").End()
values, err := prefetchUniqueIndices(ctx, txn, rows)
if err != nil {
return err
@@ -257,18 +252,13 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D
}
for _, uk := range r.uniqueKeys {
- val, err := txn.Get(ctx, uk.newKey)
+ _, handle, err := tables.FetchDuplicatedHandle(ctx, uk.newKey, true, txn, e.Table.Meta().ID, uk.commonHandle)
if err != nil {
- if kv.IsErrNotFound(err) {
- continue
- }
return err
}
- handle, err := tablecodec.DecodeHandleInUniqueIndexValue(val, uk.commonHandle)
- if err != nil {
- return err
+ if handle == nil {
+ continue
}
-
err = e.updateDupRow(ctx, i, txn, r, handle, e.OnDuplicate)
if err != nil {
if kv.IsErrNotFound(err) {
@@ -329,9 +319,10 @@ func (e *InsertExec) Next(ctx context.Context, req *chunk.Chunk) error {
// Close implements the Executor Close interface.
func (e *InsertExec) Close() error {
+ if e.runtimeStats != nil && e.stats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
+ }
defer e.memTracker.ReplaceBytesUsed(0)
- e.ctx.GetSessionVars().CurrInsertValues = chunk.Row{}
- e.ctx.GetSessionVars().CurrInsertBatchExtraCols = e.ctx.GetSessionVars().CurrInsertBatchExtraCols[0:0:0]
e.setMessage()
if e.SelectExec != nil {
return e.SelectExec.Close()
diff --git a/executor/insert_common.go b/executor/insert_common.go
index 5bb7feb2441da..87938ff9af27b 100644
--- a/executor/insert_common.go
+++ b/executor/insert_common.go
@@ -22,7 +22,6 @@ import (
"sync"
"time"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/expression"
@@ -32,6 +31,7 @@ import (
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
+ "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/sessiontxn"
@@ -45,6 +45,7 @@ import (
"github.com/pingcap/tidb/util/execdetails"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/memory"
+ "github.com/pingcap/tidb/util/tracing"
"github.com/tikv/client-go/v2/txnkv/txnsnapshot"
"go.uber.org/zap"
)
@@ -388,7 +389,7 @@ func (e *InsertValues) evalRow(ctx context.Context, list []expression.Expression
e.evalBuffer.SetDatum(offset, val1)
}
// Row may lack of generated column, autoIncrement column, empty column here.
- return e.fillRow(ctx, row, hasValue)
+ return e.fillRow(ctx, row, hasValue, rowIdx)
}
var emptyRow chunk.Row
@@ -422,7 +423,7 @@ func (e *InsertValues) fastEvalRow(ctx context.Context, list []expression.Expres
offset := e.insertColumns[i].Offset
row[offset], hasValue[offset] = val1, true
}
- return e.fillRow(ctx, row, hasValue)
+ return e.fillRow(ctx, row, hasValue, rowIdx)
}
// setValueForRefColumn set some default values for the row to eval the row value with other columns,
@@ -537,7 +538,7 @@ func (e *InsertValues) doBatchInsert(ctx context.Context) error {
return ErrBatchInsertFail.GenWithStack("BatchInsert failed with error: %v", err)
}
e.memTracker.Consume(-int64(txn.Size()))
- e.ctx.StmtCommit()
+ e.ctx.StmtCommit(ctx)
if err := sessiontxn.NewTxnInStmt(ctx, e.ctx); err != nil {
// We should return a special error for batch insert.
return ErrBatchInsertFail.GenWithStack("BatchInsert failed with error: %v", err)
@@ -562,7 +563,7 @@ func (e *InsertValues) getRow(ctx context.Context, vals []types.Datum) ([]types.
hasValue[offset] = true
}
- return e.fillRow(ctx, row, hasValue)
+ return e.fillRow(ctx, row, hasValue, 0)
}
// getColDefaultValue gets the column default value.
@@ -647,7 +648,7 @@ func (e *InsertValues) fillColValue(ctx context.Context, datum types.Datum, idx
// `insert|replace values` can guarantee consecutive autoID in a batch.
// Other statements like `insert select from` don't guarantee consecutive autoID.
// https://dev.mysql.com/doc/refman/8.0/en/innodb-auto-increment-handling.html
-func (e *InsertValues) fillRow(ctx context.Context, row []types.Datum, hasValue []bool) ([]types.Datum, error) {
+func (e *InsertValues) fillRow(ctx context.Context, row []types.Datum, hasValue []bool, rowIdx int) ([]types.Datum, error) {
gCols := make([]*table.Column, 0)
tCols := e.Table.Cols()
if e.hasExtraHandle {
@@ -693,6 +694,9 @@ func (e *InsertValues) fillRow(ctx context.Context, row []types.Datum, hasValue
for i, gCol := range gCols {
colIdx := gCol.ColumnInfo.Offset
val, err := e.GenExprs[i].Eval(chunk.MutRowFromDatums(row).ToRow())
+ if err != nil && gCol.FieldType.IsArray() {
+ return nil, completeError(tbl, gCol.Offset, rowIdx, err)
+ }
if e.ctx.GetSessionVars().StmtCtx.HandleTruncate(err) != nil {
return nil, err
}
@@ -708,6 +712,29 @@ func (e *InsertValues) fillRow(ctx context.Context, row []types.Datum, hasValue
return row, nil
}
+func completeError(tbl *model.TableInfo, offset int, rowIdx int, err error) error {
+ name := "expression_index"
+ for _, idx := range tbl.Indices {
+ for _, column := range idx.Columns {
+ if column.Offset == offset {
+ name = idx.Name.O
+ break
+ }
+ }
+ }
+
+ if expression.ErrInvalidJSONForFuncIndex.Equal(err) {
+ return expression.ErrInvalidJSONForFuncIndex.GenWithStackByArgs(name)
+ }
+ if types.ErrOverflow.Equal(err) {
+ return expression.ErrDataOutOfRangeFuncIndex.GenWithStackByArgs(name, rowIdx+1)
+ }
+ if types.ErrDataTooLong.Equal(err) {
+ return expression.ErrFuncIndexDataIsTooLong.GenWithStackByArgs(name)
+ }
+ return err
+}
+
// isAutoNull can help judge whether a datum is AutoIncrement Null quickly.
// This used to help lazyFillAutoIncrement to find consecutive N datum backwards for batch autoID alloc.
func (e *InsertValues) isAutoNull(ctx context.Context, d types.Datum, col *table.Column) bool {
@@ -745,7 +772,16 @@ func setDatumAutoIDAndCast(ctx sessionctx.Context, d *types.Datum, id int64, col
var err error
*d, err = table.CastValue(ctx, *d, col.ToInfo(), false, false)
if err == nil && d.GetInt64() < id {
- // Auto ID is out of range, the truncated ID is possible to duplicate with an existing ID.
+ // Auto ID is out of range.
+ sc := ctx.GetSessionVars().StmtCtx
+ insertPlan, ok := sc.GetPlan().(*core.Insert)
+ if ok && sc.TruncateAsWarning && len(insertPlan.OnDuplicate) > 0 {
+ // Fix issue #38950: AUTO_INCREMENT is incompatible with mysql
+ // An auto id out of range error occurs in `insert ignore into ... on duplicate ...`.
+ // We should allow the SQL to be executed successfully.
+ return nil
+ }
+ // The truncated ID is possible to duplicate with an existing ID.
// To prevent updating unrelated rows in the REPLACE statement, it is better to throw an error.
return autoid.ErrAutoincReadFailed
}
@@ -778,7 +814,8 @@ func (e *InsertValues) lazyAdjustAutoIncrementDatum(ctx context.Context, rows []
}
// Use the value if it's not null and not 0.
if recordID != 0 {
- err = e.Table.Allocators(e.ctx).Get(autoid.RowIDAllocType).Rebase(ctx, recordID, true)
+ alloc := e.Table.Allocators(e.ctx).Get(autoid.AutoIncrementType)
+ err = alloc.Rebase(ctx, recordID, true)
if err != nil {
return nil, err
}
@@ -871,7 +908,7 @@ func (e *InsertValues) adjustAutoIncrementDatum(ctx context.Context, d types.Dat
}
// Use the value if it's not null and not 0.
if recordID != 0 {
- err = e.Table.Allocators(e.ctx).Get(autoid.RowIDAllocType).Rebase(ctx, recordID, true)
+ err = e.Table.Allocators(e.ctx).Get(autoid.AutoIncrementType).Rebase(ctx, recordID, true)
if err != nil {
return types.Datum{}, err
}
@@ -1007,7 +1044,7 @@ func (e *InsertValues) allocAutoRandomID(ctx context.Context, fieldType *types.F
if err != nil {
return 0, err
}
- currentShard := e.ctx.GetSessionVars().TxnCtx.GetCurrentShard(1)
+ currentShard := e.ctx.GetSessionVars().GetCurrentShard(1)
return shardFmt.Compose(currentShard, autoRandomID), nil
}
@@ -1091,12 +1128,7 @@ func (e *InsertValues) collectRuntimeStatsEnabled() bool {
BasicRuntimeStats: e.runtimeStats,
SnapshotRuntimeStats: snapshotStats,
AllocatorRuntimeStats: autoid.NewAllocatorRuntimeStats(),
- FKCheckStats: &FKCheckRuntimeStats{},
- }
- for _, fkc := range e.fkChecks {
- fkc.stats = e.stats.FKCheckStats
}
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
return true
}
@@ -1110,11 +1142,7 @@ func (e *InsertValues) batchCheckAndInsert(ctx context.Context, rows [][]types.D
replace bool) error {
// all the rows will be checked, so it is safe to set BatchCheck = true
e.ctx.GetSessionVars().StmtCtx.BatchCheck = true
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("InsertValues.batchCheckAndInsert", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- opentracing.ContextWithSpan(ctx, span1)
- }
+ defer tracing.StartRegion(ctx, "InsertValues.batchCheckAndInsert").End()
start := time.Now()
// Get keys need to be checked.
toBeCheckedRows, err := getKeysNeedCheck(ctx, e.ctx, e.Table, rows)
@@ -1299,7 +1327,6 @@ type InsertRuntimeStat struct {
CheckInsertTime time.Duration
Prefetch time.Duration
FKCheckTime time.Duration
- FKCheckStats *FKCheckRuntimeStats
}
func (e *InsertRuntimeStat) String() string {
@@ -1341,10 +1368,8 @@ func (e *InsertRuntimeStat) String() string {
execdetails.FormatDuration(e.CheckInsertTime),
execdetails.FormatDuration(e.CheckInsertTime-e.Prefetch),
execdetails.FormatDuration(e.Prefetch)))
- if e.FKCheckStats != nil && e.FKCheckStats.Keys > 0 {
- buf.WriteString(fmt.Sprintf(", fk_check: %v, fk_num: %v",
- execdetails.FormatDuration(e.FKCheckTime),
- e.FKCheckStats.Keys))
+ if e.FKCheckTime > 0 {
+ buf.WriteString(fmt.Sprintf(", fk_check: %v", execdetails.FormatDuration(e.FKCheckTime)))
}
if e.SnapshotRuntimeStats != nil {
if rpc := e.SnapshotRuntimeStats.String(); len(rpc) > 0 {
@@ -1376,10 +1401,6 @@ func (e *InsertRuntimeStat) Clone() execdetails.RuntimeStats {
if e.AllocatorRuntimeStats != nil {
newRs.AllocatorRuntimeStats = e.AllocatorRuntimeStats.Clone()
}
- if e.FKCheckStats != nil {
- fkCheckStats := *e.FKCheckStats
- newRs.FKCheckStats = &fkCheckStats
- }
return newRs
}
diff --git a/executor/insert_test.go b/executor/insert_test.go
index 53d0c62cc9776..acd8d6736b219 100644
--- a/executor/insert_test.go
+++ b/executor/insert_test.go
@@ -419,6 +419,34 @@ func TestInsertValueForCastDecimalField(t *testing.T) {
tk.MustQuery(`select cast(a as decimal) from t1;`).Check(testkit.Rows(`9999999999`))
}
+func TestInsertForMultiValuedIndex(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec(`drop table if exists t1;`)
+ tk.MustExec(`create table t1(a json, b int, unique index idx((cast(a as signed array))));`)
+ tk.MustExec(`insert into t1 values ('[1,11]', 1);`)
+ tk.MustExec(`insert into t1 values ('[2, 22]', 2);`)
+ tk.MustQuery(`select * from t1;`).Check(testkit.Rows(`[1, 11] 1`, `[2, 22] 2`))
+ tk.MustGetErrMsg(`insert into t1 values ('[2, 222]', 2);`, "[kv:1062]Duplicate entry '2' for key 't1.idx'")
+ tk.MustExec(`replace into t1 values ('[1, 10]', 10)`)
+ tk.MustQuery(`select * from t1;`).Check(testkit.Rows(`[2, 22] 2`, `[1, 10] 10`))
+ tk.MustExec(`replace into t1 values ('[1, 2]', 1)`)
+ tk.MustQuery(`select * from t1;`).Check(testkit.Rows(`[1, 2] 1`))
+ tk.MustExec(`replace into t1 values ('[1, 11]', 1)`)
+ tk.MustExec(`insert into t1 values ('[2, 22]', 2);`)
+ tk.MustQuery(`select * from t1;`).Check(testkit.Rows(`[1, 11] 1`, `[2, 22] 2`))
+ tk.MustExec(`insert ignore into t1 values ('[1]', 2);`)
+ tk.MustQuery(`select * from t1;`).Check(testkit.Rows(`[1, 11] 1`, `[2, 22] 2`))
+ tk.MustExec(`insert ignore into t1 values ('[1, 2]', 2);`)
+ tk.MustQuery(`select * from t1;`).Check(testkit.Rows(`[1, 11] 1`, `[2, 22] 2`))
+ tk.MustExec(`insert into t1 values ('[2]', 2) on duplicate key update b = 10;`)
+ tk.MustQuery(`select * from t1;`).Check(testkit.Rows(`[1, 11] 1`, `[2, 22] 10`))
+ tk.MustGetErrMsg(`insert into t1 values ('[2, 1]', 2) on duplicate key update a = '[1,2]';`, "[kv:1062]Duplicate entry '[1, 2]' for key 't1.idx'")
+ tk.MustGetErrMsg(`insert into t1 values ('[1,2]', 2) on duplicate key update a = '[1,2]';`, "[kv:1062]Duplicate entry '[1, 2]' for key 't1.idx'")
+ tk.MustGetErrMsg(`insert into t1 values ('[11, 22]', 2) on duplicate key update a = '[1,2]';`, "[kv:1062]Duplicate entry '[1, 2]' for key 't1.idx'")
+}
+
func TestInsertDateTimeWithTimeZone(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -589,435 +617,6 @@ func TestAllowInvalidDates(t *testing.T) {
runWithMode("ALLOW_INVALID_DATES")
}
-func TestInsertWithAutoidSchema(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec(`use test`)
- tk.MustExec(`create table t1(id int primary key auto_increment, n int);`)
- tk.MustExec(`create table t2(id int unsigned primary key auto_increment, n int);`)
- tk.MustExec(`create table t3(id tinyint primary key auto_increment, n int);`)
- tk.MustExec(`create table t4(id int primary key, n float auto_increment, key I_n(n));`)
- tk.MustExec(`create table t5(id int primary key, n float unsigned auto_increment, key I_n(n));`)
- tk.MustExec(`create table t6(id int primary key, n double auto_increment, key I_n(n));`)
- tk.MustExec(`create table t7(id int primary key, n double unsigned auto_increment, key I_n(n));`)
- // test for inserting multiple values
- tk.MustExec(`create table t8(id int primary key auto_increment, n int);`)
-
- tests := []struct {
- insert string
- query string
- result [][]interface{}
- }{
- {
- `insert into t1(id, n) values(1, 1)`,
- `select * from t1 where id = 1`,
- testkit.Rows(`1 1`),
- },
- {
- `insert into t1(n) values(2)`,
- `select * from t1 where id = 2`,
- testkit.Rows(`2 2`),
- },
- {
- `insert into t1(n) values(3)`,
- `select * from t1 where id = 3`,
- testkit.Rows(`3 3`),
- },
- {
- `insert into t1(id, n) values(-1, 4)`,
- `select * from t1 where id = -1`,
- testkit.Rows(`-1 4`),
- },
- {
- `insert into t1(n) values(5)`,
- `select * from t1 where id = 4`,
- testkit.Rows(`4 5`),
- },
- {
- `insert into t1(id, n) values('5', 6)`,
- `select * from t1 where id = 5`,
- testkit.Rows(`5 6`),
- },
- {
- `insert into t1(n) values(7)`,
- `select * from t1 where id = 6`,
- testkit.Rows(`6 7`),
- },
- {
- `insert into t1(id, n) values(7.4, 8)`,
- `select * from t1 where id = 7`,
- testkit.Rows(`7 8`),
- },
- {
- `insert into t1(id, n) values(7.5, 9)`,
- `select * from t1 where id = 8`,
- testkit.Rows(`8 9`),
- },
- {
- `insert into t1(n) values(9)`,
- `select * from t1 where id = 9`,
- testkit.Rows(`9 9`),
- },
- // test last insert id
- {
- `insert into t1 values(3000, -1), (null, -2)`,
- `select * from t1 where id = 3000`,
- testkit.Rows(`3000 -1`),
- },
- {
- `;`,
- `select * from t1 where id = 3001`,
- testkit.Rows(`3001 -2`),
- },
- {
- `;`,
- `select last_insert_id()`,
- testkit.Rows(`3001`),
- },
- {
- `insert into t2(id, n) values(1, 1)`,
- `select * from t2 where id = 1`,
- testkit.Rows(`1 1`),
- },
- {
- `insert into t2(n) values(2)`,
- `select * from t2 where id = 2`,
- testkit.Rows(`2 2`),
- },
- {
- `insert into t2(n) values(3)`,
- `select * from t2 where id = 3`,
- testkit.Rows(`3 3`),
- },
- {
- `insert into t3(id, n) values(1, 1)`,
- `select * from t3 where id = 1`,
- testkit.Rows(`1 1`),
- },
- {
- `insert into t3(n) values(2)`,
- `select * from t3 where id = 2`,
- testkit.Rows(`2 2`),
- },
- {
- `insert into t3(n) values(3)`,
- `select * from t3 where id = 3`,
- testkit.Rows(`3 3`),
- },
- {
- `insert into t3(id, n) values(-1, 4)`,
- `select * from t3 where id = -1`,
- testkit.Rows(`-1 4`),
- },
- {
- `insert into t3(n) values(5)`,
- `select * from t3 where id = 4`,
- testkit.Rows(`4 5`),
- },
- {
- `insert into t4(id, n) values(1, 1)`,
- `select * from t4 where id = 1`,
- testkit.Rows(`1 1`),
- },
- {
- `insert into t4(id) values(2)`,
- `select * from t4 where id = 2`,
- testkit.Rows(`2 2`),
- },
- {
- `insert into t4(id, n) values(3, -1)`,
- `select * from t4 where id = 3`,
- testkit.Rows(`3 -1`),
- },
- {
- `insert into t4(id) values(4)`,
- `select * from t4 where id = 4`,
- testkit.Rows(`4 3`),
- },
- {
- `insert into t4(id, n) values(5, 5.5)`,
- `select * from t4 where id = 5`,
- testkit.Rows(`5 5.5`),
- },
- {
- `insert into t4(id) values(6)`,
- `select * from t4 where id = 6`,
- testkit.Rows(`6 7`),
- },
- {
- `insert into t4(id, n) values(7, '7.7')`,
- `select * from t4 where id = 7`,
- testkit.Rows(`7 7.7`),
- },
- {
- `insert into t4(id) values(8)`,
- `select * from t4 where id = 8`,
- testkit.Rows(`8 9`),
- },
- {
- `insert into t4(id, n) values(9, 10.4)`,
- `select * from t4 where id = 9`,
- testkit.Rows(`9 10.4`),
- },
- {
- `insert into t4(id) values(10)`,
- `select * from t4 where id = 10`,
- testkit.Rows(`10 11`),
- },
- {
- `insert into t5(id, n) values(1, 1)`,
- `select * from t5 where id = 1`,
- testkit.Rows(`1 1`),
- },
- {
- `insert into t5(id) values(2)`,
- `select * from t5 where id = 2`,
- testkit.Rows(`2 2`),
- },
- {
- `insert into t5(id) values(3)`,
- `select * from t5 where id = 3`,
- testkit.Rows(`3 3`),
- },
- {
- `insert into t6(id, n) values(1, 1)`,
- `select * from t6 where id = 1`,
- testkit.Rows(`1 1`),
- },
- {
- `insert into t6(id) values(2)`,
- `select * from t6 where id = 2`,
- testkit.Rows(`2 2`),
- },
- {
- `insert into t6(id, n) values(3, -1)`,
- `select * from t6 where id = 3`,
- testkit.Rows(`3 -1`),
- },
- {
- `insert into t6(id) values(4)`,
- `select * from t6 where id = 4`,
- testkit.Rows(`4 3`),
- },
- {
- `insert into t6(id, n) values(5, 5.5)`,
- `select * from t6 where id = 5`,
- testkit.Rows(`5 5.5`),
- },
- {
- `insert into t6(id) values(6)`,
- `select * from t6 where id = 6`,
- testkit.Rows(`6 7`),
- },
- {
- `insert into t6(id, n) values(7, '7.7')`,
- `select * from t4 where id = 7`,
- testkit.Rows(`7 7.7`),
- },
- {
- `insert into t6(id) values(8)`,
- `select * from t4 where id = 8`,
- testkit.Rows(`8 9`),
- },
- {
- `insert into t6(id, n) values(9, 10.4)`,
- `select * from t6 where id = 9`,
- testkit.Rows(`9 10.4`),
- },
- {
- `insert into t6(id) values(10)`,
- `select * from t6 where id = 10`,
- testkit.Rows(`10 11`),
- },
- {
- `insert into t7(id, n) values(1, 1)`,
- `select * from t7 where id = 1`,
- testkit.Rows(`1 1`),
- },
- {
- `insert into t7(id) values(2)`,
- `select * from t7 where id = 2`,
- testkit.Rows(`2 2`),
- },
- {
- `insert into t7(id) values(3)`,
- `select * from t7 where id = 3`,
- testkit.Rows(`3 3`),
- },
-
- // the following is test for insert multiple values.
- {
- `insert into t8(n) values(1),(2)`,
- `select * from t8 where id = 1`,
- testkit.Rows(`1 1`),
- },
- {
- `;`,
- `select * from t8 where id = 2`,
- testkit.Rows(`2 2`),
- },
- {
- `;`,
- `select last_insert_id();`,
- testkit.Rows(`1`),
- },
- // test user rebase and auto alloc mixture.
- {
- `insert into t8 values(null, 3),(-1, -1),(null,4),(null, 5)`,
- `select * from t8 where id = 3`,
- testkit.Rows(`3 3`),
- },
- // -1 won't rebase allocator here cause -1 < base.
- {
- `;`,
- `select * from t8 where id = -1`,
- testkit.Rows(`-1 -1`),
- },
- {
- `;`,
- `select * from t8 where id = 4`,
- testkit.Rows(`4 4`),
- },
- {
- `;`,
- `select * from t8 where id = 5`,
- testkit.Rows(`5 5`),
- },
- {
- `;`,
- `select last_insert_id();`,
- testkit.Rows(`3`),
- },
- {
- `insert into t8 values(null, 6),(10, 7),(null, 8)`,
- `select * from t8 where id = 6`,
- testkit.Rows(`6 6`),
- },
- // 10 will rebase allocator here.
- {
- `;`,
- `select * from t8 where id = 10`,
- testkit.Rows(`10 7`),
- },
- {
- `;`,
- `select * from t8 where id = 11`,
- testkit.Rows(`11 8`),
- },
- {
- `;`,
- `select last_insert_id()`,
- testkit.Rows(`6`),
- },
- // fix bug for last_insert_id should be first allocated id in insert rows (skip the rebase id).
- {
- `insert into t8 values(100, 9),(null,10),(null,11)`,
- `select * from t8 where id = 100`,
- testkit.Rows(`100 9`),
- },
- {
- `;`,
- `select * from t8 where id = 101`,
- testkit.Rows(`101 10`),
- },
- {
- `;`,
- `select * from t8 where id = 102`,
- testkit.Rows(`102 11`),
- },
- {
- `;`,
- `select last_insert_id()`,
- testkit.Rows(`101`),
- },
- // test with sql_mode: NO_AUTO_VALUE_ON_ZERO.
- {
- `;`,
- `select @@sql_mode`,
- testkit.Rows(`ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION`),
- },
- {
- `;`,
- "set session sql_mode = `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO`",
- nil,
- },
- {
- `insert into t8 values (0, 12), (null, 13)`,
- `select * from t8 where id = 0`,
- testkit.Rows(`0 12`),
- },
- {
- `;`,
- `select * from t8 where id = 103`,
- testkit.Rows(`103 13`),
- },
- {
- `;`,
- `select last_insert_id()`,
- testkit.Rows(`103`),
- },
- // test without sql_mode: NO_AUTO_VALUE_ON_ZERO.
- {
- `;`,
- "set session sql_mode = `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION`",
- nil,
- },
- // value 0 will be substitute by autoid.
- {
- `insert into t8 values (0, 14), (null, 15)`,
- `select * from t8 where id = 104`,
- testkit.Rows(`104 14`),
- },
- {
- `;`,
- `select * from t8 where id = 105`,
- testkit.Rows(`105 15`),
- },
- {
- `;`,
- `select last_insert_id()`,
- testkit.Rows(`104`),
- },
- // last test : auto increment allocation can find in retryInfo.
- {
- `retry : insert into t8 values (null, 16), (null, 17)`,
- `select * from t8 where id = 1000`,
- testkit.Rows(`1000 16`),
- },
- {
- `;`,
- `select * from t8 where id = 1001`,
- testkit.Rows(`1001 17`),
- },
- {
- `;`,
- `select last_insert_id()`,
- // this insert doesn't has the last_insert_id, should be same as the last insert case.
- testkit.Rows(`104`),
- },
- }
-
- for _, tt := range tests {
- if strings.HasPrefix(tt.insert, "retry : ") {
- // it's the last retry insert case, change the sessionVars.
- retryInfo := &variable.RetryInfo{Retrying: true}
- retryInfo.AddAutoIncrementID(1000)
- retryInfo.AddAutoIncrementID(1001)
- tk.Session().GetSessionVars().RetryInfo = retryInfo
- tk.MustExec(tt.insert[8:])
- tk.Session().GetSessionVars().RetryInfo = &variable.RetryInfo{}
- } else {
- tk.MustExec(tt.insert)
- }
- if tt.query == "set session sql_mode = `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO`" ||
- tt.query == "set session sql_mode = `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION`" {
- tk.MustExec(tt.query)
- } else {
- tk.MustQuery(tt.query).Check(tt.result)
- }
- }
-}
-
func TestPartitionInsertOnDuplicate(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -1139,75 +738,6 @@ func TestInsertFloatOverflow(t *testing.T) {
tk.MustExec("drop table if exists t,t1")
}
-// TestAutoIDIncrementAndOffset There is a potential issue in MySQL: when the value of auto_increment_offset is greater
-// than that of auto_increment_increment, the value of auto_increment_offset is ignored
-// (https://dev.mysql.com/doc/refman/8.0/en/replication-options-master.html#sysvar_auto_increment_increment),
-// This issue is a flaw of the implementation of MySQL and it doesn't exist in TiDB.
-func TestAutoIDIncrementAndOffset(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec(`use test`)
- // Test for offset is larger than increment.
- tk.Session().GetSessionVars().AutoIncrementIncrement = 5
- tk.Session().GetSessionVars().AutoIncrementOffset = 10
- tk.MustExec(`create table io (a int key auto_increment)`)
- tk.MustExec(`insert into io values (null),(null),(null)`)
- tk.MustQuery(`select * from io`).Check(testkit.Rows("10", "15", "20"))
- tk.MustExec(`drop table io`)
-
- // Test handle is PK.
- tk.MustExec(`create table io (a int key auto_increment)`)
- tk.Session().GetSessionVars().AutoIncrementOffset = 10
- tk.Session().GetSessionVars().AutoIncrementIncrement = 2
- tk.MustExec(`insert into io values (),(),()`)
- tk.MustQuery(`select * from io`).Check(testkit.Rows("10", "12", "14"))
- tk.MustExec(`delete from io`)
-
- // Test reset the increment.
- tk.Session().GetSessionVars().AutoIncrementIncrement = 5
- tk.MustExec(`insert into io values (),(),()`)
- tk.MustQuery(`select * from io`).Check(testkit.Rows("15", "20", "25"))
- tk.MustExec(`delete from io`)
-
- tk.Session().GetSessionVars().AutoIncrementIncrement = 10
- tk.MustExec(`insert into io values (),(),()`)
- tk.MustQuery(`select * from io`).Check(testkit.Rows("30", "40", "50"))
- tk.MustExec(`delete from io`)
-
- tk.Session().GetSessionVars().AutoIncrementIncrement = 5
- tk.MustExec(`insert into io values (),(),()`)
- tk.MustQuery(`select * from io`).Check(testkit.Rows("55", "60", "65"))
- tk.MustExec(`drop table io`)
-
- // Test handle is not PK.
- tk.Session().GetSessionVars().AutoIncrementIncrement = 2
- tk.Session().GetSessionVars().AutoIncrementOffset = 10
- tk.MustExec(`create table io (a int, b int auto_increment, key(b))`)
- tk.MustExec(`insert into io(b) values (null),(null),(null)`)
- // AutoID allocation will take increment and offset into consideration.
- tk.MustQuery(`select b from io`).Check(testkit.Rows("10", "12", "14"))
- // HandleID allocation will ignore the increment and offset.
- tk.MustQuery(`select _tidb_rowid from io`).Check(testkit.Rows("15", "16", "17"))
- tk.MustExec(`delete from io`)
-
- tk.Session().GetSessionVars().AutoIncrementIncrement = 10
- tk.MustExec(`insert into io(b) values (null),(null),(null)`)
- tk.MustQuery(`select b from io`).Check(testkit.Rows("20", "30", "40"))
- tk.MustQuery(`select _tidb_rowid from io`).Check(testkit.Rows("41", "42", "43"))
-
- // Test invalid value.
- tk.Session().GetSessionVars().AutoIncrementIncrement = -1
- tk.Session().GetSessionVars().AutoIncrementOffset = -2
- tk.MustGetErrMsg(`insert into io(b) values (null),(null),(null)`,
- "[autoid:8060]Invalid auto_increment settings: auto_increment_increment: -1, auto_increment_offset: -2, both of them must be in range [1..65535]")
- tk.MustExec(`delete from io`)
-
- tk.Session().GetSessionVars().AutoIncrementIncrement = 65536
- tk.Session().GetSessionVars().AutoIncrementOffset = 65536
- tk.MustGetErrMsg(`insert into io(b) values (null),(null),(null)`,
- "[autoid:8060]Invalid auto_increment settings: auto_increment_increment: 65536, auto_increment_offset: 65536, both of them must be in range [1..65535]")
-}
-
// Fix https://github.com/pingcap/tidb/issues/32601.
func TestTextTooLongError(t *testing.T) {
store := testkit.CreateMockStore(t)
@@ -1393,6 +923,22 @@ func TestInsertErrorMsg(t *testing.T) {
tk.MustExec(`create table t (a int primary key, b datetime, d date)`)
tk.MustContainErrMsg(`insert into t values (1, '2019-02-11 30:00:00', '2019-01-31')`,
"Incorrect datetime value: '2019-02-11 30:00:00' for column 'b' at row 1")
+
+ // test for Issue #35289
+ tk.MustExec("CREATE TABLE t1 (a BINARY(16) PRIMARY KEY);")
+ tk.MustExec(`INSERT INTO t1 VALUES (AES_ENCRYPT('a','a'));`)
+ err := tk.ExecToErr(`INSERT INTO t1 VALUES (AES_ENCRYPT('a','a'));`)
+ require.Error(t, err, `ERROR 1062 (23000): Duplicate entry '{ W]\xA1\x06u\x9D\xBD\xB1\xA3.\xE2\xD9\xA7t' for key 't1.PRIMARY'`)
+
+ tk.MustExec(`INSERT INTO t1 VALUES (AES_ENCRYPT('b','b'));`)
+ err = tk.ExecToErr(`INSERT INTO t1 VALUES (AES_ENCRYPT('b','b'));`)
+ require.Error(t, err, "ERROR 1062 (23000): Duplicate entry '\\x0C\\x1E\\x8DG`\\xEB\\x93 F&BC\\xF0\\xB5\\xF4\\xB7' for key 't1.PRIMARY'")
+
+ tk.MustExec("drop table if exists t1")
+ tk.MustExec("create table t1 (a bit primary key) engine=innodb;")
+ tk.MustExec("insert into t1 values (b'0');")
+ err = tk.ExecToErr(`insert into t1 values (b'0');`)
+ require.Error(t, err, `ERROR 1062 (23000): Duplicate entry '\x00' for key 't1.PRIMARY'`)
}
func TestIssue16366(t *testing.T) {
@@ -1554,8 +1100,7 @@ func TestInsertRuntimeStat(t *testing.T) {
stats.Merge(stats.Clone())
require.Equal(t, "prepare: 6s, check_insert: {total_time: 4s, mem_insert_time: 2s, prefetch: 2s}", stats.String())
stats.FKCheckTime = time.Second
- stats.FKCheckStats = &executor.FKCheckRuntimeStats{Keys: 20}
- require.Equal(t, "prepare: 6s, check_insert: {total_time: 4s, mem_insert_time: 2s, prefetch: 2s, fk_check: 1s, fk_num: 20}", stats.String())
+ require.Equal(t, "prepare: 6s, check_insert: {total_time: 4s, mem_insert_time: 2s, prefetch: 2s, fk_check: 1s}", stats.String())
}
func TestDuplicateEntryMessage(t *testing.T) {
diff --git a/executor/inspection_result_test.go b/executor/inspection_result_test.go
index 67e5d98216b1b..4d5da7ba287cc 100644
--- a/executor/inspection_result_test.go
+++ b/executor/inspection_result_test.go
@@ -179,7 +179,7 @@ func TestInspectionResult(t *testing.T) {
}
func parseTime(t *testing.T, se session.Session, str string) types.Time {
- time, err := types.ParseTime(se.GetSessionVars().StmtCtx, str, mysql.TypeDatetime, types.MaxFsp)
+ time, err := types.ParseTime(se.GetSessionVars().StmtCtx, str, mysql.TypeDatetime, types.MaxFsp, nil)
require.NoError(t, err)
return time
}
@@ -338,7 +338,7 @@ func TestThresholdCheckInspection2(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
datetime := func(s string) types.Time {
- time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp)
+ time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp, nil)
require.NoError(t, err)
return time
}
@@ -421,7 +421,7 @@ func TestThresholdCheckInspection3(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
datetime := func(s string) types.Time {
- time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp)
+ time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp, nil)
require.NoError(t, err)
return time
}
@@ -628,7 +628,7 @@ func TestNodeLoadInspection(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
datetime := func(s string) types.Time {
- time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp)
+ time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp, nil)
require.NoError(t, err)
return time
}
@@ -704,7 +704,7 @@ func TestConfigCheckOfStorageBlockCacheSize(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
datetime := func(s string) types.Time {
- time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp)
+ time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp, nil)
require.NoError(t, err)
return time
}
diff --git a/executor/inspection_summary_test.go b/executor/inspection_summary_test.go
index 5a28bbde464b0..bad9cfe1b21ba 100644
--- a/executor/inspection_summary_test.go
+++ b/executor/inspection_summary_test.go
@@ -51,7 +51,7 @@ func TestInspectionSummary(t *testing.T) {
defer func() { require.NoError(t, failpoint.Disable(fpName)) }()
datetime := func(s string) types.Time {
- time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp)
+ time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp, nil)
require.NoError(t, err)
return time
}
diff --git a/executor/issuetest/BUILD.bazel b/executor/issuetest/BUILD.bazel
new file mode 100644
index 0000000000000..12282131b98e0
--- /dev/null
+++ b/executor/issuetest/BUILD.bazel
@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+ name = "issuetest_test",
+ timeout = "short",
+ srcs = [
+ "executor_issue_test.go",
+ "main_test.go",
+ ],
+ flaky = True,
+ shard_count = 50,
+ deps = [
+ "//autoid_service",
+ "//config",
+ "//kv",
+ "//meta/autoid",
+ "//parser/auth",
+ "//parser/charset",
+ "//parser/mysql",
+ "//sessionctx/variable",
+ "//statistics",
+ "//testkit",
+ "//util",
+ "@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_stretchr_testify//require",
+ "@com_github_tikv_client_go_v2//tikv",
+ "@org_uber_go_goleak//:goleak",
+ ],
+)
diff --git a/executor/executor_issue_test.go b/executor/issuetest/executor_issue_test.go
similarity index 86%
rename from executor/executor_issue_test.go
rename to executor/issuetest/executor_issue_test.go
index 5d2912f78f449..b97df5a5c11c1 100644
--- a/executor/executor_issue_test.go
+++ b/executor/issuetest/executor_issue_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package executor_test
+package issuetest_test
import (
"context"
@@ -22,6 +22,7 @@ import (
"testing"
"github.com/pingcap/failpoint"
+ _ "github.com/pingcap/tidb/autoid_service"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/auth"
@@ -532,6 +533,7 @@ func TestFix31537(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
+ tk.MustExec("set @@foreign_key_checks=0")
tk.MustExec(`CREATE TABLE trade (
t_id bigint(16) NOT NULL AUTO_INCREMENT,
t_dts datetime NOT NULL,
@@ -684,6 +686,9 @@ func TestIssue22231(t *testing.T) {
tk.MustQuery("select cast('2020-05-28 23:59:59 00:00:00' as datetime)").Check(testkit.Rows("2020-05-28 23:59:59"))
tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1292 Truncated incorrect datetime value: '2020-05-28 23:59:59 00:00:00'"))
tk.MustExec("drop table if exists t_issue_22231")
+
+ tk.MustQuery("SELECT CAST(\"1111111111-\" AS DATE);")
+ tk.MustQuery("SHOW WARNINGS").Check(testkit.Rows("Warning 1292 Incorrect datetime value: '1111111111-'"))
}
// TestIssue2612 is related with https://github.com/pingcap/tidb/issues/2612
@@ -1237,3 +1242,148 @@ func TestIssue33214(t *testing.T) {
}
}
}
+
+func TestIssue982(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t;")
+ tk.MustExec("create table t (c int auto_increment, key(c)) auto_id_cache 1;")
+ tk.MustExec("insert into t values();")
+ tk.MustExec("insert into t values();")
+ tk.MustQuery("select * from t;").Check(testkit.Rows("1", "2"))
+}
+
+func TestIssue24627(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ for _, sql := range []string{
+ "create table test(id float primary key clustered AUTO_INCREMENT, col1 int);",
+ "create table test(id float primary key nonclustered AUTO_INCREMENT, col1 int) AUTO_ID_CACHE 1;",
+ } {
+ tk.MustExec("drop table if exists test;")
+ tk.MustExec(sql)
+ tk.MustExec("replace into test(col1) values(1);")
+ tk.MustExec("replace into test(col1) values(2);")
+ tk.MustQuery("select * from test;").Check(testkit.Rows("1 1", "2 2"))
+ tk.MustExec("drop table test")
+ }
+
+ for _, sql := range []string{
+ "create table test2(id double primary key clustered AUTO_INCREMENT, col1 int);",
+ "create table test2(id double primary key nonclustered AUTO_INCREMENT, col1 int) AUTO_ID_CACHE 1;",
+ } {
+ tk.MustExec(sql)
+ tk.MustExec("replace into test2(col1) values(1);")
+ tk.MustExec("insert into test2(col1) values(1);")
+ tk.MustExec("replace into test2(col1) values(1);")
+ tk.MustExec("insert into test2(col1) values(1);")
+ tk.MustExec("replace into test2(col1) values(1);")
+ tk.MustExec("replace into test2(col1) values(1);")
+ tk.MustQuery("select * from test2").Check(testkit.Rows("1 1", "2 1", "3 1", "4 1", "5 1", "6 1"))
+ tk.MustExec("drop table test2")
+ }
+}
+
+func TestIssue39618(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1;")
+ tk.MustExec(`CREATE TABLE t1 (
+ c_int int(11) NOT NULL,
+ c_str varbinary(40) NOT NULL,
+ c_datetime datetime DEFAULT NULL,
+ c_timestamp timestamp NULL DEFAULT NULL,
+ c_double double DEFAULT NULL,
+ c_decimal decimal(12,6) DEFAULT NULL,
+ c_enum enum('blue','green','red','yellow','white','orange','purple') DEFAULT NULL,
+ PRIMARY KEY (c_int,c_str) /*T![clustered_index] CLUSTERED */,
+ KEY c_int_2 (c_int),
+ KEY c_decimal (c_decimal),
+ KEY c_datetime (c_datetime)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin
+PARTITION BY LIST COLUMNS(c_int)
+(PARTITION p0 VALUES IN (1,5,9,13,17,21,25,29,33,37),
+ PARTITION p1 VALUES IN (2,6,10,14,18,22,26,30,34,38),
+ PARTITION p2 VALUES IN (3,7,11,15,19,23,27,31,35,39),
+ PARTITION p3 VALUES IN (4,8,12,16,20,24,28,32,36,40));`)
+ tk.MustExec("INSERT INTO t1 VALUES (3,'bold goldberg','2020-01-07 12:08:19','2020-06-19 08:13:35',0.941002,5.303000,'yellow'),(1,'crazy wescoff','2020-03-24 21:51:02','2020-06-19 08:13:35',47.565275,6.313000,'orange'),(5,'relaxed gagarin','2020-05-20 11:36:26','2020-06-19 08:13:35',38.948617,3.143000,'green'),(9,'gifted vaughan','2020-04-09 16:19:45','2020-06-19 08:13:35',95.922976,8.708000,'yellow'),(2,'focused taussig','2020-05-17 17:58:34','2020-06-19 08:13:35',4.137803,4.902000,'white'),(6,'fervent yonath','2020-05-26 03:55:25','2020-06-19 08:13:35',72.394272,6.491000,'white'),(18,'mystifying bhaskara','2020-02-19 10:41:48','2020-06-19 08:13:35',10.832397,9.707000,'red'),(4,'goofy saha','2020-03-11 13:24:31','2020-06-19 08:13:35',39.007216,2.446000,'blue'),(20,'mystifying bhaskara','2020-04-03 11:33:27','2020-06-19 08:13:35',85.190386,6.787000,'blue');")
+
+ tk.MustExec("DROP TABLE IF EXISTS t2;")
+ tk.MustExec(`CREATE TABLE t2 (
+ c_int int(11) NOT NULL,
+ c_str varbinary(40) NOT NULL,
+ c_datetime datetime DEFAULT NULL,
+ c_timestamp timestamp NULL DEFAULT NULL,
+ c_double double DEFAULT NULL,
+ c_decimal decimal(12,6) DEFAULT NULL,
+ c_enum enum('blue','green','red','yellow','white','orange','purple') DEFAULT NULL,
+ PRIMARY KEY (c_int,c_str) /*T![clustered_index] CLUSTERED */,
+ KEY c_int_2 (c_int),
+ KEY c_decimal (c_decimal),
+ KEY c_datetime (c_datetime)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin
+PARTITION BY LIST COLUMNS(c_int)
+(PARTITION p0 VALUES IN (1,5,9,13,17,21,25,29,33,37),
+ PARTITION p1 VALUES IN (2,6,10,14,18,22,26,30,34,38),
+ PARTITION p2 VALUES IN (3,7,11,15,19,23,27,31,35,39),
+ PARTITION p3 VALUES IN (4,8,12,16,20,24,28,32,36,40));
+`)
+
+ tk.MustExec("INSERT INTO t2 VALUES (1,'crazy wescoff','2020-03-24 21:51:02','2020-04-01 12:11:56',47.565275,6.313000,'orange'),(1,'unruffled johnson','2020-06-30 03:42:58','2020-06-14 00:16:50',35.444084,1.090000,'red'),(5,'relaxed gagarin','2020-05-20 11:36:26','2020-02-19 12:25:48',38.948617,3.143000,'green'),(9,'eloquent archimedes','2020-02-16 04:20:21','2020-05-23 15:42:33',32.310878,5.855000,'orange'),(9,'gifted vaughan','2020-04-09 16:19:45','2020-05-15 01:42:16',95.922976,8.708000,'yellow'),(13,'dreamy benz','2020-04-27 17:43:44','2020-03-27 06:33:03',39.539233,4.823000,'red'),(3,'bold goldberg','2020-01-07 12:08:19','2020-03-10 18:37:09',0.941002,5.303000,'yellow'),(3,'youthful yonath','2020-01-12 17:10:39','2020-06-10 15:13:44',66.288511,6.046000,'white'),(7,'upbeat bhabha','2020-04-29 01:17:05','2020-03-11 22:58:43',23.316987,9.026000,'yellow'),(11,'quizzical ritchie','2020-05-16 08:21:36','2020-03-05 19:23:25',75.019379,0.260000,'purple'),(2,'dazzling kepler','2020-04-11 04:38:59','2020-05-06 04:42:32',78.798503,2.274000,'purple'),(2,'focused taussig','2020-05-17 17:58:34','2020-02-25 09:11:03',4.137803,4.902000,'white'),(2,'sharp ptolemy',NULL,'2020-05-17 18:04:19',NULL,5.573000,'purple'),(6,'fervent yonath','2020-05-26 03:55:25','2020-05-06 14:23:44',72.394272,6.491000,'white'),(10,'musing wu','2020-04-03 11:33:27','2020-05-24 06:11:56',85.190386,6.787000,'blue'),(8,'hopeful keller','2020-02-19 10:41:48','2020-04-19 17:10:36',10.832397,9.707000,'red'),(12,'exciting boyd',NULL,'2020-03-28 18:27:23',NULL,9.249000,'blue');")
+
+ tk.MustExec("set tidb_txn_assertion_level=strict;")
+ tk.MustExec("begin")
+ tk.MustExec("delete t1, t2 from t1, t2 where t1.c_enum in ('blue');")
+ tk.MustExec("commit")
+}
+
+func TestIssue40158(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1")
+ tk.MustExec("create table t1 (_id int PRIMARY KEY, c1 char, index (c1));")
+ tk.MustExec("insert into t1 values (1, null);")
+ tk.MustQuery("select * from t1 where c1 is null and _id < 1;").Check(testkit.Rows())
+}
+
+func TestIssue40596(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec(`CREATE TABLE t1 (
+ c1 double DEFAULT '1.335088259490289',
+ c2 set('mj','4s7ht','z','3i','b26','9','cg11','uvzcp','c','ns','fl9') NOT NULL DEFAULT 'mj,z,3i,9,cg11,c',
+ PRIMARY KEY (c2) /*T![clustered_index] CLUSTERED */,
+ KEY i1 (c1),
+ KEY i2 (c1),
+ KEY i3 (c1)
+) ENGINE=InnoDB DEFAULT CHARSET=gbk COLLATE=gbk_chinese_ci;`)
+ tk.MustExec("INSERT INTO t1 VALUES (634.2783557491367,''),(2000.5041449792013,'4s7ht'),(634.2783557491367,'3i'),(634.2783557491367,'9'),(7803.173688589342,'uvzcp'),(634.2783557491367,'ns'),(634.2783557491367,'fl9');")
+ tk.MustExec(`CREATE TABLE t2 (
+ c3 decimal(56,16) DEFAULT '931359772706767457132645278260455518957.9866038319986886',
+ c4 set('3bqx','g','6op3','2g','jf','arkd3','y0b','jdy','1g','ff5z','224b') DEFAULT '3bqx,2g,ff5z,224b',
+ c5 smallint(6) NOT NULL DEFAULT '-25973',
+ c6 year(4) DEFAULT '2122',
+ c7 text DEFAULT NULL,
+ PRIMARY KEY (c5) /*T![clustered_index] CLUSTERED */,
+ KEY i4 (c6),
+ KEY i5 (c5)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT=''
+PARTITION BY HASH (c5) PARTITIONS 4;`)
+ tk.MustExec("INSERT INTO t2 VALUES (465.0000000000000000,'jdy',-8542,2008,'FgZXe');")
+ tk.MustExec("set @@sql_mode='';")
+ tk.MustExec("set tidb_partition_prune_mode=dynamic;")
+ tk.MustExec("analyze table t1;")
+ tk.MustExec("analyze table t2;")
+
+ // No nil pointer panic
+ tk.MustQuery("select /*+ inl_join( t1 , t2 ) */ avg( t2.c5 ) as r0 , repeat( t2.c7 , t2.c5 ) as r1 , locate( t2.c7 , t2.c7 ) as r2 , unhex( t1.c1 ) as r3 from t1 right join t2 on t1.c2 = t2.c5 where not( t2.c5 in ( -7860 ,-13384 ,-12940 ) ) and not( t1.c2 between '4s7ht' and 'mj' );").Check(testkit.Rows(" "))
+ // Again, a simpler reproduce.
+ tk.MustQuery("select /*+ inl_join (t1, t2) */ t2.c5 from t1 right join t2 on t1.c2 = t2.c5 where not( t1.c2 between '4s7ht' and 'mj' );").Check(testkit.Rows())
+}
diff --git a/executor/issuetest/main_test.go b/executor/issuetest/main_test.go
new file mode 100644
index 0000000000000..daecbf2f06859
--- /dev/null
+++ b/executor/issuetest/main_test.go
@@ -0,0 +1,45 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package issuetest
+
+import (
+ "testing"
+
+ "github.com/pingcap/tidb/config"
+ "github.com/pingcap/tidb/meta/autoid"
+ "github.com/tikv/client-go/v2/tikv"
+ "go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+ autoid.SetStep(5000)
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.Instance.SlowThreshold = 30000 // 30s
+ conf.TiKVClient.AsyncCommit.SafeWindow = 0
+ conf.TiKVClient.AsyncCommit.AllowedClockDrift = 0
+ conf.Experimental.AllowsExpressionIndex = true
+ })
+ tikv.EnableFailpoints()
+
+ opts := []goleak.Option{
+ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
+ goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
+ goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
+ goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
+ goleak.IgnoreTopFunction("gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun"),
+ goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"),
+ }
+ goleak.VerifyTestMain(m, opts...)
+}
diff --git a/executor/join.go b/executor/join.go
index 95ecee42c02d4..91533be2259cf 100644
--- a/executor/join.go
+++ b/executor/join.go
@@ -46,8 +46,35 @@ var (
_ Executor = &NestedLoopApplyExec{}
)
+type hashJoinCtx struct {
+ sessCtx sessionctx.Context
+ allocPool chunk.Allocator
+ // concurrency is the number of partition, build and join workers.
+ concurrency uint
+ joinResultCh chan *hashjoinWorkerResult
+ // closeCh add a lock for closing executor.
+ closeCh chan struct{}
+ finished atomic.Bool
+ useOuterToBuild bool
+ isOuterJoin bool
+ isNullEQ []bool
+ buildFinished chan error
+ rowContainer *hashRowContainer
+ joinType plannercore.JoinType
+ outerMatchedStatus []*bitmap.ConcurrentBitmap
+ stats *hashJoinRuntimeStats
+ probeTypes []*types.FieldType
+ buildTypes []*types.FieldType
+ outerFilter expression.CNFExprs
+ isNullAware bool
+ memTracker *memory.Tracker // track memory usage.
+ diskTracker *disk.Tracker // track disk usage.
+}
+
// probeSideTupleFetcher reads tuples from probeSideExec and send them to probeWorkers.
type probeSideTupleFetcher struct {
+ *hashJoinCtx
+
probeSideExec Executor
probeChkResourceCh chan *probeChkResource
probeResultChs []chan *chunk.Chunk
@@ -55,6 +82,11 @@ type probeSideTupleFetcher struct {
}
type probeWorker struct {
+ hashJoinCtx *hashJoinCtx
+ workerID uint
+
+ probeKeyColIdx []int
+ probeNAKeyColIdx []int
// We pre-alloc and reuse the Rows and RowPtrs for each probe goroutine, to avoid allocation frequently
buildSideRows []chunk.Row
buildSideRowPtrs []chunk.RowPtr
@@ -67,51 +99,31 @@ type probeWorker struct {
// for every naaj probe worker, pre-allocate the int slice for store the join column index to check.
needCheckBuildRowPos []int
needCheckProbeRowPos []int
+ probeChkResourceCh chan *probeChkResource
+ joinChkResourceCh chan *chunk.Chunk
+ probeResultCh chan *chunk.Chunk
+}
+
+type buildWorker struct {
+ hashJoinCtx *hashJoinCtx
+ buildSideExec Executor
+ buildKeyColIdx []int
+ buildNAKeyColIdx []int
}
// HashJoinExec implements the hash join algorithm.
type HashJoinExec struct {
baseExecutor
+ *hashJoinCtx
- probeSideTupleFetcher
- probeWorkers []probeWorker
- buildSideExec Executor
- buildSideEstCount float64
- outerFilter expression.CNFExprs
- probeKeys []*expression.Column
- probeNAKeys []*expression.Column
- buildKeys []*expression.Column
- buildNAKeys []*expression.Column
- isNullEQ []bool
- probeTypes []*types.FieldType
- buildTypes []*types.FieldType
+ probeSideTupleFetcher *probeSideTupleFetcher
+ probeWorkers []*probeWorker
+ buildWorker *buildWorker
- // concurrency is the number of partition, build and join workers.
- concurrency uint
- rowContainer *hashRowContainer
- buildFinished chan error
-
- // closeCh add a lock for closing executor.
- closeCh chan struct{}
- worker util.WaitGroupWrapper
- waiter util.WaitGroupWrapper
- joinType plannercore.JoinType
+ workerWg util.WaitGroupWrapper
+ waiterWg util.WaitGroupWrapper
- joinChkResourceCh []chan *chunk.Chunk
- joinResultCh chan *hashjoinWorkerResult
-
- memTracker *memory.Tracker // track memory usage.
- diskTracker *disk.Tracker // track disk usage.
-
- outerMatchedStatus []*bitmap.ConcurrentBitmap
- useOuterToBuild bool
-
- prepared bool
- isOuterJoin bool
-
- finished atomic.Bool
-
- stats *hashJoinRuntimeStats
+ prepared bool
}
// probeChkResource stores the result of the join probe side fetch worker,
@@ -152,14 +164,13 @@ func (e *HashJoinExec) Close() error {
for i := range e.probeSideTupleFetcher.probeResultChs {
channel.Clear(e.probeSideTupleFetcher.probeResultChs[i])
}
- for i := range e.joinChkResourceCh {
- close(e.joinChkResourceCh[i])
- channel.Clear(e.joinChkResourceCh[i])
+ for i := range e.probeWorkers {
+ close(e.probeWorkers[i].joinChkResourceCh)
+ channel.Clear(e.probeWorkers[i].joinChkResourceCh)
}
e.probeSideTupleFetcher.probeChkResourceCh = nil
- e.joinChkResourceCh = nil
terror.Call(e.rowContainer.Close)
- e.waiter.Wait()
+ e.waiterWg.Wait()
}
e.outerMatchedStatus = e.outerMatchedStatus[:0]
for _, w := range e.probeWorkers {
@@ -167,11 +178,15 @@ func (e *HashJoinExec) Close() error {
w.buildSideRowPtrs = nil
w.needCheckBuildRowPos = nil
w.needCheckProbeRowPos = nil
+ w.joinChkResourceCh = nil
}
if e.stats != nil && e.rowContainer != nil {
e.stats.hashStat = *e.rowContainer.stat
}
+ if e.stats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
+ }
err := e.baseExecutor.Close()
return err
}
@@ -184,60 +199,53 @@ func (e *HashJoinExec) Open(ctx context.Context) error {
return err
}
e.prepared = false
- e.memTracker = memory.NewTracker(e.id, -1)
- e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker)
+ e.hashJoinCtx.memTracker = memory.NewTracker(e.id, -1)
+ e.hashJoinCtx.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker)
e.diskTracker = disk.NewTracker(e.id, -1)
e.diskTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.DiskTracker)
- e.worker = util.WaitGroupWrapper{}
- e.waiter = util.WaitGroupWrapper{}
+ e.workerWg = util.WaitGroupWrapper{}
+ e.waiterWg = util.WaitGroupWrapper{}
e.closeCh = make(chan struct{})
e.finished.Store(false)
- if e.probeTypes == nil {
- e.probeTypes = retTypes(e.probeSideTupleFetcher.probeSideExec)
- }
- if e.buildTypes == nil {
- e.buildTypes = retTypes(e.buildSideExec)
- }
if e.runtimeStats != nil {
e.stats = &hashJoinRuntimeStats{
concurrent: int(e.concurrency),
}
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
return nil
}
// fetchProbeSideChunks get chunks from fetches chunks from the big table in a background goroutine
// and sends the chunks to multiple channels which will be read by multiple join workers.
-func (e *HashJoinExec) fetchProbeSideChunks(ctx context.Context) {
+func (fetcher *probeSideTupleFetcher) fetchProbeSideChunks(ctx context.Context, maxChunkSize int) {
hasWaitedForBuild := false
for {
- if e.finished.Load() {
+ if fetcher.finished.Load() {
return
}
var probeSideResource *probeChkResource
var ok bool
select {
- case <-e.closeCh:
+ case <-fetcher.closeCh:
return
- case probeSideResource, ok = <-e.probeSideTupleFetcher.probeChkResourceCh:
+ case probeSideResource, ok = <-fetcher.probeChkResourceCh:
if !ok {
return
}
}
probeSideResult := probeSideResource.chk
- if e.isOuterJoin {
- required := int(atomic.LoadInt64(&e.probeSideTupleFetcher.requiredRows))
- probeSideResult.SetRequiredRows(required, e.maxChunkSize)
+ if fetcher.isOuterJoin {
+ required := int(atomic.LoadInt64(&fetcher.requiredRows))
+ probeSideResult.SetRequiredRows(required, maxChunkSize)
}
- err := Next(ctx, e.probeSideTupleFetcher.probeSideExec, probeSideResult)
+ err := Next(ctx, fetcher.probeSideExec, probeSideResult)
failpoint.Inject("ConsumeRandomPanic", nil)
if err != nil {
- e.joinResultCh <- &hashjoinWorkerResult{
+ fetcher.joinResultCh <- &hashjoinWorkerResult{
err: err,
}
return
@@ -248,23 +256,18 @@ func (e *HashJoinExec) fetchProbeSideChunks(ctx context.Context) {
probeSideResult.Reset()
}
})
- if probeSideResult.NumRows() == 0 && !e.useOuterToBuild {
- e.finished.Store(true)
+ if probeSideResult.NumRows() == 0 && !fetcher.useOuterToBuild {
+ fetcher.finished.Store(true)
}
- emptyBuild, buildErr := e.wait4BuildSide()
+ emptyBuild, buildErr := fetcher.wait4BuildSide()
if buildErr != nil {
- e.joinResultCh <- &hashjoinWorkerResult{
+ fetcher.joinResultCh <- &hashjoinWorkerResult{
err: buildErr,
}
return
} else if emptyBuild {
return
}
- // after building is finished. the hash null bucket slice is allocated and determined.
- // copy it for multi probe worker.
- for _, w := range e.probeWorkers {
- w.rowContainerForProbe.hashNANullBucket = e.rowContainer.hashNANullBucket
- }
hasWaitedForBuild = true
}
@@ -276,16 +279,16 @@ func (e *HashJoinExec) fetchProbeSideChunks(ctx context.Context) {
}
}
-func (e *HashJoinExec) wait4BuildSide() (emptyBuild bool, err error) {
+func (fetcher *probeSideTupleFetcher) wait4BuildSide() (emptyBuild bool, err error) {
select {
- case <-e.closeCh:
+ case <-fetcher.closeCh:
return true, nil
- case err := <-e.buildFinished:
+ case err := <-fetcher.buildFinished:
if err != nil {
return false, err
}
}
- if e.rowContainer.Len() == uint64(0) && (e.joinType == plannercore.InnerJoin || e.joinType == plannercore.SemiJoin) {
+ if fetcher.rowContainer.Len() == uint64(0) && (fetcher.joinType == plannercore.InnerJoin || fetcher.joinType == plannercore.SemiJoin) {
return true, nil
}
return false, nil
@@ -293,7 +296,7 @@ func (e *HashJoinExec) wait4BuildSide() (emptyBuild bool, err error) {
// fetchBuildSideRows fetches all rows from build side executor, and append them
// to e.buildSideResult.
-func (e *HashJoinExec) fetchBuildSideRows(ctx context.Context, chkCh chan<- *chunk.Chunk, errCh chan<- error, doneCh <-chan struct{}) {
+func (w *buildWorker) fetchBuildSideRows(ctx context.Context, chkCh chan<- *chunk.Chunk, errCh chan<- error, doneCh <-chan struct{}) {
defer close(chkCh)
var err error
failpoint.Inject("issue30289", func(val failpoint.Value) {
@@ -303,12 +306,13 @@ func (e *HashJoinExec) fetchBuildSideRows(ctx context.Context, chkCh chan<- *chu
return
}
})
+ sessVars := w.hashJoinCtx.sessCtx.GetSessionVars()
for {
- if e.finished.Load() {
+ if w.hashJoinCtx.finished.Load() {
return
}
- chk := e.ctx.GetSessionVars().GetNewChunkWithCapacity(e.buildSideExec.base().retFieldTypes, e.ctx.GetSessionVars().MaxChunkSize, e.ctx.GetSessionVars().MaxChunkSize, e.AllocPool)
- err = Next(ctx, e.buildSideExec, chk)
+ chk := sessVars.GetNewChunkWithCapacity(w.buildSideExec.base().retFieldTypes, sessVars.MaxChunkSize, sessVars.MaxChunkSize, w.hashJoinCtx.allocPool)
+ err = Next(ctx, w.buildSideExec, chk)
if err != nil {
errCh <- errors.Trace(err)
return
@@ -321,7 +325,7 @@ func (e *HashJoinExec) fetchBuildSideRows(ctx context.Context, chkCh chan<- *chu
select {
case <-doneCh:
return
- case <-e.closeCh:
+ case <-w.hashJoinCtx.closeCh:
return
case chkCh <- chk:
}
@@ -329,12 +333,18 @@ func (e *HashJoinExec) fetchBuildSideRows(ctx context.Context, chkCh chan<- *chu
}
func (e *HashJoinExec) initializeForProbe() {
+ // e.joinResultCh is for transmitting the join result chunks to the main
+ // thread.
+ e.joinResultCh = make(chan *hashjoinWorkerResult, e.concurrency+1)
+
+ e.probeSideTupleFetcher.hashJoinCtx = e.hashJoinCtx
// e.probeSideTupleFetcher.probeResultChs is for transmitting the chunks which store the data of
// probeSideExec, it'll be written by probe side worker goroutine, and read by join
// workers.
e.probeSideTupleFetcher.probeResultChs = make([]chan *chunk.Chunk, e.concurrency)
for i := uint(0); i < e.concurrency; i++ {
e.probeSideTupleFetcher.probeResultChs[i] = make(chan *chunk.Chunk, 1)
+ e.probeWorkers[i].probeResultCh = e.probeSideTupleFetcher.probeResultChs[i]
}
// e.probeChkResourceCh is for transmitting the used probeSideExec chunks from
@@ -347,51 +357,44 @@ func (e *HashJoinExec) initializeForProbe() {
}
}
- // e.joinChkResourceCh is for transmitting the reused join result chunks
- // from the main thread to join worker goroutines.
- e.joinChkResourceCh = make([]chan *chunk.Chunk, e.concurrency)
+ // e.probeWorker.joinChkResourceCh is for transmitting the reused join result chunks
+ // from the main thread to probe worker goroutines.
for i := uint(0); i < e.concurrency; i++ {
- e.joinChkResourceCh[i] = make(chan *chunk.Chunk, 1)
- e.joinChkResourceCh[i] <- newFirstChunk(e)
+ e.probeWorkers[i].joinChkResourceCh = make(chan *chunk.Chunk, 1)
+ e.probeWorkers[i].joinChkResourceCh <- newFirstChunk(e)
+ e.probeWorkers[i].probeChkResourceCh = e.probeSideTupleFetcher.probeChkResourceCh
}
-
- // e.joinResultCh is for transmitting the join result chunks to the main
- // thread.
- e.joinResultCh = make(chan *hashjoinWorkerResult, e.concurrency+1)
}
func (e *HashJoinExec) fetchAndProbeHashTable(ctx context.Context) {
e.initializeForProbe()
- e.worker.RunWithRecover(func() {
+ e.workerWg.RunWithRecover(func() {
defer trace.StartRegion(ctx, "HashJoinProbeSideFetcher").End()
- e.fetchProbeSideChunks(ctx)
- }, e.handleProbeSideFetcherPanic)
-
- probeKeyColIdx := make([]int, len(e.probeKeys))
- probeNAKeColIdx := make([]int, len(e.probeNAKeys))
- for i := range e.probeKeys {
- probeKeyColIdx[i] = e.probeKeys[i].Index
- }
- for i := range e.probeNAKeys {
- probeNAKeColIdx[i] = e.probeNAKeys[i].Index
- }
+ e.probeSideTupleFetcher.fetchProbeSideChunks(ctx, e.maxChunkSize)
+ }, e.probeSideTupleFetcher.handleProbeSideFetcherPanic)
for i := uint(0); i < e.concurrency; i++ {
- workID := i
- e.worker.RunWithRecover(func() {
+ workerID := i
+ e.workerWg.RunWithRecover(func() {
defer trace.StartRegion(ctx, "HashJoinWorker").End()
- e.runJoinWorker(workID, probeKeyColIdx, probeNAKeColIdx)
- }, e.handleJoinWorkerPanic)
+ e.probeWorkers[workerID].runJoinWorker()
+ }, e.probeWorkers[workerID].handleProbeWorkerPanic)
}
- e.waiter.RunWithRecover(e.waitJoinWorkersAndCloseResultChan, nil)
+ e.waiterWg.RunWithRecover(e.waitJoinWorkersAndCloseResultChan, nil)
}
-func (e *HashJoinExec) handleProbeSideFetcherPanic(r interface{}) {
- for i := range e.probeSideTupleFetcher.probeResultChs {
- close(e.probeSideTupleFetcher.probeResultChs[i])
+func (fetcher *probeSideTupleFetcher) handleProbeSideFetcherPanic(r interface{}) {
+ for i := range fetcher.probeResultChs {
+ close(fetcher.probeResultChs[i])
}
if r != nil {
- e.joinResultCh <- &hashjoinWorkerResult{err: errors.Errorf("%v", r)}
+ fetcher.joinResultCh <- &hashjoinWorkerResult{err: errors.Errorf("%v", r)}
+ }
+}
+
+func (w *probeWorker) handleProbeWorkerPanic(r interface{}) {
+ if r != nil {
+ w.hashJoinCtx.joinResultCh <- &hashjoinWorkerResult{err: errors.Errorf("probeWorker[%d] meets error: %v", w.workerID, r)}
}
}
@@ -402,27 +405,27 @@ func (e *HashJoinExec) handleJoinWorkerPanic(r interface{}) {
}
// Concurrently handling unmatched rows from the hash table
-func (e *HashJoinExec) handleUnmatchedRowsFromHashTable(workerID uint) {
- ok, joinResult := e.getNewJoinResult(workerID)
+func (w *probeWorker) handleUnmatchedRowsFromHashTable() {
+ ok, joinResult := w.getNewJoinResult()
if !ok {
return
}
- numChks := e.rowContainer.NumChunks()
- for i := int(workerID); i < numChks; i += int(e.concurrency) {
- chk, err := e.rowContainer.GetChunk(i)
+ numChks := w.rowContainerForProbe.NumChunks()
+ for i := int(w.workerID); i < numChks; i += int(w.hashJoinCtx.concurrency) {
+ chk, err := w.rowContainerForProbe.GetChunk(i)
if err != nil {
// Catching the error and send it
joinResult.err = err
- e.joinResultCh <- joinResult
+ w.hashJoinCtx.joinResultCh <- joinResult
return
}
for j := 0; j < chk.NumRows(); j++ {
- if !e.outerMatchedStatus[i].UnsafeIsSet(j) { // process unmatched outer rows
- e.probeWorkers[workerID].joiner.onMissMatch(false, chk.GetRow(j), joinResult.chk)
+ if !w.hashJoinCtx.outerMatchedStatus[i].UnsafeIsSet(j) { // process unmatched outer rows
+ w.joiner.onMissMatch(false, chk.GetRow(j), joinResult.chk)
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return
}
@@ -433,32 +436,32 @@ func (e *HashJoinExec) handleUnmatchedRowsFromHashTable(workerID uint) {
if joinResult == nil {
return
} else if joinResult.err != nil || (joinResult.chk != nil && joinResult.chk.NumRows() > 0) {
- e.joinResultCh <- joinResult
+ w.hashJoinCtx.joinResultCh <- joinResult
}
}
func (e *HashJoinExec) waitJoinWorkersAndCloseResultChan() {
- e.worker.Wait()
+ e.workerWg.Wait()
if e.useOuterToBuild {
// Concurrently handling unmatched rows from the hash table at the tail
for i := uint(0); i < e.concurrency; i++ {
var workerID = i
- e.worker.RunWithRecover(func() { e.handleUnmatchedRowsFromHashTable(workerID) }, e.handleJoinWorkerPanic)
+ e.workerWg.RunWithRecover(func() { e.probeWorkers[workerID].handleUnmatchedRowsFromHashTable() }, e.handleJoinWorkerPanic)
}
- e.worker.Wait()
+ e.workerWg.Wait()
}
close(e.joinResultCh)
}
-func (e *HashJoinExec) runJoinWorker(workerID uint, probeKeyColIdx, probeNAKeyColIdx []int) {
+func (w *probeWorker) runJoinWorker() {
probeTime := int64(0)
- if e.stats != nil {
+ if w.hashJoinCtx.stats != nil {
start := time.Now()
defer func() {
t := time.Since(start)
- atomic.AddInt64(&e.stats.probe, probeTime)
- atomic.AddInt64(&e.stats.fetchAndProbe, int64(t))
- e.stats.setMaxFetchAndProbeTime(int64(t))
+ atomic.AddInt64(&w.hashJoinCtx.stats.probe, probeTime)
+ atomic.AddInt64(&w.hashJoinCtx.stats.fetchAndProbe, int64(t))
+ w.hashJoinCtx.stats.setMaxFetchAndProbeTime(int64(t))
}()
}
@@ -466,38 +469,38 @@ func (e *HashJoinExec) runJoinWorker(workerID uint, probeKeyColIdx, probeNAKeyCo
probeSideResult *chunk.Chunk
selected = make([]bool, 0, chunk.InitialCapacity)
)
- ok, joinResult := e.getNewJoinResult(workerID)
+ ok, joinResult := w.getNewJoinResult()
if !ok {
return
}
// Read and filter probeSideResult, and join the probeSideResult with the build side rows.
emptyProbeSideResult := &probeChkResource{
- dest: e.probeSideTupleFetcher.probeResultChs[workerID],
+ dest: w.probeResultCh,
}
hCtx := &hashContext{
- allTypes: e.probeTypes,
- keyColIdx: probeKeyColIdx,
- naKeyColIdx: probeNAKeyColIdx,
+ allTypes: w.hashJoinCtx.probeTypes,
+ keyColIdx: w.probeKeyColIdx,
+ naKeyColIdx: w.probeNAKeyColIdx,
}
for ok := true; ok; {
- if e.finished.Load() {
+ if w.hashJoinCtx.finished.Load() {
break
}
select {
- case <-e.closeCh:
+ case <-w.hashJoinCtx.closeCh:
return
- case probeSideResult, ok = <-e.probeSideTupleFetcher.probeResultChs[workerID]:
+ case probeSideResult, ok = <-w.probeResultCh:
}
failpoint.Inject("ConsumeRandomPanic", nil)
if !ok {
break
}
start := time.Now()
- if e.useOuterToBuild {
- ok, joinResult = e.join2ChunkForOuterHashJoin(workerID, probeSideResult, hCtx, e.probeWorkers[workerID].rowContainerForProbe, joinResult)
+ if w.hashJoinCtx.useOuterToBuild {
+ ok, joinResult = w.join2ChunkForOuterHashJoin(probeSideResult, hCtx, joinResult)
} else {
- ok, joinResult = e.join2Chunk(workerID, probeSideResult, hCtx, e.probeWorkers[workerID].rowContainerForProbe, joinResult, selected)
+ ok, joinResult = w.join2Chunk(probeSideResult, hCtx, joinResult, selected)
}
probeTime += int64(time.Since(start))
if !ok {
@@ -505,22 +508,22 @@ func (e *HashJoinExec) runJoinWorker(workerID uint, probeKeyColIdx, probeNAKeyCo
}
probeSideResult.Reset()
emptyProbeSideResult.chk = probeSideResult
- e.probeSideTupleFetcher.probeChkResourceCh <- emptyProbeSideResult
+ w.probeChkResourceCh <- emptyProbeSideResult
}
// note joinResult.chk may be nil when getNewJoinResult fails in loops
if joinResult == nil {
return
} else if joinResult.err != nil || (joinResult.chk != nil && joinResult.chk.NumRows() > 0) {
- e.joinResultCh <- joinResult
+ w.hashJoinCtx.joinResultCh <- joinResult
} else if joinResult.chk != nil && joinResult.chk.NumRows() == 0 {
- e.joinChkResourceCh[workerID] <- joinResult.chk
+ w.joinChkResourceCh <- joinResult.chk
}
}
-func (e *HashJoinExec) joinMatchedProbeSideRow2ChunkForOuterHashJoin(workerID uint, probeKey uint64, probeSideRow chunk.Row, hCtx *hashContext, rowContainer *hashRowContainer, joinResult *hashjoinWorkerResult) (bool, *hashjoinWorkerResult) {
+func (w *probeWorker) joinMatchedProbeSideRow2ChunkForOuterHashJoin(probeKey uint64, probeSideRow chunk.Row, hCtx *hashContext, joinResult *hashjoinWorkerResult) (bool, *hashjoinWorkerResult) {
var err error
- e.probeWorkers[workerID].buildSideRows, e.probeWorkers[workerID].buildSideRowPtrs, err = rowContainer.GetMatchedRowsAndPtrs(probeKey, probeSideRow, hCtx, e.probeWorkers[workerID].buildSideRows, e.probeWorkers[workerID].buildSideRowPtrs, true)
- buildSideRows, rowsPtrs := e.probeWorkers[workerID].buildSideRows, e.probeWorkers[workerID].buildSideRowPtrs
+ w.buildSideRows, w.buildSideRowPtrs, err = w.rowContainerForProbe.GetMatchedRowsAndPtrs(probeKey, probeSideRow, hCtx, w.buildSideRows, w.buildSideRowPtrs, true)
+ buildSideRows, rowsPtrs := w.buildSideRows, w.buildSideRowPtrs
if err != nil {
joinResult.err = err
return false, joinResult
@@ -529,25 +532,25 @@ func (e *HashJoinExec) joinMatchedProbeSideRow2ChunkForOuterHashJoin(workerID ui
return true, joinResult
}
- iter := e.probeWorkers[workerID].rowIters
+ iter := w.rowIters
iter.Reset(buildSideRows)
var outerMatchStatus []outerRowStatusFlag
rowIdx, ok := 0, false
for iter.Begin(); iter.Current() != iter.End(); {
- outerMatchStatus, err = e.probeWorkers[workerID].joiner.tryToMatchOuters(iter, probeSideRow, joinResult.chk, outerMatchStatus)
+ outerMatchStatus, err = w.joiner.tryToMatchOuters(iter, probeSideRow, joinResult.chk, outerMatchStatus)
if err != nil {
joinResult.err = err
return false, joinResult
}
for i := range outerMatchStatus {
if outerMatchStatus[i] == outerRowMatched {
- e.outerMatchedStatus[rowsPtrs[rowIdx+i].ChkIdx].Set(int(rowsPtrs[rowIdx+i].RowIdx))
+ w.hashJoinCtx.outerMatchedStatus[rowsPtrs[rowIdx+i].ChkIdx].Set(int(rowsPtrs[rowIdx+i].RowIdx))
}
}
rowIdx += len(outerMatchStatus)
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -557,8 +560,7 @@ func (e *HashJoinExec) joinMatchedProbeSideRow2ChunkForOuterHashJoin(workerID ui
}
// joinNAALOSJMatchProbeSideRow2Chunk implement the matching logic for NA-AntiLeftOuterSemiJoin
-func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKey uint64, probeKeyNullBits *bitmap.ConcurrentBitmap, probeSideRow chunk.Row, hCtx *hashContext,
- rowContainer *hashRowContainer, joinResult *hashjoinWorkerResult) (bool, *hashjoinWorkerResult) {
+func (w *probeWorker) joinNAALOSJMatchProbeSideRow2Chunk(probeKey uint64, probeKeyNullBits *bitmap.ConcurrentBitmap, probeSideRow chunk.Row, hCtx *hashContext, joinResult *hashjoinWorkerResult) (bool, *hashjoinWorkerResult) {
var (
err error
ok bool
@@ -568,17 +570,17 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
// because AntiLeftOuterSemiJoin cares about the scalar value. If we both have a match from null
// bucket and same key bucket, we should return the result as from same-key bucket
// rather than from null bucket.
- e.probeWorkers[workerID].buildSideRows, err = rowContainer.GetMatchedRows(probeKey, probeSideRow, hCtx, e.probeWorkers[workerID].buildSideRows)
- buildSideRows := e.probeWorkers[workerID].buildSideRows
+ w.buildSideRows, err = w.rowContainerForProbe.GetMatchedRows(probeKey, probeSideRow, hCtx, w.buildSideRows)
+ buildSideRows := w.buildSideRows
if err != nil {
joinResult.err = err
return false, joinResult
}
if len(buildSideRows) != 0 {
- iter1 := e.probeWorkers[workerID].rowIters
+ iter1 := w.rowIters
iter1.Reset(buildSideRows)
for iter1.Begin(); iter1.Current() != iter1.End(); {
- matched, _, err := e.probeWorkers[workerID].joiner.tryToMatchInners(probeSideRow, iter1, joinResult.chk, LeftNotNullRightNotNull)
+ matched, _, err := w.joiner.tryToMatchInners(probeSideRow, iter1, joinResult.chk, LeftNotNullRightNotNull)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -589,8 +591,8 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
return true, joinResult
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -598,8 +600,8 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
}
}
// step2: match the null bucket secondly.
- e.probeWorkers[workerID].buildSideRows, err = rowContainer.GetNullBucketRows(hCtx, probeSideRow, probeKeyNullBits, e.probeWorkers[workerID].buildSideRows, e.probeWorkers[workerID].needCheckBuildRowPos, e.probeWorkers[workerID].needCheckProbeRowPos)
- buildSideRows = e.probeWorkers[workerID].buildSideRows
+ w.buildSideRows, err = w.rowContainerForProbe.GetNullBucketRows(hCtx, probeSideRow, probeKeyNullBits, w.buildSideRows, w.needCheckBuildRowPos, w.needCheckProbeRowPos)
+ buildSideRows = w.buildSideRows
if err != nil {
joinResult.err = err
return false, joinResult
@@ -607,13 +609,13 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
if len(buildSideRows) == 0 {
// when reach here, it means we couldn't find a valid same key match from same-key bucket yet
// and the null bucket is empty. so the result should be .
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideRow, joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideRow, joinResult.chk)
return true, joinResult
}
- iter2 := e.probeWorkers[workerID].rowIters
+ iter2 := w.rowIters
iter2.Reset(buildSideRows)
for iter2.Begin(); iter2.Current() != iter2.End(); {
- matched, _, err := e.probeWorkers[workerID].joiner.tryToMatchInners(probeSideRow, iter2, joinResult.chk, LeftNotNullRightHasNull)
+ matched, _, err := w.joiner.tryToMatchInners(probeSideRow, iter2, joinResult.chk, LeftNotNullRightHasNull)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -624,8 +626,8 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
return true, joinResult
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -635,7 +637,7 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
// case1: x NOT IN (empty set): if other key bucket don't have the valid rows yet.
// case2: x NOT IN (l,m,n...): if other key bucket do have the valid rows.
// both cases mean the result should be
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideRow, joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideRow, joinResult.chk)
return true, joinResult
}
// when left side has null values, all we want is to find a valid build side rows (past other condition)
@@ -643,17 +645,17 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
// case1: NOT IN (empty set): ----------------------> result is .
// case2: NOT IN (at least a valid inner row) ------------------> result is .
// Step1: match null bucket (assumption that null bucket is quite smaller than all hash table bucket rows)
- e.probeWorkers[workerID].buildSideRows, err = rowContainer.GetNullBucketRows(hCtx, probeSideRow, probeKeyNullBits, e.probeWorkers[workerID].buildSideRows, e.probeWorkers[workerID].needCheckBuildRowPos, e.probeWorkers[workerID].needCheckProbeRowPos)
- buildSideRows := e.probeWorkers[workerID].buildSideRows
+ w.buildSideRows, err = w.rowContainerForProbe.GetNullBucketRows(hCtx, probeSideRow, probeKeyNullBits, w.buildSideRows, w.needCheckBuildRowPos, w.needCheckProbeRowPos)
+ buildSideRows := w.buildSideRows
if err != nil {
joinResult.err = err
return false, joinResult
}
if len(buildSideRows) != 0 {
- iter1 := e.probeWorkers[workerID].rowIters
+ iter1 := w.rowIters
iter1.Reset(buildSideRows)
for iter1.Begin(); iter1.Current() != iter1.End(); {
- matched, _, err := e.probeWorkers[workerID].joiner.tryToMatchInners(probeSideRow, iter1, joinResult.chk, LeftHasNullRightHasNull)
+ matched, _, err := w.joiner.tryToMatchInners(probeSideRow, iter1, joinResult.chk, LeftHasNullRightHasNull)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -664,8 +666,8 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
return true, joinResult
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -673,8 +675,8 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
}
}
// Step2: match all hash table bucket build rows (use probeKeyNullBits to filter if any).
- e.probeWorkers[workerID].buildSideRows, err = rowContainer.GetAllMatchedRows(hCtx, probeSideRow, probeKeyNullBits, e.probeWorkers[workerID].buildSideRows, e.probeWorkers[workerID].needCheckBuildRowPos, e.probeWorkers[workerID].needCheckProbeRowPos)
- buildSideRows = e.probeWorkers[workerID].buildSideRows
+ w.buildSideRows, err = w.rowContainerForProbe.GetAllMatchedRows(hCtx, probeSideRow, probeKeyNullBits, w.buildSideRows, w.needCheckBuildRowPos, w.needCheckProbeRowPos)
+ buildSideRows = w.buildSideRows
if err != nil {
joinResult.err = err
return false, joinResult
@@ -682,13 +684,13 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
if len(buildSideRows) == 0 {
// when reach here, it means we couldn't return it quickly in null bucket, and same-bucket is empty,
// which means x NOT IN (empty set) or x NOT IN (l,m,n), the result should be
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideRow, joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideRow, joinResult.chk)
return true, joinResult
}
- iter2 := e.probeWorkers[workerID].rowIters
+ iter2 := w.rowIters
iter2.Reset(buildSideRows)
for iter2.Begin(); iter2.Current() != iter2.End(); {
- matched, _, err := e.probeWorkers[workerID].joiner.tryToMatchInners(probeSideRow, iter2, joinResult.chk, LeftHasNullRightNotNull)
+ matched, _, err := w.joiner.tryToMatchInners(probeSideRow, iter2, joinResult.chk, LeftHasNullRightNotNull)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -699,8 +701,8 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
return true, joinResult
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -709,13 +711,12 @@ func (e *HashJoinExec) joinNAALOSJMatchProbeSideRow2Chunk(workerID uint, probeKe
// step3: if we couldn't return it quickly in null bucket and all hash bucket, here means only one cases:
// case1: NOT IN (empty set):
// empty set comes from no rows from all bucket can pass other condition. the result should be
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideRow, joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideRow, joinResult.chk)
return true, joinResult
}
// joinNAASJMatchProbeSideRow2Chunk implement the matching logic for NA-AntiSemiJoin
-func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey uint64, probeKeyNullBits *bitmap.ConcurrentBitmap, probeSideRow chunk.Row, hCtx *hashContext,
- rowContainer *hashRowContainer, joinResult *hashjoinWorkerResult) (bool, *hashjoinWorkerResult) {
+func (w *probeWorker) joinNAASJMatchProbeSideRow2Chunk(probeKey uint64, probeKeyNullBits *bitmap.ConcurrentBitmap, probeSideRow chunk.Row, hCtx *hashContext, joinResult *hashjoinWorkerResult) (bool, *hashjoinWorkerResult) {
var (
err error
ok bool
@@ -723,17 +724,17 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
if probeKeyNullBits == nil {
// step1: match null bucket first.
// need fetch the "valid" rows every time. (nullBits map check is necessary)
- e.probeWorkers[workerID].buildSideRows, err = rowContainer.GetNullBucketRows(hCtx, probeSideRow, probeKeyNullBits, e.probeWorkers[workerID].buildSideRows, e.probeWorkers[workerID].needCheckBuildRowPos, e.probeWorkers[workerID].needCheckProbeRowPos)
- buildSideRows := e.probeWorkers[workerID].buildSideRows
+ w.buildSideRows, err = w.rowContainerForProbe.GetNullBucketRows(hCtx, probeSideRow, probeKeyNullBits, w.buildSideRows, w.needCheckBuildRowPos, w.needCheckProbeRowPos)
+ buildSideRows := w.buildSideRows
if err != nil {
joinResult.err = err
return false, joinResult
}
if len(buildSideRows) != 0 {
- iter1 := e.probeWorkers[workerID].rowIters
+ iter1 := w.rowIters
iter1.Reset(buildSideRows)
for iter1.Begin(); iter1.Current() != iter1.End(); {
- matched, _, err := e.probeWorkers[workerID].joiner.tryToMatchInners(probeSideRow, iter1, joinResult.chk)
+ matched, _, err := w.joiner.tryToMatchInners(probeSideRow, iter1, joinResult.chk)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -744,8 +745,8 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
return true, joinResult
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -753,8 +754,8 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
}
}
// step2: then same key bucket.
- e.probeWorkers[workerID].buildSideRows, err = rowContainer.GetMatchedRows(probeKey, probeSideRow, hCtx, e.probeWorkers[workerID].buildSideRows)
- buildSideRows = e.probeWorkers[workerID].buildSideRows
+ w.buildSideRows, err = w.rowContainerForProbe.GetMatchedRows(probeKey, probeSideRow, hCtx, w.buildSideRows)
+ buildSideRows = w.buildSideRows
if err != nil {
joinResult.err = err
return false, joinResult
@@ -762,13 +763,13 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
if len(buildSideRows) == 0 {
// when reach here, it means we couldn't return it quickly in null bucket, and same-bucket is empty,
// which means x NOT IN (empty set), accept the rhs row.
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideRow, joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideRow, joinResult.chk)
return true, joinResult
}
- iter2 := e.probeWorkers[workerID].rowIters
+ iter2 := w.rowIters
iter2.Reset(buildSideRows)
for iter2.Begin(); iter2.Current() != iter2.End(); {
- matched, _, err := e.probeWorkers[workerID].joiner.tryToMatchInners(probeSideRow, iter2, joinResult.chk)
+ matched, _, err := w.joiner.tryToMatchInners(probeSideRow, iter2, joinResult.chk)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -779,8 +780,8 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
return true, joinResult
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -790,7 +791,7 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
// case1: x NOT IN (empty set): if other key bucket don't have the valid rows yet.
// case2: x NOT IN (l,m,n...): if other key bucket do have the valid rows.
// both cases should accept the rhs row.
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideRow, joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideRow, joinResult.chk)
return true, joinResult
}
// when left side has null values, all we want is to find a valid build side rows (passed from other condition)
@@ -798,17 +799,17 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
// case1: NOT IN (empty set): ----------------------> accept rhs row.
// case2: NOT IN (at least a valid inner row) ------------------> unknown result, refuse rhs row.
// Step1: match null bucket (assumption that null bucket is quite smaller than all hash table bucket rows)
- e.probeWorkers[workerID].buildSideRows, err = rowContainer.GetNullBucketRows(hCtx, probeSideRow, probeKeyNullBits, e.probeWorkers[workerID].buildSideRows, e.probeWorkers[workerID].needCheckBuildRowPos, e.probeWorkers[workerID].needCheckProbeRowPos)
- buildSideRows := e.probeWorkers[workerID].buildSideRows
+ w.buildSideRows, err = w.rowContainerForProbe.GetNullBucketRows(hCtx, probeSideRow, probeKeyNullBits, w.buildSideRows, w.needCheckBuildRowPos, w.needCheckProbeRowPos)
+ buildSideRows := w.buildSideRows
if err != nil {
joinResult.err = err
return false, joinResult
}
if len(buildSideRows) != 0 {
- iter1 := e.probeWorkers[workerID].rowIters
+ iter1 := w.rowIters
iter1.Reset(buildSideRows)
for iter1.Begin(); iter1.Current() != iter1.End(); {
- matched, _, err := e.probeWorkers[workerID].joiner.tryToMatchInners(probeSideRow, iter1, joinResult.chk)
+ matched, _, err := w.joiner.tryToMatchInners(probeSideRow, iter1, joinResult.chk)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -819,8 +820,8 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
return true, joinResult
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -828,8 +829,8 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
}
}
// Step2: match all hash table bucket build rows.
- e.probeWorkers[workerID].buildSideRows, err = rowContainer.GetAllMatchedRows(hCtx, probeSideRow, probeKeyNullBits, e.probeWorkers[workerID].buildSideRows, e.probeWorkers[workerID].needCheckBuildRowPos, e.probeWorkers[workerID].needCheckProbeRowPos)
- buildSideRows = e.probeWorkers[workerID].buildSideRows
+ w.buildSideRows, err = w.rowContainerForProbe.GetAllMatchedRows(hCtx, probeSideRow, probeKeyNullBits, w.buildSideRows, w.needCheckBuildRowPos, w.needCheckProbeRowPos)
+ buildSideRows = w.buildSideRows
if err != nil {
joinResult.err = err
return false, joinResult
@@ -837,13 +838,13 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
if len(buildSideRows) == 0 {
// when reach here, it means we couldn't return it quickly in null bucket, and same-bucket is empty,
// which means NOT IN (empty set) or NOT IN (no valid rows) accept the rhs row.
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideRow, joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideRow, joinResult.chk)
return true, joinResult
}
- iter2 := e.probeWorkers[workerID].rowIters
+ iter2 := w.rowIters
iter2.Reset(buildSideRows)
for iter2.Begin(); iter2.Current() != iter2.End(); {
- matched, _, err := e.probeWorkers[workerID].joiner.tryToMatchInners(probeSideRow, iter2, joinResult.chk)
+ matched, _, err := w.joiner.tryToMatchInners(probeSideRow, iter2, joinResult.chk)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -854,8 +855,8 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
return true, joinResult
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -864,7 +865,7 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
// step3: if we couldn't return it quickly in null bucket and all hash bucket, here means only one cases:
// case1: NOT IN (empty set):
// empty set comes from no rows from all bucket can pass other condition. we should accept the rhs row.
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideRow, joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideRow, joinResult.chk)
return true, joinResult
}
@@ -887,38 +888,37 @@ func (e *HashJoinExec) joinNAASJMatchProbeSideRow2Chunk(workerID uint, probeKey
//
// For NA-AntiLeftOuterSemiJoin, we couldn't match null-bucket first, because once y set has a same key x and null
// key, we should return the result as left side row appended with a scalar value 0 which is from same key matching failure.
-func (e *HashJoinExec) joinNAAJMatchProbeSideRow2Chunk(workerID uint, probeKey uint64, probeKeyNullBits *bitmap.ConcurrentBitmap, probeSideRow chunk.Row, hCtx *hashContext,
- rowContainer *hashRowContainer, joinResult *hashjoinWorkerResult) (bool, *hashjoinWorkerResult) {
- NAAntiSemiJoin := e.joinType == plannercore.AntiSemiJoin && len(e.buildNAKeys) > 0
- NAAntiLeftOuterSemiJoin := e.joinType == plannercore.AntiLeftOuterSemiJoin && len(e.buildNAKeys) > 0
+func (w *probeWorker) joinNAAJMatchProbeSideRow2Chunk(probeKey uint64, probeKeyNullBits *bitmap.ConcurrentBitmap, probeSideRow chunk.Row, hCtx *hashContext, joinResult *hashjoinWorkerResult) (bool, *hashjoinWorkerResult) {
+ NAAntiSemiJoin := w.hashJoinCtx.joinType == plannercore.AntiSemiJoin && w.hashJoinCtx.isNullAware
+ NAAntiLeftOuterSemiJoin := w.hashJoinCtx.joinType == plannercore.AntiLeftOuterSemiJoin && w.hashJoinCtx.isNullAware
if NAAntiSemiJoin {
- return e.joinNAASJMatchProbeSideRow2Chunk(workerID, probeKey, probeKeyNullBits, probeSideRow, hCtx, rowContainer, joinResult)
+ return w.joinNAASJMatchProbeSideRow2Chunk(probeKey, probeKeyNullBits, probeSideRow, hCtx, joinResult)
}
if NAAntiLeftOuterSemiJoin {
- return e.joinNAALOSJMatchProbeSideRow2Chunk(workerID, probeKey, probeKeyNullBits, probeSideRow, hCtx, rowContainer, joinResult)
+ return w.joinNAALOSJMatchProbeSideRow2Chunk(probeKey, probeKeyNullBits, probeSideRow, hCtx, joinResult)
}
// shouldn't be here, not a valid NAAJ.
return false, joinResult
}
-func (e *HashJoinExec) joinMatchedProbeSideRow2Chunk(workerID uint, probeKey uint64, probeSideRow chunk.Row, hCtx *hashContext,
- rowContainer *hashRowContainer, joinResult *hashjoinWorkerResult) (bool, *hashjoinWorkerResult) {
+func (w *probeWorker) joinMatchedProbeSideRow2Chunk(probeKey uint64, probeSideRow chunk.Row, hCtx *hashContext,
+ joinResult *hashjoinWorkerResult) (bool, *hashjoinWorkerResult) {
var err error
- e.probeWorkers[workerID].buildSideRows, err = rowContainer.GetMatchedRows(probeKey, probeSideRow, hCtx, e.probeWorkers[workerID].buildSideRows)
- buildSideRows := e.probeWorkers[workerID].buildSideRows
+ w.buildSideRows, err = w.rowContainerForProbe.GetMatchedRows(probeKey, probeSideRow, hCtx, w.buildSideRows)
+ buildSideRows := w.buildSideRows
if err != nil {
joinResult.err = err
return false, joinResult
}
if len(buildSideRows) == 0 {
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideRow, joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideRow, joinResult.chk)
return true, joinResult
}
- iter := e.probeWorkers[workerID].rowIters
+ iter := w.rowIters
iter.Reset(buildSideRows)
hasMatch, hasNull, ok := false, false, false
for iter.Begin(); iter.Current() != iter.End(); {
- matched, isNull, err := e.probeWorkers[workerID].joiner.tryToMatchInners(probeSideRow, iter, joinResult.chk)
+ matched, isNull, err := w.joiner.tryToMatchInners(probeSideRow, iter, joinResult.chk)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -927,36 +927,36 @@ func (e *HashJoinExec) joinMatchedProbeSideRow2Chunk(workerID uint, probeKey uin
hasNull = hasNull || isNull
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
}
}
if !hasMatch {
- e.probeWorkers[workerID].joiner.onMissMatch(hasNull, probeSideRow, joinResult.chk)
+ w.joiner.onMissMatch(hasNull, probeSideRow, joinResult.chk)
}
return true, joinResult
}
-func (e *HashJoinExec) getNewJoinResult(workerID uint) (bool, *hashjoinWorkerResult) {
+func (w *probeWorker) getNewJoinResult() (bool, *hashjoinWorkerResult) {
joinResult := &hashjoinWorkerResult{
- src: e.joinChkResourceCh[workerID],
+ src: w.joinChkResourceCh,
}
ok := true
select {
- case <-e.closeCh:
+ case <-w.hashJoinCtx.closeCh:
ok = false
- case joinResult.chk, ok = <-e.joinChkResourceCh[workerID]:
+ case joinResult.chk, ok = <-w.joinChkResourceCh:
}
return ok, joinResult
}
-func (e *HashJoinExec) join2Chunk(workerID uint, probeSideChk *chunk.Chunk, hCtx *hashContext, rowContainer *hashRowContainer, joinResult *hashjoinWorkerResult,
+func (w *probeWorker) join2Chunk(probeSideChk *chunk.Chunk, hCtx *hashContext, joinResult *hashjoinWorkerResult,
selected []bool) (ok bool, _ *hashjoinWorkerResult) {
var err error
- selected, err = expression.VectorizedFilter(e.ctx, e.outerFilter, chunk.NewIterator4Chunk(probeSideChk), selected)
+ selected, err = expression.VectorizedFilter(w.hashJoinCtx.sessCtx, w.hashJoinCtx.outerFilter, chunk.NewIterator4Chunk(probeSideChk), selected)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -967,8 +967,8 @@ func (e *HashJoinExec) join2Chunk(workerID uint, probeSideChk *chunk.Chunk, hCtx
// By now, path 1 and 2 won't be conducted at the same time.
// 1: write the row data of join key to hashVals. (normal EQ key should ignore the null values.) null-EQ for Except statement is an exception.
for keyIdx, i := range hCtx.keyColIdx {
- ignoreNull := len(e.isNullEQ) > keyIdx && e.isNullEQ[keyIdx]
- err = codec.HashChunkSelected(rowContainer.sc, hCtx.hashVals, probeSideChk, hCtx.allTypes[keyIdx], i, hCtx.buf, hCtx.hasNull, selected, ignoreNull)
+ ignoreNull := len(w.hashJoinCtx.isNullEQ) > keyIdx && w.hashJoinCtx.isNullEQ[keyIdx]
+ err = codec.HashChunkSelected(w.rowContainerForProbe.sc, hCtx.hashVals, probeSideChk, hCtx.allTypes[keyIdx], i, hCtx.buf, hCtx.hasNull, selected, ignoreNull)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -978,7 +978,7 @@ func (e *HashJoinExec) join2Chunk(workerID uint, probeSideChk *chunk.Chunk, hCtx
isNAAJ := len(hCtx.naKeyColIdx) > 0
for keyIdx, i := range hCtx.naKeyColIdx {
// NAAJ won't ignore any null values, but collect them up to probe.
- err = codec.HashChunkSelected(rowContainer.sc, hCtx.hashVals, probeSideChk, hCtx.allTypes[keyIdx], i, hCtx.buf, hCtx.hasNull, selected, false)
+ err = codec.HashChunkSelected(w.rowContainerForProbe.sc, hCtx.hashVals, probeSideChk, hCtx.allTypes[keyIdx], i, hCtx.buf, hCtx.hasNull, selected, false)
if err != nil {
joinResult.err = err
return false, joinResult
@@ -996,7 +996,7 @@ func (e *HashJoinExec) join2Chunk(workerID uint, probeSideChk *chunk.Chunk, hCtx
}
for i := range selected {
- killed := atomic.LoadUint32(&e.ctx.GetSessionVars().Killed) == 1
+ killed := atomic.LoadUint32(&w.hashJoinCtx.sessCtx.GetSessionVars().Killed) == 1
failpoint.Inject("killedInJoin2Chunk", func(val failpoint.Value) {
if val.(bool) {
killed = true
@@ -1009,13 +1009,13 @@ func (e *HashJoinExec) join2Chunk(workerID uint, probeSideChk *chunk.Chunk, hCtx
if isNAAJ {
if !selected[i] {
// since this is the case of using inner to build, so for an outer row unselected, we should fill the result when it's outer join.
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideChk.GetRow(i), joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideChk.GetRow(i), joinResult.chk)
}
if hCtx.naHasNull[i] {
// here means the probe join connecting column has null value in it and this is special for matching all the hash buckets
// for it. (probeKey is not necessary here)
probeRow := probeSideChk.GetRow(i)
- ok, joinResult = e.joinNAAJMatchProbeSideRow2Chunk(workerID, 0, hCtx.naColNullBitMap[i].Clone(), probeRow, hCtx, rowContainer, joinResult)
+ ok, joinResult = w.joinNAAJMatchProbeSideRow2Chunk(0, hCtx.naColNullBitMap[i].Clone(), probeRow, hCtx, joinResult)
if !ok {
return false, joinResult
}
@@ -1023,7 +1023,7 @@ func (e *HashJoinExec) join2Chunk(workerID uint, probeSideChk *chunk.Chunk, hCtx
// here means the probe join connecting column without null values, where we should match same key bucket and null bucket for it at its order.
// step1: process same key matched probe side rows
probeKey, probeRow := hCtx.hashVals[i].Sum64(), probeSideChk.GetRow(i)
- ok, joinResult = e.joinNAAJMatchProbeSideRow2Chunk(workerID, probeKey, nil, probeRow, hCtx, rowContainer, joinResult)
+ ok, joinResult = w.joinNAAJMatchProbeSideRow2Chunk(probeKey, nil, probeRow, hCtx, joinResult)
if !ok {
return false, joinResult
}
@@ -1031,18 +1031,18 @@ func (e *HashJoinExec) join2Chunk(workerID uint, probeSideChk *chunk.Chunk, hCtx
} else {
// since this is the case of using inner to build, so for an outer row unselected, we should fill the result when it's outer join.
if !selected[i] || hCtx.hasNull[i] { // process unmatched probe side rows
- e.probeWorkers[workerID].joiner.onMissMatch(false, probeSideChk.GetRow(i), joinResult.chk)
+ w.joiner.onMissMatch(false, probeSideChk.GetRow(i), joinResult.chk)
} else { // process matched probe side rows
probeKey, probeRow := hCtx.hashVals[i].Sum64(), probeSideChk.GetRow(i)
- ok, joinResult = e.joinMatchedProbeSideRow2Chunk(workerID, probeKey, probeRow, hCtx, rowContainer, joinResult)
+ ok, joinResult = w.joinMatchedProbeSideRow2Chunk(probeKey, probeRow, hCtx, joinResult)
if !ok {
return false, joinResult
}
}
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -1052,17 +1052,17 @@ func (e *HashJoinExec) join2Chunk(workerID uint, probeSideChk *chunk.Chunk, hCtx
}
// join2ChunkForOuterHashJoin joins chunks when using the outer to build a hash table (refer to outer hash join)
-func (e *HashJoinExec) join2ChunkForOuterHashJoin(workerID uint, probeSideChk *chunk.Chunk, hCtx *hashContext, rowContainer *hashRowContainer, joinResult *hashjoinWorkerResult) (ok bool, _ *hashjoinWorkerResult) {
+func (w *probeWorker) join2ChunkForOuterHashJoin(probeSideChk *chunk.Chunk, hCtx *hashContext, joinResult *hashjoinWorkerResult) (ok bool, _ *hashjoinWorkerResult) {
hCtx.initHash(probeSideChk.NumRows())
for keyIdx, i := range hCtx.keyColIdx {
- err := codec.HashChunkColumns(rowContainer.sc, hCtx.hashVals, probeSideChk, hCtx.allTypes[keyIdx], i, hCtx.buf, hCtx.hasNull)
+ err := codec.HashChunkColumns(w.rowContainerForProbe.sc, hCtx.hashVals, probeSideChk, hCtx.allTypes[keyIdx], i, hCtx.buf, hCtx.hasNull)
if err != nil {
joinResult.err = err
return false, joinResult
}
}
for i := 0; i < probeSideChk.NumRows(); i++ {
- killed := atomic.LoadUint32(&e.ctx.GetSessionVars().Killed) == 1
+ killed := atomic.LoadUint32(&w.hashJoinCtx.sessCtx.GetSessionVars().Killed) == 1
failpoint.Inject("killedInJoin2ChunkForOuterHashJoin", func(val failpoint.Value) {
if val.(bool) {
killed = true
@@ -1073,13 +1073,13 @@ func (e *HashJoinExec) join2ChunkForOuterHashJoin(workerID uint, probeSideChk *c
return false, joinResult
}
probeKey, probeRow := hCtx.hashVals[i].Sum64(), probeSideChk.GetRow(i)
- ok, joinResult = e.joinMatchedProbeSideRow2ChunkForOuterHashJoin(workerID, probeKey, probeRow, hCtx, rowContainer, joinResult)
+ ok, joinResult = w.joinMatchedProbeSideRow2ChunkForOuterHashJoin(probeKey, probeRow, hCtx, joinResult)
if !ok {
return false, joinResult
}
if joinResult.chk.IsFull() {
- e.joinResultCh <- joinResult
- ok, joinResult = e.getNewJoinResult(workerID)
+ w.hashJoinCtx.joinResultCh <- joinResult
+ ok, joinResult = w.getNewJoinResult()
if !ok {
return false, joinResult
}
@@ -1095,20 +1095,12 @@ func (e *HashJoinExec) join2ChunkForOuterHashJoin(workerID uint, probeSideChk *c
func (e *HashJoinExec) Next(ctx context.Context, req *chunk.Chunk) (err error) {
if !e.prepared {
e.buildFinished = make(chan error, 1)
- buildKeyColIdx := make([]int, len(e.buildKeys))
- for i := range e.buildKeys {
- buildKeyColIdx[i] = e.buildKeys[i].Index
- }
- buildNAKeyColIdx := make([]int, len(e.buildNAKeys))
- for i := range e.buildNAKeys {
- buildNAKeyColIdx[i] = e.buildNAKeys[i].Index
- }
hCtx := &hashContext{
allTypes: e.buildTypes,
- keyColIdx: buildKeyColIdx,
- naKeyColIdx: buildNAKeyColIdx,
+ keyColIdx: e.buildWorker.buildKeyColIdx,
+ naKeyColIdx: e.buildWorker.buildNAKeyColIdx,
}
- e.rowContainer = newHashRowContainer(e.ctx, int(e.buildSideEstCount), hCtx, retTypes(e.buildSideExec))
+ e.rowContainer = newHashRowContainer(e.ctx, hCtx, retTypes(e.buildWorker.buildSideExec))
// we shallow copies rowContainer for each probe worker to avoid lock contention
for i := uint(0); i < e.concurrency; i++ {
if i == 0 {
@@ -1120,7 +1112,7 @@ func (e *HashJoinExec) Next(ctx context.Context, req *chunk.Chunk) (err error) {
for i := uint(0); i < e.concurrency; i++ {
e.probeWorkers[i].rowIters = chunk.NewIterator4Slice([]chunk.Row{}).(*chunk.Iterator4Slice)
}
- e.worker.RunWithRecover(func() {
+ e.workerWg.RunWithRecover(func() {
defer trace.StartRegion(ctx, "HashJoinHashTableBuilder").End()
e.fetchAndBuildHashTable(ctx)
}, e.handleFetchAndBuildHashTablePanic)
@@ -1163,10 +1155,10 @@ func (e *HashJoinExec) fetchAndBuildHashTable(ctx context.Context) {
buildSideResultCh := make(chan *chunk.Chunk, 1)
doneCh := make(chan struct{})
fetchBuildSideRowsOk := make(chan error, 1)
- e.worker.RunWithRecover(
+ e.workerWg.RunWithRecover(
func() {
defer trace.StartRegion(ctx, "HashJoinBuildSideFetcher").End()
- e.fetchBuildSideRows(ctx, buildSideResultCh, fetchBuildSideRowsOk, doneCh)
+ e.buildWorker.fetchBuildSideRows(ctx, buildSideResultCh, fetchBuildSideRowsOk, doneCh)
},
func(r interface{}) {
if r != nil {
@@ -1177,7 +1169,7 @@ func (e *HashJoinExec) fetchAndBuildHashTable(ctx context.Context) {
)
// TODO: Parallel build hash table. Currently not support because `unsafeHashTable` is not thread-safe.
- err := e.buildHashTableForList(buildSideResultCh)
+ err := e.buildWorker.buildHashTableForList(buildSideResultCh)
if err != nil {
e.buildFinished <- errors.Trace(err)
close(doneCh)
@@ -1195,41 +1187,42 @@ func (e *HashJoinExec) fetchAndBuildHashTable(ctx context.Context) {
}
// buildHashTableForList builds hash table from `list`.
-func (e *HashJoinExec) buildHashTableForList(buildSideResultCh <-chan *chunk.Chunk) error {
+func (w *buildWorker) buildHashTableForList(buildSideResultCh <-chan *chunk.Chunk) error {
var err error
var selected []bool
- e.rowContainer.GetMemTracker().AttachTo(e.memTracker)
- e.rowContainer.GetMemTracker().SetLabel(memory.LabelForBuildSideResult)
- e.rowContainer.GetDiskTracker().AttachTo(e.diskTracker)
- e.rowContainer.GetDiskTracker().SetLabel(memory.LabelForBuildSideResult)
+ rowContainer := w.hashJoinCtx.rowContainer
+ rowContainer.GetMemTracker().AttachTo(w.hashJoinCtx.memTracker)
+ rowContainer.GetMemTracker().SetLabel(memory.LabelForBuildSideResult)
+ rowContainer.GetDiskTracker().AttachTo(w.hashJoinCtx.diskTracker)
+ rowContainer.GetDiskTracker().SetLabel(memory.LabelForBuildSideResult)
if variable.EnableTmpStorageOnOOM.Load() {
- actionSpill := e.rowContainer.ActionSpill()
+ actionSpill := rowContainer.ActionSpill()
failpoint.Inject("testRowContainerSpill", func(val failpoint.Value) {
if val.(bool) {
- actionSpill = e.rowContainer.rowContainer.ActionSpillForTest()
+ actionSpill = rowContainer.rowContainer.ActionSpillForTest()
defer actionSpill.(*chunk.SpillDiskAction).WaitForTest()
}
})
- e.ctx.GetSessionVars().MemTracker.FallbackOldAndSetNewAction(actionSpill)
+ w.hashJoinCtx.sessCtx.GetSessionVars().MemTracker.FallbackOldAndSetNewAction(actionSpill)
}
for chk := range buildSideResultCh {
- if e.finished.Load() {
+ if w.hashJoinCtx.finished.Load() {
return nil
}
- if !e.useOuterToBuild {
- err = e.rowContainer.PutChunk(chk, e.isNullEQ)
+ if !w.hashJoinCtx.useOuterToBuild {
+ err = rowContainer.PutChunk(chk, w.hashJoinCtx.isNullEQ)
} else {
var bitMap = bitmap.NewConcurrentBitmap(chk.NumRows())
- e.outerMatchedStatus = append(e.outerMatchedStatus, bitMap)
- e.memTracker.Consume(bitMap.BytesConsumed())
- if len(e.outerFilter) == 0 {
- err = e.rowContainer.PutChunk(chk, e.isNullEQ)
+ w.hashJoinCtx.outerMatchedStatus = append(w.hashJoinCtx.outerMatchedStatus, bitMap)
+ w.hashJoinCtx.memTracker.Consume(bitMap.BytesConsumed())
+ if len(w.hashJoinCtx.outerFilter) == 0 {
+ err = w.hashJoinCtx.rowContainer.PutChunk(chk, w.hashJoinCtx.isNullEQ)
} else {
- selected, err = expression.VectorizedFilter(e.ctx, e.outerFilter, chunk.NewIterator4Chunk(chk), selected)
+ selected, err = expression.VectorizedFilter(w.hashJoinCtx.sessCtx, w.hashJoinCtx.outerFilter, chunk.NewIterator4Chunk(chk), selected)
if err != nil {
return err
}
- err = e.rowContainer.PutChunkSelected(chk, selected, e.isNullEQ)
+ err = rowContainer.PutChunkSelected(chk, selected, w.hashJoinCtx.isNullEQ)
}
}
failpoint.Inject("ConsumeRandomPanic", nil)
@@ -1283,7 +1276,6 @@ func (e *NestedLoopApplyExec) Close() error {
e.memTracker = nil
if e.runtimeStats != nil {
runtimeStats := newJoinRuntimeStats()
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, runtimeStats)
if e.canUseCache {
var hitRatio float64
if e.cacheAccessCounter > 0 {
@@ -1294,6 +1286,7 @@ func (e *NestedLoopApplyExec) Close() error {
runtimeStats.setCacheInfo(false, 0)
}
runtimeStats.SetConcurrencyInfo(execdetails.NewConcurrencyInfo("Concurrency", 0))
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, runtimeStats)
}
return e.outerExec.Close()
}
@@ -1551,6 +1544,17 @@ func (e *joinRuntimeStats) Tp() int {
return execdetails.TpJoinRuntimeStats
}
+func (e *joinRuntimeStats) Clone() execdetails.RuntimeStats {
+ newJRS := &joinRuntimeStats{
+ RuntimeStatsWithConcurrencyInfo: e.RuntimeStatsWithConcurrencyInfo,
+ applyCache: e.applyCache,
+ cache: e.cache,
+ hasHashStat: e.hasHashStat,
+ hashStat: e.hashStat,
+ }
+ return newJRS
+}
+
type hashJoinRuntimeStats struct {
fetchAndBuildHashTable time.Duration
hashStat hashStatistic
diff --git a/executor/join_test.go b/executor/join_test.go
index be006d7dc0063..6f56d0a18dc8e 100644
--- a/executor/join_test.go
+++ b/executor/join_test.go
@@ -1479,7 +1479,7 @@ func TestIndexNestedLoopHashJoin(t *testing.T) {
" └─TableRowIDScan 27.00 cop[tikv] table:l2 keep order:false"))
tk.MustQuery("select * from t l1 where exists ( select * from t l2 where l2.l_orderkey = l1.l_orderkey and l2.l_suppkey <> l1.l_suppkey )order by `l_orderkey`,`l_linenumber`;").Check(testkit.Rows("0 0 0 0", "0 1 0 1", "0 2 0 0", "1 0 1 0", "1 1 1 1", "1 2 1 0", "2 0 0 0", "2 1 0 1", "2 2 0 0"))
tk.MustQuery("desc format = 'brief' select count(*) from t l1 where exists ( select * from t l2 where l2.l_orderkey = l1.l_orderkey and l2.l_suppkey <> l1.l_suppkey );").Check(testkit.Rows(
- "HashAgg 1.00 root funcs:count(1)->Column#11",
+ "StreamAgg 1.00 root funcs:count(1)->Column#11",
"└─IndexHashJoin 7.20 root semi join, inner:IndexLookUp, outer key:test.t.l_orderkey, inner key:test.t.l_orderkey, equal cond:eq(test.t.l_orderkey, test.t.l_orderkey), other cond:ne(test.t.l_suppkey, test.t.l_suppkey)",
" ├─TableReader(Build) 9.00 root data:Selection",
" │ └─Selection 9.00 cop[tikv] not(isnull(test.t.l_suppkey))",
@@ -2307,16 +2307,14 @@ func TestIssue18070(t *testing.T) {
tk.MustExec("insert into t1 values(1),(2)")
tk.MustExec("insert into t2 values(1),(1),(2),(2)")
tk.MustExec("set @@tidb_mem_quota_query=1000")
- err := tk.QueryToErr("select /*+ inl_hash_join(t1)*/ * from t1 join t2 on t1.a = t2.a;")
- require.True(t, strings.Contains(err.Error(), "Out Of Memory Quota!"))
+ tk.MustContainErrMsg("select /*+ inl_hash_join(t1)*/ * from t1 join t2 on t1.a = t2.a;", "Out Of Memory Quota!")
fpName := "github.com/pingcap/tidb/executor/mockIndexMergeJoinOOMPanic"
require.NoError(t, failpoint.Enable(fpName, `panic("ERROR 1105 (HY000): Out Of Memory Quota![conn_id=1]")`))
defer func() {
require.NoError(t, failpoint.Disable(fpName))
}()
- err = tk.QueryToErr("select /*+ inl_merge_join(t1)*/ * from t1 join t2 on t1.a = t2.a;")
- require.True(t, strings.Contains(err.Error(), "Out Of Memory Quota!"))
+ tk.MustContainErrMsg("select /*+ inl_merge_join(t1)*/ * from t1 join t2 on t1.a = t2.a;", "Out Of Memory Quota!")
}
func TestIssue18564(t *testing.T) {
@@ -2894,3 +2892,20 @@ func TestOuterJoin(t *testing.T) {
),
)
}
+
+func TestCartesianJoinPanic(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int)")
+ tk.MustExec("insert into t values(1)")
+ tk.MustExec("set tidb_mem_quota_query = 1 << 30")
+ tk.MustExec("set global tidb_mem_oom_action = 'CANCEL'")
+ tk.MustExec("set global tidb_enable_tmp_storage_on_oom = off;")
+ for i := 0; i < 14; i++ {
+ tk.MustExec("insert into t select * from t")
+ }
+ err := tk.QueryToErr("desc analyze select * from t t1, t t2, t t3, t t4, t t5, t t6;")
+ require.NotNil(t, err)
+ require.True(t, strings.Contains(err.Error(), "Out Of Memory Quota!"))
+}
diff --git a/executor/load_data.go b/executor/load_data.go
index e11137c3916ae..85e04aaad3144 100644
--- a/executor/load_data.go
+++ b/executor/load_data.go
@@ -15,20 +15,28 @@
package executor
import (
+ "bufio"
"bytes"
"context"
"fmt"
+ "io"
+ "path/filepath"
"strings"
+ "sync"
"sync/atomic"
"time"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
+ backup "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/tidb/br/pkg/storage"
"github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
+ "github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/sessiontxn"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
@@ -40,13 +48,15 @@ import (
var (
null = []byte("NULL")
taskQueueSize = 16 // the maximum number of pending tasks to commit in queue
+ // InTest is a flag that bypass gcs authentication in unit tests.
+ InTest bool
)
// LoadDataExec represents a load data executor.
type LoadDataExec struct {
baseExecutor
- IsLocal bool
+ FileLocRef ast.FileLocRefTp
OnDuplicate ast.OnDuplicateKeyHandlingType
loadDataInfo *LoadDataInfo
}
@@ -54,32 +64,76 @@ type LoadDataExec struct {
// Next implements the Executor Next interface.
func (e *LoadDataExec) Next(ctx context.Context, req *chunk.Chunk) error {
req.GrowAndReset(e.maxChunkSize)
- // TODO: support load data without local field.
- if !e.IsLocal {
- return errors.New("Load Data: don't support load data without local field")
- }
- e.loadDataInfo.OnDuplicate = e.OnDuplicate
// TODO: support lines terminated is "".
if len(e.loadDataInfo.LinesInfo.Terminated) == 0 {
return errors.New("Load Data: don't support load data terminated is nil")
}
-
- sctx := e.loadDataInfo.ctx
- val := sctx.Value(LoadDataVarKey)
- if val != nil {
- sctx.SetValue(LoadDataVarKey, nil)
- return errors.New("Load Data: previous load data option isn't closed normal")
- }
if e.loadDataInfo.Path == "" {
return errors.New("Load Data: infile path is empty")
}
- sctx.SetValue(LoadDataVarKey, e.loadDataInfo)
+ if !e.loadDataInfo.Table.Meta().IsBaseTable() {
+ return errors.New("can only load data into base tables")
+ }
+ switch e.FileLocRef {
+ case ast.FileLocServerOrRemote:
+ u, err := storage.ParseRawURL(e.loadDataInfo.Path)
+ if err != nil {
+ return err
+ }
+ var filename string
+ u.Path, filename = filepath.Split(u.Path)
+ b, err := storage.ParseBackendFromURL(u, nil)
+ if err != nil {
+ return err
+ }
+ if b.GetLocal() != nil {
+ return errors.Errorf("Load Data: don't support load data from tidb-server's disk")
+ }
+ return e.loadFromRemote(ctx, b, filename)
+ case ast.FileLocClient:
+ // let caller use handleQuerySpecial to read data in this connection
+ sctx := e.loadDataInfo.ctx
+ val := sctx.Value(LoadDataVarKey)
+ if val != nil {
+ sctx.SetValue(LoadDataVarKey, nil)
+ return errors.New("Load Data: previous load data option wasn't closed normally")
+ }
+ sctx.SetValue(LoadDataVarKey, e.loadDataInfo)
+ }
return nil
}
+func (e *LoadDataExec) loadFromRemote(
+ ctx context.Context,
+ b *backup.StorageBackend,
+ filename string,
+) error {
+ opt := &storage.ExternalStorageOptions{}
+ if InTest {
+ opt.NoCredentials = true
+ }
+ s, err := storage.New(ctx, b, opt)
+ if err != nil {
+ return err
+ }
+ fileReader, err := s.Open(ctx, filename)
+ if err != nil {
+ return err
+ }
+ defer fileReader.Close()
+ reader := bufio.NewReader(fileReader)
+
+ return e.loadDataInfo.Load(ctx, func() ([]byte, error) {
+ return reader.ReadBytes('\n')
+ })
+}
+
// Close implements the Executor Close interface.
func (e *LoadDataExec) Close() error {
+ if e.runtimeStats != nil && e.loadDataInfo != nil && e.loadDataInfo.stats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.loadDataInfo.stats)
+ }
return nil
}
@@ -100,6 +154,7 @@ type CommitTask struct {
}
// LoadDataInfo saves the information of loading data operation.
+// TODO: rename it and remove unnecessary public methods.
type LoadDataInfo struct {
*InsertValues
@@ -129,6 +184,111 @@ type FieldMapping struct {
UserVar *ast.VariableExpr
}
+// Load reads from readerFn and do load data job.
+func (e *LoadDataInfo) Load(ctx context.Context, readerFn func() ([]byte, error)) error {
+ e.InitQueues()
+ e.SetMaxRowsInBatch(uint64(e.Ctx.GetSessionVars().DMLBatchSize))
+ e.StartStopWatcher()
+ // let stop watcher goroutine quit
+ defer e.ForceQuit()
+ err := sessiontxn.NewTxn(ctx, e.Ctx)
+ if err != nil {
+ return err
+ }
+ // processStream process input data, enqueue commit task
+ wg := new(sync.WaitGroup)
+ wg.Add(1)
+ go processStream(ctx, readerFn, e, wg)
+ err = e.CommitWork(ctx)
+ wg.Wait()
+ return err
+}
+
+// processStream process input stream from network
+func processStream(ctx context.Context, readerFn func() ([]byte, error), loadDataInfo *LoadDataInfo, wg *sync.WaitGroup) {
+ var err error
+ var shouldBreak bool
+ var prevData, curData []byte
+ defer func() {
+ r := recover()
+ if r != nil {
+ logutil.Logger(ctx).Error("process routine panicked",
+ zap.Reflect("r", r),
+ zap.Stack("stack"))
+ }
+ if err != nil || r != nil {
+ loadDataInfo.ForceQuit()
+ } else {
+ loadDataInfo.CloseTaskQueue()
+ }
+ wg.Done()
+ }()
+ for {
+ curData, err = readerFn()
+ if err != nil {
+ if terror.ErrorNotEqual(err, io.EOF) {
+ logutil.Logger(ctx).Error("read data for LOAD DATA failed", zap.Error(err))
+ break
+ }
+ err = nil
+ }
+ if len(curData) == 0 {
+ loadDataInfo.Drained = true
+ shouldBreak = true
+ if len(prevData) == 0 {
+ break
+ }
+ }
+ select {
+ case <-loadDataInfo.QuitCh:
+ err = errors.New("processStream forced to quit")
+ default:
+ }
+ if err != nil {
+ break
+ }
+ // prepare batch and enqueue task
+ prevData, err = insertDataWithCommit(ctx, prevData, curData, loadDataInfo)
+ if err != nil {
+ break
+ }
+ if shouldBreak {
+ break
+ }
+ }
+ if err != nil {
+ logutil.Logger(ctx).Error("load data process stream error", zap.Error(err))
+ return
+ }
+ if err = loadDataInfo.EnqOneTask(ctx); err != nil {
+ logutil.Logger(ctx).Error("load data process stream error", zap.Error(err))
+ return
+ }
+}
+
+func insertDataWithCommit(ctx context.Context, prevData,
+ curData []byte, loadDataInfo *LoadDataInfo) ([]byte, error) {
+ var err error
+ var reachLimit bool
+ for {
+ prevData, reachLimit, err = loadDataInfo.InsertData(ctx, prevData, curData)
+ if err != nil {
+ return nil, err
+ }
+ if !reachLimit {
+ break
+ }
+ // push into commit task queue
+ err = loadDataInfo.EnqOneTask(ctx)
+ if err != nil {
+ return prevData, err
+ }
+ curData = prevData
+ prevData = nil
+ }
+ return prevData, nil
+}
+
// reorderColumns reorder the e.insertColumns according to the order of columnNames
// Note: We must ensure there must be one-to-one mapping between e.insertColumns and columnNames in terms of column name.
func (e *LoadDataInfo) reorderColumns(columnNames []string) error {
@@ -313,7 +473,7 @@ func (e *LoadDataInfo) CommitOneTask(ctx context.Context, task CommitTask) error
var err error
defer func() {
if err != nil {
- e.Ctx.StmtRollback()
+ e.Ctx.StmtRollback(ctx, false)
}
}()
err = e.CheckAndInsertOneBatch(ctx, task.rows, task.cnt)
@@ -324,7 +484,7 @@ func (e *LoadDataInfo) CommitOneTask(ctx context.Context, task CommitTask) error
failpoint.Inject("commitOneTaskErr", func() error {
return errors.New("mock commit one task error")
})
- e.Ctx.StmtCommit()
+ e.Ctx.StmtCommit(ctx)
// Make sure process stream routine never use invalid txn
e.txnInUse.Lock()
defer e.txnInUse.Unlock()
@@ -350,7 +510,7 @@ func (e *LoadDataInfo) CommitWork(ctx context.Context) error {
e.ForceQuit()
}
if err != nil {
- e.ctx.StmtRollback()
+ e.ctx.StmtRollback(ctx, false)
}
}()
var tasks uint64
diff --git a/executor/load_stats.go b/executor/load_stats.go
index d59651e29200a..e292eb817519d 100644
--- a/executor/load_stats.go
+++ b/executor/load_stats.go
@@ -52,7 +52,7 @@ func (k loadStatsVarKeyType) String() string {
const LoadStatsVarKey loadStatsVarKeyType = 0
// Next implements the Executor Next interface.
-func (e *LoadStatsExec) Next(ctx context.Context, req *chunk.Chunk) error {
+func (e *LoadStatsExec) Next(_ context.Context, req *chunk.Chunk) error {
req.GrowAndReset(e.maxChunkSize)
if len(e.info.Path) == 0 {
return errors.New("Load Stats: file path is empty")
@@ -72,7 +72,7 @@ func (e *LoadStatsExec) Close() error {
}
// Open implements the Executor Open interface.
-func (e *LoadStatsExec) Open(ctx context.Context) error {
+func (e *LoadStatsExec) Open(_ context.Context) error {
return nil
}
diff --git a/executor/loadremotetest/BUILD.bazel b/executor/loadremotetest/BUILD.bazel
new file mode 100644
index 0000000000000..5db2476b844c2
--- /dev/null
+++ b/executor/loadremotetest/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+ name = "loadremotetest_test",
+ timeout = "short",
+ srcs = [
+ "main_test.go",
+ "one_csv_test.go",
+ "util_test.go",
+ ],
+ flaky = True,
+ deps = [
+ "//executor",
+ "//kv",
+ "//testkit",
+ "@com_github_fsouza_fake_gcs_server//fakestorage",
+ "@com_github_stretchr_testify//suite",
+ "@org_uber_go_goleak//:goleak",
+ ],
+)
diff --git a/executor/loadremotetest/main_test.go b/executor/loadremotetest/main_test.go
new file mode 100644
index 0000000000000..82f95ed8dd112
--- /dev/null
+++ b/executor/loadremotetest/main_test.go
@@ -0,0 +1,33 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loadremotetest
+
+import (
+ "testing"
+
+ "go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+ opts := []goleak.Option{
+ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
+ goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
+ goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"),
+ goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
+ goleak.IgnoreTopFunction("internal/poll.runtime_pollWait"),
+ goleak.IgnoreTopFunction("net.(*netFD).connect.func2"),
+ }
+ goleak.VerifyTestMain(m, opts...)
+}
diff --git a/executor/loadremotetest/one_csv_test.go b/executor/loadremotetest/one_csv_test.go
new file mode 100644
index 0000000000000..db1709f23fc0f
--- /dev/null
+++ b/executor/loadremotetest/one_csv_test.go
@@ -0,0 +1,57 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loadremotetest
+
+import (
+ "fmt"
+
+ "github.com/pingcap/tidb/testkit"
+)
+
+func (s *mockGCSSuite) TestLoadCSV() {
+ s.tk.MustExec("DROP DATABASE IF EXISTS load_csv;")
+ s.tk.MustExec("CREATE DATABASE load_csv;")
+ s.tk.MustExec("CREATE TABLE load_csv.t (i INT, s varchar(32));")
+
+ // no-new-line-at-end
+ sql := fmt.Sprintf(`LOAD DATA INFILE 'gcs://test-bucket/no-new-line-at-end.csv?endpoint=%s' INTO TABLE load_csv.t
+ FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"'
+ LINES TERMINATED BY '\n' IGNORE 1 LINES;`, gcsEndpoint)
+ s.tk.MustExec(sql)
+ s.tk.MustQuery("SELECT * FROM load_csv.t;").Check(testkit.Rows(
+ "100 test100",
+ "101 \"",
+ "102 😄😄😄😄😄",
+ "104 ",
+ ))
+ s.tk.MustExec("TRUNCATE TABLE load_csv.t;")
+
+ // new-line-at-end
+ sql = fmt.Sprintf(`LOAD DATA INFILE 'gcs://test-bucket/new-line-at-end.csv?endpoint=%s' INTO TABLE load_csv.t
+ FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"'
+ LINES TERMINATED BY '\n' IGNORE 1 LINES;`, gcsEndpoint)
+ s.tk.MustExec(sql)
+ s.tk.MustQuery("SELECT * FROM load_csv.t;").Check(testkit.Rows(
+ "100 test100",
+ "101 \"",
+ "102 😄😄😄😄😄",
+ "104 ",
+ ))
+ s.tk.MustExec("TRUNCATE TABLE load_csv.t;")
+
+ // can't read file at tidb-server
+ sql = "LOAD DATA INFILE '/etc/passwd' INTO TABLE load_csv.t;"
+ s.tk.MustContainErrMsg(sql, "don't support load data from tidb-server")
+}
diff --git a/executor/loadremotetest/util_test.go b/executor/loadremotetest/util_test.go
new file mode 100644
index 0000000000000..713a3883fa616
--- /dev/null
+++ b/executor/loadremotetest/util_test.go
@@ -0,0 +1,87 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loadremotetest
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/fsouza/fake-gcs-server/fakestorage"
+ "github.com/pingcap/tidb/executor"
+ "github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/stretchr/testify/suite"
+)
+
+type mockGCSSuite struct {
+ suite.Suite
+
+ server *fakestorage.Server
+ store kv.Storage
+ tk *testkit.TestKit
+}
+
+var (
+ gcsHost = "127.0.0.1"
+ gcsPort = uint16(4443)
+ gcsEndpoint = fmt.Sprintf("http://%s:%d", gcsHost, gcsPort)
+)
+
+func TestLoadRemote(t *testing.T) {
+ suite.Run(t, &mockGCSSuite{})
+}
+
+func (s *mockGCSSuite) SetupSuite() {
+ objects := []fakestorage.Object{
+ {
+ BucketName: "test-bucket",
+ Name: "no-new-line-at-end.csv",
+ Content: []byte(`i,s
+100,"test100"
+101,"\""
+102,"😄😄😄😄😄"
+104,""`),
+ },
+ {
+ BucketName: "test-bucket",
+ Name: "new-line-at-end.csv",
+ Content: []byte(`i,s
+100,"test100"
+101,"\""
+102,"😄😄😄😄😄"
+104,""
+`),
+ },
+ }
+
+ var err error
+ opt := fakestorage.Options{
+ InitialObjects: objects,
+ Scheme: "http",
+ Host: gcsHost,
+ Port: gcsPort,
+ PublicHost: gcsHost,
+ }
+ s.server, err = fakestorage.NewServerWithOptions(opt)
+ s.Require().NoError(err)
+ s.store = testkit.CreateMockStore(s.T())
+ s.tk = testkit.NewTestKit(s.T(), s.store)
+ executor.InTest = true
+}
+
+func (s *mockGCSSuite) TearDownSuite() {
+ s.server.Stop()
+ executor.InTest = false
+}
diff --git a/executor/lock_stats.go b/executor/lock_stats.go
index 540670e8119fc..ccf6df123db07 100644
--- a/executor/lock_stats.go
+++ b/executor/lock_stats.go
@@ -44,7 +44,7 @@ func (k lockStatsVarKeyType) String() string {
const LockStatsVarKey lockStatsVarKeyType = 0
// Next implements the Executor Next interface.
-func (e *LockStatsExec) Next(ctx context.Context, req *chunk.Chunk) error {
+func (e *LockStatsExec) Next(_ context.Context, _ *chunk.Chunk) error {
do := domain.GetDomain(e.ctx)
is := do.InfoSchema()
h := do.StatsHandle()
@@ -87,7 +87,7 @@ func (e *LockStatsExec) Close() error {
}
// Open implements the Executor Open interface.
-func (e *LockStatsExec) Open(ctx context.Context) error {
+func (e *LockStatsExec) Open(_ context.Context) error {
return nil
}
@@ -109,7 +109,7 @@ func (k unlockStatsVarKeyType) String() string {
const UnlockStatsVarKey unlockStatsVarKeyType = 0
// Next implements the Executor Next interface.
-func (e *UnlockStatsExec) Next(ctx context.Context, req *chunk.Chunk) error {
+func (e *UnlockStatsExec) Next(_ context.Context, _ *chunk.Chunk) error {
do := domain.GetDomain(e.ctx)
is := do.InfoSchema()
h := do.StatsHandle()
@@ -152,6 +152,6 @@ func (e *UnlockStatsExec) Close() error {
}
// Open implements the Executor Open interface.
-func (e *UnlockStatsExec) Open(ctx context.Context) error {
+func (e *UnlockStatsExec) Open(_ context.Context) error {
return nil
}
diff --git a/executor/main_test.go b/executor/main_test.go
index 512eced5b50fa..4bf8cfb8f1bac 100644
--- a/executor/main_test.go
+++ b/executor/main_test.go
@@ -33,6 +33,7 @@ var prepareMergeSuiteData testdata.TestData
var aggMergeSuiteData testdata.TestData
var executorSuiteData testdata.TestData
var pointGetSuiteData testdata.TestData
+var slowQuerySuiteData testdata.TestData
func TestMain(m *testing.M) {
testsetup.SetupForCommonTest()
@@ -40,10 +41,12 @@ func TestMain(m *testing.M) {
testDataMap.LoadTestSuiteData("testdata", "executor_suite")
testDataMap.LoadTestSuiteData("testdata", "prepare_suite")
testDataMap.LoadTestSuiteData("testdata", "point_get_suite")
+ testDataMap.LoadTestSuiteData("testdata", "slow_query_suite")
aggMergeSuiteData = testDataMap["agg_suite"]
executorSuiteData = testDataMap["executor_suite"]
prepareMergeSuiteData = testDataMap["prepare_suite"]
pointGetSuiteData = testDataMap["point_get_suite"]
+ slowQuerySuiteData = testDataMap["slow_query_suite"]
autoid.SetStep(5000)
config.UpdateGlobal(func(conf *config.Config) {
diff --git a/executor/mem_reader.go b/executor/mem_reader.go
index 3790a5d862539..91bc6140b1295 100644
--- a/executor/mem_reader.go
+++ b/executor/mem_reader.go
@@ -17,7 +17,6 @@ package executor
import (
"context"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/distsql"
"github.com/pingcap/tidb/expression"
@@ -34,6 +33,7 @@ import (
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/rowcodec"
+ "github.com/pingcap/tidb/util/tracing"
)
type memReader interface {
@@ -65,11 +65,7 @@ type memIndexReader struct {
}
func buildMemIndexReader(ctx context.Context, us *UnionScanExec, idxReader *IndexReaderExecutor) *memIndexReader {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("buildMemIndexReader", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- opentracing.ContextWithSpan(ctx, span1)
- }
+ defer tracing.StartRegion(ctx, "buildMemIndexReader").End()
kvRanges := idxReader.kvRanges
outputOffset := make([]int, 0, len(us.columns))
for _, col := range idxReader.outputColumns {
@@ -90,11 +86,7 @@ func buildMemIndexReader(ctx context.Context, us *UnionScanExec, idxReader *Inde
}
func (m *memIndexReader) getMemRows(ctx context.Context) ([][]types.Datum, error) {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("memIndexReader.getMemRows", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- opentracing.ContextWithSpan(ctx, span1)
- }
+ defer tracing.StartRegion(ctx, "memIndexReader.getMemRows").End()
tps := make([]*types.FieldType, 0, len(m.index.Columns)+1)
cols := m.table.Columns
for _, col := range m.index.Columns {
@@ -190,11 +182,7 @@ type allocBuf struct {
}
func buildMemTableReader(ctx context.Context, us *UnionScanExec, tblReader *TableReaderExecutor) *memTableReader {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("buildMemTableReader", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- opentracing.ContextWithSpan(ctx, span1)
- }
+ defer tracing.StartRegion(ctx, "buildMemTableReader").End()
colIDs := make(map[int64]int, len(us.columns))
for i, col := range us.columns {
colIDs[col.ID] = i
@@ -235,11 +223,7 @@ func buildMemTableReader(ctx context.Context, us *UnionScanExec, tblReader *Tabl
// TODO: Try to make memXXXReader lazy, There is no need to decode many rows when parent operator only need 1 row.
func (m *memTableReader) getMemRows(ctx context.Context) ([][]types.Datum, error) {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("memTableReader.getMemRows", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- opentracing.ContextWithSpan(ctx, span1)
- }
+ defer tracing.StartRegion(ctx, "memTableReader.getMemRows").End()
mutableRow := chunk.MutRowFromTypes(m.retFieldTypes)
resultRows := make([]types.Datum, len(m.columns))
m.offsets = make([]int, len(m.columns))
@@ -490,11 +474,8 @@ type memIndexLookUpReader struct {
}
func buildMemIndexLookUpReader(ctx context.Context, us *UnionScanExec, idxLookUpReader *IndexLookUpExecutor) *memIndexLookUpReader {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("buildMemIndexLookUpReader", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- opentracing.ContextWithSpan(ctx, span1)
- }
+ defer tracing.StartRegion(ctx, "buildMemIndexLookUpReader").End()
+
kvRanges := idxLookUpReader.kvRanges
outputOffset := []int{len(idxLookUpReader.index.Columns)}
memIdxReader := &memIndexReader{
@@ -527,11 +508,9 @@ func buildMemIndexLookUpReader(ctx context.Context, us *UnionScanExec, idxLookUp
}
func (m *memIndexLookUpReader) getMemRows(ctx context.Context) ([][]types.Datum, error) {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("memIndexLookUpReader.getMemRows", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "memIndexLookUpReader.getMemRows")
+ defer r.End()
+
kvRanges := [][]kv.KeyRange{m.idxReader.kvRanges}
tbls := []table.Table{m.table}
if m.partitionMode {
@@ -595,6 +574,7 @@ type memIndexMergeReader struct {
retFieldTypes []*types.FieldType
indexMergeReader *IndexMergeReaderExecutor
memReaders []memReader
+ isIntersection bool
// partition mode
partitionMode bool // if it is accessing a partition table
@@ -603,11 +583,7 @@ type memIndexMergeReader struct {
}
func buildMemIndexMergeReader(ctx context.Context, us *UnionScanExec, indexMergeReader *IndexMergeReaderExecutor) *memIndexMergeReader {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("buildMemIndexMergeReader", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- opentracing.ContextWithSpan(ctx, span1)
- }
+ defer tracing.StartRegion(ctx, "buildMemIndexMergeReader").End()
indexCount := len(indexMergeReader.indexes)
memReaders := make([]memReader, 0, indexCount)
for i := 0; i < indexCount; i++ {
@@ -651,6 +627,7 @@ func buildMemIndexMergeReader(ctx context.Context, us *UnionScanExec, indexMerge
retFieldTypes: retTypes(us),
indexMergeReader: indexMergeReader,
memReaders: memReaders,
+ isIntersection: indexMergeReader.isIntersection,
partitionMode: indexMergeReader.partitionTableMode,
partitionTables: indexMergeReader.prunedPartitions,
@@ -659,11 +636,8 @@ func buildMemIndexMergeReader(ctx context.Context, us *UnionScanExec, indexMerge
}
func (m *memIndexMergeReader) getMemRows(ctx context.Context) ([][]types.Datum, error) {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("memIndexMergeReader.getMemRows", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "memIndexMergeReader.getMemRows")
+ defer r.End()
tbls := []table.Table{m.table}
// [partNum][indexNum][rangeNum]
var kvRanges [][][]kv.KeyRange
@@ -676,11 +650,20 @@ func (m *memIndexMergeReader) getMemRows(ctx context.Context) ([][]types.Datum,
} else {
kvRanges = append(kvRanges, m.indexMergeReader.keyRanges)
}
+ if len(kvRanges) != len(tbls) {
+ return nil, errors.Errorf("length of tbls(size: %d) should be equals to length of kvRanges(size: %d)", len(tbls), len(kvRanges))
+ }
tblKVRanges := make([]kv.KeyRange, 0, 16)
numHandles := 0
+ var handles []kv.Handle
+ var err error
for i, tbl := range tbls {
- handles, err := m.unionHandles(kvRanges[i])
+ if m.isIntersection {
+ handles, err = m.intersectionHandles(kvRanges[i])
+ } else {
+ handles, err = m.unionHandles(kvRanges[i])
+ }
if err != nil {
return nil, err
}
@@ -716,7 +699,7 @@ func (m *memIndexMergeReader) getMemRows(ctx context.Context) ([][]types.Datum,
return memTblReader.getMemRows(ctx)
}
-// Union all handles of different Indexes.
+// Union all handles of all partial paths.
func (m *memIndexMergeReader) unionHandles(kvRanges [][]kv.KeyRange) (finalHandles []kv.Handle, err error) {
if len(m.memReaders) != len(kvRanges) {
return nil, errors.Errorf("len(kvRanges) should be equal to len(memReaders)")
@@ -747,6 +730,44 @@ func (m *memIndexMergeReader) unionHandles(kvRanges [][]kv.KeyRange) (finalHandl
return finalHandles, nil
}
+// Intersect handles of each partial paths.
+func (m *memIndexMergeReader) intersectionHandles(kvRanges [][]kv.KeyRange) (finalHandles []kv.Handle, err error) {
+ if len(m.memReaders) != len(kvRanges) {
+ return nil, errors.Errorf("len(kvRanges) should be equal to len(memReaders)")
+ }
+
+ hMap := kv.NewHandleMap()
+ var handles []kv.Handle
+ for i, reader := range m.memReaders {
+ switch r := reader.(type) {
+ case *memTableReader:
+ r.kvRanges = kvRanges[i]
+ case *memIndexReader:
+ r.kvRanges = kvRanges[i]
+ default:
+ return nil, errors.New("memReader have to be memTableReader or memIndexReader")
+ }
+ if handles, err = reader.getMemRowsHandle(); err != nil {
+ return nil, err
+ }
+ for _, h := range handles {
+ if cntPtr, ok := hMap.Get(h); !ok {
+ cnt := 1
+ hMap.Set(h, &cnt)
+ } else {
+ *(cntPtr.(*int)) += 1
+ }
+ }
+ }
+ hMap.Range(func(h kv.Handle, val interface{}) bool {
+ if *(val.(*int)) == len(m.memReaders) {
+ finalHandles = append(finalHandles, h)
+ }
+ return true
+ })
+ return finalHandles, nil
+}
+
func (m *memIndexMergeReader) getMemRowsHandle() ([]kv.Handle, error) {
return nil, errors.New("getMemRowsHandle has not been implemented for memIndexMergeReader")
}
diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go
index dc215e71fe3bb..a299438da0777 100644
--- a/executor/memtable_reader.go
+++ b/executor/memtable_reader.go
@@ -29,7 +29,6 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/diagnosticspb"
- "github.com/pingcap/log"
"github.com/pingcap/sysutil"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/infoschema"
@@ -47,10 +46,10 @@ import (
"github.com/pingcap/tidb/util/execdetails"
"github.com/pingcap/tidb/util/pdapi"
"github.com/pingcap/tidb/util/set"
- "go.uber.org/zap"
"golang.org/x/exp/slices"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
)
const clusterLogBatchSize = 256
@@ -138,7 +137,7 @@ func (e *MemTableReaderExec) Next(ctx context.Context, req *chunk.Chunk) error {
// Close implements the Executor Close interface.
func (e *MemTableReaderExec) Close() error {
if stats := e.retriever.getRuntimeStats(); stats != nil && e.runtimeStats != nil {
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, stats)
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, stats)
}
return e.retriever.close()
}
@@ -177,7 +176,7 @@ func fetchClusterConfig(sctx sessionctx.Context, nodeTypes, nodeAddrs set.String
if err != nil {
return nil, err
}
- serversInfo = filterClusterServerInfo(serversInfo, nodeTypes, nodeAddrs)
+ serversInfo = infoschema.FilterClusterServerInfo(serversInfo, nodeTypes, nodeAddrs)
//nolint: prealloc
var finalRows [][]types.Datum
wg := sync.WaitGroup{}
@@ -310,108 +309,12 @@ func (e *clusterServerInfoRetriever) retrieve(ctx context.Context, sctx sessionc
return nil, nil
}
e.retrieved = true
-
serversInfo, err := infoschema.GetClusterServerInfo(sctx)
if err != nil {
return nil, err
}
- serversInfo = filterClusterServerInfo(serversInfo, e.extractor.NodeTypes, e.extractor.Instances)
-
- type result struct {
- idx int
- rows [][]types.Datum
- err error
- }
- wg := sync.WaitGroup{}
- ch := make(chan result, len(serversInfo))
- infoTp := e.serverInfoType
- finalRows := make([][]types.Datum, 0, len(serversInfo)*10)
- for i, srv := range serversInfo {
- address := srv.Address
- remote := address
- if srv.ServerType == "tidb" {
- remote = srv.StatusAddr
- }
- wg.Add(1)
- go func(index int, remote, address, serverTP string) {
- util.WithRecovery(func() {
- defer wg.Done()
- items, err := getServerInfoByGRPC(ctx, remote, infoTp)
- if err != nil {
- ch <- result{idx: index, err: err}
- return
- }
- partRows := serverInfoItemToRows(items, serverTP, address)
- ch <- result{idx: index, rows: partRows}
- }, nil)
- }(i, remote, address, srv.ServerType)
- }
- wg.Wait()
- close(ch)
- // Keep the original order to make the result more stable
- var results []result //nolint: prealloc
- for result := range ch {
- if result.err != nil {
- sctx.GetSessionVars().StmtCtx.AppendWarning(result.err)
- continue
- }
- results = append(results, result)
- }
- slices.SortFunc(results, func(i, j result) bool { return i.idx < j.idx })
- for _, result := range results {
- finalRows = append(finalRows, result.rows...)
- }
- return finalRows, nil
-}
-
-func serverInfoItemToRows(items []*diagnosticspb.ServerInfoItem, tp, addr string) [][]types.Datum {
- rows := make([][]types.Datum, 0, len(items))
- for _, v := range items {
- for _, item := range v.Pairs {
- row := types.MakeDatums(
- tp,
- addr,
- v.Tp,
- v.Name,
- item.Key,
- item.Value,
- )
- rows = append(rows, row)
- }
- }
- return rows
-}
-
-func getServerInfoByGRPC(ctx context.Context, address string, tp diagnosticspb.ServerInfoType) ([]*diagnosticspb.ServerInfoItem, error) {
- opt := grpc.WithInsecure()
- security := config.GetGlobalConfig().Security
- if len(security.ClusterSSLCA) != 0 {
- clusterSecurity := security.ClusterSecurity()
- tlsConfig, err := clusterSecurity.ToTLSConfig()
- if err != nil {
- return nil, errors.Trace(err)
- }
- opt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
- }
- conn, err := grpc.Dial(address, opt)
- if err != nil {
- return nil, err
- }
- defer func() {
- err := conn.Close()
- if err != nil {
- log.Error("close grpc connection error", zap.Error(err))
- }
- }()
-
- cli := diagnosticspb.NewDiagnosticsClient(conn)
- ctx, cancel := context.WithTimeout(ctx, time.Second*10)
- defer cancel()
- r, err := cli.ServerInfo(ctx, &diagnosticspb.ServerInfoRequest{Tp: tp})
- if err != nil {
- return nil, err
- }
- return r.Items, nil
+ serversInfo = infoschema.FilterClusterServerInfo(serversInfo, e.extractor.NodeTypes, e.extractor.Instances)
+ return infoschema.FetchClusterServerInfoWithoutPrivilegeCheck(ctx, sctx, serversInfo, e.serverInfoType, true)
}
func parseFailpointServerInfo(s string) []infoschema.ServerInfo {
@@ -428,28 +331,6 @@ func parseFailpointServerInfo(s string) []infoschema.ServerInfo {
return serversInfo
}
-func filterClusterServerInfo(serversInfo []infoschema.ServerInfo, nodeTypes, addresses set.StringSet) []infoschema.ServerInfo {
- if len(nodeTypes) == 0 && len(addresses) == 0 {
- return serversInfo
- }
-
- filterServers := make([]infoschema.ServerInfo, 0, len(serversInfo))
- for _, srv := range serversInfo {
- // Skip some node type which has been filtered in WHERE clause
- // e.g: SELECT * FROM cluster_config WHERE type='tikv'
- if len(nodeTypes) > 0 && !nodeTypes.Exist(srv.ServerType) {
- continue
- }
- // Skip some node address which has been filtered in WHERE clause
- // e.g: SELECT * FROM cluster_config WHERE address='192.16.8.12:2379'
- if len(addresses) > 0 && !addresses.Exist(srv.Address) {
- continue
- }
- filterServers = append(filterServers, srv)
- }
- return filterServers
-}
-
type clusterLogRetriever struct {
isDrained bool
retrieving bool
@@ -515,7 +396,7 @@ func (e *clusterLogRetriever) initialize(ctx context.Context, sctx sessionctx.Co
instances := e.extractor.Instances
nodeTypes := e.extractor.NodeTypes
- serversInfo = filterClusterServerInfo(serversInfo, nodeTypes, instances)
+ serversInfo = infoschema.FilterClusterServerInfo(serversInfo, nodeTypes, instances)
var levels = make([]diagnosticspb.LogLevel, 0, len(e.extractor.LogLevels))
for l := range e.extractor.LogLevels {
@@ -551,7 +432,7 @@ func (e *clusterLogRetriever) startRetrieving(
serversInfo []infoschema.ServerInfo,
req *diagnosticspb.SearchLogRequest) ([]chan logStreamResult, error) {
// gRPC options
- opt := grpc.WithInsecure()
+ opt := grpc.WithTransportCredentials(insecure.NewCredentials())
security := config.GetGlobalConfig().Security
if len(security.ClusterSSLCA) != 0 {
clusterSecurity := security.ClusterSecurity()
diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go
index 870a4193fb3b2..f6d98d4ec24fc 100644
--- a/executor/memtable_reader_test.go
+++ b/executor/memtable_reader_test.go
@@ -29,6 +29,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/fn"
"github.com/pingcap/sysutil"
+ "github.com/pingcap/tidb/executor"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/util/pdapi"
pmodel "github.com/prometheus/common/model"
@@ -56,7 +57,7 @@ func TestMetricTableData(t *testing.T) {
}
matrix = append(matrix, &pmodel.SampleStream{Metric: metric, Values: []pmodel.SamplePair{v1}})
- ctx := context.WithValue(context.Background(), "__mockMetricsPromData", matrix)
+ ctx := context.WithValue(context.Background(), executor.MockMetricsPromDataKey{}, matrix)
ctx = failpoint.WithHook(ctx, func(ctx context.Context, fpname string) bool {
return fpname == fpName
})
diff --git a/executor/metrics_reader.go b/executor/metrics_reader.go
index 314616785d60f..d9e0bd39f1128 100644
--- a/executor/metrics_reader.go
+++ b/executor/metrics_reader.go
@@ -89,9 +89,12 @@ func (e *MetricRetriever) retrieve(ctx context.Context, sctx sessionctx.Context)
return totalRows, nil
}
+// MockMetricsPromDataKey is for test
+type MockMetricsPromDataKey struct{}
+
func (e *MetricRetriever) queryMetric(ctx context.Context, sctx sessionctx.Context, queryRange promv1.Range, quantile float64) (result pmodel.Value, err error) {
failpoint.InjectContext(ctx, "mockMetricsPromData", func() {
- failpoint.Return(ctx.Value("__mockMetricsPromData").(pmodel.Matrix), nil)
+ failpoint.Return(ctx.Value(MockMetricsPromDataKey{}).(pmodel.Matrix), nil)
})
// Add retry to avoid network error.
diff --git a/executor/mpp_gather.go b/executor/mpp_gather.go
index 42526774dbdd5..6460e7e3de267 100644
--- a/executor/mpp_gather.go
+++ b/executor/mpp_gather.go
@@ -16,6 +16,7 @@ package executor
import (
"context"
+ "time"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
@@ -26,6 +27,7 @@ import (
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tipb/go-tipb"
"go.uber.org/zap"
)
@@ -38,6 +40,18 @@ func useMPPExecution(ctx sessionctx.Context, tr *plannercore.PhysicalTableReader
return ok
}
+func getMPPQueryID(ctx sessionctx.Context) uint64 {
+ mppQueryInfo := &ctx.GetSessionVars().StmtCtx.MPPQueryInfo
+ mppQueryInfo.QueryID.CompareAndSwap(0, plannercore.AllocMPPQueryID())
+ return mppQueryInfo.QueryID.Load()
+}
+
+func getMPPQueryTS(ctx sessionctx.Context) uint64 {
+ mppQueryInfo := &ctx.GetSessionVars().StmtCtx.MPPQueryInfo
+ mppQueryInfo.QueryTS.CompareAndSwap(0, uint64(time.Now().UnixNano()))
+ return mppQueryInfo.QueryTS.Load()
+}
+
// MPPGather dispatch MPP tasks and read data from root tasks.
type MPPGather struct {
// following fields are construct needed
@@ -45,10 +59,13 @@ type MPPGather struct {
is infoschema.InfoSchema
originalPlan plannercore.PhysicalPlan
startTS uint64
+ mppQueryID kv.MPPQueryID
mppReqs []*kv.MPPDispatchRequest
respIter distsql.SelectResult
+
+ memTracker *memory.Tracker
}
func (e *MPPGather) appendMPPDispatchReq(pf *plannercore.Fragment) error {
@@ -67,7 +84,9 @@ func (e *MPPGather) appendMPPDispatchReq(pf *plannercore.Fragment) error {
for _, mppTask := range pf.ExchangeSender.Tasks {
if mppTask.PartitionTableIDs != nil {
err = updateExecutorTableID(context.Background(), dagReq.RootExecutor, true, mppTask.PartitionTableIDs)
- } else {
+ } else if !mppTask.IsDisaggregatedTiFlashStaticPrune {
+ // If isDisaggregatedTiFlashStaticPrune is true, it means this TableScan is under PartitionUnoin,
+ // tableID in TableScan is already the physical table id of this partition, no need to update again.
err = updateExecutorTableID(context.Background(), dagReq.RootExecutor, true, []int64{mppTask.TableID})
}
if err != nil {
@@ -77,18 +96,24 @@ func (e *MPPGather) appendMPPDispatchReq(pf *plannercore.Fragment) error {
if err != nil {
return errors.Trace(err)
}
+
logutil.BgLogger().Info("Dispatch mpp task", zap.Uint64("timestamp", mppTask.StartTs),
- zap.Int64("ID", mppTask.ID), zap.String("address", mppTask.Meta.GetAddress()),
- zap.String("plan", plannercore.ToString(pf.ExchangeSender)))
+ zap.Int64("ID", mppTask.ID), zap.Uint64("QueryTs", mppTask.MppQueryID.QueryTs), zap.Uint64("LocalQueryId", mppTask.MppQueryID.LocalQueryID),
+ zap.Uint64("ServerID", mppTask.MppQueryID.ServerID), zap.String("address", mppTask.Meta.GetAddress()),
+ zap.String("plan", plannercore.ToString(pf.ExchangeSender)),
+ zap.Int64("mpp-version", mppTask.MppVersion.ToInt64()),
+ zap.String("exchange-compression-mode", pf.ExchangeSender.CompressionMode.Name()),
+ )
req := &kv.MPPDispatchRequest{
- Data: pbData,
- Meta: mppTask.Meta,
- ID: mppTask.ID,
- IsRoot: pf.IsRoot,
- Timeout: 10,
- SchemaVar: e.is.SchemaMetaVersion(),
- StartTs: e.startTS,
- State: kv.MppTaskReady,
+ Data: pbData,
+ Meta: mppTask.Meta,
+ ID: mppTask.ID,
+ IsRoot: pf.IsRoot,
+ Timeout: 10,
+ SchemaVar: e.is.SchemaMetaVersion(),
+ StartTs: e.startTS,
+ MppQueryID: mppTask.MppQueryID,
+ State: kv.MppTaskReady,
}
e.mppReqs = append(e.mppReqs, req)
}
@@ -109,7 +134,7 @@ func (e *MPPGather) Open(ctx context.Context) (err error) {
// TODO: Move the construct tasks logic to planner, so we can see the explain results.
sender := e.originalPlan.(*plannercore.PhysicalExchangeSender)
planIDs := collectPlanIDS(e.originalPlan, nil)
- frags, err := plannercore.GenerateRootMPPTasks(e.ctx, e.startTS, sender, e.is)
+ frags, err := plannercore.GenerateRootMPPTasks(e.ctx, e.startTS, e.mppQueryID, sender, e.is)
if err != nil {
return errors.Trace(err)
}
@@ -124,7 +149,7 @@ func (e *MPPGather) Open(ctx context.Context) (err error) {
failpoint.Return(errors.Errorf("The number of tasks is not right, expect %d tasks but actually there are %d tasks", val.(int), len(e.mppReqs)))
}
})
- e.respIter, err = distsql.DispatchMPPTasks(ctx, e.ctx, e.mppReqs, e.retFieldTypes, planIDs, e.id, e.startTS)
+ e.respIter, err = distsql.DispatchMPPTasks(ctx, e.ctx, e.mppReqs, e.retFieldTypes, planIDs, e.id, e.startTS, e.mppQueryID, e.memTracker)
if err != nil {
return errors.Trace(err)
}
diff --git a/executor/oomtest/BUILD.bazel b/executor/oomtest/BUILD.bazel
index 2fd8fe00f7d23..d6d7915a0b634 100644
--- a/executor/oomtest/BUILD.bazel
+++ b/executor/oomtest/BUILD.bazel
@@ -2,7 +2,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test")
go_test(
name = "oomtest_test",
- timeout = "short",
+ timeout = "moderate",
srcs = ["oom_test.go"],
flaky = True,
race = "on",
diff --git a/executor/oomtest/oom_test.go b/executor/oomtest/oom_test.go
index 8a9f2c0263d76..fc95bb47ceab8 100644
--- a/executor/oomtest/oom_test.go
+++ b/executor/oomtest/oom_test.go
@@ -222,6 +222,11 @@ func (h *oomCapture) Write(entry zapcore.Entry, fields []zapcore.Field) error {
h.tracker = str[begin+len("8001]") : end]
return nil
}
+ // They are just common background task and not related to the oom.
+ if entry.Message == "SetTiFlashGroupConfig" ||
+ entry.Message == "record table item load status failed due to not finding item" {
+ return nil
+ }
h.mu.Lock()
h.tracker = entry.Message
diff --git a/executor/parallel_apply.go b/executor/parallel_apply.go
index 2c4499b14818c..a0d418cc0e441 100644
--- a/executor/parallel_apply.go
+++ b/executor/parallel_apply.go
@@ -176,7 +176,6 @@ func (e *ParallelNestedLoopApplyExec) Close() error {
if e.runtimeStats != nil {
runtimeStats := newJoinRuntimeStats()
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, runtimeStats)
if e.useCache {
var hitRatio float64
if e.cacheAccessCounter > 0 {
@@ -187,6 +186,7 @@ func (e *ParallelNestedLoopApplyExec) Close() error {
runtimeStats.setCacheInfo(false, 0)
}
runtimeStats.SetConcurrencyInfo(execdetails.NewConcurrencyInfo("Concurrency", e.concurrency))
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, runtimeStats)
}
return err
}
diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go
index 50bb68a7b5235..85b096c28ff0d 100644
--- a/executor/partition_table_test.go
+++ b/executor/partition_table_test.go
@@ -17,6 +17,7 @@ package executor_test
import (
"fmt"
"math/rand"
+ "strconv"
"strings"
"testing"
"time"
@@ -84,7 +85,7 @@ partition p2 values less than (10))`)
// Table reader: one partition
tk.MustQuery("select * from pt where c > 8").Check(testkit.Rows("9 9"))
// Table reader: more than one partition
- tk.MustQuery("select * from pt where c < 2 or c >= 9").Check(testkit.Rows("0 0", "9 9"))
+ tk.MustQuery("select * from pt where c < 2 or c >= 9").Sort().Check(testkit.Rows("0 0", "9 9"))
// Index reader
tk.MustQuery("select c from pt").Sort().Check(testkit.Rows("0", "2", "4", "6", "7", "9", ""))
@@ -96,7 +97,7 @@ partition p2 values less than (10))`)
tk.MustQuery("select /*+ use_index(pt, i_id) */ * from pt").Sort().Check(testkit.Rows("0 0", "2 2", "4 4", "6 6", "7 7", "9 9", " "))
tk.MustQuery("select /*+ use_index(pt, i_id) */ * from pt where id < 4 and c > 10").Check(testkit.Rows())
tk.MustQuery("select /*+ use_index(pt, i_id) */ * from pt where id < 10 and c > 8").Check(testkit.Rows("9 9"))
- tk.MustQuery("select /*+ use_index(pt, i_id) */ * from pt where id < 10 and c < 2 or c >= 9").Check(testkit.Rows("0 0", "9 9"))
+ tk.MustQuery("select /*+ use_index(pt, i_id) */ * from pt where id < 10 and c < 2 or c >= 9").Sort().Check(testkit.Rows("0 0", "9 9"))
// Index Merge
tk.MustExec("set @@tidb_enable_index_merge = 1")
@@ -356,7 +357,7 @@ func TestPartitionInfoDisable(t *testing.T) {
tk.MustQuery("select * from t_info_null where (date = '2020-10-02' or date = '2020-10-06') and app = 'xxx' and media = '19003006'").Check(testkit.Rows())
}
-func TestOrderByandLimit(t *testing.T) {
+func TestOrderByAndLimit(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -377,14 +378,148 @@ func TestOrderByandLimit(t *testing.T) {
// regular table
tk.MustExec("create table tregular(a int, b int, index idx_a(a))")
+ // range partition table with int pk
+ tk.MustExec(`create table trange_intpk(a int primary key, b int) partition by range(a) (
+ partition p0 values less than(300),
+ partition p1 values less than (500),
+ partition p2 values less than(1100));`)
+
+ // hash partition table with int pk
+ tk.MustExec("create table thash_intpk(a int primary key, b int) partition by hash(a) partitions 4;")
+
+ // regular table with int pk
+ tk.MustExec("create table tregular_intpk(a int primary key, b int)")
+
+ // range partition table with clustered index
+ tk.MustExec(`create table trange_clustered(a int, b int, primary key(a, b) clustered) partition by range(a) (
+ partition p0 values less than(300),
+ partition p1 values less than (500),
+ partition p2 values less than(1100));`)
+
+ // hash partition table with clustered index
+ tk.MustExec("create table thash_clustered(a int, b int, primary key(a, b) clustered) partition by hash(a) partitions 4;")
+
+ // regular table with clustered index
+ tk.MustExec("create table tregular_clustered(a int, b int, primary key(a, b) clustered)")
+
+ listVals := make([]int, 0, 2000)
+
+ for i := 0; i < 2000; i++ {
+ listVals = append(listVals, i)
+ }
+ rand.Shuffle(len(listVals), func(i, j int) {
+ listVals[i], listVals[j] = listVals[j], listVals[i]
+ })
+
+ var listVals1, listVals2, listVals3 string
+
+ for i := 0; i <= 600; i++ {
+ listVals1 += strconv.Itoa(listVals[i])
+ if i != 600 {
+ listVals1 += ","
+ }
+ }
+ for i := 601; i <= 1200; i++ {
+ listVals2 += strconv.Itoa(listVals[i])
+ if i != 1200 {
+ listVals2 += ","
+ }
+ }
+ for i := 1201; i <= 1999; i++ {
+ listVals3 += strconv.Itoa(listVals[i])
+ if i != 1999 {
+ listVals3 += ","
+ }
+ }
+
+ tk.MustExec(fmt.Sprintf(`create table tlist_intpk(a int primary key, b int) partition by list(a)(
+ partition p1 values in (%s),
+ partition p2 values in (%s),
+ partition p3 values in (%s)
+ )`, listVals1, listVals2, listVals3))
+ tk.MustExec(fmt.Sprintf(`create table tlist(a int, b int, index idx_a(a)) partition by list(a)(
+ partition p1 values in (%s),
+ partition p2 values in (%s),
+ partition p3 values in (%s)
+ )`, listVals1, listVals2, listVals3))
+ tk.MustExec(fmt.Sprintf(`create table tlist_clustered(a int, b int, primary key(a, b)) partition by list(a)(
+ partition p1 values in (%s),
+ partition p2 values in (%s),
+ partition p3 values in (%s)
+ )`, listVals1, listVals2, listVals3))
+
// generate some random data to be inserted
vals := make([]string, 0, 2000)
for i := 0; i < 2000; i++ {
vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(1100), rand.Intn(2000)))
}
- tk.MustExec("insert into trange values " + strings.Join(vals, ","))
- tk.MustExec("insert into thash values " + strings.Join(vals, ","))
- tk.MustExec("insert into tregular values " + strings.Join(vals, ","))
+
+ dedupValsA := make([]string, 0, 2000)
+ dedupMapA := make(map[int]struct{}, 2000)
+ for i := 0; i < 2000; i++ {
+ valA := rand.Intn(1100)
+ if _, ok := dedupMapA[valA]; ok {
+ continue
+ }
+ dedupValsA = append(dedupValsA, fmt.Sprintf("(%v, %v)", valA, rand.Intn(2000)))
+ dedupMapA[valA] = struct{}{}
+ }
+
+ dedupValsAB := make([]string, 0, 2000)
+ dedupMapAB := make(map[string]struct{}, 2000)
+ for i := 0; i < 2000; i++ {
+ val := fmt.Sprintf("(%v, %v)", rand.Intn(1100), rand.Intn(2000))
+ if _, ok := dedupMapAB[val]; ok {
+ continue
+ }
+ dedupValsAB = append(dedupValsAB, val)
+ dedupMapAB[val] = struct{}{}
+ }
+
+ valInserted := strings.Join(vals, ",")
+ valDedupAInserted := strings.Join(dedupValsA, ",")
+ valDedupABInserted := strings.Join(dedupValsAB, ",")
+
+ tk.MustExec("insert into trange values " + valInserted)
+ tk.MustExec("insert into thash values " + valInserted)
+ tk.MustExec("insert into tlist values" + valInserted)
+ tk.MustExec("insert into tregular values " + valInserted)
+ tk.MustExec("insert into trange_intpk values " + valDedupAInserted)
+ tk.MustExec("insert into thash_intpk values " + valDedupAInserted)
+ tk.MustExec("insert into tlist_intpk values " + valDedupAInserted)
+ tk.MustExec("insert into tregular_intpk values " + valDedupAInserted)
+ tk.MustExec("insert into trange_clustered values " + valDedupABInserted)
+ tk.MustExec("insert into thash_clustered values " + valDedupABInserted)
+ tk.MustExec("insert into tlist_clustered values " + valDedupABInserted)
+ tk.MustExec("insert into tregular_clustered values " + valDedupABInserted)
+
+ tk.MustExec("analyze table trange")
+ tk.MustExec("analyze table trange_intpk")
+ tk.MustExec("analyze table trange_clustered")
+ tk.MustExec("analyze table thash")
+ tk.MustExec("analyze table thash_intpk")
+ tk.MustExec("analyze table thash_clustered")
+ tk.MustExec("analyze table tregular")
+ tk.MustExec("analyze table tregular_intpk")
+ tk.MustExec("analyze table tregular_clustered")
+ tk.MustExec("analyze table tlist")
+ tk.MustExec("analyze table tlist_intpk")
+ tk.MustExec("analyze table tlist_clustered")
+
+ // Create virtual tiflash replica info.
+ dom := domain.GetDomain(tk.Session())
+ is := dom.InfoSchema()
+ db, exists := is.SchemaByName(model.NewCIStr("test_orderby_limit"))
+ require.True(t, exists)
+ for _, tblInfo := range db.Tables {
+ if strings.HasPrefix(tblInfo.Name.L, "tr") || strings.HasPrefix(tblInfo.Name.L, "thash") || strings.HasPrefix(tblInfo.Name.L, "tlist") {
+ tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{
+ Count: 1,
+ Available: true,
+ }
+ }
+ }
+ tk.MustExec("set @@session.tidb_isolation_read_engines=\"tikv\"")
// test indexLookUp
for i := 0; i < 100; i++ {
@@ -398,6 +533,34 @@ func TestOrderByandLimit(t *testing.T) {
tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
}
+ // test indexLookUp with order property pushed down.
+ for i := 0; i < 100; i++ {
+ // explain select * from t where a > {y} use index(idx_a) order by a limit {x}; // check if IndexLookUp is used
+ // select * from t where a > {y} use index(idx_a) order by a limit {x}; // it can return the correct result
+ x := rand.Intn(1099)
+ y := rand.Intn(2000) + 1
+ // Since we only use order by a not order by a, b, the result is not stable when we read both a and b.
+ // We cut the max element so that the result can be stable.
+ maxEle := tk.MustQuery(fmt.Sprintf("select ifnull(max(a), 1100) from (select * from tregular use index(idx_a) where a > %v order by a limit %v) t", x, y)).Rows()[0][0]
+ queryRangePartitionWithLimitHint := fmt.Sprintf("select /*+ LIMIT_TO_COP() */ * from trange use index(idx_a) where a > %v and a < greatest(%v+1, %v) order by a limit %v", x, x+1, maxEle, y)
+ queryHashPartitionWithLimitHint := fmt.Sprintf("select /*+ LIMIT_TO_COP() */ * from thash use index(idx_a) where a > %v and a < greatest(%v+1, %v) order by a limit %v", x, x+1, maxEle, y)
+ queryListPartitionWithLimitHint := fmt.Sprintf("select /*+ LIMIT_TO_COP() */ * from thash use index(idx_a) where a > %v and a < greatest(%v+1, %v) order by a limit %v", x, x+1, maxEle, y)
+ queryRegular := fmt.Sprintf("select * from tregular use index(idx_a) where a > %v and a < greatest(%v+1, %v) order by a limit %v;", x, x+1, maxEle, y)
+ require.True(t, tk.HasPlan(queryRangePartitionWithLimitHint, "Limit"))
+ require.True(t, tk.HasPlan(queryRangePartitionWithLimitHint, "IndexLookUp"))
+ require.True(t, tk.HasPlan(queryHashPartitionWithLimitHint, "Limit"))
+ require.True(t, tk.HasPlan(queryHashPartitionWithLimitHint, "IndexLookUp"))
+ require.True(t, tk.HasPlan(queryListPartitionWithLimitHint, "Limit"))
+ require.True(t, tk.HasPlan(queryListPartitionWithLimitHint, "IndexLookUp"))
+ require.True(t, tk.HasPlan(queryRangePartitionWithLimitHint, "TopN")) // but not fully pushed
+ require.True(t, tk.HasPlan(queryHashPartitionWithLimitHint, "TopN"))
+ require.True(t, tk.HasPlan(queryListPartitionWithLimitHint, "TopN"))
+ regularResult := tk.MustQuery(queryRegular).Sort().Rows()
+ tk.MustQuery(queryRangePartitionWithLimitHint).Sort().Check(regularResult)
+ tk.MustQuery(queryHashPartitionWithLimitHint).Sort().Check(regularResult)
+ tk.MustQuery(queryListPartitionWithLimitHint).Sort().Check(regularResult)
+ }
+
// test tableReader
for i := 0; i < 100; i++ {
// explain select * from t where a > {y} ignore index(idx_a) order by a limit {x}; // check if IndexLookUp is used
@@ -410,6 +573,119 @@ func TestOrderByandLimit(t *testing.T) {
tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
}
+ // test tableReader with order property pushed down.
+ for i := 0; i < 100; i++ {
+ // explain select * from t where a > {y} ignore index(idx_a) order by a limit {x}; // check if IndexLookUp is used
+ // select * from t where a > {y} ignore index(idx_a) order by a limit {x}; // it can return the correct result
+ x := rand.Intn(1099)
+ y := rand.Intn(2000) + 1
+ queryRangePartition := fmt.Sprintf("select /*+ LIMIT_TO_COP() */ * from trange ignore index(idx_a) where a > %v order by a, b limit %v;", x, y)
+ queryHashPartition := fmt.Sprintf("select /*+ LIMIT_TO_COP() */ * from thash ignore index(idx_a) where a > %v order by a, b limit %v;", x, y)
+ queryListPartition := fmt.Sprintf("select /*+ LIMIT_TO_COP() */ * from tlist ignore index(idx_a) where a > %v order by a, b limit %v;", x, y)
+ queryRegular := fmt.Sprintf("select * from tregular ignore index(idx_a) where a > %v order by a, b limit %v;", x, y)
+ require.True(t, tk.HasPlan(queryRangePartition, "TableReader")) // check if tableReader is used
+ require.True(t, tk.HasPlan(queryHashPartition, "TableReader"))
+ require.True(t, tk.HasPlan(queryListPartition, "TableReader"))
+ require.False(t, tk.HasPlan(queryRangePartition, "Limit")) // check if order property is not pushed
+ require.False(t, tk.HasPlan(queryHashPartition, "Limit"))
+ require.False(t, tk.HasPlan(queryListPartition, "Limit"))
+ regularResult := tk.MustQuery(queryRegular).Sort().Rows()
+ tk.MustQuery(queryRangePartition).Sort().Check(regularResult)
+ tk.MustQuery(queryHashPartition).Sort().Check(regularResult)
+ tk.MustQuery(queryListPartition).Sort().Check(regularResult)
+
+ // test int pk
+ // To be simplified, we only read column a.
+ queryRangePartition = fmt.Sprintf("select /*+ LIMIT_TO_COP() */ a from trange_intpk use index(primary) where a > %v order by a limit %v", x, y)
+ queryHashPartition = fmt.Sprintf("select /*+ LIMIT_TO_COP() */ a from thash_intpk use index(primary) where a > %v order by a limit %v", x, y)
+ queryListPartition = fmt.Sprintf("select /*+ LIMIT_TO_COP() */ a from tlist_intpk use index(primary) where a > %v order by a limit %v", x, y)
+ queryRegular = fmt.Sprintf("select a from tregular_intpk where a > %v order by a limit %v", x, y)
+ require.True(t, tk.HasPlan(queryRangePartition, "TableReader"))
+ require.True(t, tk.HasPlan(queryHashPartition, "TableReader"))
+ require.True(t, tk.HasPlan(queryListPartition, "TableReader"))
+ require.True(t, tk.HasPlan(queryRangePartition, "Limit")) // check if order property is not pushed
+ require.True(t, tk.HasPlan(queryHashPartition, "Limit"))
+ require.True(t, tk.HasPlan(queryListPartition, "Limit"))
+ regularResult = tk.MustQuery(queryRegular).Rows()
+ tk.MustQuery(queryRangePartition).Check(regularResult)
+ tk.MustQuery(queryHashPartition).Check(regularResult)
+ tk.MustQuery(queryListPartition).Check(regularResult)
+
+ // test clustered index
+ queryRangePartition = fmt.Sprintf("select /*+ LIMIT_TO_COP() */ * from trange_clustered use index(primary) where a > %v order by a, b limit %v;", x, y)
+ queryHashPartition = fmt.Sprintf("select /*+ LIMIT_TO_COP() */ * from thash_clustered use index(primary) where a > %v order by a, b limit %v;", x, y)
+ queryListPartition = fmt.Sprintf("select /*+ LIMIT_TO_COP() */ * from tlist_clustered use index(primary) where a > %v order by a, b limit %v;", x, y)
+ queryRegular = fmt.Sprintf("select * from tregular_clustered where a > %v order by a, b limit %v;", x, y)
+ require.True(t, tk.HasPlan(queryRangePartition, "TableReader")) // check if tableReader is used
+ require.True(t, tk.HasPlan(queryHashPartition, "TableReader"))
+ require.True(t, tk.HasPlan(queryListPartition, "TableReader"))
+ require.True(t, tk.HasPlan(queryRangePartition, "Limit")) // check if order property is pushed
+ require.True(t, tk.HasPlan(queryHashPartition, "Limit"))
+ require.True(t, tk.HasPlan(queryListPartition, "Limit"))
+ require.True(t, tk.HasPlan(queryRangePartition, "TopN")) // but not fully pushed
+ require.True(t, tk.HasPlan(queryHashPartition, "TopN"))
+ require.True(t, tk.HasPlan(queryListPartition, "TopN"))
+ regularResult = tk.MustQuery(queryRegular).Rows()
+ tk.MustQuery(queryRangePartition).Check(regularResult)
+ tk.MustQuery(queryHashPartition).Check(regularResult)
+ tk.MustQuery(queryListPartition).Check(regularResult)
+
+ tk.MustExec(" set @@tidb_allow_mpp=1;")
+ tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash,tikv\"")
+ queryPartitionWithTiFlash := fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_intpk]) */ * from trange_intpk where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ // but order is not pushed
+ require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_intpk]) */ /*+ LIMIT_TO_COP() */ * from trange_intpk where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ // but order is not pushed
+ require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_clustered]) */ * from trange_clustered where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_clustered]) */ /*+ LIMIT_TO_COP() */ * from trange_clustered where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash))
+ // but order is not pushed
+ require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_intpk]) */ * from thash_intpk where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_intpk]) */ /*+ LIMIT_TO_COP() */ * from thash_intpk where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash))
+ // but order is not pushed
+ require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_clustered]) */ * from thash_clustered where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_clustered]) */ /*+ LIMIT_TO_COP() */ * from thash_clustered where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash))
+ // but order is not pushed
+ require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[tlist_intpk]) */ * from tlist_intpk where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[tlist_intpk]) */ /*+ LIMIT_TO_COP() */ * from tlist_intpk where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash))
+ // but order is not pushed
+ require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[tlist_clustered]) */ * from tlist_clustered where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[tlist_clustered]) */ /*+ LIMIT_TO_COP() */ * from tlist_clustered where a > %v order by a limit %v", x, y)
+ // check if tiflash is used
+ require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash))
+ // but order is not pushed
+ require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows()))
+ tk.MustExec(" set @@tidb_allow_mpp=0;")
+ tk.MustExec("set @@session.tidb_isolation_read_engines=\"tikv\"")
+ }
+
// test indexReader
for i := 0; i < 100; i++ {
// explain select a from t where a > {y} use index(idx_a) order by a limit {x}; // check if IndexLookUp is used
@@ -422,6 +698,24 @@ func TestOrderByandLimit(t *testing.T) {
tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
}
+ // test indexReader with order property pushed down.
+ for i := 0; i < 100; i++ {
+ // explain select a from t where a > {y} use index(idx_a) order by a limit {x}; // check if IndexLookUp is used
+ // select a from t where a > {y} use index(idx_a) order by a limit {x}; // it can return the correct result
+ x := rand.Intn(1099)
+ y := rand.Intn(2000) + 1
+ queryRangePartition := fmt.Sprintf("select /*+ LIMIT_TO_COP() */ a from trange use index(idx_a) where a > %v order by a limit %v;", x, y)
+ queryHashPartition := fmt.Sprintf("select /*+ LIMIT_TO_COP() */ a from trange use index(idx_a) where a > %v order by a limit %v;", x, y)
+ queryRegular := fmt.Sprintf("select a from tregular use index(idx_a) where a > %v order by a limit %v;", x, y)
+ require.True(t, tk.HasPlan(queryRangePartition, "IndexReader")) // check if indexReader is used
+ require.True(t, tk.HasPlan(queryHashPartition, "IndexReader"))
+ require.True(t, tk.HasPlan(queryRangePartition, "Limit")) // check if order property is pushed
+ require.True(t, tk.HasPlan(queryHashPartition, "Limit"))
+ regularResult := tk.MustQuery(queryRegular).Sort().Rows()
+ tk.MustQuery(queryRangePartition).Sort().Check(regularResult)
+ tk.MustQuery(queryHashPartition).Sort().Check(regularResult)
+ }
+
// test indexMerge
for i := 0; i < 100; i++ {
// explain select /*+ use_index_merge(t) */ * from t where a > 2 or b < 5 order by a limit {x}; // check if IndexMerge is used
@@ -434,6 +728,17 @@ func TestOrderByandLimit(t *testing.T) {
}
}
+func TestOrderByOnUnsignedPk(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table tunsigned_hash(a bigint unsigned primary key) partition by hash(a) partitions 6")
+ tk.MustExec("insert into tunsigned_hash values(25), (9279808998424041135)")
+ tk.MustQuery("select min(a) from tunsigned_hash").Check(testkit.Rows("25"))
+ tk.MustQuery("select max(a) from tunsigned_hash").Check(testkit.Rows("9279808998424041135"))
+}
+
func TestBatchGetandPointGetwithHashPartition(t *testing.T) {
store := testkit.CreateMockStore(t)
@@ -2834,7 +3139,7 @@ partition p1 values less than (7),
partition p2 values less than (10))`)
tk.MustExec("alter table p add unique idx(id)")
tk.MustExec("insert into p values (1,3), (3,4), (5,6), (7,9)")
- tk.MustQuery("select id from p use index (idx)").Check(testkit.Rows("1", "3", "5", "7"))
+ tk.MustQuery("select id from p use index (idx) order by id").Check(testkit.Rows("1", "3", "5", "7"))
}
func TestGlobalIndexDoubleRead(t *testing.T) {
@@ -3531,11 +3836,11 @@ func TestPartitionTableExplain(t *testing.T) {
" └─IndexFullScan 1.00 cop[tikv] table:t, partition:p1, index:b(b) keep order:false"))
tk.MustQuery(`explain format = 'brief' select * from t,t2 where t2.a = 1 and t2.b = t.b and t.a = 1`).Check(testkit.Rows(
"HashJoin 1.00 root inner join, equal:[eq(testpartitiontableexplain.t.b, testpartitiontableexplain.t2.b)]",
- "├─TableReader(Build) 1.00 root data:Selection",
- "│ └─Selection 1.00 cop[tikv] eq(testpartitiontableexplain.t2.a, 1), not(isnull(testpartitiontableexplain.t2.b))",
- "│ └─TableFullScan 3.00 cop[tikv] table:t2 keep order:false",
- "└─Selection(Probe) 1.00 root not(isnull(testpartitiontableexplain.t.b))",
- " └─Point_Get 1.00 root table:t, partition:p1 handle:1"))
+ `├─Selection(Build) 1.00 root not(isnull(testpartitiontableexplain.t.b))`,
+ `│ └─Point_Get 1.00 root table:t, partition:p1 handle:1`,
+ `└─TableReader(Probe) 1.00 root data:Selection`,
+ ` └─Selection 1.00 cop[tikv] eq(testpartitiontableexplain.t2.a, 1), not(isnull(testpartitiontableexplain.t2.b))`,
+ ` └─TableFullScan 3.00 cop[tikv] table:t2 keep order:false`))
tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'")
tk.MustExec(`analyze table t`)
@@ -3628,3 +3933,72 @@ func TestIssue21732(t *testing.T) {
})
}
}
+
+func TestIssue39999(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec(`create schema test39999`)
+ tk.MustExec(`use test39999`)
+ tk.MustExec(`drop table if exists c, t`)
+ tk.MustExec("CREATE TABLE `c` (" +
+ "`serial_id` varchar(24)," +
+ "`occur_trade_date` date," +
+ "`txt_account_id` varchar(24)," +
+ "`capital_sub_class` varchar(10)," +
+ "`occur_amount` decimal(16,2)," +
+ "`broker` varchar(10)," +
+ "PRIMARY KEY (`txt_account_id`,`occur_trade_date`,`serial_id`) /*T![clustered_index] CLUSTERED */," +
+ "KEY `idx_serial_id` (`serial_id`)" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci " +
+ "PARTITION BY RANGE COLUMNS(`serial_id`) (" +
+ "PARTITION `p202209` VALUES LESS THAN ('20221001')," +
+ "PARTITION `p202210` VALUES LESS THAN ('20221101')," +
+ "PARTITION `p202211` VALUES LESS THAN ('20221201')" +
+ ")")
+
+ tk.MustExec("CREATE TABLE `t` ( " +
+ "`txn_account_id` varchar(24), " +
+ "`account_id` varchar(32), " +
+ "`broker` varchar(10), " +
+ "PRIMARY KEY (`txn_account_id`) /*T![clustered_index] CLUSTERED */ " +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci")
+
+ tk.MustExec("INSERT INTO `c` (serial_id, txt_account_id, capital_sub_class, occur_trade_date, occur_amount, broker) VALUES ('2022111700196920','04482786','CUST','2022-11-17',-2.01,'0009')")
+ tk.MustExec("INSERT INTO `t` VALUES ('04482786','1142927','0009')")
+
+ tk.MustExec(`set tidb_partition_prune_mode='dynamic'`)
+ tk.MustExec(`analyze table c`)
+ tk.MustExec(`analyze table t`)
+ query := `select
+ /*+ inl_join(c) */
+ c.occur_amount
+from
+ c
+ join t on c.txt_account_id = t.txn_account_id
+ and t.broker = '0009'
+ and c.occur_trade_date = '2022-11-17'`
+ tk.MustQuery("explain " + query).Check(testkit.Rows(""+
+ "IndexJoin_22 1.00 root inner join, inner:TableReader_21, outer key:test39999.t.txn_account_id, inner key:test39999.c.txt_account_id, equal cond:eq(test39999.t.txn_account_id, test39999.c.txt_account_id)",
+ "├─TableReader_27(Build) 1.00 root data:Selection_26",
+ "│ └─Selection_26 1.00 cop[tikv] eq(test39999.t.broker, \"0009\")",
+ "│ └─TableFullScan_25 1.00 cop[tikv] table:t keep order:false",
+ "└─TableReader_21(Probe) 1.00 root partition:all data:Selection_20",
+ " └─Selection_20 1.00 cop[tikv] eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)",
+ " └─TableRangeScan_19 1.00 cop[tikv] table:c range: decided by [eq(test39999.c.txt_account_id, test39999.t.txn_account_id) eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false"))
+ tk.MustQuery(query).Check(testkit.Rows("-2.01"))
+
+ // Add the missing partition key part.
+ tk.MustExec(`alter table t add column serial_id varchar(24) default '2022111700196920'`)
+ query += ` and c.serial_id = t.serial_id`
+ tk.MustQuery(query).Check(testkit.Rows("-2.01"))
+ tk.MustQuery("explain " + query).Check(testkit.Rows(""+
+ `IndexJoin_20 0.80 root inner join, inner:TableReader_19, outer key:test39999.t.txn_account_id, test39999.t.serial_id, inner key:test39999.c.txt_account_id, test39999.c.serial_id, equal cond:eq(test39999.t.serial_id, test39999.c.serial_id), eq(test39999.t.txn_account_id, test39999.c.txt_account_id)`,
+ `├─TableReader_25(Build) 0.80 root data:Selection_24`,
+ `│ └─Selection_24 0.80 cop[tikv] eq(test39999.t.broker, "0009"), not(isnull(test39999.t.serial_id))`,
+ `│ └─TableFullScan_23 1.00 cop[tikv] table:t keep order:false`,
+ `└─TableReader_19(Probe) 0.80 root partition:all data:Selection_18`,
+ ` └─Selection_18 0.80 cop[tikv] eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)`,
+ ` └─TableRangeScan_17 0.80 cop[tikv] table:c range: decided by [eq(test39999.c.txt_account_id, test39999.t.txn_account_id) eq(test39999.c.serial_id, test39999.t.serial_id) eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false`))
+}
diff --git a/executor/plan_replayer.go b/executor/plan_replayer.go
index fec3de1867933..f8a7ffe90fcca 100644
--- a/executor/plan_replayer.go
+++ b/executor/plan_replayer.go
@@ -18,25 +18,24 @@ import (
"archive/zip"
"bytes"
"context"
- "crypto/rand"
- "encoding/base64"
"encoding/json"
"fmt"
"os"
- "path/filepath"
"strings"
- "time"
"github.com/BurntSushi/toml"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/infoschema"
+ "github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/sessiontxn"
"github.com/pingcap/tidb/statistics/handle"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/replayer"
"github.com/pingcap/tidb/util/sqlexec"
"go.uber.org/zap"
)
@@ -47,8 +46,16 @@ var _ Executor = &PlanReplayerLoadExec{}
// PlanReplayerExec represents a plan replayer executor.
type PlanReplayerExec struct {
baseExecutor
- DumpInfo *PlanReplayerDumpInfo
- endFlag bool
+ CaptureInfo *PlanReplayerCaptureInfo
+ DumpInfo *PlanReplayerDumpInfo
+ endFlag bool
+}
+
+// PlanReplayerCaptureInfo indicates capture info
+type PlanReplayerCaptureInfo struct {
+ SQLDigest string
+ PlanDigest string
+ Remove bool
}
// PlanReplayerDumpInfo indicates dump info
@@ -67,6 +74,12 @@ func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error {
if e.endFlag {
return nil
}
+ if e.CaptureInfo != nil {
+ if e.CaptureInfo.Remove {
+ return e.removeCaptureTask(ctx)
+ }
+ return e.registerCaptureTask(ctx)
+ }
err := e.createFile()
if err != nil {
return err
@@ -93,50 +106,69 @@ func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error {
return nil
}
-func (e *PlanReplayerExec) createFile() error {
- var err error
- e.DumpInfo.File, e.DumpInfo.FileName, err = GeneratePlanReplayerFile()
+func (e *PlanReplayerExec) removeCaptureTask(ctx context.Context) error {
+ ctx1 := kv.WithInternalSourceType(ctx, kv.InternalTxnStats)
+ exec := e.ctx.(sqlexec.RestrictedSQLExecutor)
+ _, _, err := exec.ExecRestrictedSQL(ctx1, nil, fmt.Sprintf("delete from mysql.plan_replayer_task where sql_digest = '%s' and plan_digest = '%s'",
+ e.CaptureInfo.SQLDigest, e.CaptureInfo.PlanDigest))
if err != nil {
+ logutil.BgLogger().Warn("remove mysql.plan_replayer_status record failed",
+ zap.Error(err))
return err
}
+ err = domain.GetDomain(e.ctx).GetPlanReplayerHandle().CollectPlanReplayerTask()
+ if err != nil {
+ logutil.BgLogger().Warn("collect task failed", zap.Error(err))
+ }
+ logutil.BgLogger().Info("collect plan replayer task success")
+ e.endFlag = true
return nil
}
-// GeneratePlanReplayerFile generates plan replayer file
-func GeneratePlanReplayerFile() (*os.File, string, error) {
- path := domain.GetPlanReplayerDirName()
- err := os.MkdirAll(path, os.ModePerm)
+func (e *PlanReplayerExec) registerCaptureTask(ctx context.Context) error {
+ ctx1 := kv.WithInternalSourceType(ctx, kv.InternalTxnStats)
+ exists, err := domain.CheckPlanReplayerTaskExists(ctx1, e.ctx, e.CaptureInfo.SQLDigest, e.CaptureInfo.PlanDigest)
if err != nil {
- return nil, "", errors.AddStack(err)
+ return err
+ }
+ if exists {
+ return errors.New("plan replayer capture task already exists")
}
- fileName, err := generatePlanReplayerFileName()
+ exec := e.ctx.(sqlexec.RestrictedSQLExecutor)
+ _, _, err = exec.ExecRestrictedSQL(ctx1, nil, fmt.Sprintf("insert into mysql.plan_replayer_task (sql_digest, plan_digest) values ('%s','%s')",
+ e.CaptureInfo.SQLDigest, e.CaptureInfo.PlanDigest))
if err != nil {
- return nil, "", errors.AddStack(err)
+ logutil.BgLogger().Warn("insert mysql.plan_replayer_status record failed",
+ zap.Error(err))
+ return err
}
- zf, err := os.Create(filepath.Join(path, fileName))
+ err = domain.GetDomain(e.ctx).GetPlanReplayerHandle().CollectPlanReplayerTask()
if err != nil {
- return nil, "", errors.AddStack(err)
+ logutil.BgLogger().Warn("collect task failed", zap.Error(err))
}
- return zf, fileName, err
+ logutil.BgLogger().Info("collect plan replayer task success")
+ e.endFlag = true
+ return nil
}
-func generatePlanReplayerFileName() (string, error) {
- // Generate key and create zip file
- time := time.Now().UnixNano()
- b := make([]byte, 16)
- //nolint: gosec
- _, err := rand.Read(b)
+func (e *PlanReplayerExec) createFile() error {
+ var err error
+ e.DumpInfo.File, e.DumpInfo.FileName, err = replayer.GeneratePlanReplayerFile(false, false, false)
if err != nil {
- return "", err
+ return err
}
- key := base64.URLEncoding.EncodeToString(b)
- return fmt.Sprintf("replayer_%v_%v.zip", key, time), nil
+ return nil
}
func (e *PlanReplayerDumpInfo) dump(ctx context.Context) (err error) {
fileName := e.FileName
zf := e.File
+ startTS, err := sessiontxn.GetTxnManager(e.ctx).GetStmtReadTS()
+ if err != nil {
+ return err
+ }
task := &domain.PlanReplayerDumpTask{
+ StartTS: startTS,
FileName: fileName,
Zf: zf,
SessionVars: e.ctx.GetSessionVars(),
@@ -378,21 +410,23 @@ func createSchemaAndItems(ctx sessionctx.Context, f *zip.File) error {
if err != nil {
return errors.AddStack(err)
}
- sqls := strings.Split(buf.String(), ";")
- if len(sqls) != 3 {
- return errors.New("plan replayer: create schema and tables failed")
- }
+ originText := buf.String()
+ index1 := strings.Index(originText, ";")
+ createDatabaseSQL := originText[:index1+1]
+ index2 := strings.Index(originText[index1+1:], ";")
+ useDatabaseSQL := originText[index1+1:][:index2+1]
+ createTableSQL := originText[index1+1:][index2+1:]
c := context.Background()
// create database if not exists
- _, err = ctx.(sqlexec.SQLExecutor).Execute(c, sqls[0])
+ _, err = ctx.(sqlexec.SQLExecutor).Execute(c, createDatabaseSQL)
logutil.BgLogger().Debug("plan replayer: skip error", zap.Error(err))
// use database
- _, err = ctx.(sqlexec.SQLExecutor).Execute(c, sqls[1])
+ _, err = ctx.(sqlexec.SQLExecutor).Execute(c, useDatabaseSQL)
if err != nil {
return err
}
// create table or view
- _, err = ctx.(sqlexec.SQLExecutor).Execute(c, sqls[2])
+ _, err = ctx.(sqlexec.SQLExecutor).Execute(c, createTableSQL)
if err != nil {
return err
}
@@ -439,6 +473,9 @@ func (e *PlanReplayerLoadInfo) Update(data []byte) error {
// build schema and table first
for _, zipFile := range z.File {
+ if zipFile.Name == fmt.Sprintf("schema/%v", domain.PlanReplayerSchemaMetaFile) {
+ continue
+ }
path := strings.Split(zipFile.Name, "/")
if len(path) == 2 && strings.Compare(path[0], "schema") == 0 {
err = createSchemaAndItems(e.Ctx, zipFile)
diff --git a/executor/point_get.go b/executor/point_get.go
index f65c52c06cb1b..8efda04a970e5 100644
--- a/executor/point_get.go
+++ b/executor/point_get.go
@@ -74,13 +74,13 @@ func (b *executorBuilder) buildPointGet(p *plannercore.PointGetPlan) Executor {
if b.ctx.GetSessionVars().IsReplicaReadClosestAdaptive() {
e.snapshot.SetOption(kv.ReplicaReadAdjuster, newReplicaReadAdjuster(e.ctx, p.GetAvgRowSize()))
}
+ e.snapshot.SetOption(kv.ResourceGroupName, b.ctx.GetSessionVars().ResourceGroupName)
if e.runtimeStats != nil {
snapshotStats := &txnsnapshot.SnapshotRuntimeStats{}
e.stats = &runtimeStatsWithSnapshot{
SnapshotRuntimeStats: snapshotStats,
}
e.snapshot.SetOption(kv.CollectRuntimeStats, snapshotStats)
- b.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
if p.IndexInfo != nil {
@@ -194,6 +194,9 @@ func (e *PointGetExecutor) Open(context.Context) error {
// Close implements the Executor interface.
func (e *PointGetExecutor) Close() error {
+ if e.stats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
+ }
if e.runtimeStats != nil && e.snapshot != nil {
e.snapshot.SetOption(kv.CollectRuntimeStats, nil)
}
@@ -347,8 +350,8 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error {
return err
}
- err = FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex,
- e.schema, e.columns, e.ctx, req)
+ err = table.FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex,
+ e.schema.Columns, e.columns, e.ctx, req)
if err != nil {
return err
}
diff --git a/executor/point_get_test.go b/executor/point_get_test.go
index c615c3a75cb1a..8f13675457481 100644
--- a/executor/point_get_test.go
+++ b/executor/point_get_test.go
@@ -825,3 +825,20 @@ func TestPointGetIssue25167(t *testing.T) {
tk.MustExec("insert into t values (1)")
tk.MustQuery("select * from t as of timestamp @a where a = 1").Check(testkit.Rows())
}
+
+func TestPointGetIssue40194(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t1(id int primary key, v int)")
+ tk.MustExec("insert into t1 values(1, 10)")
+ tk.MustExec("prepare s from 'select * from t1 where id=1'")
+ tk.MustExec("set @@tidb_enable_plan_replayer_capture=1")
+ tk.MustQuery("execute s").Check(testkit.Rows("1 10"))
+ tk.MustQuery("execute s").Check(testkit.Rows("1 10"))
+
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk2.MustExec("update t1 set v=v+1")
+ tk.MustQuery("execute s").Check(testkit.Rows("1 11"))
+}
diff --git a/executor/prepared.go b/executor/prepared.go
index ae742a30d8bb5..6a5025e0d539b 100644
--- a/executor/prepared.go
+++ b/executor/prepared.go
@@ -53,8 +53,6 @@ type PrepareExec struct {
Fields []*ast.ResultField
Stmt interface{}
- IsGeneralStmt bool
-
// If it's generated from executing "prepare stmt from '...'", the process is parse -> plan -> executor
// If it's generated from the prepare protocol, the process is session.PrepareStmt -> NewPrepareExec
// They both generate a PrepareExec struct, but the second case needs to reset the statement context while the first already do that.
@@ -117,7 +115,7 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error {
return err
}
}
- stmt, p, paramCnt, err := plannercore.GeneratePlanCacheStmtWithAST(ctx, e.ctx, stmt0)
+ stmt, p, paramCnt, err := plannercore.GeneratePlanCacheStmtWithAST(ctx, e.ctx, stmt0.Text(), stmt0)
if err != nil {
return err
}
@@ -135,19 +133,15 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error {
if !isNoResultPlan(p) {
e.Fields = colNames2ResultFields(p.Schema(), p.OutputNames(), vars.CurrentDB)
}
- if e.ID == 0 && !e.IsGeneralStmt {
+ if e.ID == 0 {
e.ID = vars.GetNextPreparedStmtID()
}
- if e.name != "" && !e.IsGeneralStmt {
+ if e.name != "" {
vars.PreparedStmtNameToID[e.name] = e.ID
}
e.ParamCount = paramCnt
e.Stmt = stmt
- if e.IsGeneralStmt {
- vars.AddGeneralPlanCacheStmt(e.sqlText, stmt)
- return nil
- }
return vars.AddPreparedStmt(e.ID, stmt)
}
diff --git a/executor/prepared_test.go b/executor/prepared_test.go
index 8ee6a0aa867c4..739ba1c0cd659 100644
--- a/executor/prepared_test.go
+++ b/executor/prepared_test.go
@@ -18,11 +18,13 @@ import (
"fmt"
"strconv"
"strings"
+ "sync/atomic"
"testing"
"github.com/pingcap/tidb/parser/auth"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
+ "github.com/pingcap/tidb/parser/terror"
plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/server"
"github.com/pingcap/tidb/sessionctx/variable"
@@ -1255,3 +1257,18 @@ func TestIssue31141(t *testing.T) {
tk.MustExec("set @@tidb_txn_mode = 'optimistic'")
tk.MustExec("prepare stmt1 from 'do 1'")
}
+
+func TestMaxPreparedStmtCount(t *testing.T) {
+ oldVal := atomic.LoadInt64(&variable.PreparedStmtCount)
+ atomic.StoreInt64(&variable.PreparedStmtCount, 0)
+ defer func() {
+ atomic.StoreInt64(&variable.PreparedStmtCount, oldVal)
+ }()
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.max_prepared_stmt_count = 2")
+ tk.MustExec("prepare stmt1 from 'select ? as num from dual'")
+ tk.MustExec("prepare stmt2 from 'select ? as num from dual'")
+ err := tk.ExecToErr("prepare stmt3 from 'select ? as num from dual'")
+ require.True(t, terror.ErrorEqual(err, variable.ErrMaxPreparedStmtCountReached))
+}
diff --git a/executor/projection.go b/executor/projection.go
index 27ce1bafc0b8a..c6c33d02651ff 100644
--- a/executor/projection.go
+++ b/executor/projection.go
@@ -94,7 +94,7 @@ func (e *ProjectionExec) Open(ctx context.Context) error {
return e.open(ctx)
}
-func (e *ProjectionExec) open(ctx context.Context) error {
+func (e *ProjectionExec) open(_ context.Context) error {
e.prepared = false
e.parentReqRows = int64(e.maxChunkSize)
@@ -364,14 +364,14 @@ func (f *projectionInputFetcher) run(ctx context.Context) {
}()
for {
- input := readProjectionInput(f.inputCh, f.globalFinishCh)
- if input == nil {
+ input, isNil := readProjection[*projectionInput](f.inputCh, f.globalFinishCh)
+ if isNil {
return
}
targetWorker := input.targetWorker
- output = readProjectionOutput(f.outputCh, f.globalFinishCh)
- if output == nil {
+ output, isNil = readProjection[*projectionOutput](f.outputCh, f.globalFinishCh)
+ if isNil {
f.proj.memTracker.Consume(-input.chk.MemoryUsage())
return
}
@@ -431,13 +431,13 @@ func (w *projectionWorker) run(ctx context.Context) {
w.proj.wg.Done()
}()
for {
- input := readProjectionInput(w.inputCh, w.globalFinishCh)
- if input == nil {
+ input, isNil := readProjection[*projectionInput](w.inputCh, w.globalFinishCh)
+ if isNil {
return
}
- output = readProjectionOutput(w.outputCh, w.globalFinishCh)
- if output == nil {
+ output, isNil = readProjection[*projectionOutput](w.outputCh, w.globalFinishCh)
+ if isNil {
return
}
@@ -462,26 +462,14 @@ func recoveryProjection(output *projectionOutput, r interface{}) {
logutil.BgLogger().Error("projection executor panicked", zap.String("error", fmt.Sprintf("%v", r)), zap.Stack("stack"))
}
-func readProjectionInput(inputCh <-chan *projectionInput, finishCh <-chan struct{}) *projectionInput {
+func readProjection[T any](ch <-chan T, finishCh <-chan struct{}) (t T, isNil bool) {
select {
case <-finishCh:
- return nil
- case input, ok := <-inputCh:
- if !ok {
- return nil
- }
- return input
- }
-}
-
-func readProjectionOutput(outputCh <-chan *projectionOutput, finishCh <-chan struct{}) *projectionOutput {
- select {
- case <-finishCh:
- return nil
- case output, ok := <-outputCh:
+ return t, true
+ case t, ok := <-ch:
if !ok {
- return nil
+ return t, true
}
- return output
+ return t, false
}
}
diff --git a/executor/recover_test.go b/executor/recover_test.go
index aad1c93d9fb87..92aa83ec99d3e 100644
--- a/executor/recover_test.go
+++ b/executor/recover_test.go
@@ -296,6 +296,43 @@ func TestRecoverTableMeetError(t *testing.T) {
tk.MustContainErrMsg("select * from t_recover", "Table 'test_recover.t_recover' doesn't exist")
}
+func TestRecoverTablePrivilege(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ timeBeforeDrop, _, safePointSQL, resetGC := MockGC(tk)
+ defer resetGC()
+
+ // Set GC safe point
+ tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t_recover")
+ tk.MustExec("create table t_recover (a int);")
+ tk.MustExec("drop table t_recover")
+
+ // Recover without drop/create privilege.
+ tk.MustExec("CREATE USER 'testrecovertable'@'localhost';")
+ newTk := testkit.NewTestKit(t, store)
+ require.NoError(t, newTk.Session().Auth(&auth.UserIdentity{Username: "testrecovertable", Hostname: "localhost"}, nil, nil))
+ newTk.MustGetErrCode("recover table t_recover", errno.ErrTableaccessDenied)
+ newTk.MustGetErrCode("flashback table t_recover", errno.ErrTableaccessDenied)
+
+ // Got drop privilege, still failed.
+ tk.MustExec("grant drop on *.* to 'testrecovertable'@'localhost';")
+ newTk.MustGetErrCode("recover table t_recover", errno.ErrTableaccessDenied)
+ newTk.MustGetErrCode("flashback table t_recover", errno.ErrTableaccessDenied)
+
+ // Got select, create and drop privilege, execute success.
+ tk.MustExec("grant select,create on *.* to 'testrecovertable'@'localhost';")
+ newTk.MustExec("use test")
+ newTk.MustExec("recover table t_recover")
+ newTk.MustExec("drop table t_recover")
+ newTk.MustExec("flashback table t_recover")
+
+ tk.MustExec("drop user 'testrecovertable'@'localhost';")
+}
+
func TestRecoverClusterMeetError(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -307,9 +344,7 @@ func TestRecoverClusterMeetError(t *testing.T) {
injectSafeTS := oracle.GoTimeToTS(flashbackTs.Add(10 * time.Second))
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockFlashbackTest", `return(true)`))
- require.NoError(t, failpoint.Enable("tikvclient/injectSafeTS",
- fmt.Sprintf("return(%v)", injectSafeTS)))
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS",
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/injectSafeTS",
fmt.Sprintf("return(%v)", injectSafeTS)))
// Get GC safe point error.
@@ -332,13 +367,21 @@ func TestRecoverClusterMeetError(t *testing.T) {
newTk.MustGetErrCode(fmt.Sprintf("flashback cluster to timestamp '%s'", time.Now().Add(0-30*time.Second)), errno.ErrPrivilegeCheckFail)
tk.MustExec("drop user 'testflashback'@'localhost';")
- // Flashback failed because of ddl history.
- tk.MustExec("use test;")
- tk.MustExec("create table t(a int);")
- tk.MustMatchErrMsg(fmt.Sprintf("flashback cluster to timestamp '%s'", flashbackTs), "Detected schema change due to another DDL job during \\[.*, now\\), can't do flashback")
+ // detect modify system table
+ nowTS, err := tk.Session().GetStore().GetOracle().GetTimestamp(context.Background(), &oracle.Option{})
+ require.NoError(t, err)
+ tk.MustExec("truncate table mysql.stats_meta")
+ errorMsg := fmt.Sprintf("[ddl:-1]Detected modified system table during [%s, now), can't do flashback", oracle.GetTimeFromTS(nowTS).String())
+ tk.MustGetErrMsg(fmt.Sprintf("flashback cluster to timestamp '%s'", oracle.GetTimeFromTS(nowTS)), errorMsg)
+
+ // update tidb_server_version
+ nowTS, err = tk.Session().GetStore().GetOracle().GetTimestamp(context.Background(), &oracle.Option{})
+ require.NoError(t, err)
+ tk.MustExec("update mysql.tidb set VARIABLE_VALUE=VARIABLE_VALUE+1 where VARIABLE_NAME='tidb_server_version'")
+ errorMsg = fmt.Sprintf("[ddl:-1]Detected TiDB upgrade during [%s, now), can't do flashback", oracle.GetTimeFromTS(nowTS).String())
+ tk.MustGetErrMsg(fmt.Sprintf("flashback cluster to timestamp '%s'", oracle.GetTimeFromTS(nowTS)), errorMsg)
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS"))
- require.NoError(t, failpoint.Disable("tikvclient/injectSafeTS"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/injectSafeTS"))
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockFlashbackTest"))
}
@@ -347,6 +390,7 @@ func TestFlashbackWithSafeTs(t *testing.T) {
tk := testkit.NewTestKit(t, store)
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockFlashbackTest", `return(true)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/changeFlashbackGetMinSafeTimeTimeout", `return(0)`))
timeBeforeDrop, _, safePointSQL, resetGC := MockGC(tk)
defer resetGC()
@@ -371,9 +415,8 @@ func TestFlashbackWithSafeTs(t *testing.T) {
compareWithSafeTS: 0,
},
{
- name: "10 seconds ago to now, safeTS 5 secs ago",
- // Add flashbackTs.Add(-500*time.Millisecond) to avoid flashback time range overlapped.
- sql: fmt.Sprintf("flashback cluster to timestamp '%s'", flashbackTs.Add(-500*time.Millisecond)),
+ name: "10 seconds ago to now, safeTS 5 secs ago",
+ sql: fmt.Sprintf("flashback cluster to timestamp '%s'", flashbackTs),
injectSafeTS: oracle.GoTimeToTS(flashbackTs.Add(10 * time.Second)),
compareWithSafeTS: -1,
},
@@ -386,19 +429,55 @@ func TestFlashbackWithSafeTs(t *testing.T) {
}
for _, testcase := range testcases {
t.Log(testcase.name)
- require.NoError(t, failpoint.Enable("tikvclient/injectSafeTS",
- fmt.Sprintf("return(%v)", testcase.injectSafeTS)))
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS",
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/injectSafeTS",
fmt.Sprintf("return(%v)", testcase.injectSafeTS)))
if testcase.compareWithSafeTS == 1 {
+ start := time.Now()
tk.MustContainErrMsg(testcase.sql,
- "cannot set flashback timestamp to too close to present time")
+ "cannot set flashback timestamp after min-resolved-ts")
+ // When set `flashbackGetMinSafeTimeTimeout` = 0, no retry for `getStoreGlobalMinSafeTS`.
+ require.Less(t, time.Since(start), time.Second)
} else {
tk.MustExec(testcase.sql)
}
}
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS"))
- require.NoError(t, failpoint.Disable("tikvclient/injectSafeTS"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/injectSafeTS"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockFlashbackTest"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/changeFlashbackGetMinSafeTimeTimeout"))
+}
+
+func TestFlashbackRetryGetMinSafeTime(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockFlashbackTest", `return(true)`))
+
+ timeBeforeDrop, _, safePointSQL, resetGC := MockGC(tk)
+ defer resetGC()
+
+ // Set GC safe point.
+ tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
+
+ time.Sleep(time.Second)
+ ts, _ := tk.Session().GetStore().GetOracle().GetTimestamp(context.Background(), &oracle.Option{})
+ flashbackTs := oracle.GetTimeFromTS(ts)
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/injectSafeTS",
+ fmt.Sprintf("return(%v)", oracle.GoTimeToTS(flashbackTs.Add(-10*time.Minute)))))
+
+ go func() {
+ time.Sleep(2 * time.Second)
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/injectSafeTS",
+ fmt.Sprintf("return(%v)", oracle.GoTimeToTS(flashbackTs.Add(10*time.Minute)))))
+ }()
+
+ start := time.Now()
+ tk.MustExec(fmt.Sprintf("flashback cluster to timestamp '%s'", flashbackTs))
+ duration := time.Since(start)
+ require.Greater(t, duration, 2*time.Second)
+ require.Less(t, duration, 5*time.Second)
+
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/injectSafeTS"))
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockFlashbackTest"))
}
@@ -457,6 +536,26 @@ func TestFlashbackSchema(t *testing.T) {
tk.MustExec("use test2")
tk.MustQuery("select a from t order by a").Check(testkit.Rows("1", "2", "3"))
tk.MustQuery("select a from t1 order by a").Check(testkit.Rows("4", "5", "6"))
+
+ tk.MustExec("drop database if exists t_recover")
+ tk.MustExec("create database t_recover")
+ tk.MustExec("drop database t_recover")
+
+ // Recover without drop/create privilege.
+ tk.MustExec("CREATE USER 'testflashbackschema'@'localhost';")
+ newTk := testkit.NewTestKit(t, store)
+ require.NoError(t, newTk.Session().Auth(&auth.UserIdentity{Username: "testflashbackschema", Hostname: "localhost"}, nil, nil))
+ newTk.MustGetErrCode("flashback database t_recover", errno.ErrDBaccessDenied)
+
+ // Got drop privilege, still failed.
+ tk.MustExec("grant drop on *.* to 'testflashbackschema'@'localhost';")
+ newTk.MustGetErrCode("flashback database t_recover", errno.ErrDBaccessDenied)
+
+ // Got create and drop privilege, execute success.
+ tk.MustExec("grant create on *.* to 'testflashbackschema'@'localhost';")
+ newTk.MustExec("flashback schema t_recover")
+
+ tk.MustExec("drop user 'testflashbackschema'@'localhost';")
}
// MockGC is used to make GC work in the test environment.
diff --git a/executor/replace.go b/executor/replace.go
index 221cbf87b2504..801ff40ebd252 100644
--- a/executor/replace.go
+++ b/executor/replace.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/tidb/meta/autoid"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/sessionctx/stmtctx"
+ "github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
@@ -43,6 +44,9 @@ type ReplaceExec struct {
// Close implements the Executor Close interface.
func (e *ReplaceExec) Close() error {
e.setMessage()
+ if e.runtimeStats != nil && e.stats != nil {
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
+ }
if e.SelectExec != nil {
return e.SelectExec.Close()
}
@@ -89,6 +93,10 @@ func (e *ReplaceExec) removeRow(ctx context.Context, txn kv.Transaction, handle
if err != nil {
return false, err
}
+ err = onRemoveRowForFK(e.ctx, oldRow, e.fkChecks, e.fkCascades)
+ if err != nil {
+ return false, err
+ }
e.ctx.GetSessionVars().StmtCtx.AddAffectedRows(1)
return false, nil
}
@@ -169,16 +177,12 @@ func (e *ReplaceExec) replaceRow(ctx context.Context, r toBeCheckedRow) error {
// 3. error: the error.
func (e *ReplaceExec) removeIndexRow(ctx context.Context, txn kv.Transaction, r toBeCheckedRow) (bool, bool, error) {
for _, uk := range r.uniqueKeys {
- val, err := txn.Get(ctx, uk.newKey)
+ _, handle, err := tables.FetchDuplicatedHandle(ctx, uk.newKey, true, txn, e.Table.Meta().ID, uk.commonHandle)
if err != nil {
- if kv.IsErrNotFound(err) {
- continue
- }
return false, false, err
}
- handle, err := tablecodec.DecodeHandleInUniqueIndexValue(val, uk.commonHandle)
- if err != nil {
- return false, true, err
+ if handle == nil {
+ continue
}
rowUnchanged, err := e.removeRow(ctx, txn, handle, r)
if err != nil {
@@ -268,3 +272,18 @@ func (e *ReplaceExec) setMessage() {
stmtCtx.SetMessage(msg)
}
}
+
+// GetFKChecks implements WithForeignKeyTrigger interface.
+func (e *ReplaceExec) GetFKChecks() []*FKCheckExec {
+ return e.fkChecks
+}
+
+// GetFKCascades implements WithForeignKeyTrigger interface.
+func (e *ReplaceExec) GetFKCascades() []*FKCascadeExec {
+ return e.fkCascades
+}
+
+// HasFKCascades implements WithForeignKeyTrigger interface.
+func (e *ReplaceExec) HasFKCascades() bool {
+ return len(e.fkCascades) > 0
+}
diff --git a/executor/sample.go b/executor/sample.go
index e7eb9bd223639..7f64c365d599e 100644
--- a/executor/sample.go
+++ b/executor/sample.go
@@ -17,7 +17,6 @@ package executor
import (
"context"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/kv"
@@ -28,6 +27,7 @@ import (
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
decoder "github.com/pingcap/tidb/util/rowDecoder"
+ "github.com/pingcap/tidb/util/tracing"
"github.com/tikv/client-go/v2/tikv"
"golang.org/x/exp/slices"
)
@@ -47,10 +47,7 @@ type TableSampleExecutor struct {
// Open initializes necessary variables for using this executor.
func (e *TableSampleExecutor) Open(ctx context.Context) error {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("TableSampleExecutor.Open", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- }
+ defer tracing.StartRegion(ctx, "TableSampleExecutor.Open").End()
return nil
}
diff --git a/executor/seqtest/BUILD.bazel b/executor/seqtest/BUILD.bazel
index 64046c0852123..6d582ded16630 100644
--- a/executor/seqtest/BUILD.bazel
+++ b/executor/seqtest/BUILD.bazel
@@ -2,7 +2,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test")
go_test(
name = "seqtest_test",
- timeout = "moderate",
+ timeout = "short",
srcs = [
"main_test.go",
"prepared_test.go",
diff --git a/executor/seqtest/main_test.go b/executor/seqtest/main_test.go
index 299cc69a8a26f..067680fb23365 100644
--- a/executor/seqtest/main_test.go
+++ b/executor/seqtest/main_test.go
@@ -31,7 +31,7 @@ func TestMain(m *testing.M) {
opts := []goleak.Option{
goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
- goleak.IgnoreTopFunction("github.com/pingcap/tidb/executor.readProjectionInput"),
+ goleak.IgnoreTopFunction("github.com/pingcap/tidb/executor.readProjection[...]"),
goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
}
diff --git a/executor/seqtest/prepared_test.go b/executor/seqtest/prepared_test.go
index 11dae5e570767..3c0f180e54f22 100644
--- a/executor/seqtest/prepared_test.go
+++ b/executor/seqtest/prepared_test.go
@@ -280,11 +280,11 @@ func TestPreparedLimitOffset(t *testing.T) {
r.Check(testkit.Rows("2"))
tk.MustExec(`set @a=1.1`)
- r = tk.MustQuery(`execute stmt_test_1 using @a, @b;`)
- r.Check(testkit.Rows("2"))
+ _, err := tk.Exec(`execute stmt_test_1 using @a, @b;`)
+ require.True(t, plannercore.ErrWrongArguments.Equal(err))
tk.MustExec(`set @c="-1"`)
- _, err := tk.Exec("execute stmt_test_1 using @c, @c")
+ _, err = tk.Exec("execute stmt_test_1 using @c, @c")
require.True(t, plannercore.ErrWrongArguments.Equal(err))
stmtID, _, _, err := tk.Session().PrepareStmt("select id from prepare_test limit ?")
@@ -334,7 +334,7 @@ func TestPrepareWithAggregation(t *testing.T) {
tk.MustExec(fmt.Sprintf(`set @@tidb_enable_prepared_plan_cache=%v`, flag))
se, err := session.CreateSession4TestWithOpt(store, &session.Opt{
- PreparedPlanCache: plannercore.NewLRUPlanCache(100, 0.1, math.MaxUint64, plannercore.PickPlanFromBucket, tk.Session()),
+ PreparedPlanCache: plannercore.NewLRUPlanCache(100, 0.1, math.MaxUint64, tk.Session()),
})
require.NoError(t, err)
tk.SetSession(se)
@@ -420,14 +420,14 @@ func TestPreparedInsert(t *testing.T) {
err = counter.Write(pb)
require.NoError(t, err)
hit := pb.GetCounter().GetValue()
- require.Equal(t, float64(1), hit)
+ require.Equal(t, float64(0), hit) // insert-values-stmt cannot use the plan cache
}
tk.MustExec(`set @a=3,@b=3; execute stmt_insert using @a, @b;`)
if flag {
err = counter.Write(pb)
require.NoError(t, err)
hit := pb.GetCounter().GetValue()
- require.Equal(t, float64(2), hit)
+ require.Equal(t, float64(0), hit)
}
result := tk.MustQuery("select id, c1 from prepare_test where id = ?", 1)
@@ -443,21 +443,21 @@ func TestPreparedInsert(t *testing.T) {
err = counter.Write(pb)
require.NoError(t, err)
hit := pb.GetCounter().GetValue()
- require.Equal(t, float64(2), hit)
+ require.Equal(t, float64(0), hit)
}
tk.MustExec(`set @a=2; execute stmt_insert_select using @a;`)
if flag {
err = counter.Write(pb)
require.NoError(t, err)
hit := pb.GetCounter().GetValue()
- require.Equal(t, float64(3), hit)
+ require.Equal(t, float64(1), hit)
}
tk.MustExec(`set @a=3; execute stmt_insert_select using @a;`)
if flag {
err = counter.Write(pb)
require.NoError(t, err)
hit := pb.GetCounter().GetValue()
- require.Equal(t, float64(4), hit)
+ require.Equal(t, float64(2), hit)
}
result = tk.MustQuery("select id, c1 from prepare_test where id = ?", 101)
@@ -599,7 +599,7 @@ func TestPrepareDealloc(t *testing.T) {
tk.MustExec(`set @@tidb_enable_prepared_plan_cache=true`)
se, err := session.CreateSession4TestWithOpt(store, &session.Opt{
- PreparedPlanCache: plannercore.NewLRUPlanCache(3, 0.1, math.MaxUint64, plannercore.PickPlanFromBucket, tk.Session()),
+ PreparedPlanCache: plannercore.NewLRUPlanCache(3, 0.1, math.MaxUint64, tk.Session()),
})
require.NoError(t, err)
tk.SetSession(se)
@@ -767,3 +767,108 @@ func TestPreparedIssue17419(t *testing.T) {
// _, ok := tk1.Session().ShowProcess().Plan.(*plannercore.Execute)
// require.True(t, ok)
}
+
+func TestLimitUnsupportedCase(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, key(a))")
+ tk.MustExec("prepare stmt from 'select * from t limit ?'")
+
+ tk.MustExec("set @a = 1.2")
+ tk.MustGetErrMsg("execute stmt using @a", "[planner:1210]Incorrect arguments to LIMIT")
+ tk.MustExec("set @a = 1.")
+ tk.MustGetErrMsg("execute stmt using @a", "[planner:1210]Incorrect arguments to LIMIT")
+ tk.MustExec("set @a = '0'")
+ tk.MustGetErrMsg("execute stmt using @a", "[planner:1210]Incorrect arguments to LIMIT")
+ tk.MustExec("set @a = '1'")
+ tk.MustGetErrMsg("execute stmt using @a", "[planner:1210]Incorrect arguments to LIMIT")
+ tk.MustExec("set @a = 1_2")
+ tk.MustGetErrMsg("execute stmt using @a", "[planner:1210]Incorrect arguments to LIMIT")
+}
+
+func TestIssue38323(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(id int, k int);")
+
+ tk.MustExec("prepare stmt from 'explain select * from t where id = ? and k = ? group by id, k';")
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 skip plan-cache: not a SELECT/UPDATE/INSERT/DELETE/SET statement"))
+ tk.MustExec("set @a = 1;")
+ tk.MustExec("execute stmt using @a, @a")
+ tk.MustQuery("execute stmt using @a, @a").Check(tk.MustQuery("explain select * from t where id = 1 and k = 1 group by id, k").Rows())
+
+ tk.MustExec("prepare stmt from 'explain select * from t where ? = id and ? = k group by id, k';")
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 skip plan-cache: not a SELECT/UPDATE/INSERT/DELETE/SET statement"))
+ tk.MustExec("set @a = 1;")
+ tk.MustQuery("execute stmt using @a, @a").Check(tk.MustQuery("explain select * from t where 1 = id and 1 = k group by id, k").Rows())
+}
+
+func TestSetPlanCacheLimitSwitch(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk.MustQuery("select @@session.tidb_enable_plan_cache_for_param_limit").Check(testkit.Rows("1"))
+ tk.MustQuery("select @@global.tidb_enable_plan_cache_for_param_limit").Check(testkit.Rows("1"))
+
+ tk.MustExec("set @@session.tidb_enable_plan_cache_for_param_limit = OFF;")
+ tk.MustQuery("select @@session.tidb_enable_plan_cache_for_param_limit").Check(testkit.Rows("0"))
+
+ tk.MustExec("set @@session.tidb_enable_plan_cache_for_param_limit = 1;")
+ tk.MustQuery("select @@session.tidb_enable_plan_cache_for_param_limit").Check(testkit.Rows("1"))
+
+ tk.MustExec("set @@global.tidb_enable_plan_cache_for_param_limit = off;")
+ tk.MustQuery("select @@global.tidb_enable_plan_cache_for_param_limit").Check(testkit.Rows("0"))
+
+ tk.MustExec("set @@global.tidb_enable_plan_cache_for_param_limit = ON;")
+ tk.MustQuery("select @@global.tidb_enable_plan_cache_for_param_limit").Check(testkit.Rows("1"))
+
+ tk.MustGetErrMsg("set @@global.tidb_enable_plan_cache_for_param_limit = '';", "[variable:1231]Variable 'tidb_enable_plan_cache_for_param_limit' can't be set to the value of ''")
+ tk.MustGetErrMsg("set @@global.tidb_enable_plan_cache_for_param_limit = 11;", "[variable:1231]Variable 'tidb_enable_plan_cache_for_param_limit' can't be set to the value of '11'")
+ tk.MustGetErrMsg("set @@global.tidb_enable_plan_cache_for_param_limit = enabled;", "[variable:1231]Variable 'tidb_enable_plan_cache_for_param_limit' can't be set to the value of 'enabled'")
+ tk.MustGetErrMsg("set @@global.tidb_enable_plan_cache_for_param_limit = disabled;", "[variable:1231]Variable 'tidb_enable_plan_cache_for_param_limit' can't be set to the value of 'disabled'")
+ tk.MustGetErrMsg("set @@global.tidb_enable_plan_cache_for_param_limit = open;", "[variable:1231]Variable 'tidb_enable_plan_cache_for_param_limit' can't be set to the value of 'open'")
+}
+
+func TestPlanCacheLimitSwitchEffective(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, key(a))")
+
+ checkIfCached := func(res string) {
+ tk.MustExec("set @a = 1")
+ tk.MustExec("execute stmt using @a")
+ tk.MustExec("execute stmt using @a")
+ tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows(res))
+ }
+
+ // before prepare
+ tk.MustExec("set @@session.tidb_enable_plan_cache_for_param_limit = OFF")
+ tk.MustExec("prepare stmt from 'select * from t limit ?'")
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 skip plan-cache: query has 'limit ?' is un-cacheable"))
+ checkIfCached("0")
+ tk.MustExec("deallocate prepare stmt")
+
+ // after prepare
+ tk.MustExec("set @@session.tidb_enable_plan_cache_for_param_limit = ON")
+ tk.MustExec("prepare stmt from 'select * from t limit ?'")
+ tk.MustExec("set @@session.tidb_enable_plan_cache_for_param_limit = OFF")
+ checkIfCached("0")
+ tk.MustExec("execute stmt using @a")
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 skip plan-cache: the switch 'tidb_enable_plan_cache_for_param_limit' is off"))
+ tk.MustExec("deallocate prepare stmt")
+
+ // after execute
+ tk.MustExec("set @@session.tidb_enable_plan_cache_for_param_limit = ON")
+ tk.MustExec("prepare stmt from 'select * from t limit ?'")
+ checkIfCached("1")
+ tk.MustExec("set @@session.tidb_enable_plan_cache_for_param_limit = OFF")
+ checkIfCached("0")
+ tk.MustExec("deallocate prepare stmt")
+}
diff --git a/executor/seqtest/seq_executor_test.go b/executor/seqtest/seq_executor_test.go
index 7f16fd68da5e7..3e692a39a19df 100644
--- a/executor/seqtest/seq_executor_test.go
+++ b/executor/seqtest/seq_executor_test.go
@@ -624,7 +624,7 @@ func TestShowStatsHealthy(t *testing.T) {
require.NoError(t, err)
err = do.StatsHandle().Update(do.InfoSchema())
require.NoError(t, err)
- tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 19"))
+ tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 0"))
tk.MustExec("analyze table t")
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100"))
tk.MustExec("delete from t")
@@ -773,43 +773,45 @@ func HelperTestAdminShowNextID(t *testing.T, store kv.Storage, str string) {
tk.MustExec("create table t(id int, c int)")
// Start handle is 1.
r := tk.MustQuery(str + " t next_row_id")
- r.Check(testkit.Rows("test t _tidb_rowid 1 AUTO_INCREMENT"))
+ r.Check(testkit.Rows("test t _tidb_rowid 1 _TIDB_ROWID"))
// Row ID is step + 1.
tk.MustExec("insert into t values(1, 1)")
r = tk.MustQuery(str + " t next_row_id")
- r.Check(testkit.Rows("test t _tidb_rowid 11 AUTO_INCREMENT"))
+ r.Check(testkit.Rows("test t _tidb_rowid 11 _TIDB_ROWID"))
// Row ID is original + step.
for i := 0; i < int(step); i++ {
tk.MustExec("insert into t values(10000, 1)")
}
r = tk.MustQuery(str + " t next_row_id")
- r.Check(testkit.Rows("test t _tidb_rowid 21 AUTO_INCREMENT"))
+ r.Check(testkit.Rows("test t _tidb_rowid 21 _TIDB_ROWID"))
tk.MustExec("drop table t")
// test for a table with the primary key
tk.MustExec("create table tt(id int primary key auto_increment, c int)")
// Start handle is 1.
r = tk.MustQuery(str + " tt next_row_id")
- r.Check(testkit.Rows("test tt id 1 AUTO_INCREMENT"))
+ r.Check(testkit.Rows("test tt id 1 _TIDB_ROWID", "test tt id 1 AUTO_INCREMENT"))
// After rebasing auto ID, row ID is 20 + step + 1.
tk.MustExec("insert into tt values(20, 1)")
r = tk.MustQuery(str + " tt next_row_id")
- r.Check(testkit.Rows("test tt id 31 AUTO_INCREMENT"))
+ r.Check(testkit.Rows("test tt id 31 _TIDB_ROWID", "test tt id 1 AUTO_INCREMENT"))
// test for renaming the table
tk.MustExec("drop database if exists test1")
tk.MustExec("create database test1")
tk.MustExec("rename table test.tt to test1.tt")
tk.MustExec("use test1")
r = tk.MustQuery(str + " tt next_row_id")
- r.Check(testkit.Rows("test1 tt id 31 AUTO_INCREMENT"))
+ r.Check(testkit.Rows("test1 tt id 31 _TIDB_ROWID", "test1 tt id 1 AUTO_INCREMENT"))
tk.MustExec("insert test1.tt values ()")
r = tk.MustQuery(str + " tt next_row_id")
- r.Check(testkit.Rows("test1 tt id 41 AUTO_INCREMENT"))
+ r.Check(testkit.Rows("test1 tt id 41 _TIDB_ROWID", "test1 tt id 1 AUTO_INCREMENT"))
tk.MustExec("drop table tt")
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (a int auto_increment primary key nonclustered, b int);")
- tk.MustQuery("show table t next_row_id;").Check(testkit.Rows("test1 t _tidb_rowid 1 AUTO_INCREMENT"))
+ tk.MustQuery("show table t next_row_id;").Check(testkit.Rows(
+ "test1 t _tidb_rowid 1 _TIDB_ROWID",
+ "test1 t _tidb_rowid 1 AUTO_INCREMENT"))
tk.MustExec("set @@allow_auto_random_explicit_insert = true")
@@ -830,19 +832,19 @@ func HelperTestAdminShowNextID(t *testing.T, store kv.Storage, str string) {
// Test for a sequence.
tk.MustExec("create sequence seq1 start 15 cache 57")
r = tk.MustQuery(str + " seq1 next_row_id")
- r.Check(testkit.Rows("test1 seq1 _tidb_rowid 1 AUTO_INCREMENT", "test1 seq1 15 SEQUENCE"))
+ r.Check(testkit.Rows("test1 seq1 _tidb_rowid 1 _TIDB_ROWID", "test1 seq1 15 SEQUENCE"))
r = tk.MustQuery("select nextval(seq1)")
r.Check(testkit.Rows("15"))
r = tk.MustQuery(str + " seq1 next_row_id")
- r.Check(testkit.Rows("test1 seq1 _tidb_rowid 1 AUTO_INCREMENT", "test1 seq1 72 SEQUENCE"))
+ r.Check(testkit.Rows("test1 seq1 _tidb_rowid 1 _TIDB_ROWID", "test1 seq1 72 SEQUENCE"))
r = tk.MustQuery("select nextval(seq1)")
r.Check(testkit.Rows("16"))
r = tk.MustQuery(str + " seq1 next_row_id")
- r.Check(testkit.Rows("test1 seq1 _tidb_rowid 1 AUTO_INCREMENT", "test1 seq1 72 SEQUENCE"))
+ r.Check(testkit.Rows("test1 seq1 _tidb_rowid 1 _TIDB_ROWID", "test1 seq1 72 SEQUENCE"))
r = tk.MustQuery("select setval(seq1, 96)")
r.Check(testkit.Rows("96"))
r = tk.MustQuery(str + " seq1 next_row_id")
- r.Check(testkit.Rows("test1 seq1 _tidb_rowid 1 AUTO_INCREMENT", "test1 seq1 97 SEQUENCE"))
+ r.Check(testkit.Rows("test1 seq1 _tidb_rowid 1 _TIDB_ROWID", "test1 seq1 97 SEQUENCE"))
}
func TestNoHistoryWhenDisableRetry(t *testing.T) {
@@ -898,7 +900,7 @@ func TestPrepareMaxParamCountCheck(t *testing.T) {
require.NoError(t, err)
bigSQL, bigParams := generateBatchSQL(math.MaxUint16 + 2)
- _, err = tk.Exec(bigSQL, bigParams...)
+ err = tk.ExecToErr(bigSQL, bigParams...)
require.Error(t, err)
require.EqualError(t, err, "[executor:1390]Prepared statement contains too many placeholders")
}
@@ -938,7 +940,7 @@ func TestBatchInsertDelete(t *testing.T) {
atomic.StoreUint64(&kv.TxnTotalSizeLimit, originLimit)
}()
// Set the limitation to a small value, make it easier to reach the limitation.
- atomic.StoreUint64(&kv.TxnTotalSizeLimit, 5800)
+ atomic.StoreUint64(&kv.TxnTotalSizeLimit, 5900)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -985,16 +987,12 @@ func TestBatchInsertDelete(t *testing.T) {
// Test tidb_batch_insert could not work if enable-batch-dml is disabled.
tk.MustExec("set @@session.tidb_batch_insert=1;")
- _, err = tk.Exec("insert into batch_insert (c) select * from batch_insert;")
- require.Error(t, err)
- require.True(t, kv.ErrTxnTooLarge.Equal(err))
+ tk.MustGetErrCode("insert into batch_insert (c) select * from batch_insert;", errno.ErrTxnTooLarge)
tk.MustExec("set @@session.tidb_batch_insert=0;")
// for on duplicate key
- _, err = tk.Exec(`insert into batch_insert_on_duplicate select * from batch_insert_on_duplicate as tt
- on duplicate key update batch_insert_on_duplicate.id=batch_insert_on_duplicate.id+1000;`)
- require.Error(t, err)
- require.Truef(t, kv.ErrTxnTooLarge.Equal(err), "%v", err)
+ tk.MustGetErrCode(`insert into batch_insert_on_duplicate select * from batch_insert_on_duplicate as tt
+ on duplicate key update batch_insert_on_duplicate.id=batch_insert_on_duplicate.id+1000;`, errno.ErrTxnTooLarge)
r = tk.MustQuery("select count(*) from batch_insert;")
r.Check(testkit.Rows("320"))
@@ -1020,17 +1018,14 @@ func TestBatchInsertDelete(t *testing.T) {
tk.MustExec("set @@session.tidb_dml_batch_size=50;")
// for on duplicate key
- _, err = tk.Exec(`insert into batch_insert_on_duplicate select * from batch_insert_on_duplicate as tt
+ tk.MustExec(`insert into batch_insert_on_duplicate select * from batch_insert_on_duplicate as tt
on duplicate key update batch_insert_on_duplicate.id=batch_insert_on_duplicate.id+1000;`)
- require.NoError(t, err)
r = tk.MustQuery("select count(*) from batch_insert_on_duplicate;")
r.Check(testkit.Rows("320"))
// Disable BachInsert mode in transition.
tk.MustExec("begin;")
- _, err = tk.Exec("insert into batch_insert (c) select * from batch_insert;")
- require.Error(t, err)
- require.True(t, kv.ErrTxnTooLarge.Equal(err))
+ tk.MustGetErrCode("insert into batch_insert (c) select * from batch_insert;", errno.ErrTxnTooLarge)
tk.MustExec("rollback;")
r = tk.MustQuery("select count(*) from batch_insert;")
r.Check(testkit.Rows("640"))
diff --git a/executor/set.go b/executor/set.go
index 2396356c1245b..a7b5ff19da91c 100644
--- a/executor/set.go
+++ b/executor/set.go
@@ -284,7 +284,16 @@ func (e *SetExecutor) getVarValue(ctx context.Context, v *expression.VarAssignme
if err != nil || nativeVal.IsNull() {
return "", err
}
- return nativeVal.ToString()
+
+ value, err = nativeVal.ToString()
+ if err != nil {
+ return "", err
+ }
+
+ // We need to clone the string because the value is constructed by `hack.String` in Datum which reuses the under layer `[]byte`
+ // instead of allocating some new spaces. The `[]byte` in Datum will be reused in `chunk.Chunk` by different statements in session.
+ // If we do not clone the value, the system variable will have a risk to be modified by other statements.
+ return strings.Clone(value), nil
}
func (e *SetExecutor) loadSnapshotInfoSchemaIfNeeded(name string, snapshotTS uint64) error {
diff --git a/executor/set_config.go b/executor/set_config.go
index 531a61acdc4a0..b508b55eb2fc7 100644
--- a/executor/set_config.go
+++ b/executor/set_config.go
@@ -102,7 +102,7 @@ func (s *SetConfigExec) Next(ctx context.Context, req *chunk.Chunk) error {
if s.p.Instance != "" {
nodeAddrs.Insert(s.p.Instance)
}
- serversInfo = filterClusterServerInfo(serversInfo, nodeTypes, nodeAddrs)
+ serversInfo = infoschema.FilterClusterServerInfo(serversInfo, nodeTypes, nodeAddrs)
if s.p.Instance != "" && len(serversInfo) == 0 {
return errors.Errorf("instance %v is not found in this cluster", s.p.Instance)
}
diff --git a/executor/set_test.go b/executor/set_test.go
index 697209d64836a..c50105cf6fd63 100644
--- a/executor/set_test.go
+++ b/executor/set_test.go
@@ -20,7 +20,6 @@ import (
"errors"
"fmt"
"io"
- "io/ioutil"
"net/http"
"strconv"
"testing"
@@ -646,7 +645,7 @@ func TestSetVar(t *testing.T) {
tk.MustQuery("select @@tidb_enable_tso_follower_proxy").Check(testkit.Rows("0"))
require.Error(t, tk.ExecToErr("set tidb_enable_tso_follower_proxy = 1"))
- tk.MustQuery("select @@tidb_enable_historical_stats").Check(testkit.Rows("0"))
+ tk.MustQuery("select @@tidb_enable_historical_stats").Check(testkit.Rows("1"))
tk.MustExec("set global tidb_enable_historical_stats = 1")
tk.MustQuery("select @@tidb_enable_historical_stats").Check(testkit.Rows("1"))
tk.MustExec("set global tidb_enable_historical_stats = 0")
@@ -767,51 +766,51 @@ func TestSetVar(t *testing.T) {
tk.MustGetErrCode("set global init_connect = 'invalidstring'", mysql.ErrWrongTypeForVar)
tk.MustExec("set global init_connect = 'select now(); select timestamp()'")
- // test variable 'tidb_enable_general_plan_cache'
+ // test variable 'tidb_enable_non_prepared_plan_cache'
// global scope
- tk.MustQuery("select @@global.tidb_enable_general_plan_cache").Check(testkit.Rows("0")) // default value
- tk.MustExec("set global tidb_enable_general_plan_cache = 1")
- tk.MustQuery("select @@global.tidb_enable_general_plan_cache").Check(testkit.Rows("1"))
- tk.MustExec("set global tidb_enable_general_plan_cache = 0")
- tk.MustQuery("select @@global.tidb_enable_general_plan_cache").Check(testkit.Rows("0"))
+ tk.MustQuery("select @@global.tidb_enable_non_prepared_plan_cache").Check(testkit.Rows("0")) // default value
+ tk.MustExec("set global tidb_enable_non_prepared_plan_cache = 1")
+ tk.MustQuery("select @@global.tidb_enable_non_prepared_plan_cache").Check(testkit.Rows("1"))
+ tk.MustExec("set global tidb_enable_non_prepared_plan_cache = 0")
+ tk.MustQuery("select @@global.tidb_enable_non_prepared_plan_cache").Check(testkit.Rows("0"))
// session scope
- tk.MustQuery("select @@session.tidb_enable_general_plan_cache").Check(testkit.Rows("0")) // default value
- tk.MustExec("set session tidb_enable_general_plan_cache = 1")
- tk.MustQuery("select @@session.tidb_enable_general_plan_cache").Check(testkit.Rows("1"))
- tk.MustExec("set session tidb_enable_general_plan_cache = 0")
- tk.MustQuery("select @@session.tidb_enable_general_plan_cache").Check(testkit.Rows("0"))
+ tk.MustQuery("select @@session.tidb_enable_non_prepared_plan_cache").Check(testkit.Rows("0")) // default value
+ tk.MustExec("set session tidb_enable_non_prepared_plan_cache = 1")
+ tk.MustQuery("select @@session.tidb_enable_non_prepared_plan_cache").Check(testkit.Rows("1"))
+ tk.MustExec("set session tidb_enable_non_prepared_plan_cache = 0")
+ tk.MustQuery("select @@session.tidb_enable_non_prepared_plan_cache").Check(testkit.Rows("0"))
- // test variable 'tidb_general_plan_cache-size'
+ // test variable 'tidb_non_prepared_plan_cache-size'
// global scope
- tk.MustQuery("select @@global.tidb_general_plan_cache_size").Check(testkit.Rows("100")) // default value
- tk.MustExec("set global tidb_general_plan_cache_size = 200")
- tk.MustQuery("select @@global.tidb_general_plan_cache_size").Check(testkit.Rows("200"))
- tk.MustExec("set global tidb_general_plan_cache_size = 200000000") // overflow
- tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1292 Truncated incorrect tidb_general_plan_cache_size value: '200000000'"))
- tk.MustQuery("select @@global.tidb_general_plan_cache_size").Check(testkit.Rows("100000"))
+ tk.MustQuery("select @@global.tidb_non_prepared_plan_cache_size").Check(testkit.Rows("100")) // default value
+ tk.MustExec("set global tidb_non_prepared_plan_cache_size = 200")
+ tk.MustQuery("select @@global.tidb_non_prepared_plan_cache_size").Check(testkit.Rows("200"))
+ tk.MustExec("set global tidb_non_prepared_plan_cache_size = 200000000") // overflow
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1292 Truncated incorrect tidb_non_prepared_plan_cache_size value: '200000000'"))
+ tk.MustQuery("select @@global.tidb_non_prepared_plan_cache_size").Check(testkit.Rows("100000"))
// session scope
- tk.MustQuery("select @@session.tidb_general_plan_cache_size").Check(testkit.Rows("100")) // default value
- tk.MustExec("set session tidb_general_plan_cache_size = 300")
- tk.MustQuery("select @@session.tidb_general_plan_cache_size").Check(testkit.Rows("300"))
- tk.MustExec("set session tidb_general_plan_cache_size = -1") // underflow
- tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1292 Truncated incorrect tidb_general_plan_cache_size value: '-1'"))
- tk.MustQuery("select @@session.tidb_general_plan_cache_size").Check(testkit.Rows("1"))
+ tk.MustQuery("select @@session.tidb_non_prepared_plan_cache_size").Check(testkit.Rows("100")) // default value
+ tk.MustExec("set session tidb_non_prepared_plan_cache_size = 300")
+ tk.MustQuery("select @@session.tidb_non_prepared_plan_cache_size").Check(testkit.Rows("300"))
+ tk.MustExec("set session tidb_non_prepared_plan_cache_size = -1") // underflow
+ tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1292 Truncated incorrect tidb_non_prepared_plan_cache_size value: '-1'"))
+ tk.MustQuery("select @@session.tidb_non_prepared_plan_cache_size").Check(testkit.Rows("1"))
// test variable 'foreign_key_checks'
// global scope
- tk.MustQuery("select @@global.foreign_key_checks").Check(testkit.Rows("0")) // default value
- tk.MustExec("set global foreign_key_checks = 1")
- tk.MustQuery("select @@global.foreign_key_checks").Check(testkit.Rows("1"))
+ tk.MustQuery("select @@global.foreign_key_checks").Check(testkit.Rows("1")) // default value
+ tk.MustExec("set global foreign_key_checks = 0")
+ tk.MustQuery("select @@global.foreign_key_checks").Check(testkit.Rows("0"))
// session scope
- tk.MustQuery("select @@session.foreign_key_checks").Check(testkit.Rows("0")) // default value
- tk.MustExec("set session foreign_key_checks = 1")
- tk.MustQuery("select @@session.foreign_key_checks").Check(testkit.Rows("1"))
+ tk.MustQuery("select @@session.foreign_key_checks").Check(testkit.Rows("1")) // default value
+ tk.MustExec("set session foreign_key_checks = 0")
+ tk.MustQuery("select @@session.foreign_key_checks").Check(testkit.Rows("0"))
- // test variable 'foreign_key_checks'
+ // test variable 'tidb_enable_foreign_key'
// global scope
- tk.MustQuery("select @@global.tidb_enable_foreign_key").Check(testkit.Rows("0")) // default value
- tk.MustExec("set global tidb_enable_foreign_key = 1")
- tk.MustQuery("select @@global.tidb_enable_foreign_key").Check(testkit.Rows("1"))
+ tk.MustQuery("select @@global.tidb_enable_foreign_key").Check(testkit.Rows("1")) // default value
+ tk.MustExec("set global tidb_enable_foreign_key = 0")
+ tk.MustQuery("select @@global.tidb_enable_foreign_key").Check(testkit.Rows("0"))
// test variable 'tidb_opt_force_inline_cte'
tk.MustQuery("select @@session.tidb_opt_force_inline_cte").Check(testkit.Rows("0")) // default value is 0
@@ -853,6 +852,23 @@ func TestSetVar(t *testing.T) {
tk.MustQuery("select @@global.tidb_opt_range_max_size").Check(testkit.Rows("1048576"))
tk.MustExec("set session tidb_opt_range_max_size = 2097152")
tk.MustQuery("select @@session.tidb_opt_range_max_size").Check(testkit.Rows("2097152"))
+
+ // test for password validation
+ tk.MustQuery("SELECT @@GLOBAL.validate_password.enable").Check(testkit.Rows("0"))
+ tk.MustQuery("SELECT @@GLOBAL.validate_password.length").Check(testkit.Rows("8"))
+ tk.MustExec("SET GLOBAL validate_password.length = 3")
+ tk.MustQuery("SELECT @@GLOBAL.validate_password.length").Check(testkit.Rows("4"))
+ tk.MustExec("SET GLOBAL validate_password.mixed_case_count = 2")
+ tk.MustQuery("SELECT @@GLOBAL.validate_password.length").Check(testkit.Rows("6"))
+
+ // test tidb_cdc_write_source
+ require.Equal(t, uint64(0), tk.Session().GetSessionVars().CDCWriteSource)
+ tk.MustQuery("select @@tidb_cdc_write_source").Check(testkit.Rows("0"))
+ tk.MustExec("set @@session.tidb_cdc_write_source = 2")
+ tk.MustQuery("select @@tidb_cdc_write_source").Check(testkit.Rows("2"))
+ require.Equal(t, uint64(2), tk.Session().GetSessionVars().CDCWriteSource)
+ tk.MustExec("set @@session.tidb_cdc_write_source = 0")
+ require.Equal(t, uint64(0), tk.Session().GetSessionVars().CDCWriteSource)
}
func TestGetSetNoopVars(t *testing.T) {
@@ -1407,14 +1423,11 @@ func TestValidateSetVar(t *testing.T) {
tk.MustExec("set @@innodb_lock_wait_timeout = 1073741825")
tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1292|Truncated incorrect innodb_lock_wait_timeout value: '1073741825'"))
- tk.MustExec("set @@global.validate_password_number_count=-1")
- tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1292|Truncated incorrect validate_password_number_count value: '-1'"))
-
- tk.MustExec("set @@global.validate_password_length=-1")
- tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1292|Truncated incorrect validate_password_length value: '-1'"))
+ tk.MustExec("set @@global.validate_password.number_count=-1")
+ tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1292|Truncated incorrect validate_password.number_count value: '-1'"))
- tk.MustExec("set @@global.validate_password_length=8")
- tk.MustQuery("show warnings").Check(testkit.Rows())
+ tk.MustExec("set @@global.validate_password.length=-1")
+ tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1292|Truncated incorrect validate_password.length value: '-1'"))
err = tk.ExecToErr("set @@tx_isolation=''")
require.True(t, terror.ErrorEqual(err, variable.ErrWrongValueForVar), fmt.Sprintf("err %v", err))
@@ -1788,7 +1801,7 @@ func TestSetClusterConfig(t *testing.T) {
httpCnt = 0
tk.Session().SetValue(executor.TestSetConfigHTTPHandlerKey, func(req *http.Request) (*http.Response, error) {
httpCnt++
- body, err := ioutil.ReadAll(req.Body)
+ body, err := io.ReadAll(req.Body)
require.NoError(t, err)
// The `raftstore.` prefix is stripped.
require.JSONEq(t, `{"server.snap-max-write-bytes-per-sec":"500MB"}`, string(body))
@@ -2058,3 +2071,59 @@ func TestSetChunkReuseVariable(t *testing.T) {
// error value
tk.MustGetErrCode("set @@tidb_enable_reuse_chunk=s;", errno.ErrWrongValueForVar)
}
+
+func TestSetMppVersionVariable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustQuery("select @@session.mpp_version").Check(testkit.Rows("UNSPECIFIED"))
+ tk.MustExec("SET SESSION mpp_version = -1")
+ tk.MustQuery("select @@session.mpp_version").Check(testkit.Rows("-1"))
+ tk.MustExec("SET SESSION mpp_version = 0")
+ tk.MustQuery("select @@session.mpp_version").Check(testkit.Rows("0"))
+ tk.MustExec("SET SESSION mpp_version = 1")
+ tk.MustQuery("select @@session.mpp_version").Check(testkit.Rows("1"))
+ tk.MustExec("SET SESSION mpp_version = unspecified")
+ tk.MustQuery("select @@session.mpp_version").Check(testkit.Rows("unspecified"))
+ {
+ tk.MustGetErrMsg("SET SESSION mpp_version = 2", "incorrect value: 2. mpp_version options: -1 (unspecified), 0, 1")
+ }
+ {
+ tk.MustExec("SET GLOBAL mpp_version = 1")
+ tk.MustQuery("select @@global.mpp_version").Check(testkit.Rows("1"))
+ tk.MustExec("SET GLOBAL mpp_version = -1")
+ tk.MustQuery("select @@global.mpp_version").Check(testkit.Rows("-1"))
+ }
+}
+
+func TestSetMppExchangeCompressionModeVariable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustGetErrMsg(
+ "SET SESSION mpp_exchange_compression_mode = 123",
+ "incorrect value: `123`. mpp_exchange_compression_mode options: NONE, FAST, HIGH_COMPRESSION, UNSPECIFIED")
+ tk.MustQuery("select @@session.mpp_exchange_compression_mode").Check(testkit.Rows("UNSPECIFIED"))
+
+ tk.MustExec("SET SESSION mpp_exchange_compression_mode = none")
+ tk.MustQuery("select @@session.mpp_exchange_compression_mode").Check(testkit.Rows("none"))
+ tk.MustExec("SET SESSION mpp_exchange_compression_mode = fast")
+ tk.MustQuery("select @@session.mpp_exchange_compression_mode").Check(testkit.Rows("fast"))
+ tk.MustExec("SET SESSION mpp_exchange_compression_mode = HIGH_COMPRESSION")
+ tk.MustQuery("select @@session.mpp_exchange_compression_mode").Check(testkit.Rows("HIGH_COMPRESSION"))
+
+ {
+ tk.MustExec("SET GLOBAL mpp_exchange_compression_mode = none")
+ tk.MustQuery("select @@global.mpp_exchange_compression_mode").Check(testkit.Rows("none"))
+ }
+ {
+ tk.MustExec("SET mpp_version = 0")
+ tk.MustExec("SET mpp_exchange_compression_mode = unspecified")
+ require.Equal(t, len(tk.Session().GetSessionVars().StmtCtx.GetWarnings()), 0)
+ }
+ {
+ tk.MustExec("SET mpp_version = 0")
+ tk.MustExec("SET mpp_exchange_compression_mode = HIGH_COMPRESSION")
+ warnings := tk.Session().GetSessionVars().StmtCtx.GetWarnings()
+ require.Equal(t, len(warnings), 1)
+ require.Equal(t, warnings[0].Err.Error(), "mpp exchange compression won't work under current mpp version 0")
+ }
+}
diff --git a/executor/show.go b/executor/show.go
index 72bd63a785623..130f743d914bb 100644
--- a/executor/show.go
+++ b/executor/show.go
@@ -39,9 +39,11 @@ import (
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/auth"
"github.com/pingcap/tidb/parser/charset"
+ parserformat "github.com/pingcap/tidb/parser/format"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/terror"
+ "github.com/pingcap/tidb/parser/tidb"
field_types "github.com/pingcap/tidb/parser/types"
plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/plugin"
@@ -78,16 +80,17 @@ var etcdDialTimeout = 5 * time.Second
type ShowExec struct {
baseExecutor
- Tp ast.ShowStmtType // Databases/Tables/Columns/....
- DBName model.CIStr
- Table *ast.TableName // Used for showing columns.
- Partition model.CIStr // Used for showing partition
- Column *ast.ColumnName // Used for `desc table column`.
- IndexName model.CIStr // Used for show table regions.
- Flag int // Some flag parsed from sql, such as FULL.
- Roles []*auth.RoleIdentity // Used for show grants.
- User *auth.UserIdentity // Used by show grants, show create user.
- Extractor plannercore.ShowPredicateExtractor
+ Tp ast.ShowStmtType // Databases/Tables/Columns/....
+ DBName model.CIStr
+ Table *ast.TableName // Used for showing columns.
+ Partition model.CIStr // Used for showing partition
+ Column *ast.ColumnName // Used for `desc table column`.
+ IndexName model.CIStr // Used for show table regions.
+ ResourceGroupName model.CIStr // Used for showing resource group
+ Flag int // Some flag parsed from sql, such as FULL.
+ Roles []*auth.RoleIdentity // Used for show grants.
+ User *auth.UserIdentity // Used by show grants, show create user.
+ Extractor plannercore.ShowPredicateExtractor
is infoschema.InfoSchema
@@ -179,6 +182,8 @@ func (e *ShowExec) fetchAll(ctx context.Context) error {
return e.fetchShowCreateDatabase()
case ast.ShowCreatePlacementPolicy:
return e.fetchShowCreatePlacementPolicy()
+ case ast.ShowCreateResourceGroup:
+ return e.fetchShowCreateResourceGroup()
case ast.ShowDatabases:
return e.fetchShowDatabases()
case ast.ShowDrainerStatus:
@@ -304,12 +309,16 @@ func (v *visibleChecker) Leave(in ast.Node) (out ast.Node, ok bool) {
}
func (e *ShowExec) fetchShowBind() error {
- var bindRecords []*bindinfo.BindRecord
+ var tmp []*bindinfo.BindRecord
if !e.GlobalScope {
handle := e.ctx.Value(bindinfo.SessionBindInfoKeyType).(*bindinfo.SessionHandle)
- bindRecords = handle.GetAllBindRecord()
+ tmp = handle.GetAllBindRecord()
} else {
- bindRecords = domain.GetDomain(e.ctx).BindHandle().GetAllBindRecord()
+ tmp = domain.GetDomain(e.ctx).BindHandle().GetAllBindRecord()
+ }
+ bindRecords := make([]*bindinfo.BindRecord, 0)
+ for _, bindRecord := range tmp {
+ bindRecords = append(bindRecords, bindRecord.Copy())
}
// Remove the invalid bindRecord.
ind := 0
@@ -368,6 +377,8 @@ func (e *ShowExec) fetchShowBind() error {
hint.Charset,
hint.Collation,
hint.Source,
+ hint.SQLDigest,
+ hint.PlanDigest,
})
}
}
@@ -1006,8 +1017,9 @@ func ConstructResultOfShowCreateTable(ctx sessionctx.Context, tableInfo *model.T
}
buf.WriteString(" DEFAULT NULL")
}
- case "CURRENT_TIMESTAMP":
- buf.WriteString(" DEFAULT CURRENT_TIMESTAMP")
+ case "CURRENT_TIMESTAMP", "CURRENT_DATE":
+ buf.WriteString(" DEFAULT ")
+ buf.WriteString(defaultValue.(string))
if col.GetDecimal() > 0 {
buf.WriteString(fmt.Sprintf("(%d)", col.GetDecimal()))
}
@@ -1222,6 +1234,54 @@ func ConstructResultOfShowCreateTable(ctx sessionctx.Context, tableInfo *model.T
// add partition info here.
ddl.AppendPartitionInfo(tableInfo.Partition, buf, sqlMode)
+
+ if tableInfo.TTLInfo != nil {
+ restoreFlags := parserformat.RestoreStringSingleQuotes | parserformat.RestoreNameBackQuotes | parserformat.RestoreTiDBSpecialComment
+ restoreCtx := parserformat.NewRestoreCtx(restoreFlags, buf)
+
+ restoreCtx.WritePlain(" ")
+ err = restoreCtx.WriteWithSpecialComments(tidb.FeatureIDTTL, func() error {
+ columnName := ast.ColumnName{Name: tableInfo.TTLInfo.ColumnName}
+ timeUnit := ast.TimeUnitExpr{Unit: ast.TimeUnitType(tableInfo.TTLInfo.IntervalTimeUnit)}
+ restoreCtx.WriteKeyWord("TTL")
+ restoreCtx.WritePlain("=")
+ restoreCtx.WriteName(columnName.String())
+ restoreCtx.WritePlainf(" + INTERVAL %s ", tableInfo.TTLInfo.IntervalExprStr)
+ return timeUnit.Restore(restoreCtx)
+ })
+
+ if err != nil {
+ return err
+ }
+
+ restoreCtx.WritePlain(" ")
+ err = restoreCtx.WriteWithSpecialComments(tidb.FeatureIDTTL, func() error {
+ restoreCtx.WriteKeyWord("TTL_ENABLE")
+ restoreCtx.WritePlain("=")
+ if tableInfo.TTLInfo.Enable {
+ restoreCtx.WriteString("ON")
+ } else {
+ restoreCtx.WriteString("OFF")
+ }
+ return nil
+ })
+
+ if err != nil {
+ return err
+ }
+
+ restoreCtx.WritePlain(" ")
+ err = restoreCtx.WriteWithSpecialComments(tidb.FeatureIDTTL, func() error {
+ restoreCtx.WriteKeyWord("TTL_JOB_INTERVAL")
+ restoreCtx.WritePlain("=")
+ restoreCtx.WriteString(tableInfo.TTLInfo.JobInterval)
+ return nil
+ })
+
+ if err != nil {
+ return err
+ }
+ }
return nil
}
@@ -1397,6 +1457,11 @@ func ConstructResultOfShowCreatePlacementPolicy(policyInfo *model.PolicyInfo) st
return fmt.Sprintf("CREATE PLACEMENT POLICY `%s` %s", policyInfo.Name.O, policyInfo.PlacementSettings.String())
}
+// constructResultOfShowCreateResourceGroup constructs the result for show create resource group.
+func constructResultOfShowCreateResourceGroup(resourceGroup *model.ResourceGroupInfo) string {
+ return fmt.Sprintf("CREATE RESOURCE GROUP `%s` %s", resourceGroup.Name.O, resourceGroup.ResourceGroupSettings.String())
+}
+
// fetchShowCreateDatabase composes show create database result.
func (e *ShowExec) fetchShowCreateDatabase() error {
checker := privilege.GetPrivilegeManager(e.ctx)
@@ -1430,6 +1495,17 @@ func (e *ShowExec) fetchShowCreatePlacementPolicy() error {
return nil
}
+// fetchShowCreateResourceGroup composes show create resource group result.
+func (e *ShowExec) fetchShowCreateResourceGroup() error {
+ group, found := e.is.ResourceGroupByName(e.ResourceGroupName)
+ if !found {
+ return infoschema.ErrResourceGroupNotExists.GenWithStackByArgs(e.ResourceGroupName.O)
+ }
+ showCreate := constructResultOfShowCreateResourceGroup(group)
+ e.appendRow([]interface{}{e.ResourceGroupName.O, showCreate})
+ return nil
+}
+
func (e *ShowExec) fetchShowCollation() error {
var (
fieldPatternsLike collate.WildcardPattern
@@ -1487,7 +1563,11 @@ func (e *ShowExec) fetchShowCreateUser(ctx context.Context) error {
exec := e.ctx.(sqlexec.RestrictedSQLExecutor)
- rows, _, err := exec.ExecRestrictedSQL(ctx, nil, `SELECT plugin, Account_locked, JSON_UNQUOTE(JSON_EXTRACT(user_attributes, '$.metadata')), Token_issuer
+ rows, _, err := exec.ExecRestrictedSQL(ctx, nil,
+ `SELECT plugin, Account_locked, user_attributes->>'$.metadata', Token_issuer,
+ Password_reuse_history, Password_reuse_time, Password_expired, Password_lifetime,
+ user_attributes->>'$.Password_locking.failed_login_attempts',
+ user_attributes->>'$.Password_locking.password_lock_time_days'
FROM %n.%n WHERE User=%? AND Host=%?`,
mysql.SystemDB, mysql.UserTable, userName, strings.ToLower(hostName))
if err != nil {
@@ -1513,7 +1593,7 @@ func (e *ShowExec) fetchShowCreateUser(ctx context.Context) error {
userAttributes := rows[0].GetString(2)
if len(userAttributes) > 0 {
- userAttributes = " ATTRIBUTE " + userAttributes
+ userAttributes = fmt.Sprintf(" ATTRIBUTE '%s'", userAttributes)
}
tokenIssuer := rows[0].GetString(3)
@@ -1521,6 +1601,47 @@ func (e *ShowExec) fetchShowCreateUser(ctx context.Context) error {
tokenIssuer = " token_issuer " + tokenIssuer
}
+ var passwordHistory string
+ if rows[0].IsNull(4) {
+ passwordHistory = "DEFAULT"
+ } else {
+ passwordHistory = strconv.FormatUint(rows[0].GetUint64(4), 10)
+ }
+
+ var passwordReuseInterval string
+ if rows[0].IsNull(5) {
+ passwordReuseInterval = "DEFAULT"
+ } else {
+ passwordReuseInterval = strconv.FormatUint(rows[0].GetUint64(5), 10) + " DAY"
+ }
+
+ passwordExpired := rows[0].GetEnum(6).String()
+ passwordLifetime := int64(-1)
+ if !rows[0].IsNull(7) {
+ passwordLifetime = rows[0].GetInt64(7)
+ }
+ passwordExpiredStr := "PASSWORD EXPIRE DEFAULT"
+ if passwordExpired == "Y" {
+ passwordExpiredStr = "PASSWORD EXPIRE"
+ } else if passwordLifetime == 0 {
+ passwordExpiredStr = "PASSWORD EXPIRE NEVER"
+ } else if passwordLifetime > 0 {
+ passwordExpiredStr = fmt.Sprintf("PASSWORD EXPIRE INTERVAL %d DAY", passwordLifetime)
+ }
+
+ failedLoginAttempts := rows[0].GetString(8)
+ if len(failedLoginAttempts) > 0 {
+ failedLoginAttempts = " FAILED_LOGIN_ATTEMPTS " + failedLoginAttempts
+ }
+
+ passwordLockTimeDays := rows[0].GetString(9)
+ if len(passwordLockTimeDays) > 0 {
+ if passwordLockTimeDays == "-1" {
+ passwordLockTimeDays = " PASSWORD_LOCK_TIME UNBOUNDED"
+ } else {
+ passwordLockTimeDays = " PASSWORD_LOCK_TIME " + passwordLockTimeDays
+ }
+ }
rows, _, err = exec.ExecRestrictedSQL(ctx, nil, `SELECT Priv FROM %n.%n WHERE User=%? AND Host=%?`, mysql.SystemDB, mysql.GlobalPrivTable, userName, hostName)
if err != nil {
return errors.Trace(err)
@@ -1544,8 +1665,8 @@ func (e *ShowExec) fetchShowCreateUser(ctx context.Context) error {
}
// FIXME: the returned string is not escaped safely
- showStr := fmt.Sprintf("CREATE USER '%s'@'%s' IDENTIFIED WITH '%s'%s REQUIRE %s%s PASSWORD EXPIRE DEFAULT ACCOUNT %s%s",
- e.User.Username, e.User.Hostname, authplugin, authStr, require, tokenIssuer, accountLocked, userAttributes)
+ showStr := fmt.Sprintf("CREATE USER '%s'@'%s' IDENTIFIED WITH '%s'%s REQUIRE %s%s %s ACCOUNT %s PASSWORD HISTORY %s PASSWORD REUSE INTERVAL %s%s%s%s",
+ e.User.Username, e.User.Hostname, authplugin, authStr, require, tokenIssuer, passwordExpiredStr, accountLocked, passwordHistory, passwordReuseInterval, failedLoginAttempts, passwordLockTimeDays, userAttributes)
e.appendRow([]interface{}{showStr})
return nil
}
@@ -1596,13 +1717,16 @@ func (e *ShowExec) fetchShowGrants() error {
func (e *ShowExec) fetchShowPrivileges() error {
e.appendRow([]interface{}{"Alter", "Tables", "To alter the table"})
e.appendRow([]interface{}{"Alter routine", "Functions,Procedures", "To alter or drop stored functions/procedures"})
+ e.appendRow([]interface{}{"Config", "Server Admin", "To use SHOW CONFIG and SET CONFIG statements"})
e.appendRow([]interface{}{"Create", "Databases,Tables,Indexes", "To create new databases and tables"})
e.appendRow([]interface{}{"Create routine", "Databases", "To use CREATE FUNCTION/PROCEDURE"})
+ e.appendRow([]interface{}{"Create role", "Server Admin", "To create new roles"})
e.appendRow([]interface{}{"Create temporary tables", "Databases", "To use CREATE TEMPORARY TABLE"})
e.appendRow([]interface{}{"Create view", "Tables", "To create new views"})
e.appendRow([]interface{}{"Create user", "Server Admin", "To create new users"})
e.appendRow([]interface{}{"Delete", "Tables", "To delete existing rows"})
e.appendRow([]interface{}{"Drop", "Databases,Tables", "To drop databases, tables, and views"})
+ e.appendRow([]interface{}{"Drop role", "Server Admin", "To drop roles"})
e.appendRow([]interface{}{"Event", "Server Admin", "To create, alter, drop and execute events"})
e.appendRow([]interface{}{"Execute", "Functions,Procedures", "To execute stored routines"})
e.appendRow([]interface{}{"File", "File access on server", "To read and write files on the server"})
diff --git a/executor/showtest/BUILD.bazel b/executor/showtest/BUILD.bazel
index 807e00c8e88ec..1882c92e0627d 100644
--- a/executor/showtest/BUILD.bazel
+++ b/executor/showtest/BUILD.bazel
@@ -11,6 +11,7 @@ go_test(
race = "on",
shard_count = 45,
deps = [
+ "//autoid_service",
"//config",
"//executor",
"//infoschema",
diff --git a/executor/showtest/show_test.go b/executor/showtest/show_test.go
index 3566b1589a271..6ff3db919ee9f 100644
--- a/executor/showtest/show_test.go
+++ b/executor/showtest/show_test.go
@@ -21,6 +21,7 @@ import (
"testing"
"github.com/pingcap/failpoint"
+ _ "github.com/pingcap/tidb/autoid_service"
"github.com/pingcap/tidb/executor"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/parser/auth"
@@ -142,7 +143,8 @@ func TestShowCreateTable(t *testing.T) {
"`d` datetime(4) default current_timestamp(4),\n" +
"`e` varchar(20) default 'cUrrent_tImestamp',\n" +
"`f` datetime(2) default current_timestamp(2) on update current_timestamp(2),\n" +
- "`g` timestamp(2) default current_timestamp(2) on update current_timestamp(2))")
+ "`g` timestamp(2) default current_timestamp(2) on update current_timestamp(2),\n" +
+ "`h` date default current_date )")
tk.MustQuery("show create table `t`").Check(testkit.RowsWithSep("|",
""+
"t CREATE TABLE `t` (\n"+
@@ -152,7 +154,8 @@ func TestShowCreateTable(t *testing.T) {
" `d` datetime(4) DEFAULT CURRENT_TIMESTAMP(4),\n"+
" `e` varchar(20) DEFAULT 'cUrrent_tImestamp',\n"+
" `f` datetime(2) DEFAULT CURRENT_TIMESTAMP(2) ON UPDATE CURRENT_TIMESTAMP(2),\n"+
- " `g` timestamp(2) DEFAULT CURRENT_TIMESTAMP(2) ON UPDATE CURRENT_TIMESTAMP(2)\n"+
+ " `g` timestamp(2) DEFAULT CURRENT_TIMESTAMP(2) ON UPDATE CURRENT_TIMESTAMP(2),\n"+
+ " `h` date DEFAULT CURRENT_DATE\n"+
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin",
))
tk.MustExec("drop table t")
@@ -317,9 +320,10 @@ func TestShowCreateTable(t *testing.T) {
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin",
))
- // TiDB defaults (and only supports) foreign_key_checks=0
+ // set @@foreign_key_checks=0,
// This means that the child table can be created before the parent table.
// This behavior is required for mysqldump restores.
+ tk.MustExec("set @@foreign_key_checks=0")
tk.MustExec(`DROP TABLE IF EXISTS parent, child`)
tk.MustExec(`CREATE TABLE child (id INT NOT NULL PRIMARY KEY auto_increment, parent_id INT NOT NULL, INDEX par_ind (parent_id), CONSTRAINT child_ibfk_1 FOREIGN KEY (parent_id) REFERENCES parent(id))`)
tk.MustExec(`CREATE TABLE parent ( id INT NOT NULL PRIMARY KEY auto_increment )`)
@@ -330,7 +334,7 @@ func TestShowCreateTable(t *testing.T) {
" `parent_id` int(11) NOT NULL,\n"+
" PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n"+
" KEY `par_ind` (`parent_id`),\n"+
- " CONSTRAINT `child_ibfk_1` FOREIGN KEY (`parent_id`) REFERENCES `test`.`parent` (`id`) /* FOREIGN KEY INVALID */\n"+
+ " CONSTRAINT `child_ibfk_1` FOREIGN KEY (`parent_id`) REFERENCES `test`.`parent` (`id`)\n"+
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin",
))
@@ -344,7 +348,7 @@ func TestShowCreateTable(t *testing.T) {
" `parent_id` int(11) NOT NULL,\n"+
" PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n"+
" KEY `par_ind` (`parent_id`),\n"+
- " CONSTRAINT `child_ibfk_1` FOREIGN KEY (`parent_id`) REFERENCES `test`.`parent` (`id`) ON DELETE RESTRICT ON UPDATE CASCADE /* FOREIGN KEY INVALID */\n"+
+ " CONSTRAINT `child_ibfk_1` FOREIGN KEY (`parent_id`) REFERENCES `test`.`parent` (`id`) ON DELETE RESTRICT ON UPDATE CASCADE\n"+
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin",
))
@@ -357,7 +361,8 @@ func TestShowCreateTable(t *testing.T) {
" `id` int(11) NOT NULL,\n" +
" `b` int(11) DEFAULT NULL,\n" +
" PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" +
- " CONSTRAINT `fk` FOREIGN KEY (`b`) REFERENCES `test1`.`t1` (`id`) /* FOREIGN KEY INVALID */\n" +
+ " KEY `fk` (`b`),\n" +
+ " CONSTRAINT `fk` FOREIGN KEY (`b`) REFERENCES `test1`.`t1` (`id`)\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
// Test issue #20327
@@ -963,6 +968,8 @@ func TestShow2(t *testing.T) {
c_timestamp_default timestamp default current_timestamp,
c_timestamp_default_3 timestamp(3) default current_timestamp(3),
c_timestamp_default_4 timestamp(3) default current_timestamp(3) on update current_timestamp(3),
+ c_date_default date default current_date,
+ c_date_default_2 date default (curdate()),
c_blob blob,
c_tinyblob tinyblob,
c_mediumblob mediumblob,
@@ -999,6 +1006,8 @@ func TestShow2(t *testing.T) {
"[c_timestamp_default timestamp YES CURRENT_TIMESTAMP select,insert,update,references ]\n" +
"[c_timestamp_default_3 timestamp(3) YES CURRENT_TIMESTAMP(3) select,insert,update,references ]\n" +
"[c_timestamp_default_4 timestamp(3) YES CURRENT_TIMESTAMP(3) DEFAULT_GENERATED on update CURRENT_TIMESTAMP(3) select,insert,update,references ]\n" +
+ "[c_date_default date YES CURRENT_DATE select,insert,update,references ]\n" +
+ "[c_date_default_2 date YES CURRENT_DATE select,insert,update,references ]\n" +
"[c_blob blob YES select,insert,update,references ]\n" +
"[c_tinyblob tinyblob YES select,insert,update,references ]\n" +
"[c_mediumblob mediumblob YES select,insert,update,references ]\n" +
@@ -1057,11 +1066,11 @@ func TestShowCreateUser(t *testing.T) {
// Create a new user.
tk.MustExec(`CREATE USER 'test_show_create_user'@'%' IDENTIFIED BY 'root';`)
tk.MustQuery("show create user 'test_show_create_user'@'%'").
- Check(testkit.Rows(`CREATE USER 'test_show_create_user'@'%' IDENTIFIED WITH 'mysql_native_password' AS '*81F5E21E35407D884A6CD4A731AEBFB6AF209E1B' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK`))
+ Check(testkit.Rows(`CREATE USER 'test_show_create_user'@'%' IDENTIFIED WITH 'mysql_native_password' AS '*81F5E21E35407D884A6CD4A731AEBFB6AF209E1B' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`))
tk.MustExec(`CREATE USER 'test_show_create_user'@'localhost' IDENTIFIED BY 'test';`)
tk.MustQuery("show create user 'test_show_create_user'@'localhost';").
- Check(testkit.Rows(`CREATE USER 'test_show_create_user'@'localhost' IDENTIFIED WITH 'mysql_native_password' AS '*94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK`))
+ Check(testkit.Rows(`CREATE USER 'test_show_create_user'@'localhost' IDENTIFIED WITH 'mysql_native_password' AS '*94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`))
// Case: the user exists but the host portion doesn't match
err := tk.QueryToErr("show create user 'test_show_create_user'@'asdf';")
@@ -1073,10 +1082,10 @@ func TestShowCreateUser(t *testing.T) {
tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "127.0.0.1", AuthUsername: "root", AuthHostname: "%"}, nil, nil)
tk.MustQuery("show create user current_user").
- Check(testkit.Rows("CREATE USER 'root'@'127.0.0.1' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK"))
+ Check(testkit.Rows("CREATE USER 'root'@'127.0.0.1' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT"))
tk.MustQuery("show create user current_user()").
- Check(testkit.Rows("CREATE USER 'root'@'127.0.0.1' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK"))
+ Check(testkit.Rows("CREATE USER 'root'@'127.0.0.1' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT"))
tk.MustExec("create user 'check_priv'")
@@ -1089,9 +1098,9 @@ func TestShowCreateUser(t *testing.T) {
// "show create user" for current user doesn't check privileges.
tk1.MustQuery("show create user current_user").
- Check(testkit.Rows("CREATE USER 'check_priv'@'127.0.0.1' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK"))
+ Check(testkit.Rows("CREATE USER 'check_priv'@'127.0.0.1' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT"))
- // Creating users with `IDENTIFIED WITH 'caching_sha2_password'`
+ // Creating users with `IDENTIFIED WITH 'caching_sha2_password'`.
tk.MustExec("CREATE USER 'sha_test'@'%' IDENTIFIED WITH 'caching_sha2_password' BY 'temp_passwd'")
// Compare only the start of the output as the salt changes every time.
@@ -1102,29 +1111,57 @@ func TestShowCreateUser(t *testing.T) {
// Compare only the start of the output as the salt changes every time.
rows = tk.MustQuery("SHOW CREATE USER 'sock'@'%'")
- require.Equal(t, "CREATE USER 'sock'@'%' IDENTIFIED WITH 'auth_socket' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK", rows.Rows()[0][0].(string))
+ require.Equal(t, "CREATE USER 'sock'@'%' IDENTIFIED WITH 'auth_socket' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT", rows.Rows()[0][0].(string))
tk.MustExec("CREATE USER 'sock2'@'%' IDENTIFIED WITH 'auth_socket' AS 'sock3'")
// Compare only the start of the output as the salt changes every time.
rows = tk.MustQuery("SHOW CREATE USER 'sock2'@'%'")
- require.Equal(t, "CREATE USER 'sock2'@'%' IDENTIFIED WITH 'auth_socket' AS 'sock3' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK", rows.Rows()[0][0].(string))
+ require.Equal(t, "CREATE USER 'sock2'@'%' IDENTIFIED WITH 'auth_socket' AS 'sock3' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT", rows.Rows()[0][0].(string))
- // Test ACCOUNT LOCK/UNLOCK
+ // Test ACCOUNT LOCK/UNLOCK.
tk.MustExec("CREATE USER 'lockness'@'%' IDENTIFIED BY 'monster' ACCOUNT LOCK")
rows = tk.MustQuery("SHOW CREATE USER 'lockness'@'%'")
- require.Equal(t, "CREATE USER 'lockness'@'%' IDENTIFIED WITH 'mysql_native_password' AS '*BC05309E7FE12AFD4EBB9FFE7E488A6320F12FF3' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT LOCK", rows.Rows()[0][0].(string))
+ require.Equal(t, "CREATE USER 'lockness'@'%' IDENTIFIED WITH 'mysql_native_password' AS '*BC05309E7FE12AFD4EBB9FFE7E488A6320F12FF3' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT LOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT", rows.Rows()[0][0].(string))
- // Test COMMENT and ATTRIBUTE
+ // Test COMMENT and ATTRIBUTE.
tk.MustExec("CREATE USER commentUser COMMENT '1234'")
- tk.MustQuery("SHOW CREATE USER commentUser").Check(testkit.Rows(`CREATE USER 'commentUser'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK ATTRIBUTE {"comment": "1234"}`))
+ tk.MustQuery("SHOW CREATE USER commentUser").Check(testkit.Rows(`CREATE USER 'commentUser'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT ATTRIBUTE '{"comment": "1234"}'`))
tk.MustExec(`CREATE USER attributeUser attribute '{"name": "Tom", "age": 19}'`)
- tk.MustQuery("SHOW CREATE USER attributeUser").Check(testkit.Rows(`CREATE USER 'attributeUser'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK ATTRIBUTE {"age": 19, "name": "Tom"}`))
+ tk.MustQuery("SHOW CREATE USER attributeUser").Check(testkit.Rows(`CREATE USER 'attributeUser'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT ATTRIBUTE '{"age": 19, "name": "Tom"}'`))
- // Creating users with 'IDENTIFIED WITH 'tidb_auth_token''
+ // Creating users with IDENTIFIED WITH 'tidb_auth_token'.
tk.MustExec(`CREATE USER 'token_user'@'%' IDENTIFIED WITH 'tidb_auth_token' ATTRIBUTE '{"email": "user@pingcap.com"}'`)
- tk.MustQuery("SHOW CREATE USER token_user").Check(testkit.Rows(`CREATE USER 'token_user'@'%' IDENTIFIED WITH 'tidb_auth_token' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK ATTRIBUTE {"email": "user@pingcap.com"}`))
+ tk.MustQuery("SHOW CREATE USER token_user").Check(testkit.Rows(`CREATE USER 'token_user'@'%' IDENTIFIED WITH 'tidb_auth_token' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT ATTRIBUTE '{"email": "user@pingcap.com"}'`))
tk.MustExec(`ALTER USER 'token_user'@'%' REQUIRE token_issuer 'issuer-ABC'`)
- tk.MustQuery("SHOW CREATE USER token_user").Check(testkit.Rows(`CREATE USER 'token_user'@'%' IDENTIFIED WITH 'tidb_auth_token' AS '' REQUIRE NONE token_issuer issuer-ABC PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK ATTRIBUTE {"email": "user@pingcap.com"}`))
+ tk.MustQuery("SHOW CREATE USER token_user").Check(testkit.Rows(`CREATE USER 'token_user'@'%' IDENTIFIED WITH 'tidb_auth_token' AS '' REQUIRE NONE token_issuer issuer-ABC PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT ATTRIBUTE '{"email": "user@pingcap.com"}'`))
+
+ // create users with password reuse.
+ tk.MustExec(`CREATE USER 'reuse_user'@'%' IDENTIFIED WITH 'tidb_auth_token' PASSWORD HISTORY 5 PASSWORD REUSE INTERVAL 3 DAY`)
+ tk.MustQuery("SHOW CREATE USER reuse_user").Check(testkit.Rows(`CREATE USER 'reuse_user'@'%' IDENTIFIED WITH 'tidb_auth_token' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY 5 PASSWORD REUSE INTERVAL 3 DAY`))
+ tk.MustExec(`ALTER USER 'reuse_user'@'%' PASSWORD HISTORY 50`)
+ tk.MustQuery("SHOW CREATE USER reuse_user").Check(testkit.Rows(`CREATE USER 'reuse_user'@'%' IDENTIFIED WITH 'tidb_auth_token' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY 50 PASSWORD REUSE INTERVAL 3 DAY`))
+ tk.MustExec(`ALTER USER 'reuse_user'@'%' PASSWORD REUSE INTERVAL 31 DAY`)
+ tk.MustQuery("SHOW CREATE USER reuse_user").Check(testkit.Rows(`CREATE USER 'reuse_user'@'%' IDENTIFIED WITH 'tidb_auth_token' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY 50 PASSWORD REUSE INTERVAL 31 DAY`))
+
+ tk.MustExec("CREATE USER 'jeffrey1'@'localhost' PASSWORD EXPIRE")
+ tk.MustQuery("SHOW CREATE USER 'jeffrey1'@'localhost'").Check(testkit.Rows(`CREATE USER 'jeffrey1'@'localhost' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`))
+ tk.MustExec("CREATE USER 'jeffrey2'@'localhost' PASSWORD EXPIRE DEFAULT")
+ tk.MustQuery("SHOW CREATE USER 'jeffrey2'@'localhost'").Check(testkit.Rows(`CREATE USER 'jeffrey2'@'localhost' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`))
+ tk.MustExec("CREATE USER 'jeffrey3'@'localhost' PASSWORD EXPIRE NEVER")
+ tk.MustQuery("SHOW CREATE USER 'jeffrey3'@'localhost'").Check(testkit.Rows(`CREATE USER 'jeffrey3'@'localhost' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE NEVER ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`))
+ tk.MustExec("CREATE USER 'jeffrey4'@'localhost' PASSWORD EXPIRE INTERVAL 180 DAY")
+ tk.MustQuery("SHOW CREATE USER 'jeffrey4'@'localhost'").Check(testkit.Rows(`CREATE USER 'jeffrey4'@'localhost' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE INTERVAL 180 DAY ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`))
+
+ tk.MustExec("CREATE USER failed_login_user")
+ tk.MustQuery("SHOW CREATE USER failed_login_user").Check(testkit.Rows(`CREATE USER 'failed_login_user'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`))
+ tk.MustExec("ALTER USER failed_login_user FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 2")
+ tk.MustQuery("SHOW CREATE USER failed_login_user").Check(testkit.Rows(`CREATE USER 'failed_login_user'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 2`))
+ tk.MustExec("ALTER USER failed_login_user PASSWORD_LOCK_TIME UNBOUNDED")
+ tk.MustQuery("SHOW CREATE USER failed_login_user").Check(testkit.Rows(`CREATE USER 'failed_login_user'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME UNBOUNDED`))
+ tk.MustExec("ALTER USER failed_login_user comment 'testcomment'")
+ tk.MustQuery("SHOW CREATE USER failed_login_user").Check(testkit.Rows(`CREATE USER 'failed_login_user'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME UNBOUNDED ATTRIBUTE '{"comment": "testcomment"}'`))
+ tk.MustExec("ALTER USER failed_login_user ATTRIBUTE '{\"attribute\": \"testattribute\"}'")
+ tk.MustQuery("SHOW CREATE USER failed_login_user").Check(testkit.Rows(`CREATE USER 'failed_login_user'@'%' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME UNBOUNDED ATTRIBUTE '{"attribute": "testattribute", "comment": "testcomment"}'`))
}
func TestUnprivilegedShow(t *testing.T) {
@@ -1485,7 +1522,7 @@ func TestShowBuiltin(t *testing.T) {
res := tk.MustQuery("show builtins;")
require.NotNil(t, res)
rows := res.Rows()
- const builtinFuncNum = 283
+ const builtinFuncNum = 285
require.Equal(t, builtinFuncNum, len(rows))
require.Equal(t, rows[0][0].(string), "abs")
require.Equal(t, rows[builtinFuncNum-1][0].(string), "yearweek")
@@ -1989,3 +2026,55 @@ func TestShowLimitReturnRow(t *testing.T) {
rows = result.Rows()
require.Equal(t, rows[0][2], "idx_b")
}
+
+func TestShowTTLOption(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(created_at datetime) ttl = `created_at` + INTERVAL 100 YEAR")
+ tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`created_at` + INTERVAL 100 YEAR */ /*T![ttl] TTL_ENABLE='ON' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(created_at datetime) ttl = `created_at` + INTERVAL 100 YEAR ttl_enable = 'OFF'")
+ tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`created_at` + INTERVAL 100 YEAR */ /*T![ttl] TTL_ENABLE='OFF' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t (created_at datetime) TTL = created_at + INTERVAL 3.14159 HOUR_MINUTE")
+ tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`created_at` + INTERVAL 3.14159 HOUR_MINUTE */ /*T![ttl] TTL_ENABLE='ON' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t (created_at datetime) TTL = created_at + INTERVAL \"15:20\" HOUR_MINUTE")
+ tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`created_at` + INTERVAL _utf8mb4'15:20' HOUR_MINUTE */ /*T![ttl] TTL_ENABLE='ON' */ /*T![ttl] TTL_JOB_INTERVAL='1h' */"))
+
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t (created_at datetime) TTL = created_at + INTERVAL 100 YEAR TTL_JOB_INTERVAL = '1d'")
+ tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `created_at` datetime DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![ttl] TTL=`created_at` + INTERVAL 100 YEAR */ /*T![ttl] TTL_ENABLE='ON' */ /*T![ttl] TTL_JOB_INTERVAL='1d' */"))
+}
+
+func TestShowBindingDigestField(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1, t2")
+ tk.MustExec("create table t1(id int, key(id))")
+ tk.MustExec("create table t2(id int, key(id))")
+ tk.MustExec("create binding for select * from t1, t2 where t1.id = t2.id using select /*+ merge_join(t1, t2)*/ * from t1, t2 where t1.id = t2.id")
+ result := tk.MustQuery("show bindings;")
+ rows := result.Rows()[0]
+ require.Equal(t, len(rows), 11)
+ require.Equal(t, rows[9], "ac1ceb4eb5c01f7c03e29b7d0d6ab567e563f4c93164184cde218f20d07fd77c")
+ tk.MustExec("drop binding for select * from t1, t2 where t1.id = t2.id")
+ result = tk.MustQuery("show bindings;")
+ require.Equal(t, len(result.Rows()), 0)
+
+ tk.MustExec("create global binding for select * from t1, t2 where t1.id = t2.id using select /*+ merge_join(t1, t2)*/ * from t1, t2 where t1.id = t2.id")
+ result = tk.MustQuery("show global bindings;")
+ rows = result.Rows()[0]
+ require.Equal(t, len(rows), 11)
+ require.Equal(t, rows[9], "ac1ceb4eb5c01f7c03e29b7d0d6ab567e563f4c93164184cde218f20d07fd77c")
+ tk.MustExec("drop global binding for select * from t1, t2 where t1.id = t2.id")
+ result = tk.MustQuery("show global bindings;")
+ require.Equal(t, len(result.Rows()), 0)
+}
diff --git a/executor/simple.go b/executor/simple.go
index 2ab99b101dde1..dad3a6e63781e 100644
--- a/executor/simple.go
+++ b/executor/simple.go
@@ -19,7 +19,9 @@ import (
"context"
"encoding/json"
"fmt"
+ "math"
"os"
+ "strconv"
"strings"
"syscall"
"time"
@@ -51,18 +53,23 @@ import (
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/hack"
"github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/mathutil"
+ pwdValidator "github.com/pingcap/tidb/util/password-validation"
"github.com/pingcap/tidb/util/sem"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/pingcap/tidb/util/timeutil"
"github.com/pingcap/tidb/util/tls"
"github.com/pingcap/tipb/go-tipb"
- tikvutil "github.com/tikv/client-go/v2/util"
"go.uber.org/zap"
)
+const notSpecified = -1
+
var (
- transactionDurationPessimisticRollback = metrics.TransactionDuration.WithLabelValues(metrics.LblPessimistic, metrics.LblRollback)
- transactionDurationOptimisticRollback = metrics.TransactionDuration.WithLabelValues(metrics.LblOptimistic, metrics.LblRollback)
+ transactionDurationPessimisticRollbackInternal = metrics.TransactionDuration.WithLabelValues(metrics.LblPessimistic, metrics.LblRollback, metrics.LblInternal)
+ transactionDurationPessimisticRollbackGeneral = metrics.TransactionDuration.WithLabelValues(metrics.LblPessimistic, metrics.LblRollback, metrics.LblGeneral)
+ transactionDurationOptimisticRollbackInternal = metrics.TransactionDuration.WithLabelValues(metrics.LblOptimistic, metrics.LblRollback, metrics.LblInternal)
+ transactionDurationOptimisticRollbackGeneral = metrics.TransactionDuration.WithLabelValues(metrics.LblOptimistic, metrics.LblRollback, metrics.LblGeneral)
)
// SimpleExec represents simple statement executor.
@@ -85,6 +92,33 @@ type SimpleExec struct {
staleTxnStartTS uint64
}
+type passwordOrLockOptionsInfo struct {
+ lockAccount string
+ passwordExpired string
+ passwordLifetime any
+ passwordHistory int64
+ passwordHistoryChange bool
+ passwordReuseInterval int64
+ passwordReuseIntervalChange bool
+ failedLoginAttempts int64
+ passwordLockTime int64
+ failedLoginAttemptsChange bool
+ passwordLockTimeChange bool
+}
+
+type passwordReuseInfo struct {
+ passwordHistory int64
+ passwordReuseInterval int64
+}
+
+type userInfo struct {
+ host string
+ user string
+ pLI *passwordOrLockOptionsInfo
+ pwd string
+ authString string
+}
+
func (e *baseExecutor) getSysSession() (sessionctx.Context, error) {
dom := domain.GetDomain(e.ctx)
sysSessionPool := dom.SysSessionPool()
@@ -110,6 +144,16 @@ func (e *baseExecutor) releaseSysSession(ctx context.Context, sctx sessionctx.Co
sysSessionPool.Put(sctx.(pools.Resource))
}
+// clearSysSession close the session does not return the session.
+// Since the environment variables in the session are changed, the session object is not returned.
+func clearSysSession(ctx context.Context, sctx sessionctx.Context) {
+ if sctx == nil {
+ return
+ }
+ _, _ = sctx.(sqlexec.SQLExecutor).ExecuteInternal(ctx, "rollback")
+ sctx.(pools.Resource).Close()
+}
+
// Next implements the Executor Next interface.
func (e *SimpleExec) Next(ctx context.Context, req *chunk.Chunk) (err error) {
if e.done {
@@ -772,10 +816,18 @@ func (e *SimpleExec) executeRollback(s *ast.RollbackStmt) error {
sessVars.SetInTxn(false)
if txn.Valid() {
duration := time.Since(sessVars.TxnCtx.CreateTime).Seconds()
- if sessVars.TxnCtx.IsPessimistic {
- transactionDurationPessimisticRollback.Observe(duration)
- } else {
- transactionDurationOptimisticRollback.Observe(duration)
+ isInternal := false
+ if internal := txn.GetOption(kv.RequestSourceInternal); internal != nil && internal.(bool) {
+ isInternal = true
+ }
+ if isInternal && sessVars.TxnCtx.IsPessimistic {
+ transactionDurationPessimisticRollbackInternal.Observe(duration)
+ } else if isInternal && !sessVars.TxnCtx.IsPessimistic {
+ transactionDurationOptimisticRollbackInternal.Observe(duration)
+ } else if !isInternal && sessVars.TxnCtx.IsPessimistic {
+ transactionDurationPessimisticRollbackGeneral.Observe(duration)
+ } else if !isInternal && !sessVars.TxnCtx.IsPessimistic {
+ transactionDurationOptimisticRollbackGeneral.Observe(duration)
}
sessVars.TxnCtx.ClearDelta()
return txn.Rollback()
@@ -783,6 +835,215 @@ func (e *SimpleExec) executeRollback(s *ast.RollbackStmt) error {
return nil
}
+func whetherSavePasswordHistory(plOptions *passwordOrLockOptionsInfo) bool {
+ var passwdSaveNum, passwdSaveTime int64
+ // If the user specifies a default, read the global variable.
+ if plOptions.passwordHistoryChange && plOptions.passwordHistory != notSpecified {
+ passwdSaveNum = plOptions.passwordHistory
+ } else {
+ passwdSaveNum = variable.PasswordHistory.Load()
+ }
+ if plOptions.passwordReuseIntervalChange && plOptions.passwordReuseInterval != notSpecified {
+ passwdSaveTime = plOptions.passwordReuseInterval
+ } else {
+ passwdSaveTime = variable.PasswordReuseInterval.Load()
+ }
+ return passwdSaveTime > 0 || passwdSaveNum > 0
+}
+
+type alterUserPasswordLocking struct {
+ failedLoginAttempts int64
+ passwordLockTime int64
+ failedLoginAttemptsNotFound bool
+ passwordLockTimeChangeNotFound bool
+ // containsNoOthers indicates whether User_attributes only contains one "Password_locking" element.
+ containsNoOthers bool
+}
+
+func (info *passwordOrLockOptionsInfo) loadOptions(plOption []*ast.PasswordOrLockOption) error {
+ if length := len(plOption); length > 0 {
+ // If "PASSWORD EXPIRE ..." appears many times,
+ // only the last declaration takes effect.
+ Loop:
+ for i := length - 1; i >= 0; i-- {
+ switch plOption[i].Type {
+ case ast.PasswordExpire:
+ info.passwordExpired = "Y"
+ break Loop
+ case ast.PasswordExpireDefault:
+ info.passwordLifetime = nil
+ break Loop
+ case ast.PasswordExpireNever:
+ info.passwordLifetime = 0
+ break Loop
+ case ast.PasswordExpireInterval:
+ if plOption[i].Count == 0 || plOption[i].Count > math.MaxUint16 {
+ return types.ErrWrongValue2.GenWithStackByArgs("DAY", fmt.Sprintf("%v", plOption[i].Count))
+ }
+ info.passwordLifetime = plOption[i].Count
+ break Loop
+ }
+ }
+ }
+ // only the last declaration takes effect.
+ for _, option := range plOption {
+ switch option.Type {
+ case ast.Lock:
+ info.lockAccount = "Y"
+ case ast.Unlock:
+ info.lockAccount = "N"
+ case ast.FailedLoginAttempts:
+ info.failedLoginAttempts = mathutil.Min(option.Count, math.MaxInt16)
+ info.failedLoginAttemptsChange = true
+ case ast.PasswordLockTime:
+ info.passwordLockTime = mathutil.Min(option.Count, math.MaxInt16)
+ info.passwordLockTimeChange = true
+ case ast.PasswordLockTimeUnbounded:
+ info.passwordLockTime = -1
+ info.passwordLockTimeChange = true
+ case ast.PasswordHistory:
+ info.passwordHistory = mathutil.Min(option.Count, math.MaxUint16)
+ info.passwordHistoryChange = true
+ case ast.PasswordHistoryDefault:
+ info.passwordHistory = notSpecified
+ info.passwordHistoryChange = true
+ case ast.PasswordReuseInterval:
+ info.passwordReuseInterval = mathutil.Min(option.Count, math.MaxUint16)
+ info.passwordReuseIntervalChange = true
+ case ast.PasswordReuseDefault:
+ info.passwordReuseInterval = notSpecified
+ info.passwordReuseIntervalChange = true
+ }
+ }
+ return nil
+}
+
+func createUserFailedLoginJSON(info *passwordOrLockOptionsInfo) string {
+ // Record only when either failedLoginAttempts and passwordLockTime is not 0
+ if (info.failedLoginAttemptsChange && info.failedLoginAttempts != 0) || (info.passwordLockTimeChange && info.passwordLockTime != 0) {
+ return fmt.Sprintf("\"Password_locking\": {\"failed_login_attempts\": %d,\"password_lock_time_days\": %d}",
+ info.failedLoginAttempts, info.passwordLockTime)
+ }
+ return ""
+}
+
+func alterUserFailedLoginJSON(info *alterUserPasswordLocking, lockAccount string) string {
+ // alterUserPasswordLocking is the user's actual configuration.
+ var passwordLockingArray []string
+ if info.failedLoginAttempts != 0 || info.passwordLockTime != 0 {
+ if lockAccount == "N" {
+ passwordLockingArray = append(passwordLockingArray,
+ fmt.Sprintf("\"auto_account_locked\": \"%s\"", lockAccount),
+ fmt.Sprintf("\"auto_locked_last_changed\": \"%s\"", time.Now().Format(time.UnixDate)),
+ fmt.Sprintf("\"failed_login_count\": %d", 0))
+ }
+ passwordLockingArray = append(passwordLockingArray,
+ fmt.Sprintf("\"failed_login_attempts\": %d", info.failedLoginAttempts),
+ fmt.Sprintf("\"password_lock_time_days\": %d", info.passwordLockTime))
+ }
+ if len(passwordLockingArray) > 0 {
+ return fmt.Sprintf("\"Password_locking\": {%s}", strings.Join(passwordLockingArray, ","))
+ }
+ return ""
+}
+
+func readPasswordLockingInfo(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, name string, host string, pLO *passwordOrLockOptionsInfo) (aUPL *alterUserPasswordLocking, err error) {
+ alterUserInfo := &alterUserPasswordLocking{
+ failedLoginAttempts: 0,
+ passwordLockTime: 0,
+ failedLoginAttemptsNotFound: false,
+ passwordLockTimeChangeNotFound: false,
+ containsNoOthers: false,
+ }
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, `SELECT JSON_UNQUOTE(JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_attempts')),
+ JSON_UNQUOTE(JSON_EXTRACT(user_attributes, '$.Password_locking.password_lock_time_days')),
+ JSON_LENGTH(JSON_REMOVE(user_attributes, '$.Password_locking')) FROM %n.%n WHERE User=%? AND Host=%?;`,
+ mysql.SystemDB, mysql.UserTable, name, strings.ToLower(host))
+ recordSet, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if closeErr := recordSet.Close(); closeErr != nil {
+ err = closeErr
+ }
+ }()
+ rows, err := sqlexec.DrainRecordSet(ctx, recordSet, 3)
+ if err != nil {
+ return nil, err
+ }
+
+ // Configuration priority is User Changes > User History
+ if pLO.failedLoginAttemptsChange {
+ alterUserInfo.failedLoginAttempts = pLO.failedLoginAttempts
+ } else if !rows[0].IsNull(0) {
+ str := rows[0].GetString(0)
+ alterUserInfo.failedLoginAttempts, err = strconv.ParseInt(str, 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ alterUserInfo.failedLoginAttempts = mathutil.Max(alterUserInfo.failedLoginAttempts, 0)
+ alterUserInfo.failedLoginAttempts = mathutil.Min(alterUserInfo.failedLoginAttempts, math.MaxInt16)
+ } else {
+ alterUserInfo.failedLoginAttemptsNotFound = true
+ }
+
+ if pLO.passwordLockTimeChange {
+ alterUserInfo.passwordLockTime = pLO.passwordLockTime
+ } else if !rows[0].IsNull(1) {
+ str := rows[0].GetString(1)
+ alterUserInfo.passwordLockTime, err = strconv.ParseInt(str, 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ alterUserInfo.passwordLockTime = mathutil.Max(alterUserInfo.passwordLockTime, -1)
+ alterUserInfo.passwordLockTime = mathutil.Min(alterUserInfo.passwordLockTime, math.MaxInt16)
+ } else {
+ alterUserInfo.passwordLockTimeChangeNotFound = true
+ }
+
+ alterUserInfo.containsNoOthers = rows[0].IsNull(2) || rows[0].GetInt64(2) == 0
+ return alterUserInfo, nil
+}
+
+// deletePasswordLockingAttribute deletes "$.Password_locking" in "User_attributes" when failedLoginAttempts and passwordLockTime both 0.
+func deletePasswordLockingAttribute(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, name string, host string, alterUser *alterUserPasswordLocking) error {
+ // No password_locking information.
+ if alterUser.failedLoginAttemptsNotFound && alterUser.passwordLockTimeChangeNotFound {
+ return nil
+ }
+ // Password_locking information is still in used.
+ if alterUser.failedLoginAttempts != 0 || alterUser.passwordLockTime != 0 {
+ return nil
+ }
+ sql := new(strings.Builder)
+ if alterUser.containsNoOthers {
+ // If we use JSON_REMOVE(user_attributes, '$.Password_locking') directly here, the result is not compatible with MySQL.
+ sqlexec.MustFormatSQL(sql, `UPDATE %n.%n SET user_attributes=NULL`, mysql.SystemDB, mysql.UserTable)
+ } else {
+ sqlexec.MustFormatSQL(sql, `UPDATE %n.%n SET user_attributes=JSON_REMOVE(user_attributes, '$.Password_locking') `, mysql.SystemDB, mysql.UserTable)
+ }
+ sqlexec.MustFormatSQL(sql, " WHERE Host=%? and User=%?;", host, name)
+ _, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ return err
+}
+
+func (e *SimpleExec) authUsingCleartextPwd(authOpt *ast.AuthOption, authPlugin string) bool {
+ if authOpt == nil || !authOpt.ByAuthString {
+ return false
+ }
+ return mysql.IsAuthPluginClearText(authPlugin)
+}
+
+func (e *SimpleExec) isValidatePasswordEnabled() bool {
+ validatePwdEnable, err := e.ctx.GetSessionVars().GlobalVarsAccessor.GetGlobalSysVar(variable.ValidatePasswordEnable)
+ if err != nil {
+ return false
+ }
+ return variable.TiDBOptOn(validatePwdEnable)
+}
+
func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStmt) error {
internalCtx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnPrivilege)
// Check `CREATE USER` privilege.
@@ -810,32 +1071,59 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm
return err
}
- lockAccount := "N"
- if length := len(s.PasswordOrLockOptions); length > 0 {
- // If "ACCOUNT LOCK" or "ACCOUNT UNLOCK" appears many times,
- // the last declaration takes effect.
- for i := length - 1; i >= 0; i-- {
- if s.PasswordOrLockOptions[i].Type == ast.Lock {
- lockAccount = "Y"
- break
- } else if s.PasswordOrLockOptions[i].Type == ast.Unlock {
- break
- }
- }
+ plOptions := &passwordOrLockOptionsInfo{
+ lockAccount: "N",
+ passwordExpired: "N",
+ passwordLifetime: nil,
+ passwordHistory: notSpecified,
+ passwordReuseInterval: notSpecified,
+ failedLoginAttemptsChange: false,
+ passwordLockTimeChange: false,
+ passwordHistoryChange: false,
+ passwordReuseIntervalChange: false,
}
+ err = plOptions.loadOptions(s.PasswordOrLockOptions)
+ if err != nil {
+ return err
+ }
+ PasswordLocking := createUserFailedLoginJSON(plOptions)
if s.IsCreateRole {
- lockAccount = "Y"
+ plOptions.lockAccount = "Y"
+ plOptions.passwordExpired = "Y"
}
- var userAttributes any = nil
+ var userAttributes []string
if s.CommentOrAttributeOption != nil {
if s.CommentOrAttributeOption.Type == ast.UserCommentType {
- userAttributes = fmt.Sprintf("{\"metadata\": {\"comment\": \"%s\"}}", s.CommentOrAttributeOption.Value)
+ userAttributes = append(userAttributes, fmt.Sprintf("\"metadata\": {\"comment\": \"%s\"}", s.CommentOrAttributeOption.Value))
} else if s.CommentOrAttributeOption.Type == ast.UserAttributeType {
- userAttributes = fmt.Sprintf("{\"metadata\": %s}", s.CommentOrAttributeOption.Value)
+ userAttributes = append(userAttributes, fmt.Sprintf("\"metadata\": %s", s.CommentOrAttributeOption.Value))
}
}
+ if s.ResourceGroupNameOption != nil {
+ if !variable.EnableResourceControl.Load() {
+ return infoschema.ErrResourceGroupSupportDisabled
+ }
+
+ resourceGroupName := strings.ToLower(s.ResourceGroupNameOption.Value)
+
+ // check if specified resource group exists
+ if resourceGroupName != "default" && resourceGroupName != "" {
+ _, exists := e.is.ResourceGroupByName(model.NewCIStr(resourceGroupName))
+ if !exists {
+ return infoschema.ErrResourceGroupNotExists.GenWithStackByArgs(resourceGroupName)
+ }
+ }
+ userAttributes = append(userAttributes, fmt.Sprintf("\"resource_group\": \"%s\"", resourceGroupName))
+ }
+ // If FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME are both specified to 0, a string of 0 length is generated.
+ // When inserting the attempts into json, an error occurs. This requires special handling.
+ if PasswordLocking != "" {
+ userAttributes = append(userAttributes, PasswordLocking)
+ }
+ userAttributesStr := fmt.Sprintf("{%s}", strings.Join(userAttributes, ","))
+
tokenIssuer := ""
for _, authTokenOption := range s.AuthTokenOrTLSOptions {
switch authTokenOption.Type {
@@ -845,13 +1133,26 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm
}
sql := new(strings.Builder)
- sqlexec.MustFormatSQL(sql, `INSERT INTO %n.%n (Host, User, authentication_string, plugin, user_attributes, Account_locked, Token_issuer) VALUES `, mysql.SystemDB, mysql.UserTable)
+ sqlPasswordHistory := new(strings.Builder)
+ passwordInit := true
+ // Get changed user password reuse info.
+ savePasswdHistory := whetherSavePasswordHistory(plOptions)
+ sqlTemplate := "INSERT INTO %n.%n (Host, User, authentication_string, plugin, user_attributes, Account_locked, Token_issuer, Password_expired, Password_lifetime, Password_reuse_time, Password_reuse_history) VALUES "
+ valueTemplate := "(%?, %?, %?, %?, %?, %?, %?, %?, %?"
+
+ sqlexec.MustFormatSQL(sql, sqlTemplate, mysql.SystemDB, mysql.UserTable)
+ if savePasswdHistory {
+ sqlexec.MustFormatSQL(sqlPasswordHistory, `INSERT INTO %n.%n (Host, User, Password) VALUES `, mysql.SystemDB, mysql.PasswordHistoryTable)
+ }
users := make([]*auth.UserIdentity, 0, len(s.Specs))
for _, spec := range s.Specs {
if len(spec.User.Username) > auth.UserNameMaxLength {
return ErrWrongStringLength.GenWithStackByArgs(spec.User.Username, "user name", auth.UserNameMaxLength)
}
+ if len(spec.User.Username) == 0 && plOptions.passwordExpired == "Y" {
+ return ErrPasswordExpireAnonymousUser.GenWithStackByArgs()
+ }
if len(spec.User.Hostname) > auth.HostNameMaxLength {
return ErrWrongStringLength.GenWithStackByArgs(spec.User.Hostname, "host name", auth.HostNameMaxLength)
}
@@ -874,15 +1175,25 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm
e.ctx.GetSessionVars().StmtCtx.AppendNote(err)
continue
}
+ authPlugin := mysql.AuthNativePassword
+ if spec.AuthOpt != nil && spec.AuthOpt.AuthPlugin != "" {
+ authPlugin = spec.AuthOpt.AuthPlugin
+ }
+ if e.isValidatePasswordEnabled() && !s.IsCreateRole {
+ if spec.AuthOpt == nil || !spec.AuthOpt.ByAuthString && spec.AuthOpt.HashString == "" {
+ return variable.ErrNotValidPassword.GenWithStackByArgs()
+ }
+ if e.authUsingCleartextPwd(spec.AuthOpt, authPlugin) {
+ if err := pwdValidator.ValidatePassword(e.ctx.GetSessionVars(), spec.AuthOpt.AuthString); err != nil {
+ return err
+ }
+ }
+ }
pwd, ok := spec.EncodedPassword()
if !ok {
return errors.Trace(ErrPasswordFormat)
}
- authPlugin := mysql.AuthNativePassword
- if spec.AuthOpt != nil && spec.AuthOpt.AuthPlugin != "" {
- authPlugin = spec.AuthOpt.AuthPlugin
- }
switch authPlugin {
case mysql.AuthNativePassword, mysql.AuthCachingSha2Password, mysql.AuthTiDBSM3Password, mysql.AuthSocket, mysql.AuthTiDBAuthToken:
@@ -901,7 +1212,31 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm
}
hostName := strings.ToLower(spec.User.Hostname)
- sqlexec.MustFormatSQL(sql, `(%?, %?, %?, %?, %?, %?, %?)`, hostName, spec.User.Username, pwd, authPlugin, userAttributes, lockAccount, recordTokenIssuer)
+ sqlexec.MustFormatSQL(sql, valueTemplate, hostName, spec.User.Username, pwd, authPlugin, userAttributesStr, plOptions.lockAccount, recordTokenIssuer, plOptions.passwordExpired, plOptions.passwordLifetime)
+ // add Password_reuse_time value.
+ if plOptions.passwordReuseIntervalChange && (plOptions.passwordReuseInterval != notSpecified) {
+ sqlexec.MustFormatSQL(sql, `, %?`, plOptions.passwordReuseInterval)
+ } else {
+ sqlexec.MustFormatSQL(sql, `, %?`, nil)
+ }
+ // add Password_reuse_history value.
+ if plOptions.passwordHistoryChange && (plOptions.passwordHistory != notSpecified) {
+ sqlexec.MustFormatSQL(sql, `, %?`, plOptions.passwordHistory)
+ } else {
+ sqlexec.MustFormatSQL(sql, `, %?`, nil)
+ }
+ sqlexec.MustFormatSQL(sql, `)`)
+ // The empty password does not count in the password history and is subject to reuse at any time.
+ // AuthTiDBAuthToken is the token login method on the cloud,
+ // and the Password Reuse Policy does not take effect.
+ if savePasswdHistory && len(pwd) != 0 && !strings.EqualFold(authPlugin, mysql.AuthTiDBAuthToken) {
+ if !passwordInit {
+ sqlexec.MustFormatSQL(sqlPasswordHistory, ",")
+ } else {
+ passwordInit = false
+ }
+ sqlexec.MustFormatSQL(sqlPasswordHistory, `( %?, %?, %?)`, hostName, spec.User.Username, pwd)
+ }
users = append(users, spec.User)
}
if len(users) == 0 {
@@ -920,11 +1255,23 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm
}
_, err = sqlExecutor.ExecuteInternal(internalCtx, sql.String())
if err != nil {
+ logutil.BgLogger().Warn("Fail to create user", zap.String("sql", sql.String()))
if _, rollbackErr := sqlExecutor.ExecuteInternal(internalCtx, "rollback"); rollbackErr != nil {
return rollbackErr
}
return err
}
+
+ if savePasswdHistory && !passwordInit {
+ _, err = sqlExecutor.ExecuteInternal(internalCtx, sqlPasswordHistory.String())
+ if err != nil {
+ if _, rollbackErr := sqlExecutor.ExecuteInternal(internalCtx, "rollback"); rollbackErr != nil {
+ return errors.Trace(rollbackErr)
+ }
+ return errors.Trace(err)
+ }
+ }
+
if len(privData) != 0 {
sql.Reset()
sqlexec.MustFormatSQL(sql, "INSERT IGNORE INTO %n.%n (Host, User, Priv) VALUES ", mysql.SystemDB, mysql.GlobalPrivTable)
@@ -948,7 +1295,377 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm
return domain.GetDomain(e.ctx).NotifyUpdatePrivilege()
}
+func getUserPasswordLimit(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, name string, host string, plOptions *passwordOrLockOptionsInfo) (pRI *passwordReuseInfo, err error) {
+ res := &passwordReuseInfo{notSpecified, notSpecified}
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, `SELECT Password_reuse_history,Password_reuse_time FROM %n.%n WHERE User=%? AND Host=%?;`,
+ mysql.SystemDB, mysql.UserTable, name, strings.ToLower(host))
+ // Query the specified user password reuse rules.
+ recordSet, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if closeErr := recordSet.Close(); closeErr != nil {
+ err = closeErr
+ }
+ }()
+ rows, err := sqlexec.DrainRecordSet(ctx, recordSet, 3)
+ if err != nil {
+ return nil, err
+ }
+ for _, row := range rows {
+ if !row.IsNull(0) {
+ res.passwordHistory = int64(row.GetUint64(0))
+ } else {
+ res.passwordHistory = variable.PasswordHistory.Load()
+ }
+ if !row.IsNull(1) {
+ res.passwordReuseInterval = int64(row.GetUint64(1))
+ } else {
+ res.passwordReuseInterval = variable.PasswordReuseInterval.Load()
+ }
+ }
+ if plOptions.passwordHistoryChange {
+ // If the user specifies a default, the global variable needs to be re-read.
+ if plOptions.passwordHistory != notSpecified {
+ res.passwordHistory = plOptions.passwordHistory
+ } else {
+ res.passwordHistory = variable.PasswordHistory.Load()
+ }
+ }
+ if plOptions.passwordReuseIntervalChange {
+ // If the user specifies a default, the global variable needs to be re-read.
+ if plOptions.passwordReuseInterval != notSpecified {
+ res.passwordReuseInterval = plOptions.passwordReuseInterval
+ } else {
+ res.passwordReuseInterval = variable.PasswordReuseInterval.Load()
+ }
+ }
+ return res, nil
+}
+
+// getValidTime get the boundary of password valid time.
+func getValidTime(sctx sessionctx.Context, passwordReuse *passwordReuseInfo) string {
+ nowTime := time.Now().In(sctx.GetSessionVars().TimeZone)
+ nowTimeS := nowTime.Unix()
+ beforeTimeS := nowTimeS - passwordReuse.passwordReuseInterval*24*int64(time.Hour/time.Second)
+ if beforeTimeS < 0 {
+ beforeTimeS = 0
+ }
+ return time.Unix(beforeTimeS, 0).Format("2006-01-02 15:04:05.999999999")
+}
+
+// deleteHistoricalData delete useless password history.
+// The deleted password must meet the following conditions at the same time.
+// 1. Exceeded the maximum number of saves.
+// 2. The password has exceeded the prohibition time.
+func deleteHistoricalData(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, userDetail *userInfo, maxDelRows int64, passwordReuse *passwordReuseInfo, sctx sessionctx.Context) error {
+ //never times out or no row need delete.
+ if (passwordReuse.passwordReuseInterval > math.MaxInt32) || maxDelRows == 0 {
+ return nil
+ }
+ sql := new(strings.Builder)
+ // no prohibition time.
+ if passwordReuse.passwordReuseInterval == 0 {
+ deleteTemplate := `DELETE from %n.%n WHERE User= %? AND Host= %? order by Password_timestamp ASC LIMIT `
+ deleteTemplate = deleteTemplate + strconv.FormatInt(maxDelRows, 10)
+ sqlexec.MustFormatSQL(sql, deleteTemplate, mysql.SystemDB, mysql.PasswordHistoryTable,
+ userDetail.user, strings.ToLower(userDetail.host))
+ _, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return err
+ }
+ } else {
+ beforeDate := getValidTime(sctx, passwordReuse)
+ // Deletion must satisfy 1. Exceed the prohibition time 2. Exceed the maximum number of saved records.
+ deleteTemplate := `DELETE from %n.%n WHERE User= %? AND Host= %? AND Password_timestamp < %? order by Password_timestamp ASC LIMIT `
+ deleteTemplate = deleteTemplate + strconv.FormatInt(maxDelRows, 10)
+ sql.Reset()
+ sqlexec.MustFormatSQL(sql, deleteTemplate, mysql.SystemDB, mysql.PasswordHistoryTable,
+ userDetail.user, strings.ToLower(userDetail.host), beforeDate)
+ _, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func addHistoricalData(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, userDetail *userInfo, passwordReuse *passwordReuseInfo) error {
+ if passwordReuse.passwordHistory <= 0 && passwordReuse.passwordReuseInterval <= 0 {
+ return nil
+ }
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, `INSERT INTO %n.%n (Host, User, Password) VALUES (%?, %?, %?) `, mysql.SystemDB, mysql.PasswordHistoryTable, strings.ToLower(userDetail.host), userDetail.user, userDetail.pwd)
+ _, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+// checkPasswordsMatch used to compare whether the password encrypted with mysql.AuthCachingSha2Password or mysql.AuthTiDBSM3Password is repeated.
+func checkPasswordsMatch(rows []chunk.Row, oldPwd, authPlugin string) (bool, error) {
+ for _, row := range rows {
+ if !row.IsNull(0) {
+ pwd := row.GetString(0)
+ authok, err := auth.CheckHashingPassword([]byte(pwd), oldPwd, authPlugin)
+ if err != nil {
+ logutil.BgLogger().Error("Failed to check caching_sha2_password", zap.Error(err))
+ return false, err
+ }
+ if authok {
+ return false, nil
+ }
+ }
+ }
+ return true, nil
+}
+
+func getUserPasswordNum(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, userDetail *userInfo) (deleteNum int64, err error) {
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, `SELECT count(*) FROM %n.%n WHERE User=%? AND Host=%?;`, mysql.SystemDB, mysql.PasswordHistoryTable, userDetail.user, strings.ToLower(userDetail.host))
+ recordSet, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return 0, err
+ }
+ defer func() {
+ if closeErr := recordSet.Close(); closeErr != nil {
+ err = closeErr
+ }
+ }()
+ rows, err := sqlexec.DrainRecordSet(ctx, recordSet, 3)
+ if err != nil {
+ return 0, err
+ }
+ if len(rows) != 1 {
+ err := fmt.Errorf("`%s`@`%s` is not unique, please confirm the mysql.password_history table structure", userDetail.user, strings.ToLower(userDetail.host))
+ return 0, err
+ }
+
+ return rows[0].GetInt64(0), nil
+}
+
+func fullRecordCheck(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, userDetail *userInfo, authPlugin string) (canUse bool, err error) {
+ switch authPlugin {
+ case mysql.AuthNativePassword, "":
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, `SELECT count(*) FROM %n.%n WHERE User= %? AND Host= %? AND Password = %?;`, mysql.SystemDB, mysql.PasswordHistoryTable, userDetail.user, strings.ToLower(userDetail.host), userDetail.pwd)
+ recordSet, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return false, err
+ }
+ defer func() {
+ if closeErr := recordSet.Close(); closeErr != nil {
+ err = closeErr
+ }
+ }()
+ rows, err := sqlexec.DrainRecordSet(ctx, recordSet, 3)
+ if err != nil {
+ return false, err
+ }
+ if rows[0].GetInt64(0) == 0 {
+ return true, nil
+ }
+ return false, nil
+ case mysql.AuthCachingSha2Password, mysql.AuthTiDBSM3Password:
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, `SELECT Password FROM %n.%n WHERE User= %? AND Host= %? ;`, mysql.SystemDB, mysql.PasswordHistoryTable, userDetail.user, strings.ToLower(userDetail.host))
+ recordSet, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return false, err
+ }
+ defer func() {
+ if closeErr := recordSet.Close(); closeErr != nil {
+ err = closeErr
+ }
+ }()
+ rows, err := sqlexec.DrainRecordSet(ctx, recordSet, variable.DefMaxChunkSize)
+ if err != nil {
+ return false, err
+ }
+ return checkPasswordsMatch(rows, userDetail.authString, authPlugin)
+ default:
+ return false, ErrPluginIsNotLoaded.GenWithStackByArgs(authPlugin)
+ }
+}
+
+func checkPasswordHistoryRule(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, userDetail *userInfo, passwordReuse *passwordReuseInfo, authPlugin string) (canUse bool, err error) {
+ switch authPlugin {
+ case mysql.AuthNativePassword, "":
+ sql := new(strings.Builder)
+ // Exceeded the maximum number of saved items, only check the ones within the limit.
+ checkRows := `SELECT count(*) FROM (SELECT Password FROM %n.%n WHERE User=%? AND Host=%? ORDER BY Password_timestamp DESC LIMIT `
+ checkRows = checkRows + strconv.FormatInt(passwordReuse.passwordHistory, 10)
+ checkRows = checkRows + ` ) as t where t.Password = %? `
+ sqlexec.MustFormatSQL(sql, checkRows, mysql.SystemDB, mysql.PasswordHistoryTable, userDetail.user, strings.ToLower(userDetail.host), userDetail.pwd)
+ recordSet, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return false, err
+ }
+ defer func() {
+ if closeErr := recordSet.Close(); closeErr != nil {
+ err = closeErr
+ }
+ }()
+ rows, err := sqlexec.DrainRecordSet(ctx, recordSet, 3)
+ if err != nil {
+ return false, err
+ }
+ if rows[0].GetInt64(0) != 0 {
+ return false, nil
+ }
+ return true, nil
+ case mysql.AuthCachingSha2Password, mysql.AuthTiDBSM3Password:
+ sql := new(strings.Builder)
+ checkRows := `SELECT Password FROM %n.%n WHERE User=%? AND Host=%? ORDER BY Password_timestamp DESC LIMIT `
+ checkRows = checkRows + strconv.FormatInt(passwordReuse.passwordHistory, 10)
+ sqlexec.MustFormatSQL(sql, checkRows, mysql.SystemDB, mysql.PasswordHistoryTable, userDetail.user, strings.ToLower(userDetail.host))
+ recordSet, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return false, err
+ }
+ defer func() {
+ if closeErr := recordSet.Close(); closeErr != nil {
+ err = closeErr
+ }
+ }()
+ rows, err := sqlexec.DrainRecordSet(ctx, recordSet, variable.DefMaxChunkSize)
+ if err != nil {
+ return false, err
+ }
+ return checkPasswordsMatch(rows, userDetail.authString, authPlugin)
+ default:
+ return false, ErrPluginIsNotLoaded.GenWithStackByArgs(authPlugin)
+ }
+}
+
+func checkPasswordTimeRule(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, userDetail *userInfo, passwordReuse *passwordReuseInfo,
+ sctx sessionctx.Context, authPlugin string) (canUse bool, err error) {
+ beforeDate := getValidTime(sctx, passwordReuse)
+ switch authPlugin {
+ case mysql.AuthNativePassword, "":
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, `SELECT count(*) FROM %n.%n WHERE User=%? AND Host=%? AND Password = %? AND Password_timestamp >= %?;`,
+ mysql.SystemDB, mysql.PasswordHistoryTable, userDetail.user, strings.ToLower(userDetail.host), userDetail.pwd, beforeDate)
+ recordSet, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return false, err
+ }
+ defer func() {
+ if closeErr := recordSet.Close(); closeErr != nil {
+ err = closeErr
+ }
+ }()
+ rows, err := sqlexec.DrainRecordSet(ctx, recordSet, 3)
+ if err != nil {
+ return false, err
+ }
+ if rows[0].GetInt64(0) == 0 {
+ return true, nil
+ }
+ case mysql.AuthCachingSha2Password, mysql.AuthTiDBSM3Password:
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, `SELECT Password FROM %n.%n WHERE User=%? AND Host=%? AND Password_timestamp >= %?;`, mysql.SystemDB, mysql.PasswordHistoryTable, userDetail.user, strings.ToLower(userDetail.host), beforeDate)
+ recordSet, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
+ if err != nil {
+ return false, err
+ }
+ defer func() {
+ if closeErr := recordSet.Close(); closeErr != nil {
+ err = closeErr
+ }
+ }()
+ rows, err := sqlexec.DrainRecordSet(ctx, recordSet, variable.DefMaxChunkSize)
+ if err != nil {
+ return false, err
+ }
+ return checkPasswordsMatch(rows, userDetail.authString, authPlugin)
+ default:
+ return false, ErrPluginIsNotLoaded.GenWithStackByArgs(authPlugin)
+ }
+ return false, nil
+}
+
+func passwordVerification(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, userDetail *userInfo, passwordReuse *passwordReuseInfo, sctx sessionctx.Context, authPlugin string) (bool, int64, error) {
+ passwordNum, err := getUserPasswordNum(ctx, sqlExecutor, userDetail)
+ if err != nil {
+ return false, 0, err
+ }
+
+ // the maximum number of records that can be deleted.
+ canDeleteNum := passwordNum - passwordReuse.passwordHistory + 1
+ if canDeleteNum < 0 {
+ canDeleteNum = 0
+ }
+
+ if passwordReuse.passwordHistory <= 0 && passwordReuse.passwordReuseInterval <= 0 {
+ return true, canDeleteNum, nil
+ }
+
+ // The maximum number of saves has not been exceeded.
+ // There are too many retention days, and it is impossible to time out in one's lifetime.
+ if (passwordNum <= passwordReuse.passwordHistory) || (passwordReuse.passwordReuseInterval > math.MaxInt32) {
+ passChecking, err := fullRecordCheck(ctx, sqlExecutor, userDetail, authPlugin)
+ return passChecking, canDeleteNum, err
+ }
+
+ if passwordReuse.passwordHistory > 0 {
+ passChecking, err := checkPasswordHistoryRule(ctx, sqlExecutor, userDetail, passwordReuse, authPlugin)
+ if err != nil || !passChecking {
+ return false, 0, err
+ }
+ }
+ if passwordReuse.passwordReuseInterval > 0 {
+ passChecking, err := checkPasswordTimeRule(ctx, sqlExecutor, userDetail, passwordReuse, sctx, authPlugin)
+ if err != nil || !passChecking {
+ return false, 0, err
+ }
+ }
+ return true, canDeleteNum, nil
+}
+
+func checkPasswordReusePolicy(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, userDetail *userInfo, sctx sessionctx.Context, authPlugin string) error {
+ if strings.EqualFold(authPlugin, mysql.AuthTiDBAuthToken) {
+ // AuthTiDBAuthToken is the token login method on the cloud,
+ // and the Password Reuse Policy does not take effect.
+ return nil
+ }
+ // read password reuse info from mysql.user and global variables.
+ passwdReuseInfo, err := getUserPasswordLimit(ctx, sqlExecutor, userDetail.user, userDetail.host, userDetail.pLI)
+ if err != nil {
+ return err
+ }
+ // check whether password can be used.
+ res, maxDelNum, err := passwordVerification(ctx, sqlExecutor, userDetail, passwdReuseInfo, sctx, authPlugin)
+ if err != nil {
+ return err
+ }
+ if !res {
+ return ErrExistsInHistoryPassword.GenWithStackByArgs(userDetail.user, userDetail.host)
+ }
+ err = deleteHistoricalData(ctx, sqlExecutor, userDetail, maxDelNum, passwdReuseInfo, sctx)
+ if err != nil {
+ return err
+ }
+ // insert password history.
+ err = addHistoricalData(ctx, sqlExecutor, userDetail, passwdReuseInfo)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt) error {
+ disableSandBoxMode := false
+ var err error
+ if e.ctx.InSandBoxMode() {
+ if err = e.checkSandboxMode(s.Specs); err != nil {
+ return err
+ }
+ disableSandBoxMode = true
+ }
ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnPrivilege)
if s.CurrentAuth != nil {
user := e.ctx.GetSessionVars().User
@@ -965,19 +1682,20 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt)
s.Specs = []*ast.UserSpec{spec}
}
- lockAccount := ""
- if len(s.PasswordOrLockOptions) > 0 {
- // If "ACCOUNT LOCK" or "ACCOUNT UNLOCK" appears many times,
- // the last declaration takes effect.
- for i := len(s.PasswordOrLockOptions) - 1; i >= 0; i-- {
- if s.PasswordOrLockOptions[i].Type == ast.Lock {
- lockAccount = "Y"
- break
- } else if s.PasswordOrLockOptions[i].Type == ast.Unlock {
- lockAccount = "N"
- break
- }
- }
+ plOptions := passwordOrLockOptionsInfo{
+ lockAccount: "",
+ passwordExpired: "",
+ passwordLifetime: notSpecified,
+ passwordHistory: notSpecified,
+ passwordReuseInterval: notSpecified,
+ failedLoginAttemptsChange: false,
+ passwordLockTimeChange: false,
+ passwordHistoryChange: false,
+ passwordReuseIntervalChange: false,
+ }
+ err = plOptions.loadOptions(s.PasswordOrLockOptions)
+ if err != nil {
+ return err
}
privData, err := tlsOption2GlobalPriv(s.AuthTokenOrTLSOptions)
@@ -986,6 +1704,7 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt)
}
failedUsers := make([]string, 0, len(s.Specs))
+ needRollback := false
checker := privilege.GetPrivilegeManager(e.ctx)
if checker == nil {
return errors.New("could not load privilege checker")
@@ -1003,6 +1722,24 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt)
}
}
+ sysSession, err := e.getSysSession()
+ defer clearSysSession(ctx, sysSession)
+ if err != nil {
+ return err
+ }
+ sqlExecutor := sysSession.(sqlexec.SQLExecutor)
+ // session isolation level changed to READ-COMMITTED.
+ // When tidb is at the RR isolation level, executing `begin` will obtain a consistent state.
+ // When operating the same user concurrently, it may happen that historical versions are read.
+ // In order to avoid this risk, change the isolation level to RC.
+ _, err = sqlExecutor.ExecuteInternal(ctx, "set tx_isolation = 'READ-COMMITTED'")
+ if err != nil {
+ return err
+ }
+ if _, err := sqlExecutor.ExecuteInternal(ctx, "BEGIN PESSIMISTIC"); err != nil {
+ return err
+ }
+
for _, spec := range s.Specs {
user := e.ctx.GetSessionVars().User
if spec.User.CurrentUser || ((user != nil) && (user.Username == spec.User.Username) && (user.AuthHostname == spec.User.Hostname)) {
@@ -1036,7 +1773,7 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt)
}
}
- exists, err := userExists(ctx, e.ctx, spec.User.Username, spec.User.Hostname)
+ exists, err := userExistsInternal(ctx, sqlExecutor, spec.User.Username, spec.User.Hostname)
if err != nil {
return err
}
@@ -1057,25 +1794,23 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt)
RequireAuthTokenOptions
)
authTokenOptionHandler := NoNeedAuthTokenOptions
- if currentAuthPlugin, err := e.userAuthPlugin(spec.User.Username, spec.User.Hostname); err != nil {
+ currentAuthPlugin, err := privilege.GetPrivilegeManager(e.ctx).GetAuthPlugin(spec.User.Username, spec.User.Hostname)
+ if err != nil {
return err
- } else if currentAuthPlugin == mysql.AuthTiDBAuthToken {
+ }
+ if currentAuthPlugin == mysql.AuthTiDBAuthToken {
authTokenOptionHandler = OptionalAuthTokenOptions
}
- exec := e.ctx.(sqlexec.RestrictedSQLExecutor)
type alterField struct {
expr string
- value string
+ value any
}
var fields []alterField
if spec.AuthOpt != nil {
+ fields = append(fields, alterField{"password_last_changed=current_timestamp()", nil})
if spec.AuthOpt.AuthPlugin == "" {
- authplugin, err := e.userAuthPlugin(spec.User.Username, spec.User.Hostname)
- if err != nil {
- return err
- }
- spec.AuthOpt.AuthPlugin = authplugin
+ spec.AuthOpt.AuthPlugin = currentAuthPlugin
}
switch spec.AuthOpt.AuthPlugin {
case mysql.AuthNativePassword, mysql.AuthCachingSha2Password, mysql.AuthTiDBSM3Password, mysql.AuthSocket, "":
@@ -1087,28 +1822,122 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt)
default:
return ErrPluginIsNotLoaded.GenWithStackByArgs(spec.AuthOpt.AuthPlugin)
}
+ // changing the auth method prunes history.
+ if spec.AuthOpt.AuthPlugin != currentAuthPlugin {
+ // delete password history from mysql.password_history.
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, `DELETE FROM %n.%n WHERE Host = %? and User = %?;`, mysql.SystemDB, mysql.PasswordHistoryTable, spec.User.Hostname, spec.User.Username)
+ if _, err := sqlExecutor.ExecuteInternal(ctx, sql.String()); err != nil {
+ failedUsers = append(failedUsers, spec.User.String())
+ needRollback = true
+ break
+ }
+ }
+ if e.isValidatePasswordEnabled() && e.authUsingCleartextPwd(spec.AuthOpt, spec.AuthOpt.AuthPlugin) {
+ if err := pwdValidator.ValidatePassword(e.ctx.GetSessionVars(), spec.AuthOpt.AuthString); err != nil {
+ return err
+ }
+ }
pwd, ok := spec.EncodedPassword()
if !ok {
return errors.Trace(ErrPasswordFormat)
}
- fields = append(fields,
- alterField{"authentication_string=%?", pwd},
- alterField{"plugin=%?", spec.AuthOpt.AuthPlugin},
- )
+ // for Support Password Reuse Policy.
+ // The empty password does not count in the password history and is subject to reuse at any time.
+ // https://dev.mysql.com/doc/refman/8.0/en/password-management.html#password-reuse-policy
+ if len(pwd) != 0 {
+ userDetail := &userInfo{
+ host: spec.User.Hostname,
+ user: spec.User.Username,
+ pLI: &plOptions,
+ pwd: pwd,
+ authString: spec.AuthOpt.AuthString,
+ }
+ err := checkPasswordReusePolicy(ctx, sqlExecutor, userDetail, e.ctx, spec.AuthOpt.AuthPlugin)
+ if err != nil {
+ return err
+ }
+ }
+ fields = append(fields, alterField{"authentication_string=%?", pwd})
+ if spec.AuthOpt.AuthPlugin != "" {
+ fields = append(fields, alterField{"plugin=%?", spec.AuthOpt.AuthPlugin})
+ }
+ if spec.AuthOpt.ByAuthString || spec.AuthOpt.ByHashString {
+ if plOptions.passwordExpired == "" {
+ plOptions.passwordExpired = "N"
+ }
+ }
+ }
+
+ if len(plOptions.lockAccount) != 0 {
+ fields = append(fields, alterField{"account_locked=%?", plOptions.lockAccount})
+ }
+
+ // support alter Password_reuse_history and Password_reuse_time.
+ if plOptions.passwordHistoryChange {
+ if plOptions.passwordHistory == notSpecified {
+ fields = append(fields, alterField{"Password_reuse_history = NULL ", ""})
+ } else {
+ fields = append(fields, alterField{"Password_reuse_history = %? ", strconv.FormatInt(plOptions.passwordHistory, 10)})
+ }
+ }
+ if plOptions.passwordReuseIntervalChange {
+ if plOptions.passwordReuseInterval == notSpecified {
+ fields = append(fields, alterField{"Password_reuse_time = NULL ", ""})
+ } else {
+ fields = append(fields, alterField{"Password_reuse_time = %? ", strconv.FormatInt(plOptions.passwordReuseInterval, 10)})
+ }
}
- if len(lockAccount) != 0 {
- fields = append(fields, alterField{"account_locked=%?", lockAccount})
+ passwordLockingInfo, err := readPasswordLockingInfo(ctx, sqlExecutor, spec.User.Username, spec.User.Hostname, &plOptions)
+ if err != nil {
+ return err
}
+ passwordLockingStr := alterUserFailedLoginJSON(passwordLockingInfo, plOptions.lockAccount)
+ if len(plOptions.passwordExpired) != 0 {
+ if len(spec.User.Username) == 0 && plOptions.passwordExpired == "Y" {
+ return ErrPasswordExpireAnonymousUser.GenWithStackByArgs()
+ }
+ fields = append(fields, alterField{"password_expired=%?", plOptions.passwordExpired})
+ }
+ if plOptions.passwordLifetime != notSpecified {
+ fields = append(fields, alterField{"password_lifetime=%?", plOptions.passwordLifetime})
+ }
+
+ var newAttributes []string
if s.CommentOrAttributeOption != nil {
- newAttributesStr := ""
if s.CommentOrAttributeOption.Type == ast.UserCommentType {
- newAttributesStr = fmt.Sprintf(`{"metadata": {"comment": "%s"}}`, s.CommentOrAttributeOption.Value)
+ newAttributes = append(newAttributes, fmt.Sprintf(`"metadata": {"comment": "%s"}`, s.CommentOrAttributeOption.Value))
} else {
- newAttributesStr = fmt.Sprintf(`{"metadata": %s}`, s.CommentOrAttributeOption.Value)
+ newAttributes = append(newAttributes, fmt.Sprintf(`"metadata": %s`, s.CommentOrAttributeOption.Value))
}
- fields = append(fields, alterField{"user_attributes=json_merge_patch(user_attributes, %?)", newAttributesStr})
+ }
+ if s.ResourceGroupNameOption != nil {
+ if !variable.EnableResourceControl.Load() {
+ return infoschema.ErrResourceGroupSupportDisabled
+ }
+
+ // check if specified resource group exists
+ resourceGroupName := strings.ToLower(s.ResourceGroupNameOption.Value)
+ if resourceGroupName != "default" && s.ResourceGroupNameOption.Value != "" {
+ _, exists := e.is.ResourceGroupByName(model.NewCIStr(resourceGroupName))
+ if !exists {
+ return infoschema.ErrResourceGroupNotExists.GenWithStackByArgs(resourceGroupName)
+ }
+ }
+
+ newAttributes = append(newAttributes, fmt.Sprintf(`"resource_group": "%s"`, resourceGroupName))
+ }
+ if passwordLockingStr != "" {
+ newAttributes = append(newAttributes, passwordLockingStr)
+ }
+ if length := len(newAttributes); length > 0 {
+ if length > 1 || passwordLockingStr == "" {
+ passwordLockingInfo.containsNoOthers = false
+ }
+ newAttributesStr := fmt.Sprintf("{%s}", strings.Join(newAttributes, ","))
+ fields = append(fields, alterField{"user_attributes=json_merge_patch(coalesce(user_attributes, '{}'), %?)", newAttributesStr})
}
switch authTokenOptionHandler {
@@ -1144,31 +1973,35 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt)
}
}
sqlexec.MustFormatSQL(sql, " WHERE Host=%? and User=%?;", spec.User.Hostname, spec.User.Username)
- _, _, err := exec.ExecRestrictedSQL(ctx, nil, sql.String())
+ _, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
if err != nil {
failedUsers = append(failedUsers, spec.User.String())
+ needRollback = true
continue
}
}
+ // Remove useless Password_locking from User_attributes.
+ err = deletePasswordLockingAttribute(ctx, sqlExecutor, spec.User.Username, spec.User.Hostname, passwordLockingInfo)
+ if err != nil {
+ failedUsers = append(failedUsers, spec.User.String())
+ needRollback = true
+ continue
+ }
+
if len(privData) > 0 {
- _, _, err := exec.ExecRestrictedSQL(ctx, nil, "INSERT INTO %n.%n (Host, User, Priv) VALUES (%?,%?,%?) ON DUPLICATE KEY UPDATE Priv = values(Priv)", mysql.SystemDB, mysql.GlobalPrivTable, spec.User.Hostname, spec.User.Username, string(hack.String(privData)))
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, "INSERT INTO %n.%n (Host, User, Priv) VALUES (%?,%?,%?) ON DUPLICATE KEY UPDATE Priv = values(Priv)", mysql.SystemDB, mysql.GlobalPrivTable, spec.User.Hostname, spec.User.Username, string(hack.String(privData)))
+ _, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
if err != nil {
failedUsers = append(failedUsers, spec.User.String())
+ needRollback = true
}
}
}
if len(failedUsers) > 0 {
- // Commit the transaction even if we returns error
- txn, err := e.ctx.Txn(true)
- if err != nil {
- return err
- }
- err = txn.Commit(tikvutil.SetSessionID(context.TODO(), e.ctx.GetSessionVars().ConnectionID))
- if err != nil {
- return err
- }
- if !s.IfExists {
+ // Compatible with MySQL 8.0, `ALTER USER` realizes atomic operation.
+ if !s.IfExists || needRollback {
return ErrCannotUser.GenWithStackByArgs("ALTER USER", strings.Join(failedUsers, ","))
}
for _, user := range failedUsers {
@@ -1176,7 +2009,30 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt)
e.ctx.GetSessionVars().StmtCtx.AppendNote(err)
}
}
- return domain.GetDomain(e.ctx).NotifyUpdatePrivilege()
+ if _, err := sqlExecutor.ExecuteInternal(ctx, "commit"); err != nil {
+ return err
+ }
+ if err = domain.GetDomain(e.ctx).NotifyUpdatePrivilege(); err != nil {
+ return err
+ }
+ if disableSandBoxMode {
+ e.ctx.DisableSandBoxMode()
+ }
+ return nil
+}
+
+func (e *SimpleExec) checkSandboxMode(specs []*ast.UserSpec) error {
+ for _, spec := range specs {
+ if spec.AuthOpt == nil {
+ continue
+ }
+ if spec.AuthOpt.ByAuthString || spec.AuthOpt.ByHashString {
+ if spec.User.CurrentUser || e.ctx.GetSessionVars().User.Username == spec.User.Username {
+ return nil
+ }
+ }
+ }
+ return errMustChangePassword.GenWithStackByArgs()
}
func (e *SimpleExec) executeGrantRole(ctx context.Context, s *ast.GrantRoleStmt) error {
@@ -1246,10 +2102,9 @@ func (e *SimpleExec) executeRenameUser(s *ast.RenameUserStmt) error {
}
sqlExecutor := sysSession.(sqlexec.SQLExecutor)
- if _, err := sqlExecutor.ExecuteInternal(ctx, "begin"); err != nil {
+ if _, err := sqlExecutor.ExecuteInternal(ctx, "BEGIN PESSIMISTIC"); err != nil {
return err
}
-
for _, userToUser := range s.UserToUsers {
oldUser, newUser := userToUser.OldUser, userToUser.NewUser
if len(newUser.Username) > auth.UserNameMaxLength {
@@ -1322,6 +2177,12 @@ func (e *SimpleExec) executeRenameUser(s *ast.RenameUserStmt) error {
break
}
+ // rename passwordhistory from PasswordHistoryTable.
+ if err = renameUserHostInSystemTable(sqlExecutor, mysql.PasswordHistoryTable, "USER", "HOST", userToUser); err != nil {
+ failedUser = oldUser.String() + " TO " + newUser.String() + " " + mysql.PasswordHistoryTable + " error"
+ break
+ }
+
// rename relationship from mysql.global_grants
// TODO: add global_grants into the parser
// TODO: need update columns_priv once we implement columns_priv functionality.
@@ -1426,6 +2287,14 @@ func (e *SimpleExec) executeDropUser(ctx context.Context, s *ast.DropUserStmt) e
break
}
+ // delete password history from mysql.password_history.
+ sql.Reset()
+ sqlexec.MustFormatSQL(sql, `DELETE FROM %n.%n WHERE Host = %? and User = %?;`, mysql.SystemDB, mysql.PasswordHistoryTable, strings.ToLower(user.Hostname), user.Username)
+ if _, err = sqlExecutor.ExecuteInternal(internalCtx, sql.String()); err != nil {
+ failedUsers = append(failedUsers, user.String())
+ break
+ }
+
// delete privileges from mysql.global_priv
sql.Reset()
sqlexec.MustFormatSQL(sql, `DELETE FROM %n.%n WHERE Host = %? and User = %?;`, mysql.SystemDB, mysql.GlobalPrivTable, user.Hostname, user.Username)
@@ -1546,7 +2415,7 @@ func userExists(ctx context.Context, sctx sessionctx.Context, name string, host
// use the same internal executor to read within the same transaction, otherwise same as userExists
func userExistsInternal(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, name string, host string) (bool, error) {
sql := new(strings.Builder)
- sqlexec.MustFormatSQL(sql, `SELECT * FROM %n.%n WHERE User=%? AND Host=%?;`, mysql.SystemDB, mysql.UserTable, name, strings.ToLower(host))
+ sqlexec.MustFormatSQL(sql, `SELECT * FROM %n.%n WHERE User=%? AND Host=%? FOR UPDATE;`, mysql.SystemDB, mysql.UserTable, name, strings.ToLower(host))
recordSet, err := sqlExecutor.ExecuteInternal(ctx, sql.String())
if err != nil {
return false, err
@@ -1564,18 +2433,29 @@ func userExistsInternal(ctx context.Context, sqlExecutor sqlexec.SQLExecutor, na
return rows > 0, err
}
-func (e *SimpleExec) userAuthPlugin(name string, host string) (string, error) {
- pm := privilege.GetPrivilegeManager(e.ctx)
- authplugin, err := pm.GetAuthPlugin(name, host)
+func (e *SimpleExec) executeSetPwd(ctx context.Context, s *ast.SetPwdStmt) error {
+ ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnPrivilege)
+ sysSession, err := e.getSysSession()
+ defer clearSysSession(ctx, sysSession)
if err != nil {
- return "", err
+ return err
+ }
+
+ sqlExecutor := sysSession.(sqlexec.SQLExecutor)
+ // session isolation level changed to READ-COMMITTED.
+ // When tidb is at the RR isolation level, executing `begin` will obtain a consistent state.
+ // When operating the same user concurrently, it may happen that historical versions are read.
+ // In order to avoid this risk, change the isolation level to RC.
+ _, err = sqlExecutor.ExecuteInternal(ctx, "set tx_isolation = 'READ-COMMITTED'")
+ if err != nil {
+ return err
+ }
+ if _, err := sqlExecutor.ExecuteInternal(ctx, "BEGIN PESSIMISTIC"); err != nil {
+ return err
}
- return authplugin, nil
-}
-func (e *SimpleExec) executeSetPwd(ctx context.Context, s *ast.SetPwdStmt) error {
- ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnPrivilege)
var u, h string
+ disableSandboxMode := false
if s.User == nil || s.User.CurrentUser {
if e.ctx.GetSessionVars().User == nil {
return errors.New("Session error is empty")
@@ -1591,18 +2471,31 @@ func (e *SimpleExec) executeSetPwd(ctx context.Context, s *ast.SetPwdStmt) error
u = s.User.Username
h = s.User.Hostname
}
- exists, err := userExists(ctx, e.ctx, u, h)
+ exists, err := userExistsInternal(ctx, sqlExecutor, u, h)
if err != nil {
return err
}
if !exists {
return errors.Trace(ErrPasswordNoMatch)
}
+ if e.ctx.InSandBoxMode() {
+ if s.User == nil || s.User.CurrentUser ||
+ e.ctx.GetSessionVars().User.AuthUsername == u && e.ctx.GetSessionVars().User.AuthHostname == strings.ToLower(h) {
+ disableSandboxMode = true
+ } else {
+ return errMustChangePassword.GenWithStackByArgs()
+ }
+ }
- authplugin, err := e.userAuthPlugin(u, h)
+ authplugin, err := privilege.GetPrivilegeManager(e.ctx).GetAuthPlugin(u, h)
if err != nil {
return err
}
+ if e.isValidatePasswordEnabled() {
+ if err := pwdValidator.ValidatePassword(e.ctx.GetSessionVars(), s.Password); err != nil {
+ return err
+ }
+ }
var pwd string
switch authplugin {
case mysql.AuthCachingSha2Password, mysql.AuthTiDBSM3Password:
@@ -1614,13 +2507,47 @@ func (e *SimpleExec) executeSetPwd(ctx context.Context, s *ast.SetPwdStmt) error
pwd = auth.EncodePassword(s.Password)
}
+ // for Support Password Reuse Policy.
+ plOptions := &passwordOrLockOptionsInfo{
+ lockAccount: "",
+ passwordHistory: notSpecified,
+ passwordReuseInterval: notSpecified,
+ passwordHistoryChange: false,
+ passwordReuseIntervalChange: false,
+ }
+ // The empty password does not count in the password history and is subject to reuse at any time.
+ // https://dev.mysql.com/doc/refman/8.0/en/password-management.html#password-reuse-policy
+ if len(pwd) != 0 {
+ userDetail := &userInfo{
+ host: h,
+ user: u,
+ pLI: plOptions,
+ pwd: pwd,
+ authString: s.Password,
+ }
+ err := checkPasswordReusePolicy(ctx, sqlExecutor, userDetail, e.ctx, authplugin)
+ if err != nil {
+ return err
+ }
+ }
// update mysql.user
- exec := e.ctx.(sqlexec.RestrictedSQLExecutor)
- _, _, err = exec.ExecRestrictedSQL(ctx, nil, `UPDATE %n.%n SET authentication_string=%? WHERE User=%? AND Host=%?;`, mysql.SystemDB, mysql.UserTable, pwd, u, strings.ToLower(h))
+ sql := new(strings.Builder)
+ sqlexec.MustFormatSQL(sql, `UPDATE %n.%n SET authentication_string=%?,password_expired='N',password_last_changed=current_timestamp() WHERE User=%? AND Host=%?;`, mysql.SystemDB, mysql.UserTable, pwd, u, strings.ToLower(h))
+ _, err = sqlExecutor.ExecuteInternal(ctx, sql.String())
if err != nil {
return err
}
- return domain.GetDomain(e.ctx).NotifyUpdatePrivilege()
+ if _, err := sqlExecutor.ExecuteInternal(ctx, "commit"); err != nil {
+ return err
+ }
+ err = domain.GetDomain(e.ctx).NotifyUpdatePrivilege()
+ if err != nil {
+ return err
+ }
+ if disableSandboxMode {
+ e.ctx.DisableSandBoxMode()
+ }
+ return nil
}
func (e *SimpleExec) executeKillStmt(ctx context.Context, s *ast.KillStmt) error {
diff --git a/executor/simple_test.go b/executor/simple_test.go
index 8b284fb9b42e5..0ededa68e2253 100644
--- a/executor/simple_test.go
+++ b/executor/simple_test.go
@@ -21,13 +21,13 @@ import (
"testing"
"github.com/pingcap/tidb/config"
+ "github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/auth"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/server"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/util"
"github.com/stretchr/testify/require"
- tikvutil "github.com/tikv/client-go/v2/util"
)
func TestKillStmt(t *testing.T) {
@@ -86,7 +86,7 @@ func TestKillStmt(t *testing.T) {
func TestUserAttributes(t *testing.T) {
store, _ := testkit.CreateMockStoreAndDomain(t)
rootTK := testkit.NewTestKit(t, store)
- ctx := context.WithValue(context.Background(), tikvutil.RequestSourceKey, tikvutil.RequestSource{RequestSourceInternal: true})
+ ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnPrivilege)
// https://dev.mysql.com/doc/refman/8.0/en/create-user.html#create-user-comments-attributes
rootTK.MustExec(`CREATE USER testuser COMMENT '1234'`)
@@ -96,7 +96,7 @@ func TestUserAttributes(t *testing.T) {
require.Error(t, err)
rootTK.MustQuery(`SELECT user_attributes FROM mysql.user WHERE user = 'testuser'`).Check(testkit.Rows(`{"metadata": {"comment": "1234"}}`))
rootTK.MustQuery(`SELECT user_attributes FROM mysql.user WHERE user = 'testuser1'`).Check(testkit.Rows(`{"metadata": {"age": 19, "name": "Tom"}}`))
- rootTK.MustQuery(`SELECT user_attributes FROM mysql.user WHERE user = 'testuser2'`).Check(testkit.Rows(``))
+ rootTK.MustQuery(`SELECT user_attributes FROM mysql.user WHERE user = 'testuser2'`).Check(testkit.Rows(`{}`))
rootTK.MustQueryWithContext(ctx, `SELECT attribute FROM information_schema.user_attributes WHERE user = 'testuser'`).Check(testkit.Rows(`{"comment": "1234"}`))
rootTK.MustQueryWithContext(ctx, `SELECT attribute FROM information_schema.user_attributes WHERE user = 'testuser1'`).Check(testkit.Rows(`{"age": 19, "name": "Tom"}`))
rootTK.MustQueryWithContext(ctx, `SELECT attribute->>"$.age" AS age, attribute->>"$.name" AS name FROM information_schema.user_attributes WHERE user = 'testuser1'`).Check(testkit.Rows(`19 Tom`))
@@ -105,7 +105,9 @@ func TestUserAttributes(t *testing.T) {
// https://dev.mysql.com/doc/refman/8.0/en/alter-user.html#alter-user-comments-attributes
rootTK.MustExec(`ALTER USER testuser1 ATTRIBUTE '{"age": 20, "sex": "male"}'`)
rootTK.MustQueryWithContext(ctx, `SELECT attribute FROM information_schema.user_attributes WHERE user = 'testuser1'`).Check(testkit.Rows(`{"age": 20, "name": "Tom", "sex": "male"}`))
- rootTK.MustExec(`ALTER USER testuser1 ATTRIBUTE '{"sex": null}'`)
+ rootTK.MustExec(`ALTER USER testuser1 ATTRIBUTE '{"hobby": "soccer"}'`)
+ rootTK.MustQueryWithContext(ctx, `SELECT attribute FROM information_schema.user_attributes WHERE user = 'testuser1'`).Check(testkit.Rows(`{"age": 20, "hobby": "soccer", "name": "Tom", "sex": "male"}`))
+ rootTK.MustExec(`ALTER USER testuser1 ATTRIBUTE '{"sex": null, "hobby": null}'`)
rootTK.MustQueryWithContext(ctx, `SELECT attribute FROM information_schema.user_attributes WHERE user = 'testuser1'`).Check(testkit.Rows(`{"age": 20, "name": "Tom"}`))
rootTK.MustExec(`ALTER USER testuser1 COMMENT '5678'`)
rootTK.MustQueryWithContext(ctx, `SELECT attribute FROM information_schema.user_attributes WHERE user = 'testuser1'`).Check(testkit.Rows(`{"age": 20, "comment": "5678", "name": "Tom"}`))
@@ -121,4 +123,13 @@ func TestUserAttributes(t *testing.T) {
tk.MustQueryWithContext(ctx, `SELECT user, host, attribute FROM information_schema.user_attributes ORDER BY user`).Check(
testkit.Rows("root % ", "testuser % {\"comment\": \"1234\"}", "testuser1 % {\"age\": 20, \"name\": \"Tom\"}", "testuser2 % "))
tk.MustGetErrCode(`SELECT user, host, user_attributes FROM mysql.user ORDER BY user`, mysql.ErrTableaccessDenied)
+
+ // https://github.com/pingcap/tidb/issues/39207
+ rootTK.MustExec("create user usr1@'%' identified by 'passord'")
+ rootTK.MustExec("alter user usr1 comment 'comment1'")
+ rootTK.MustQuery("select user_attributes from mysql.user where user = 'usr1'").Check(testkit.Rows(`{"metadata": {"comment": "comment1"}}`))
+ rootTK.MustExec("set global tidb_enable_resource_control = 'on'")
+ rootTK.MustExec("CREATE RESOURCE GROUP rg1 ru_per_sec = 100")
+ rootTK.MustExec("alter user usr1 resource group rg1")
+ rootTK.MustQuery("select user_attributes from mysql.user where user = 'usr1'").Check(testkit.Rows(`{"metadata": {"comment": "comment1"}, "resource_group": "rg1"}`))
}
diff --git a/executor/simpletest/BUILD.bazel b/executor/simpletest/BUILD.bazel
index 0f8c010c0bc99..2da6aa029e3cd 100644
--- a/executor/simpletest/BUILD.bazel
+++ b/executor/simpletest/BUILD.bazel
@@ -5,6 +5,7 @@ go_test(
timeout = "short",
srcs = [
"main_test.go",
+ "password_management_test.go",
"simple_test.go",
],
flaky = True,
@@ -12,17 +13,23 @@ go_test(
shard_count = 30,
deps = [
"//config",
+ "//domain",
+ "//errno",
"//executor",
+ "//kv",
"//parser/auth",
"//parser/model",
"//parser/mysql",
"//parser/terror",
"//planner/core",
+ "//privilege/privileges",
"//session",
"//sessionctx",
+ "//sessionctx/variable",
"//statistics/handle",
"//store/mockstore",
"//testkit",
+ "//util/sqlexec",
"@com_github_pingcap_errors//:errors",
"@com_github_stretchr_testify//require",
"@io_opencensus_go//stats/view",
diff --git a/executor/simpletest/password_management_test.go b/executor/simpletest/password_management_test.go
new file mode 100644
index 0000000000000..1e79662b5d5a3
--- /dev/null
+++ b/executor/simpletest/password_management_test.go
@@ -0,0 +1,1363 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package simpletest
+
+import (
+ "bytes"
+ "crypto/sha1"
+ "encoding/json"
+ "fmt"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/pingcap/tidb/domain"
+ "github.com/pingcap/tidb/errno"
+ "github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/parser/auth"
+ "github.com/pingcap/tidb/parser/mysql"
+ "github.com/pingcap/tidb/privilege/privileges"
+ "github.com/pingcap/tidb/sessionctx/variable"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/pingcap/tidb/util/sqlexec"
+ "github.com/stretchr/testify/require"
+)
+
+func TestValidatePassword(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ subtk := testkit.NewTestKit(t, store)
+ err := tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil)
+ require.NoError(t, err)
+ tk.MustExec("CREATE USER ''@'localhost'")
+ tk.MustExec("GRANT ALL PRIVILEGES ON mysql.* TO ''@'localhost';")
+ err = subtk.Session().Auth(&auth.UserIdentity{Hostname: "localhost"}, nil, nil)
+ require.NoError(t, err)
+
+ authPlugins := []string{mysql.AuthNativePassword, mysql.AuthCachingSha2Password, mysql.AuthTiDBSM3Password}
+ tk.MustQuery("SELECT @@global.validate_password.enable").Check(testkit.Rows("0"))
+ tk.MustExec("SET GLOBAL validate_password.enable = 1")
+ tk.MustQuery("SELECT @@global.validate_password.enable").Check(testkit.Rows("1"))
+
+ for _, authPlugin := range authPlugins {
+ tk.MustExec("DROP USER IF EXISTS testuser")
+ tk.MustExec(fmt.Sprintf("CREATE USER testuser IDENTIFIED WITH %s BY '!Abc12345678'", authPlugin))
+
+ tk.MustExec("SET GLOBAL validate_password.policy = 'LOW'")
+ // check user name
+ tk.MustQuery("SELECT @@global.validate_password.check_user_name").Check(testkit.Rows("1"))
+ tk.MustContainErrMsg("ALTER USER testuser IDENTIFIED BY '!Abcdroot1234'", "Password Contains User Name")
+ tk.MustContainErrMsg("ALTER USER testuser IDENTIFIED BY '!Abcdtoor1234'", "Password Contains Reversed User Name")
+ tk.MustExec("SET PASSWORD FOR 'testuser' = 'testuser'") // password the same as the user name, but run by root
+ tk.MustExec("ALTER USER testuser IDENTIFIED BY 'testuser'")
+ tk.MustExec("SET GLOBAL validate_password.check_user_name = 0")
+ tk.MustExec("ALTER USER testuser IDENTIFIED BY '!Abcdroot1234'")
+ tk.MustExec("ALTER USER testuser IDENTIFIED BY '!Abcdtoor1234'")
+ tk.MustExec("SET GLOBAL validate_password.check_user_name = 1")
+
+ // LOW: Length
+ tk.MustExec("SET GLOBAL validate_password.length = 8")
+ tk.MustQuery("SELECT @@global.validate_password.length").Check(testkit.Rows("8"))
+ tk.MustContainErrMsg("ALTER USER testuser IDENTIFIED BY '1234567'", "Require Password Length: 8")
+ tk.MustExec("SET GLOBAL validate_password.length = 12")
+ tk.MustContainErrMsg("ALTER USER testuser IDENTIFIED BY '!Abcdefg123'", "Require Password Length: 12")
+ tk.MustExec("ALTER USER testuser IDENTIFIED BY '!Abcdefg1234'")
+ tk.MustExec("SET GLOBAL validate_password.length = 8")
+
+ // MEDIUM: Length; numeric, lowercase/uppercase, and special characters
+ tk.MustExec("SET GLOBAL validate_password.policy = 'MEDIUM'")
+ tk.MustExec("ALTER USER testuser IDENTIFIED BY '!Abc1234567'")
+ tk.MustContainErrMsg("ALTER USER testuser IDENTIFIED BY '!ABC1234567'", "Require Password Lowercase Count: 1")
+ tk.MustContainErrMsg("ALTER USER testuser IDENTIFIED BY '!abc1234567'", "Require Password Uppercase Count: 1")
+ tk.MustContainErrMsg("ALTER USER testuser IDENTIFIED BY '!ABCDabcd'", "Require Password Digit Count: 1")
+ tk.MustContainErrMsg("ALTER USER testuser IDENTIFIED BY 'Abc1234567'", "Require Password Non-alphanumeric Count: 1")
+ tk.MustExec("SET GLOBAL validate_password.special_char_count = 0")
+ tk.MustExec("ALTER USER testuser IDENTIFIED BY 'Abc1234567'")
+ tk.MustExec("SET GLOBAL validate_password.special_char_count = 1")
+ tk.MustExec("SET GLOBAL validate_password.length = 3")
+ tk.MustQuery("SELECT @@GLOBAL.validate_password.length").Check(testkit.Rows("4"))
+
+ // STRONG: Length; numeric, lowercase/uppercase, and special characters; dictionary file
+ tk.MustExec("SET GLOBAL validate_password.policy = 'STRONG'")
+ tk.MustExec("ALTER USER testuser IDENTIFIED BY '!Abc1234567'")
+ tk.MustExec(fmt.Sprintf("SET GLOBAL validate_password.dictionary = '%s'", "1234;5678"))
+ tk.MustExec("ALTER USER testuser IDENTIFIED BY '!Abc123567'")
+ tk.MustExec("ALTER USER testuser IDENTIFIED BY '!Abc43218765'")
+ tk.MustContainErrMsg("ALTER USER testuser IDENTIFIED BY '!Abc1234567'", "Password contains word in the dictionary")
+ tk.MustExec("SET GLOBAL validate_password.dictionary = ''")
+ tk.MustExec("ALTER USER testuser IDENTIFIED BY '!Abc1234567'")
+
+ // "IDENTIFIED AS 'xxx'" is not affected by validation
+ tk.MustExec(fmt.Sprintf("ALTER USER testuser IDENTIFIED WITH '%s' AS ''", authPlugin))
+ }
+ tk.MustContainErrMsg("CREATE USER 'testuser1'@'localhost'", "Your password does not satisfy the current policy requirements")
+ tk.MustContainErrMsg("CREATE USER 'testuser1'@'localhost' IDENTIFIED WITH 'caching_sha2_password'", "Your password does not satisfy the current policy requirements")
+ tk.MustContainErrMsg("CREATE USER 'testuser1'@'localhost' IDENTIFIED WITH 'caching_sha2_password' AS ''", "Your password does not satisfy the current policy requirements")
+
+ // if the username is '', all password can pass the check_user_name
+ subtk.MustQuery("SELECT user(), current_user()").Check(testkit.Rows("@localhost @localhost"))
+ subtk.MustQuery("SELECT @@global.validate_password.check_user_name").Check(testkit.Rows("1"))
+ subtk.MustQuery("SELECT @@global.validate_password.enable").Check(testkit.Rows("1"))
+ tk.MustExec("SET GLOBAL validate_password.number_count = 0")
+ tk.MustExec("SET GLOBAL validate_password.special_char_count = 0")
+ tk.MustExec("SET GLOBAL validate_password.mixed_case_count = 0")
+ tk.MustExec("SET GLOBAL validate_password.length = 0")
+ subtk.MustExec("ALTER USER ''@'localhost' IDENTIFIED BY ''")
+ subtk.MustExec("ALTER USER ''@'localhost' IDENTIFIED BY 'abcd'")
+
+ // CREATE ROLE is not affected by password validation
+ tk.MustExec("SET GLOBAL validate_password.enable = 1")
+ tk.MustExec("SET GLOBAL validate_password.number_count = default")
+ tk.MustExec("SET GLOBAL validate_password.special_char_count = default")
+ tk.MustExec("SET GLOBAL validate_password.mixed_case_count = default")
+ tk.MustExec("SET GLOBAL validate_password.length = default")
+ tk.MustExec("CREATE ROLE role1")
+}
+
+func expectedPasswordExpiration(t *testing.T, tk *testkit.TestKit, testuser, expired string, lifetime string) {
+ res := tk.MustQuery(fmt.Sprintf("SELECT password_expired, password_last_changed, password_lifetime FROM mysql.user WHERE user = '%s'", testuser))
+ rows := res.Rows()
+ require.NotEmpty(t, rows)
+ row := rows[0]
+ require.Equal(t, 3, len(row))
+ require.Equal(t, expired, row[0].(string), testuser)
+ require.True(t, len(row[1].(string)) > 0, testuser)
+ require.Equal(t, lifetime, row[2].(string), testuser)
+}
+
+func TestPasswordExpiration(t *testing.T) {
+ store, _ := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+
+ // CREATE USER
+ tk.MustExec(`CREATE USER testuser`)
+ expectedPasswordExpiration(t, tk, "testuser", "N", "")
+ tk.MustExec(`CREATE USER testuser1 PASSWORD EXPIRE`)
+ expectedPasswordExpiration(t, tk, "testuser1", "Y", "")
+ tk.MustExec(`CREATE USER testuser2 PASSWORD EXPIRE DEFAULT`)
+ expectedPasswordExpiration(t, tk, "testuser2", "N", "")
+ tk.MustExec(`CREATE USER testuser3 PASSWORD EXPIRE NEVER`)
+ expectedPasswordExpiration(t, tk, "testuser3", "N", "0")
+ tk.MustExec(`CREATE USER testuser4 PASSWORD EXPIRE INTERVAL 3 DAY`)
+ expectedPasswordExpiration(t, tk, "testuser4", "N", "3")
+ tk.MustExec(`CREATE ROLE role1`)
+ expectedPasswordExpiration(t, tk, "role1", "Y", "")
+
+ // ALTER USER
+ testcases := []struct {
+ user string
+ expired string
+ }{
+ {"testuser", "N"},
+ {"testuser1", "Y"},
+ {"testuser2", "N"},
+ {"testuser3", "N"},
+ {"testuser4", "N"},
+ {"role1", "Y"},
+ }
+ for _, testcase := range testcases {
+ tk.MustExec(fmt.Sprintf("ALTER USER %s PASSWORD EXPIRE NEVER", testcase.user))
+ expectedPasswordExpiration(t, tk, testcase.user, testcase.expired, "0")
+ tk.MustExec(fmt.Sprintf("ALTER USER %s PASSWORD EXPIRE DEFAULT", testcase.user))
+ expectedPasswordExpiration(t, tk, testcase.user, testcase.expired, "")
+ tk.MustExec(fmt.Sprintf("ALTER USER %s PASSWORD EXPIRE INTERVAL 3 DAY", testcase.user))
+ expectedPasswordExpiration(t, tk, testcase.user, testcase.expired, "3")
+ tk.MustExec(fmt.Sprintf("ALTER USER %s PASSWORD EXPIRE", testcase.user))
+ expectedPasswordExpiration(t, tk, testcase.user, "Y", "3")
+ tk.MustExec(fmt.Sprintf("ALTER USER %s IDENTIFIED BY '' PASSWORD EXPIRE", testcase.user))
+ expectedPasswordExpiration(t, tk, testcase.user, "Y", "3")
+ tk.MustExec(fmt.Sprintf("ALTER USER %s IDENTIFIED WITH 'mysql_native_password' AS ''", testcase.user))
+ expectedPasswordExpiration(t, tk, testcase.user, "N", "3")
+ tk.MustExec(fmt.Sprintf("ALTER USER %s IDENTIFIED BY ''", testcase.user))
+ expectedPasswordExpiration(t, tk, testcase.user, "N", "3")
+ }
+
+ // SET PASSWORD
+ tk.MustExec("ALTER USER testuser PASSWORD EXPIRE")
+ expectedPasswordExpiration(t, tk, "testuser", "Y", "3")
+ tk.MustExec("SET PASSWORD FOR testuser = '1234'")
+ expectedPasswordExpiration(t, tk, "testuser", "N", "3")
+
+ tk.MustGetErrCode(`CREATE USER ''@localhost IDENTIFIED BY 'pass' PASSWORD EXPIRE`, mysql.ErrPasswordExpireAnonymousUser)
+ tk.MustExec(`CREATE USER ''@localhost IDENTIFIED BY 'pass'`)
+ tk.MustGetErrCode(`ALTER USER ''@localhost PASSWORD EXPIRE`, mysql.ErrPasswordExpireAnonymousUser)
+
+ // different cleartext authentication plugin
+ for _, authplugin := range []string{mysql.AuthNativePassword, mysql.AuthCachingSha2Password, mysql.AuthTiDBSM3Password} {
+ tk.MustExec("DROP USER IF EXISTS 'u1'@'localhost'")
+ tk.MustExec(fmt.Sprintf("CREATE USER 'u1'@'localhost' IDENTIFIED WITH '%s'", authplugin))
+ tk.MustExec("ALTER USER 'u1'@'localhost' IDENTIFIED BY 'pass'")
+ tk.MustExec("ALTER USER 'u1'@'localhost' PASSWORD EXPIRE")
+ tk.MustQuery("SELECT password_expired FROM mysql.user WHERE user = 'u1'").Check(testkit.Rows("Y"))
+ }
+}
+
+// Test cases that related to PASSWORD VALIDATION, PASSWORD EXPIRATION, PASSWORD REUSE POLICY, and PASSWORD FAILED-LOGIN TRACK.
+func TestPasswordManagement(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("SET GLOBAL validate_password.enable = 1")
+
+ // PASSWORD VALIDATION can work with user-specified PASSWORD REUSE POLICY.
+ tk.MustExec("CREATE USER u1 IDENTIFIED BY '!Abc1234' password history 1")
+ tk.MustGetErrCode("ALTER USER u1 IDENTIFIED BY '!Abc1234'", errno.ErrExistsInHistoryPassword)
+ tk.MustGetErrCode("ALTER USER u1 IDENTIFIED BY '!abc1234'", errno.ErrNotValidPassword)
+
+ // PASSWORD VALIDATION can work with global PASSWORD REUSE POLICY.
+ tk.MustExec("SET GLOBAL password_history = 1")
+ tk.MustExec("DROP USER u1")
+ tk.MustExec("CREATE USER u1 IDENTIFIED BY '!Abc1234'")
+ tk.MustGetErrCode("ALTER USER u1 IDENTIFIED BY '!Abc1234'", errno.ErrExistsInHistoryPassword)
+ tk.MustGetErrCode("ALTER USER u1 IDENTIFIED BY '!abc1234'", errno.ErrNotValidPassword)
+
+ // PASSWORD EXPIRATION can work with ACCOUNT LOCK.
+ // PASSWORD EXPIRE NEVER and ACCOUNT UNLOCK take effect.
+ tk.MustExec(`ALTER USER u1 ACCOUNT LOCK PASSWORD EXPIRE NEVER PASSWORD EXPIRE NEVER ACCOUNT UNLOCK ACCOUNT LOCK ACCOUNT LOCK ACCOUNT UNLOCK;`)
+ tk.MustQuery(`SELECT password_expired, password_lifetime, account_locked FROM mysql.user WHERE USER='u1';`).Check(
+ testkit.Rows("N 0 N"))
+
+ // PASSWORD EXPIRATION can work with PASSWORD REUSE POLICY
+ tk.MustExec(`create user u2 identified by '!Abc1234' password expire password reuse interval default password expire never password
+ reuse interval 3 day password history 5 password history default password expire default`)
+ tk.MustQuery(`select password_expired, password_lifetime, password_reuse_history, password_reuse_time from mysql.user where user = 'u2'`).Check(
+ testkit.Rows("N 3"))
+ tk.MustExec(`alter user u2 password expire default password reuse interval 3 day password history default
+ password expire never password expire interval 5 day password reuse interval default password expire password history 5`)
+ tk.MustQuery(`select password_expired, password_lifetime, password_reuse_history, password_reuse_time from mysql.user where user = 'u2'`).Check(
+ testkit.Rows("Y 5 "))
+ tk.MustExec(`alter user u2 identified by '!Abc12345'`)
+ tk.MustQuery(`select password_expired, password_lifetime, password_reuse_history, password_reuse_time from mysql.user where user = 'u2'`).Check(
+ testkit.Rows("N 5 "))
+
+ // PASSWORD FAILED-LOGIN TRACK can work with USER COMMENT and USER ATTRIBUTE
+ tk.MustExec(`CREATE USER u3 IDENTIFIED BY '!Abc12345' FAILED_LOGIN_ATTEMPTS 4 PASSWORD_LOCK_TIME 3 COMMENT 'Some statements to test create user'`)
+ tk.MustQuery(`select user_attributes->>"$.metadata" from mysql.user where user = 'u3'`).Check(testkit.Rows(`{"comment": "Some statements to test create user"}`))
+ tk.MustQuery(`select user_attributes->>"$.Password_locking" from mysql.user where user = 'u3'`).Check(testkit.Rows(`{"failed_login_attempts": 4, "password_lock_time_days": 3}`))
+ tk.MustExec(`ALTER USER u3 FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME unbounded FAILED_LOGIN_ATTEMPTS 5 PASSWORD_LOCK_TIME 5 ATTRIBUTE '{"name": "John", "age": 19}'`)
+ tk.MustQuery(`select user_attributes->>"$.metadata" from mysql.user where user = 'u3'`).Check(testkit.Rows(`{"age": 19, "comment": "Some statements to test create user", "name": "John"}`))
+ tk.MustQuery(`select user_attributes->>"$.Password_locking" from mysql.user where user = 'u3'`).Check(testkit.Rows(`{"failed_login_attempts": 5, "password_lock_time_days": 5}`))
+
+ tk.MustExec("SET GLOBAL validate_password.enable = 0")
+
+ rootTK := testkit.NewTestKit(t, store)
+ // Password Strength Check.
+ rootTK.MustExec(`set global validate_password.enable = ON`)
+ rootTK.MustExec(`drop user u2`)
+ rootTK.MustGetErrCode(`create user u2 identified by 'u2' PASSWORD EXPIRE INTERVAL 2 DAY password history 2
+ password reuse interval 2 day FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 1`, 1819)
+ rootTK.MustGetErrCode(`create user u2`, 1819)
+ rootTK.MustGetErrCode(`create user u2 identified by 'u2222222' PASSWORD EXPIRE INTERVAL 2 DAY password history 2
+ password reuse interval 2 day FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 1`, 1819)
+ rootTK.MustGetErrCode(`create user u2 identified by 'Uu2222222' PASSWORD EXPIRE INTERVAL 2 DAY password history 2
+ password reuse interval 2 day FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 1`, 1819)
+ rootTK.MustGetErrCode(`create user u2 identified by 'Uu3222222' PASSWORD EXPIRE INTERVAL 2 DAY password history 2
+ password reuse interval 2 day FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 1`, 1819)
+ rootTK.MustExec(`create user u2 identified by 'Uu3@22222' PASSWORD EXPIRE INTERVAL 2 DAY password history 2
+ password reuse interval 2 day FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 1`)
+ rootTK.MustQuery(`Select count(*) from mysql.password_history where user = 'u2' and host = '%'`).Check(testkit.Rows("1"))
+ result := rootTK.MustQuery(`Select authentication_string from mysql.user where user = 'u2' and host = '%'`)
+ result.Check(testkit.Rows(auth.EncodePassword("Uu3@22222")))
+ // Disable password reuse.
+ rootTK.MustGetErrCode(`Alter user u2 identified by 'Uu3@22222'`, 3638)
+ rootTK.MustGetErrCode(`Set password for 'u2' = 'Uu3@22222'`, 3638)
+ // Password Strength Check.
+ rootTK.MustGetErrCode(`Alter user u2 identified by 'U2'`, 1819)
+ rootTK.MustGetErrCode(`Set password for 'u2' = 'U2'`, 1819)
+ // Did not modify successfully.
+ result = rootTK.MustQuery(`Select authentication_string from mysql.user where user = 'u2' and host = '%'`)
+ result.Check(testkit.Rows(auth.EncodePassword("Uu3@22222")))
+ // Auto-lock in effect.
+ err := tk.Session().Auth(&auth.UserIdentity{Username: "u2", Hostname: "%"}, sha1Password(""), nil)
+ require.ErrorContains(t, err, "Account is blocked for 1 day(s) (1 day(s) remaining) due to 1 consecutive failed logins.")
+ result = rootTK.MustQuery(`SELECT
+ JSON_UNQUOTE(JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_count')),
+ JSON_UNQUOTE(JSON_EXTRACT(user_attributes, '$.Password_locking.auto_account_locked')) from mysql.user where user = 'u2' and host = '%'`)
+ result.Check(testkit.Rows(`1 Y`))
+ rootTK.MustExec(`ALTER user u2 account unlock`)
+
+ // Unlock in effect.
+ result = rootTK.MustQuery(`SELECT
+ JSON_UNQUOTE(JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_count')),
+ JSON_UNQUOTE(JSON_EXTRACT(user_attributes, '$.Password_locking.auto_account_locked')) from mysql.user where user = 'u2' and host = '%'`)
+ result.Check(testkit.Rows(`0 N`))
+
+ rootTK.MustExec(`set global validate_password.enable = OFF`)
+ rootTK.MustExec(`update mysql.user set Password_last_changed = date_sub(Password_last_changed,interval '3 0:0:1' DAY_SECOND) where user = 'u2' and host = '%'`)
+ err = domain.GetDomain(rootTK.Session()).NotifyUpdatePrivilege()
+ require.NoError(t, err)
+ // Password expires and takes effect.
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "u2", Hostname: "%"}, sha1Password("Uu3@22222"), nil)
+ require.ErrorContains(t, err, "Your password has expired.")
+ variable.IsSandBoxModeEnabled.Store(true)
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "u2", Hostname: "%"}, sha1Password("Uu3@22222"), nil)
+ require.NoError(t, err)
+ require.True(t, tk.Session().InSandBoxMode())
+
+ rootTK.MustExec(`set global validate_password.enable = ON`)
+ // Forbid other users to change password.
+ tk.MustGetErrCode(`Alter user root identified by 'Uu3@22222'`, 1820)
+ // Disable password reuse.
+ tk.MustGetErrCode(`Alter user u2 identified by 'Uu3@22222'`, 3638)
+ tk.MustGetErrCode(`set password = 'Uu3@22222'`, 3638)
+ // Password Strength Check.
+ tk.MustGetErrCode(`Alter user u2 identified by 'U2'`, 1819)
+ tk.MustGetErrCode(`set password = 'U2'`, 1819)
+ tk.MustExec(`Set password = 'Uu3@22223'`)
+ require.False(t, tk.Session().InSandBoxMode())
+ rootTK.MustQuery(`Select count(*) from mysql.password_history where user = 'u2' and host = '%'`).Check(testkit.Rows("2"))
+ result = rootTK.MustQuery(`Select authentication_string from mysql.user where user = 'u2' and host = '%'`)
+ result.Check(testkit.Rows(auth.EncodePassword("Uu3@22223")))
+ tk = testkit.NewTestKit(t, store)
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "u2", Hostname: "%"}, sha1Password("Uu3@22223"), nil)
+ require.NoError(t, err)
+}
+
+// Test basic CREATE/ALTER USER with failed-login track.
+func TestFailedLoginTrackingBasic(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ createUserTestCases := []struct {
+ sql string
+ rsJSON string
+ user string
+ }{
+ {"CREATE USER 'u1'@'localhost' IDENTIFIED BY 'password' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 3;",
+ "{\"failed_login_attempts\": 3, \"password_lock_time_days\": 3}", "u1"},
+ {"CREATE USER 'u2'@'localhost' IDENTIFIED BY 'password' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME UNBOUNDED;",
+ "{\"failed_login_attempts\": 3, \"password_lock_time_days\": -1}", "u2"},
+ {"CREATE USER 'u3'@'localhost' IDENTIFIED BY 'password' FAILED_LOGIN_ATTEMPTS 3;",
+ "{\"failed_login_attempts\": 3, \"password_lock_time_days\": 0}", "u3"},
+ {"CREATE USER 'u4'@'localhost' IDENTIFIED BY 'password' PASSWORD_LOCK_TIME 3;",
+ "{\"failed_login_attempts\": 0, \"password_lock_time_days\": 3}", "u4"},
+ {"CREATE USER 'u5'@'localhost' IDENTIFIED BY 'password' PASSWORD_LOCK_TIME UNBOUNDED;",
+ "{\"failed_login_attempts\": 0, \"password_lock_time_days\": -1}", "u5"},
+ }
+ for _, tc := range createUserTestCases {
+ tk.MustExec(tc.sql)
+ sql := fmt.Sprintf("SELECT user_attributes->>\"$.Password_locking\" from mysql.user WHERE USER = '%s' AND HOST = 'localhost' for update", tc.user)
+ tk.MustQuery(sql).Check(testkit.Rows(tc.rsJSON))
+ }
+
+ alterUserTestCases := []struct {
+ sql string
+ user string
+ failedLoginAttempts int64
+ passwordLockTimeDays int64
+ failedLoginCount int64
+ comment string
+ }{
+ {"ALTER USER 'u1'@'localhost' FAILED_LOGIN_ATTEMPTS 4 PASSWORD_LOCK_TIME 6;", "u1",
+ 4, 6, 0, ""},
+ {"ALTER USER 'u2'@'localhost' FAILED_LOGIN_ATTEMPTS 4 PASSWORD_LOCK_TIME UNBOUNDED;",
+ "u2", 4, -1, 0, ""},
+ {"ALTER USER 'u3'@'localhost' PASSWORD_LOCK_TIME 6;",
+ "u3", 3, 6, 0, ""},
+ {"ALTER USER 'u4'@'localhost' FAILED_LOGIN_ATTEMPTS 4;",
+ "u4", 4, 3, 0, ""},
+ {"ALTER USER 'u4'@'localhost' PASSWORD_LOCK_TIME UNBOUNDED;",
+ "u4", 4, -1, 0, ""},
+ {"ALTER USER 'u5'@'localhost' ACCOUNT UNLOCK FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 6;",
+ "u5", 3, 6, 0, ""},
+ {"ALTER USER 'u5'@'localhost' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 6 COMMENT 'Something';",
+ "u5", 3, 6, 0, "Something"},
+ }
+ for _, tc := range alterUserTestCases {
+ tk.MustExec(tc.sql)
+ sql := fmt.Sprintf("SELECT user_attributes from mysql.user WHERE USER = '%s' AND HOST = 'localhost' for update", tc.user)
+ rs := tk.MustQuery(sql)
+ buf := bytes.NewBufferString("")
+ for _, row := range rs.Rows() {
+ _, err := fmt.Fprintf(buf, "%s\n", row)
+ require.NoError(t, err)
+ }
+ str := buf.String()
+ var ua []userAttributes
+ err := json.Unmarshal([]byte(str), &ua)
+ require.NoError(t, err)
+ require.Equal(t, tc.failedLoginAttempts, ua[0].PasswordLocking.FailedLoginAttempts, tc.sql, str)
+ require.Equal(t, tc.passwordLockTimeDays, ua[0].PasswordLocking.PasswordLockTimeDays, tc.sql, str)
+ require.Equal(t, tc.failedLoginCount, ua[0].PasswordLocking.FailedLoginCount, tc.sql, str)
+ require.Equal(t, tc.comment, ua[0].Metadata.Comment, tc.sql, str)
+ }
+
+ tk.MustExec("CREATE USER 'u6'@'localhost' IDENTIFIED BY 'password' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 3;")
+ tk.MustQuery(" SHOW CREATE USER 'u6'@'localhost';").Check(
+ testkit.Rows("CREATE USER 'u6'@'localhost' IDENTIFIED WITH 'mysql_native_password' AS '*2470C0C06DEE42FD1618BB99005ADCA2EC9D1E19' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 3"))
+
+ tk.MustExec("CREATE USER 'u7'@'localhost' IDENTIFIED BY 'password';")
+ tk.MustQuery(" SHOW CREATE USER 'u7'@'localhost';").Check(
+ testkit.Rows("CREATE USER 'u7'@'localhost' IDENTIFIED WITH 'mysql_native_password' AS '*2470C0C06DEE42FD1618BB99005ADCA2EC9D1E19' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT"))
+
+ tk.MustExec("CREATE USER 'u8'@'localhost' IDENTIFIED BY 'password' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME UNBOUNDED;")
+ tk.MustQuery(" SHOW CREATE USER 'u8'@'localhost';").Check(
+ testkit.Rows("CREATE USER 'u8'@'localhost' IDENTIFIED WITH 'mysql_native_password' AS '*2470C0C06DEE42FD1618BB99005ADCA2EC9D1E19' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME UNBOUNDED"))
+
+ tk.MustExec("ALTER USER 'u4'@'localhost' PASSWORD_LOCK_TIME 0 FAILED_LOGIN_ATTEMPTS 0")
+ tk.MustQuery("select user_attributes from mysql.user where user = 'u4' and host = 'localhost'").Check(testkit.Rows(``))
+ tk.MustExec("ALTER USER 'u4'@'localhost' account unlock")
+ tk.MustQuery("select user_attributes from mysql.user where user = 'u4' and host = 'localhost'").Check(testkit.Rows(``))
+ tk.MustExec("ALTER USER 'u4'@'localhost' PASSWORD_LOCK_TIME 6")
+ tk.MustQuery("select user_attributes from mysql.user where user = 'u4' and host = 'localhost'").Check(testkit.Rows(`{"Password_locking": {"failed_login_attempts": 0, "password_lock_time_days": 6}}`))
+}
+
+func TestUserReuseControl(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ rootTK := testkit.NewTestKit(t, store)
+ rootTK.MustQuery(`show variables like "password_history"`).Check(testkit.Rows("password_history 0"))
+ rootTK.MustQuery(`show variables like "password_reuse_interval"`).Check(testkit.Rows("password_reuse_interval 0"))
+ rootTK.MustExec(`set global password_history = -1`)
+ rootTK.MustExec(`set global password_reuse_interval = -1`)
+ rootTK.MustQuery(`show variables like "password_history"`).Check(testkit.Rows("password_history 0"))
+ rootTK.MustQuery(`show variables like "password_reuse_interval"`).Check(testkit.Rows("password_reuse_interval 0"))
+ rootTK.MustExec(`set global password_history = 4294967295`)
+ rootTK.MustExec(`set global password_reuse_interval = 4294967295`)
+ rootTK.MustQuery(`show variables like "password_history"`).Check(testkit.Rows("password_history 4294967295"))
+ rootTK.MustQuery(`show variables like "password_reuse_interval"`).Check(testkit.Rows("password_reuse_interval 4294967295"))
+ rootTK.MustExec(`set global password_history = 4294967296`)
+ rootTK.MustExec(`set global password_reuse_interval = 4294967296`)
+ rootTK.MustQuery(`show variables like "password_history"`).Check(testkit.Rows("password_history 4294967295"))
+ rootTK.MustQuery(`show variables like "password_reuse_interval"`).Check(testkit.Rows("password_reuse_interval 4294967295"))
+ rootTK.MustGetErrCode(`set session password_history = 42949`, 1229)
+ rootTK.MustGetErrCode(`set session password_reuse_interval = 42949`, 1229)
+}
+
+func TestUserReuseInfo(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ rootTK := testkit.NewTestKit(t, store)
+ rootTK.MustExec(`CREATE USER testReuse`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(` `))
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD HISTORY 5`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`5 `))
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD HISTORY 0`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`0 `))
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD HISTORY DEFAULT`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(` `))
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD HISTORY 65536`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`65535 `))
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD REUSE INTERVAL 5 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`65535 5`))
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD REUSE INTERVAL 0 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`65535 0`))
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD REUSE INTERVAL DEFAULT`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`65535 `))
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD REUSE INTERVAL 65536 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`65535 65535`))
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD HISTORY 6 PASSWORD REUSE INTERVAL 6 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`6 6`))
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD HISTORY 6 PASSWORD HISTORY 7 `)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`7 6`))
+
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`CREATE USER testReuse PASSWORD HISTORY 5`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`5 `))
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`CREATE USER testReuse PASSWORD REUSE INTERVAL 5 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(` 5`))
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`CREATE USER testReuse PASSWORD REUSE INTERVAL 5 DAY PASSWORD REUSE INTERVAL 6 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(` 6`))
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`CREATE USER testReuse PASSWORD HISTORY 5 PASSWORD REUSE INTERVAL 6 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`5 6`))
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`CREATE USER testReuse PASSWORD REUSE INTERVAL 6 DAY PASSWORD HISTORY 5`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`5 6`))
+
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustGetErrCode(`CREATE USER testReuse PASSWORD HISTORY -5`, 1064)
+ rootTK.MustGetErrCode(`CREATE USER testReuse PASSWORD REUSE INTERVAL -6 DAY`, 1064)
+ rootTK.MustExec(`CREATE USER testReuse PASSWORD HISTORY 65535 PASSWORD REUSE INTERVAL 65535 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`65535 65535`))
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`CREATE USER testReuse PASSWORD HISTORY 65536 PASSWORD REUSE INTERVAL 65536 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`65535 65535`))
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`CREATE USER testReuse PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(` `))
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`CREATE USER testReuse PASSWORD HISTORY 0 PASSWORD REUSE INTERVAL 0 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(`0 0`))
+}
+
+func TestUserReuseFunction(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ rootTK := testkit.NewTestKit(t, store)
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+ rootTK.MustExec(`set global password_history = 1;`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustExec(`DROP USER testReuse`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+
+ rootTK.MustExec(`set global password_history = 0;`)
+ rootTK.MustExec(`set global password_reuse_interval = 1;`)
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`2`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test2'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`3`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test3'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`4`))
+ rootTK.MustExec(`update mysql.password_history set Password_timestamp = date_sub(Password_timestamp,interval '1 0:0:1' DAY_SECOND)`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustExec(`drop USER testReuse `)
+
+ rootTK.MustExec(`set global password_reuse_interval = 0;`)
+ //password nil is not stored
+ rootTK.MustExec(`CREATE USER testReuse PASSWORD HISTORY 5 PASSWORD REUSE INTERVAL 6 DAY`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+ rootTK.MustExec(`drop USER testReuse `)
+
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test' PASSWORD HISTORY 5`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test2'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test3'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`4`))
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustExec(`alter USER testReuse identified by 'test4'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`5`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test5'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`5`))
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test1'`, 3638)
+ rootTK.MustExec(`alter USER testReuse identified by 'test'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`5`))
+ rootTK.MustExec(`drop USER testReuse`)
+
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test' PASSWORD HISTORY 5 PASSWORD REUSE INTERVAL 3 DAY`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test2'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test3'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test4'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test5'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`6`))
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustExec(`update mysql.password_history set Password_timestamp = date_sub(Password_timestamp,interval '3 0:0:1' DAY_SECOND) where user = 'testReuse' order by Password_timestamp asc limit 1`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`6`))
+ rootTK.MustExec(`drop USER testReuse`)
+
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test' PASSWORD HISTORY 5 PASSWORD REUSE INTERVAL 3 DAY`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test2'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test3'`)
+ rootTK.MustExec(`update mysql.password_history set Password_timestamp = date_sub(Password_timestamp,interval '3 0:0:1' DAY_SECOND) where user = 'testReuse' order by Password_timestamp asc limit 1`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`4`))
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustExec(`ALTER USER testReuse PASSWORD HISTORY 3`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`4`))
+ rootTK.MustExec(`drop USER testReuse`)
+
+ rootTK.MustExec(`set global password_history = 1;`)
+ rootTK.MustExec(`set global password_reuse_interval = 1;`)
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test' PASSWORD HISTORY 0 PASSWORD REUSE INTERVAL 0 DAY`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+ rootTK.MustExec(`drop USER testReuse`)
+
+ rootTK.MustExec(`set global password_history = 0;`)
+ rootTK.MustExec(`set global password_reuse_interval = 360000000;`)
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustGetErrCode(`set PASSWORD FOR testReuse = 'test'`, 3638)
+ rootTK.MustExec(`alter USER testReuse identified by ''`)
+ rootTK.MustExec(`alter USER testReuse identified by ''`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`2`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test2'`)
+ rootTK.MustExec(`set global password_reuse_interval = 4294967295;`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test3'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`4`))
+ rootTK.MustExec(`set PASSWORD FOR testReuse = 'test4'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`5`))
+ rootTK.MustExec(`drop USER testReuse`)
+
+ rootTK.MustExec(`set global password_reuse_interval = 0;`)
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test' PASSWORD HISTORY 5`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`2`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test1' PASSWORD HISTORY 0`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test1' PASSWORD HISTORY 2 PASSWORD REUSE INTERVAL 1 DAY`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test2'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test3'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test1' PASSWORD HISTORY 2 PASSWORD REUSE INTERVAL 0 DAY`)
+
+ // Support password and default value modification at the same time.
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`set global password_history = 1`)
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test' PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustGetErrCode(`ALTER USER testReuse identified by 'test' PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`, 3638)
+ rootTK.MustExec(`ALTER USER testReuse identified by 'test1' PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+}
+
+func TestUserReuseDifferentAuth(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ rootTK := testkit.NewTestKit(t, store)
+ // test caching_sha2_password.
+ rootTK.MustExec(`CREATE USER testReuse identified with 'caching_sha2_password' by 'test' PASSWORD HISTORY 1 `)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustGetErrCode(`set password for testReuse = 'test'`, 3638)
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustExec(`alter USER testReuse identified with 'tidb_sm3_password'`)
+ // changing the auth method prunes history.
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`CREATE USER testReuse identified with 'tidb_sm3_password' by 'test' PASSWORD HISTORY 1 `)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustGetErrCode(`set password for testReuse = 'test'`, 3638)
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustExec(`alter USER testReuse identified with 'caching_sha2_password'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustExec(`CREATE USER testReuse identified with 'caching_sha2_password' by 'test' PASSWORD REUSE INTERVAL 1 DAY`)
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustGetErrCode(`set password for testReuse = 'test'`, 3638)
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test2'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test3'`)
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustExec(`update mysql.password_history set Password_timestamp = date_sub(Password_timestamp,interval '1 0:0:1' DAY_SECOND) where user = 'testReuse' order by Password_timestamp asc limit 1`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test'`)
+
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustGetErrCode(`CREATE USER testReuse identified with 'mysql_clear_password' by 'test' PASSWORD REUSE INTERVAL 1 DAY`, 1524)
+ rootTK.MustGetErrCode(`CREATE USER testReuse identified with 'tidb_session_token' by 'test' PASSWORD REUSE INTERVAL 1 DAY`, 1524)
+ // no password.
+ rootTK.MustExec(`CREATE USER testReuse identified with 'auth_socket' by 'test' PASSWORD REUSE INTERVAL 1 DAY`)
+ rootTK.MustExec(`ALTER USER testReuse identified by 'test' `)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+ rootTK.MustQuery(`SELECT authentication_string FROM mysql.user WHERE user = 'testReuse'`).Check(testkit.Rows(""))
+ rootTK.MustExec(`ALTER USER testReuse identified with 'caching_sha2_password' by 'test' `)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ // AuthTiDBAuthToken is the token login method on the cloud,
+ // and the Password Reuse Policy does not take effect.
+ rootTK.MustExec(`drop USER testReuse`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+ rootTK.MustExec(`CREATE USER testReuse identified with 'tidb_auth_token' by 'test' PASSWORD REUSE INTERVAL 1 DAY`)
+ rootTK.MustExec(`ALTER USER testReuse identified by 'test' `)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+ rootTK.MustExec(`set password for testReuse = 'test'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+ rootTK.MustExec(`ALTER USER testReuse identified with 'caching_sha2_password' by 'test' `)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustGetErrCode(`alter USER testReuse identified by 'test'`, 3638)
+ rootTK.MustGetErrCode(`set password for testReuse = 'test'`, 3638)
+ rootTK.MustExec(`drop USER testReuse`)
+}
+
+func TestUserReuseMultiuser(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ rootTK := testkit.NewTestKit(t, store)
+ //alter multi user success
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test', testReuse1 identified by 'test', testReuse2 identified by 'test' PASSWORD HISTORY 65535 PASSWORD REUSE INTERVAL 65535 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user like 'testReuse%'`).Check(testkit.Rows(`65535 65535`, `65535 65535`, `65535 65535`))
+ rootTK.MustExec(`ALTER USER testReuse identified by 'test1', testReuse1 identified by 'test1', testReuse2 identified by 'test1' PASSWORD HISTORY 3 PASSWORD REUSE INTERVAL 3 DAY`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user like 'testReuse%'`).Check(testkit.Rows(`3 3`, `3 3`, `3 3`))
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user like 'testReuse%' group by user`).Check(testkit.Rows(`2`, `2`, `2`))
+ //alter multi user fail
+ rootTK.MustExec(`CREATE USER testReuse3 identified by 'test'`)
+ rootTK.MustQuery(`SELECT Password_reuse_history,Password_reuse_time FROM mysql.user WHERE user like 'testReuse%'`).Check(testkit.Rows(`3 3`, `3 3`, `3 3`, ` `))
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user like 'testReuse%' group by user`).Check(testkit.Rows(`2`, `2`, `2`))
+ rootTK.MustGetErrCode(`ALTER USER testReuse identified by 'test1', testReuse3 identified by 'test1'`, 3638)
+ //drop user
+ rootTK.MustExec(`drop User testReuse, testReuse1, testReuse2, testReuse3`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user like 'testReuse%' `).Check(testkit.Rows(`0`))
+}
+
+func TestUserReuseRename(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ rootTK := testkit.NewTestKit(t, store)
+ rootTK.MustExec(`CREATE USER testReuse identified by 'test' PASSWORD HISTORY 5`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`1`))
+ rootTK.MustExec(`alter USER testReuse identified by 'test1'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test2'`)
+ rootTK.MustExec(`alter USER testReuse identified by 'test3'`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`4`))
+ rootTK.MustExec(`rename USER testReuse to testReuse1`)
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse'`).Check(testkit.Rows(`0`))
+ rootTK.MustQuery(`SELECT count(*) FROM mysql.password_history WHERE user = 'testReuse1'`).Check(testkit.Rows(`4`))
+}
+
+func TestUserAlterUser(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ rootTK := testkit.NewTestKit(t, store)
+ rootTK.MustExec(`CREATE USER test1 IDENTIFIED WITH 'mysql_native_password' BY '1234'`)
+ alterUserSQL := `ALTER USER 'test1' IDENTIFIED BY '222', 'test_not_exist'@'localhost' IDENTIFIED BY '111';`
+ rootTK.MustGetErrCode(alterUserSQL, mysql.ErrCannotUser)
+ result := rootTK.MustQuery(`SELECT authentication_string FROM mysql.User WHERE User="test1" and Host="%"`)
+ result.Check(testkit.Rows(auth.EncodePassword("1234")))
+ alterUserSQL = `ALTER USER IF EXISTS 'test1' IDENTIFIED BY '222', 'test_not_exist'@'localhost' IDENTIFIED BY '111';`
+ rootTK.MustExec(alterUserSQL)
+ rootTK.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Note|3162|User 'test_not_exist'@'localhost' does not exist."))
+ result = rootTK.MustQuery(`SELECT authentication_string FROM mysql.User WHERE User="test1" and Host="%"`)
+ result.Check(testkit.Rows(auth.EncodePassword("222")))
+}
+
+func sha1Password(s string) []byte {
+ crypt := sha1.New()
+ crypt.Write([]byte(s))
+ hashStage1 := crypt.Sum(nil)
+ crypt.Reset()
+ crypt.Write(hashStage1)
+ hashStage2 := crypt.Sum(nil)
+ crypt.Reset()
+ crypt.Write(hashStage2)
+ hashStage3 := crypt.Sum(nil)
+ for i := range hashStage3 {
+ hashStage3[i] ^= hashStage1[i]
+ }
+ return hashStage3
+}
+
+func TestFailedLoginTracking(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ // Set FAILED_LOGIN_ATTEMPTS to 1, and check error messages after login failure once.
+ createAndCheck(tk, "CREATE USER 'testu1'@'localhost' IDENTIFIED BY 'testu1' FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 1",
+ "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": 1}}", "testu1")
+ err := tk.Session().Auth(&auth.UserIdentity{Username: "testu1", Hostname: "localhost"}, sha1Password("password"), nil)
+ lds := strconv.FormatInt(1, 10)
+ errTarget := privileges.GenerateAccountAutoLockErr(1, "testu1", "localhost", lds, lds)
+ require.Equal(t, err.Error(), errTarget.Error())
+ checkAuthUser(t, tk, "testu1", 1, "Y")
+
+ // Check the login error message after the account is locked.
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "testu1", Hostname: "localhost"}, sha1Password("password"), nil)
+ require.Equal(t, err.Error(), errTarget.Error())
+ checkAuthUser(t, tk, "testu1", 1, "Y")
+
+ // Set FAILED_LOGIN_ATTEMPTS to 1 and PASSWORD_LOCK_TIME to UNBOUNDED. Check error messages after failed login once.
+ createAndCheck(tk, "CREATE USER 'testu2'@'localhost' IDENTIFIED BY 'testu2' FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME UNBOUNDED",
+ "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": -1}}", "testu2")
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "testu2", Hostname: "localhost"}, sha1Password("password"), nil)
+ errTarget = privileges.GenerateAccountAutoLockErr(1, "testu2", "localhost", "unlimited", "unlimited")
+ require.Equal(t, err.Error(), errTarget.Error())
+ checkAuthUser(t, tk, "testu2", 1, "Y")
+
+ // Check the login error message after the account is locked.
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "testu2", Hostname: "localhost"}, sha1Password("password"), nil)
+ require.Equal(t, err.Error(), errTarget.Error())
+ checkAuthUser(t, tk, "testu2", 1, "Y")
+
+ // Set FAILED_LOGIN_ATTEMPTS to 0 or PASSWORD_LOCK_TIME to 0. Check error messages after failed login once.
+ createAndCheck(tk, "CREATE USER 'testu3'@'localhost' IDENTIFIED BY 'testu3' FAILED_LOGIN_ATTEMPTS 0 PASSWORD_LOCK_TIME UNBOUNDED",
+ "{\"Password_locking\": {\"failed_login_attempts\": 0, \"password_lock_time_days\": -1}}", "testu3")
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "testu3", Hostname: "localhost"}, sha1Password("password"), nil)
+ require.ErrorContains(t, err, "Access denied for user 'testu3'@'localhost' (using password: YES)")
+ checkAuthUser(t, tk, "testu3", 0, "")
+ createAndCheck(tk, "CREATE USER 'testu4'@'localhost' IDENTIFIED BY 'testu4' FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 0",
+ "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": 0}}", "testu4")
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "testu4", Hostname: "localhost"}, sha1Password("password"), nil)
+ require.ErrorContains(t, err, "Access denied for user 'testu4'@'localhost' (using password: YES)")
+ checkAuthUser(t, tk, "testu4", 0, "")
+ tk.MustExec("CREATE USER 'testu5'@'localhost' IDENTIFIED BY 'testu5' FAILED_LOGIN_ATTEMPTS 0 PASSWORD_LOCK_TIME 0")
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "testu5", Hostname: "localhost"}, sha1Password("password"), nil)
+ require.ErrorContains(t, err, "Access denied for user 'testu5'@'localhost' (using password: YES)")
+ tk.MustQuery("select user_attributes from mysql.user where user= 'testu5' and host = 'localhost'").Check(testkit.Rows("{}"))
+
+ tk.MustExec("DROP USER 'testu1'@'localhost', 'testu2'@'localhost', 'testu3'@'localhost', 'testu4'@'localhost', 'testu5'@'localhost'")
+
+ // Create user specifying only comment.
+ tk.MustExec("CREATE USER 'testu1'@'localhost' IDENTIFIED BY 'testu1' comment 'testcomment' ")
+ tk.MustQuery("select user_attributes from mysql.user where user= 'testu1' and host = 'localhost'").
+ Check(testkit.Rows("{\"metadata\": {\"comment\": \"testcomment\"}}"))
+
+ // Create user specifying only attribute.
+ tk.MustExec("create user testu2@'localhost' identified by 'testu2' ATTRIBUTE '{\"attribute\":\"testattribute\"}'")
+ tk.MustQuery("select user_attributes from mysql.user where user= 'testu2' and host = 'localhost'").
+ Check(testkit.Rows("{\"metadata\": {\"attribute\": \"testattribute\"}}"))
+
+ // Create user specified comment and FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME.
+ tk.MustExec("create user testu3@'localhost' identified by 'testu3' FAILED_LOGIN_ATTEMPTS 1 " +
+ "PASSWORD_LOCK_TIME 1 comment 'testcomment'")
+ checkUserUserAttributes(tk, "testu3", "localhost", "1 1 {\"comment\": \"testcomment\"}")
+
+ // Create user specified attribute and FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME.
+ tk.MustExec("create user testu4@'localhost' identified by 'testu4' FAILED_LOGIN_ATTEMPTS 1 " +
+ "PASSWORD_LOCK_TIME 1 ATTRIBUTE '{\"attribute\":\"testattribute\"}'")
+ checkUserUserAttributes(tk, "testu4", "localhost", "1 1 {\"attribute\": \"testattribute\"}")
+
+ // Create user specified comment, FAILED_LOGIN_ATTEMPTS, and PASSWORD_LOCK_TIME,
+ // and confirm the user_attributes column value after login fails.
+ tk.MustExec("create user testu5@'localhost' identified by 'testu5' FAILED_LOGIN_ATTEMPTS 2 " +
+ "PASSWORD_LOCK_TIME 1 comment 'testcomment'")
+ checkUserUserAttributes(tk, "testu5", "localhost", "2 1 {\"comment\": \"testcomment\"}")
+
+ // Confirm the user_attributes value after login failure once.
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu5", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu5", "localhost", "2 \"N\" 1 1 {\"comment\": \"testcomment\"}")
+
+ // After the number of failed login attempts reaches FAILED_LOGIN_ATTEMPTS, check the account lock status.
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu5", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu5", "localhost", "2 \"Y\" 2 1 {\"comment\": \"testcomment\"}")
+ // After the account is locked, manually unlock the account and check the user_attributes value.
+ tk.MustExec("alter user testu5@'localhost' account unlock")
+ checkUserUserAttributes(tk, "testu5", "localhost", "2 \"N\" 0 1 {\"comment\": \"testcomment\"}")
+
+ // Create user specified comment, FAILED_LOGIN_ATTEMPTS, and PASSWORD_LOCK_TIME,
+ // and confirm the user_attributes column value after login fails.
+ tk.MustExec("create user testu6@'localhost' identified by '' FAILED_LOGIN_ATTEMPTS 2 PASSWORD_LOCK_TIME 1 " +
+ "comment 'testcomment'")
+ // Confirm the user_attributes value after login failure once.
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu6", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu6", "localhost", "2 \"N\" 1 1 {\"comment\": \"testcomment\"}")
+
+ // After the number of failed login attempts reaches FAILED_LOGIN_ATTEMPTS, check the account lock status.
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu6", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu6", "localhost", "2 \"Y\" 2 1 {\"comment\": \"testcomment\"}")
+
+ // After the account is automatically locked, change the lock time and check
+ // the user_attributes value after logging in successfully.
+ changeAutoLockedLastChanged(tk, "-72h1s", "testu6")
+ sk1 := testkit.NewTestKit(t, store)
+ require.NoError(t, sk1.Session().Auth(&auth.UserIdentity{Username: "testu6", Hostname: "localhost"}, nil, nil))
+ checkUserUserAttributes(tk, "testu6", "localhost", "3 \"N\" 0 3 {\"comment\": \"testcomment\"}")
+
+ // Create user specified attributes, FAILED_LOGIN_ATTEMPTS, and PASSWORD_LOCK_TIME,
+ // and confirm the user_attributes column value after login fails.
+ tk.MustExec("create user testu7@'localhost' identified by 'testu7' FAILED_LOGIN_ATTEMPTS 2 PASSWORD_LOCK_TIME 1 " +
+ "ATTRIBUTE '{\"attribute\":\"testattribute\"}'")
+ checkUserUserAttributes(tk, "testu7", "localhost", "2 1 {\"attribute\": \"testattribute\"}")
+
+ // Confirm the user_attributes value after login failure once.
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu7", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu7", "localhost", "2 \"N\" 1 1 {\"attribute\": \"testattribute\"}")
+
+ // After the number of failed login attempts reaches FAILED_LOGIN_ATTEMPTS, check the account lock status.
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu7", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu7", "localhost", "2 \"Y\" 2 1 {\"attribute\": \"testattribute\"}")
+
+ // After the account is locked, manually unlock the account and check the user_attributes value.
+ tk.MustExec("alter user testu7@'localhost' account unlock")
+ checkUserUserAttributes(tk, "testu7", "localhost", "2 \"N\" 0 1 {\"attribute\": \"testattribute\"}")
+
+ tk.MustExec("create user testu8@'localhost' identified by '' FAILED_LOGIN_ATTEMPTS 2 PASSWORD_LOCK_TIME 1" +
+ " ATTRIBUTE '{\"attribute\":\"testattribute\"}'")
+ // Confirm the user_attributes value after login failure once.
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu8", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu8", "localhost", "2 \"N\" 1 1 {\"attribute\": \"testattribute\"}")
+
+ // After the number of failed login attempts reaches FAILED_LOGIN_ATTEMPTS, check the account lock status.
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu8", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu8", "localhost", "2 \"Y\" 2 1 {\"attribute\": \"testattribute\"}")
+
+ // After the account is automatically locked, change the lock time and check
+ // the user_attributes value after logging in successfully.
+ changeAutoLockedLastChanged(tk, "-72h1s", "testu8")
+ sk2 := testkit.NewTestKit(t, store)
+ require.NoError(t, sk2.Session().Auth(&auth.UserIdentity{Username: "testu8", Hostname: "localhost"}, nil, nil))
+ checkUserUserAttributes(tk, "testu8", "localhost", "3 \"N\" 0 3 {\"attribute\": \"testattribute\"}")
+
+ // FAILED_LOGIN_ATTEMPTS is set to 2 . check user_attributes value after
+ // the user login fails once ,and login success at second time.
+ tk.MustExec("create user testu9@'localhost' identified by '' FAILED_LOGIN_ATTEMPTS 2 PASSWORD_LOCK_TIME 1" +
+ " comment 'testcomment'")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu9", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu9", "localhost", "2 \"N\" 1 1 {\"comment\": \"testcomment\"}")
+ sk3 := testkit.NewTestKit(t, store)
+ require.NoError(t, sk3.Session().Auth(&auth.UserIdentity{Username: "testu9", Hostname: "localhost"},
+ nil, nil))
+ checkUserUserAttributes(tk, "testu9", "localhost", "2 \"N\" 0 1 {\"comment\": \"testcomment\"}")
+
+ // FAILED_LOGIN_ATTEMPTS or PASSWORD_LOCK_TIME is set to 0. Check user_attributes value after login fail.
+ tk.MustExec("create user testu10@'localhost' identified by '' FAILED_LOGIN_ATTEMPTS 2 PASSWORD_LOCK_TIME 0 " +
+ "comment 'testcomment'")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu10", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu10", "localhost", "2 0 {\"comment\": \"testcomment\"}")
+
+ tk.MustExec("create user testu11@'localhost' identified by '' FAILED_LOGIN_ATTEMPTS 0 PASSWORD_LOCK_TIME 2 " +
+ "comment 'testcomment'")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu11", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu11", "localhost", "0 2 {\"comment\": \"testcomment\"}")
+
+ // The account is automatically locked after the user specifies FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME.
+ // Change FAILED_LOGIN_ATTEMPTS or PASSWORD_LOCK_TIME to 0, and check whether the user can login.
+ tk.MustExec("create user testu12@'localhost' identified by '' FAILED_LOGIN_ATTEMPTS 2 PASSWORD_LOCK_TIME 1 " +
+ "comment 'testcomment'")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu12", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu12", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu12", "localhost", "2 \"Y\" 2 1 {\"comment\": \"testcomment\"}")
+ tk.MustExec("alter user testu12@'localhost' FAILED_LOGIN_ATTEMPTS 0")
+ checkUserUserAttributes(tk, "testu12", "localhost", "0 \"Y\" 2 1 {\"comment\": \"testcomment\"}")
+ sk4 := testkit.NewTestKit(t, store)
+ require.NoError(t, sk4.Session().Auth(&auth.UserIdentity{Username: "testu12", Hostname: "localhost"},
+ nil, nil))
+
+ rootk := testkit.NewTestKit(t, store)
+ createAndCheck(tk, "CREATE USER 'u6'@'localhost' IDENTIFIED BY '' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 3",
+ "{\"Password_locking\": {\"failed_login_attempts\": 3, \"password_lock_time_days\": 3}}", "u6")
+ createAndCheck(tk, "CREATE USER 'u5'@'localhost' IDENTIFIED BY '' FAILED_LOGIN_ATTEMPTS 60 PASSWORD_LOCK_TIME 3",
+ "{\"Password_locking\": {\"failed_login_attempts\": 60, \"password_lock_time_days\": 3}}", "u5")
+
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 1, "N")
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, nil, nil))
+ checkAuthUser(t, rootk, "u6", 0, "N")
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "localhost"}, nil, nil))
+
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 1, "N")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 2, "N")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 3, "Y")
+
+ changeAutoLockedLastChanged(rootk, "-72h1s", "u6")
+ loadUser(t, tk, 1, rootk)
+ checkAuthUser(t, rootk, "u6", 3, "Y")
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, nil, nil))
+ checkAuthUser(t, rootk, "u6", 0, "N")
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "localhost"}, nil, nil))
+
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 1, "N")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 2, "N")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 3, "Y")
+ alterAndCheck(t, rootk, "ALTER USER 'u6'@'localhost' ACCOUNT UNLOCK;", "u6", 3, 3, 0)
+ loadUser(t, tk, 2, rootk)
+ checkAuthUser(t, rootk, "u6", 0, "N")
+
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 1, "N")
+ alterAndCheck(t, rootk, "ALTER USER 'u6'@'localhost' ACCOUNT UNLOCK;", "u6", 3, 3, 0)
+ checkAuthUser(t, rootk, "u6", 0, "N")
+
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 1, "N")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 2, "N")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 3, "Y")
+ changeAutoLockedLastChanged(rootk, "-72h1s", "u6")
+ loadUser(t, tk, 3, rootk)
+ checkAuthUser(t, rootk, "u6", 3, "Y")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u6", 1, "N")
+
+ createAndCheck(rootk, "CREATE USER 'u1'@'localhost' IDENTIFIED BY '' FAILED_LOGIN_ATTEMPTS 3",
+ "{\"Password_locking\": {\"failed_login_attempts\": 3, \"password_lock_time_days\": 0}}", "u1")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u1", 0, "")
+ alterAndCheck(t, rootk, "ALTER USER 'u1'@'localhost' PASSWORD_LOCK_TIME 6;", "u1", 3, 6, 0)
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u1", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u1", 1, "N")
+}
+
+func TestFailedLoginTrackingAlterUser(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ // Create user specifying only comment.
+ tk.MustExec("CREATE USER 'testu1'@'localhost' IDENTIFIED BY 'testu1' comment 'testcomment' ")
+ tk.MustQuery("select user_attributes from mysql.user where user= 'testu1' and host = 'localhost'").
+ Check(testkit.Rows("{\"metadata\": {\"comment\": \"testcomment\"}}"))
+ tk.MustExec("Alter USER 'testu1'@'localhost' comment ''")
+ tk.MustQuery("select user_attributes from mysql.user where user= 'testu1' and host = 'localhost'").
+ Check(testkit.Rows("{\"metadata\": {\"comment\": \"\"}}"))
+
+ // Create user specifying only attribute.
+ tk.MustExec("CREATE USER 'testu2'@'localhost' IDENTIFIED BY 'testu2' ATTRIBUTE '{\"attribute\":\"testattribute\"}'")
+ tk.MustQuery("select user_attributes from mysql.user where user= 'testu2' and host = 'localhost'").
+ Check(testkit.Rows("{\"metadata\": {\"attribute\": \"testattribute\"}}"))
+ tk.MustExec("Alter USER 'testu2'@'localhost' ATTRIBUTE '{\"attribute\":\"test\"}'")
+ tk.MustQuery("select user_attributes from mysql.user where user= 'testu2' and host = 'localhost'").
+ Check(testkit.Rows("{\"metadata\": {\"attribute\": \"test\"}}"))
+
+ // Create a user and specify FAILED_LOGIN_ATTEMPTS, PASSWORD_LOCK_TIME, and COMMENT.
+ // Check the user_attributes value after alter user.
+ tk.MustExec("CREATE USER 'testu3'@'localhost' IDENTIFIED BY 'testu3' FAILED_LOGIN_ATTEMPTS 1 " +
+ "PASSWORD_LOCK_TIME 1 comment 'testcomment'")
+ checkUserUserAttributes(tk, "testu3", "localhost", "1 1 {\"comment\": \"testcomment\"}")
+ tk.MustExec("alter user 'testu3'@'localhost' FAILED_LOGIN_ATTEMPTS 0")
+ checkUserUserAttributes(tk, "testu3", "localhost", "0 1 {\"comment\": \"testcomment\"}")
+ tk.MustExec("alter user 'testu3'@'localhost' PASSWORD_LOCK_TIME 0")
+ tk.MustQuery("select JSON_EXTRACT(user_attributes, '$.Password_locking')," +
+ "JSON_EXTRACT(user_attributes, '$.metadata')from mysql.user where user= 'testu3' and host = 'localhost'").
+ Check(testkit.Rows(" {\"comment\": \"testcomment\"}"))
+
+ // Create a user and specify FAILED_LOGIN_ATTEMPTS, PASSWORD_LOCK_TIME, and ATTRIBUTE.
+ // Check the user_attributes value after alter user.
+ tk.MustExec("CREATE USER 'testu4'@'localhost' IDENTIFIED BY 'testu4' FAILED_LOGIN_ATTEMPTS 1 " +
+ "PASSWORD_LOCK_TIME 1 ATTRIBUTE '{\"attribute\":\"testattribute\"}'")
+ checkUserUserAttributes(tk, "testu4", "localhost", "1 1 {\"attribute\": \"testattribute\"}")
+ tk.MustExec("alter user 'testu4'@'localhost' FAILED_LOGIN_ATTEMPTS 0")
+ checkUserUserAttributes(tk, "testu4", "localhost", "0 1 {\"attribute\": \"testattribute\"}")
+ tk.MustExec("alter user 'testu4'@'localhost' PASSWORD_LOCK_TIME 0")
+ tk.MustQuery("select JSON_EXTRACT(user_attributes, '$.Password_locking')," +
+ "JSON_EXTRACT(user_attributes, '$.metadata')from mysql.user where user= 'testu4' and host = 'localhost'").
+ Check(testkit.Rows(" {\"attribute\": \"testattribute\"}"))
+
+ // Create a user and specify FAILED_LOGIN_ATTEMPTS, PASSWORD_LOCK_TIME, and ATTRIBUTE.
+ // Check the user_attributes value after alter user.
+ tk.MustExec("CREATE USER 'testu5'@'localhost' IDENTIFIED BY 'testu5' FAILED_LOGIN_ATTEMPTS 1 " +
+ "PASSWORD_LOCK_TIME 1 ATTRIBUTE '{\"attribute\":\"testattribute\"}'")
+ checkUserUserAttributes(tk, "testu5", "localhost", "1 1 {\"attribute\": \"testattribute\"}")
+ tk.MustExec("alter user 'testu5'@'localhost' FAILED_LOGIN_ATTEMPTS 0 PASSWORD_LOCK_TIME 0 ATTRIBUTE '{\"attribute\":\"test\"}'")
+ tk.MustQuery("select JSON_EXTRACT(user_attributes, '$.Password_locking')," +
+ "JSON_EXTRACT(user_attributes, '$.metadata')from mysql.user where user= 'testu5' and host = 'localhost'").
+ Check(testkit.Rows(" {\"attribute\": \"test\"}"))
+
+ // Create a user to specify a comment, modify the user to add an ATTRIBUTE,
+ // modify the user to delete a comment, and check the user_attributes value.
+ tk.MustExec("CREATE USER 'testu6'@'localhost' IDENTIFIED BY 'testu6' FAILED_LOGIN_ATTEMPTS 1 " +
+ "PASSWORD_LOCK_TIME 1 comment 'testcomment'")
+ checkUserUserAttributes(tk, "testu6", "localhost", "1 1 {\"comment\": \"testcomment\"}")
+ tk.MustExec("alter user 'testu6'@'localhost' ATTRIBUTE '{\"attribute\": \"testattribute\"}'")
+ checkUserUserAttributes(tk, "testu6", "localhost", "1 1 {\"attribute\": \"testattribute\", \"comment\": \"testcomment\"}")
+ tk.MustExec("alter user 'testu6'@'localhost' ATTRIBUTE '{\"comment\": null}'")
+ checkUserUserAttributes(tk, "testu6", "localhost", "1 1 {\"attribute\": \"testattribute\"}")
+
+ // After consecutive login failures and the account is locked,
+ // change the values of FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME to 0 and check the user_attributes value
+ tk.MustExec("CREATE USER 'testu7'@'localhost' IDENTIFIED BY 'testu7' FAILED_LOGIN_ATTEMPTS 1 " +
+ "PASSWORD_LOCK_TIME 1 comment 'testcomment'")
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu7", Hostname: "localhost"},
+ sha1Password("password"), nil))
+ checkUserUserAttributes(tk, "testu7", "localhost", "1 \"Y\" 1 1 {\"comment\": \"testcomment\"}")
+ tk.MustExec("alter user 'testu7'@'localhost' FAILED_LOGIN_ATTEMPTS 0 PASSWORD_LOCK_TIME 0")
+ tk.MustQuery("select JSON_EXTRACT(user_attributes, '$.Password_locking'),JSON_EXTRACT(user_attributes,'$.metadata') " +
+ "from mysql.user where user='testu7' and host ='localhost'").Check(testkit.Rows(" {\"comment\": \"testcomment\"}"))
+
+ // Create a user and specify FAILED_LOGIN_ATTEMPTS, PASSWORD_LOCK_TIME.
+ // Check the user_attributes value after alter user.
+ tk.MustExec("CREATE USER 'testu8'@'localhost' IDENTIFIED BY 'testu5' FAILED_LOGIN_ATTEMPTS 1 " +
+ "PASSWORD_LOCK_TIME 1")
+ checkUserUserAttributes(tk, "testu8", "localhost", "1 1 ")
+ tk.MustExec("alter user 'testu8'@'localhost' FAILED_LOGIN_ATTEMPTS 0 PASSWORD_LOCK_TIME 0")
+ tk.MustQuery("select user_attributes from mysql.user where user= 'testu8' and host = 'localhost'").
+ Check(testkit.Rows(""))
+
+ // Specify only FAILED_LOGIN_ATTEMPTS one attribute when creating user.
+ // Change the value to 0 and check the user_attributes value.
+ tk.MustExec("CREATE USER 'testu9'@'localhost' IDENTIFIED BY 'testu9' FAILED_LOGIN_ATTEMPTS 1 ")
+ tk.MustQuery("select JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_attempts') " +
+ "from mysql.user where user='testu9' and host ='localhost'").Check(testkit.Rows("1"))
+ tk.MustExec("ALTER USER 'testu9'@'localhost' FAILED_LOGIN_ATTEMPTS 0")
+ tk.MustQuery("select user_attributes from mysql.user where user='testu9' and host ='localhost'").Check(testkit.Rows(""))
+
+ // Specify only PASSWORD_LOCK_TIME one attribute when creating user.
+ // Change the value to 0 and check the user_attributes value.
+ tk.MustExec("CREATE USER 'testu10'@'localhost' IDENTIFIED BY 'testu10' PASSWORD_LOCK_TIME 1 ")
+ tk.MustQuery("select JSON_EXTRACT(user_attributes, '$.Password_locking.password_lock_time_days') " +
+ "from mysql.user where user='testu10' and host ='localhost'").Check(testkit.Rows("1"))
+ tk.MustExec("ALTER USER 'testu10'@'localhost' PASSWORD_LOCK_TIME 0")
+ tk.MustQuery("select user_attributes from mysql.user where user='testu10' and host ='localhost'").Check(testkit.Rows(""))
+
+ // Specify FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME attributes when creating user ,
+ // change the values of the two attributes to 0, and check the value of user_attributes.
+ tk.MustExec("CREATE USER 'testu11'@'localhost' IDENTIFIED BY 'testu11' FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 1 ")
+ tk.MustQuery("select JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_attempts')," +
+ "JSON_EXTRACT(user_attributes, '$.Password_locking.password_lock_time_days') " +
+ "from mysql.user where user='testu11' and host ='localhost'").Check(testkit.Rows("1 1"))
+ tk.MustExec("ALTER USER 'testu11'@'localhost' PASSWORD_LOCK_TIME 0")
+ tk.MustQuery("select JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_attempts')," +
+ "JSON_EXTRACT(user_attributes, '$.Password_locking.password_lock_time_days') " +
+ "from mysql.user where user='testu11' and host ='localhost'").Check(testkit.Rows("1 0"))
+ tk.MustExec("ALTER USER 'testu11'@'localhost' FAILED_LOGIN_ATTEMPTS 0")
+ tk.MustQuery("select user_attributes " +
+ "from mysql.user where user='testu11' and host ='localhost'").Check(testkit.Rows(""))
+
+ rootTK := testkit.NewTestKit(t, store)
+ sql := new(strings.Builder)
+ checkUserAttributes := "select JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_attempts')," +
+ "JSON_EXTRACT(user_attributes, '$.Password_locking.auto_account_locked')," +
+ "JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_count')," +
+ "JSON_EXTRACT(user_attributes, '$.Password_locking.password_lock_time_days')," +
+ "JSON_EXTRACT(user_attributes, '$.metadata')from mysql.user where user= %? and host = %?"
+ err := domain.GetDomain(rootTK.Session()).NotifyUpdatePrivilege()
+ require.NoError(t, err)
+ rootTK.MustExec(`CREATE USER test1 IDENTIFIED BY '1234' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 3 COMMENT 'test'`)
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "test1", Hostname: "%"}, sha1Password("1234"), nil)
+ require.NoError(t, err)
+ sqlexec.MustFormatSQL(sql, checkUserAttributes, "test1", "%")
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`3 3 {"comment": "test"}`))
+ tk = testkit.NewTestKit(t, store)
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "test1", Hostname: "%"}, sha1Password(""), nil)
+ require.Error(t, err)
+
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`3 "N" 1 3 {"comment": "test"}`))
+ rootTK.MustExec(`Alter user test1 FAILED_LOGIN_ATTEMPTS 4 `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`4 "N" 1 3 {"comment": "test"}`))
+ rootTK.MustExec(`Alter user test1 PASSWORD_LOCK_TIME 4 `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`4 "N" 1 4 {"comment": "test"}`))
+ rootTK.MustExec(`Alter user test1 COMMENT 'test1' `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`4 "N" 1 4 {"comment": "test1"}`))
+ rootTK.MustExec(`Alter user test1 FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 3 COMMENT 'test'`)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`3 "N" 1 3 {"comment": "test"}`))
+
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "test1", Hostname: "%"}, sha1Password(""), nil)
+ require.Error(t, err)
+ err = tk.Session().Auth(&auth.UserIdentity{Username: "test1", Hostname: "%"}, sha1Password(""), nil)
+ require.Error(t, err)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`3 "Y" 3 3 {"comment": "test"}`))
+ rootTK.MustExec(`Alter user test1 FAILED_LOGIN_ATTEMPTS 4 `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`4 "Y" 3 3 {"comment": "test"}`))
+ rootTK.MustExec(`Alter user test1 PASSWORD_LOCK_TIME 4 `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`4 "Y" 3 4 {"comment": "test"}`))
+ rootTK.MustExec(`Alter user test1 COMMENT 'test2' `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`4 "Y" 3 4 {"comment": "test2"}`))
+ rootTK.MustExec(`Alter user test1 account unlock `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`4 "N" 0 4 {"comment": "test2"}`))
+
+ rootTK.MustExec(`Alter user test1 FAILED_LOGIN_ATTEMPTS 0 `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`0 "N" 0 4 {"comment": "test2"}`))
+ rootTK.MustExec(`Alter user test1 PASSWORD_LOCK_TIME 0 `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(` {"comment": "test2"}`))
+
+ rootTK.MustExec(`Alter user test1 account unlock `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(` {"comment": "test2"}`))
+ rootTK.MustExec(`Alter user test1 FAILED_LOGIN_ATTEMPTS 4 `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`4 0 {"comment": "test2"}`))
+ rootTK.MustExec(`Alter user test1 account unlock `)
+ rootTK.MustQuery(sql.String()).Check(testkit.Rows(`4 "N" 0 0 {"comment": "test2"}`))
+}
+
+func TestFailedLoginTrackingCheckPrivilges(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ createAndCheck(tk, "CREATE USER 'testu1'@'localhost' IDENTIFIED BY '' FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 1",
+ "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": 1}}", "testu1")
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu1", Hostname: "localhost"}, nil, nil))
+ // Specify FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME attributes when creating user ,
+ // Check user privileges after successful login.
+ tk.MustQuery(`show grants`).Check(testkit.Rows("GRANT USAGE ON *.* TO 'testu1'@'localhost'"))
+ tk.MustQuery(`select user()`).Check(testkit.Rows("testu1@localhost"))
+}
+
+func TestUserPassword(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec(`set global validate_password.enable = ON`)
+
+ testcases := []struct {
+ errSQL string
+ sucSQL string
+ user string
+ host string
+ rsJSON string
+ simplePassword string
+ strongPassword string
+ }{
+ {
+ "CREATE USER 'u1'@'localhost' IDENTIFIED BY 'qwe123' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 4;",
+ "CREATE USER 'u1'@'localhost' IDENTIFIED BY '!@#HASHhs123' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 4;",
+ "u1",
+ "localhost",
+ "{\"Password_locking\": {\"failed_login_attempts\": 3, \"password_lock_time_days\": 4}}",
+ "qwe123",
+ "!@#HASHhs123",
+ },
+ {
+ `CREATE USER 'u2'@'localhost' IDENTIFIED BY 'qwe123' FAILED_LOGIN_ATTEMPTS 4 PASSWORD_LOCK_TIME 3 COMMENT 'Some statements to test create user'`,
+ `CREATE USER 'u2'@'localhost' IDENTIFIED BY '!@#HASHhs123' FAILED_LOGIN_ATTEMPTS 4 PASSWORD_LOCK_TIME 3 COMMENT 'Some statements to test create user'`,
+ "u2",
+ "localhost",
+ "{\"Password_locking\": {\"failed_login_attempts\": 4, \"password_lock_time_days\": 3}, \"metadata\": {\"comment\": \"Some statements to test create user\"}}",
+ "qwe123",
+ "!@#HASHhs123",
+ },
+ }
+ for _, tc := range testcases {
+ tk := testkit.NewTestKit(t, store)
+ rootk := testkit.NewTestKit(t, store)
+ createAndCheckToErr(t, rootk, tc.errSQL, tc.user)
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: tc.user, Hostname: tc.host}, sha1Password(tc.simplePassword), nil))
+ createAndCheck(rootk, tc.sucSQL, tc.rsJSON, tc.user)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: tc.user, Hostname: tc.host}, sha1Password(tc.strongPassword), nil))
+ }
+}
+
+func TestPasswordExpiredAndTacking(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ user := "u3"
+ host := "localhost"
+ tk.MustExec(`set global validate_password.enable = ON`)
+ tk = testkit.NewTestKit(t, store)
+ createAndCheckToErr(t, tk, `CREATE USER 'u3'@'localhost' IDENTIFIED BY 'qwe123' PASSWORD EXPIRE INTERVAL 3 DAY FAILED_LOGIN_ATTEMPTS 4 PASSWORD_LOCK_TIME 3 COMMENT 'Some statements to test create user'`, user)
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: user, Hostname: host}, sha1Password("qwe123"), nil))
+ tk = testkit.NewTestKit(t, store)
+ createAndCheck(tk, `CREATE USER 'u3'@'localhost' IDENTIFIED BY '!@#HASHhs123' PASSWORD EXPIRE INTERVAL 3 DAY FAILED_LOGIN_ATTEMPTS 4 PASSWORD_LOCK_TIME 3 COMMENT 'Some statements to test create user'`,
+ "{\"Password_locking\": {\"failed_login_attempts\": 4, \"password_lock_time_days\": 3}, \"metadata\": {\"comment\": \"Some statements to test create user\"}}", user)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: user, Hostname: host}, sha1Password("!@#HASHhs123"), nil))
+
+ tk = testkit.NewTestKit(t, store)
+ tk.MustExec(fmt.Sprintf("ALTER USER '%s'@'%s' PASSWORD EXPIRE NEVER", user, host))
+ tk = testkit.NewTestKit(t, store)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: user, Hostname: host}, sha1Password("!@#HASHhs123"), nil))
+
+ loginFailedAncCheck(t, store, user, host, "password", 1, "N")
+ loginSucAncCheck(t, store, user, host, "!@#HASHhs123", 0, "N")
+ loginFailedAncCheck(t, store, user, host, "password", 1, "N")
+ loginFailedAncCheck(t, store, user, host, "password", 2, "N")
+ loginFailedAncCheck(t, store, user, host, "password", 3, "N")
+ loginFailedAncCheck(t, store, user, host, "password", 4, "Y")
+
+ tk = testkit.NewTestKit(t, store)
+ tk.MustExec(fmt.Sprintf("ALTER USER '%s'@'%s' PASSWORD EXPIRE", user, host))
+ tk = testkit.NewTestKit(t, store)
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: user, Hostname: host}, sha1Password("!@#HASHhs123"), nil))
+}
+
+func loginFailedAncCheck(t *testing.T, store kv.Storage, user, host, password string, failedLoginCount int64, autoAccountLocked string) {
+ tk := testkit.NewTestKit(t, store)
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: user, Hostname: host}, sha1Password(password), nil))
+ checkAuthUser(t, tk, user, failedLoginCount, autoAccountLocked)
+}
+
+func loginSucAncCheck(t *testing.T, store kv.Storage, user, host, password string, failedLoginCount int64, autoAccountLocked string) {
+ tk := testkit.NewTestKit(t, store)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: user, Hostname: host}, sha1Password(password), nil))
+ tk = testkit.NewTestKit(t, store)
+ checkAuthUser(t, tk, user, failedLoginCount, autoAccountLocked)
+}
+
+func loadUser(t *testing.T, tk *testkit.TestKit, useCount int64, rootk *testkit.TestKit) {
+ require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u5", Hostname: "localhost"}, sha1Password("password"), nil))
+ checkAuthUser(t, rootk, "u5", useCount, "N")
+}
+
+func changeAutoLockedLastChanged(tk *testkit.TestKit, ds, user string) {
+ SQL := "UPDATE `mysql`.`User` SET user_attributes=json_merge_patch(user_attributes, '{\"Password_locking\": {\"failed_login_attempts\": 3," +
+ "\"password_lock_time_days\": 3,\"auto_account_locked\": \"Y\",\"failed_login_count\": 3,\"auto_locked_last_changed\": \"%s\"}}') " +
+ "WHERE Host='localhost' and User='%s'"
+ d, _ := time.ParseDuration(ds)
+ changeTime := time.Now().Add(d).Format(time.UnixDate)
+ SQL = fmt.Sprintf(SQL, changeTime, user)
+ tk.MustExec(SQL)
+ domain.GetDomain(tk.Session()).NotifyUpdatePrivilege()
+}
+
+func checkUserUserAttributes(tk *testkit.TestKit, user, host, row string) {
+ sqlTemplate := "select JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_attempts')," +
+ "JSON_EXTRACT(user_attributes, '$.Password_locking.auto_account_locked')," +
+ "JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_count')," +
+ "JSON_EXTRACT(user_attributes, '$.Password_locking.password_lock_time_days')," +
+ "JSON_EXTRACT(user_attributes, '$.metadata')from mysql.user where user= %? and host = %?"
+ userAttributesSQL := new(strings.Builder)
+ sqlexec.MustFormatSQL(userAttributesSQL, sqlTemplate, user, host)
+ tk.MustQuery(userAttributesSQL.String()).Check(testkit.Rows(row))
+}
+
+func alterAndCheck(t *testing.T, tk *testkit.TestKit, sql string, user string, failedLoginAttempts, passwordLockTimeDays, failedLoginCount int64) {
+ tk.MustExec(sql)
+ userAttributesSQL := selectSQL(user)
+ resBuff := bytes.NewBufferString("")
+ rs := tk.MustQuery(userAttributesSQL)
+ for _, row := range rs.Rows() {
+ _, err := fmt.Fprintf(resBuff, "%s\n", row)
+ require.NoError(t, err)
+ }
+ err := checkUser(t, resBuff.String(), failedLoginAttempts, passwordLockTimeDays, failedLoginCount)
+ require.NoError(t, err)
+}
+
+func checkUser(t *testing.T, rs string, failedLoginAttempts, passwordLockTimeDays, failedLoginCount int64) error {
+ var ua []userAttributes
+ if err := json.Unmarshal([]byte(rs), &ua); err != nil {
+ return err
+ }
+ require.Equal(t, failedLoginAttempts, ua[0].PasswordLocking.FailedLoginAttempts)
+ require.Equal(t, passwordLockTimeDays, ua[0].PasswordLocking.PasswordLockTimeDays)
+ require.Equal(t, failedLoginCount, ua[0].PasswordLocking.FailedLoginCount)
+ return nil
+}
+
+func createAndCheck(tk *testkit.TestKit, sql, rsJSON, user string) {
+ tk.MustExec(sql)
+ sql = selectSQL(user)
+ tk.MustQuery(sql).Check(testkit.Rows(rsJSON))
+}
+
+func createAndCheckToErr(t *testing.T, tk *testkit.TestKit, sql, user string) {
+ tk.MustExecToErr(sql)
+ sql = selectSQL(user)
+ require.Equal(t, 0, len(tk.MustQuery(sql).Rows()))
+}
+
+func checkAuthUser(t *testing.T, tk *testkit.TestKit, user string, failedLoginCount int64, autoAccountLocked string) {
+ userAttributesSQL := selectSQL(user)
+ resBuff := bytes.NewBufferString("")
+ rs := tk.MustQuery(userAttributesSQL)
+ for _, row := range rs.Rows() {
+ _, err := fmt.Fprintf(resBuff, "%s\n", row)
+ require.NoError(t, err)
+ }
+ var ua []userAttributes
+ err := json.Unmarshal(resBuff.Bytes(), &ua)
+ require.NoError(t, err)
+ require.Equal(t, failedLoginCount, ua[0].PasswordLocking.FailedLoginCount)
+ require.Equal(t, autoAccountLocked, ua[0].PasswordLocking.AutoAccountLocked)
+}
+
+func selectSQL(user string) string {
+ userAttributesSQL := new(strings.Builder)
+ sqlexec.MustFormatSQL(userAttributesSQL, "SELECT user_attributes from mysql.user WHERE USER = %? AND HOST = 'localhost' for update", user)
+ return userAttributesSQL.String()
+}
+
+type passwordLocking struct {
+ FailedLoginAttempts int64 `json:"failed_login_attempts"`
+ PasswordLockTimeDays int64 `json:"password_lock_time_days"`
+ AutoAccountLocked string `json:"auto_account_locked"`
+ FailedLoginCount int64 `json:"failed_login_count"`
+ AutoLockedLastChanged string `json:"auto_locked_last_changed"`
+}
+
+type metadata struct {
+ Comment string `json:"comment"`
+}
+
+type userAttributes struct {
+ PasswordLocking passwordLocking `json:"Password_locking"`
+ Metadata metadata `json:"metadata"`
+}
diff --git a/executor/simpletest/simple_test.go b/executor/simpletest/simple_test.go
index 738a5808be6fa..06f7e72c030d5 100644
--- a/executor/simpletest/simple_test.go
+++ b/executor/simpletest/simple_test.go
@@ -747,7 +747,7 @@ func TestUser(t *testing.T) {
alterUserSQL = `ALTER USER 'test1'@'localhost' IDENTIFIED BY '222', 'test_not_exist'@'localhost' IDENTIFIED BY '111';`
tk.MustGetErrCode(alterUserSQL, mysql.ErrCannotUser)
result = tk.MustQuery(`SELECT authentication_string FROM mysql.User WHERE User="test1" and Host="localhost"`)
- result.Check(testkit.Rows(auth.EncodePassword("222")))
+ result.Check(testkit.Rows(auth.EncodePassword("111")))
alterUserSQL = `ALTER USER 'test4'@'localhost' IDENTIFIED WITH 'auth_socket';`
tk.MustExec(alterUserSQL)
result = tk.MustQuery(`SELECT plugin FROM mysql.User WHERE User="test4" and Host="localhost"`)
diff --git a/executor/slow_query.go b/executor/slow_query.go
index 395d8f4eba8ac..f7878839ae24b 100644
--- a/executor/slow_query.go
+++ b/executor/slow_query.go
@@ -52,6 +52,8 @@ import (
"golang.org/x/exp/slices"
)
+type signalsKey struct{}
+
// ParseSlowLogBatchSize is the batch size of slow-log lines for a worker to parse, exported for testing.
var ParseSlowLogBatchSize = 64
@@ -273,33 +275,7 @@ func (sc *slowLogChecker) isTimeValid(t types.Time) bool {
}
func getOneLine(reader *bufio.Reader) ([]byte, error) {
- var resByte []byte
- lineByte, isPrefix, err := reader.ReadLine()
- if isPrefix {
- // Need to read more data.
- resByte = make([]byte, len(lineByte), len(lineByte)*2)
- } else {
- resByte = make([]byte, len(lineByte))
- }
- // Use copy here to avoid shallow copy problem.
- copy(resByte, lineByte)
- if err != nil {
- return resByte, err
- }
-
- var tempLine []byte
- for isPrefix {
- tempLine, isPrefix, err = reader.ReadLine()
- resByte = append(resByte, tempLine...) // nozero
- // Use the max value of max_allowed_packet to check the single line length.
- if len(resByte) > int(variable.MaxOfMaxAllowedPacket) {
- return resByte, errors.Errorf("single line length exceeds limit: %v", variable.MaxOfMaxAllowedPacket)
- }
- if err != nil {
- return resByte, err
- }
- }
- return resByte, err
+ return util.ReadLine(reader, int(variable.MaxOfMaxAllowedPacket))
}
type offset struct {
@@ -474,7 +450,7 @@ func (e *slowQueryRetriever) parseSlowLog(ctx context.Context, sctx sessionctx.C
}
failpoint.Inject("mockReadSlowLogSlow", func(val failpoint.Value) {
if val.(bool) {
- signals := ctx.Value("signals").([]chan int)
+ signals := ctx.Value(signalsKey{}).([]chan int)
signals[0] <- 1
<-signals[1]
}
@@ -622,6 +598,9 @@ func (e *slowQueryRetriever) parseLog(ctx context.Context, sctx sessionctx.Conte
valid = e.setColumnValue(sctx, row, tz, variable.SlowLogHostStr, host, e.checker, fileLine)
} else if strings.HasPrefix(line, variable.SlowLogCopBackoffPrefix) {
valid = e.setColumnValue(sctx, row, tz, variable.SlowLogBackoffDetail, line, e.checker, fileLine)
+ } else if strings.HasPrefix(line, variable.SlowLogWarnings) {
+ line = line[len(variable.SlowLogWarnings+variable.SlowLogSpaceMarkStr):]
+ valid = e.setColumnValue(sctx, row, tz, variable.SlowLogWarnings, line, e.checker, fileLine)
} else {
fields, values := splitByColon(line)
for i := 0; i < len(fields); i++ {
@@ -781,7 +760,7 @@ func getColumnValueFactoryByName(sctx sessionctx.Context, colName string, column
}, nil
case variable.SlowLogUserStr, variable.SlowLogHostStr, execdetails.BackoffTypesStr, variable.SlowLogDBStr, variable.SlowLogIndexNamesStr, variable.SlowLogDigestStr,
variable.SlowLogStatsInfoStr, variable.SlowLogCopProcAddr, variable.SlowLogCopWaitAddr, variable.SlowLogPlanDigest,
- variable.SlowLogPrevStmt, variable.SlowLogQuerySQLStr:
+ variable.SlowLogPrevStmt, variable.SlowLogQuerySQLStr, variable.SlowLogWarnings:
return func(row []types.Datum, value string, tz *time.Location, checker *slowLogChecker) (valid bool, err error) {
row[columnIdx] = types.NewStringDatum(value)
return true, nil
diff --git a/executor/slow_query_sql_test.go b/executor/slow_query_sql_test.go
index a77e32e3bfb16..cdf13f2a9ccf9 100644
--- a/executor/slow_query_sql_test.go
+++ b/executor/slow_query_sql_test.go
@@ -23,6 +23,7 @@ import (
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/executor"
"github.com/pingcap/tidb/parser/auth"
+ "github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/testkit/testdata"
"github.com/pingcap/tidb/util/logutil"
@@ -46,8 +47,6 @@ func TestSlowQueryWithoutSlowLog(t *testing.T) {
}
func TestSlowQuerySensitiveQuery(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
originCfg := config.GetGlobalConfig()
newCfg := *originCfg
@@ -57,11 +56,15 @@ func TestSlowQuerySensitiveQuery(t *testing.T) {
newCfg.Log.SlowQueryFile = f.Name()
config.StoreGlobalConfig(&newCfg)
defer func() {
- tk.MustExec("set tidb_slow_log_threshold=300;")
config.StoreGlobalConfig(originCfg)
require.NoError(t, os.Remove(newCfg.Log.SlowQueryFile))
}()
require.NoError(t, logutil.InitLogger(newCfg.Log.ToLogConfig()))
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ defer func() {
+ tk.MustExec("set tidb_slow_log_threshold=300;")
+ }()
tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", f.Name()))
tk.MustExec("set tidb_slow_log_threshold=0;")
@@ -80,8 +83,6 @@ func TestSlowQuerySensitiveQuery(t *testing.T) {
}
func TestSlowQueryPrepared(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
originCfg := config.GetGlobalConfig()
newCfg := *originCfg
@@ -91,12 +92,16 @@ func TestSlowQueryPrepared(t *testing.T) {
newCfg.Log.SlowQueryFile = f.Name()
config.StoreGlobalConfig(&newCfg)
defer func() {
- tk.MustExec("set tidb_slow_log_threshold=300;")
- tk.MustExec("set tidb_redact_log=0;")
config.StoreGlobalConfig(originCfg)
require.NoError(t, os.Remove(newCfg.Log.SlowQueryFile))
}()
require.NoError(t, logutil.InitLogger(newCfg.Log.ToLogConfig()))
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ defer func() {
+ tk.MustExec("set tidb_slow_log_threshold=300;")
+ tk.MustExec("set tidb_redact_log=0;")
+ }()
tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", f.Name()))
tk.MustExec("set tidb_slow_log_threshold=0;")
@@ -116,8 +121,6 @@ func TestSlowQueryPrepared(t *testing.T) {
}
func TestLogSlowLogIndex(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
f, err := os.CreateTemp("", "tidb-slow-*.log")
require.NoError(t, err)
require.NoError(t, f.Close())
@@ -127,6 +130,8 @@ func TestLogSlowLogIndex(t *testing.T) {
conf.Log.SlowQueryFile = f.Name()
})
require.NoError(t, logutil.InitLogger(config.GetGlobalConfig().Log.ToLogConfig()))
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", f.Name()))
tk.MustExec("use test")
@@ -140,9 +145,6 @@ func TestLogSlowLogIndex(t *testing.T) {
}
func TestSlowQuery(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
-
f, err := os.CreateTemp("", "tidb-slow-*.log")
require.NoError(t, err)
_, err = f.WriteString(`
@@ -197,6 +199,8 @@ SELECT original_sql, bind_sql, default_db, status, create_time, update_time, cha
require.NoError(t, os.Remove(newCfg.Log.SlowQueryFile))
}()
require.NoError(t, logutil.InitLogger(newCfg.Log.ToLogConfig()))
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", f.Name()))
tk.MustQuery("select count(*) from `information_schema`.`slow_query` where time > '2020-10-16 20:08:13' and time < '2020-10-16 21:08:13'").Check(testkit.Rows("1"))
@@ -208,10 +212,6 @@ SELECT original_sql, bind_sql, default_db, status, create_time, update_time, cha
}
func TestIssue37066(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
- require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
-
originCfg := config.GetGlobalConfig()
newCfg := *originCfg
f, err := os.CreateTemp("", "tidb-slow-*.log")
@@ -224,6 +224,9 @@ func TestIssue37066(t *testing.T) {
require.NoError(t, os.Remove(newCfg.Log.SlowQueryFile))
}()
require.NoError(t, logutil.InitLogger(newCfg.Log.ToLogConfig()))
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", f.Name()))
tk.MustExec("set tidb_slow_log_threshold=0;")
defer func() {
@@ -281,3 +284,61 @@ func TestIssue37066(t *testing.T) {
}
}
}
+
+func TestWarningsInSlowQuery(t *testing.T) {
+ // Prepare the slow log
+ originCfg := config.GetGlobalConfig()
+ newCfg := *originCfg
+ f, err := os.CreateTemp("", "tidb-slow-*.log")
+ require.NoError(t, err)
+ newCfg.Log.SlowQueryFile = f.Name()
+ config.StoreGlobalConfig(&newCfg)
+ defer func() {
+ config.StoreGlobalConfig(originCfg)
+ require.NoError(t, f.Close())
+ require.NoError(t, os.Remove(newCfg.Log.SlowQueryFile))
+ }()
+ require.NoError(t, logutil.InitLogger(newCfg.Log.ToLogConfig()))
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", f.Name()))
+ tk.MustExec("set tidb_slow_log_threshold=0;")
+ defer func() {
+ tk.MustExec("set tidb_slow_log_threshold=300;")
+ }()
+
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, b int, c int, d int, e int, f int, g int, h set('11', '22', '33')," +
+ "primary key (a), unique key c_d_e (c, d, e), unique key f (f), unique key f_g (f, g), key g (g))")
+ tbl, err := dom.InfoSchema().TableByName(model.CIStr{O: "test", L: "test"}, model.CIStr{O: "t", L: "t"})
+ require.NoError(t, err)
+ tbl.Meta().TiFlashReplica = &model.TiFlashReplicaInfo{Count: 1, Available: true}
+
+ var input []string
+ var output []struct {
+ SQL string
+ Result string
+ }
+ slowQuerySuiteData.LoadTestCases(t, &input, &output)
+ for i, test := range input {
+ comment := fmt.Sprintf("case:%v sql:%s", i, test)
+ if len(test) < 6 || test[:6] != "select" {
+ tk.MustExec(test)
+ } else {
+ tk.MustQuery(test)
+ }
+ res := testdata.ConvertRowsToStrings(
+ tk.MustQuery("select warnings from information_schema.slow_query " +
+ `where query = "` + test + `;" ` +
+ "order by time desc limit 1").Rows(),
+ )
+ require.Lenf(t, res, 1, comment)
+
+ testdata.OnRecord(func() {
+ output[i].SQL = test
+ output[i].Result = res[0]
+ })
+ require.Equal(t, output[i].Result, res[0])
+ }
+}
diff --git a/executor/slow_query_test.go b/executor/slow_query_test.go
index 32412d2f1ac70..fe2a5b68a329a 100644
--- a/executor/slow_query_test.go
+++ b/executor/slow_query_test.go
@@ -55,7 +55,7 @@ func parseLog(retriever *slowQueryRetriever, sctx sessionctx.Context, reader *bu
}
func newSlowQueryRetriever() (*slowQueryRetriever, error) {
- newISBuilder, err := infoschema.NewBuilder(nil, nil).InitWithDBInfos(nil, nil, 0)
+ newISBuilder, err := infoschema.NewBuilder(nil, nil).InitWithDBInfos(nil, nil, nil, 0)
if err != nil {
return nil, err
}
@@ -160,7 +160,7 @@ select * from t;`
expectRecordString := `2019-04-28 15:24:04.309074,` +
`405888132465033227,root,localhost,0,57,0.12,0.216905,` +
`0,0,0,0,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0.38,0.021,0,0,0,1,637,0,10,10,10,10,100,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,` +
- `0.1,0.2,0.03,127.0.0.1:20160,0.05,0.6,0.8,0.0.0.0:20160,70724,65536,0,0,0,0,0,` +
+ `0.1,0.2,0.03,127.0.0.1:20160,0.05,0.6,0.8,0.0.0.0:20160,70724,65536,0,0,0,0,0,,` +
`Cop_backoff_regionMiss_total_times: 200 Cop_backoff_regionMiss_total_time: 0.2 Cop_backoff_regionMiss_max_time: 0.2 Cop_backoff_regionMiss_max_addr: 127.0.0.1 Cop_backoff_regionMiss_avg_time: 0.2 Cop_backoff_regionMiss_p90_time: 0.2 Cop_backoff_rpcPD_total_times: 200 Cop_backoff_rpcPD_total_time: 0.2 Cop_backoff_rpcPD_max_time: 0.2 Cop_backoff_rpcPD_max_addr: 127.0.0.1 Cop_backoff_rpcPD_avg_time: 0.2 Cop_backoff_rpcPD_p90_time: 0.2 Cop_backoff_rpcTiKV_total_times: 200 Cop_backoff_rpcTiKV_total_time: 0.2 Cop_backoff_rpcTiKV_max_time: 0.2 Cop_backoff_rpcTiKV_max_addr: 127.0.0.1 Cop_backoff_rpcTiKV_avg_time: 0.2 Cop_backoff_rpcTiKV_p90_time: 0.2,` +
`0,0,1,0,1,1,0,,60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4,` +
`,update t set i = 1;,select * from t;`
@@ -183,7 +183,7 @@ select * from t;`
expectRecordString = `2019-04-28 15:24:04.309074,` +
`405888132465033227,root,localhost,0,57,0.12,0.216905,` +
`0,0,0,0,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0.38,0.021,0,0,0,1,637,0,10,10,10,10,100,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,` +
- `0.1,0.2,0.03,127.0.0.1:20160,0.05,0.6,0.8,0.0.0.0:20160,70724,65536,0,0,0,0,0,` +
+ `0.1,0.2,0.03,127.0.0.1:20160,0.05,0.6,0.8,0.0.0.0:20160,70724,65536,0,0,0,0,0,,` +
`Cop_backoff_regionMiss_total_times: 200 Cop_backoff_regionMiss_total_time: 0.2 Cop_backoff_regionMiss_max_time: 0.2 Cop_backoff_regionMiss_max_addr: 127.0.0.1 Cop_backoff_regionMiss_avg_time: 0.2 Cop_backoff_regionMiss_p90_time: 0.2 Cop_backoff_rpcPD_total_times: 200 Cop_backoff_rpcPD_total_time: 0.2 Cop_backoff_rpcPD_max_time: 0.2 Cop_backoff_rpcPD_max_addr: 127.0.0.1 Cop_backoff_rpcPD_avg_time: 0.2 Cop_backoff_rpcPD_p90_time: 0.2 Cop_backoff_rpcTiKV_total_times: 200 Cop_backoff_rpcTiKV_total_time: 0.2 Cop_backoff_rpcTiKV_max_time: 0.2 Cop_backoff_rpcTiKV_max_addr: 127.0.0.1 Cop_backoff_rpcTiKV_avg_time: 0.2 Cop_backoff_rpcTiKV_p90_time: 0.2,` +
`0,0,1,0,1,1,0,,60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4,` +
`,update t set i = 1;,select * from t;`
@@ -666,7 +666,7 @@ select * from t;`
retriever, err := newSlowQueryRetriever()
require.NoError(t, err)
var signal1, signal2 = make(chan int, 1), make(chan int, 1)
- ctx := context.WithValue(context.Background(), "signals", []chan int{signal1, signal2})
+ ctx := context.WithValue(context.Background(), signalsKey{}, []chan int{signal1, signal2})
ctx, cancel := context.WithCancel(ctx)
err = failpoint.Enable("github.com/pingcap/tidb/executor/mockReadSlowLogSlow", "return(true)")
require.NoError(t, err)
diff --git a/executor/stmtsummary.go b/executor/stmtsummary.go
new file mode 100644
index 0000000000000..de7cbef6e1aa6
--- /dev/null
+++ b/executor/stmtsummary.go
@@ -0,0 +1,408 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package executor
+
+import (
+ "context"
+
+ "github.com/pingcap/tidb/config"
+ "github.com/pingcap/tidb/infoschema"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/parser/mysql"
+ plannercore "github.com/pingcap/tidb/planner/core"
+ "github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/types"
+ "github.com/pingcap/tidb/util/execdetails"
+ "github.com/pingcap/tidb/util/set"
+ "github.com/pingcap/tidb/util/stmtsummary"
+ stmtsummaryv2 "github.com/pingcap/tidb/util/stmtsummary/v2"
+)
+
+const (
+ defaultRetrieveCount = 1024
+)
+
+func buildStmtSummaryRetriever(
+ ctx sessionctx.Context,
+ table *model.TableInfo,
+ columns []*model.ColumnInfo,
+ extractor *plannercore.StatementsSummaryExtractor,
+) memTableRetriever {
+ if extractor == nil {
+ extractor = &plannercore.StatementsSummaryExtractor{}
+ }
+ if extractor.Digests.Empty() {
+ extractor.Digests = nil
+ }
+
+ var retriever memTableRetriever
+ if extractor.SkipRequest {
+ retriever = &dummyRetriever{}
+ } else if config.GetGlobalConfig().Instance.StmtSummaryEnablePersistent {
+ retriever = &stmtSummaryRetrieverV2{
+ stmtSummary: stmtsummaryv2.GlobalStmtSummary,
+ table: table,
+ columns: columns,
+ digests: extractor.Digests,
+ timeRanges: buildTimeRanges(extractor.CoarseTimeRange),
+ }
+ } else {
+ retriever = &stmtSummaryRetriever{
+ table: table,
+ columns: columns,
+ digests: extractor.Digests,
+ }
+ }
+
+ return retriever
+}
+
+type dummyRetriever struct {
+ dummyCloser
+}
+
+func (e *dummyRetriever) retrieve(_ context.Context, _ sessionctx.Context) ([][]types.Datum, error) {
+ return nil, nil
+}
+
+// stmtSummaryRetriever is used to retrieve statements summary.
+type stmtSummaryRetriever struct {
+ table *model.TableInfo
+ columns []*model.ColumnInfo
+ digests set.StringSet
+
+ // lazily initialized
+ rowsReader *rowsReader
+}
+
+func (e *stmtSummaryRetriever) retrieve(ctx context.Context, sctx sessionctx.Context) ([][]types.Datum, error) {
+ if err := e.ensureRowsReader(sctx); err != nil {
+ return nil, err
+ }
+ return e.rowsReader.read(defaultRetrieveCount)
+}
+
+func (e *stmtSummaryRetriever) close() error {
+ if e.rowsReader != nil {
+ return e.rowsReader.close()
+ }
+ return nil
+}
+
+func (e *stmtSummaryRetriever) getRuntimeStats() execdetails.RuntimeStats {
+ return nil
+}
+
+func (e *stmtSummaryRetriever) ensureRowsReader(sctx sessionctx.Context) error {
+ if e.rowsReader != nil {
+ return nil
+ }
+
+ var err error
+ if isEvictedTable(e.table.Name.O) {
+ e.rowsReader, err = e.initEvictedRowsReader(sctx)
+ } else {
+ e.rowsReader, err = e.initSummaryRowsReader(sctx)
+ }
+
+ return err
+}
+
+func (e *stmtSummaryRetriever) initEvictedRowsReader(sctx sessionctx.Context) (*rowsReader, error) {
+ if err := checkPrivilege(sctx); err != nil {
+ return nil, err
+ }
+
+ rows := stmtsummary.StmtSummaryByDigestMap.ToEvictedCountDatum()
+ if !isClusterTable(e.table.Name.O) {
+ // rows are full-columned, so we need to adjust them to the required columns.
+ return newSimpleRowsReader(adjustColumns(rows, e.columns, e.table)), nil
+ }
+
+ // Additional column `INSTANCE` for cluster table
+ rows, err := infoschema.AppendHostInfoToRows(sctx, rows)
+ if err != nil {
+ return nil, err
+ }
+ // rows are full-columned, so we need to adjust them to the required columns.
+ return newSimpleRowsReader(adjustColumns(rows, e.columns, e.table)), nil
+}
+
+func (e *stmtSummaryRetriever) initSummaryRowsReader(sctx sessionctx.Context) (*rowsReader, error) {
+ vars := sctx.GetSessionVars()
+ user := vars.User
+ tz := vars.StmtCtx.TimeZone
+ columns := e.columns
+ priv := hasPriv(sctx, mysql.ProcessPriv)
+ instanceAddr, err := clusterTableInstanceAddr(sctx, e.table.Name.O)
+ if err != nil {
+ return nil, err
+ }
+
+ reader := stmtsummary.NewStmtSummaryReader(user, priv, columns, instanceAddr, tz)
+ if e.digests != nil {
+ // set checker to filter out statements not matching the given digests
+ checker := stmtsummary.NewStmtSummaryChecker(e.digests)
+ reader.SetChecker(checker)
+ }
+
+ var rows [][]types.Datum
+ if isCurrentTable(e.table.Name.O) {
+ rows = reader.GetStmtSummaryCurrentRows()
+ }
+ if isHistoryTable(e.table.Name.O) {
+ rows = reader.GetStmtSummaryHistoryRows()
+ }
+ return newSimpleRowsReader(rows), nil
+}
+
+// stmtSummaryRetriever is used to retrieve statements summary when
+// config.GetGlobalConfig().Instance.StmtSummaryEnablePersistent is true
+type stmtSummaryRetrieverV2 struct {
+ stmtSummary *stmtsummaryv2.StmtSummary
+ table *model.TableInfo
+ columns []*model.ColumnInfo
+ digests set.StringSet
+ timeRanges []*stmtsummaryv2.StmtTimeRange
+
+ // lazily initialized
+ rowsReader *rowsReader
+}
+
+func (r *stmtSummaryRetrieverV2) retrieve(ctx context.Context, sctx sessionctx.Context) ([][]types.Datum, error) {
+ if err := r.ensureRowsReader(ctx, sctx); err != nil {
+ return nil, err
+ }
+ return r.rowsReader.read(defaultRetrieveCount)
+}
+
+func (r *stmtSummaryRetrieverV2) close() error {
+ if r.rowsReader != nil {
+ return r.rowsReader.close()
+ }
+ return nil
+}
+
+func (r *stmtSummaryRetrieverV2) getRuntimeStats() execdetails.RuntimeStats {
+ return nil
+}
+
+func (r *stmtSummaryRetrieverV2) ensureRowsReader(ctx context.Context, sctx sessionctx.Context) error {
+ if r.rowsReader != nil {
+ return nil
+ }
+
+ var err error
+ if isEvictedTable(r.table.Name.O) {
+ r.rowsReader, err = r.initEvictedRowsReader(sctx)
+ } else {
+ r.rowsReader, err = r.initSummaryRowsReader(ctx, sctx)
+ }
+
+ return err
+}
+
+func (r *stmtSummaryRetrieverV2) initEvictedRowsReader(sctx sessionctx.Context) (*rowsReader, error) {
+ if err := checkPrivilege(sctx); err != nil {
+ return nil, err
+ }
+
+ var rows [][]types.Datum
+
+ row := r.stmtSummary.Evicted()
+ if row != nil {
+ rows = append(rows, row)
+ }
+ if !isClusterTable(r.table.Name.O) {
+ // rows are full-columned, so we need to adjust them to the required columns.
+ return newSimpleRowsReader(adjustColumns(rows, r.columns, r.table)), nil
+ }
+
+ // Additional column `INSTANCE` for cluster table
+ rows, err := infoschema.AppendHostInfoToRows(sctx, rows)
+ if err != nil {
+ return nil, err
+ }
+ // rows are full-columned, so we need to adjust them to the required columns.
+ return newSimpleRowsReader(adjustColumns(rows, r.columns, r.table)), nil
+}
+
+func (r *stmtSummaryRetrieverV2) initSummaryRowsReader(ctx context.Context, sctx sessionctx.Context) (*rowsReader, error) {
+ vars := sctx.GetSessionVars()
+ user := vars.User
+ tz := vars.StmtCtx.TimeZone
+ stmtSummary := r.stmtSummary
+ columns := r.columns
+ timeRanges := r.timeRanges
+ digests := r.digests
+ priv := hasPriv(sctx, mysql.ProcessPriv)
+ instanceAddr, err := clusterTableInstanceAddr(sctx, r.table.Name.O)
+ if err != nil {
+ return nil, err
+ }
+
+ mem := stmtsummaryv2.NewMemReader(stmtSummary, columns, instanceAddr, tz, user, priv, digests, timeRanges)
+ memRows := mem.Rows()
+
+ var rowsReader *rowsReader
+ if isCurrentTable(r.table.Name.O) {
+ rowsReader = newSimpleRowsReader(memRows)
+ }
+ if isHistoryTable(r.table.Name.O) {
+ // history table should return all rows including mem and disk
+ concurrent := sctx.GetSessionVars().Concurrency.DistSQLScanConcurrency()
+ history, err := stmtsummaryv2.NewHistoryReader(ctx, columns, instanceAddr, tz, user, priv, digests, timeRanges, concurrent)
+ if err != nil {
+ return nil, err
+ }
+ rowsReader = newRowsReader(memRows, history)
+ }
+
+ return rowsReader, nil
+}
+
+type rowsPuller interface {
+ Closeable
+ Rows() ([][]types.Datum, error)
+}
+
+type rowsReader struct {
+ puller rowsPuller
+ rows [][]types.Datum
+}
+
+func newSimpleRowsReader(rows [][]types.Datum) *rowsReader {
+ return &rowsReader{rows: rows}
+}
+
+func newRowsReader(rows [][]types.Datum, puller rowsPuller) *rowsReader {
+ return &rowsReader{puller: puller, rows: rows}
+}
+
+func (r *rowsReader) read(maxCount int) ([][]types.Datum, error) {
+ if err := r.pull(); err != nil {
+ return nil, err
+ }
+
+ if maxCount >= len(r.rows) {
+ ret := r.rows
+ r.rows = nil
+ return ret, nil
+ }
+ ret := r.rows[:maxCount]
+ r.rows = r.rows[maxCount:]
+ return ret, nil
+}
+
+func (r *rowsReader) pull() error {
+ if r.puller == nil {
+ return nil
+ }
+ // there are remaining rows
+ if len(r.rows) > 0 {
+ return nil
+ }
+
+ rows, err := r.puller.Rows()
+ if err != nil {
+ return err
+ }
+ // pulled new rows from the puller
+ if len(rows) != 0 {
+ r.rows = rows
+ return nil
+ }
+
+ // reach the end of the puller
+ err = r.puller.Close()
+ if err != nil {
+ return err
+ }
+ r.puller = nil
+ return nil
+}
+
+func (r *rowsReader) close() error {
+ if r.puller != nil {
+ return r.puller.Close()
+ }
+ return nil
+}
+
+func isClusterTable(originalTableName string) bool {
+ switch originalTableName {
+ case infoschema.ClusterTableStatementsSummary,
+ infoschema.ClusterTableStatementsSummaryHistory,
+ infoschema.ClusterTableStatementsSummaryEvicted:
+ return true
+ }
+
+ return false
+}
+
+func isCurrentTable(originalTableName string) bool {
+ switch originalTableName {
+ case infoschema.TableStatementsSummary,
+ infoschema.ClusterTableStatementsSummary:
+ return true
+ }
+
+ return false
+}
+
+func isHistoryTable(originalTableName string) bool {
+ switch originalTableName {
+ case infoschema.TableStatementsSummaryHistory,
+ infoschema.ClusterTableStatementsSummaryHistory:
+ return true
+ }
+
+ return false
+}
+
+func isEvictedTable(originalTableName string) bool {
+ switch originalTableName {
+ case infoschema.TableStatementsSummaryEvicted,
+ infoschema.ClusterTableStatementsSummaryEvicted:
+ return true
+ }
+
+ return false
+}
+
+func checkPrivilege(sctx sessionctx.Context) error {
+ if !hasPriv(sctx, mysql.ProcessPriv) {
+ return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS")
+ }
+ return nil
+}
+
+func clusterTableInstanceAddr(sctx sessionctx.Context, originalTableName string) (string, error) {
+ if isClusterTable(originalTableName) {
+ return infoschema.GetInstanceAddr(sctx)
+ }
+ return "", nil
+}
+
+func buildTimeRanges(tr *plannercore.TimeRange) []*stmtsummaryv2.StmtTimeRange {
+ if tr == nil {
+ return nil
+ }
+
+ return []*stmtsummaryv2.StmtTimeRange{{
+ Begin: tr.StartTime.Unix(),
+ End: tr.EndTime.Unix(),
+ }}
+}
diff --git a/executor/stmtsummary_test.go b/executor/stmtsummary_test.go
new file mode 100644
index 0000000000000..a09966c5f3d59
--- /dev/null
+++ b/executor/stmtsummary_test.go
@@ -0,0 +1,182 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package executor
+
+import (
+ "context"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/pingcap/tidb/infoschema"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/types"
+ "github.com/pingcap/tidb/util"
+ "github.com/pingcap/tidb/util/mock"
+ stmtsummaryv2 "github.com/pingcap/tidb/util/stmtsummary/v2"
+ "github.com/stretchr/testify/require"
+)
+
+func TestStmtSummaryRetriverV2_TableStatementsSummary(t *testing.T) {
+ infoSchemaBuilder, err := infoschema.NewBuilder(nil, nil).InitWithDBInfos(nil, nil, nil, 0)
+ require.NoError(t, err)
+ infoSchema := infoSchemaBuilder.Build()
+ table, err := infoSchema.TableByName(util.InformationSchemaName, model.NewCIStr(infoschema.TableStatementsSummary))
+ require.NoError(t, err)
+ columns := table.Meta().Columns
+
+ stmtSummary := stmtsummaryv2.NewStmtSummary4Test(1000)
+ defer stmtSummary.Close()
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest1"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest1"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest2"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest2"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest3"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest3"))
+
+ retriever := stmtSummaryRetrieverV2{
+ stmtSummary: stmtSummary,
+ table: table.Meta(),
+ columns: columns,
+ }
+ defer func() {
+ require.NoError(t, retriever.close())
+ }()
+
+ ctx := context.Background()
+ sctx := mock.NewContext()
+ sctx.GetSessionVars().TimeZone, _ = time.LoadLocation("Asia/Shanghai")
+
+ var results [][]types.Datum
+ for {
+ rows, err := retriever.retrieve(ctx, sctx)
+ require.NoError(t, err)
+ if len(rows) == 0 {
+ break
+ }
+ results = append(results, rows...)
+ }
+ require.Len(t, results, 3)
+}
+
+func TestStmtSummaryRetriverV2_TableStatementsSummaryEvicted(t *testing.T) {
+ infoSchemaBuilder, err := infoschema.NewBuilder(nil, nil).InitWithDBInfos(nil, nil, nil, 0)
+ require.NoError(t, err)
+ infoSchema := infoSchemaBuilder.Build()
+ table, err := infoSchema.TableByName(util.InformationSchemaName, model.NewCIStr(infoschema.TableStatementsSummaryEvicted))
+ require.NoError(t, err)
+ columns := table.Meta().Columns
+
+ stmtSummary := stmtsummaryv2.NewStmtSummary4Test(1)
+ defer stmtSummary.Close()
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest1"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest1"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest2"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest2"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest3"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest3"))
+
+ retriever := stmtSummaryRetrieverV2{
+ stmtSummary: stmtSummary,
+ table: table.Meta(),
+ columns: columns,
+ }
+ defer func() {
+ require.NoError(t, retriever.close())
+ }()
+
+ ctx := context.Background()
+ sctx := mock.NewContext()
+ sctx.GetSessionVars().TimeZone, _ = time.LoadLocation("Asia/Shanghai")
+
+ var results [][]types.Datum
+ for {
+ rows, err := retriever.retrieve(ctx, sctx)
+ require.NoError(t, err)
+ if len(rows) == 0 {
+ break
+ }
+ results = append(results, rows...)
+ }
+ require.Len(t, results, 1)
+ require.Equal(t, int64(2), results[0][2].GetInt64())
+}
+
+func TestStmtSummaryRetriverV2_TableStatementsSummaryHistory(t *testing.T) {
+ filename1 := "tidb-statements-2022-12-27T16-21-20.245.log"
+ filename2 := "tidb-statements.log"
+
+ file, err := os.Create(filename1)
+ require.NoError(t, err)
+ defer func() {
+ require.NoError(t, os.Remove(filename1))
+ }()
+ _, err = file.WriteString("{\"begin\":1672128520,\"end\":1672128530,\"digest\":\"digest1\",\"exec_count\":1}\n")
+ require.NoError(t, err)
+ _, err = file.WriteString("{\"begin\":1672129270,\"end\":1672129280,\"digest\":\"digest2\",\"exec_count\":2}\n")
+ require.NoError(t, err)
+ require.NoError(t, file.Close())
+
+ file, err = os.Create(filename2)
+ require.NoError(t, err)
+ defer func() {
+ require.NoError(t, os.Remove(filename2))
+ }()
+ _, err = file.WriteString("{\"begin\":1672129270,\"end\":1672129280,\"digest\":\"digest3\",\"exec_count\":3}\n")
+ require.NoError(t, err)
+ _, err = file.WriteString("{\"begin\":1672129380,\"end\":1672129390,\"digest\":\"digest4\",\"exec_count\":4}\n")
+ require.NoError(t, err)
+ require.NoError(t, file.Close())
+
+ stmtSummary := stmtsummaryv2.NewStmtSummary4Test(2)
+ defer stmtSummary.Close()
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest1"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest1"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest2"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest2"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest3"))
+ stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest3"))
+
+ infoSchemaBuilder, err := infoschema.NewBuilder(nil, nil).InitWithDBInfos(nil, nil, nil, 0)
+ require.NoError(t, err)
+ infoSchema := infoSchemaBuilder.Build()
+ table, err := infoSchema.TableByName(util.InformationSchemaName, model.NewCIStr(infoschema.TableStatementsSummaryHistory))
+ require.NoError(t, err)
+ columns := table.Meta().Columns
+
+ retriever := stmtSummaryRetrieverV2{
+ stmtSummary: stmtSummary,
+ table: table.Meta(),
+ columns: columns,
+ }
+ defer func() {
+ require.NoError(t, retriever.close())
+ }()
+
+ ctx := context.Background()
+ sctx := mock.NewContext()
+ sctx.GetSessionVars().TimeZone, _ = time.LoadLocation("Asia/Shanghai")
+
+ var results [][]types.Datum
+ for {
+ rows, err := retriever.retrieve(ctx, sctx)
+ require.NoError(t, err)
+ if len(rows) == 0 {
+ break
+ }
+ results = append(results, rows...)
+ }
+ require.Len(t, results, 7)
+}
diff --git a/executor/table_reader.go b/executor/table_reader.go
index 500b3ba3ab771..3e29dfe27b053 100644
--- a/executor/table_reader.go
+++ b/executor/table_reader.go
@@ -19,7 +19,6 @@ import (
"context"
"time"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/distsql"
"github.com/pingcap/tidb/domain"
@@ -39,6 +38,7 @@ import (
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/ranger"
"github.com/pingcap/tidb/util/stringutil"
+ "github.com/pingcap/tidb/util/tracing"
"github.com/pingcap/tipb/go-tipb"
"golang.org/x/exp/slices"
)
@@ -61,7 +61,7 @@ func (sr selectResultHook) SelectResult(ctx context.Context, sctx sessionctx.Con
}
type kvRangeBuilder interface {
- buildKeyRange(ranges []*ranger.Range) ([]kv.KeyRange, error)
+ buildKeyRange(ranges []*ranger.Range) ([][]kv.KeyRange, error)
buildKeyRangeSeparately(ranges []*ranger.Range) ([]int64, [][]kv.KeyRange, error)
}
@@ -135,11 +135,8 @@ func (e *TableReaderExecutor) setDummy() {
// Open initializes necessary variables for using this executor.
func (e *TableReaderExecutor) Open(ctx context.Context) error {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("TableReaderExecutor.Open", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "TableReaderExecutor.Open")
+ defer r.End()
failpoint.Inject("mockSleepInTableReaderNext", func(v failpoint.Value) {
ms := v.(int)
time.Sleep(time.Millisecond * time.Duration(ms))
@@ -205,13 +202,13 @@ func (e *TableReaderExecutor) Open(ctx context.Context) error {
if err != nil {
return err
}
- e.kvRanges = append(e.kvRanges, kvReq.KeyRanges...)
+ e.kvRanges = kvReq.KeyRanges.AppendSelfTo(e.kvRanges)
if len(secondPartRanges) != 0 {
kvReq, err = e.buildKVReq(ctx, secondPartRanges)
if err != nil {
return err
}
- e.kvRanges = append(e.kvRanges, kvReq.KeyRanges...)
+ e.kvRanges = kvReq.KeyRanges.AppendSelfTo(e.kvRanges)
}
return nil
}
@@ -256,7 +253,7 @@ func (e *TableReaderExecutor) Next(ctx context.Context, req *chunk.Chunk) error
return err
}
- err := FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, e.schema, e.columns, e.ctx, req)
+ err := table.FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, e.schema.Columns, e.columns, e.ctx, req)
if err != nil {
return err
}
@@ -314,10 +311,10 @@ func (e *TableReaderExecutor) buildResp(ctx context.Context, ranges []*ranger.Ra
if err != nil {
return nil, err
}
- slices.SortFunc(kvReq.KeyRanges, func(i, j kv.KeyRange) bool {
+ kvReq.KeyRanges.SortByFunc(func(i, j kv.KeyRange) bool {
return bytes.Compare(i.StartKey, j.StartKey) < 0
})
- e.kvRanges = append(e.kvRanges, kvReq.KeyRanges...)
+ e.kvRanges = kvReq.KeyRanges.AppendSelfTo(e.kvRanges)
result, err := e.SelectResult(ctx, e.ctx, kvReq, retTypes(e), e.feedback, getPhysicalPlanIDs(e.plans), e.id)
if err != nil {
@@ -409,7 +406,7 @@ func (e *TableReaderExecutor) buildKVReq(ctx context.Context, ranges []*ranger.R
if err != nil {
return nil, err
}
- reqBuilder = builder.SetKeyRanges(kvRange)
+ reqBuilder = builder.SetPartitionKeyRanges(kvRange)
} else {
reqBuilder = builder.SetHandleRanges(e.ctx.GetSessionVars().StmtCtx, getPhysicalTableID(e.table), e.table.Meta() != nil && e.table.Meta().IsCommonHandle, ranges, e.feedback)
}
diff --git a/executor/testdata/executor_suite_out.json b/executor/testdata/executor_suite_out.json
index c06ee935e8b37..e0fbfecb07095 100644
--- a/executor/testdata/executor_suite_out.json
+++ b/executor/testdata/executor_suite_out.json
@@ -674,7 +674,7 @@
{
"SQL": "select count(*) from t as t1 inner join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL",
"Plan": [
- "HashAgg 1.00 root funcs:count(1)->Column#7",
+ "StreamAgg 1.00 root funcs:count(1)->Column#7",
"└─MergeJoin 0.00 root inner join, left key:test.t.c1, right key:test.t.c1",
" ├─TableDual(Build) 0.00 root rows:0",
" └─TableDual(Probe) 0.00 root rows:0"
@@ -704,7 +704,7 @@
{
"SQL": "select count(*) from t as t1 left join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL",
"Plan": [
- "HashAgg 1.00 root funcs:count(1)->Column#7",
+ "StreamAgg 1.00 root funcs:count(1)->Column#7",
"└─MergeJoin 0.00 root left outer join, left key:test.t.c1, right key:test.t.c1",
" ├─TableDual(Build) 0.00 root rows:0",
" └─TableDual(Probe) 0.00 root rows:0"
diff --git a/executor/testdata/slow_query_suite_in.json b/executor/testdata/slow_query_suite_in.json
new file mode 100644
index 0000000000000..4b2c95b6aa378
--- /dev/null
+++ b/executor/testdata/slow_query_suite_in.json
@@ -0,0 +1,19 @@
+[
+ {
+ "name": "TestWarningsInSlowQuery",
+ "cases": [
+ "insert into t(a) value (1)",
+ "select /*+ READ_FROM_STORAGE(TIKV[t]) */ a/0 from t",
+ "create binding for select * from t where c = 10 using select * from t use index (c_d_e) where c = 10",
+ "select * from t where c = 10",
+ "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where a > 1 order by f",
+ "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where f > 1",
+ "select /*+ READ_FROM_STORAGE(TIKV[t]) */ f from t where f > 1",
+ "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where f > 3 and g = 5",
+ "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where g = 5 order by f",
+ "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where d = 3 order by c, e",
+ "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where h = '11,22'",
+ "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where a > rand()*100"
+ ]
+ }
+]
diff --git a/executor/testdata/slow_query_suite_out.json b/executor/testdata/slow_query_suite_out.json
new file mode 100644
index 0000000000000..adf6606709859
--- /dev/null
+++ b/executor/testdata/slow_query_suite_out.json
@@ -0,0 +1,55 @@
+[
+ {
+ "Name": "TestWarningsInSlowQuery",
+ "Cases": [
+ {
+ "SQL": "insert into t(a) value (1)",
+ "Result": ""
+ },
+ {
+ "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ a/0 from t",
+ "Result": "[{\"Level\":\"Warning\",\"Message\":\"Division by 0\"}]"
+ },
+ {
+ "SQL": "create binding for select * from t where c = 10 using select * from t use index (c_d_e) where c = 10",
+ "Result": ""
+ },
+ {
+ "SQL": "select * from t where c = 10",
+ "Result": "[{\"Level\":\"Note\",\"Message\":\"Using the bindSQL: SELECT * FROM `test`.`t` USE INDEX (`c_d_e`) WHERE `c` = 10\",\"IsExtra\":true}]"
+ },
+ {
+ "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where a > 1 order by f",
+ "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash),f,f_g] remain after pruning paths for t given Prop{SortItems: [{test.t.f asc}], TaskTp: rootTask}\",\"IsExtra\":true}]"
+ },
+ {
+ "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where f > 1",
+ "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash),f,f_g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
+ },
+ {
+ "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ f from t where f > 1",
+ "Result": "[{\"Level\":\"Note\",\"Message\":\"[t(tiflash),f,f_g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
+ },
+ {
+ "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where f > 3 and g = 5",
+ "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash),f_g,g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
+ },
+ {
+ "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where g = 5 order by f",
+ "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash),g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash),f_g,g] remain after pruning paths for t given Prop{SortItems: [{test.t.f asc}], TaskTp: rootTask}\",\"IsExtra\":true}]"
+ },
+ {
+ "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where d = 3 order by c, e",
+ "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash),c_d_e] remain after pruning paths for t given Prop{SortItems: [{test.t.c asc} {test.t.e asc}], TaskTp: rootTask}\",\"IsExtra\":true}]"
+ },
+ {
+ "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where h = '11,22'",
+ "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
+ },
+ {
+ "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where a > rand()*100",
+ "Result": "[{\"Level\":\"Warning\",\"Message\":\"Scalar function 'rand'(signature: Rand, return type: double) is not supported to push down to storage layer now.\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Scalar function 'rand'(signature: Rand, return type: double) is not supported to push down to tiflash now.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
+ }
+ ]
+ }
+]
diff --git a/executor/tiflashtest/BUILD.bazel b/executor/tiflashtest/BUILD.bazel
index 5223fa79cc2d9..0e6244f1b2f90 100644
--- a/executor/tiflashtest/BUILD.bazel
+++ b/executor/tiflashtest/BUILD.bazel
@@ -14,13 +14,16 @@ go_test(
"//config",
"//domain",
"//executor",
+ "//kv",
"//meta/autoid",
"//parser/terror",
+ "//planner/core",
"//store/mockstore",
"//store/mockstore/unistore",
"//testkit",
"//testkit/external",
"//util/israce",
+ "//util/tiflashcompute",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/metapb",
diff --git a/executor/tiflashtest/tiflash_test.go b/executor/tiflashtest/tiflash_test.go
index becdb2ec1956c..e703d66ed33cc 100644
--- a/executor/tiflashtest/tiflash_test.go
+++ b/executor/tiflashtest/tiflash_test.go
@@ -27,14 +27,18 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/metapb"
+ "github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/executor"
+ "github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/terror"
+ plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/store/mockstore"
"github.com/pingcap/tidb/store/mockstore/unistore"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/testkit/external"
"github.com/pingcap/tidb/util/israce"
+ "github.com/pingcap/tidb/util/tiflashcompute"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/testutils"
)
@@ -266,14 +270,9 @@ func TestMppExecution(t *testing.T) {
tk.MustExec("begin")
tk.MustQuery("select count(*) from ( select * from t2 group by a, b) A group by A.b").Check(testkit.Rows("3"))
tk.MustQuery("select count(*) from t1 where t1.a+100 > ( select count(*) from t2 where t1.a=t2.a and t1.b=t2.b) group by t1.b").Check(testkit.Rows("4"))
- txn, err := tk.Session().Txn(true)
- require.NoError(t, err)
- ts := txn.StartTS()
- taskID := tk.Session().GetSessionVars().AllocMPPTaskID(ts)
- require.Equal(t, int64(6), taskID)
- tk.MustExec("commit")
- taskID = tk.Session().GetSessionVars().AllocMPPTaskID(ts + 1)
+ taskID := plannercore.AllocMPPTaskID(tk.Session())
require.Equal(t, int64(1), taskID)
+ tk.MustExec("commit")
failpoint.Enable("github.com/pingcap/tidb/executor/checkTotalMPPTasks", `return(3)`)
// all the data is related to one store, so there are three tasks.
@@ -462,6 +461,7 @@ func TestPartitionTable(t *testing.T) {
store := testkit.CreateMockStore(t, withMockTiFlash(2))
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
+ tk.MustExec("set tidb_cost_model_version=1")
tk.MustExec("drop table if exists t")
tk.MustExec("drop table if exists t1")
tk.MustExec("drop table if exists t2")
@@ -630,10 +630,41 @@ func TestDispatchTaskRetry(t *testing.T) {
require.NoError(t, err)
tk.MustExec("set @@session.tidb_enforce_mpp=ON")
require.Nil(t, failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/mppDispatchTimeout", "3*return(true)"))
- tk.MustQuery("select count(*) from t").Check(testkit.Rows("4"))
+ tk.MustQuery("select count(*) from t group by b").Check(testkit.Rows("4"))
require.Nil(t, failpoint.Disable("github.com/pingcap/tidb/store/mockstore/unistore/mppDispatchTimeout"))
}
+func TestMppVersionError(t *testing.T) {
+ store := testkit.CreateMockStore(t, withMockTiFlash(2))
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int not null primary key, b int not null)")
+ tk.MustExec("alter table t set tiflash replica 1")
+ tk.MustExec("insert into t values(1,0),(2,0),(3,0),(4,0)")
+ tb := external.GetTableByName(t, tk, "test", "t")
+ err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
+ require.NoError(t, err)
+ tk.MustExec("set @@session.tidb_enforce_mpp=ON")
+ {
+ item := fmt.Sprintf("return(%d)", kv.GetNewestMppVersion()+1)
+ require.Nil(t, failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/MppVersionError", item))
+ }
+ {
+ err := tk.QueryToErr("select count(*) from t group by b")
+ require.Error(t, err)
+ }
+ require.Nil(t, failpoint.Disable("github.com/pingcap/tidb/store/mockstore/unistore/MppVersionError"))
+ {
+ item := fmt.Sprintf("return(%d)", kv.GetNewestMppVersion())
+ require.Nil(t, failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/MppVersionError", item))
+ }
+ {
+ tk.MustQuery("select count(*) from t group by b").Check(testkit.Rows("4"))
+ }
+ require.Nil(t, failpoint.Disable("github.com/pingcap/tidb/store/mockstore/unistore/MppVersionError"))
+}
+
func TestCancelMppTasks(t *testing.T) {
var hang = "github.com/pingcap/tidb/store/mockstore/unistore/mppRecvHang"
store := testkit.CreateMockStore(t, withMockTiFlash(2))
@@ -1041,7 +1072,7 @@ func TestTiFlashPartitionTableBroadcastJoin(t *testing.T) {
}
}
-func TestForbidTiflashDuringStaleRead(t *testing.T) {
+func TestTiflashSupportStaleRead(t *testing.T) {
store := testkit.CreateMockStore(t, withMockTiFlash(2))
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -1073,8 +1104,8 @@ func TestForbidTiflashDuringStaleRead(t *testing.T) {
fmt.Fprintf(resBuff, "%s\n", row)
}
res = resBuff.String()
- require.NotContains(t, res, "tiflash")
- require.Contains(t, res, "tikv")
+ require.Contains(t, res, "tiflash")
+ require.NotContains(t, res, "tikv")
}
func TestForbidTiFlashIfExtraPhysTableIDIsNeeded(t *testing.T) {
@@ -1206,6 +1237,62 @@ func TestAggPushDownCountStar(t *testing.T) {
tk.MustQuery("select count(*) from c, o where c.c_id=o.c_id").Check(testkit.Rows("5"))
}
+func TestGroupStreamAggOnTiFlash(t *testing.T) {
+ store := testkit.CreateMockStore(t, withMockTiFlash(2))
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists foo")
+ tk.MustExec("create table foo(a int, b int, c int, d int, primary key(a,b,c,d))")
+ tk.MustExec("alter table foo set tiflash replica 1")
+ tk.MustExec("insert into foo values(1,2,3,1),(1,2,3,6),(1,2,3,5)," +
+ "(1,2,3,2),(1,2,3,4),(1,2,3,7),(1,2,3,3),(1,2,3,0)")
+ tb := external.GetTableByName(t, tk, "test", "foo")
+ err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
+ require.NoError(t, err)
+ tk.MustExec("set @@tidb_allow_mpp=0")
+ sql := "select a,b,c,count(*) from foo group by a,b,c order by a,b,c"
+ tk.MustQuery(sql).Check(testkit.Rows("1 2 3 8"))
+ rows := tk.MustQuery("explain " + sql).Rows()
+
+ for _, row := range rows {
+ resBuff := bytes.NewBufferString("")
+ fmt.Fprintf(resBuff, "%s\n", row)
+ res := resBuff.String()
+ // StreamAgg with group keys on TiFlash is not supported
+ if strings.Contains(res, "tiflash") {
+ require.NotContains(t, res, "StreamAgg")
+ }
+ }
+}
+
+// TestIssue41014 test issue that can't find proper physical plan
+func TestIssue41014(t *testing.T) {
+ store := testkit.CreateMockStore(t, withMockTiFlash(2))
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test;")
+ tk.MustExec("CREATE TABLE `tai1` (\n `aid` int(11) DEFAULT NULL,\n `rid` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")
+ tk.MustExec("CREATE TABLE `tai2` (\n `rid` int(11) DEFAULT NULL,\n `prilan` varchar(20) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")
+ tk.MustExec("alter table tai1 set tiflash replica 1")
+ tk.MustExec("alter table tai2 set tiflash replica 1")
+ tk.MustExec("alter table tai2 add index idx((lower(prilan)));")
+ tk.MustExec("set @@tidb_opt_distinct_agg_push_down = 1;")
+
+ tk.MustQuery("explain select count(distinct tai1.aid) as cb from tai1 inner join tai2 on tai1.rid = tai2.rid where lower(prilan) LIKE LOWER('%python%');").Check(
+ testkit.Rows("HashAgg_11 1.00 root funcs:count(distinct test.tai1.aid)->Column#8",
+ "└─HashJoin_15 9990.00 root inner join, equal:[eq(test.tai2.rid, test.tai1.rid)]",
+ " ├─Selection_20(Build) 8000.00 root like(lower(test.tai2.prilan), \"%python%\", 92)",
+ " │ └─Projection_19 10000.00 root test.tai2.rid, lower(test.tai2.prilan)",
+ " │ └─TableReader_18 9990.00 root data:Selection_17",
+ " │ └─Selection_17 9990.00 cop[tikv] not(isnull(test.tai2.rid))",
+ " │ └─TableFullScan_16 10000.00 cop[tikv] table:tai2 keep order:false, stats:pseudo",
+ " └─TableReader_23(Probe) 9990.00 root data:Selection_22",
+ " └─Selection_22 9990.00 cop[tikv] not(isnull(test.tai1.rid))",
+ " └─TableFullScan_21 10000.00 cop[tikv] table:tai1 keep order:false, stats:pseudo"))
+ tk.MustQuery("select count(distinct tai1.aid) as cb from tai1 inner join tai2 on tai1.rid = tai2.rid where lower(prilan) LIKE LOWER('%python%');").Check(
+ testkit.Rows("0"))
+}
+
func TestTiflashEmptyDynamicPruneResult(t *testing.T) {
store := testkit.CreateMockStore(t, withMockTiFlash(2))
tk := testkit.NewTestKit(t, store)
@@ -1226,3 +1313,142 @@ func TestTiflashEmptyDynamicPruneResult(t *testing.T) {
tk.MustQuery("select /*+ read_from_storage(tiflash[t2]) */ * from IDT_RP24833 partition(p2) t2 where t2. col1 <= -8448770111093677011;").Check(testkit.Rows())
tk.MustQuery("select /*+ read_from_storage(tiflash[t1, t2]) */ * from IDT_RP24833 partition(p3, p4) t1 join IDT_RP24833 partition(p2) t2 on t1.col1 = t2.col1 where t1. col1 between -8448770111093677011 and -8448770111093677011 and t2. col1 <= -8448770111093677011;").Check(testkit.Rows())
}
+
+func TestDisaggregatedTiFlash(t *testing.T) {
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.DisaggregatedTiFlash = true
+ })
+ defer config.UpdateGlobal(func(conf *config.Config) {
+ conf.DisaggregatedTiFlash = false
+ })
+ err := tiflashcompute.InitGlobalTopoFetcher(tiflashcompute.TestASStr, "", "", false)
+ require.NoError(t, err)
+
+ store := testkit.CreateMockStore(t, withMockTiFlash(2))
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(c1 int)")
+ tk.MustExec("alter table t set tiflash replica 1")
+ tb := external.GetTableByName(t, tk, "test", "t")
+ err = domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
+ require.NoError(t, err)
+ tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
+
+ err = tk.ExecToErr("select * from t;")
+ // Expect error, because TestAutoScaler return empty topo.
+ require.Contains(t, err.Error(), "Cannot find proper topo from AutoScaler")
+
+ err = tiflashcompute.InitGlobalTopoFetcher(tiflashcompute.AWSASStr, "", "", false)
+ require.NoError(t, err)
+ err = tk.ExecToErr("select * from t;")
+ // Expect error, because AWSAutoScaler is not setup, so http request will fail.
+ require.Contains(t, err.Error(), "[util:1815]Internal : get tiflash_compute topology failed")
+}
+
+// todo: remove this after AutoScaler is stable.
+func TestDisaggregatedTiFlashNonAutoScaler(t *testing.T) {
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.DisaggregatedTiFlash = true
+ conf.UseAutoScaler = false
+ })
+ defer config.UpdateGlobal(func(conf *config.Config) {
+ conf.DisaggregatedTiFlash = false
+ conf.UseAutoScaler = true
+ })
+
+ // Setting globalTopoFetcher to nil to can make sure cannot fetch topo from AutoScaler.
+ err := tiflashcompute.InitGlobalTopoFetcher(tiflashcompute.InvalidASStr, "", "", false)
+ require.Contains(t, err.Error(), "unexpected topo fetch type. expect: mock or aws or gcp, got invalid")
+
+ store := testkit.CreateMockStore(t, withMockTiFlash(2))
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(c1 int)")
+ tk.MustExec("alter table t set tiflash replica 1")
+ tb := external.GetTableByName(t, tk, "test", "t")
+ err = domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
+ require.NoError(t, err)
+ tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
+
+ err = tk.ExecToErr("select * from t;")
+ // This error message means we use PD instead of AutoScaler.
+ require.Contains(t, err.Error(), "tiflash_compute node is unavailable")
+}
+
+func TestDisaggregatedTiFlashQuery(t *testing.T) {
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.DisaggregatedTiFlash = true
+ })
+ defer config.UpdateGlobal(func(conf *config.Config) {
+ conf.DisaggregatedTiFlash = false
+ })
+
+ store := testkit.CreateMockStore(t, withMockTiFlash(2))
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists tbl_1")
+ tk.MustExec(`create table tbl_1 ( col_1 bigint not null default -1443635317331776148,
+ col_2 text ( 176 ) collate utf8mb4_bin not null,
+ col_3 decimal ( 8, 3 ),
+ col_4 varchar ( 128 ) collate utf8mb4_bin not null,
+ col_5 varchar ( 377 ) collate utf8mb4_bin,
+ col_6 double,
+ col_7 varchar ( 459 ) collate utf8mb4_bin,
+ col_8 tinyint default -88 ) charset utf8mb4 collate utf8mb4_bin ;`)
+ tk.MustExec("alter table tbl_1 set tiflash replica 1")
+ tb := external.GetTableByName(t, tk, "test", "tbl_1")
+ err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
+ require.NoError(t, err)
+ tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
+
+ tk.MustExec("explain select max( tbl_1.col_1 ) as r0 , sum( tbl_1.col_1 ) as r1 , sum( tbl_1.col_8 ) as r2 from tbl_1 where tbl_1.col_8 != 68 or tbl_1.col_3 between null and 939 order by r0,r1,r2;")
+
+ tk.MustExec("set @@tidb_partition_prune_mode = 'static';")
+ tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
+ tk.MustExec("create table t1(c1 int, c2 int) partition by hash(c1) partitions 3")
+ tk.MustExec("insert into t1 values(1, 1), (2, 2), (3, 3)")
+ tk.MustExec("alter table t1 set tiflash replica 1")
+ tb = external.GetTableByName(t, tk, "test", "t1")
+ err = domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
+ require.NoError(t, err)
+ tk.MustQuery("explain select * from t1 where c1 < 2").Check(testkit.Rows(
+ "PartitionUnion_10 9970.00 root ",
+ "├─TableReader_15 3323.33 root MppVersion: 1, data:ExchangeSender_14",
+ "│ └─ExchangeSender_14 3323.33 mpp[tiflash] ExchangeType: PassThrough",
+ "│ └─Selection_13 3323.33 mpp[tiflash] lt(test.t1.c1, 2)",
+ "│ └─TableFullScan_12 10000.00 mpp[tiflash] table:t1, partition:p0 keep order:false, stats:pseudo",
+ "├─TableReader_19 3323.33 root MppVersion: 1, data:ExchangeSender_18",
+ "│ └─ExchangeSender_18 3323.33 mpp[tiflash] ExchangeType: PassThrough",
+ "│ └─Selection_17 3323.33 mpp[tiflash] lt(test.t1.c1, 2)",
+ "│ └─TableFullScan_16 10000.00 mpp[tiflash] table:t1, partition:p1 keep order:false, stats:pseudo",
+ "└─TableReader_23 3323.33 root MppVersion: 1, data:ExchangeSender_22",
+ " └─ExchangeSender_22 3323.33 mpp[tiflash] ExchangeType: PassThrough",
+ " └─Selection_21 3323.33 mpp[tiflash] lt(test.t1.c1, 2)",
+ " └─TableFullScan_20 10000.00 mpp[tiflash] table:t1, partition:p2 keep order:false, stats:pseudo"))
+ // tk.MustQuery("select * from t1 where c1 < 2").Check(testkit.Rows("1 1"))
+}
+
+func TestMPPMemoryTracker(t *testing.T) {
+ store := testkit.CreateMockStore(t, withMockTiFlash(2))
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set tidb_mem_quota_query = 1 << 30")
+ tk.MustExec("set global tidb_mem_oom_action = 'CANCEL'")
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int);")
+ tk.MustExec("insert into t values (1);")
+ tk.MustExec("alter table t set tiflash replica 1")
+ tb := external.GetTableByName(t, tk, "test", "t")
+ err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
+ require.NoError(t, err)
+ tk.MustExec("set tidb_enforce_mpp = on;")
+ tk.MustQuery("select * from t").Check(testkit.Rows("1"))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/store/copr/testMPPOOMPanic", `return(true)`))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/store/copr/testMPPOOMPanic"))
+ }()
+ err = tk.QueryToErr("select * from t")
+ require.NotNil(t, err)
+ require.True(t, strings.Contains(err.Error(), "Out Of Memory Quota!"))
+}
diff --git a/executor/union_scan.go b/executor/union_scan.go
index a23cd8b8c7873..f3a2b82c70812 100644
--- a/executor/union_scan.go
+++ b/executor/union_scan.go
@@ -19,7 +19,6 @@ import (
"fmt"
"runtime/trace"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/model"
@@ -30,6 +29,7 @@ import (
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/collate"
+ "github.com/pingcap/tidb/util/tracing"
)
// UnionScanExec merges the rows from dirty table and the rows from distsql request.
@@ -71,11 +71,9 @@ type UnionScanExec struct {
// Open implements the Executor Open interface.
func (us *UnionScanExec) Open(ctx context.Context) error {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("UnionScanExec.Open", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "UnionScanExec.Open")
+ defer r.End()
+
if err := us.baseExecutor.Open(ctx); err != nil {
return err
}
diff --git a/executor/update.go b/executor/update.go
index cf0a6ae2e33f4..0068392653cdd 100644
--- a/executor/update.go
+++ b/executor/update.go
@@ -434,6 +434,7 @@ func (e *UpdateExec) Close() error {
if err == nil && txn.Valid() && txn.GetSnapshot() != nil {
txn.GetSnapshot().SetOption(kv.CollectRuntimeStats, nil)
}
+ defer e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
return e.children[0].Close()
}
@@ -463,7 +464,6 @@ func (e *UpdateExec) collectRuntimeStatsEnabled() bool {
SnapshotRuntimeStats: &txnsnapshot.SnapshotRuntimeStats{},
AllocatorRuntimeStats: autoid.NewAllocatorRuntimeStats(),
}
- e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats)
}
return true
}
diff --git a/executor/write.go b/executor/write.go
index 01359b56a2571..0a3fbfda4795e 100644
--- a/executor/write.go
+++ b/executor/write.go
@@ -18,7 +18,6 @@ import (
"context"
"strings"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/expression"
@@ -34,6 +33,7 @@ import (
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/memory"
+ "github.com/pingcap/tidb/util/tracing"
)
var (
@@ -52,17 +52,9 @@ var (
// 2. err (error) : error in the update.
func updateRecord(ctx context.Context, sctx sessionctx.Context, h kv.Handle, oldData, newData []types.Datum, modified []bool, t table.Table,
onDup bool, memTracker *memory.Tracker, fkChecks []*FKCheckExec, fkCascades []*FKCascadeExec) (bool, error) {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("executor.updateRecord", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
- txn, err := sctx.Txn(false)
- if err != nil {
- return false, err
- }
- memUsageOfTxnState := txn.Size()
- defer memTracker.Consume(int64(txn.Size() - memUsageOfTxnState))
+ r, ctx := tracing.StartRegionEx(ctx, "executor.updateRecord")
+ defer r.End()
+
sc := sctx.GetSessionVars().StmtCtx
changed, handleChanged := false, false
// onUpdateSpecified is for "UPDATE SET ts_field = old_value", the
@@ -115,7 +107,7 @@ func updateRecord(ctx context.Context, sctx sessionctx.Context, h kv.Handle, old
if err != nil {
return false, err
}
- if err = t.Allocators(sctx).Get(autoid.RowIDAllocType).Rebase(ctx, recordID, true); err != nil {
+ if err = t.Allocators(sctx).Get(autoid.AutoIncrementType).Rebase(ctx, recordID, true); err != nil {
return false, err
}
}
@@ -166,7 +158,7 @@ func updateRecord(ctx context.Context, sctx sessionctx.Context, h kv.Handle, old
// Fill values into on-update-now fields, only if they are really changed.
for i, col := range t.Cols() {
if mysql.HasOnUpdateNowFlag(col.GetFlag()) && !modified[i] && !onUpdateSpecified[i] {
- if v, err := expression.GetTimeValue(sctx, strings.ToUpper(ast.CurrentTimestamp), col.GetType(), col.GetDecimal()); err == nil {
+ if v, err := expression.GetTimeValue(sctx, strings.ToUpper(ast.CurrentTimestamp), col.GetType(), col.GetDecimal(), nil); err == nil {
newData[i] = v
modified[i] = true
} else {
@@ -207,7 +199,7 @@ func updateRecord(ctx context.Context, sctx sessionctx.Context, h kv.Handle, old
}
} else {
// Update record to new value and update index.
- if err = t.UpdateRecord(ctx, sctx, h, oldData, newData, modified); err != nil {
+ if err := t.UpdateRecord(ctx, sctx, h, oldData, newData, modified); err != nil {
if terr, ok := errors.Cause(err).(*terror.Error); sctx.GetSessionVars().StmtCtx.IgnoreNoPartition && ok && terr.Code() == errno.ErrNoPartitionForGivenValue {
return false, nil
}
diff --git a/executor/writetest/BUILD.bazel b/executor/writetest/BUILD.bazel
new file mode 100644
index 0000000000000..5ef1885be7ec6
--- /dev/null
+++ b/executor/writetest/BUILD.bazel
@@ -0,0 +1,39 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+ name = "writetest_test",
+ timeout = "short",
+ srcs = [
+ "main_test.go",
+ "write_test.go",
+ ],
+ flaky = True,
+ race = "on",
+ shard_count = 50,
+ deps = [
+ "//config",
+ "//executor",
+ "//kv",
+ "//meta/autoid",
+ "//parser/model",
+ "//parser/mysql",
+ "//planner/core",
+ "//session",
+ "//sessionctx",
+ "//sessionctx/stmtctx",
+ "//sessionctx/variable",
+ "//sessiontxn",
+ "//store/mockstore",
+ "//table",
+ "//table/tables",
+ "//testkit",
+ "//types",
+ "//util",
+ "//util/mock",
+ "@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_stretchr_testify//require",
+ "@com_github_tikv_client_go_v2//tikv",
+ "@io_opencensus_go//stats/view",
+ "@org_uber_go_goleak//:goleak",
+ ],
+)
diff --git a/executor/writetest/main_test.go b/executor/writetest/main_test.go
new file mode 100644
index 0000000000000..075e1f91b488a
--- /dev/null
+++ b/executor/writetest/main_test.go
@@ -0,0 +1,60 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package writetest
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/pingcap/tidb/config"
+ "github.com/pingcap/tidb/meta/autoid"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/tikv/client-go/v2/tikv"
+ "go.opencensus.io/stats/view"
+ "go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+ autoid.SetStep(5000)
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.Log.SlowThreshold = 30000 // 30s
+ conf.TiKVClient.AsyncCommit.SafeWindow = 0
+ conf.TiKVClient.AsyncCommit.AllowedClockDrift = 0
+ conf.Experimental.AllowsExpressionIndex = true
+ })
+ tikv.EnableFailpoints()
+
+ opts := []goleak.Option{
+ goleak.Cleanup(func(_ int) {
+ view.Stop()
+ }),
+ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
+ goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
+ goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
+ goleak.IgnoreTopFunction("gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun"),
+ goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"),
+ }
+
+ goleak.VerifyTestMain(m, opts...)
+}
+
+func fillData(tk *testkit.TestKit, table string) {
+ tk.MustExec("use test")
+ tk.MustExec(fmt.Sprintf("create table %s(id int not null default 1, name varchar(255), PRIMARY KEY(id));", table))
+
+ // insert data
+ tk.MustExec(fmt.Sprintf("insert INTO %s VALUES (1, \"hello\");", table))
+ tk.MustExec(fmt.Sprintf("insert into %s values (2, \"hello\");", table))
+}
diff --git a/executor/write_test.go b/executor/writetest/write_test.go
similarity index 98%
rename from executor/write_test.go
rename to executor/writetest/write_test.go
index 32aa261c5518d..9aca357032b68 100644
--- a/executor/write_test.go
+++ b/executor/writetest/write_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package executor_test
+package writetest
import (
"context"
@@ -592,6 +592,25 @@ commit;`
tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1526 Table has no partition for value 3"))
}
+func TestIssue38950(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ var cfg kv.InjectionConfig
+ tk := testkit.NewTestKit(t, kv.NewInjectedStore(store, &cfg))
+ tk.MustExec("use test;")
+ tk.MustExec("drop table if exists t; create table t (id smallint auto_increment primary key);")
+ tk.MustExec("alter table t add column c1 int default 1;")
+ tk.MustExec("insert ignore into t(id) values (194626268);")
+ require.Empty(t, tk.Session().LastMessage())
+
+ tk.MustQuery("select * from t").Check(testkit.Rows("32767 1"))
+
+ tk.MustExec("insert ignore into t(id) values ('*') on duplicate key update c1 = 2;")
+ require.Equal(t, int64(2), int64(tk.Session().AffectedRows()))
+ require.Empty(t, tk.Session().LastMessage())
+
+ tk.MustQuery("select * from t").Check(testkit.Rows("32767 2"))
+}
+
func TestInsertOnDup(t *testing.T) {
store := testkit.CreateMockStore(t)
var cfg kv.InjectionConfig
@@ -1877,7 +1896,7 @@ func checkCases(tests []testCase, ld *executor.LoadDataInfo, t *testing.T, tk *t
}
ld.SetMessage()
require.Equal(t, tt.expectedMsg, tk.Session().LastMessage())
- ctx.StmtCommit()
+ ctx.StmtCommit(context.Background())
txn, err := ctx.Txn(true)
require.NoError(t, err)
err = txn.Commit(context.Background())
@@ -2334,7 +2353,7 @@ func TestLoadDataIntoPartitionedTable(t *testing.T) {
require.NoError(t, err)
ld.SetMaxRowsInBatch(20000)
ld.SetMessage()
- ctx.StmtCommit()
+ ctx.StmtCommit(context.Background())
txn, err := ctx.Txn(true)
require.NoError(t, err)
err = txn.Commit(context.Background())
@@ -4299,3 +4318,23 @@ func TestIssueInsertPrefixIndexForNonUTF8Collation(t *testing.T) {
tk.MustExec("insert into t3 select 'abc '")
tk.MustGetErrCode("insert into t3 select 'abc d'", 1062)
}
+
+func TestMutipleReplaceAndInsertInOneSession(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t_securities(id bigint not null auto_increment primary key, security_id varchar(8), market_id smallint, security_type int, unique key uu(security_id, market_id))")
+ tk.MustExec(`insert into t_securities (security_id, market_id, security_type) values ("1", 2, 7), ("7", 1, 7) ON DUPLICATE KEY UPDATE security_type = VALUES(security_type)`)
+ tk.MustExec(`replace into t_securities (security_id, market_id, security_type) select security_id+1, 1, security_type from t_securities where security_id="7";`)
+ tk.MustExec(`INSERT INTO t_securities (security_id, market_id, security_type) values ("1", 2, 7), ("7", 1, 7) ON DUPLICATE KEY UPDATE security_type = VALUES(security_type)`)
+
+ tk.MustQuery("select * from t_securities").Sort().Check(testkit.Rows("1 1 2 7", "2 7 1 7", "3 8 1 7"))
+
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk2.MustExec(`insert into t_securities (security_id, market_id, security_type) values ("1", 2, 7), ("7", 1, 7) ON DUPLICATE KEY UPDATE security_type = VALUES(security_type)`)
+ tk2.MustExec(`insert into t_securities (security_id, market_id, security_type) select security_id+2, 1, security_type from t_securities where security_id="7";`)
+ tk2.MustExec(`INSERT INTO t_securities (security_id, market_id, security_type) values ("1", 2, 7), ("7", 1, 7) ON DUPLICATE KEY UPDATE security_type = VALUES(security_type)`)
+
+ tk2.MustQuery("select * from t_securities").Sort().Check(testkit.Rows("1 1 2 7", "2 7 1 7", "3 8 1 7", "8 9 1 7"))
+}
diff --git a/expression/BUILD.bazel b/expression/BUILD.bazel
index 032c44054dba2..17436f517a673 100644
--- a/expression/BUILD.bazel
+++ b/expression/BUILD.bazel
@@ -97,6 +97,7 @@ go_library(
"//util/mathutil",
"//util/mock",
"//util/parser",
+ "//util/password-validation",
"//util/plancodec",
"//util/printer",
"//util/sem",
@@ -110,6 +111,7 @@ go_library(
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_tipb//go-tipb",
+ "@com_github_pkg_errors//:errors",
"@com_github_tikv_client_go_v2//oracle",
"@org_golang_x_exp//slices",
"@org_golang_x_tools//container/intsets",
@@ -120,7 +122,7 @@ go_library(
go_test(
name = "expression_test",
- timeout = "short",
+ timeout = "moderate",
srcs = [
"bench_test.go",
"builtin_arithmetic_test.go",
@@ -175,6 +177,7 @@ go_test(
"integration_serial_test.go",
"integration_test.go",
"main_test.go",
+ "multi_valued_index_test.go",
"scalar_function_test.go",
"schema_test.go",
"typeinfer_test.go",
diff --git a/expression/builtin.go b/expression/builtin.go
index f10cf9aa3dfa9..66abac551a3e6 100644
--- a/expression/builtin.go
+++ b/expression/builtin.go
@@ -873,7 +873,9 @@ var funcs = map[string]functionClass{
ast.JSONMerge: &jsonMergeFunctionClass{baseFunctionClass{ast.JSONMerge, 2, -1}},
ast.JSONObject: &jsonObjectFunctionClass{baseFunctionClass{ast.JSONObject, 0, -1}},
ast.JSONArray: &jsonArrayFunctionClass{baseFunctionClass{ast.JSONArray, 0, -1}},
+ ast.JSONMemberOf: &jsonMemberOfFunctionClass{baseFunctionClass{ast.JSONMemberOf, 2, 2}},
ast.JSONContains: &jsonContainsFunctionClass{baseFunctionClass{ast.JSONContains, 2, 3}},
+ ast.JSONOverlaps: &jsonOverlapsFunctionClass{baseFunctionClass{ast.JSONOverlaps, 2, 2}},
ast.JSONContainsPath: &jsonContainsPathFunctionClass{baseFunctionClass{ast.JSONContainsPath, 3, -1}},
ast.JSONValid: &jsonValidFunctionClass{baseFunctionClass{ast.JSONValid, 1, 1}},
ast.JSONArrayAppend: &jsonArrayAppendFunctionClass{baseFunctionClass{ast.JSONArrayAppend, 3, -1}},
diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go
index 0dd7e20cce24e..c2d57fdeab9cc 100644
--- a/expression/builtin_cast.go
+++ b/expression/builtin_cast.go
@@ -23,6 +23,7 @@
package expression
import (
+ "fmt"
"math"
"strconv"
"strings"
@@ -35,6 +36,7 @@ import (
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
@@ -407,6 +409,149 @@ func (c *castAsDurationFunctionClass) getFunction(ctx sessionctx.Context, args [
return sig, nil
}
+type castAsArrayFunctionClass struct {
+ baseFunctionClass
+
+ tp *types.FieldType
+}
+
+func (c *castAsArrayFunctionClass) verifyArgs(args []Expression) error {
+ if err := c.baseFunctionClass.verifyArgs(args); err != nil {
+ return err
+ }
+
+ if args[0].GetType().EvalType() != types.ETJson {
+ return ErrInvalidTypeForJSON.GenWithStackByArgs(1, "cast_as_array")
+ }
+
+ return nil
+}
+
+func (c *castAsArrayFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (sig builtinFunc, err error) {
+ if err := c.verifyArgs(args); err != nil {
+ return nil, err
+ }
+ arrayType := c.tp.ArrayType()
+ switch arrayType.GetType() {
+ case mysql.TypeYear, mysql.TypeJSON, mysql.TypeDouble, mysql.TypeFloat, mysql.TypeNewDecimal:
+ return nil, ErrNotSupportedYet.GenWithStackByArgs(fmt.Sprintf("CAST-ing data to array of %s", arrayType.String()))
+ }
+ if arrayType.EvalType() == types.ETString && arrayType.GetCharset() != charset.CharsetUTF8MB4 && arrayType.GetCharset() != charset.CharsetBin {
+ return nil, ErrNotSupportedYet.GenWithStackByArgs("specifying charset for multi-valued index")
+ }
+ if arrayType.EvalType() == types.ETString && arrayType.GetFlen() == types.UnspecifiedLength {
+ return nil, ErrNotSupportedYet.GenWithStackByArgs("CAST-ing data to array of char/binary BLOBs")
+ }
+
+ bf, err := newBaseBuiltinFunc(ctx, c.funcName, args, c.tp)
+ if err != nil {
+ return nil, err
+ }
+ sig = &castJSONAsArrayFunctionSig{bf}
+ return sig, nil
+}
+
+type castJSONAsArrayFunctionSig struct {
+ baseBuiltinFunc
+}
+
+func (b *castJSONAsArrayFunctionSig) Clone() builtinFunc {
+ newSig := &castJSONAsArrayFunctionSig{}
+ newSig.cloneFrom(&b.baseBuiltinFunc)
+ return newSig
+}
+
+// fakeSctx is used to ignore the sql mode, `cast as array` should always return error if any.
+var fakeSctx = &stmtctx.StatementContext{InInsertStmt: true}
+
+func (b *castJSONAsArrayFunctionSig) evalJSON(row chunk.Row) (res types.BinaryJSON, isNull bool, err error) {
+ val, isNull, err := b.args[0].EvalJSON(b.ctx, row)
+ if isNull || err != nil {
+ return res, isNull, err
+ }
+
+ if val.TypeCode == types.JSONTypeCodeObject {
+ return types.BinaryJSON{}, false, ErrNotSupportedYet.GenWithStackByArgs("CAST-ing JSON OBJECT type to array")
+ }
+
+ arrayVals := make([]any, 0, 8)
+ ft := b.tp.ArrayType()
+ f := convertJSON2Tp(ft.EvalType())
+ if f == nil {
+ return types.BinaryJSON{}, false, ErrNotSupportedYet.GenWithStackByArgs(fmt.Sprintf("CAS-ing data to array of %s", ft.String()))
+ }
+ if val.TypeCode != types.JSONTypeCodeArray {
+ item, err := f(fakeSctx, val, ft)
+ if err != nil {
+ return types.BinaryJSON{}, false, err
+ }
+ arrayVals = append(arrayVals, item)
+ } else {
+ for i := 0; i < val.GetElemCount(); i++ {
+ item, err := f(fakeSctx, val.ArrayGetElem(i), ft)
+ if err != nil {
+ return types.BinaryJSON{}, false, err
+ }
+ arrayVals = append(arrayVals, item)
+ }
+ }
+ return types.CreateBinaryJSON(arrayVals), false, nil
+}
+
+// ConvertJSON2Tp returns a function that can convert JSON to the specified type.
+func ConvertJSON2Tp(v types.BinaryJSON, targetType *types.FieldType) (any, error) {
+ convertFunc := convertJSON2Tp(targetType.EvalType())
+ if convertFunc == nil {
+ return nil, ErrInvalidJSONForFuncIndex
+ }
+ return convertFunc(fakeSctx, v, targetType)
+}
+
+func convertJSON2Tp(evalType types.EvalType) func(*stmtctx.StatementContext, types.BinaryJSON, *types.FieldType) (any, error) {
+ switch evalType {
+ case types.ETString:
+ return func(sc *stmtctx.StatementContext, item types.BinaryJSON, tp *types.FieldType) (any, error) {
+ if item.TypeCode != types.JSONTypeCodeString {
+ return nil, ErrInvalidJSONForFuncIndex
+ }
+ return types.ProduceStrWithSpecifiedTp(string(item.GetString()), tp, sc, false)
+ }
+ case types.ETInt:
+ return func(sc *stmtctx.StatementContext, item types.BinaryJSON, tp *types.FieldType) (any, error) {
+ if item.TypeCode != types.JSONTypeCodeInt64 && item.TypeCode != types.JSONTypeCodeUint64 {
+ return nil, ErrInvalidJSONForFuncIndex
+ }
+ jsonToInt, err := types.ConvertJSONToInt(sc, item, mysql.HasUnsignedFlag(tp.GetFlag()), tp.GetType())
+ if mysql.HasUnsignedFlag(tp.GetFlag()) {
+ return uint64(jsonToInt), err
+ }
+ return jsonToInt, err
+ }
+ case types.ETDatetime:
+ return func(sc *stmtctx.StatementContext, item types.BinaryJSON, tp *types.FieldType) (any, error) {
+ if (tp.GetType() == mysql.TypeDatetime && item.TypeCode != types.JSONTypeCodeDatetime) || (tp.GetType() == mysql.TypeDate && item.TypeCode != types.JSONTypeCodeDate) {
+ return nil, ErrInvalidJSONForFuncIndex
+ }
+ res := item.GetTime()
+ res.SetType(tp.GetType())
+ if tp.GetType() == mysql.TypeDate {
+ // Truncate hh:mm:ss part if the type is Date.
+ res.SetCoreTime(types.FromDate(res.Year(), res.Month(), res.Day(), 0, 0, 0, 0))
+ }
+ return res, nil
+ }
+ case types.ETDuration:
+ return func(sc *stmtctx.StatementContext, item types.BinaryJSON, tp *types.FieldType) (any, error) {
+ if item.TypeCode != types.JSONTypeCodeDuration {
+ return nil, ErrInvalidJSONForFuncIndex
+ }
+ return item.GetDuration(), nil
+ }
+ default:
+ return nil
+ }
+}
+
type castAsJSONFunctionClass struct {
baseFunctionClass
@@ -1299,7 +1444,7 @@ func (b *builtinCastStringAsTimeSig) evalTime(row chunk.Row) (res types.Time, is
return res, isNull, err
}
sc := b.ctx.GetSessionVars().StmtCtx
- res, err = types.ParseTime(sc, val, b.tp.GetType(), b.tp.GetDecimal())
+ res, err = types.ParseTime(sc, val, b.tp.GetType(), b.tp.GetDecimal(), nil)
if err != nil {
return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err)
}
@@ -1722,7 +1867,11 @@ func (b *builtinCastJSONAsStringSig) evalString(row chunk.Row) (res string, isNu
if isNull || err != nil {
return res, isNull, err
}
- return val.String(), false, nil
+ s, err := types.ProduceStrWithSpecifiedTp(val.String(), b.tp, b.ctx.GetSessionVars().StmtCtx, false)
+ if err != nil {
+ return res, false, err
+ }
+ return s, false, nil
}
type builtinCastJSONAsTimeSig struct {
@@ -1770,7 +1919,7 @@ func (b *builtinCastJSONAsTimeSig) evalTime(row chunk.Row) (res types.Time, isNu
return res, false, err
}
sc := b.ctx.GetSessionVars().StmtCtx
- res, err = types.ParseTime(sc, s, b.tp.GetType(), b.tp.GetDecimal())
+ res, err = types.ParseTime(sc, s, b.tp.GetType(), b.tp.GetDecimal(), nil)
if err != nil {
return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err)
}
@@ -1910,6 +2059,13 @@ func BuildCastCollationFunction(ctx sessionctx.Context, expr Expression, ec *Exp
// BuildCastFunction builds a CAST ScalarFunction from the Expression.
func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldType) (res Expression) {
+ res, err := BuildCastFunctionWithCheck(ctx, expr, tp)
+ terror.Log(err)
+ return
+}
+
+// BuildCastFunctionWithCheck builds a CAST ScalarFunction from the Expression and return error if any.
+func BuildCastFunctionWithCheck(ctx sessionctx.Context, expr Expression, tp *types.FieldType) (res Expression, err error) {
argType := expr.GetType()
// If source argument's nullable, then target type should be nullable
if !mysql.HasNotNullFlag(argType.GetFlag()) {
@@ -1929,7 +2085,11 @@ func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldT
case types.ETDuration:
fc = &castAsDurationFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp}
case types.ETJson:
- fc = &castAsJSONFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp}
+ if tp.IsArray() {
+ fc = &castAsArrayFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp}
+ } else {
+ fc = &castAsJSONFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp}
+ }
case types.ETString:
fc = &castAsStringFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp}
if expr.GetType().GetType() == mysql.TypeBit {
@@ -1937,7 +2097,6 @@ func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldT
}
}
f, err := fc.getFunction(ctx, []Expression{expr})
- terror.Log(err)
res = &ScalarFunction{
FuncName: model.NewCIStr(ast.Cast),
RetType: tp,
@@ -1946,10 +2105,10 @@ func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldT
// We do not fold CAST if the eval type of this scalar function is ETJson
// since we may reset the flag of the field type of CastAsJson later which
// would affect the evaluation of it.
- if tp.EvalType() != types.ETJson {
+ if tp.EvalType() != types.ETJson && err == nil {
res = FoldConstant(res)
}
- return res
+ return res, err
}
// WrapWithCastAsInt wraps `expr` with `cast` if the return type of expr is not
diff --git a/expression/builtin_cast_test.go b/expression/builtin_cast_test.go
index f8e789a527620..b33c70e19d60f 100644
--- a/expression/builtin_cast_test.go
+++ b/expression/builtin_cast_test.go
@@ -719,6 +719,13 @@ func TestCastFuncSig(t *testing.T) {
3,
chunk.MutRowFromDatums([]types.Datum{types.NewStringDatum("你好world")}),
},
+ // cast json as string
+ {
+ &Column{RetType: types.NewFieldType(mysql.TypeJSON), Index: 0},
+ fmt.Sprintf(`"%s`, curTimeString[:2]),
+ 3,
+ chunk.MutRowFromDatums([]types.Datum{jsonTime}),
+ },
}
for i, c := range castToStringCases2 {
args := []Expression{c.before}
@@ -741,6 +748,8 @@ func TestCastFuncSig(t *testing.T) {
case 5:
stringFunc.tp.SetCharset(charset.CharsetUTF8)
sig = &builtinCastStringAsStringSig{stringFunc}
+ case 6:
+ sig = &builtinCastJSONAsStringSig{stringFunc}
}
res, isNull, err := sig.evalString(c.row.ToRow())
require.False(t, isNull)
@@ -1610,3 +1619,59 @@ func TestCastBinaryStringAsJSONSig(t *testing.T) {
require.Equal(t, tt.resultStr, res.String())
}
}
+
+func TestCastArrayFunc(t *testing.T) {
+ ctx := createContext(t)
+ tbl := []struct {
+ input interface{}
+ expected interface{}
+ tp *types.FieldType
+ success bool
+ buildFuncSuccess bool
+ }{
+ {
+ []interface{}{int64(-1), int64(2), int64(3)},
+ []interface{}{int64(-1), int64(2), int64(3)},
+ types.NewFieldTypeBuilder().SetType(mysql.TypeLonglong).SetCharset(charset.CharsetBin).SetCollate(charset.CollationBin).SetArray(true).BuildP(),
+ true,
+ true,
+ },
+ {
+ []interface{}{int64(-1), int64(2), int64(3)},
+ nil,
+ types.NewFieldTypeBuilder().SetType(mysql.TypeString).SetCharset(charset.CharsetUTF8MB4).SetCollate(charset.CollationUTF8MB4).SetArray(true).BuildP(),
+ false,
+ true,
+ },
+ {
+ []interface{}{"1"},
+ nil,
+ types.NewFieldTypeBuilder().SetType(mysql.TypeLonglong).SetCharset(charset.CharsetBin).SetCollate(charset.CharsetBin).SetArray(true).BuildP(),
+ false,
+ true,
+ },
+ }
+ for _, tt := range tbl {
+ f, err := BuildCastFunctionWithCheck(ctx, datumsToConstants(types.MakeDatums(types.CreateBinaryJSON(tt.input)))[0], tt.tp)
+ if tt.buildFuncSuccess {
+ require.NoError(t, err, tt.input)
+ } else {
+ require.Error(t, err, tt.input)
+ continue
+ }
+
+ val, isNull, err := f.EvalJSON(ctx, chunk.Row{})
+ if tt.success {
+ require.NoError(t, err, tt.input)
+ if tt.expected == nil {
+ require.True(t, isNull, tt.input)
+ } else {
+ j1 := types.CreateBinaryJSON(tt.expected)
+ cmp := types.CompareBinaryJSON(j1, val)
+ require.Equal(t, 0, cmp, tt.input)
+ }
+ } else {
+ require.Error(t, err, tt.input)
+ }
+ }
+}
diff --git a/expression/builtin_cast_vec.go b/expression/builtin_cast_vec.go
index 8b82d86b776e5..f6b4266e8e534 100644
--- a/expression/builtin_cast_vec.go
+++ b/expression/builtin_cast_vec.go
@@ -529,7 +529,7 @@ func (b *builtinCastJSONAsTimeSig) vecEvalTime(input *chunk.Chunk, result *chunk
if err != nil {
return err
}
- tm, err := types.ParseTime(stmtCtx, s, b.tp.GetType(), fsp)
+ tm, err := types.ParseTime(stmtCtx, s, b.tp.GetType(), fsp, nil)
if err != nil {
if err = handleInvalidTimeError(b.ctx, err); err != nil {
return err
@@ -1186,7 +1186,11 @@ func (b *builtinCastJSONAsStringSig) vecEvalString(input *chunk.Chunk, result *c
result.AppendNull()
continue
}
- result.AppendString(buf.GetJSON(i).String())
+ s, err := types.ProduceStrWithSpecifiedTp(buf.GetJSON(i).String(), b.tp, b.ctx.GetSessionVars().StmtCtx, false)
+ if err != nil {
+ return err
+ }
+ result.AppendString(s)
}
return nil
}
@@ -1762,7 +1766,7 @@ func (b *builtinCastStringAsTimeSig) vecEvalTime(input *chunk.Chunk, result *chu
if result.IsNull(i) {
continue
}
- tm, err := types.ParseTime(stmtCtx, buf.GetString(i), b.tp.GetType(), fsp)
+ tm, err := types.ParseTime(stmtCtx, buf.GetString(i), b.tp.GetType(), fsp, nil)
if err != nil {
if err = handleInvalidTimeError(b.ctx, err); err != nil {
return err
diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go
index 3baa86b635ba6..bed48c0e59096 100644
--- a/expression/builtin_compare.go
+++ b/expression/builtin_compare.go
@@ -28,6 +28,7 @@ import (
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tipb/go-tipb"
+ "github.com/pkg/errors"
)
var (
@@ -1555,6 +1556,37 @@ func RefineComparedConstant(ctx sessionctx.Context, targetFieldType types.FieldT
return con, false
}
+// Since the argument refining of cmp functions can bring some risks to the plan-cache, the optimizer
+// needs to decide to whether to skip the refining or skip plan-cache for safety.
+// For example, `unsigned_int_col > ?(-1)` can be refined to `True`, but the validation of this result
+// can be broken if the parameter changes to 1 after.
+func allowCmpArgsRefining4PlanCache(ctx sessionctx.Context, args []Expression) (allowRefining bool) {
+ if !MaybeOverOptimized4PlanCache(ctx, args) {
+ return true // plan-cache disabled or no parameter in these args
+ }
+
+ // For these 2 cases below which may affect the index selection a lot, skip plan-cache,
+ // and for all other cases, skip the refining.
+ // 1. int-expr string-const
+ // 2. int-expr float/double/decimal-const
+ for conIdx := 0; conIdx < 2; conIdx++ {
+ if args[1-conIdx].GetType().EvalType() != types.ETInt {
+ continue // not a int-expr
+ }
+ if _, isCon := args[conIdx].(*Constant); !isCon {
+ continue // not a constant
+ }
+ conType := args[conIdx].GetType().EvalType()
+ if conType == types.ETString || conType == types.ETReal || conType == types.ETDecimal {
+ reason := errors.Errorf("skip plan-cache: '%v' may be converted to INT", args[conIdx].String())
+ ctx.GetSessionVars().StmtCtx.SetSkipPlanCache(reason)
+ return true
+ }
+ }
+
+ return false
+}
+
// refineArgs will rewrite the arguments if the compare expression is `int column non-int constant` or
// `non-int constant int column`. E.g., `a < 1.1` will be rewritten to `a < 2`. It also handles comparing year type
// with int constant if the int constant falls into a sensible year representation.
@@ -1564,25 +1596,17 @@ func (c *compareFunctionClass) refineArgs(ctx sessionctx.Context, args []Express
arg0Type, arg1Type := args[0].GetType(), args[1].GetType()
arg0IsInt := arg0Type.EvalType() == types.ETInt
arg1IsInt := arg1Type.EvalType() == types.ETInt
- arg0IsString := arg0Type.EvalType() == types.ETString
- arg1IsString := arg1Type.EvalType() == types.ETString
arg0, arg0IsCon := args[0].(*Constant)
arg1, arg1IsCon := args[1].(*Constant)
isExceptional, finalArg0, finalArg1 := false, args[0], args[1]
isPositiveInfinite, isNegativeInfinite := false, false
- if MaybeOverOptimized4PlanCache(ctx, args) {
- // To keep the result be compatible with MySQL, refine `int non-constant str constant`
- // here and skip this refine operation in all other cases for safety.
- if (arg0IsInt && !arg0IsCon && arg1IsString && arg1IsCon) || (arg1IsInt && !arg1IsCon && arg0IsString && arg0IsCon) {
- ctx.GetSessionVars().StmtCtx.SkipPlanCache = true
- RemoveMutableConst(ctx, args)
- } else {
- return args
- }
- } else if ctx.GetSessionVars().StmtCtx.SkipPlanCache {
- // We should remove the mutable constant for correctness, because its value may be changed.
- RemoveMutableConst(ctx, args)
+
+ if !allowCmpArgsRefining4PlanCache(ctx, args) {
+ return args
}
+ // We should remove the mutable constant for correctness, because its value may be changed.
+ RemoveMutableConst(ctx, args)
+
// int non-constant [cmp] non-int constant
if arg0IsInt && !arg0IsCon && !arg1IsInt && arg1IsCon {
arg1, isExceptional = RefineComparedConstant(ctx, *arg0Type, arg1, c.op)
diff --git a/expression/builtin_compare_test.go b/expression/builtin_compare_test.go
index 78229b77b1f60..849142308ec93 100644
--- a/expression/builtin_compare_test.go
+++ b/expression/builtin_compare_test.go
@@ -406,3 +406,17 @@ func TestGreatestLeastFunc(t *testing.T) {
_, err = funcs[ast.Least].getFunction(ctx, []Expression{NewZero(), NewOne()})
require.NoError(t, err)
}
+
+func TestRefineArgsWithCastEnum(t *testing.T) {
+ ctx := createContext(t)
+ zeroUintConst := primitiveValsToConstants(ctx, []interface{}{uint64(0)})[0]
+ enumType := types.NewFieldTypeBuilder().SetType(mysql.TypeEnum).SetElems([]string{"1", "2", "3"}).AddFlag(mysql.EnumSetAsIntFlag).Build()
+ enumCol := &Column{RetType: &enumType}
+
+ f := funcs[ast.EQ].(*compareFunctionClass)
+ require.NotNil(t, f)
+
+ args := f.refineArgsByUnsignedFlag(ctx, []Expression{zeroUintConst, enumCol})
+ require.Equal(t, zeroUintConst, args[0])
+ require.Equal(t, enumCol, args[1])
+}
diff --git a/expression/builtin_encryption.go b/expression/builtin_encryption.go
index a206a9d4970bb..fb451f9714cd4 100644
--- a/expression/builtin_encryption.go
+++ b/expression/builtin_encryption.go
@@ -37,6 +37,7 @@ import (
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/encrypt"
+ pwdValidator "github.com/pingcap/tidb/util/password-validation"
"github.com/pingcap/tipb/go-tipb"
)
@@ -73,6 +74,7 @@ var (
_ builtinFunc = &builtinSHA2Sig{}
_ builtinFunc = &builtinUncompressSig{}
_ builtinFunc = &builtinUncompressedLengthSig{}
+ _ builtinFunc = &builtinValidatePasswordStrengthSig{}
)
// aesModeAttr indicates that the key length and iv attribute for specific block_encryption_mode.
@@ -728,7 +730,6 @@ func (c *sm3FunctionClass) getFunction(ctx sessionctx.Context, args []Expression
bf.tp.SetCollate(collate)
bf.tp.SetFlen(40)
sig := &builtinSM3Sig{bf}
- //sig.setPbCode(tipb.ScalarFuncSig_SM3) // TODO
return sig, nil
}
@@ -1010,5 +1011,66 @@ type validatePasswordStrengthFunctionClass struct {
}
func (c *validatePasswordStrengthFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) {
- return nil, errFunctionNotExists.GenWithStackByArgs("FUNCTION", "VALIDATE_PASSWORD_STRENGTH")
+ if err := c.verifyArgs(args); err != nil {
+ return nil, err
+ }
+ bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETInt, types.ETString)
+ if err != nil {
+ return nil, err
+ }
+ bf.tp.SetFlen(21)
+ sig := &builtinValidatePasswordStrengthSig{bf}
+ return sig, nil
+}
+
+type builtinValidatePasswordStrengthSig struct {
+ baseBuiltinFunc
+}
+
+func (b *builtinValidatePasswordStrengthSig) Clone() builtinFunc {
+ newSig := &builtinValidatePasswordStrengthSig{}
+ newSig.cloneFrom(&b.baseBuiltinFunc)
+ return newSig
+}
+
+// evalInt evals VALIDATE_PASSWORD_STRENGTH(str).
+// See https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_validate-password-strength
+func (b *builtinValidatePasswordStrengthSig) evalInt(row chunk.Row) (int64, bool, error) {
+ globalVars := b.ctx.GetSessionVars().GlobalVarsAccessor
+ str, isNull, err := b.args[0].EvalString(b.ctx, row)
+ if err != nil || isNull {
+ return 0, true, err
+ } else if len([]rune(str)) < 4 {
+ return 0, false, nil
+ }
+ if validation, err := globalVars.GetGlobalSysVar(variable.ValidatePasswordEnable); err != nil {
+ return 0, true, err
+ } else if !variable.TiDBOptOn(validation) {
+ return 0, false, nil
+ }
+ return b.validateStr(str, &globalVars)
+}
+
+func (b *builtinValidatePasswordStrengthSig) validateStr(str string, globalVars *variable.GlobalVarAccessor) (int64, bool, error) {
+ if warn, err := pwdValidator.ValidateUserNameInPassword(str, b.ctx.GetSessionVars()); err != nil {
+ return 0, true, err
+ } else if len(warn) > 0 {
+ return 0, false, nil
+ }
+ if warn, err := pwdValidator.ValidatePasswordLowPolicy(str, globalVars); err != nil {
+ return 0, true, err
+ } else if len(warn) > 0 {
+ return 25, false, nil
+ }
+ if warn, err := pwdValidator.ValidatePasswordMediumPolicy(str, globalVars); err != nil {
+ return 0, true, err
+ } else if len(warn) > 0 {
+ return 50, false, nil
+ }
+ if ok, err := pwdValidator.ValidateDictionaryPassword(str, globalVars); err != nil {
+ return 0, true, err
+ } else if !ok {
+ return 75, false, nil
+ }
+ return 100, false, nil
}
diff --git a/expression/builtin_encryption_test.go b/expression/builtin_encryption_test.go
index 0f74ab611aa48..087fb3f35e466 100644
--- a/expression/builtin_encryption_test.go
+++ b/expression/builtin_encryption_test.go
@@ -15,12 +15,14 @@
package expression
import (
+ "context"
"encoding/hex"
"fmt"
"strings"
"testing"
"github.com/pingcap/tidb/parser/ast"
+ "github.com/pingcap/tidb/parser/auth"
"github.com/pingcap/tidb/parser/charset"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/terror"
@@ -631,6 +633,55 @@ func TestUncompressLength(t *testing.T) {
}
}
+func TestValidatePasswordStrength(t *testing.T) {
+ ctx := createContext(t)
+ ctx.GetSessionVars().User = &auth.UserIdentity{Username: "testuser"}
+ globalVarsAccessor := variable.NewMockGlobalAccessor4Tests()
+ ctx.GetSessionVars().GlobalVarsAccessor = globalVarsAccessor
+ err := globalVarsAccessor.SetGlobalSysVar(context.Background(), variable.ValidatePasswordDictionary, "1234")
+ require.NoError(t, err)
+
+ tests := []struct {
+ in interface{}
+ expect interface{}
+ }{
+ {nil, nil},
+ {"123", 0},
+ {"testuser123", 0},
+ {"resutset123", 0},
+ {"12345", 25},
+ {"12345678", 50},
+ {"!Abc12345678", 75},
+ {"!Abc87654321", 100},
+ }
+
+ fc := funcs[ast.ValidatePasswordStrength]
+ // disable password validation
+ for _, test := range tests {
+ arg := types.NewDatum(test.in)
+ f, err := fc.getFunction(ctx, datumsToConstants([]types.Datum{arg}))
+ require.NoErrorf(t, err, "%v", test)
+ out, err := evalBuiltinFunc(f, chunk.Row{})
+ require.NoErrorf(t, err, "%v", test)
+ if test.expect == nil {
+ require.Equal(t, types.NewDatum(nil), out)
+ } else {
+ require.Equalf(t, types.NewDatum(0), out, "%v", test)
+ }
+ }
+ // enable password validation
+ err = globalVarsAccessor.SetGlobalSysVar(context.Background(), variable.ValidatePasswordEnable, "ON")
+ require.NoError(t, err)
+ for _, test := range tests {
+ arg := types.NewDatum(test.in)
+ f, err := fc.getFunction(ctx, datumsToConstants([]types.Datum{arg}))
+ require.NoErrorf(t, err, "%v", test)
+ out, err := evalBuiltinFunc(f, chunk.Row{})
+ require.NoErrorf(t, err, "%v", test)
+ require.Equalf(t, types.NewDatum(test.expect), out, "%v", test)
+ }
+}
+
func TestPassword(t *testing.T) {
ctx := createContext(t)
cases := []struct {
diff --git a/expression/builtin_encryption_vec.go b/expression/builtin_encryption_vec.go
index e9a1d45ae67be..ff71913f8d70b 100644
--- a/expression/builtin_encryption_vec.go
+++ b/expression/builtin_encryption_vec.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/parser/auth"
+ "github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/encrypt"
@@ -863,3 +864,45 @@ func (b *builtinUncompressedLengthSig) vecEvalInt(input *chunk.Chunk, result *ch
}
return nil
}
+
+func (b *builtinValidatePasswordStrengthSig) vectorized() bool {
+ return true
+}
+
+func (b *builtinValidatePasswordStrengthSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) error {
+ n := input.NumRows()
+ buf, err := b.bufAllocator.get()
+ if err != nil {
+ return err
+ }
+ defer b.bufAllocator.put(buf)
+ if err := b.args[0].VecEvalString(b.ctx, input, buf); err != nil {
+ return err
+ }
+
+ result.ResizeInt64(n, false)
+ result.MergeNulls(buf)
+ i64s := result.Int64s()
+ globalVars := b.ctx.GetSessionVars().GlobalVarsAccessor
+ enableValidation := false
+ validation, err := globalVars.GetGlobalSysVar(variable.ValidatePasswordEnable)
+ if err != nil {
+ return err
+ }
+ enableValidation = variable.TiDBOptOn(validation)
+ for i := 0; i < n; i++ {
+ if result.IsNull(i) {
+ continue
+ }
+ if !enableValidation {
+ i64s[i] = 0
+ } else if score, isNull, err := b.validateStr(buf.GetString(i), &globalVars); err != nil {
+ return err
+ } else if !isNull {
+ i64s[i] = score
+ } else {
+ result.SetNull(i, true)
+ }
+ }
+ return nil
+}
diff --git a/expression/builtin_encryption_vec_test.go b/expression/builtin_encryption_vec_test.go
index c6caa1eb60d51..46395e51bcb6b 100644
--- a/expression/builtin_encryption_vec_test.go
+++ b/expression/builtin_encryption_vec_test.go
@@ -75,6 +75,9 @@ var vecBuiltinEncryptionCases = map[string][]vecExprBenchCase{
ast.Decode: {
{retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString, types.ETString}, geners: []dataGenerator{newRandLenStrGener(10, 20)}},
},
+ ast.ValidatePasswordStrength: {
+ {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETString}},
+ },
}
func TestVectorizedBuiltinEncryptionFunc(t *testing.T) {
diff --git a/expression/builtin_json.go b/expression/builtin_json.go
index e317fa88e952a..e9f803dcc86df 100644
--- a/expression/builtin_json.go
+++ b/expression/builtin_json.go
@@ -43,7 +43,9 @@ var (
_ functionClass = &jsonMergeFunctionClass{}
_ functionClass = &jsonObjectFunctionClass{}
_ functionClass = &jsonArrayFunctionClass{}
+ _ functionClass = &jsonMemberOfFunctionClass{}
_ functionClass = &jsonContainsFunctionClass{}
+ _ functionClass = &jsonOverlapsFunctionClass{}
_ functionClass = &jsonContainsPathFunctionClass{}
_ functionClass = &jsonValidFunctionClass{}
_ functionClass = &jsonArrayAppendFunctionClass{}
@@ -71,7 +73,9 @@ var (
_ builtinFunc = &builtinJSONReplaceSig{}
_ builtinFunc = &builtinJSONRemoveSig{}
_ builtinFunc = &builtinJSONMergeSig{}
+ _ builtinFunc = &builtinJSONMemberOfSig{}
_ builtinFunc = &builtinJSONContainsSig{}
+ _ builtinFunc = &builtinJSONOverlapsSig{}
_ builtinFunc = &builtinJSONStorageSizeSig{}
_ builtinFunc = &builtinJSONDepthSig{}
_ builtinFunc = &builtinJSONSearchSig{}
@@ -740,6 +744,68 @@ func jsonModify(ctx sessionctx.Context, args []Expression, row chunk.Row, mt typ
return res, false, nil
}
+type jsonMemberOfFunctionClass struct {
+ baseFunctionClass
+}
+
+type builtinJSONMemberOfSig struct {
+ baseBuiltinFunc
+}
+
+func (b *builtinJSONMemberOfSig) Clone() builtinFunc {
+ newSig := &builtinJSONMemberOfSig{}
+ newSig.cloneFrom(&b.baseBuiltinFunc)
+ return newSig
+}
+
+func (c *jsonMemberOfFunctionClass) verifyArgs(args []Expression) error {
+ if err := c.baseFunctionClass.verifyArgs(args); err != nil {
+ return err
+ }
+ if evalType := args[1].GetType().EvalType(); evalType != types.ETJson && evalType != types.ETString {
+ return types.ErrInvalidJSONData.GenWithStackByArgs(2, "member of")
+ }
+ return nil
+}
+
+func (c *jsonMemberOfFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) {
+ if err := c.verifyArgs(args); err != nil {
+ return nil, err
+ }
+ argTps := []types.EvalType{types.ETJson, types.ETJson}
+ bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETInt, argTps...)
+ if err != nil {
+ return nil, err
+ }
+ DisableParseJSONFlag4Expr(args[0])
+ sig := &builtinJSONMemberOfSig{bf}
+ return sig, nil
+}
+
+func (b *builtinJSONMemberOfSig) evalInt(row chunk.Row) (res int64, isNull bool, err error) {
+ target, isNull, err := b.args[0].EvalJSON(b.ctx, row)
+ if isNull || err != nil {
+ return res, isNull, err
+ }
+ obj, isNull, err := b.args[1].EvalJSON(b.ctx, row)
+ if isNull || err != nil {
+ return res, isNull, err
+ }
+
+ if obj.TypeCode != types.JSONTypeCodeArray {
+ return boolToInt64(types.CompareBinaryJSON(obj, target) == 0), false, nil
+ }
+
+ elemCount := obj.GetElemCount()
+ for i := 0; i < elemCount; i++ {
+ if types.CompareBinaryJSON(obj.ArrayGetElem(i), target) == 0 {
+ return 1, false, nil
+ }
+ }
+
+ return 0, false, nil
+}
+
type jsonContainsFunctionClass struct {
baseFunctionClass
}
@@ -820,6 +886,62 @@ func (b *builtinJSONContainsSig) evalInt(row chunk.Row) (res int64, isNull bool,
return 0, false, nil
}
+type jsonOverlapsFunctionClass struct {
+ baseFunctionClass
+}
+
+type builtinJSONOverlapsSig struct {
+ baseBuiltinFunc
+}
+
+func (b *builtinJSONOverlapsSig) Clone() builtinFunc {
+ newSig := &builtinJSONOverlapsSig{}
+ newSig.cloneFrom(&b.baseBuiltinFunc)
+ return newSig
+}
+
+func (c *jsonOverlapsFunctionClass) verifyArgs(args []Expression) error {
+ if err := c.baseFunctionClass.verifyArgs(args); err != nil {
+ return err
+ }
+ if evalType := args[0].GetType().EvalType(); evalType != types.ETJson && evalType != types.ETString {
+ return types.ErrInvalidJSONData.GenWithStackByArgs(1, "json_overlaps")
+ }
+ if evalType := args[1].GetType().EvalType(); evalType != types.ETJson && evalType != types.ETString {
+ return types.ErrInvalidJSONData.GenWithStackByArgs(2, "json_overlaps")
+ }
+ return nil
+}
+
+func (c *jsonOverlapsFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) {
+ if err := c.verifyArgs(args); err != nil {
+ return nil, err
+ }
+
+ argTps := []types.EvalType{types.ETJson, types.ETJson}
+ bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETInt, argTps...)
+ if err != nil {
+ return nil, err
+ }
+ sig := &builtinJSONOverlapsSig{bf}
+ return sig, nil
+}
+
+func (b *builtinJSONOverlapsSig) evalInt(row chunk.Row) (res int64, isNull bool, err error) {
+ obj, isNull, err := b.args[0].EvalJSON(b.ctx, row)
+ if isNull || err != nil {
+ return res, isNull, err
+ }
+ target, isNull, err := b.args[1].EvalJSON(b.ctx, row)
+ if isNull || err != nil {
+ return res, isNull, err
+ }
+ if types.OverlapsBinaryJSON(obj, target) {
+ return 1, false, nil
+ }
+ return 0, false, nil
+}
+
type jsonValidFunctionClass struct {
baseFunctionClass
}
diff --git a/expression/builtin_json_test.go b/expression/builtin_json_test.go
index 3e142860fdd27..23da60f380652 100644
--- a/expression/builtin_json_test.go
+++ b/expression/builtin_json_test.go
@@ -386,6 +386,47 @@ func TestJSONRemove(t *testing.T) {
}
}
+func TestJSONMemberOf(t *testing.T) {
+ ctx := createContext(t)
+ fc := funcs[ast.JSONMemberOf]
+ tbl := []struct {
+ input []interface{}
+ expected interface{}
+ err error
+ }{
+ {[]interface{}{`1`, `a:1`}, 1, types.ErrInvalidJSONText},
+
+ {[]interface{}{1, `[1, 2]`}, 1, nil},
+ {[]interface{}{1, `[1]`}, 1, nil},
+ {[]interface{}{1, `[0]`}, 0, nil},
+ {[]interface{}{1, `[1]`}, 1, nil},
+ {[]interface{}{1, `[[1]]`}, 0, nil},
+ {[]interface{}{"1", `[1]`}, 0, nil},
+ {[]interface{}{"1", `["1"]`}, 1, nil},
+ {[]interface{}{`{"a":1}`, `{"a":1}`}, 0, nil},
+ {[]interface{}{`{"a":1}`, `[{"a":1}]`}, 0, nil},
+ {[]interface{}{`{"a":1}`, `[{"a":1}, 1]`}, 0, nil},
+ {[]interface{}{`{"a":1}`, `["{\"a\":1}"]`}, 1, nil},
+ {[]interface{}{`{"a":1}`, `["{\"a\":1}", 1]`}, 1, nil},
+ }
+ for _, tt := range tbl {
+ args := types.MakeDatums(tt.input...)
+ f, err := fc.getFunction(ctx, datumsToConstants(args))
+ require.NoError(t, err, tt.input)
+ d, err := evalBuiltinFunc(f, chunk.Row{})
+ if tt.err == nil {
+ require.NoError(t, err, tt.input)
+ if tt.expected == nil {
+ require.True(t, d.IsNull(), tt.input)
+ } else {
+ require.Equal(t, int64(tt.expected.(int)), d.GetInt64(), tt.input)
+ }
+ } else {
+ require.True(t, tt.err.(*terror.Error).Equal(err), tt.input)
+ }
+ }
+}
+
func TestJSONContains(t *testing.T) {
ctx := createContext(t)
fc := funcs[ast.JSONContains]
@@ -466,6 +507,71 @@ func TestJSONContains(t *testing.T) {
}
}
+func TestJSONOverlaps(t *testing.T) {
+ ctx := createContext(t)
+ fc := funcs[ast.JSONOverlaps]
+ tbl := []struct {
+ input []any
+ expected any
+ err error
+ }{
+ {[]any{`[1,2,[1,3]]`, `a:1`}, 1, types.ErrInvalidJSONText},
+ {[]any{`a:1`, `1`}, 1, types.ErrInvalidJSONText},
+ {[]any{nil, `1`}, nil, nil},
+ {[]any{`1`, nil}, nil, nil},
+
+ {[]any{`[1, 2]`, `[2,3]`}, 1, nil},
+ {[]any{`[1, 2]`, `[2]`}, 1, nil},
+ {[]any{`[1, 2]`, `2`}, 1, nil},
+ {[]any{`[{"a":1}]`, `{"a":1}`}, 1, nil},
+ {[]any{`[{"a":1}]`, `{"a":1,"b":2}`}, 0, nil},
+ {[]any{`[{"a":1}]`, `{"a":2}`}, 0, nil},
+ {[]any{`{"a":[1,2]}`, `{"a":[1]}`}, 0, nil},
+ {[]any{`{"a":[1,2]}`, `{"a":[2,1]}`}, 0, nil},
+ {[]any{`[1,1,1]`, `1`}, 1, nil},
+ {[]any{`1`, `1`}, 1, nil},
+ {[]any{`0`, `1`}, 0, nil},
+ {[]any{`[[1,2], 3]`, `[1,[2,3]]`}, 0, nil},
+ {[]any{`[[1,2], 3]`, `[1,3]`}, 1, nil},
+ {[]any{`{"a":1,"b":10,"d":10}`, `{"a":5,"e":10,"f":1,"d":20}`}, 0, nil},
+ {[]any{`[4,5,"6",7]`, `6`}, 0, nil},
+ {[]any{`[4,5,6,7]`, `"6"`}, 0, nil},
+
+ {[]any{`[2,3]`, `[1, 2]`}, 1, nil},
+ {[]any{`[2]`, `[1, 2]`}, 1, nil},
+ {[]any{`2`, `[1, 2]`}, 1, nil},
+ {[]any{`{"a":1}`, `[{"a":1}]`}, 1, nil},
+ {[]any{`{"a":1,"b":2}`, `[{"a":1}]`}, 0, nil},
+ {[]any{`{"a":2}`, `[{"a":1}]`}, 0, nil},
+ {[]any{`{"a":[1]}`, `{"a":[1,2]}`}, 0, nil},
+ {[]any{`{"a":[2,1]}`, `{"a":[1,2]}`}, 0, nil},
+ {[]any{`1`, `[1,1,1]`}, 1, nil},
+ {[]any{`1`, `1`}, 1, nil},
+ {[]any{`1`, `0`}, 0, nil},
+ {[]any{`[1,[2,3]]`, `[[1,2], 3]`}, 0, nil},
+ {[]any{`[1,3]`, `[[1,2], 3]`}, 1, nil},
+ {[]any{`{"a":5,"e":10,"f":1,"d":20}`, `{"a":1,"b":10,"d":10}`}, 0, nil},
+ {[]any{`6`, `[4,5,"6",7]`}, 0, nil},
+ {[]any{`"6"`, `[4,5,6,7]`}, 0, nil},
+ }
+ for _, tt := range tbl {
+ args := types.MakeDatums(tt.input...)
+ f, err := fc.getFunction(ctx, datumsToConstants(args))
+ require.NoError(t, err, tt.input)
+ d, err := evalBuiltinFunc(f, chunk.Row{})
+ if tt.err == nil {
+ require.NoError(t, err, tt.input)
+ if tt.expected == nil {
+ require.True(t, d.IsNull(), tt.input)
+ } else {
+ require.Equal(t, int64(tt.expected.(int)), d.GetInt64(), tt.input)
+ }
+ } else {
+ require.True(t, tt.err.(*terror.Error).Equal(err), tt.input)
+ }
+ }
+}
+
func TestJSONContainsPath(t *testing.T) {
ctx := createContext(t)
fc := funcs[ast.JSONContainsPath]
diff --git a/expression/builtin_json_vec.go b/expression/builtin_json_vec.go
index fb24808ff2c73..0610a1f6ea3ca 100644
--- a/expression/builtin_json_vec.go
+++ b/expression/builtin_json_vec.go
@@ -274,6 +274,59 @@ func (b *builtinJSONArraySig) vecEvalJSON(input *chunk.Chunk, result *chunk.Colu
return nil
}
+func (b *builtinJSONMemberOfSig) vectorized() bool {
+ return true
+}
+
+func (b *builtinJSONMemberOfSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) error {
+ nr := input.NumRows()
+
+ targetCol, err := b.bufAllocator.get()
+ if err != nil {
+ return err
+ }
+ defer b.bufAllocator.put(targetCol)
+
+ if err := b.args[0].VecEvalJSON(b.ctx, input, targetCol); err != nil {
+ return err
+ }
+
+ objCol, err := b.bufAllocator.get()
+ if err != nil {
+ return err
+ }
+ defer b.bufAllocator.put(objCol)
+
+ if err := b.args[1].VecEvalJSON(b.ctx, input, objCol); err != nil {
+ return err
+ }
+
+ result.ResizeInt64(nr, false)
+ resI64s := result.Int64s()
+
+ result.MergeNulls(targetCol, objCol)
+ for i := 0; i < nr; i++ {
+ if result.IsNull(i) {
+ continue
+ }
+ obj := objCol.GetJSON(i)
+ target := targetCol.GetJSON(i)
+ if obj.TypeCode != types.JSONTypeCodeArray {
+ resI64s[i] = boolToInt64(types.CompareBinaryJSON(obj, target) == 0)
+ } else {
+ elemCount := obj.GetElemCount()
+ for j := 0; j < elemCount; j++ {
+ if types.CompareBinaryJSON(obj.ArrayGetElem(j), target) == 0 {
+ resI64s[i] = 1
+ break
+ }
+ }
+ }
+ }
+
+ return nil
+}
+
func (b *builtinJSONContainsSig) vectorized() bool {
return true
}
@@ -359,6 +412,51 @@ func (b *builtinJSONContainsSig) vecEvalInt(input *chunk.Chunk, result *chunk.Co
return nil
}
+func (b *builtinJSONOverlapsSig) vectorized() bool {
+ return true
+}
+
+func (b *builtinJSONOverlapsSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) error {
+ nr := input.NumRows()
+
+ objCol, err := b.bufAllocator.get()
+ if err != nil {
+ return err
+ }
+ defer b.bufAllocator.put(objCol)
+
+ if err := b.args[0].VecEvalJSON(b.ctx, input, objCol); err != nil {
+ return err
+ }
+
+ targetCol, err := b.bufAllocator.get()
+ if err != nil {
+ return err
+ }
+ defer b.bufAllocator.put(targetCol)
+
+ if err := b.args[1].VecEvalJSON(b.ctx, input, targetCol); err != nil {
+ return err
+ }
+
+ result.ResizeInt64(nr, false)
+ resI64s := result.Int64s()
+
+ result.MergeNulls(objCol, targetCol)
+ for i := 0; i < nr; i++ {
+ if result.IsNull(i) {
+ continue
+ }
+ if types.OverlapsBinaryJSON(objCol.GetJSON(i), targetCol.GetJSON(i)) {
+ resI64s[i] = 1
+ } else {
+ resI64s[i] = 0
+ }
+ }
+
+ return nil
+}
+
func (b *builtinJSONQuoteSig) vectorized() bool {
return true
}
diff --git a/expression/builtin_op.go b/expression/builtin_op.go
index cbb2afcc9377a..e15d1730c8cd9 100644
--- a/expression/builtin_op.go
+++ b/expression/builtin_op.go
@@ -718,7 +718,7 @@ func (c *unaryNotFunctionClass) getFunction(ctx sessionctx.Context, args []Expre
argTp := args[0].GetType().EvalType()
if argTp == types.ETTimestamp || argTp == types.ETDatetime || argTp == types.ETDuration {
argTp = types.ETInt
- } else if argTp == types.ETJson || argTp == types.ETString {
+ } else if argTp == types.ETString {
argTp = types.ETReal
}
@@ -739,6 +739,10 @@ func (c *unaryNotFunctionClass) getFunction(ctx sessionctx.Context, args []Expre
case types.ETInt:
sig = &builtinUnaryNotIntSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_UnaryNotInt)
+ case types.ETJson:
+ ctx.GetSessionVars().StmtCtx.AppendWarning(errJSONInBooleanContext)
+ sig = &builtinUnaryNotJSONSig{bf}
+ sig.setPbCode(tipb.ScalarFuncSig_UnaryNotJSON)
default:
return nil, errors.Errorf("unexpected types.EvalType %v", argTp)
}
@@ -808,6 +812,28 @@ func (b *builtinUnaryNotIntSig) evalInt(row chunk.Row) (int64, bool, error) {
return 0, false, nil
}
+type builtinUnaryNotJSONSig struct {
+ baseBuiltinFunc
+}
+
+func (b *builtinUnaryNotJSONSig) Clone() builtinFunc {
+ newSig := &builtinUnaryNotJSONSig{}
+ newSig.cloneFrom(&b.baseBuiltinFunc)
+ return newSig
+}
+
+func (b *builtinUnaryNotJSONSig) evalInt(row chunk.Row) (int64, bool, error) {
+ arg, isNull, err := b.args[0].EvalJSON(b.ctx, row)
+ if isNull || err != nil {
+ return 0, true, err
+ }
+
+ if types.CompareBinaryJSON(arg, types.CreateBinaryJSON(int64(0))) == 0 {
+ return 1, false, nil
+ }
+ return 0, false, nil
+}
+
type unaryMinusFunctionClass struct {
baseFunctionClass
}
diff --git a/expression/builtin_op_test.go b/expression/builtin_op_test.go
index 04024c90cb70f..c771b95da89d3 100644
--- a/expression/builtin_op_test.go
+++ b/expression/builtin_op_test.go
@@ -462,6 +462,8 @@ func TestUnaryNot(t *testing.T) {
{[]interface{}{"0.3"}, 0, false, false},
{[]interface{}{types.NewDecFromFloatForTest(0.3)}, 0, false, false},
{[]interface{}{nil}, 0, true, false},
+ {[]interface{}{types.CreateBinaryJSON(int64(0))}, 1, false, false},
+ {[]interface{}{types.CreateBinaryJSON(map[string]interface{}{"test": "test"})}, 0, false, false},
{[]interface{}{errors.New("must error")}, 0, false, true},
}
diff --git a/expression/builtin_other.go b/expression/builtin_other.go
index 0f4bd85d45b43..c5bd6738aa9df 100644
--- a/expression/builtin_other.go
+++ b/expression/builtin_other.go
@@ -165,7 +165,7 @@ func (c *inFunctionClass) verifyArgs(ctx sessionctx.Context, args []Expression)
case columnType.GetType() == mysql.TypeBit && constant.Value.Kind() == types.KindInt64:
if constant.Value.GetInt64() < 0 {
if MaybeOverOptimized4PlanCache(ctx, args) {
- ctx.GetSessionVars().StmtCtx.SkipPlanCache = true
+ ctx.GetSessionVars().StmtCtx.SetSkipPlanCache(errors.Errorf("skip plan-cache: Bit Column in (%v)", constant.Value.GetInt64()))
}
continue
}
diff --git a/expression/builtin_time.go b/expression/builtin_time.go
index 83f18d1c1653a..4349129e2dba6 100644
--- a/expression/builtin_time.go
+++ b/expression/builtin_time.go
@@ -2774,7 +2774,7 @@ func (du *baseDateArithmetical) getDateFromString(ctx sessionctx.Context, args [
}
sc := ctx.GetSessionVars().StmtCtx
- date, err := types.ParseTime(sc, dateStr, dateTp, types.MaxFsp)
+ date, err := types.ParseTime(sc, dateStr, dateTp, types.MaxFsp, nil)
if err != nil {
err = handleInvalidTimeError(ctx, err)
if err != nil {
@@ -3184,7 +3184,7 @@ func (du *baseDateArithmetical) vecGetDateFromString(b *baseBuiltinFunc, input *
dateTp = mysql.TypeDatetime
}
- date, err := types.ParseTime(sc, dateStr, dateTp, types.MaxFsp)
+ date, err := types.ParseTime(sc, dateStr, dateTp, types.MaxFsp, nil)
if err != nil {
err = handleInvalidTimeError(b.ctx, err)
if err != nil {
@@ -4348,7 +4348,7 @@ func (b *builtinTimestamp1ArgSig) evalTime(row chunk.Row) (types.Time, bool, err
if b.isFloat {
tm, err = types.ParseTimeFromFloatString(sc, s, mysql.TypeDatetime, types.GetFsp(s))
} else {
- tm, err = types.ParseTime(sc, s, mysql.TypeDatetime, types.GetFsp(s))
+ tm, err = types.ParseTime(sc, s, mysql.TypeDatetime, types.GetFsp(s), nil)
}
if err != nil {
return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err)
@@ -4380,7 +4380,7 @@ func (b *builtinTimestamp2ArgsSig) evalTime(row chunk.Row) (types.Time, bool, er
if b.isFloat {
tm, err = types.ParseTimeFromFloatString(sc, arg0, mysql.TypeDatetime, types.GetFsp(arg0))
} else {
- tm, err = types.ParseTime(sc, arg0, mysql.TypeDatetime, types.GetFsp(arg0))
+ tm, err = types.ParseTime(sc, arg0, mysql.TypeDatetime, types.GetFsp(arg0), nil)
}
if err != nil {
return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err)
@@ -4431,7 +4431,7 @@ func (c *timestampLiteralFunctionClass) getFunction(ctx sessionctx.Context, args
if !timestampPattern.MatchString(str) {
return nil, types.ErrWrongValue.GenWithStackByArgs(types.DateTimeStr, str)
}
- tm, err := types.ParseTime(ctx.GetSessionVars().StmtCtx, str, mysql.TypeDatetime, types.GetFsp(str))
+ tm, err := types.ParseTime(ctx.GetSessionVars().StmtCtx, str, mysql.TypeDatetime, types.GetFsp(str), nil)
if err != nil {
return nil, err
}
@@ -4539,7 +4539,7 @@ func isDuration(str string) bool {
// strDatetimeAddDuration adds duration to datetime string, returns a string value.
func strDatetimeAddDuration(sc *stmtctx.StatementContext, d string, arg1 types.Duration) (result string, isNull bool, err error) {
- arg0, err := types.ParseTime(sc, d, mysql.TypeDatetime, types.MaxFsp)
+ arg0, err := types.ParseTime(sc, d, mysql.TypeDatetime, types.MaxFsp, nil)
if err != nil {
// Return a warning regardless of the sql_mode, this is compatible with MySQL.
sc.AppendWarning(err)
@@ -4576,7 +4576,7 @@ func strDurationAddDuration(sc *stmtctx.StatementContext, d string, arg1 types.D
// strDatetimeSubDuration subtracts duration from datetime string, returns a string value.
func strDatetimeSubDuration(sc *stmtctx.StatementContext, d string, arg1 types.Duration) (result string, isNull bool, err error) {
- arg0, err := types.ParseTime(sc, d, mysql.TypeDatetime, types.MaxFsp)
+ arg0, err := types.ParseTime(sc, d, mysql.TypeDatetime, types.MaxFsp, nil)
if err != nil {
// Return a warning regardless of the sql_mode, this is compatible with MySQL.
sc.AppendWarning(err)
@@ -6087,7 +6087,7 @@ func (c *timestampAddFunctionClass) getFunction(ctx sessionctx.Context, args []E
if err := c.verifyArgs(args); err != nil {
return nil, err
}
- bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETString, types.ETString, types.ETInt, types.ETDatetime)
+ bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETString, types.ETString, types.ETReal, types.ETDatetime)
if err != nil {
return nil, err
}
@@ -6120,6 +6120,82 @@ func (b *builtinTimestampAddSig) Clone() builtinFunc {
return newSig
}
+var (
+ minDatetimeInGoTime, _ = types.MinDatetime.GoTime(time.Local)
+ minDatetimeNanos = float64(minDatetimeInGoTime.Unix())*1e9 + float64(minDatetimeInGoTime.Nanosecond())
+ maxDatetimeInGoTime, _ = types.MaxDatetime.GoTime(time.Local)
+ maxDatetimeNanos = float64(maxDatetimeInGoTime.Unix())*1e9 + float64(maxDatetimeInGoTime.Nanosecond())
+ minDatetimeMonths = float64(types.MinDatetime.Year()*12 + types.MinDatetime.Month() - 1) // 0001-01-01 00:00:00
+ maxDatetimeMonths = float64(types.MaxDatetime.Year()*12 + types.MaxDatetime.Month() - 1) // 9999-12-31 00:00:00
+)
+
+func validAddTime(nano1 float64, nano2 float64) bool {
+ return nano1+nano2 >= minDatetimeNanos && nano1+nano2 <= maxDatetimeNanos
+}
+
+func validAddMonth(month1 float64, year, month int) bool {
+ tmp := month1 + float64(year)*12 + float64(month-1)
+ return tmp >= minDatetimeMonths && tmp <= maxDatetimeMonths
+}
+
+func addUnitToTime(unit string, t time.Time, v float64) (time.Time, bool, error) {
+ s := math.Trunc(v * 1000000)
+ // round to the nearest int
+ v = math.Round(v)
+ var tb time.Time
+ nano := float64(t.Unix())*1e9 + float64(t.Nanosecond())
+ switch unit {
+ case "MICROSECOND":
+ if !validAddTime(v*float64(time.Microsecond), nano) {
+ return tb, true, nil
+ }
+ tb = t.Add(time.Duration(v) * time.Microsecond)
+ case "SECOND":
+ if !validAddTime(s*float64(time.Microsecond), nano) {
+ return tb, true, nil
+ }
+ tb = t.Add(time.Duration(s) * time.Microsecond)
+ case "MINUTE":
+ if !validAddTime(v*float64(time.Minute), nano) {
+ return tb, true, nil
+ }
+ tb = t.Add(time.Duration(v) * time.Minute)
+ case "HOUR":
+ if !validAddTime(v*float64(time.Hour), nano) {
+ return tb, true, nil
+ }
+ tb = t.Add(time.Duration(v) * time.Hour)
+ case "DAY":
+ if !validAddTime(v*24*float64(time.Hour), nano) {
+ return tb, true, nil
+ }
+ tb = t.AddDate(0, 0, int(v))
+ case "WEEK":
+ if !validAddTime(v*24*7*float64(time.Hour), nano) {
+ return tb, true, nil
+ }
+ tb = t.AddDate(0, 0, 7*int(v))
+ case "MONTH":
+ if !validAddMonth(v, t.Year(), int(t.Month())) {
+ return tb, true, nil
+ }
+ tb = t.AddDate(0, int(v), 0)
+ case "QUARTER":
+ if !validAddMonth(v*3, t.Year(), int(t.Month())) {
+ return tb, true, nil
+ }
+ tb = t.AddDate(0, 3*int(v), 0)
+ case "YEAR":
+ if !validAddMonth(v*12, t.Year(), int(t.Month())) {
+ return tb, true, nil
+ }
+ tb = t.AddDate(int(v), 0, 0)
+ default:
+ return tb, false, types.ErrWrongValue.GenWithStackByArgs(types.TimeStr, unit)
+ }
+ return tb, false, nil
+}
+
// evalString evals a builtinTimestampAddSig.
// See https://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_timestampadd
func (b *builtinTimestampAddSig) evalString(row chunk.Row) (string, bool, error) {
@@ -6127,7 +6203,7 @@ func (b *builtinTimestampAddSig) evalString(row chunk.Row) (string, bool, error)
if isNull || err != nil {
return "", isNull, err
}
- v, isNull, err := b.args[1].EvalInt(b.ctx, row)
+ v, isNull, err := b.args[1].EvalReal(b.ctx, row)
if isNull || err != nil {
return "", isNull, err
}
@@ -6140,30 +6216,17 @@ func (b *builtinTimestampAddSig) evalString(row chunk.Row) (string, bool, error)
b.ctx.GetSessionVars().StmtCtx.AppendWarning(err)
return "", true, nil
}
- var tb time.Time
+ tb, overflow, err := addUnitToTime(unit, tm1, v)
+ if err != nil {
+ return "", true, err
+ }
+ if overflow {
+ return "", true, handleInvalidTimeError(b.ctx, types.ErrDatetimeFunctionOverflow.GenWithStackByArgs("datetime"))
+ }
fsp := types.DefaultFsp
- switch unit {
- case "MICROSECOND":
- tb = tm1.Add(time.Duration(v) * time.Microsecond)
+ // use MaxFsp when microsecond is not zero
+ if tb.Nanosecond()/1000 != 0 {
fsp = types.MaxFsp
- case "SECOND":
- tb = tm1.Add(time.Duration(v) * time.Second)
- case "MINUTE":
- tb = tm1.Add(time.Duration(v) * time.Minute)
- case "HOUR":
- tb = tm1.Add(time.Duration(v) * time.Hour)
- case "DAY":
- tb = tm1.AddDate(0, 0, int(v))
- case "WEEK":
- tb = tm1.AddDate(0, 0, 7*int(v))
- case "MONTH":
- tb = tm1.AddDate(0, int(v), 0)
- case "QUARTER":
- tb = tm1.AddDate(0, 3*int(v), 0)
- case "YEAR":
- tb = tm1.AddDate(int(v), 0, 0)
- default:
- return "", true, types.ErrWrongValue.GenWithStackByArgs(types.TimeStr, unit)
}
r := types.NewTime(types.FromGoTime(tb), b.resolveType(arg.Type(), unit), fsp)
if err = r.Check(b.ctx.GetSessionVars().StmtCtx); err != nil {
diff --git a/expression/builtin_time_test.go b/expression/builtin_time_test.go
index 55973f9dab5f8..2010bf0a225b5 100644
--- a/expression/builtin_time_test.go
+++ b/expression/builtin_time_test.go
@@ -2506,7 +2506,7 @@ func TestTimestampAdd(t *testing.T) {
ctx := createContext(t)
tests := []struct {
unit string
- interval int64
+ interval float64
date interface{}
expect string
}{
@@ -2514,11 +2514,22 @@ func TestTimestampAdd(t *testing.T) {
{"WEEK", 1, "2003-01-02 23:59:59", "2003-01-09 23:59:59"},
{"MICROSECOND", 1, 950501, "1995-05-01 00:00:00.000001"},
{"DAY", 28768, 0, ""},
+ {"QUARTER", 3, "1995-05-01", "1996-02-01 00:00:00"},
+ {"SECOND", 1.1, "1995-05-01", "1995-05-01 00:00:01.100000"},
+ {"SECOND", -1, "1995-05-01", "1995-04-30 23:59:59"},
+ {"SECOND", -1.1, "1995-05-01", "1995-04-30 23:59:58.900000"},
+ {"SECOND", 9.9999e-6, "1995-05-01", "1995-05-01 00:00:00.000009"},
+ {"SECOND", 9.9999e-7, "1995-05-01", "1995-05-01 00:00:00"},
+ {"SECOND", -9.9999e-6, "1995-05-01", "1995-04-30 23:59:59.999991"},
+ {"SECOND", -9.9999e-7, "1995-05-01", "1995-05-01 00:00:00"},
+ {"MINUTE", 1.5, "1995-05-01 00:00:00", "1995-05-01 00:02:00"},
+ {"MINUTE", 1.5, "1995-05-01 00:00:00.000000", "1995-05-01 00:02:00"},
+ {"MICROSECOND", -100, "1995-05-01 00:00:00.0001", "1995-05-01 00:00:00"},
}
fc := funcs[ast.TimestampAdd]
for _, test := range tests {
- dat := []types.Datum{types.NewStringDatum(test.unit), types.NewIntDatum(test.interval), types.NewDatum(test.date)}
+ dat := []types.Datum{types.NewStringDatum(test.unit), types.NewFloat64Datum(test.interval), types.NewDatum(test.date)}
f, err := fc.getFunction(ctx, datumsToConstants(dat))
require.NoError(t, err)
d, err := evalBuiltinFunc(f, chunk.Row{})
diff --git a/expression/builtin_time_vec.go b/expression/builtin_time_vec.go
index 0d6b4321095f5..34f3fb87f14c0 100644
--- a/expression/builtin_time_vec.go
+++ b/expression/builtin_time_vec.go
@@ -1133,7 +1133,6 @@ func (b *builtinStrToDateDurationSig) vecEvalDuration(input *chunk.Chunk, result
result.MergeNulls(bufStrings, bufFormats)
d64s := result.GoDurations()
sc := b.ctx.GetSessionVars().StmtCtx
- hasNoZeroDateMode := b.ctx.GetSessionVars().SQLMode.HasNoZeroDateMode()
for i := 0; i < n; i++ {
if result.IsNull(i) {
continue
@@ -1147,13 +1146,6 @@ func (b *builtinStrToDateDurationSig) vecEvalDuration(input *chunk.Chunk, result
result.SetNull(i, true)
continue
}
- if hasNoZeroDateMode && (t.Year() == 0 || t.Month() == 0 || t.Day() == 0) {
- if err := handleInvalidTimeError(b.ctx, types.ErrWrongValue.GenWithStackByArgs(types.DateTimeStr, t.String())); err != nil {
- return err
- }
- result.SetNull(i, true)
- continue
- }
t.SetFsp(b.tp.GetDecimal())
dur, err := t.ConvertToDuration()
if err != nil {
@@ -1703,7 +1695,7 @@ func (b *builtinTimestampAddSig) vecEvalString(input *chunk.Chunk, result *chunk
return err
}
defer b.bufAllocator.put(buf1)
- if err := b.args[1].VecEvalInt(b.ctx, input, buf1); err != nil {
+ if err := b.args[1].VecEvalReal(b.ctx, input, buf1); err != nil {
return err
}
@@ -1717,7 +1709,7 @@ func (b *builtinTimestampAddSig) vecEvalString(input *chunk.Chunk, result *chunk
}
result.ReserveString(n)
- nums := buf1.Int64s()
+ nums := buf1.Float64s()
ds := buf2.Times()
for i := 0; i < n; i++ {
if buf.IsNull(i) || buf1.IsNull(i) || buf2.IsNull(i) {
@@ -1735,30 +1727,21 @@ func (b *builtinTimestampAddSig) vecEvalString(input *chunk.Chunk, result *chunk
result.AppendNull()
continue
}
- var tb time.Time
+ tb, overflow, err := addUnitToTime(unit, tm1, v)
+ if err != nil {
+ return err
+ }
+ if overflow {
+ if err = handleInvalidTimeError(b.ctx, types.ErrDatetimeFunctionOverflow.GenWithStackByArgs("datetime")); err != nil {
+ return err
+ }
+ result.AppendNull()
+ continue
+ }
fsp := types.DefaultFsp
- switch unit {
- case "MICROSECOND":
- tb = tm1.Add(time.Duration(v) * time.Microsecond)
+ // use MaxFsp when microsecond is not zero
+ if tb.Nanosecond()/1000 != 0 {
fsp = types.MaxFsp
- case "SECOND":
- tb = tm1.Add(time.Duration(v) * time.Second)
- case "MINUTE":
- tb = tm1.Add(time.Duration(v) * time.Minute)
- case "HOUR":
- tb = tm1.Add(time.Duration(v) * time.Hour)
- case "DAY":
- tb = tm1.AddDate(0, 0, int(v))
- case "WEEK":
- tb = tm1.AddDate(0, 0, 7*int(v))
- case "MONTH":
- tb = tm1.AddDate(0, int(v), 0)
- case "QUARTER":
- tb = tm1.AddDate(0, 3*int(v), 0)
- case "YEAR":
- tb = tm1.AddDate(int(v), 0, 0)
- default:
- return types.ErrWrongValue.GenWithStackByArgs(types.TimeStr, unit)
}
r := types.NewTime(types.FromGoTime(tb), b.resolveType(arg.Type(), unit), fsp)
if err = r.Check(b.ctx.GetSessionVars().StmtCtx); err != nil {
@@ -2677,7 +2660,7 @@ func (b *builtinTimestamp1ArgSig) vecEvalTime(input *chunk.Chunk, result *chunk.
if b.isFloat {
tm, err = types.ParseTimeFromFloatString(sc, s, mysql.TypeDatetime, types.GetFsp(s))
} else {
- tm, err = types.ParseTime(sc, s, mysql.TypeDatetime, types.GetFsp(s))
+ tm, err = types.ParseTime(sc, s, mysql.TypeDatetime, types.GetFsp(s), nil)
}
if err != nil {
if err = handleInvalidTimeError(b.ctx, err); err != nil {
@@ -2730,7 +2713,7 @@ func (b *builtinTimestamp2ArgsSig) vecEvalTime(input *chunk.Chunk, result *chunk
if b.isFloat {
tm, err = types.ParseTimeFromFloatString(sc, arg0, mysql.TypeDatetime, types.GetFsp(arg0))
} else {
- tm, err = types.ParseTime(sc, arg0, mysql.TypeDatetime, types.GetFsp(arg0))
+ tm, err = types.ParseTime(sc, arg0, mysql.TypeDatetime, types.GetFsp(arg0), nil)
}
if err != nil {
if err = handleInvalidTimeError(b.ctx, err); err != nil {
diff --git a/expression/collation.go b/expression/collation.go
index eebab0aa5bc1f..8b11d3198a40e 100644
--- a/expression/collation.go
+++ b/expression/collation.go
@@ -299,14 +299,6 @@ func deriveCollation(ctx sessionctx.Context, funcName string, args []Expression,
return ec, nil
}
-// DeriveCollationFromExprs derives collation information from these expressions.
-// Deprecated, use CheckAndDeriveCollationFromExprs instead.
-// TODO: remove this function after the all usage is replaced by CheckAndDeriveCollationFromExprs
-func DeriveCollationFromExprs(ctx sessionctx.Context, exprs ...Expression) (dstCharset, dstCollation string) {
- collation := inferCollation(exprs...)
- return collation.Charset, collation.Collation
-}
-
// CheckAndDeriveCollationFromExprs derives collation information from these expressions, return error if derives collation error.
func CheckAndDeriveCollationFromExprs(ctx sessionctx.Context, funcName string, evalType types.EvalType, args ...Expression) (et *ExprCollation, err error) {
ec := inferCollation(args...)
diff --git a/expression/constant_fold.go b/expression/constant_fold.go
index f43cb75a1ff3c..0d73757b3161b 100644
--- a/expression/constant_fold.go
+++ b/expression/constant_fold.go
@@ -178,7 +178,12 @@ func foldConstant(expr Expression) (Expression, bool) {
}
}
if !allConstArg {
- if !hasNullArg || !sc.InNullRejectCheck || x.FuncName.L == ast.NullEQ {
+ // try to optimize on the situation when not all arguments are const
+ // for most functions, if one of the arguments are NULL, the result can be a constant (NULL or something else)
+ //
+ // NullEQ and ConcatWS are excluded, because they could have different value when the non-constant value is
+ // 1 or NULL. For example, concat_ws(NULL, NULL) gives NULL, but concat_ws(1, NULL) gives ''
+ if !hasNullArg || !sc.InNullRejectCheck || x.FuncName.L == ast.NullEQ || x.FuncName.L == ast.ConcatWS {
return expr, isDeferredConst
}
constArgs := make([]Expression, len(args))
diff --git a/expression/constant_test.go b/expression/constant_test.go
index 82c3d8e489fdf..2c2e226eabb5c 100644
--- a/expression/constant_test.go
+++ b/expression/constant_test.go
@@ -217,6 +217,15 @@ func TestConstantFolding(t *testing.T) {
condition: newFunction(ast.LT, newColumn(0), newFunction(ast.Plus, newColumn(1), newFunction(ast.Plus, newLonglong(2), newLonglong(1)))),
result: "lt(Column#0, plus(Column#1, 3))",
},
+ {
+ condition: func() Expression {
+ expr := newFunction(ast.ConcatWS, newColumn(0), NewNull())
+ function := expr.(*ScalarFunction)
+ function.GetCtx().GetSessionVars().StmtCtx.InNullRejectCheck = true
+ return function
+ }(),
+ result: "concat_ws(cast(Column#0, var_string(20)), )",
+ },
}
for _, tt := range tests {
newConds := FoldConstant(tt.condition)
diff --git a/expression/errors.go b/expression/errors.go
index 0db38645f78d4..536ef065d7664 100644
--- a/expression/errors.go
+++ b/expression/errors.go
@@ -37,6 +37,10 @@ var (
ErrInvalidTableSample = dbterror.ClassExpression.NewStd(mysql.ErrInvalidTableSample)
ErrInternal = dbterror.ClassOptimizer.NewStd(mysql.ErrInternal)
ErrNoDB = dbterror.ClassOptimizer.NewStd(mysql.ErrNoDB)
+ ErrNotSupportedYet = dbterror.ClassExpression.NewStd(mysql.ErrNotSupportedYet)
+ ErrInvalidJSONForFuncIndex = dbterror.ClassExpression.NewStd(mysql.ErrInvalidJSONValueForFuncIndex)
+ ErrDataOutOfRangeFuncIndex = dbterror.ClassExpression.NewStd(mysql.ErrDataOutOfRangeFunctionalIndex)
+ ErrFuncIndexDataIsTooLong = dbterror.ClassExpression.NewStd(mysql.ErrFunctionalIndexDataIsTooLong)
// All the un-exported errors are defined here:
errFunctionNotExists = dbterror.ClassExpression.NewStd(mysql.ErrSpDoesNotExist)
@@ -56,6 +60,7 @@ var (
errSpecificAccessDenied = dbterror.ClassExpression.NewStd(mysql.ErrSpecificAccessDenied)
errUserLockDeadlock = dbterror.ClassExpression.NewStd(mysql.ErrUserLockDeadlock)
errUserLockWrongName = dbterror.ClassExpression.NewStd(mysql.ErrUserLockWrongName)
+ errJSONInBooleanContext = dbterror.ClassExpression.NewStd(mysql.ErrJSONInBooleanContext)
// Sequence usage privilege check.
errSequenceAccessDenied = dbterror.ClassExpression.NewStd(mysql.ErrTableaccessDenied)
diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go
index 802d559eb3c80..6756601be0e04 100644
--- a/expression/expr_to_pb_test.go
+++ b/expression/expr_to_pb_test.go
@@ -528,6 +528,18 @@ func TestExprPushDownToFlash(t *testing.T) {
require.NoError(t, err)
exprs = append(exprs, function)
+ // json_extract
+ function, err = NewFunction(mock.NewContext(), ast.JSONExtract, types.NewFieldType(mysql.TypeJSON), jsonColumn, stringColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
+ // json_unquote argument is cast(json as string)
+ subFunc, subErr := NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeString), jsonColumn)
+ require.NoError(t, subErr)
+ function, err = NewFunction(mock.NewContext(), ast.JSONUnquote, types.NewFieldType(mysql.TypeString), subFunc)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
// lpad
function, err = NewFunction(mock.NewContext(), ast.Lpad, types.NewFieldType(mysql.TypeString), stringColumn, int32Column, stringColumn)
require.NoError(t, err)
@@ -558,6 +570,14 @@ func TestExprPushDownToFlash(t *testing.T) {
require.NoError(t, err)
exprs = append(exprs, function)
+ // ExtractDuration
+ extractDurationUnitCol := new(Constant)
+ extractDurationUnitCol.Value = types.NewStringDatum("microsecond")
+ extractDurationUnitCol.RetType = types.NewFieldType(mysql.TypeString)
+ function, err = NewFunction(mock.NewContext(), ast.Extract, types.NewFieldType(mysql.TypeLonglong), extractDurationUnitCol, durationColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
// CastIntAsInt
function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeLonglong), intColumn)
require.NoError(t, err)
@@ -631,6 +651,11 @@ func TestExprPushDownToFlash(t *testing.T) {
require.NoError(t, err)
exprs = append(exprs, function)
+ // CastJsonAsString
+ function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeString), jsonColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
// CastIntAsTime
function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeDatetime), intColumn)
require.NoError(t, err)
@@ -950,6 +975,11 @@ func TestExprPushDownToFlash(t *testing.T) {
exprs = exprs[:0]
+ // json_unquote's argument is not cast(json as string)
+ function, err = NewFunction(mock.NewContext(), ast.JSONUnquote, types.NewFieldType(mysql.TypeString), stringColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
// Substring2Args: can not be pushed
function, err = NewFunction(mock.NewContext(), ast.Substr, types.NewFieldType(mysql.TypeString), binaryStringColumn, intColumn)
require.NoError(t, err)
@@ -1065,6 +1095,26 @@ func TestExprPushDownToFlash(t *testing.T) {
require.NoError(t, err)
exprs = append(exprs, function)
+ // regexp_like: supported
+ function, err = NewFunction(mock.NewContext(), ast.RegexpLike, types.NewFieldType(mysql.TypeLonglong), binaryStringColumn, binaryStringColumn, binaryStringColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
+ // regexp_instr: supported
+ function, err = NewFunction(mock.NewContext(), ast.RegexpInStr, types.NewFieldType(mysql.TypeLonglong), stringColumn, stringColumn, intColumn, intColumn, intColumn, stringColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
+ // regexp_substr: supported
+ function, err = NewFunction(mock.NewContext(), ast.RegexpSubstr, types.NewFieldType(mysql.TypeString), stringColumn, stringColumn, intColumn, intColumn, stringColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
+ // regexp_replace: supported
+ function, err = NewFunction(mock.NewContext(), ast.RegexpReplace, types.NewFieldType(mysql.TypeString), stringColumn, stringColumn, stringColumn, intColumn, intColumn, stringColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
// greatest
function, err = NewFunction(mock.NewContext(), ast.Greatest, types.NewFieldType(mysql.TypeLonglong), int32Column, intColumn)
require.NoError(t, err)
@@ -1074,6 +1124,10 @@ func TestExprPushDownToFlash(t *testing.T) {
require.NoError(t, err)
exprs = append(exprs, function)
+ function, err = NewFunction(mock.NewContext(), ast.Greatest, types.NewFieldType(mysql.TypeString), stringColumn, stringColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
// least
function, err = NewFunction(mock.NewContext(), ast.Least, types.NewFieldType(mysql.TypeLonglong), int32Column, intColumn)
require.NoError(t, err)
@@ -1082,6 +1136,11 @@ func TestExprPushDownToFlash(t *testing.T) {
function, err = NewFunction(mock.NewContext(), ast.Least, types.NewFieldType(mysql.TypeDouble), float32Column, intColumn)
require.NoError(t, err)
exprs = append(exprs, function)
+
+ function, err = NewFunction(mock.NewContext(), ast.Least, types.NewFieldType(mysql.TypeString), stringColumn, stringColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
// is true
function, err = NewFunction(mock.NewContext(), ast.IsTruthWithoutNull, types.NewFieldType(mysql.TypeLonglong), int32Column)
require.NoError(t, err)
@@ -1194,6 +1253,11 @@ func TestExprPushDownToFlash(t *testing.T) {
require.Equal(t, tipb.ScalarFuncSig_CastTimeAsDuration, function.(*ScalarFunction).Function.PbCode())
exprs = append(exprs, function)
+ // Unhex
+ function, err = NewFunction(mock.NewContext(), ast.Unhex, types.NewFieldType(mysql.TypeString), stringColumn)
+ require.NoError(t, err)
+ exprs = append(exprs, function)
+
pushed, remained = PushDownExprs(sc, exprs, client, kv.TiFlash)
require.Len(t, pushed, len(exprs))
require.Len(t, remained, 0)
diff --git a/expression/expression.go b/expression/expression.go
index fefa1c403c959..e3ab332f8a00b 100644
--- a/expression/expression.go
+++ b/expression/expression.go
@@ -59,7 +59,7 @@ var EvalAstExpr func(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, e
// RewriteAstExpr rewrites ast expression directly.
// Note: initialized in planner/core
// import expression and planner/core together to use EvalAstExpr
-var RewriteAstExpr func(sctx sessionctx.Context, expr ast.ExprNode, schema *Schema, names types.NameSlice) (Expression, error)
+var RewriteAstExpr func(sctx sessionctx.Context, expr ast.ExprNode, schema *Schema, names types.NameSlice, allowCastArray bool) (Expression, error)
// VecExpr contains all vectorized evaluation methods.
type VecExpr interface {
@@ -816,7 +816,7 @@ func SplitDNFItems(onExpr Expression) []Expression {
// If the Expression is a non-constant value, it means the result is unknown.
func EvaluateExprWithNull(ctx sessionctx.Context, schema *Schema, expr Expression) Expression {
if MaybeOverOptimized4PlanCache(ctx, []Expression{expr}) {
- return expr
+ ctx.GetSessionVars().StmtCtx.SetSkipPlanCache(errors.New("skip plan-cache: %v affects null check"))
}
if ctx.GetSessionVars().StmtCtx.InNullRejectCheck {
expr, _ = evaluateExprWithNullInNullRejectCheck(ctx, schema, expr)
@@ -998,7 +998,7 @@ func ColumnInfos2ColumnsAndNames(ctx sessionctx.Context, dbName, tblName model.C
if err != nil {
return nil, nil, errors.Trace(err)
}
- e, err := RewriteAstExpr(ctx, expr, mockSchema, names)
+ e, err := RewriteAstExpr(ctx, expr, mockSchema, names, true)
if err != nil {
return nil, nil, errors.Trace(err)
}
@@ -1071,8 +1071,7 @@ func scalarExprSupportedByTiKV(sf *ScalarFunction) bool {
// json functions.
ast.JSONType, ast.JSONExtract, ast.JSONObject, ast.JSONArray, ast.JSONMerge, ast.JSONSet,
ast.JSONInsert /*ast.JSONReplace,*/, ast.JSONRemove, ast.JSONLength,
- // FIXME: JSONUnquote is incompatible with Coprocessor
- ast.JSONUnquote, ast.JSONContains,
+ ast.JSONUnquote, ast.JSONContains, ast.JSONValid,
// date functions.
ast.Date, ast.Week /* ast.YearWeek, ast.ToSeconds */, ast.DateDiff,
@@ -1151,10 +1150,10 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool {
ast.Sqrt, ast.Log, ast.Log2, ast.Log10, ast.Ln, ast.Exp, ast.Pow, ast.Sign,
ast.Radians, ast.Degrees, ast.Conv, ast.CRC32,
- ast.JSONLength, ast.Repeat,
+ ast.JSONLength, ast.JSONExtract, ast.JSONUnquote, ast.Repeat,
ast.InetNtoa, ast.InetAton, ast.Inet6Ntoa, ast.Inet6Aton,
ast.Coalesce, ast.ASCII, ast.Length, ast.Trim, ast.Position, ast.Format, ast.Elt,
- ast.LTrim, ast.RTrim, ast.Lpad, ast.Rpad, ast.Regexp,
+ ast.LTrim, ast.RTrim, ast.Lpad, ast.Rpad,
ast.Hour, ast.Minute, ast.Second, ast.MicroSecond,
ast.TimeToSec:
switch function.Function.PbCode() {
@@ -1164,6 +1163,18 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool {
tipb.ScalarFuncSig_IfDuration,
tipb.ScalarFuncSig_CaseWhenDuration:
return false
+ case tipb.ScalarFuncSig_JsonUnquoteSig:
+ // TiFlash json_unquote now only supports json string generated by cast(json as string)
+ if childFunc, ok := function.GetArgs()[0].(*ScalarFunction); ok {
+ return childFunc.Function.PbCode() == tipb.ScalarFuncSig_CastJsonAsString
+ }
+ return false
+ }
+ return true
+ case ast.Regexp, ast.RegexpLike, ast.RegexpInStr, ast.RegexpSubstr, ast.RegexpReplace:
+ funcCharset, funcCollation := function.Function.CharsetAndCollation()
+ if funcCharset == charset.CharsetBin && funcCollation == charset.CollationBin {
+ return false
}
return true
case ast.Substr, ast.Substring, ast.Left, ast.Right, ast.CharLength, ast.SubstringIndex, ast.Reverse:
@@ -1195,7 +1206,7 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool {
tipb.ScalarFuncSig_CastStringAsDecimal /*, tipb.ScalarFuncSig_CastDurationAsDecimal, tipb.ScalarFuncSig_CastJsonAsDecimal*/ :
return function.RetType.IsDecimalValid()
case tipb.ScalarFuncSig_CastDecimalAsString, tipb.ScalarFuncSig_CastIntAsString, tipb.ScalarFuncSig_CastRealAsString, tipb.ScalarFuncSig_CastTimeAsString,
- tipb.ScalarFuncSig_CastStringAsString /*, tipb.ScalarFuncSig_CastDurationAsString, tipb.ScalarFuncSig_CastJsonAsString*/ :
+ tipb.ScalarFuncSig_CastStringAsString, tipb.ScalarFuncSig_CastJsonAsString /*, tipb.ScalarFuncSig_CastDurationAsString*/ :
return true
case tipb.ScalarFuncSig_CastDecimalAsTime, tipb.ScalarFuncSig_CastIntAsTime, tipb.ScalarFuncSig_CastRealAsTime, tipb.ScalarFuncSig_CastTimeAsTime,
tipb.ScalarFuncSig_CastStringAsTime /*, tipb.ScalarFuncSig_CastDurationAsTime, tipb.ScalarFuncSig_CastJsonAsTime*/ :
@@ -1227,7 +1238,7 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool {
}
case ast.Extract:
switch function.Function.PbCode() {
- case tipb.ScalarFuncSig_ExtractDatetime:
+ case tipb.ScalarFuncSig_ExtractDatetime, tipb.ScalarFuncSig_ExtractDuration:
return true
}
case ast.Replace:
@@ -1251,12 +1262,12 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool {
case ast.Least, ast.Greatest:
switch function.Function.PbCode() {
case tipb.ScalarFuncSig_GreatestInt, tipb.ScalarFuncSig_GreatestReal,
- tipb.ScalarFuncSig_LeastInt, tipb.ScalarFuncSig_LeastReal:
+ tipb.ScalarFuncSig_LeastInt, tipb.ScalarFuncSig_LeastReal, tipb.ScalarFuncSig_LeastString, tipb.ScalarFuncSig_GreatestString:
return true
}
case ast.IsTruthWithNull, ast.IsTruthWithoutNull, ast.IsFalsity:
return true
- case ast.Hex, ast.Bin:
+ case ast.Hex, ast.Unhex, ast.Bin:
return true
case ast.GetFormat:
return true
@@ -1347,12 +1358,15 @@ func canScalarFuncPushDown(scalarFunc *ScalarFunction, pc PbConverter, storeType
panic(errors.Errorf("unspecified PbCode: %T", scalarFunc.Function))
})
}
+ storageName := storeType.Name()
+ if storeType == kv.UnSpecified {
+ storageName = "storage layer"
+ }
+ warnErr := errors.New("Scalar function '" + scalarFunc.FuncName.L + "'(signature: " + scalarFunc.Function.PbCode().String() + ", return type: " + scalarFunc.RetType.CompactStr() + ") is not supported to push down to " + storageName + " now.")
if pc.sc.InExplainStmt {
- storageName := storeType.Name()
- if storeType == kv.UnSpecified {
- storageName = "storage layer"
- }
- pc.sc.AppendWarning(errors.New("Scalar function '" + scalarFunc.FuncName.L + "'(signature: " + scalarFunc.Function.PbCode().String() + ", return type: " + scalarFunc.RetType.CompactStr() + ") is not supported to push down to " + storageName + " now."))
+ pc.sc.AppendWarning(warnErr)
+ } else {
+ pc.sc.AppendExtraWarning(warnErr)
}
return false
}
@@ -1382,14 +1396,20 @@ func canExprPushDown(expr Expression, pc PbConverter, storeType kv.StoreType, ca
if expr.GetType().GetType() == mysql.TypeEnum && canEnumPush {
break
}
+ warnErr := errors.New("Expression about '" + expr.String() + "' can not be pushed to TiFlash because it contains unsupported calculation of type '" + types.TypeStr(expr.GetType().GetType()) + "'.")
if pc.sc.InExplainStmt {
- pc.sc.AppendWarning(errors.New("Expression about '" + expr.String() + "' can not be pushed to TiFlash because it contains unsupported calculation of type '" + types.TypeStr(expr.GetType().GetType()) + "'."))
+ pc.sc.AppendWarning(warnErr)
+ } else {
+ pc.sc.AppendExtraWarning(warnErr)
}
return false
case mysql.TypeNewDecimal:
if !expr.GetType().IsDecimalValid() {
+ warnErr := errors.New("Expression about '" + expr.String() + "' can not be pushed to TiFlash because it contains invalid decimal('" + strconv.Itoa(expr.GetType().GetFlen()) + "','" + strconv.Itoa(expr.GetType().GetDecimal()) + "').")
if pc.sc.InExplainStmt {
- pc.sc.AppendWarning(errors.New("Expression about '" + expr.String() + "' can not be pushed to TiFlash because it contains invalid decimal('" + strconv.Itoa(expr.GetType().GetFlen()) + "','" + strconv.Itoa(expr.GetType().GetDecimal()) + "')."))
+ pc.sc.AppendWarning(warnErr)
+ } else {
+ pc.sc.AppendExtraWarning(warnErr)
}
return false
}
@@ -1554,6 +1574,10 @@ func Args2Expressions4Test(args ...interface{}) []Expression {
ft = types.NewFieldType(mysql.TypeDouble)
case types.KindString:
ft = types.NewFieldType(mysql.TypeVarString)
+ case types.KindMysqlTime:
+ ft = types.NewFieldType(mysql.TypeTimestamp)
+ case types.KindBytes:
+ ft = types.NewFieldType(mysql.TypeBlob)
default:
exprs[i] = nil
continue
diff --git a/expression/expression_test.go b/expression/expression_test.go
index 79ee1970ba800..4a24cb5eef759 100644
--- a/expression/expression_test.go
+++ b/expression/expression_test.go
@@ -75,8 +75,9 @@ func TestEvaluateExprWithNullAndParameters(t *testing.T) {
ltWithParam, err := newFunctionForTest(ctx, ast.LT, col0, param)
require.NoError(t, err)
res = EvaluateExprWithNull(ctx, schema, ltWithParam)
- _, isScalarFunc := res.(*ScalarFunction)
- require.True(t, isScalarFunc) // the expression with parameters is not evaluated
+ _, isConst := res.(*Constant)
+ require.True(t, isConst) // this expression is evaluated and skip-plan cache flag is set.
+ require.True(t, !ctx.GetSessionVars().StmtCtx.UseCache)
}
func TestEvaluateExprWithNullNoChangeRetType(t *testing.T) {
diff --git a/expression/function_traits.go b/expression/function_traits.go
index aba61d9b2d92a..4d6fa98da6a99 100644
--- a/expression/function_traits.go
+++ b/expression/function_traits.go
@@ -19,8 +19,8 @@ import (
"github.com/pingcap/tidb/parser/opcode"
)
-// GeneralPlanCacheableOp stores function which can be cached to general plan cache.
-var GeneralPlanCacheableOp = map[string]struct{}{
+// NonPreparedPlanCacheableOp stores function which can be cached to non-prepared plan cache.
+var NonPreparedPlanCacheableOp = map[string]struct{}{
ast.LogicAnd: {},
ast.LogicOr: {},
ast.GE: {},
diff --git a/expression/helper.go b/expression/helper.go
index c9534be9493ff..f286ee644209a 100644
--- a/expression/helper.go
+++ b/expression/helper.go
@@ -74,7 +74,7 @@ func getTimeCurrentTimeStamp(ctx sessionctx.Context, tp byte, fsp int) (t types.
return value, err
}
value.SetCoreTime(types.FromGoTime(defaultTime.Truncate(time.Duration(math.Pow10(9-fsp)) * time.Nanosecond)))
- if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime {
+ if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime || tp == mysql.TypeDate {
err = value.ConvertTimeZone(time.Local, ctx.GetSessionVars().Location())
if err != nil {
return value, err
@@ -84,22 +84,22 @@ func getTimeCurrentTimeStamp(ctx sessionctx.Context, tp byte, fsp int) (t types.
}
// GetTimeValue gets the time value with type tp.
-func GetTimeValue(ctx sessionctx.Context, v interface{}, tp byte, fsp int) (d types.Datum, err error) {
+func GetTimeValue(ctx sessionctx.Context, v interface{}, tp byte, fsp int, explicitTz *time.Location) (d types.Datum, err error) {
var value types.Time
sc := ctx.GetSessionVars().StmtCtx
switch x := v.(type) {
case string:
- upperX := strings.ToUpper(x)
- if upperX == strings.ToUpper(ast.CurrentTimestamp) {
+ lowerX := strings.ToLower(x)
+ if lowerX == ast.CurrentTimestamp || lowerX == ast.CurrentDate {
if value, err = getTimeCurrentTimeStamp(ctx, tp, fsp); err != nil {
return d, err
}
- } else if upperX == types.ZeroDatetimeStr {
+ } else if lowerX == types.ZeroDatetimeStr {
value, err = types.ParseTimeFromNum(sc, 0, tp, fsp)
terror.Log(err)
} else {
- value, err = types.ParseTime(sc, x, tp, fsp)
+ value, err = types.ParseTime(sc, x, tp, fsp, explicitTz)
if err != nil {
return d, err
}
@@ -107,7 +107,7 @@ func GetTimeValue(ctx sessionctx.Context, v interface{}, tp byte, fsp int) (d ty
case *driver.ValueExpr:
switch x.Kind() {
case types.KindString:
- value, err = types.ParseTime(sc, x.GetString(), tp, fsp)
+ value, err = types.ParseTime(sc, x.GetString(), tp, fsp, nil)
if err != nil {
return d, err
}
@@ -122,8 +122,8 @@ func GetTimeValue(ctx sessionctx.Context, v interface{}, tp byte, fsp int) (d ty
return d, errDefaultValue
}
case *ast.FuncCallExpr:
- if x.FnName.L == ast.CurrentTimestamp {
- d.SetString(strings.ToUpper(ast.CurrentTimestamp), mysql.DefaultCollationName)
+ if x.FnName.L == ast.CurrentTimestamp || x.FnName.L == ast.CurrentDate {
+ d.SetString(strings.ToUpper(x.FnName.L), mysql.DefaultCollationName)
return d, nil
}
return d, errDefaultValue
diff --git a/expression/helper_test.go b/expression/helper_test.go
index 3f58f115c998a..4c9a5e3eda3b5 100644
--- a/expression/helper_test.go
+++ b/expression/helper_test.go
@@ -32,7 +32,7 @@ import (
func TestGetTimeValue(t *testing.T) {
ctx := mock.NewContext()
- v, err := GetTimeValue(ctx, "2012-12-12 00:00:00", mysql.TypeTimestamp, types.MinFsp)
+ v, err := GetTimeValue(ctx, "2012-12-12 00:00:00", mysql.TypeTimestamp, types.MinFsp, nil)
require.NoError(t, err)
require.Equal(t, types.KindMysqlTime, v.Kind())
@@ -42,7 +42,7 @@ func TestGetTimeValue(t *testing.T) {
sessionVars := ctx.GetSessionVars()
err = sessionVars.SetSystemVar("timestamp", "0")
require.NoError(t, err)
- v, err = GetTimeValue(ctx, "2012-12-12 00:00:00", mysql.TypeTimestamp, types.MinFsp)
+ v, err = GetTimeValue(ctx, "2012-12-12 00:00:00", mysql.TypeTimestamp, types.MinFsp, nil)
require.NoError(t, err)
require.Equal(t, types.KindMysqlTime, v.Kind())
@@ -51,7 +51,7 @@ func TestGetTimeValue(t *testing.T) {
err = sessionVars.SetSystemVar("timestamp", "0")
require.NoError(t, err)
- v, err = GetTimeValue(ctx, "2012-12-12 00:00:00", mysql.TypeTimestamp, types.MinFsp)
+ v, err = GetTimeValue(ctx, "2012-12-12 00:00:00", mysql.TypeTimestamp, types.MinFsp, nil)
require.NoError(t, err)
require.Equal(t, types.KindMysqlTime, v.Kind())
@@ -60,7 +60,7 @@ func TestGetTimeValue(t *testing.T) {
err = sessionVars.SetSystemVar("timestamp", "")
require.Error(t, err, "Incorrect argument type to variable 'timestamp'")
- v, err = GetTimeValue(ctx, "2012-12-12 00:00:00", mysql.TypeTimestamp, types.MinFsp)
+ v, err = GetTimeValue(ctx, "2012-12-12 00:00:00", mysql.TypeTimestamp, types.MinFsp, nil)
require.NoError(t, err)
require.Equal(t, types.KindMysqlTime, v.Kind())
@@ -98,7 +98,7 @@ func TestGetTimeValue(t *testing.T) {
for i, tbl := range tbls {
comment := fmt.Sprintf("expr: %d", i)
- v, err := GetTimeValue(ctx, tbl.Expr, mysql.TypeTimestamp, types.MinFsp)
+ v, err := GetTimeValue(ctx, tbl.Expr, mysql.TypeTimestamp, types.MinFsp, nil)
require.NoError(t, err)
switch v.Kind() {
@@ -120,7 +120,7 @@ func TestGetTimeValue(t *testing.T) {
}
for _, tbl := range errTbl {
- _, err := GetTimeValue(ctx, tbl.Expr, mysql.TypeTimestamp, types.MinFsp)
+ _, err := GetTimeValue(ctx, tbl.Expr, mysql.TypeTimestamp, types.MinFsp, nil)
require.Error(t, err)
}
}
@@ -163,7 +163,7 @@ func TestCurrentTimestampTimeZone(t *testing.T) {
require.NoError(t, err)
err = sessionVars.SetSystemVar("time_zone", "+00:00")
require.NoError(t, err)
- v, err := GetTimeValue(ctx, ast.CurrentTimestamp, mysql.TypeTimestamp, types.MinFsp)
+ v, err := GetTimeValue(ctx, ast.CurrentTimestamp, mysql.TypeTimestamp, types.MinFsp, nil)
require.NoError(t, err)
require.EqualValues(t, types.NewTime(
types.FromDate(1970, 1, 1, 0, 20, 34, 0),
@@ -174,7 +174,7 @@ func TestCurrentTimestampTimeZone(t *testing.T) {
// would get different value.
err = sessionVars.SetSystemVar("time_zone", "+08:00")
require.NoError(t, err)
- v, err = GetTimeValue(ctx, ast.CurrentTimestamp, mysql.TypeTimestamp, types.MinFsp)
+ v, err = GetTimeValue(ctx, ast.CurrentTimestamp, mysql.TypeTimestamp, types.MinFsp, nil)
require.NoError(t, err)
require.EqualValues(t, types.NewTime(
types.FromDate(1970, 1, 1, 8, 20, 34, 0),
diff --git a/expression/integration_serial_test.go b/expression/integration_serial_test.go
index a497d9ef54600..c50aa687659a9 100644
--- a/expression/integration_serial_test.go
+++ b/expression/integration_serial_test.go
@@ -3762,15 +3762,6 @@ func TestSetVariables(t *testing.T) {
_, err = tk.Exec("set @@global.max_prepared_stmt_count='';")
require.Error(t, err)
require.Error(t, err, variable.ErrWrongTypeForVar.GenWithStackByArgs("max_prepared_stmt_count").Error())
-
- tk.MustExec("set @@global.tidb_enable_concurrent_ddl=1")
- tk.MustQuery("select @@global.tidb_enable_concurrent_ddl").Check(testkit.Rows("1"))
- require.True(t, variable.EnableConcurrentDDL.Load())
- tk.MustExec("set @@global.tidb_enable_concurrent_ddl=0")
- tk.MustQuery("select @@global.tidb_enable_concurrent_ddl").Check(testkit.Rows("0"))
- require.False(t, variable.EnableConcurrentDDL.Load())
- testkit.NewTestKit(t, store).MustQuery("select @@global.tidb_enable_concurrent_ddl").Check(testkit.Rows("0"))
- tk.MustExec("set @@global.tidb_enable_concurrent_ddl=1")
}
func TestPreparePlanCache(t *testing.T) {
@@ -3799,7 +3790,7 @@ func TestPreparePlanCacheOnCachedTable(t *testing.T) {
var err error
se, err := session.CreateSession4TestWithOpt(store, &session.Opt{
- PreparedPlanCache: plannercore.NewLRUPlanCache(100, 0.1, math.MaxUint64, plannercore.PickPlanFromBucket, tk.Session()),
+ PreparedPlanCache: plannercore.NewLRUPlanCache(100, 0.1, math.MaxUint64, tk.Session()),
})
require.NoError(t, err)
tk.SetSession(se)
diff --git a/expression/integration_test.go b/expression/integration_test.go
index d3b307fb9653b..bb8a0d622bf66 100644
--- a/expression/integration_test.go
+++ b/expression/integration_test.go
@@ -968,6 +968,7 @@ func TestEncryptionBuiltin(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
+ tk.Session().GetSessionVars().User = &auth.UserIdentity{Username: "root"}
ctx := context.Background()
// for password
@@ -1143,6 +1144,25 @@ func TestEncryptionBuiltin(t *testing.T) {
tk.MustQuery("SELECT RANDOM_BYTES(1024);")
result = tk.MustQuery("SELECT RANDOM_BYTES(NULL);")
result.Check(testkit.Rows(""))
+
+ // for VALIDATE_PASSWORD_STRENGTH
+ tk.MustExec(fmt.Sprintf("SET GLOBAL validate_password.dictionary='%s'", "password"))
+ tk.MustExec("SET GLOBAL validate_password.enable = 1")
+ tk.MustQuery("SELECT validate_password_strength('root')").Check(testkit.Rows("0"))
+ tk.MustQuery("SELECT validate_password_strength('toor')").Check(testkit.Rows("0"))
+ tk.MustQuery("SELECT validate_password_strength('ROOT')").Check(testkit.Rows("25"))
+ tk.MustQuery("SELECT validate_password_strength('TOOR')").Check(testkit.Rows("25"))
+ tk.MustQuery("SELECT validate_password_strength('fooHoHo%1')").Check(testkit.Rows("100"))
+ tk.MustQuery("SELECT validate_password_strength('pass')").Check(testkit.Rows("25"))
+ tk.MustQuery("SELECT validate_password_strength('password')").Check(testkit.Rows("50"))
+ tk.MustQuery("SELECT validate_password_strength('password0000')").Check(testkit.Rows("50"))
+ tk.MustQuery("SELECT validate_password_strength('password1A#')").Check(testkit.Rows("75"))
+ tk.MustQuery("SELECT validate_password_strength('PA12wrd!#')").Check(testkit.Rows("100"))
+ tk.MustQuery("SELECT VALIDATE_PASSWORD_STRENGTH(REPEAT(\"aA1#\", 26))").Check(testkit.Rows("100"))
+ tk.MustQuery("SELECT validate_password_strength(null)").Check(testkit.Rows(""))
+ tk.MustQuery("SELECT validate_password_strength('null')").Check(testkit.Rows("25"))
+ tk.MustQuery("SELECT VALIDATE_PASSWORD_STRENGTH( 0x6E616E646F73617135234552 )").Check(testkit.Rows("100"))
+ tk.MustQuery("SELECT VALIDATE_PASSWORD_STRENGTH(CAST(0xd2 AS BINARY(10)))").Check(testkit.Rows("50"))
}
func TestOpBuiltin(t *testing.T) {
@@ -1208,8 +1228,7 @@ func TestDatetimeOverflow(t *testing.T) {
}
for _, sql := range overflowSQLs {
- _, err := tk.Exec(sql)
- require.Error(t, err, "[types:1441]Datetime function: datetime field overflow")
+ tk.MustGetErrMsg(sql, "[types:1441]Datetime function: datetime field overflow")
}
tk.MustExec("set sql_mode=''")
@@ -1696,8 +1715,7 @@ func TestArithmeticBuiltin(t *testing.T) {
tk.MustQuery("select v from t;").Check(testkit.Rows(""))
tk.MustQuery("select 0.000 % 0.11234500000000000000;").Check(testkit.Rows("0.00000000000000000000"))
- _, err = tk.Exec("INSERT INTO t VALUE(12 MOD 0);")
- require.True(t, terror.ErrorEqual(err, expression.ErrDivisionByZero))
+ tk.MustGetDBError("INSERT INTO t VALUE(12 MOD 0);", expression.ErrDivisionByZero)
tk.MustQuery("select sum(1.2e2) * 0.1").Check(testkit.Rows("12"))
tk.MustExec("drop table if exists t")
@@ -1940,8 +1958,7 @@ func TestCompareBuiltin(t *testing.T) {
tk.MustQuery("select * from t").Check(testkit.Rows("1991-05-06 13:59:28"))
// insert an nonexistent time
tk.MustExec("set time_zone = 'America/Los_Angeles'")
- _, err := tk.Exec("insert into t value('2011-03-13 02:00:00')")
- require.Error(t, err)
+ tk.MustExecToErr("insert into t value('2011-03-13 02:00:00')")
// reset timezone to a +8 offset
tk.MustExec("set time_zone = '+08:00'")
tk.MustQuery("select * from t").Check(testkit.Rows("1991-05-06 12:59:28"))
@@ -2105,11 +2122,9 @@ func TestAggregationBuiltinGroupConcat(t *testing.T) {
result.Check(testkit.Rows("hello,h"))
tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning 1260 Some rows were cut by GROUPCONCAT(test.t.a)"))
- _, err := tk.Exec("insert into d select group_concat(a) from t")
- require.Equal(t, errors.ErrCode(mysql.ErrCutValueGroupConcat), errors.Cause(err).(*terror.Error).Code())
+ tk.MustGetErrCode("insert into d select group_concat(a) from t", mysql.ErrCutValueGroupConcat)
- _, err = tk.Exec("set sql_mode=''")
- require.NoError(t, err)
+ tk.MustExec("set sql_mode=''")
tk.MustExec("insert into d select group_concat(a) from t")
tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning 1260 Some rows were cut by GROUPCONCAT(test.t.a)"))
tk.MustQuery("select * from d").Check(testkit.Rows("hello,h"))
@@ -2193,6 +2208,22 @@ func TestAggregationBuiltinJSONObjectAgg(t *testing.T) {
result.Check(testkit.Rows(`{"first": "json_objectagg_test"}`))
result = tk.MustQuery("select json_objectagg(a, null) from t group by a order by a;")
result.Check(testkit.Rows(`{"1": null}`))
+
+ // For issue: https://github.com/pingcap/tidb/issues/39806
+ // Optimization shouldn't rewrite the flag of `castStringAsJson`.
+ tk.MustQuery(`
+ select a from (
+ select JSON_OBJECT('number', number, 'name', name) 'a' from
+ (
+ select 1 as number, 'name-1' as name union
+ (select 2, 'name-2' ) union
+ (select 3, 'name-3' ) union
+ (select 4, 'name-4' ) union
+ (select 5, 'name-5' ) union
+ (select 6, 'name-2' )
+ ) temp1
+ ) temp
+ where a ->> '$.number' = 1`).Check(testkit.Rows(`{"name": "name-1", "number": 1}`))
}
func TestOtherBuiltin(t *testing.T) {
@@ -2354,23 +2385,18 @@ func TestDateBuiltin(t *testing.T) {
r = tk.MustQuery("select date'731124', date '011124'")
r.Check(testkit.Rows("1973-11-24 2001-11-24"))
- _, err = tk.Exec("select date '0000-00-00 00:00:00';")
+ err = tk.ExecToErr("select date '0000-00-00 00:00:00';")
require.Error(t, err)
require.True(t, terror.ErrorEqual(err, types.ErrWrongValue.GenWithStackByArgs(types.DateStr, "0000-00-00 00:00:00")))
- _, err = tk.Exec("select date '2017-99-99';")
- require.Error(t, err)
- require.True(t, terror.ErrorEqual(err, types.ErrWrongValue), "err: %v", err)
-
- _, err = tk.Exec("select date '2017-2-31';")
- require.Error(t, err)
- require.True(t, terror.ErrorEqual(err, types.ErrWrongValue), "err: %v", err)
+ tk.MustGetDBError("select date '2017-99-99';", types.ErrWrongValue)
+ tk.MustGetDBError("select date '2017-2-31';", types.ErrWrongValue)
- _, err = tk.Exec("select date '201712-31';")
+ err = tk.ExecToErr("select date '201712-31';")
require.Error(t, err)
require.True(t, terror.ErrorEqual(err, types.ErrWrongValue.GenWithStackByArgs(types.DateStr, "201712-31")), "err: %v", err)
- _, err = tk.Exec("select date 'abcdefg';")
+ err = tk.ExecToErr("select date 'abcdefg';")
require.Error(t, err)
require.True(t, terror.ErrorEqual(err, types.ErrWrongValue.GenWithStackByArgs(types.DateStr, "abcdefg")), "err: %v", err)
}
@@ -2382,8 +2408,7 @@ func TestJSONBuiltin(t *testing.T) {
tk.MustExec("USE test;")
tk.MustExec("DROP TABLE IF EXISTS t;")
tk.MustExec("CREATE TABLE `my_collection` ( `doc` json DEFAULT NULL, `_id` varchar(32) GENERATED ALWAYS AS (JSON_UNQUOTE(JSON_EXTRACT(doc,'$._id'))) STORED NOT NULL, PRIMARY KEY (`_id`))")
- _, err := tk.Exec("UPDATE `test`.`my_collection` SET doc=JSON_SET(doc) WHERE (JSON_EXTRACT(doc,'$.name') = 'clare');")
- require.Error(t, err)
+ tk.MustExecToErr("UPDATE `test`.`my_collection` SET doc=JSON_SET(doc) WHERE (JSON_EXTRACT(doc,'$.name') = 'clare');")
r := tk.MustQuery("select json_valid(null);")
r.Check(testkit.Rows(""))
@@ -2459,20 +2484,19 @@ func TestTimeLiteral(t *testing.T) {
r = tk.MustQuery("select time '20 20:20';")
r.Check(testkit.Rows("500:20:00"))
- _, err := tk.Exec("select time '2017-01-01 00:00:00';")
+ err := tk.ExecToErr("select time '2017-01-01 00:00:00';")
require.Error(t, err)
require.True(t, terror.ErrorEqual(err, types.ErrWrongValue.GenWithStackByArgs(types.TimeStr, "2017-01-01 00:00:00")))
- _, err = tk.Exec("select time '071231235959.999999';")
+ err = tk.ExecToErr("select time '071231235959.999999';")
require.Error(t, err)
require.True(t, terror.ErrorEqual(err, types.ErrWrongValue.GenWithStackByArgs(types.TimeStr, "071231235959.999999")))
- _, err = tk.Exec("select time '20171231235959.999999';")
+ err = tk.ExecToErr("select time '20171231235959.999999';")
require.Error(t, err)
require.True(t, terror.ErrorEqual(err, types.ErrWrongValue.GenWithStackByArgs(types.TimeStr, "20171231235959.999999")))
- _, err = tk.Exec("select ADDDATE('2008-01-34', -1);")
- require.NoError(t, err)
+ tk.MustExec("select ADDDATE('2008-01-34', -1);")
tk.MustQuery("Show warnings;").Check(testkit.RowsWithSep("|",
"Warning|1292|Incorrect datetime value: '2008-01-34'"))
}
@@ -2502,15 +2526,15 @@ func TestTimestampLiteral(t *testing.T) {
r = tk.MustQuery("select timestamp '2017@01@0001 00~00~00.333';")
r.Check(testkit.Rows("2017-01-01 00:00:00.333"))
- _, err := tk.Exec("select timestamp '00:00:00';")
+ err := tk.ExecToErr("select timestamp '00:00:00';")
require.Error(t, err)
require.True(t, terror.ErrorEqual(err, types.ErrWrongValue.GenWithStackByArgs(types.DateTimeStr, "00:00:00")))
- _, err = tk.Exec("select timestamp '1992-01-03';")
+ err = tk.ExecToErr("select timestamp '1992-01-03';")
require.Error(t, err)
require.True(t, terror.ErrorEqual(err, types.ErrWrongValue.GenWithStackByArgs(types.DateTimeStr, "1992-01-03")))
- _, err = tk.Exec("select timestamp '20171231235959.999999';")
+ err = tk.ExecToErr("select timestamp '20171231235959.999999';")
require.Error(t, err)
require.True(t, terror.ErrorEqual(err, types.ErrWrongValue.GenWithStackByArgs(types.DateTimeStr, "20171231235959.999999")))
}
@@ -2701,12 +2725,36 @@ func TestFuncJSON(t *testing.T) {
// #16267
tk.MustQuery(`select json_array(922337203685477580) = json_array(922337203685477581);`).Check(testkit.Rows("0"))
+ tk.MustQuery("select json_overlaps('[[1,2], 3]', '[1, 3]');").Check(testkit.Rows("1"))
+ tk.MustQuery("select json_overlaps('[{\"a\":1}]', '{\"a\":1}');").Check(testkit.Rows("1"))
+ tk.MustQuery("select json_overlaps('{\"a\":1}', '[{\"a\":1}]');").Check(testkit.Rows("1"))
+ tk.MustQuery("select json_overlaps('[1,[2,3]]', '[[1,2], 3]');").Check(testkit.Rows("0"))
+ tk.MustQuery("select json_overlaps('{\"a\":[1,2]}', '{\"a\":[2,1]}');").Check(testkit.Rows("0"))
+ tk.MustQuery("select json_overlaps('{\"a\":[1,2]}', '{\"a\":[2,1]}');").Check(testkit.Rows("0"))
+
// #10461
tk.MustExec("drop table if exists tx1")
tk.MustExec("create table tx1(id int key, a double, b double, c double, d double)")
tk.MustExec("insert into tx1 values (1, 0.1, 0.2, 0.3, 0.0)")
tk.MustQuery("select a+b, c from tx1").Check(testkit.Rows("0.30000000000000004 0.3"))
tk.MustQuery("select json_array(a+b) = json_array(c) from tx1").Check(testkit.Rows("0"))
+
+ tk.MustQuery("SELECT '{\"a\":1}' MEMBER OF('{\"a\":1}');").Check(testkit.Rows("0"))
+ tk.MustQuery("SELECT '{\"a\":1}' MEMBER OF('[{\"a\":1}]');").Check(testkit.Rows("0"))
+ tk.MustQuery("SELECT 1 MEMBER OF('1');").Check(testkit.Rows("1"))
+ tk.MustQuery("SELECT '{\"a\":1}' MEMBER OF('{\"a\":1}');").Check(testkit.Rows("0"))
+ tk.MustQuery("SELECT '[4,5]' MEMBER OF('[[3,4],[4,5]]');").Check(testkit.Rows("0"))
+ tk.MustQuery("SELECT '[4,5]' MEMBER OF('[[3,4],\"[4,5]\"]');").Check(testkit.Rows("1"))
+
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a enum('a', 'b'), b time, c binary(10))")
+ tk.MustExec("insert into t values ('a', '11:00:00', 'a')")
+ tk.MustQuery("select a member of ('\"a\"') from t").Check(testkit.Rows(`1`))
+ tk.MustQuery("select b member of (json_array(cast('11:00:00' as time))) from t;").Check(testkit.Rows(`1`))
+ tk.MustQuery("select b member of ('\"11:00:00\"') from t").Check(testkit.Rows(`0`))
+ tk.MustQuery("select c member of ('\"a\"') from t").Check(testkit.Rows(`0`))
+ err = tk.QueryToErr("select 'a' member of ('a')")
+ require.Error(t, err, "ERROR 3140 (22032): Invalid JSON text: The document root must not be followed by other values.")
}
func TestColumnInfoModified(t *testing.T) {
@@ -3050,6 +3098,30 @@ func TestTiDBDecodeKeyFunc(t *testing.T) {
sql = fmt.Sprintf("select tidb_decode_key( '%s' )", hexKey)
rs = fmt.Sprintf(`{"%s":%d,"table_id":"%d"}`, tbl.Meta().GetPkName().String(), rowID, tbl.Meta().ID)
tk.MustQuery(sql).Check(testkit.Rows(rs))
+
+ // Test partition table.
+ tk.MustExec("drop table if exists t;")
+ tk.MustExec("create table t (a int primary key clustered, b int, key bk (b)) PARTITION BY RANGE (a) (PARTITION p0 VALUES LESS THAN (1), PARTITION p1 VALUES LESS THAN (2));")
+ dom = domain.GetDomain(tk.Session())
+ is = dom.InfoSchema()
+ tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+ require.NoError(t, err)
+ require.NotNil(t, tbl.Meta().Partition)
+ hexKey = buildTableRowKey(tbl.Meta().Partition.Definitions[0].ID, rowID)
+ sql = fmt.Sprintf("select tidb_decode_key( '%s' )", hexKey)
+ rs = fmt.Sprintf(`{"%s":%d,"partition_id":%d,"table_id":"%d"}`, tbl.Meta().GetPkName().String(), rowID, tbl.Meta().Partition.Definitions[0].ID, tbl.Meta().ID)
+ tk.MustQuery(sql).Check(testkit.Rows(rs))
+
+ hexKey = tablecodec.EncodeTablePrefix(tbl.Meta().Partition.Definitions[0].ID).String()
+ sql = fmt.Sprintf("select tidb_decode_key( '%s' )", hexKey)
+ rs = fmt.Sprintf(`{"partition_id":%d,"table_id":%d}`, tbl.Meta().Partition.Definitions[0].ID, tbl.Meta().ID)
+ tk.MustQuery(sql).Check(testkit.Rows(rs))
+
+ data = []types.Datum{types.NewIntDatum(100)}
+ hexKey = buildIndexKeyFromData(tbl.Meta().Partition.Definitions[0].ID, tbl.Indices()[0].Meta().ID, data)
+ sql = fmt.Sprintf("select tidb_decode_key( '%s' )", hexKey)
+ rs = fmt.Sprintf(`{"index_id":1,"index_vals":{"b":"100"},"partition_id":%d,"table_id":%d}`, tbl.Meta().Partition.Definitions[0].ID, tbl.Meta().ID)
+ tk.MustQuery(sql).Check(testkit.Rows(rs))
}
func TestTwoDecimalTruncate(t *testing.T) {
@@ -3699,18 +3771,23 @@ func TestShardIndexOnTiFlash(t *testing.T) {
}
}
}
+ tk.MustExec("set @@session.tidb_isolation_read_engines = 'tiflash'")
tk.MustExec("set @@session.tidb_enforce_mpp = 1")
rows := tk.MustQuery("explain select max(b) from t").Rows()
for _, row := range rows {
line := fmt.Sprintf("%v", row)
- require.NotContains(t, line, "tiflash")
+ if strings.Contains(line, "TableFullScan") {
+ require.Contains(t, line, "tiflash")
+ }
}
tk.MustExec("set @@session.tidb_enforce_mpp = 0")
tk.MustExec("set @@session.tidb_allow_mpp = 0")
rows = tk.MustQuery("explain select max(b) from t").Rows()
for _, row := range rows {
line := fmt.Sprintf("%v", row)
- require.NotContains(t, line, "tiflash")
+ if strings.Contains(line, "TableFullScan") {
+ require.NotContains(t, line, "mpp[tiflash]")
+ }
}
}
@@ -3718,8 +3795,6 @@ func TestExprPushdownBlacklist(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
- tk.MustQuery(`select * from mysql.expr_pushdown_blacklist`).Check(testkit.Rows(
- "date_add tiflash DST(daylight saving time) does not take effect in TiFlash date_add"))
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
@@ -3791,7 +3866,7 @@ func TestInvalidEndingStatement(t *testing.T) {
errMsgLen := len(parseErrMsg)
assertParseErr := func(sql string) {
- _, err := tk.Exec(sql)
+ err := tk.ExecToErr(sql)
require.Error(t, err)
require.Equal(t, err.Error()[:errMsgLen], parseErrMsg)
}
@@ -4110,23 +4185,17 @@ func TestNotExistFunc(t *testing.T) {
tk := testkit.NewTestKit(t, store)
// current db is empty
- _, err := tk.Exec("SELECT xxx(1)")
- require.Error(t, err, "[planner:1046]No database selected")
+ tk.MustGetErrMsg("SELECT xxx(1)", "[planner:1046]No database selected")
- _, err = tk.Exec("SELECT yyy()")
- require.Error(t, err, "[planner:1046]No database selected")
+ tk.MustGetErrMsg("SELECT yyy()", "[planner:1046]No database selected")
// current db is not empty
tk.MustExec("use test")
- _, err = tk.Exec("SELECT xxx(1)")
- require.Error(t, err, "[expression:1305]FUNCTION test.xxx does not exist")
-
- _, err = tk.Exec("SELECT yyy()")
- require.Error(t, err, "[expression:1305]FUNCTION test.yyy does not exist")
+ tk.MustGetErrMsg("SELECT xxx(1)", "[expression:1305]FUNCTION test.xxx does not exist")
+ tk.MustGetErrMsg("SELECT yyy()", "[expression:1305]FUNCTION test.yyy does not exist")
tk.MustExec("use test")
- _, err = tk.Exec("SELECT timestampliteral(rand())")
- require.Error(t, err, "[expression:1305]FUNCTION test.timestampliteral does not exist")
+ tk.MustGetErrMsg("SELECT timestampliteral(rand())", "[expression:1305]FUNCTION test.timestampliteral does not exist")
}
func TestDecodetoChunkReuse(t *testing.T) {
@@ -4925,24 +4994,6 @@ func TestIssue18525(t *testing.T) {
tk.MustQuery("select INTERVAL( ( CONVERT( -11752 USING utf8 ) ), 6558853612195285496, `col1`) from t1").Check(testkit.Rows("0", "0", "0"))
}
-func TestSchemaDMLNotChange(t *testing.T) {
- store := testkit.CreateMockStore(t)
-
- tk := testkit.NewTestKit(t, store)
- tk2 := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk.MustExec("set global tidb_enable_metadata_lock=0")
- tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
- tk2.MustExec("use test")
- tk.MustExec("drop table if exists t")
- tk.MustExec("create table t (id int primary key, c_json json);")
- tk.MustExec("insert into t values (1, '{\"k\": 1}');")
- tk.MustExec("begin")
- tk.MustExec("update t set c_json = '{\"k\": 2}' where id = 1;")
- tk2.MustExec("alter table t rename column c_json to cc_json;")
- tk.MustExec("commit")
-}
-
func TestIssue18850(t *testing.T) {
store := testkit.CreateMockStore(t)
@@ -5908,8 +5959,7 @@ func TestSecurityEnhancedMode(t *testing.T) {
// When SEM is enabled these features are restricted to all users
// regardless of what privileges they have available.
- _, err := tk.Exec("SELECT 1 INTO OUTFILE '/tmp/aaaa'")
- require.Error(t, err, "[planner:8132]Feature 'SELECT INTO' is not supported when security enhanced mode is enabled")
+ tk.MustGetErrMsg("SELECT 1 INTO OUTFILE '/tmp/aaaa'", "[planner:8132]Feature 'SELECT INTO' is not supported when security enhanced mode is enabled")
}
func TestIssue23925(t *testing.T) {
@@ -7102,29 +7152,25 @@ func TestIssue29708(t *testing.T) {
tk.MustExec("use test;")
tk.MustExec("drop table if exists t1;")
tk.MustExec("CREATE TABLE t1 (a text)character set utf8 ;")
- _, err := tk.Exec("INSERT INTO t1 VALUES (REPEAT(0125,200000000));")
- require.NotNil(t, err)
+ tk.MustExecToErr("INSERT INTO t1 VALUES (REPEAT(0125,200000000));")
tk.MustQuery("select * from t1").Check(nil)
// test vectorized build-in function
tk.MustExec("insert into t1 (a) values ('a'),('b');")
- _, err = tk.Exec("insert into t1 select REPEAT(a,200000000) from t1;")
- require.NotNil(t, err)
+ tk.MustExecToErr("insert into t1 select REPEAT(a,200000000) from t1;")
tk.MustQuery("select a from t1 order by a;").Check([][]interface{}{
{"a"},
{"b"},
})
// test cast
- _, err = tk.Exec(`insert into t1 values (cast("a" as binary(4294967295)));`)
- require.NotNil(t, err)
+ tk.MustExecToErr(`insert into t1 values (cast("a" as binary(4294967295)));`)
tk.MustQuery("select a from t1 order by a;").Check([][]interface{}{
{"a"},
{"b"},
})
- _, err = tk.Exec("INSERT IGNORE INTO t1 VALUES (REPEAT(0125,200000000));")
- require.NoError(t, err)
+ tk.MustExec("INSERT IGNORE INTO t1 VALUES (REPEAT(0125,200000000));")
tk.MustQuery("show warnings;").Check(testkit.Rows("Warning 1301 Result of repeat() was larger than max_allowed_packet (67108864) - truncated"))
tk.MustQuery("select a from t1 order by a;").Check([][]interface{}{
{nil},
@@ -7390,6 +7436,29 @@ func TestIssue31569(t *testing.T) {
tk.MustExec("drop table t")
}
+func TestTimestampAddWithFractionalSecond(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a date)")
+ tk.MustExec("insert into t values ('2021-08-20');")
+ tk.MustQuery("select timestampadd(microsecond, 1, a) from t").Check(testkit.Rows("2021-08-20 00:00:00.000001"))
+ tk.MustQuery("select timestampadd(second, 6/4, a) from t").Check(testkit.Rows("2021-08-20 00:00:01.500000"))
+ tk.MustQuery("select timestampadd(second, 9.9999e2, a) from t").Check(testkit.Rows("2021-08-20 00:16:39.990000"))
+ tk.MustQuery("select timestampadd(second, 1, '2021-08-20 00:00:01.0001')").Check(testkit.Rows("2021-08-20 00:00:02.000100"))
+ tk.MustQuery("select timestampadd(minute, 1.5, '2021-08-20 00:00:00')").Check(testkit.Rows("2021-08-20 00:02:00"))
+ tk.MustQuery("select timestampadd(minute, 1.5, '2021-08-20 00:00:00.0001')").Check(testkit.Rows("2021-08-20 00:02:00.000100"))
+ // overflow
+ tk.MustQuery("SELECT timestampadd(year,1.212208e+308,'1995-01-05 06:32:20.859724') as result").Check(testkit.Rows(""))
+ warnings := tk.Session().GetSessionVars().StmtCtx.GetWarnings()
+ require.Len(t, warnings, 1)
+ for _, warning := range warnings {
+ require.EqualError(t, warning.Err, "[types:1441]Datetime function: datetime field overflow")
+ }
+}
+
func TestDateAddForNonExistingTimestamp(t *testing.T) {
store := testkit.CreateMockStore(t)
@@ -7816,3 +7885,25 @@ func TestIfNullParamMarker(t *testing.T) {
// Should not report 'Data too long for column' error.
tk.MustExec(`execute pr1 using @a,@b;`)
}
+
+func TestIssue39146(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("CREATE TABLE `sun` ( `dest` varchar(10) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;")
+ tk.MustExec("insert into sun values('20231020');")
+ tk.MustExec("set @@sql_mode = 'ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';")
+ tk.MustExec("set @@tidb_enable_vectorized_expression = on;")
+ tk.MustQuery(`select str_to_date(substr(dest,1,6),'%H%i%s') from sun;`).Check(testkit.Rows("20:23:10"))
+ tk.MustExec("set @@tidb_enable_vectorized_expression = off;")
+ tk.MustQuery(`select str_to_date(substr(dest,1,6),'%H%i%s') from sun;`).Check(testkit.Rows("20:23:10"))
+}
+
+func TestIssue40536(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("CREATE TABLE `6bf9e76d-ab44-4031-8a07-418b10741580` (\n `e0b5f703-6cfe-49b4-bc21-16a6455e43a7` set('7','va','ung60','ow','1g','gxwz5','uhnh','k','5la1','q8d9c','1f') NOT NULL DEFAULT '7,1g,uhnh,5la1,q8d9c',\n `fbc3527f-9617-4b9d-a5dc-4be31c00d8a5` datetime DEFAULT '6449-09-28 14:39:04',\n PRIMARY KEY (`e0b5f703-6cfe-49b4-bc21-16a6455e43a7`) /*T![clustered_index] CLUSTERED */\n) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin;")
+ tk.MustExec("CREATE TABLE `8919f3f4-25be-4a1a-904a-bb5e863d8fc8` (\n `9804d5f2-cbc7-43b7-b241-ea2656dc941a` enum('s951','36d','ua65','49yru','6l2em','4ea','jf2d2','vprsc','3yl7n','hz','ov') DEFAULT '4ea',\n `323cdbcb-0c14-4362-90ab-ea42caaed6a5` year(4) NOT NULL DEFAULT '1983',\n `b9b70f39-1a02-4114-9d7d-fa6259c1b691` time DEFAULT '20:18:04',\n PRIMARY KEY (`323cdbcb-0c14-4362-90ab-ea42caaed6a5`) /*T![clustered_index] CLUSTERED */,\n KEY `a704d6bb-772b-44ea-8cb0-6f7491c1aaa6` (`323cdbcb-0c14-4362-90ab-ea42caaed6a5`,`9804d5f2-cbc7-43b7-b241-ea2656dc941a`)\n) ENGINE=InnoDB DEFAULT CHARSET=ascii COLLATE=ascii_bin;")
+ tk.MustExec("delete from `6bf9e76d-ab44-4031-8a07-418b10741580` where not( `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` in ( select `9804d5f2-cbc7-43b7-b241-ea2656dc941a` from `8919f3f4-25be-4a1a-904a-bb5e863d8fc8` where `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` in ( '1f' ) and `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` in ( '1g' ,'va' ,'uhnh' ) ) ) and not( IsNull( `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` ) );\n")
+}
diff --git a/expression/multi_valued_index_test.go b/expression/multi_valued_index_test.go
new file mode 100644
index 0000000000000..b749b4c94d9ba
--- /dev/null
+++ b/expression/multi_valued_index_test.go
@@ -0,0 +1,508 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package expression_test
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/pingcap/tidb/errno"
+ "github.com/pingcap/tidb/kv"
+ "github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/sessiontxn"
+ "github.com/pingcap/tidb/table"
+ "github.com/pingcap/tidb/tablecodec"
+ "github.com/pingcap/tidb/testkit"
+ "github.com/pingcap/tidb/types"
+ "github.com/pingcap/tidb/util/codec"
+ "github.com/stretchr/testify/require"
+)
+
+func TestMultiValuedIndexDDL(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("USE test;")
+
+ tk.MustExec("create table t(a json);")
+ tk.MustGetErrCode("select cast(a as signed array) from t", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("select json_extract(cast(a as signed array), '$[0]') from t", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("select * from t where cast(a as signed array)", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("select cast('[1,2,3]' as unsigned array);", errno.ErrNotSupportedYet)
+
+ tk.MustExec("drop table t")
+ tk.MustGetErrCode("CREATE TABLE t(x INT, KEY k ((1 AND CAST(JSON_ARRAY(x) AS UNSIGNED ARRAY))));", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(cast(f1 as unsigned array) as unsigned array))));", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("CREATE TABLE t1 (f1 json, primary key mvi((cast(cast(f1 as unsigned array) as unsigned array))));", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->>'$[*]' as unsigned array))));", errno.ErrInvalidTypeForJSON)
+ tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->'$[*]' as year array))));", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->'$[*]' as json array))));", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->'$[*]' as char(10) charset gbk array))));", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("create table t(j json, gc json as ((concat(cast(j->'$[*]' as unsigned array),\"x\"))));", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("create table t(j json, gc json as (cast(j->'$[*]' as unsigned array)));", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode(`create table t1(j json, key i1((cast(j->"$" as char array))));`, errno.ErrNotSupportedYet)
+ tk.MustGetErrCode(`create table t1(j json, key i1((cast(j->"$" as binary array))));`, errno.ErrNotSupportedYet)
+ tk.MustGetErrCode(`create table t1(j json, key i1((cast(j->"$" as float array))));`, errno.ErrNotSupportedYet)
+ tk.MustGetErrCode(`create table t1(j json, key i1((cast(j->"$" as double array))));`, errno.ErrNotSupportedYet)
+ tk.MustGetErrCode(`create table t1(j json, key i1((cast(j->"$" as decimal(4,2) array))));`, errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("create view v as select cast('[1,2,3]' as unsigned array);", errno.ErrNotSupportedYet)
+ tk.MustExec("create table t(a json, index idx((cast(a as signed array))));")
+ tk.MustExec("drop table t;")
+ tk.MustExec("create table t(a json, index idx(((cast(a as signed array)))))")
+
+ tk.MustExec("drop table t")
+ tk.MustGetErrCode("create table t(a json, b int, index idx(b, (cast(a as signed array)), (cast(a as signed array))));", errno.ErrNotSupportedYet)
+ tk.MustExec("create table t(a json, b int);")
+ tk.MustGetErrCode("create index idx on t (b, (cast(a as signed array)), (cast(a as signed array)))", errno.ErrNotSupportedYet)
+ tk.MustGetErrCode("alter table t add index idx(b, (cast(a as signed array)), (cast(a as signed array)))", errno.ErrNotSupportedYet)
+ tk.MustExec("create index idx1 on t (b, (cast(a as signed array)))")
+ tk.MustExec("alter table t add index idx2(b, (cast(a as signed array)))")
+
+ tk.MustExec("drop table t")
+ tk.MustExec("create table t(a json, b int, index idx3(b, (cast(a as signed array))));")
+ tk.MustExec("drop table t")
+ tk.MustExec("set names gbk")
+ tk.MustExec("create table t(a json, b int, index idx3(b, (cast(a as char(10) array))));")
+
+ tk.MustExec("CREATE TABLE users (id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, doc JSON);")
+ tk.MustExecToErr("CREATE TABLE t (id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, doc JSON, FOREIGN KEY fk_user_id ((cast(doc->'$[*]' as signed array))) REFERENCES users(id));")
+}
+
+func TestMultiValuedIndexDML(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("USE test;")
+
+ mode := []string{`''`, `default`}
+
+ for _, m := range mode {
+ tk.MustExec(fmt.Sprintf("set @@sql_mode=%s", m))
+
+ tk.MustExec(`drop table if exists t;`)
+ tk.MustExec(`create table t(a json, index idx((cast(a as unsigned array))));`)
+ tk.MustExec(`insert into t values ('[1,2,3]');`)
+ tk.MustGetErrCode(`insert into t values ('[-1]');`, errno.ErrDataOutOfRangeFunctionalIndex)
+ tk.MustGetErrCode(`insert into t values ('["1"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["a"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["汉字"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.2]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.0]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("11:00:00" as time)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02" as date)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02 11:00:00" as datetime)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast('{"a":1}' as json)));`, errno.ErrInvalidJSONValueForFuncIndex)
+
+ tk.MustExec(`drop table if exists t;`)
+ tk.MustExec(`create table t(a json, index idx((cast(a as signed array))));`)
+ tk.MustExec(`insert into t values ('[1,2,3]');`)
+ tk.MustExec(`insert into t values ('[-1]');`)
+ tk.MustGetErrCode(`insert into t values ('["1"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["a"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["汉字"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.2]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.0]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("11:00:00" as time)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02" as date)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02 11:00:00" as datetime)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast('{"a":1}' as json)));`, errno.ErrInvalidJSONValueForFuncIndex)
+
+ tk.MustExec(`drop table if exists t;`)
+ tk.MustExec(`create table t(a json, index idx((cast(a as char(1) array))));`)
+ tk.MustGetErrCode(`insert into t values ('[1,2,3]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[-1]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustExec(`insert into t values ('["1"]');`)
+ tk.MustExec(`insert into t values ('["a"]');`)
+ tk.MustGetErrCode(`insert into t values ('["汉字"]');`, errno.ErrFunctionalIndexDataIsTooLong)
+ tk.MustGetErrCode(`insert into t values ('[1.2]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.0]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("11:00:00" as time)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02" as date)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02 11:00:00" as datetime)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast('{"a":1}' as json)));`, errno.ErrInvalidJSONValueForFuncIndex)
+
+ tk.MustExec(`drop table if exists t;`)
+ tk.MustExec(`create table t(a json, index idx((cast(a as char(2) array))));`)
+ tk.MustGetErrCode(`insert into t values ('[1,2,3]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[-1]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustExec(`insert into t values ('["1"]');`)
+ tk.MustExec(`insert into t values ('["a"]');`)
+ tk.MustExec(`insert into t values ('["汉字"]');`)
+ tk.MustGetErrCode(`insert into t values ('[1.2]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.0]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("11:00:00" as time)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02" as date)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02 11:00:00" as datetime)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast('{"a":1}' as json)));`, errno.ErrInvalidJSONValueForFuncIndex)
+
+ tk.MustExec(`drop table if exists t;`)
+ tk.MustExec(`create table t(a json, index idx((cast(a as binary(1) array))));`)
+ tk.MustGetErrCode(`insert into t values ('[1,2,3]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[-1]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustExec(`insert into t values ('["1"]');`)
+ tk.MustExec(`insert into t values ('["a"]');`)
+ tk.MustGetErrCode(`insert into t values ('["汉字"]');`, errno.ErrFunctionalIndexDataIsTooLong)
+ tk.MustGetErrCode(`insert into t values ('[1.2]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.0]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("11:00:00" as time)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02" as date)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02 11:00:00" as datetime)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast('{"a":1}' as json)));`, errno.ErrInvalidJSONValueForFuncIndex)
+
+ tk.MustExec(`drop table if exists t;`)
+ tk.MustExec(`create table t(a json, index idx((cast(a as binary(2) array))));`)
+ tk.MustGetErrCode(`insert into t values ('[1,2,3]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[-1]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustExec(`insert into t values ('["1"]');`)
+ tk.MustExec(`insert into t values ('["a"]');`)
+ tk.MustGetErrCode(`insert into t values ('["汉字"]');`, errno.ErrFunctionalIndexDataIsTooLong)
+ tk.MustGetErrCode(`insert into t values ('[1.2]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.0]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("11:00:00" as time)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02" as date)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02 11:00:00" as datetime)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast('{"a":1}' as json)));`, errno.ErrInvalidJSONValueForFuncIndex)
+
+ tk.MustExec(`drop table if exists t;`)
+ tk.MustExec(`create table t(a json, index idx((cast(a as date array))));`)
+ tk.MustGetErrCode(`insert into t values ('[1,2,3]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[-1]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["1"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["a"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["汉字"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.2]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.0]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("11:00:00" as time)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustExec(`insert into t values (json_array(cast("2022-02-02" as date)));`)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02 11:00:00" as datetime)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast('{"a":1}' as json)));`, errno.ErrInvalidJSONValueForFuncIndex)
+
+ tk.MustExec(`drop table if exists t;`)
+ tk.MustExec(`create table t(a json, index idx((cast(a as time array))));`)
+ tk.MustGetErrCode(`insert into t values ('[1,2,3]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[-1]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["1"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["a"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["汉字"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.2]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.0]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustExec(`insert into t values (json_array(cast("11:00:00" as time)));`)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02" as date)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02 11:00:00" as datetime)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast('{"a":1}' as json)));`, errno.ErrInvalidJSONValueForFuncIndex)
+
+ tk.MustExec(`drop table if exists t;`)
+ tk.MustExec(`create table t(a json, index idx((cast(a as datetime array))));`)
+ tk.MustGetErrCode(`insert into t values ('[1,2,3]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[-1]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["1"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["a"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('["汉字"]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.2]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values ('[1.0]');`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("11:00:00" as time)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustGetErrCode(`insert into t values (json_array(cast("2022-02-02" as date)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ tk.MustExec(`insert into t values (json_array(cast("2022-02-02 11:00:00" as datetime)));`)
+ tk.MustGetErrCode(`insert into t values (json_array(cast('{"a":1}' as json)));`, errno.ErrInvalidJSONValueForFuncIndex)
+ }
+}
+
+func TestWriteMultiValuedIndex(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t1(pk int primary key, a json, index idx((cast(a as signed array))))")
+ tk.MustExec("insert into t1 values (1, '[1,2,2,3]')")
+ tk.MustExec("insert into t1 values (2, '[1,2,3]')")
+ tk.MustExec("insert into t1 values (3, '[]')")
+ tk.MustExec("insert into t1 values (4, '[2,3,4]')")
+ tk.MustExec("insert into t1 values (5, null)")
+ tk.MustExec("insert into t1 values (6, '1')")
+
+ t1, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t1"))
+ require.NoError(t, err)
+ for _, index := range t1.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, t1.IndexPrefix(), index, store, 11)
+ checkKey(t, t1.IndexPrefix(), index, store, [][]types.Datum{
+ {types.NewDatum(nil), types.NewIntDatum(5)},
+ {types.NewIntDatum(1), types.NewIntDatum(1)},
+ {types.NewIntDatum(1), types.NewIntDatum(2)},
+ {types.NewIntDatum(1), types.NewIntDatum(6)},
+ {types.NewIntDatum(2), types.NewIntDatum(1)},
+ {types.NewIntDatum(2), types.NewIntDatum(2)},
+ {types.NewIntDatum(2), types.NewIntDatum(4)},
+ {types.NewIntDatum(3), types.NewIntDatum(1)},
+ {types.NewIntDatum(3), types.NewIntDatum(2)},
+ {types.NewIntDatum(3), types.NewIntDatum(4)},
+ {types.NewIntDatum(4), types.NewIntDatum(4)},
+ })
+ }
+ }
+ tk.MustExec("delete from t1")
+ for _, index := range t1.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, t1.IndexPrefix(), index, store, 0)
+ }
+ }
+
+ tk.MustExec("drop table t1")
+ tk.MustExec("create table t1(pk int primary key, a json, index idx((cast(a as char(5) array))))")
+ tk.MustExec("insert into t1 values (1, '[\"abc\", \"abc \"]')")
+ tk.MustExec("insert into t1 values (2, '[\"b\"]')")
+ tk.MustExec("insert into t1 values (3, '[\"b \"]')")
+ tk.MustQuery("select pk from t1 where 'b ' member of (a)").Check(testkit.Rows("3"))
+
+ t1, err = dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t1"))
+ require.NoError(t, err)
+ for _, index := range t1.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, t1.IndexPrefix(), index, store, 4)
+ checkKey(t, t1.IndexPrefix(), index, store, [][]types.Datum{
+ {types.NewBytesDatum([]byte("abc")), types.NewIntDatum(1)},
+ {types.NewBytesDatum([]byte("abc ")), types.NewIntDatum(1)},
+ {types.NewBytesDatum([]byte("b")), types.NewIntDatum(2)},
+ {types.NewBytesDatum([]byte("b ")), types.NewIntDatum(3)},
+ })
+ }
+ }
+
+ tk.MustExec("update t1 set a = json_array_append(a, '$', 'bcd') where pk = 1")
+ tk.MustExec("update t1 set a = '[]' where pk = 2")
+ tk.MustExec("update t1 set a = '[\"abc\"]' where pk = 3")
+
+ for _, index := range t1.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, t1.IndexPrefix(), index, store, 4)
+ checkKey(t, t1.IndexPrefix(), index, store, [][]types.Datum{
+ {types.NewBytesDatum([]byte("abc")), types.NewIntDatum(1)},
+ {types.NewBytesDatum([]byte("abc")), types.NewIntDatum(3)},
+ {types.NewBytesDatum([]byte("abc ")), types.NewIntDatum(1)},
+ {types.NewBytesDatum([]byte("bcd")), types.NewIntDatum(1)},
+ })
+ }
+ }
+
+ tk.MustExec("delete from t1")
+ for _, index := range t1.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, t1.IndexPrefix(), index, store, 0)
+ }
+ }
+
+ tk.MustExec("drop table t1")
+ tk.MustExec("create table t1(pk int primary key, a json, index idx((cast(a as unsigned array))))")
+ tk.MustExec("insert into t1 values (1, '[1,2,2,3]')")
+ tk.MustExec("insert into t1 values (2, '[1,2,3]')")
+ tk.MustExec("insert into t1 values (3, '[]')")
+ tk.MustExec("insert into t1 values (4, '[2,3,4]')")
+ tk.MustExec("insert into t1 values (5, null)")
+ tk.MustExec("insert into t1 values (6, '1')")
+
+ t1, err = dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t1"))
+ require.NoError(t, err)
+ for _, index := range t1.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, t1.IndexPrefix(), index, store, 11)
+ checkKey(t, t1.IndexPrefix(), index, store, [][]types.Datum{
+ {types.NewDatum(nil), types.NewIntDatum(5)},
+ {types.NewUintDatum(1), types.NewIntDatum(1)},
+ {types.NewUintDatum(1), types.NewIntDatum(2)},
+ {types.NewUintDatum(1), types.NewIntDatum(6)},
+ {types.NewUintDatum(2), types.NewIntDatum(1)},
+ {types.NewUintDatum(2), types.NewIntDatum(2)},
+ {types.NewUintDatum(2), types.NewIntDatum(4)},
+ {types.NewUintDatum(3), types.NewIntDatum(1)},
+ {types.NewUintDatum(3), types.NewIntDatum(2)},
+ {types.NewUintDatum(3), types.NewIntDatum(4)},
+ {types.NewUintDatum(4), types.NewIntDatum(4)},
+ })
+ }
+ }
+ tk.MustExec("delete from t1")
+ for _, index := range t1.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, t1.IndexPrefix(), index, store, 0)
+ }
+ }
+}
+
+func TestWriteMultiValuedIndexPartitionTable(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec(`create table t1
+(
+ pk int primary key,
+ a json,
+ index idx ((cast(a as signed array)))
+) partition by range columns (pk) (partition p0 values less than (10), partition p1 values less than (20));`)
+ tk.MustExec("insert into t1 values (1, '[1,2,2,3]')")
+ tk.MustExec("insert into t1 values (11, '[1,2,3]')")
+ tk.MustExec("insert into t1 values (2, '[]')")
+ tk.MustExec("insert into t1 values (12, '[2,3,4]')")
+ tk.MustExec("insert into t1 values (3, null)")
+ tk.MustExec("insert into t1 values (13, null)")
+
+ t1, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t1"))
+ require.NoError(t, err)
+
+ expect := map[string]struct {
+ count int
+ vals [][]types.Datum
+ }{
+ "p0": {4, [][]types.Datum{
+ {types.NewDatum(nil), types.NewIntDatum(3)},
+ {types.NewIntDatum(1), types.NewIntDatum(1)},
+ {types.NewIntDatum(2), types.NewIntDatum(1)},
+ {types.NewIntDatum(3), types.NewIntDatum(1)},
+ }},
+ "p1": {7, [][]types.Datum{
+ {types.NewDatum(nil), types.NewIntDatum(13)},
+ {types.NewIntDatum(1), types.NewIntDatum(11)},
+ {types.NewIntDatum(2), types.NewIntDatum(11)},
+ {types.NewIntDatum(2), types.NewIntDatum(12)},
+ {types.NewIntDatum(3), types.NewIntDatum(11)},
+ {types.NewIntDatum(3), types.NewIntDatum(12)},
+ {types.NewIntDatum(4), types.NewIntDatum(12)},
+ }},
+ }
+
+ for _, def := range t1.Meta().GetPartitionInfo().Definitions {
+ partition := t1.(table.PartitionedTable).GetPartition(def.ID)
+ for _, index := range partition.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, partition.IndexPrefix(), index, store, expect[def.Name.L].count)
+ checkKey(t, partition.IndexPrefix(), index, store, expect[def.Name.L].vals)
+ }
+ }
+ }
+
+ tk.MustExec("delete from t1")
+ for _, def := range t1.Meta().GetPartitionInfo().Definitions {
+ partition := t1.(table.PartitionedTable).GetPartition(def.ID)
+ for _, index := range partition.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, partition.IndexPrefix(), index, store, 0)
+ }
+ }
+ }
+}
+
+func TestWriteMultiValuedIndexUnique(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t1(pk int primary key, a json, unique index idx((cast(a as signed array))))")
+ tk.MustExec("insert into t1 values (1, '[1,2,2]')")
+ tk.MustGetErrCode("insert into t1 values (2, '[1]')", errno.ErrDupEntry)
+ tk.MustExec("insert into t1 values (3, '[3,3,4]')")
+
+ t1, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t1"))
+ require.NoError(t, err)
+ for _, index := range t1.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, t1.IndexPrefix(), index, store, 4)
+ checkKey(t, t1.IndexPrefix(), index, store, [][]types.Datum{
+ {types.NewIntDatum(1)},
+ {types.NewIntDatum(2)},
+ {types.NewIntDatum(3)},
+ {types.NewIntDatum(4)},
+ })
+ }
+ }
+}
+
+func TestWriteMultiValuedIndexComposite(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t1(pk int primary key, a json, c int, d int, index idx(c, (cast(a as signed array)), d))")
+ tk.MustExec("insert into t1 values (1, '[1,2,2]', 1, 1)")
+ tk.MustExec("insert into t1 values (2, '[2,2,2]', 2, 2)")
+ tk.MustExec("insert into t1 values (3, '[3,3,4]', 3, 3)")
+ tk.MustExec("insert into t1 values (4, null, 4, 4)")
+ tk.MustExec("insert into t1 values (5, '[]', 5, 5)")
+
+ t1, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t1"))
+ require.NoError(t, err)
+ for _, index := range t1.Indices() {
+ if index.Meta().MVIndex {
+ checkCount(t, t1.IndexPrefix(), index, store, 6)
+ checkKey(t, t1.IndexPrefix(), index, store, [][]types.Datum{
+ {types.NewIntDatum(1), types.NewIntDatum(1), types.NewIntDatum(1), types.NewIntDatum(1)},
+ {types.NewIntDatum(1), types.NewIntDatum(2), types.NewIntDatum(1), types.NewIntDatum(1)},
+ {types.NewIntDatum(2), types.NewIntDatum(2), types.NewIntDatum(2), types.NewIntDatum(2)},
+ {types.NewIntDatum(3), types.NewIntDatum(3), types.NewIntDatum(3), types.NewIntDatum(3)},
+ {types.NewIntDatum(3), types.NewIntDatum(4), types.NewIntDatum(3), types.NewIntDatum(3)},
+ {types.NewIntDatum(4), types.NewDatum(nil), types.NewIntDatum(4), types.NewIntDatum(4)},
+ })
+ }
+ }
+}
+
+func checkCount(t *testing.T, prefix kv.Key, index table.Index, store kv.Storage, except int) {
+ c := 0
+ checkIndex(t, prefix, index, store, func(it kv.Iterator) {
+ c++
+ })
+ require.Equal(t, except, c)
+}
+
+func checkKey(t *testing.T, prefix kv.Key, index table.Index, store kv.Storage, except [][]types.Datum) {
+ idx := 0
+ checkIndex(t, prefix, index, store, func(it kv.Iterator) {
+ indexKey := decodeIndexKey(t, it.Key())
+ require.Equal(t, except[idx], indexKey)
+ idx++
+ })
+}
+
+func checkIndex(t *testing.T, prefix kv.Key, index table.Index, store kv.Storage, fn func(kv.Iterator)) {
+ startKey := codec.EncodeInt(prefix, index.Meta().ID)
+ prefix.Next()
+ se := testkit.NewTestKit(t, store).Session()
+ err := sessiontxn.NewTxn(context.Background(), se)
+ require.NoError(t, err)
+ txn, err := se.Txn(true)
+ require.NoError(t, err)
+ it, err := txn.Iter(startKey, prefix.PrefixNext())
+ require.NoError(t, err)
+ for it.Valid() && it.Key().HasPrefix(prefix) {
+ fn(it)
+ err = it.Next()
+ require.NoError(t, err)
+ }
+ it.Close()
+ se.Close()
+}
+
+func decodeIndexKey(t *testing.T, key kv.Key) []types.Datum {
+ var idLen = 8
+ var prefixLen = 1 + idLen /*tableID*/ + 2
+ _, _, isRecord, err := tablecodec.DecodeKeyHead(key)
+ require.NoError(t, err)
+ require.False(t, isRecord)
+ indexKey := key[prefixLen+idLen:]
+ var datumValues []types.Datum
+ for len(indexKey) > 0 {
+ remain, d, err := codec.DecodeOne(indexKey)
+ require.NoError(t, err)
+ datumValues = append(datumValues, d)
+ indexKey = remain
+ }
+ return datumValues
+}
diff --git a/expression/simple_rewriter.go b/expression/simple_rewriter.go
index 808db9f69b4cf..3343a0cbaa169 100644
--- a/expression/simple_rewriter.go
+++ b/expression/simple_rewriter.go
@@ -48,7 +48,7 @@ func ParseSimpleExprWithTableInfo(ctx sessionctx.Context, exprStr string, tableI
return nil, errors.Trace(err)
}
expr := stmts[0].(*ast.SelectStmt).Fields.Fields[0].Expr
- return RewriteSimpleExprWithTableInfo(ctx, tableInfo, expr)
+ return RewriteSimpleExprWithTableInfo(ctx, tableInfo, expr, false)
}
// ParseSimpleExprCastWithTableInfo parses simple expression string to Expression.
@@ -63,13 +63,13 @@ func ParseSimpleExprCastWithTableInfo(ctx sessionctx.Context, exprStr string, ta
}
// RewriteSimpleExprWithTableInfo rewrites simple ast.ExprNode to expression.Expression.
-func RewriteSimpleExprWithTableInfo(ctx sessionctx.Context, tbl *model.TableInfo, expr ast.ExprNode) (Expression, error) {
+func RewriteSimpleExprWithTableInfo(ctx sessionctx.Context, tbl *model.TableInfo, expr ast.ExprNode, allowCastArray bool) (Expression, error) {
dbName := model.NewCIStr(ctx.GetSessionVars().CurrentDB)
columns, names, err := ColumnInfos2ColumnsAndNames(ctx, dbName, tbl.Name, tbl.Cols(), tbl)
if err != nil {
return nil, err
}
- e, err := RewriteAstExpr(ctx, expr, NewSchema(columns...), names)
+ e, err := RewriteAstExpr(ctx, expr, NewSchema(columns...), names, allowCastArray)
if err != nil {
return nil, err
}
@@ -111,7 +111,7 @@ func ParseSimpleExprsWithNames(ctx sessionctx.Context, exprStr string, schema *S
// RewriteSimpleExprWithNames rewrites simple ast.ExprNode to expression.Expression.
func RewriteSimpleExprWithNames(ctx sessionctx.Context, expr ast.ExprNode, schema *Schema, names []*types.FieldName) (Expression, error) {
- e, err := RewriteAstExpr(ctx, expr, schema, names)
+ e, err := RewriteAstExpr(ctx, expr, schema, names, false)
if err != nil {
return nil, err
}
diff --git a/expression/util.go b/expression/util.go
index 22637d1b8617f..20de76cda383d 100644
--- a/expression/util.go
+++ b/expression/util.go
@@ -177,6 +177,31 @@ func ExtractEquivalenceColumns(result [][]Expression, exprs []Expression) [][]Ex
return result
}
+// FindUpperBound looks for column < constant or column <= constant and returns both the column
+// and constant. It return nil, 0 if the expression is not of this form.
+// It is used by derived Top N pattern and it is put here since it looks like
+// a general purpose routine. Similar routines can be added to find lower bound as well.
+func FindUpperBound(expr Expression) (*Column, int64) {
+ scalarFunction, scalarFunctionOk := expr.(*ScalarFunction)
+ if scalarFunctionOk {
+ args := scalarFunction.GetArgs()
+ if len(args) == 2 {
+ col, colOk := args[0].(*Column)
+ constant, constantOk := args[1].(*Constant)
+ if colOk && constantOk && (scalarFunction.FuncName.L == ast.LT || scalarFunction.FuncName.L == ast.LE) {
+ value, valueOk := constant.Value.GetValue().(int64)
+ if valueOk {
+ if scalarFunction.FuncName.L == ast.LT {
+ return col, value - 1
+ }
+ return col, value
+ }
+ }
+ }
+ }
+ return nil, 0
+}
+
func extractEquivalenceColumns(result [][]Expression, expr Expression) [][]Expression {
switch v := expr.(type) {
case *ScalarFunction:
@@ -415,7 +440,6 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
if v.InOperand {
newExpr = SetExprColumnInOperand(newExpr)
}
- newExpr.SetCoercibility(v.Coercibility())
return true, false, newExpr
case *ScalarFunction:
substituted := false
@@ -427,8 +451,10 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
return substituted, hasFail, v
}
if substituted {
+ flag := v.RetType.GetFlag()
e := BuildCastFunction(v.GetCtx(), newArg, v.RetType)
e.SetCoercibility(v.Coercibility())
+ e.GetType().SetFlag(flag)
return true, false, e
}
return false, false, v
@@ -436,7 +462,11 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
// cowExprRef is a copy-on-write util, args array allocation happens only
// when expr in args is changed
refExprArr := cowExprRef{v.GetArgs(), nil}
- _, coll := DeriveCollationFromExprs(v.GetCtx(), v.GetArgs()...)
+ oldCollEt, err := CheckAndDeriveCollationFromExprs(v.GetCtx(), v.FuncName.L, v.RetType.EvalType(), v.GetArgs()...)
+ if err != nil {
+ logutil.BgLogger().Error("Unexpected error happened during ColumnSubstitution", zap.Stack("stack"))
+ return false, false, v
+ }
var tmpArgForCollCheck []Expression
if collate.NewCollationEnabled() {
tmpArgForCollCheck = make([]Expression, len(v.GetArgs()))
@@ -452,9 +482,18 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
changed = false
copy(tmpArgForCollCheck, refExprArr.Result())
tmpArgForCollCheck[idx] = newFuncExpr
- _, newColl := DeriveCollationFromExprs(v.GetCtx(), tmpArgForCollCheck...)
- if coll == newColl {
- changed = checkCollationStrictness(coll, newFuncExpr.GetType().GetCollate())
+ newCollEt, err := CheckAndDeriveCollationFromExprs(v.GetCtx(), v.FuncName.L, v.RetType.EvalType(), tmpArgForCollCheck...)
+ if err != nil {
+ logutil.BgLogger().Error("Unexpected error happened during ColumnSubstitution", zap.Stack("stack"))
+ return false, failed, v
+ }
+ if oldCollEt.Collation == newCollEt.Collation {
+ if newFuncExpr.GetType().GetCollate() == arg.GetType().GetCollate() && newFuncExpr.Coercibility() == arg.Coercibility() {
+ // It's safe to use the new expression, otherwise some cases in projection push-down will be wrong.
+ changed = true
+ } else {
+ changed = checkCollationStrictness(oldCollEt.Collation, newFuncExpr.GetType().GetCollate())
+ }
}
}
hasFail = hasFail || failed || oldChanged != changed
@@ -1143,6 +1182,33 @@ func IsMutableEffectsExpr(expr Expression) bool {
return false
}
+// IsInmutableExpr checks whether this expression only consists of foldable functions and inmutable constants.
+// This expression can be evaluated by using `expr.Eval(chunk.Row{})` directly if it's inmutable.
+func IsInmutableExpr(expr Expression) bool {
+ switch x := expr.(type) {
+ case *ScalarFunction:
+ if _, ok := unFoldableFunctions[x.FuncName.L]; ok {
+ return false
+ }
+ if _, ok := mutableEffectsFunctions[x.FuncName.L]; ok {
+ return false
+ }
+ for _, arg := range x.GetArgs() {
+ if !IsInmutableExpr(arg) {
+ return false
+ }
+ }
+ return true
+ case *Constant:
+ if x.DeferredExpr != nil || x.ParamMarker != nil {
+ return false
+ }
+ return true
+ default:
+ return false
+ }
+}
+
// RemoveDupExprs removes identical exprs. Not that if expr contains functions which
// are mutable or have side effects, we cannot remove it even if it has duplicates;
// if the plan is going to be cached, we cannot remove expressions containing `?` neither.
@@ -1239,7 +1305,7 @@ func ContainCorrelatedColumn(exprs []Expression) bool {
// TODO: Do more careful check here.
func MaybeOverOptimized4PlanCache(ctx sessionctx.Context, exprs []Expression) bool {
// If we do not enable plan cache, all the optimization can work correctly.
- if !ctx.GetSessionVars().StmtCtx.UseCache || ctx.GetSessionVars().StmtCtx.SkipPlanCache {
+ if !ctx.GetSessionVars().StmtCtx.UseCache {
return false
}
return containMutableConst(ctx, exprs)
diff --git a/extension/BUILD.bazel b/extension/BUILD.bazel
index 12ca672b3c86f..f180dcf7a9955 100644
--- a/extension/BUILD.bazel
+++ b/extension/BUILD.bazel
@@ -29,6 +29,7 @@ go_library(
go_test(
name = "extension_test",
+ timeout = "short",
srcs = [
"bootstrap_test.go",
"event_listener_test.go",
@@ -37,6 +38,7 @@ go_test(
"registry_test.go",
],
embed = [":extension"],
+ flaky = True,
deps = [
"//expression",
"//parser/ast",
diff --git a/go.mod b/go.mod
index 01400b701b85b..09687001e28f1 100644
--- a/go.mod
+++ b/go.mod
@@ -3,15 +3,17 @@ module github.com/pingcap/tidb
go 1.19
require (
- cloud.google.com/go/storage v1.21.0
+ cloud.google.com/go/storage v1.27.0
+ github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0
- github.com/BurntSushi/toml v1.2.0
+ github.com/BurntSushi/toml v1.2.1
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/Jeffail/gabs/v2 v2.5.1
github.com/Masterminds/semver v1.5.0
github.com/Shopify/sarama v1.29.0
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581
+ github.com/apache/skywalking-eyes v0.4.0
github.com/ashanbrown/makezero v1.1.1
github.com/aws/aws-sdk-go v1.44.48
github.com/blacktear23/go-proxyprotocol v1.0.2
@@ -19,28 +21,29 @@ require (
github.com/charithe/durationcheck v0.0.9
github.com/cheggaaa/pb/v3 v3.0.8
github.com/cheynewallace/tabby v1.1.1
+ github.com/cloudfoundry/gosigar v1.3.6
github.com/cockroachdb/errors v1.8.1
github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5
github.com/coocood/freecache v1.2.1
github.com/coreos/go-semver v0.3.0
- github.com/daixiang0/gci v0.6.3
+ github.com/daixiang0/gci v0.9.0
github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37
- github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d
+ github.com/dgraph-io/ristretto v0.1.1
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13
github.com/docker/go-units v0.4.0
github.com/emirpasic/gods v1.18.1
github.com/fatanugraha/noloopclosure v0.1.1
- github.com/fatih/color v1.13.0
+ github.com/fatih/color v1.14.1
github.com/fsouza/fake-gcs-server v1.19.0
- github.com/go-sql-driver/mysql v1.6.0
+ github.com/go-sql-driver/mysql v1.7.0
github.com/gogo/protobuf v1.3.2
github.com/golang/mock v1.6.0
github.com/golang/protobuf v1.5.2
github.com/golang/snappy v0.0.4
- github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a
- github.com/golangci/golangci-lint v1.49.0
+ github.com/golangci/gofmt v0.0.0-20220901101216-f2edd75033f2
+ github.com/golangci/golangci-lint v1.51.0
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb
- github.com/golangci/misspell v0.3.5
+ github.com/golangci/misspell v0.4.0
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21
github.com/google/btree v1.1.2
github.com/google/pprof v0.0.0-20211122183932-1daafda22083
@@ -54,41 +57,44 @@ require (
github.com/jedib0t/go-pretty/v6 v6.2.2
github.com/jingyugao/rowserrcheck v1.1.1
github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df
- github.com/kisielk/errcheck v1.6.2
- github.com/klauspost/compress v1.15.1
- github.com/kyoh86/exportloopref v0.1.8
+ github.com/kisielk/errcheck v1.6.3
+ github.com/klauspost/compress v1.15.13
+ github.com/kyoh86/exportloopref v0.1.11
github.com/lestrrat-go/jwx/v2 v2.0.6
- github.com/mgechev/revive v1.2.4
+ github.com/mgechev/revive v1.2.5
github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7
github.com/nishanths/predeclared v0.2.2
github.com/opentracing/basictracer-go v1.0.0
github.com/opentracing/opentracing-go v1.2.0
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
- github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580
- github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278
- github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3
+ github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d
+ github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32
+ github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c
github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059
- github.com/pingcap/kvproto v0.0.0-20221103025916-e7e21f0e9cd9
- github.com/pingcap/log v1.1.1-0.20221110065318-21a4942860b3
+ github.com/pingcap/kvproto v0.0.0-20230206112125-0561adc37543
+ github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c
github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4
github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e
- github.com/pingcap/tipb v0.0.0-20221020071514-cd933387bcb5
+ github.com/pingcap/tipb v0.0.0-20230119054146-c6b7a5a1623b
github.com/pkg/errors v0.9.1
- github.com/prometheus/client_golang v1.13.0
- github.com/prometheus/client_model v0.2.0
- github.com/prometheus/common v0.37.0
+ github.com/prometheus/client_golang v1.14.0
+ github.com/prometheus/client_model v0.3.0
+ github.com/prometheus/common v0.39.0
github.com/prometheus/prometheus v0.0.0-20190525122359-d20e84d0fb64
- github.com/shirou/gopsutil/v3 v3.22.7
+ github.com/sasha-s/go-deadlock v0.2.0
+ github.com/shirou/gopsutil/v3 v3.22.12
github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0
github.com/soheilhy/cmux v0.1.5
- github.com/spf13/cobra v1.5.0
+ github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
- github.com/stretchr/testify v1.8.0
+ github.com/spkg/bom v1.0.0
+ github.com/stathat/consistent v1.0.0
+ github.com/stretchr/testify v1.8.1
github.com/tdakkota/asciicheck v0.1.1
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
- github.com/tikv/client-go/v2 v2.0.3-0.20221108030801-9c0835c80eba
- github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07
- github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144
+ github.com/tikv/client-go/v2 v2.0.6-0.20230209044022-95ebf6a86301
+ github.com/tikv/pd/client v0.0.0-20230209034200-6d23a31c24be
+ github.com/timakin/bodyclose v0.0.0-20221125081123-e39cf3fc478e
github.com/twmb/murmur3 v1.1.3
github.com/uber/jaeger-client-go v2.22.1+incompatible
github.com/vbauerster/mpb/v7 v7.5.3
@@ -100,46 +106,50 @@ require (
go.etcd.io/etcd/client/v3 v3.5.2
go.etcd.io/etcd/server/v3 v3.5.2
go.etcd.io/etcd/tests/v3 v3.5.2
- go.opencensus.io v0.23.0
+ go.opencensus.io v0.24.0
go.uber.org/atomic v1.10.0
go.uber.org/automaxprocs v1.4.0
go.uber.org/goleak v1.2.0
- go.uber.org/multierr v1.8.0
- go.uber.org/zap v1.23.0
- golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
- golang.org/x/net v0.1.0
- golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
- golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
- golang.org/x/sys v0.1.0
- golang.org/x/term v0.1.0
- golang.org/x/text v0.4.0
- golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
- golang.org/x/tools v0.1.12
- google.golang.org/api v0.74.0
- google.golang.org/grpc v1.45.0
+ go.uber.org/multierr v1.9.0
+ go.uber.org/zap v1.24.0
+ golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e
+ golang.org/x/net v0.6.0
+ golang.org/x/oauth2 v0.5.0
+ golang.org/x/sync v0.1.0
+ golang.org/x/sys v0.5.0
+ golang.org/x/term v0.5.0
+ golang.org/x/text v0.7.0
+ golang.org/x/time v0.3.0
+ golang.org/x/tools v0.5.0
+ google.golang.org/api v0.103.0
+ google.golang.org/grpc v1.52.3
gopkg.in/yaml.v2 v2.4.0
- honnef.co/go/tools v0.3.3
+ honnef.co/go/tools v0.4.0
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0
sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
)
require (
- cloud.google.com/go v0.100.2 // indirect
- cloud.google.com/go/compute v1.5.0 // indirect
- cloud.google.com/go/iam v0.1.1 // indirect
- github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 // indirect
+ cloud.google.com/go v0.105.0 // indirect
+ cloud.google.com/go/compute v1.13.0 // indirect
+ cloud.google.com/go/compute/metadata v0.2.1 // indirect
+ cloud.google.com/go/iam v0.8.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 // indirect
github.com/DataDog/zstd v1.4.5 // indirect
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
+ github.com/Masterminds/goutils v1.1.1 // indirect
+ github.com/Masterminds/semver/v3 v3.1.1 // indirect
+ github.com/Masterminds/sprig/v3 v3.2.2 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
+ github.com/bmatcuk/doublestar/v2 v2.0.4 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
- github.com/cespare/xxhash/v2 v2.1.2 // indirect
- github.com/chavacava/garif v0.0.0-20220630083739-93517212f375 // indirect
+ github.com/cespare/xxhash/v2 v2.2.0 // indirect
+ github.com/chavacava/garif v0.0.0-20221024190013-b3ef35877348 // indirect
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
github.com/cockroachdb/redact v1.0.8 // indirect
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
@@ -162,8 +172,10 @@ require (
github.com/goccy/go-json v0.9.11 // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
- github.com/google/go-cmp v0.5.8 // indirect
- github.com/googleapis/gax-go/v2 v2.2.0 // indirect
+ github.com/google/go-cmp v0.5.9 // indirect
+ github.com/google/licensecheck v0.3.1 // indirect
+ github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+ github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/gorilla/handlers v1.5.1 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
@@ -172,7 +184,9 @@ require (
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/hashicorp/go-uuid v1.0.2 // indirect
github.com/hexops/gotextdiff v1.0.3 // indirect
- github.com/inconshreveable/mousetrap v1.0.0 // indirect
+ github.com/huandu/xstrings v1.3.1 // indirect
+ github.com/imdario/mergo v0.3.11 // indirect
+ github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
@@ -191,9 +205,11 @@ require (
github.com/lestrrat-go/option v1.0.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
- github.com/mattn/go-isatty v0.0.16 // indirect
+ github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
- github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+ github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+ github.com/mitchellh/copystructure v1.0.0 // indirect
+ github.com/mitchellh/reflectwalk v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
@@ -201,23 +217,29 @@ require (
github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
+ github.com/onsi/gomega v1.23.0 // indirect
+ github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
+ github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 // indirect
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
- github.com/prometheus/procfs v0.8.0 // indirect
+ github.com/prometheus/procfs v0.9.0 // indirect
github.com/prometheus/tsdb v0.8.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
- github.com/rivo/uniseg v0.4.2 // indirect
+ github.com/rivo/uniseg v0.4.3 // indirect
github.com/rogpeppe/go-internal v1.6.1 // indirect
+ github.com/shopspring/decimal v1.2.0 // indirect
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
- github.com/stathat/consistent v1.0.0 // indirect
- github.com/tklauser/go-sysconf v0.3.10 // indirect
- github.com/tklauser/numcpus v0.4.0 // indirect
+ github.com/spaolacci/murmur3 v1.1.0 // indirect
+ github.com/spf13/cast v1.5.0 // indirect
+ github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
+ github.com/tklauser/go-sysconf v0.3.11 // indirect
+ github.com/tklauser/numcpus v0.6.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
@@ -236,16 +258,16 @@ require (
go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
go.opentelemetry.io/otel/trace v0.20.0 // indirect
go.opentelemetry.io/proto/otlp v0.7.0 // indirect
- golang.org/x/crypto v0.1.0 // indirect
- golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d // indirect
- golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
- golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+ golang.org/x/crypto v0.5.0 // indirect
+ golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
+ golang.org/x/mod v0.7.0 // indirect
+ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
- google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb // indirect
+ google.golang.org/genproto v0.0.0-20230202175211-008b39050e57 // indirect
google.golang.org/protobuf v1.28.1 // indirect
- gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+ gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
- sigs.k8s.io/yaml v1.2.0 // indirect
+ sigs.k8s.io/yaml v1.3.0 // indirect
)
replace (
diff --git a/go.sum b/go.sum
index 0730b5193bd55..02b512ade19d3 100644
--- a/go.sum
+++ b/go.sum
@@ -13,38 +13,24 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
-cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
-cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
-cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
-cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
-cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY=
-cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
-cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
-cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
-cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
-cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
-cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
-cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
-cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U=
-cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y=
-cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
+cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y=
+cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
-cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw=
-cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
-cloud.google.com/go/compute v1.5.0 h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM=
-cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
+cloud.google.com/go/compute v1.13.0 h1:AYrLkB8NPdDRslNp4Jxmzrhdr03fUAIDbiGFjLWowoU=
+cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE=
+cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=
+cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
-cloud.google.com/go/iam v0.1.1 h1:4CapQyNFjiksks1/x7jsvsygFPhihslYk5GptIrlX68=
-cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw=
+cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk=
+cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE=
+cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -54,8 +40,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-cloud.google.com/go/storage v1.21.0 h1:HwnT2u2D309SFDHQII6m18HlrCi3jAXhUMTLOWXYH14=
-cloud.google.com/go/storage v1.21.0/go.mod h1:XmRlxkgPjlBONznT2dDUU/5XlpU2OjMnKuqnZI01LAA=
+cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ=
+cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
@@ -70,8 +56,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 h1:62Ew5xXg5UCGIXDOM
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0/go.mod h1:eHWhQKXc1Gv1DvWH//UzgWjWFEo0Pp4pH2vBzjBw8Fc=
github.com/Azure/go-autorest v11.2.8+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
-github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
@@ -85,8 +71,14 @@ github.com/Jeffail/gabs/v2 v2.5.1 h1:ANfZYjpMlfTTKebycu4X1AgkVWumFVDYQl7JwOr4mDk
github.com/Jeffail/gabs/v2 v2.5.1/go.mod h1:xCn81vdHKxFUuWWAaD5jCTQDNPBMh5pPs9IJ+NcziBI=
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
+github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
+github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
+github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
+github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
+github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI=
github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
@@ -111,8 +103,9 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2c
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 h1:Q/yk4z/cHUVZfgTqtD09qeYBxHwshQAjVRX73qs8UH0=
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU=
-github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/apache/skywalking-eyes v0.4.0 h1:O13kdRU6FCEZevfD01mdhTgCZLLfPZIQ0GXZrLl7FpQ=
+github.com/apache/skywalking-eyes v0.4.0/go.mod h1:WblDbBgOLsLN0FJEBa9xj6PhuUA/J6spKYVTG4/F8Ls=
github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 h1:Jz3KVLYY5+JO7rDiX0sAuRGtuv2vG01r17Y9nLMWNUw=
github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@@ -140,6 +133,8 @@ github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8/go.mod h1:Iev9Q3MErcn+
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/blacktear23/go-proxyprotocol v1.0.2 h1:zR7PZeoU0wAkElcIXenFiy3R56WB6A+UEVi4c6RH8wo=
github.com/blacktear23/go-proxyprotocol v1.0.2/go.mod h1:FSCbgnRZrQXazBLL5snfBbrcFSMtcmUDhSRb9OfFA1o=
+github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI=
+github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw=
github.com/carlmjohnson/flagext v0.21.0 h1:/c4uK3ie786Z7caXLcIMvePNSSiH3bQVGDvmGLMme60=
github.com/carlmjohnson/flagext v0.21.0/go.mod h1:Eenv0epIUAr4NuedNmkzI8WmBmjIxZC239XcKxYS2ac=
github.com/cenk/backoff v2.0.0+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE=
@@ -151,12 +146,13 @@ github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/charithe/durationcheck v0.0.9 h1:mPP4ucLrf/rKZiIG/a9IPXHGlh8p4CzgpyTy6EEutYk=
github.com/charithe/durationcheck v0.0.9/go.mod h1:SSbRIBVfMjCi/kEB6K65XEA83D6prSM8ap1UCpNKtgg=
-github.com/chavacava/garif v0.0.0-20220630083739-93517212f375 h1:E7LT642ysztPWE0dfz43cWOvMiF42DyTRC+eZIaO4yI=
-github.com/chavacava/garif v0.0.0-20220630083739-93517212f375/go.mod h1:4m1Rv7xfuwWPNKXlThldNuJvutYM6J95wNuuVmn55To=
+github.com/chavacava/garif v0.0.0-20221024190013-b3ef35877348 h1:cy5GCEZLUCshCGCRRUjxHrDUqkB4l5cuUt3ShEckQEo=
+github.com/chavacava/garif v0.0.0-20221024190013-b3ef35877348/go.mod h1:f/miWtG3SSuTxKsNK3o58H1xl+XV6ZIfbC6p7lPPB8U=
github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA=
github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
github.com/cheynewallace/tabby v1.1.1 h1:JvUR8waht4Y0S3JF17G6Vhyt+FRhnqVCkk8l4YrOU54=
@@ -165,14 +161,10 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudfoundry/gosigar v1.3.6 h1:gIc08FbB3QPb+nAQhINIK/qhf5REKkY0FTGgRGXkcVc=
+github.com/cloudfoundry/gosigar v1.3.6/go.mod h1:lNWstu5g5gw59O09Y+wsMNFzBSnU8a0u+Sfx4dq360E=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4=
github.com/cockroachdb/cockroach v0.0.0-20170608034007-84bc9597164f/go.mod h1:xeT/CQ0qZHangbYbWShlCGAx31aV4AjGswDUjhKS6HQ=
@@ -215,13 +207,14 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
-github.com/daixiang0/gci v0.6.3 h1:wUAqXChk8HbwXn8AfxD9DYSCp9Bpz1L3e6Q4Roe+q9E=
-github.com/daixiang0/gci v0.6.3/go.mod h1:EpVfrztufwVgQRXjnX4zuNinEpLj5OmMjtu/+MB0V0c=
+github.com/daixiang0/gci v0.9.0 h1:t8XZ0vK6l0pwPoOmoGyqW2NwQlvbpAQNVvu/GRBgykM=
+github.com/daixiang0/gci v0.9.0/go.mod h1:EpVfrztufwVgQRXjnX4zuNinEpLj5OmMjtu/+MB0V0c=
github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37 h1:X6mKGhCFOxrKeeHAjv/3UvT6e5RRxW6wRdlqlV6/H4w=
github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37/go.mod h1:DC3JtzuG7kxMvJ6dZmf2ymjNyoXwgtklr7FN+Um2B0U=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -231,8 +224,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
-github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d h1:Wrc3UKTS+cffkOx0xRGFC+ZesNuTfn0ThvEC72N0krk=
-github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c=
+github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
+github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
@@ -259,11 +252,8 @@ github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FM
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
-github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE=
@@ -274,8 +264,8 @@ github.com/fatanugraha/noloopclosure v0.1.1 h1:AhepjAikNpk50qTZoipHZqeZtnyKT/C2T
github.com/fatanugraha/noloopclosure v0.1.1/go.mod h1:Mi9CiG5QvEgvPLtZLsTzjYwjIDnWAbo10r0BG7JpJII=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
-github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
-github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
+github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4=
github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
@@ -289,11 +279,11 @@ github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD
github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
-github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
+github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsouza/fake-gcs-server v1.19.0 h1:XyaGOlqo+R5sjT03x2ymk0xepaQlgwhRLTT2IopW0zA=
github.com/fsouza/fake-gcs-server v1.19.0/go.mod h1:JtXHY/QzHhtyIxsNfIuQ+XgHtRb5B/w8nqbL5O8zqo0=
github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E=
@@ -315,7 +305,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
@@ -327,8 +316,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
-github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
-github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
@@ -365,7 +354,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -393,14 +381,14 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
-github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.49.0 h1:I8WHOavragDttlLHtSraHn/h39C+R60bEQ5NoGcHQr8=
-github.com/golangci/golangci-lint v1.49.0/go.mod h1:+V/7lLv449R6w9mQ3WdV0EKh7Je/jTylMeSwBZcLeWE=
+github.com/golangci/gofmt v0.0.0-20220901101216-f2edd75033f2 h1:amWTbTGqOZ71ruzrdA+Nx5WA3tV1N0goTspwmKCQvBY=
+github.com/golangci/gofmt v0.0.0-20220901101216-f2edd75033f2/go.mod h1:9wOXstvyDRshQ9LggQuzBCGysxs3b6Uo/1MvYCR2NMs=
+github.com/golangci/golangci-lint v1.51.0 h1:M1bpDymgdaPKNzPwQdebCGki/nzvVkr2f/eUfk9C9oU=
+github.com/golangci/golangci-lint v1.51.0/go.mod h1:7taIMcmZ5ksCuRruCV/mm8Ir3sE+bIuOHVCvt1B4hi4=
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
-github.com/golangci/misspell v0.3.5 h1:pLzmVdl3VxTOncgzHcvLOKirdvcx/TydsClUQXTehjo=
-github.com/golangci/misspell v0.3.5/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/misspell v0.4.0 h1:KtVB/hTK4bbL/S6bs64rYyk8adjmh1BygbBiaAiX+a0=
+github.com/golangci/misspell v0.4.0/go.mod h1:W6O/bwV6lGDxUCChm2ykw9NQdd5bYd1Xkjo88UcWyJc=
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
@@ -421,18 +409,19 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/licensecheck v0.3.1 h1:QoxgoDkaeC4nFrtGN1jV7IPmDCHFNIVh54e5hSt6sPs=
+github.com/google/licensecheck v0.3.1/go.mod h1:ORkR35t/JjW+emNKtfJDII0zlciG9JgbT7SmsohlHmY=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ=
-github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
github.com/google/pprof v0.0.0-20180605153948-8b03ce837f34/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -441,25 +430,19 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20211122183932-1daafda22083 h1:c8EUapQFi+kjzedr4c6WqbwMdmB95+oDBWZ5XFHFYxY=
github.com/google/pprof v0.0.0-20211122183932-1daafda22083/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs=
+github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
-github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
-github.com/googleapis/gax-go/v2 v2.2.0 h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE=
-github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
+github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
+github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
github.com/gophercloud/gophercloud v0.0.0-20190301152420-fca40860790e/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -476,7 +459,6 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk=
github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc=
github.com/gostaticanalysis/comment v1.4.2 h1:hlnx5+S2fY9Zo9ePo4AhgYsYHbM2+eAv8m/s1JiCd6Q=
@@ -492,7 +474,6 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
@@ -527,15 +508,19 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs=
+github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
+github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
+github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/influxdata/influxdb v0.0.0-20170331210902-15e594fc09f1/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
@@ -601,8 +586,8 @@ github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiD
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/errcheck v1.6.2 h1:uGQ9xI8/pgc9iOoCe7kWQgRE6SBTrCGmTSf0LrEtY7c=
-github.com/kisielk/errcheck v1.6.2/go.mod h1:nXw/i/MfnvRHqXa7XXmQMUB0oNFGuBrNI8d8NLy0LPw=
+github.com/kisielk/errcheck v1.6.3 h1:dEKh+GLHcWm2oN34nMvDzn1sqI0i0WxPvrgiJA5JuM8=
+github.com/kisielk/errcheck v1.6.3/go.mod h1:nXw/i/MfnvRHqXa7XXmQMUB0oNFGuBrNI8d8NLy0LPw=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
@@ -611,8 +596,8 @@ github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
-github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
-github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0=
+github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
@@ -628,8 +613,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kyoh86/exportloopref v0.1.8 h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77LoN/M=
-github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg=
+github.com/kyoh86/exportloopref v0.1.11 h1:1Z0bcmTypkL3Q4k+IDHMWTcnCliEZcaPiIe0/ymEyhQ=
+github.com/kyoh86/exportloopref v0.1.11/go.mod h1:qkV4UF1zGl6EkF1ox8L5t9SwyeBAZ3qLMd6up458uqA=
github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80=
@@ -653,7 +638,6 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -661,9 +645,9 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
@@ -671,17 +655,20 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
-github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/maxatome/go-testdeep v1.11.0 h1:Tgh5efyCYyJFGUYiT0qxBSIDeXw0F5zSoatlou685kk=
github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
-github.com/mgechev/revive v1.2.4 h1:+2Hd/S8oO2H0Ikq2+egtNwQsVhAeELHjxjIUFX5ajLI=
-github.com/mgechev/revive v1.2.4/go.mod h1:iAWlQishqCuj4yhV24FTnKSXGpbAA+0SckXB8GQMX/Q=
+github.com/mgechev/revive v1.2.5 h1:UF9AR8pOAuwNmhXj2odp4mxv9Nx2qUIwVz8ZsU+Mbec=
+github.com/mgechev/revive v1.2.5/go.mod h1:nFOXent79jMTISAfOAasKfy0Z2Ejq0WX7Qn/KAdYopI=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.10/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
+github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@@ -690,6 +677,8 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE=
github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@@ -719,8 +708,8 @@ github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk=
github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c=
-github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@@ -733,7 +722,8 @@ github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1ls
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
+github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
+github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
github.com/opentracing-contrib/go-stdlib v0.0.0-20170113013457-1de4cc2120e7/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w=
github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo=
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
@@ -752,43 +742,43 @@ github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwp
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg=
github.com/petermattis/goid v0.0.0-20170504144140-0ded85884ba5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
+github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo=
+github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 h1:MKVFZuqFvAMiDtv3AbihOQ6rY5IE8LWflI1BuZ/hF0Y=
-github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580/go.mod h1:upwDfet29M5y5koWilbWWA6ca3Lr0YVuzwX/DK58Vdk=
+github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d h1:AEcvKyVM8CUII3bYzgz8haFXtGiqcrtXW1csu/5UELY=
+github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d/go.mod h1:p8QnkZnmyV8L/M/jzYb8rT7kv3bz9m7bn1Ju94wDifs=
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
-github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0=
github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc=
+github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 h1:R8gStypOBmpnHEx1qi//SaqxJVI4inOqljg/Aj5/390=
+github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc=
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
-github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
-github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 h1:3Dm0DWeQlwV8LbpQxP2tojHhxd9aY59KI+QN0ns6bBo=
-github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
+github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 h1:m5ZsBa5o/0CkzZXfXLaThzKuR85SnHHetqBCpzQ30h8=
+github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
-github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 h1:kJolJWbyadVeL8RKBlqmXQR7FRKPsIeU85TUYyhbhiQ=
-github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
+github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ=
+github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 h1:Pe2LbxRmbTfAoKJ65bZLmhahmvHm7n9DUxGRQT00208=
github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
-github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
-github.com/pingcap/kvproto v0.0.0-20221103025916-e7e21f0e9cd9 h1:ho5XUD8DVCnkpEj8oiTR57FXDTXnH6znyLe0gyrtzKk=
-github.com/pingcap/kvproto v0.0.0-20221103025916-e7e21f0e9cd9/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
+github.com/pingcap/kvproto v0.0.0-20230206112125-0561adc37543 h1:QcC52K9hhsP6eVmQBnSMI/b8TiOUVztbaeduTXspmeQ=
+github.com/pingcap/kvproto v0.0.0-20230206112125-0561adc37543/go.mod h1:+on3Lfk/fb1lXkud3XvskJumhSIEEgN2TTbMObUlrxE=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
-github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
-github.com/pingcap/log v1.1.1-0.20221110065318-21a4942860b3 h1:T7e5Low0BU2ZazI2dz2mh3W1qv+w8wtvq1YR8DneA0c=
-github.com/pingcap/log v1.1.1-0.20221110065318-21a4942860b3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
+github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
+github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c h1:crhkw6DD+07Bg1wYhW5Piw+kYNKZqFQqfC2puUf6gMI=
+github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM=
github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4/go.mod h1:sDCsM39cGiv2vwunZkaFA917vVkqDTGSPbbV7z4Oops=
-github.com/pingcap/tipb v0.0.0-20221020071514-cd933387bcb5 h1:Yoo8j5xQGxjlsC3yt0ndsiAz0WZXED9rzsKmEN0U0DY=
-github.com/pingcap/tipb v0.0.0-20221020071514-cd933387bcb5/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
+github.com/pingcap/tipb v0.0.0-20230119054146-c6b7a5a1623b h1:j5sw2YZY7QfgIFZEoUcn1P5cYflms1PCVVS96i+IQiI=
+github.com/pingcap/tipb v0.0.0-20230119054146-c6b7a5a1623b/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -807,31 +797,29 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU=
-github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
+github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
+github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
-github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
+github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI=
+github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
-github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
+github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
+github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/prometheus/prometheus v0.0.0-20190525122359-d20e84d0fb64 h1:3DyLm+sTAJkfLyR/1pJ3L+fU2lFufWbpcgMFlGtqeyA=
github.com/prometheus/prometheus v0.0.0-20190525122359-d20e84d0fb64/go.mod h1:oYrT4Vs22/NcnoVYXt5m4cIHP+znvgyusahVpyETKTw=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
@@ -843,8 +831,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6O
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=
-github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
+github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rlmcpherson/s3gof3r v0.5.0/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2r/Go7Bo71Rs=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@@ -859,14 +847,18 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samuel/go-zookeeper v0.0.0-20161028232340-1d7be4effb13/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/sasha-s/go-deadlock v0.0.0-20161201235124-341000892f3d/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
+github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y=
+github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA=
-github.com/shirou/gopsutil/v3 v3.22.7 h1:flKnuCMfUUrO+oAvwAd6GKZgnPzr098VA/UJ14nhJd4=
-github.com/shirou/gopsutil/v3 v3.22.7/go.mod h1:s648gW4IywYzUfE/KjXxUsqrqx/T2xO5VqOXxONeRfI=
+github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs=
+github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
+github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk=
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
@@ -880,6 +872,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -888,26 +881,34 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
+github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
-github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
+github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
+github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
+github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/spkg/bom v1.0.0 h1:S939THe0ukL5WcTGiGqkgtaW5JW+O6ITaIlpJXTYY64=
+github.com/spkg/bom v1.0.0/go.mod h1:lAz2VbTuYNcvs7iaFF8WW0ufXrHShJ7ck1fYFFbVXJs=
github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U=
github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -916,8 +917,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tdakkota/asciicheck v0.1.1 h1:PKzG7JUTUmVspQTDqtkX9eSiLGossXTybutHwTXuO0A=
github.com/tdakkota/asciicheck v0.1.1/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -927,18 +929,20 @@ github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpR
github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY=
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ=
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU=
-github.com/tikv/client-go/v2 v2.0.3-0.20221108030801-9c0835c80eba h1:nFVdyTXcQYZwQQCdSJcFI1vBFyzG1hVuZ39MAK6wqK4=
-github.com/tikv/client-go/v2 v2.0.3-0.20221108030801-9c0835c80eba/go.mod h1:X9s4ct/MLk1sFqe5mU79KClKegLFDTa/FCx3hzexGtk=
-github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc=
-github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A=
-github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 h1:kl4KhGNsJIbDHS9/4U9yQo1UcPQM0kOMJHn29EoH/Ro=
-github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
+github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
+github.com/tikv/client-go/v2 v2.0.6-0.20230209044022-95ebf6a86301 h1:kiPiEBubzB2Ef2yPfoItQpjhGepcZSlRyqrXnfs92pQ=
+github.com/tikv/client-go/v2 v2.0.6-0.20230209044022-95ebf6a86301/go.mod h1:KMEFxpYwZ3MIvW4lrSy9dXgtrLojw9c9//2XlS5TGzw=
+github.com/tikv/pd/client v0.0.0-20230209034200-6d23a31c24be h1:hauBQBHSyrUxAI0zvkTiBKd472c+Iy+aY0Jd+b9VOJ8=
+github.com/tikv/pd/client v0.0.0-20230209034200-6d23a31c24be/go.mod h1:ryhYHDwupsZHeOOF/N7So+1hbtAnuw0K2A+pKOElSVs=
+github.com/timakin/bodyclose v0.0.0-20221125081123-e39cf3fc478e h1:MV6KaVu/hzByHP0UvJ4HcMGE/8a6A4Rggc/0wx2AvJo=
+github.com/timakin/bodyclose v0.0.0-20221125081123-e39cf3fc478e/go.mod h1:27bSVNWSBOHm+qRp1T9qzaIpsWEP6TbUnei/43HK+PQ=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
-github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
-github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
+github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
+github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
-github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
-github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
+github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
+github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -983,6 +987,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -1048,17 +1053,18 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+
go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
-go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
-go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
-go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
-go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
+go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -1070,6 +1076,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -1077,8 +1084,10 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
+golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
+golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1093,10 +1102,10 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
-golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
-golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
-golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d h1:+W8Qf4iJtMGKkyAygcKohjxTk4JPsL9DpzApJ22m5Ic=
-golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
+golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e h1:SkwG94eNiiYJhbeDE018Grw09HIN/KB9NlRmZsrzfWs=
+golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
+golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a h1:Jw5wfR+h9mnIYH+OtGT2im5wV1YGGDora5vTv/aa5bE=
+golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -1110,7 +1119,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
@@ -1121,13 +1129,14 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
+golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1149,7 +1158,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -1167,46 +1175,29 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210427231257-85d9c07bbe3a/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
+golang.org/x/net v0.0.0-20220517181318-183a9ca12b87/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=
+golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1218,8 +1209,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1268,72 +1260,58 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydthsj6oOd6l9bpuFcNoUvTs=
-golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1343,11 +1321,9 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190321232350-e250d351ecad/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
@@ -1388,35 +1364,30 @@ golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU=
golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
-golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
+golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4=
+golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U=
-golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
@@ -1439,29 +1410,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
-google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
-google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
-google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
-google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo=
-google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4=
-google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
-google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
-google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
-google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
-google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
-google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
-google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM=
-google.golang.org/api v0.66.0/go.mod h1:I1dmXYpX7HGwz/ejRxwQp2qj5bFAz93HiCU1C1oYd9M=
-google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
-google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7C80=
-google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
-google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
-google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE=
-google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
+google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ=
+google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1479,7 +1429,6 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
@@ -1503,50 +1452,9 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
-google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
-google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
-google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w=
-google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220111164026-67b88f271998/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220201184016-50beb8ab5c44/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220211171837-173942840c17/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb h1:0m9wktIpOxGw+SSKmydXWB3Z3GTfcPP6+q75HCQa6HI=
-google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
+google.golang.org/genproto v0.0.0-20230202175211-008b39050e57 h1:vArvWooPH749rNHpBGgVl+U9B9dATjiEhJzcWGlovNs=
+google.golang.org/genproto v0.0.0-20230202175211-008b39050e57/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -1563,25 +1471,13 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
-google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
-google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
+google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ=
+google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1594,7 +1490,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -1618,8 +1514,9 @@ gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mN
gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
@@ -1635,6 +1532,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -1644,8 +1542,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.3.3 h1:oDx7VAwstgpYpb3wv0oxiZlxY+foCpRAwY7Vk6XpAgA=
-honnef.co/go/tools v0.3.3/go.mod h1:jzwdWgg7Jdq75wlfblQxO4neNaFFSvgc1tD5Wv8U0Yw=
+honnef.co/go/tools v0.4.0 h1:lyXVV1c8wUBJRKqI8JgIpT8TW1VDagfYYaxbKa/HoL8=
+honnef.co/go/tools v0.4.0/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA=
k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
@@ -1657,8 +1555,9 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM=
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 h1:e1sMhtVq9AfcEy8AXNb8eSg6gbzfdpYhoNqnPJa+GzI=
diff --git a/infoschema/BUILD.bazel b/infoschema/BUILD.bazel
index 53bf35a2d43e3..20a4f150e4a27 100644
--- a/infoschema/BUILD.bazel
+++ b/infoschema/BUILD.bazel
@@ -17,6 +17,7 @@ go_library(
deps = [
"//config",
"//ddl/placement",
+ "//ddl/resourcegroup",
"//domain/infosync",
"//errno",
"//kv",
@@ -50,8 +51,13 @@ go_library(
"@com_github_ngaut_pools//:pools",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_pingcap_kvproto//pkg/diagnosticspb",
"@com_github_pingcap_kvproto//pkg/metapb",
+ "@com_github_pingcap_log//:log",
"@com_github_tikv_client_go_v2//tikv",
+ "@org_golang_google_grpc//:grpc",
+ "@org_golang_google_grpc//credentials",
+ "@org_golang_google_grpc//credentials/insecure",
"@org_golang_x_exp//slices",
"@org_uber_go_zap//:zap",
],
@@ -93,8 +99,10 @@ go_test(
"//store/helper",
"//store/mockstore",
"//store/mockstore/mockstorage",
+ "//store/mockstore/unistore",
"//table",
"//testkit",
+ "//testkit/external",
"//testkit/testsetup",
"//testkit/testutil",
"//types",
@@ -104,14 +112,17 @@ go_test(
"//util/pdapi",
"//util/resourcegrouptag",
"//util/set",
+ "//util/stmtsummary",
"@com_github_gorilla_mux//:mux",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_fn//:fn",
"@com_github_pingcap_kvproto//pkg/deadlock",
+ "@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_pingcap_tipb//go-tipb",
"@com_github_prometheus_prometheus//promql",
"@com_github_stretchr_testify//require",
+ "@com_github_tikv_client_go_v2//testutils",
"@org_golang_google_grpc//:grpc",
"@org_uber_go_goleak//:goleak",
],
diff --git a/infoschema/builder.go b/infoschema/builder.go
index 8ff50a8b50435..3240e1c5c5c49 100644
--- a/infoschema/builder.go
+++ b/infoschema/builder.go
@@ -211,14 +211,22 @@ func (b *Builder) ApplyDiff(m *meta.Meta, diff *model.SchemaDiff) ([]int64, erro
return b.applyDropPolicy(diff.SchemaID), nil
case model.ActionAlterPlacementPolicy:
return b.applyAlterPolicy(m, diff)
+ case model.ActionCreateResourceGroup:
+ return nil, b.applyCreateOrAlterResourceGroup(m, diff)
+ case model.ActionAlterResourceGroup:
+ return nil, b.applyCreateOrAlterResourceGroup(m, diff)
+ case model.ActionDropResourceGroup:
+ return b.applyDropResourceGroup(m, diff), nil
case model.ActionTruncateTablePartition, model.ActionTruncateTable:
return b.applyTruncateTableOrPartition(m, diff)
case model.ActionDropTable, model.ActionDropTablePartition:
- return b.applyDropTableOrParition(m, diff)
+ return b.applyDropTableOrPartition(m, diff)
case model.ActionRecoverTable:
return b.applyRecoverTable(m, diff)
case model.ActionCreateTables:
return b.applyCreateTables(m, diff)
+ case model.ActionReorganizePartition:
+ return b.applyReorganizePartition(m, diff)
case model.ActionFlashbackCluster:
return []int64{-1}, nil
default:
@@ -269,7 +277,7 @@ func (b *Builder) applyTruncateTableOrPartition(m *meta.Meta, diff *model.Schema
return tblIDs, nil
}
-func (b *Builder) applyDropTableOrParition(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) {
+func (b *Builder) applyDropTableOrPartition(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) {
tblIDs, err := b.applyTableUpdate(m, diff)
if err != nil {
return nil, errors.Trace(err)
@@ -281,6 +289,22 @@ func (b *Builder) applyDropTableOrParition(m *meta.Meta, diff *model.SchemaDiff)
return tblIDs, nil
}
+func (b *Builder) applyReorganizePartition(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) {
+ tblIDs, err := b.applyTableUpdate(m, diff)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ for _, opt := range diff.AffectedOpts {
+ if opt.OldTableID != 0 {
+ b.deleteBundle(b.is, opt.OldTableID)
+ }
+ if opt.TableID != 0 {
+ b.markTableBundleShouldUpdate(opt.TableID)
+ }
+ }
+ return tblIDs, nil
+}
+
func (b *Builder) applyRecoverTable(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) {
tblIDs, err := b.applyTableUpdate(m, diff)
if err != nil {
@@ -501,13 +525,36 @@ func (b *Builder) copySortedTables(oldTableID, newTableID int64) {
}
}
+func (b *Builder) applyCreateOrAlterResourceGroup(m *meta.Meta, diff *model.SchemaDiff) error {
+ group, err := m.GetResourceGroup(diff.SchemaID)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if group == nil {
+ return ErrResourceGroupNotExists.GenWithStackByArgs(fmt.Sprintf("(Group ID %d)", diff.SchemaID))
+ }
+ // TODO: need mark updated?
+ b.is.setResourceGroup(group)
+ return nil
+}
+
+func (b *Builder) applyDropResourceGroup(m *meta.Meta, diff *model.SchemaDiff) []int64 {
+ group, ok := b.is.ResourceGroupByID(diff.SchemaID)
+ if !ok {
+ return nil
+ }
+ b.is.deleteResourceGroup(group.Name.L)
+ // TODO: return the related information.
+ return []int64{}
+}
+
func (b *Builder) applyCreatePolicy(m *meta.Meta, diff *model.SchemaDiff) error {
po, err := m.GetPolicy(diff.SchemaID)
if err != nil {
return errors.Trace(err)
}
if po == nil {
- return ErrPlacementPolicyExists.GenWithStackByArgs(
+ return ErrPlacementPolicyNotExists.GenWithStackByArgs(
fmt.Sprintf("(Policy ID %d)", diff.SchemaID),
)
}
@@ -529,7 +576,7 @@ func (b *Builder) applyAlterPolicy(m *meta.Meta, diff *model.SchemaDiff) ([]int6
}
if po == nil {
- return nil, ErrPlacementPolicyExists.GenWithStackByArgs(
+ return nil, ErrPlacementPolicyNotExists.GenWithStackByArgs(
fmt.Sprintf("(Policy ID %d)", diff.SchemaID),
)
}
@@ -667,6 +714,8 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i
switch tp {
case model.ActionDropTablePartition:
case model.ActionTruncateTablePartition:
+ // ReorganizePartition handle the bundles in applyReorganizePartition
+ case model.ActionReorganizePartition:
default:
pi := tblInfo.GetPartitionInfo()
if pi != nil {
@@ -693,18 +742,23 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i
ConvertCharsetCollateToLowerCaseIfNeed(tblInfo)
ConvertOldVersionUTF8ToUTF8MB4IfNeed(tblInfo)
- if len(allocs) == 0 {
+ if len(allocs.Allocs) == 0 {
allocs = autoid.NewAllocatorsFromTblInfo(b.store, dbInfo.ID, tblInfo)
} else {
tblVer := autoid.AllocOptionTableInfoVersion(tblInfo.Version)
switch tp {
case model.ActionRebaseAutoID, model.ActionModifyTableAutoIdCache:
idCacheOpt := autoid.CustomAutoIncCacheOption(tblInfo.AutoIdCache)
- newAlloc := autoid.NewAllocator(b.store, dbInfo.ID, tblInfo.ID, tblInfo.IsAutoIncColUnsigned(), autoid.RowIDAllocType, tblVer, idCacheOpt)
- allocs = append(allocs, newAlloc)
+ // If the allocator type might be AutoIncrementType, create both AutoIncrementType
+ // and RowIDAllocType allocator for it. Because auto id and row id could share the same allocator.
+ // Allocate auto id may route to allocate row id, if row id allocator is nil, the program panic!
+ for _, tp := range [2]autoid.AllocatorType{autoid.AutoIncrementType, autoid.RowIDAllocType} {
+ newAlloc := autoid.NewAllocator(b.store, dbInfo.ID, tblInfo.ID, tblInfo.IsAutoIncColUnsigned(), tp, tblVer, idCacheOpt)
+ allocs = allocs.Append(newAlloc)
+ }
case model.ActionRebaseAutoRandomBase:
newAlloc := autoid.NewAllocator(b.store, dbInfo.ID, tblInfo.ID, tblInfo.IsAutoRandomBitColUnsigned(), autoid.AutoRandomType, tblVer)
- allocs = append(allocs, newAlloc)
+ allocs = allocs.Append(newAlloc)
case model.ActionModifyColumn:
// Change column attribute from auto_increment to auto_random.
if tblInfo.ContainsAutoRandomBits() && allocs.Get(autoid.AutoRandomType) == nil {
@@ -713,7 +767,7 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i
return a.GetType() != autoid.AutoIncrementType && a.GetType() != autoid.RowIDAllocType
})
newAlloc := autoid.NewAllocator(b.store, dbInfo.ID, tblInfo.ID, tblInfo.IsAutoRandomBitColUnsigned(), autoid.AutoRandomType, tblVer)
- allocs = append(allocs, newAlloc)
+ allocs = allocs.Append(newAlloc)
}
}
}
@@ -824,6 +878,7 @@ func (b *Builder) InitWithOldInfoSchema(oldSchema InfoSchema) *Builder {
b.copySchemasMap(oldIS)
b.copyBundlesMap(oldIS)
b.copyPoliciesMap(oldIS)
+ b.copyResourceGroupMap(oldIS)
b.copyTemporaryTableIDsMap(oldIS)
b.copyReferredForeignKeyMap(oldIS)
@@ -851,6 +906,13 @@ func (b *Builder) copyPoliciesMap(oldIS *infoSchema) {
}
}
+func (b *Builder) copyResourceGroupMap(oldIS *infoSchema) {
+ is := b.is
+ for _, v := range oldIS.AllResourceGroups() {
+ is.resourceGroupMap[v.Name.L] = v
+ }
+}
+
func (b *Builder) copyTemporaryTableIDsMap(oldIS *infoSchema) {
is := b.is
if len(oldIS.temporaryTableIDs) == 0 {
@@ -892,7 +954,7 @@ func (b *Builder) getSchemaAndCopyIfNecessary(dbName string) *model.DBInfo {
}
// InitWithDBInfos initializes an empty new InfoSchema with a slice of DBInfo, all placement rules, and schema version.
-func (b *Builder) InitWithDBInfos(dbInfos []*model.DBInfo, policies []*model.PolicyInfo, schemaVersion int64) (*Builder, error) {
+func (b *Builder) InitWithDBInfos(dbInfos []*model.DBInfo, policies []*model.PolicyInfo, resourceGroups []*model.ResourceGroupInfo, schemaVersion int64) (*Builder, error) {
info := b.is
info.schemaMetaVersion = schemaVersion
// build the policies.
@@ -900,6 +962,11 @@ func (b *Builder) InitWithDBInfos(dbInfos []*model.DBInfo, policies []*model.Pol
info.setPolicy(policy)
}
+ // build the groups.
+ for _, group := range resourceGroups {
+ info.setResourceGroup(group)
+ }
+
// Maintain foreign key reference information.
for _, di := range dbInfos {
for _, t := range di.Tables {
@@ -1003,6 +1070,7 @@ func NewBuilder(store kv.Storage, factory func() (pools.Resource, error)) *Build
is: &infoSchema{
schemaMap: map[string]*schemaTables{},
policyMap: map[string]*model.PolicyInfo{},
+ resourceGroupMap: map[string]*model.ResourceGroupInfo{},
ruleBundleMap: map[int64]*placement.Bundle{},
sortedTablesBuckets: make([]sortedTables, bucketCount),
referredForeignKeyMap: make(map[SchemaAndTableName][]*model.ReferredFKInfo),
diff --git a/infoschema/cache.go b/infoschema/cache.go
index 22ea012a9be28..3ceab0bffb00d 100644
--- a/infoschema/cache.go
+++ b/infoschema/cache.go
@@ -15,6 +15,7 @@
package infoschema
import (
+ "fmt"
"sort"
"sync"
@@ -36,15 +37,27 @@ var (
// It only promised to cache the infoschema, if it is newer than all the cached.
type InfoCache struct {
mu sync.RWMutex
- // cache is sorted by SchemaVersion in descending order
- cache []InfoSchema
- // record SnapshotTS of the latest schema Insert.
- maxUpdatedSnapshotTS uint64
+ // cache is sorted by both SchemaVersion and timestamp in descending order, assume they have same order
+ cache []schemaAndTimestamp
+}
+
+type schemaAndTimestamp struct {
+ infoschema InfoSchema
+ timestamp int64
}
// NewCache creates a new InfoCache.
-func NewCache(capcity int) *InfoCache {
- return &InfoCache{cache: make([]InfoSchema, 0, capcity)}
+func NewCache(capacity int) *InfoCache {
+ return &InfoCache{
+ cache: make([]schemaAndTimestamp, 0, capacity),
+ }
+}
+
+// Reset resets the cache.
+func (h *InfoCache) Reset(capacity int) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ h.cache = make([]schemaAndTimestamp, 0, capacity)
}
// GetLatest gets the newest information schema.
@@ -54,18 +67,40 @@ func (h *InfoCache) GetLatest() InfoSchema {
getLatestCounter.Inc()
if len(h.cache) > 0 {
hitLatestCounter.Inc()
- return h.cache[0]
+ return h.cache[0].infoschema
}
return nil
}
+// GetSchemaByTimestamp returns the schema used at the specific timestamp
+func (h *InfoCache) GetSchemaByTimestamp(ts uint64) (InfoSchema, error) {
+ h.mu.RLock()
+ defer h.mu.RUnlock()
+ return h.getSchemaByTimestampNoLock(ts)
+}
+
+func (h *InfoCache) getSchemaByTimestampNoLock(ts uint64) (InfoSchema, error) {
+ i := sort.Search(len(h.cache), func(i int) bool {
+ return uint64(h.cache[i].timestamp) <= ts
+ })
+ if i < len(h.cache) {
+ return h.cache[i].infoschema, nil
+ }
+
+ return nil, fmt.Errorf("no schema cached for timestamp %d", ts)
+}
+
// GetByVersion gets the information schema based on schemaVersion. Returns nil if it is not loaded.
func (h *InfoCache) GetByVersion(version int64) InfoSchema {
h.mu.RLock()
defer h.mu.RUnlock()
+ return h.getByVersionNoLock(version)
+}
+
+func (h *InfoCache) getByVersionNoLock(version int64) InfoSchema {
getVersionCounter.Inc()
i := sort.Search(len(h.cache), func(i int) bool {
- return h.cache[i].SchemaMetaVersion() <= version
+ return h.cache[i].infoschema.SchemaMetaVersion() <= version
})
// `GetByVersion` is allowed to load the latest schema that is less than argument `version`.
@@ -86,9 +121,9 @@ func (h *InfoCache) GetByVersion(version int64) InfoSchema {
// }
// ```
- if i < len(h.cache) && (i != 0 || h.cache[i].SchemaMetaVersion() == version) {
+ if i < len(h.cache) && (i != 0 || h.cache[i].infoschema.SchemaMetaVersion() == version) {
hitVersionCounter.Inc()
- return h.cache[i]
+ return h.cache[i].infoschema
}
return nil
}
@@ -101,11 +136,9 @@ func (h *InfoCache) GetBySnapshotTS(snapshotTS uint64) InfoSchema {
defer h.mu.RUnlock()
getTSCounter.Inc()
- if snapshotTS >= h.maxUpdatedSnapshotTS {
- if len(h.cache) > 0 {
- hitTSCounter.Inc()
- return h.cache[0]
- }
+ if schema, err := h.getSchemaByTimestampNoLock(snapshotTS); err == nil {
+ hitTSCounter.Inc()
+ return schema
}
return nil
}
@@ -118,16 +151,17 @@ func (h *InfoCache) Insert(is InfoSchema, snapshotTS uint64) bool {
defer h.mu.Unlock()
version := is.SchemaMetaVersion()
+
+ // assume this is the timestamp order as well
i := sort.Search(len(h.cache), func(i int) bool {
- return h.cache[i].SchemaMetaVersion() <= version
+ return h.cache[i].infoschema.SchemaMetaVersion() <= version
})
- if h.maxUpdatedSnapshotTS < snapshotTS {
- h.maxUpdatedSnapshotTS = snapshotTS
- }
-
// cached entry
- if i < len(h.cache) && h.cache[i].SchemaMetaVersion() == version {
+ if i < len(h.cache) && h.cache[i].infoschema.SchemaMetaVersion() == version {
+ if h.cache[i].timestamp > int64(snapshotTS) {
+ h.cache[i].timestamp = int64(snapshotTS)
+ }
return true
}
@@ -135,12 +169,18 @@ func (h *InfoCache) Insert(is InfoSchema, snapshotTS uint64) bool {
// has free space, grown the slice
h.cache = h.cache[:len(h.cache)+1]
copy(h.cache[i+1:], h.cache[i:])
- h.cache[i] = is
+ h.cache[i] = schemaAndTimestamp{
+ infoschema: is,
+ timestamp: int64(snapshotTS),
+ }
return true
} else if i < len(h.cache) {
// drop older schema
copy(h.cache[i+1:], h.cache[i:])
- h.cache[i] = is
+ h.cache[i] = schemaAndTimestamp{
+ infoschema: is,
+ timestamp: int64(snapshotTS),
+ }
return true
}
// older than all cached schemas, refuse to cache it
diff --git a/infoschema/cache_test.go b/infoschema/cache_test.go
index 83506bc4794d8..5d6e0e7f4e1b1 100644
--- a/infoschema/cache_test.go
+++ b/infoschema/cache_test.go
@@ -42,7 +42,7 @@ func TestInsert(t *testing.T) {
ic.Insert(is5, 5)
require.Equal(t, is5, ic.GetByVersion(5))
require.Equal(t, is2, ic.GetByVersion(2))
- require.Nil(t, ic.GetBySnapshotTS(2))
+ require.Equal(t, is2, ic.GetBySnapshotTS(2))
require.Equal(t, is5, ic.GetBySnapshotTS(10))
// older
@@ -59,7 +59,7 @@ func TestInsert(t *testing.T) {
require.Equal(t, is5, ic.GetByVersion(5))
require.Equal(t, is2, ic.GetByVersion(2))
require.Nil(t, ic.GetByVersion(0))
- require.Nil(t, ic.GetBySnapshotTS(2))
+ require.Equal(t, is2, ic.GetBySnapshotTS(2))
require.Equal(t, is6, ic.GetBySnapshotTS(10))
// replace 2, drop 2
@@ -91,7 +91,7 @@ func TestInsert(t *testing.T) {
require.Nil(t, ic.GetByVersion(2))
require.Nil(t, ic.GetByVersion(0))
require.Nil(t, ic.GetBySnapshotTS(2))
- require.Nil(t, ic.GetBySnapshotTS(5))
+ require.Equal(t, is5, ic.GetBySnapshotTS(5))
require.Equal(t, is6, ic.GetBySnapshotTS(10))
}
@@ -129,3 +129,61 @@ func TestGetLatest(t *testing.T) {
ic.Insert(is0, 0)
require.Equal(t, is2, ic.GetLatest())
}
+
+func TestGetByTimestamp(t *testing.T) {
+ ic := infoschema.NewCache(16)
+ require.NotNil(t, ic)
+ require.Nil(t, ic.GetLatest())
+
+ is1 := infoschema.MockInfoSchemaWithSchemaVer(nil, 1)
+ ic.Insert(is1, 1)
+ require.Equal(t, is1, ic.GetLatest())
+ _, err := ic.GetSchemaByTimestamp(0)
+ require.NotNil(t, err)
+ schema, err := ic.GetSchemaByTimestamp(1)
+ require.Nil(t, err)
+ require.Equal(t, int64(1), schema.SchemaMetaVersion())
+ require.Equal(t, is1, ic.GetBySnapshotTS(1))
+ schema, err = ic.GetSchemaByTimestamp(2)
+ require.Nil(t, err)
+ require.Equal(t, int64(1), schema.SchemaMetaVersion())
+ require.Equal(t, is1, ic.GetBySnapshotTS(2))
+
+ is2 := infoschema.MockInfoSchemaWithSchemaVer(nil, 2)
+ ic.Insert(is2, 2)
+ require.Equal(t, is2, ic.GetLatest())
+ _, err = ic.GetSchemaByTimestamp(0)
+ require.NotNil(t, err)
+ schema, err = ic.GetSchemaByTimestamp(1)
+ require.Nil(t, err)
+ require.Equal(t, int64(1), schema.SchemaMetaVersion())
+ require.Equal(t, is1, ic.GetBySnapshotTS(1))
+ schema, err = ic.GetSchemaByTimestamp(2)
+ require.Nil(t, err)
+ require.Equal(t, int64(2), schema.SchemaMetaVersion())
+ require.Equal(t, is2, ic.GetBySnapshotTS(2))
+ schema, err = ic.GetSchemaByTimestamp(3)
+ require.Nil(t, err)
+ require.Equal(t, int64(2), schema.SchemaMetaVersion())
+ require.Equal(t, is2, ic.GetBySnapshotTS(3))
+
+ is0 := infoschema.MockInfoSchemaWithSchemaVer(nil, 0)
+ ic.Insert(is0, 0)
+ require.Equal(t, is2, ic.GetLatest())
+ schema, err = ic.GetSchemaByTimestamp(0)
+ require.Nil(t, err)
+ require.Equal(t, int64(0), schema.SchemaMetaVersion())
+ require.Equal(t, is0, ic.GetBySnapshotTS(0))
+ schema, err = ic.GetSchemaByTimestamp(1)
+ require.Nil(t, err)
+ require.Equal(t, int64(1), schema.SchemaMetaVersion())
+ require.Equal(t, is1, ic.GetBySnapshotTS(1))
+ schema, err = ic.GetSchemaByTimestamp(2)
+ require.Nil(t, err)
+ require.Equal(t, int64(2), schema.SchemaMetaVersion())
+ require.Equal(t, is2, ic.GetBySnapshotTS(2))
+ schema, err = ic.GetSchemaByTimestamp(3)
+ require.Nil(t, err)
+ require.Equal(t, int64(2), schema.SchemaMetaVersion())
+ require.Equal(t, is2, ic.GetBySnapshotTS(3))
+}
diff --git a/infoschema/cluster_tables_test.go b/infoschema/cluster_tables_test.go
index 5e323b2adc88a..5ae331155c5dd 100644
--- a/infoschema/cluster_tables_test.go
+++ b/infoschema/cluster_tables_test.go
@@ -31,6 +31,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/fn"
"github.com/pingcap/kvproto/pkg/deadlock"
+ "github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/executor"
@@ -40,16 +41,20 @@ import (
"github.com/pingcap/tidb/parser/auth"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/server"
- "github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/store/helper"
+ "github.com/pingcap/tidb/store/mockstore"
"github.com/pingcap/tidb/store/mockstore/mockstorage"
+ "github.com/pingcap/tidb/store/mockstore/unistore"
"github.com/pingcap/tidb/testkit"
+ "github.com/pingcap/tidb/testkit/external"
"github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/pdapi"
"github.com/pingcap/tidb/util/resourcegrouptag"
"github.com/pingcap/tidb/util/set"
+ "github.com/pingcap/tidb/util/stmtsummary"
"github.com/pingcap/tipb/go-tipb"
"github.com/stretchr/testify/require"
+ "github.com/tikv/client-go/v2/testutils"
"google.golang.org/grpc"
)
@@ -221,9 +226,12 @@ func TestSelectClusterTable(t *testing.T) {
defer s.httpServer.Close()
defer s.rpcserver.Stop()
tk := s.newTestKitWithRoot(t)
- slowLogFileName := "tidb-slow.log"
+ slowLogFileName := "tidb-slow0.log"
prepareSlowLogfile(t, slowLogFileName)
defer func() { require.NoError(t, os.Remove(slowLogFileName)) }()
+ config.UpdateGlobal(func(conf *config.Config) {
+ conf.Log.SlowQueryFile = slowLogFileName
+ })
tk.MustExec("use information_schema")
tk.MustExec("set @@global.tidb_enable_stmt_summary=1")
@@ -811,44 +819,485 @@ func (s *clusterTablesSuite) newTestKitWithRoot(t *testing.T) *testkit.TestKit {
}
func TestMDLView(t *testing.T) {
- if !variable.EnableConcurrentDDL.Load() {
- t.Skipf("test requires concurrent ddl")
+ testCases := []struct {
+ name string
+ createTable string
+ ddl string
+ queryInTxn []string
+ sqlDigest string
+ }{
+ {"add column", "create table t(a int)", "alter table test.t add column b int", []string{"select 1", "select * from t"}, "[\"begin\",\"select ?\",\"select * from `t`\"]"},
+ {"change column in 1 step", "create table t(a int)", "alter table test.t change column a b int", []string{"select 1", "select * from t"}, "[\"begin\",\"select ?\",\"select * from `t`\"]"},
}
+ for _, c := range testCases {
+ t.Run(c.name, func(t *testing.T) {
+ // setup suite
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
- // setup suite
+ tk := s.newTestKitWithRoot(t)
+ tkDDL := s.newTestKitWithRoot(t)
+ tk3 := s.newTestKitWithRoot(t)
+ tk.MustExec("use test")
+ tk.MustExec("set global tidb_enable_metadata_lock=1")
+ tk.MustExec(c.createTable)
+
+ tk.MustExec("begin")
+ for _, q := range c.queryInTxn {
+ tk.MustQuery(q)
+ }
+
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ tkDDL.MustExec(c.ddl)
+ wg.Done()
+ }()
+
+ time.Sleep(200 * time.Millisecond)
+
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", tk3.Session().GetSessionManager())
+ defer s.rpcserver.Stop()
+
+ tk3.MustQuery("select DB_NAME, QUERY, SQL_DIGESTS from mysql.tidb_mdl_view").Check(testkit.Rows(
+ strings.Join([]string{
+ "test",
+ c.ddl,
+ c.sqlDigest,
+ }, " "),
+ ))
+
+ tk.MustExec("commit")
+
+ wg.Wait()
+ })
+ }
+}
+
+func TestCreateBindingFromHistory(t *testing.T) {
s := new(clusterTablesSuite)
s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
s.startTime = time.Now()
defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t1(id int primary key, a int, b int, key(a))")
+ tk.MustExec("create table t2(id int primary key, a int, b int, key(a))")
+ var testCases = []struct {
+ sqls []string
+ hint string
+ }{
+ {
+ sqls: []string{
+ "select %s * from t1, t2 where t1.id = t2.id",
+ "select %s * from test.t1, t2 where t1.id = t2.id",
+ "select %s * from test.t1, test.t2 where t1.id = t2.id",
+ "select %s * from t1, test.t2 where t1.id = t2.id",
+ },
+ hint: "/*+ merge_join(t1, t2) */",
+ },
+ {
+ sqls: []string{
+ "select %s * from t1 where a = 1",
+ "select %s * from test.t1 where a = 1",
+ },
+ hint: "/*+ ignore_index(t, a) */",
+ },
+ }
+
+ for _, testCase := range testCases {
+ for _, bind := range testCase.sqls {
+ stmtsummary.StmtSummaryByDigestMap.Clear()
+ bindSQL := fmt.Sprintf(bind, testCase.hint)
+ tk.MustExec(bindSQL)
+ planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", bindSQL)).Rows()
+ tk.MustExec(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]))
+ showRes := tk.MustQuery("show bindings").Rows()
+ require.Equal(t, len(showRes), 1)
+ require.Equal(t, planDigest[0][0], showRes[0][10])
+ for _, sql := range testCase.sqls {
+ tk.MustExec(fmt.Sprintf(sql, ""))
+ tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1"))
+ }
+ }
+ showRes := tk.MustQuery("show bindings").Rows()
+ require.Equal(t, len(showRes), 1)
+ tk.MustExec(fmt.Sprintf("drop binding for sql digest '%s'", showRes[0][9]))
+ }
+
+ // exception cases
+ tk.MustGetErrMsg(fmt.Sprintf("create binding from history using plan digest '%s'", "1"), "can't find any plans for '1'")
+ tk.MustGetErrMsg(fmt.Sprintf("create binding from history using plan digest '%s'", ""), "plan digest is empty")
+ tk.MustExec("create binding for select * from t1, t2 where t1.id = t2.id using select /*+ merge_join(t1, t2) */ * from t1, t2 where t1.id = t2.id")
+ showRes := tk.MustQuery("show bindings").Rows()
+ require.Equal(t, showRes[0][10], "") // plan digest should be nil by create for
+}
+
+func TestCreateBindingForPrepareFromHistory(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
tk := s.newTestKitWithRoot(t)
- tkDDL := s.newTestKitWithRoot(t)
- tk3 := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
+
tk.MustExec("use test")
- tk.MustExec("set global tidb_enable_metadata_lock=1")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(id int primary key, a int, key(a))")
+
+ tk.MustExec("prepare stmt from 'select /*+ ignore_index(t,a) */ * from t where a = ?'")
+ tk.MustExec("set @a = 1")
+ tk.MustExec("execute stmt using @a")
+ planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", "select /*+ ignore_index(t,a) */ * from t where a = ? [arguments: 1]")).Rows()
+ showRes := tk.MustQuery("show bindings").Rows()
+ require.Equal(t, len(showRes), 0)
+ tk.MustExec(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0]))
+ showRes = tk.MustQuery("show bindings").Rows()
+ require.Equal(t, len(showRes), 1)
+ require.Equal(t, planDigest[0][0], showRes[0][10])
+ tk.MustExec("execute stmt using @a")
+ tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1"))
+}
+
+func TestErrorCasesCreateBindingFromHistory(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t1, t2, t3")
+ tk.MustExec("create table t1(id int)")
+ tk.MustExec("create table t2(id int)")
+ tk.MustExec("create table t3(id int)")
+
+ sql := "select * from t1 where t1.id in (select id from t2)"
+ tk.MustExec(sql)
+ planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustGetErrMsg(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0]), "can't create binding for query with sub query")
+
+ sql = "select * from t1, t2, t3 where t1.id = t2.id and t2.id = t3.id"
+ tk.MustExec(sql)
+ planDigest = tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustGetErrMsg(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0]), "can't create binding for query with more than two table join")
+}
+
+// withMockTiFlash sets the mockStore to have N TiFlash stores (naming as tiflash0, tiflash1, ...).
+func withMockTiFlash(nodes int) mockstore.MockTiKVStoreOption {
+ return mockstore.WithMultipleOptions(
+ mockstore.WithClusterInspector(func(c testutils.Cluster) {
+ mockCluster := c.(*unistore.Cluster)
+ _, _, region1 := mockstore.BootstrapWithSingleStore(c)
+ tiflashIdx := 0
+ for tiflashIdx < nodes {
+ store2 := c.AllocID()
+ peer2 := c.AllocID()
+ addr2 := fmt.Sprintf("tiflash%d", tiflashIdx)
+ mockCluster.AddStore(store2, addr2, &metapb.StoreLabel{Key: "engine", Value: "tiflash"})
+ mockCluster.AddPeer(region1, store2, peer2)
+ tiflashIdx++
+ }
+ }),
+ mockstore.WithStoreType(mockstore.EmbedUnistore),
+ )
+}
+
+func TestBindingFromHistoryWithTiFlashBindable(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.store = testkit.CreateMockStore(t, withMockTiFlash(2))
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
+
+ tk.MustExec("use test;")
+ tk.MustExec("drop table if exists t;")
tk.MustExec("create table t(a int);")
- tk.MustExec("insert into t values(1);")
+ tk.MustExec("alter table test.t set tiflash replica 1")
+ tb := external.GetTableByName(t, tk, "test", "t")
+ err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
+ require.NoError(t, err)
+ tk.MustExec("set @@session.tidb_isolation_read_engines = 'tiflash'")
- tk.MustExec("begin")
- tk.MustQuery("select 1;")
- tk.MustQuery("select * from t;")
+ sql := "select * from t"
+ tk.MustExec(sql)
+ planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustGetErrMsg(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0]), "can't create binding for query with tiflash engine")
+}
- var wg sync.WaitGroup
- wg.Add(1)
- go func() {
- tkDDL.MustExec("alter table test.t add column b int;")
- wg.Done()
- }()
+func TestSetBindingStatusBySQLDigest(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(id int, a int, key(a))")
+ sql := "select /*+ ignore_index(t, a) */ * from t where t.a = 1"
+ tk.MustExec(sql)
+ planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.cluster_statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustExec(fmt.Sprintf("create global binding from history using plan digest '%s'", planDigest[0][0]))
+ sql = "select * from t where t.a = 1"
+ tk.MustExec(sql)
+ tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1"))
+
+ sqlDigest := tk.MustQuery("show global bindings").Rows()
+ tk.MustExec(fmt.Sprintf("set binding disabled for sql digest '%s'", sqlDigest[0][9]))
+ tk.MustExec(sql)
+ tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("0"))
+ tk.MustExec(fmt.Sprintf("set binding enabled for sql digest '%s'", sqlDigest[0][9]))
+ tk.MustExec(sql)
+ tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1"))
+ tk.MustGetErrMsg("set binding enabled for sql digest '2'", "can't find any binding for '2'")
+ tk.MustGetErrMsg("set binding enabled for sql digest ''", "sql digest is empty")
+ tk.MustGetErrMsg("set binding disabled for sql digest ''", "sql digest is empty")
+}
- time.Sleep(200 * time.Millisecond)
+func TestCreateBindingWhenCloseStmtSummaryTable(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
- s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", tk3.Session().GetSessionManager())
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(id int primary key, a int, key(a))")
+ tk.MustExec("set global tidb_enable_stmt_summary = 0")
+ tk.MustExec("select /*+ ignore_index(t, a) */ * from t where a = 1")
+
+ tk.MustGetErrMsg("create binding from history using plan digest '4e3159169cc63c14b139a4e7d72eae1759875c9a9581f94bb2079aae961189cb'",
+ "can't find any plans for '4e3159169cc63c14b139a4e7d72eae1759875c9a9581f94bb2079aae961189cb'")
+}
+
+func TestCreateBindingForNotSupportedStmt(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
- tk3.MustQuery("select DB_NAME, QUERY, SQL_DIGESTS from mysql.tidb_mdl_view").Check(testkit.Rows("test alter table test.t add column b int; [\"begin\",\"select ? ;\",\"select * from `t` ;\"]"))
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(id int primary key, a int, key(a))")
+
+ sql := "admin show ddl jobs"
+ tk.MustExec(sql)
+ planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustGetErrMsg(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]), fmt.Sprintf("can't find any plans for '%s'", planDigest[0][0]))
+
+ sql = "show tables"
+ tk.MustExec(sql)
+ planDigest = tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustGetErrMsg(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]), fmt.Sprintf("can't find any plans for '%s'", planDigest[0][0]))
+
+ sql = "explain select /*+ ignore_index(t, a) */ * from t where a = 1"
+ tk.MustExec(sql)
+ planDigest = tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustGetErrMsg(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]), fmt.Sprintf("can't find any plans for '%s'", planDigest[0][0]))
+}
- tk.MustExec("commit")
+func TestCreateBindingRepeatedly(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
- wg.Wait()
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(id int primary key, a int, key(a))")
+
+ sql := "select /*+ ignore_index(t, a) */ * from t where a = 1"
+ tk.MustExec(sql)
+ planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustExec(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]))
+ binding := tk.MustQuery("show bindings").Rows()
+ loc, _ := time.LoadLocation("Asia/Shanghai")
+ createTime, _ := time.ParseInLocation("2006-01-02 15:04:05", binding[0][4].(string), loc)
+ updateTime, _ := time.ParseInLocation("2006-01-02 15:04:05", binding[0][5].(string), loc)
+
+ // binding from history cover binding from history
+ tk.MustExec(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]))
+ tk.MustExec(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]))
+ tk.MustExec(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]))
+ binding1 := tk.MustQuery("show bindings").Rows()
+ createTime1, _ := time.ParseInLocation("2006-01-02 15:04:05", binding1[0][4].(string), loc)
+ updateTime1, _ := time.ParseInLocation("2006-01-02 15:04:05", binding1[0][5].(string), loc)
+ require.Greater(t, createTime1.UnixNano(), createTime.UnixNano())
+ require.Greater(t, updateTime1.UnixNano(), updateTime.UnixNano())
+ for i := range binding1[0] {
+ if i != 4 && i != 5 {
+ require.Equal(t, binding[0][i], binding1[0][i])
+ }
+ }
+ // binding from sql cover binding from history
+ tk.MustExec("create binding for select * from t where a = 1 using select /*+ ignore_index(t, a) */ * from t where a = 1")
+ binding2 := tk.MustQuery("show bindings").Rows()
+ createTime2, _ := time.ParseInLocation("2006-01-02 15:04:05", binding2[0][4].(string), loc)
+ updateTime2, _ := time.ParseInLocation("2006-01-02 15:04:05", binding2[0][5].(string), loc)
+ require.Greater(t, createTime2.UnixNano(), createTime1.UnixNano())
+ require.Greater(t, updateTime2.UnixNano(), updateTime1.UnixNano())
+ require.Equal(t, binding2[0][8], "manual")
+ require.Equal(t, binding2[0][10], "")
+ for i := range binding2[0] {
+ if i != 1 && i != 4 && i != 5 && i != 8 && i != 10 {
+ // bind_sql, create_time, update_time, source, plan_digest may be different
+ require.Equal(t, binding1[0][i], binding2[0][i])
+ }
+ }
+ // binding from history cover binding from sql
+ tk.MustExec(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]))
+ binding3 := tk.MustQuery("show bindings").Rows()
+ createTime3, _ := time.ParseInLocation("2006-01-02 15:04:05", binding3[0][4].(string), loc)
+ updateTime3, _ := time.ParseInLocation("2006-01-02 15:04:05", binding3[0][5].(string), loc)
+ require.Greater(t, createTime3.UnixNano(), createTime2.UnixNano())
+ require.Greater(t, updateTime3.UnixNano(), updateTime2.UnixNano())
+ require.Equal(t, binding3[0][8], "history")
+ require.Equal(t, binding3[0][10], planDigest[0][0])
+ for i := range binding3[0] {
+ if i != 1 && i != 4 && i != 5 && i != 8 && i != 10 {
+ // bind_sql, create_time, update_time, source, plan_digest may be different
+ require.Equal(t, binding2[0][i], binding3[0][i])
+ }
+ }
+}
+
+func TestCreateBindingWithUsingKeyword(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t, t1, t2")
+ tk.MustExec("create table t(id int primary key, a int, key(a))")
+ tk.MustExec("create table t1(id int primary key, a int, key(a))")
+ tk.MustExec("create table t2(id int primary key, a int, key(a))")
+
+ // `JOIN` keyword and not specifying the associated columns with the `USING` keyword.
+ tk.MustGetErrMsg("CREATE GLOBAL BINDING for SELECT * FROM t t1 JOIN t t2 USING SELECT * FROM t t1 JOIN t t2;",
+ "[parser:1064]You have an error in your SQL syntax; check the manual that corresponds to your TiDB version for the right syntax to use line 1 column 67 near \"SELECT * FROM t t1 JOIN t t2;\" ")
+ sql := "SELECT * FROM t t1 JOIN t t2;"
+ tk.MustExec(sql)
+ planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustExec(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]))
+ tk.MustExec(sql)
+ tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1"))
+
+ // `DELETE` statements that contain the `USING` keyword.
+ tk.MustGetErrMsg("`CREATE GLOBAL BINDING for DELETE FROM t1 USING t1 JOIN t2 ON t1.a = t2.a USING DELETE FROM t1 USING t1 JOIN t2 ON t1.a = t2.a;",
+ "[parser:1064]You have an error in your SQL syntax; check the manual that corresponds to your TiDB version for the right syntax to use line 1 column 127 near \"`CREATE GLOBAL BINDING for DELETE FROM t1 USING t1 JOIN t2 ON t1.a = t2.a USING DELETE FROM t1 USING t1 JOIN t2 ON t1.a = t2.a;\" ")
+ sql = "DELETE FROM t1 USING t1 JOIN t2 ON t1.a = t2.a;"
+ tk.MustExec(sql)
+ planDigest = tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustExec(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]))
+ tk.MustExec(sql)
+ tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1"))
+}
+
+func TestNewCreatedBindingCanWorkWithPlanCache(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t, t1, t2")
+ tk.MustExec("create table t(id int primary key, a int, key(a))")
+
+ tk.MustExec("prepare stmt from 'select * from t where a = ?'")
+ tk.MustExec("set @a = 0")
+ tk.MustExec("execute stmt using @a")
+ tk.MustExec("execute stmt using @a")
+ tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1"))
+ sql := "select /*+ ignore_index(t, a) */ * from t where a = 1"
+ tk.MustExec(sql)
+ planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustExec(fmt.Sprintf("create session binding from history using plan digest '%s'", planDigest[0][0]))
+ tk.MustExec("execute stmt using @a")
+ tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1"))
+}
+
+func TestCreateBindingForPrepareToken(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, b time, c varchar(5))")
+
+ //some builtin functions listed in https://dev.mysql.com/doc/refman/8.0/en/function-resolution.html
+ cases := []string{
+ "select std(a) from t",
+ "select cast(a as decimal(10, 2)) from t",
+ "select bit_or(a) from t",
+ "select min(a) from t",
+ "select max(a) from t",
+ "select substr(c, 1, 2) from t",
+ }
+
+ for _, sql := range cases {
+ prep := fmt.Sprintf("prepare stmt from '%s'", sql)
+ tk.MustExec(prep)
+ tk.MustExec("execute stmt")
+ planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows()
+ tk.MustExec(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0]))
+ }
}
diff --git a/infoschema/error.go b/infoschema/error.go
index a7e4929a35bcc..2e1a45c7ae113 100644
--- a/infoschema/error.go
+++ b/infoschema/error.go
@@ -32,7 +32,11 @@ var (
ErrPlacementPolicyExists = dbterror.ClassSchema.NewStd(mysql.ErrPlacementPolicyExists)
// ErrPlacementPolicyNotExists return for placement_policy policy not exists.
ErrPlacementPolicyNotExists = dbterror.ClassSchema.NewStd(mysql.ErrPlacementPolicyNotExists)
- // ErrReservedSyntax for internal syntax.
+ // ErrResourceGroupExists return for resource group already exists.
+ ErrResourceGroupExists = dbterror.ClassSchema.NewStd(mysql.ErrResourceGroupExists)
+ // ErrResourceGroupNotExists return for resource group not exists.
+ ErrResourceGroupNotExists = dbterror.ClassSchema.NewStd(mysql.ErrResourceGroupNotExists)
+ // ErrReservedSyntax for internal syntax.
ErrReservedSyntax = dbterror.ClassSchema.NewStd(mysql.ErrReservedSyntax)
// ErrTableExists returns for table already exists.
ErrTableExists = dbterror.ClassSchema.NewStd(mysql.ErrTableExists)
@@ -64,6 +68,8 @@ var (
ErrKeyNotExists = dbterror.ClassSchema.NewStd(mysql.ErrKeyDoesNotExist)
// ErrCannotAddForeign returns for foreign key exists.
ErrCannotAddForeign = dbterror.ClassSchema.NewStd(mysql.ErrCannotAddForeign)
+ // ErrForeignKeyOnPartitioned returns for foreign key on partition table.
+ ErrForeignKeyOnPartitioned = dbterror.ClassSchema.NewStd(mysql.ErrForeignKeyOnPartitioned)
// ErrForeignKeyNotMatch returns for foreign key not match.
ErrForeignKeyNotMatch = dbterror.ClassSchema.NewStd(mysql.ErrWrongFkDef)
// ErrIndexExists returns for index already exists.
@@ -94,4 +100,6 @@ var (
ErrForeignKeyNoIndexInParent = dbterror.ClassSchema.NewStd(mysql.ErrForeignKeyNoIndexInParent)
// ErrForeignKeyColumnNotNull returns when foreign key with SET NULL constrain and the related column has not null.
ErrForeignKeyColumnNotNull = dbterror.ClassSchema.NewStd(mysql.ErrForeignKeyColumnNotNull)
+ // ErrResourceGroupSupportDisabled returns for resource group feature is disabled
+ ErrResourceGroupSupportDisabled = dbterror.ClassSchema.NewStd(mysql.ErrResourceGroupSupportDisabled)
)
diff --git a/infoschema/infoschema.go b/infoschema/infoschema.go
index d024afde40d7d..f46f7d082a66e 100644
--- a/infoschema/infoschema.go
+++ b/infoschema/infoschema.go
@@ -42,6 +42,7 @@ type InfoSchema interface {
SchemaByID(id int64) (*model.DBInfo, bool)
SchemaByTable(tableInfo *model.TableInfo) (*model.DBInfo, bool)
PolicyByName(name model.CIStr) (*model.PolicyInfo, bool)
+ ResourceGroupByName(name model.CIStr) (*model.ResourceGroupInfo, bool)
TableByID(id int64) (table.Table, bool)
AllocByID(id int64) (autoid.Allocators, bool)
AllSchemaNames() []string
@@ -60,6 +61,8 @@ type InfoSchema interface {
AllPlacementBundles() []*placement.Bundle
// AllPlacementPolicies returns all placement policies
AllPlacementPolicies() []*model.PolicyInfo
+ // AllResourceGroups returns all resource groups
+ AllResourceGroups() []*model.ResourceGroupInfo
// HasTemporaryTable returns whether information schema has temporary table
HasTemporaryTable() bool
// GetTableReferredForeignKeys gets the table's ReferredFKInfo by lowercase schema and table name.
@@ -93,6 +96,10 @@ type infoSchema struct {
policyMutex sync.RWMutex
policyMap map[string]*model.PolicyInfo
+ // resourceGroupMap stores all resource groups.
+ resourceGroupMutex sync.RWMutex
+ resourceGroupMap map[string]*model.ResourceGroupInfo
+
schemaMap map[string]*schemaTables
// sortedTablesBuckets is a slice of sortedTables, a table's bucket index is (tableID % bucketCount).
@@ -120,6 +127,7 @@ func MockInfoSchema(tbList []*model.TableInfo) InfoSchema {
result := &infoSchema{}
result.schemaMap = make(map[string]*schemaTables)
result.policyMap = make(map[string]*model.PolicyInfo)
+ result.resourceGroupMap = make(map[string]*model.ResourceGroupInfo)
result.ruleBundleMap = make(map[int64]*placement.Bundle)
result.sortedTablesBuckets = make([]sortedTables, bucketCount)
dbInfo := &model.DBInfo{ID: 0, Name: model.NewCIStr("test"), Tables: tbList}
@@ -147,6 +155,7 @@ func MockInfoSchemaWithSchemaVer(tbList []*model.TableInfo, schemaVer int64) Inf
result := &infoSchema{}
result.schemaMap = make(map[string]*schemaTables)
result.policyMap = make(map[string]*model.PolicyInfo)
+ result.resourceGroupMap = make(map[string]*model.ResourceGroupInfo)
result.ruleBundleMap = make(map[int64]*placement.Bundle)
result.sortedTablesBuckets = make([]sortedTables, bucketCount)
dbInfo := &model.DBInfo{ID: 0, Name: model.NewCIStr("test"), Tables: tbList}
@@ -235,6 +244,17 @@ func (is *infoSchema) PolicyByID(id int64) (val *model.PolicyInfo, ok bool) {
return nil, false
}
+func (is *infoSchema) ResourceGroupByID(id int64) (val *model.ResourceGroupInfo, ok bool) {
+ is.resourceGroupMutex.RLock()
+ defer is.resourceGroupMutex.RUnlock()
+ for _, v := range is.resourceGroupMap {
+ if v.ID == id {
+ return v, true
+ }
+ }
+ return nil, false
+}
+
func (is *infoSchema) SchemaByID(id int64) (val *model.DBInfo, ok bool) {
for _, v := range is.schemaMap {
if v.dbInfo.ID == id {
@@ -270,7 +290,7 @@ func (is *infoSchema) TableByID(id int64) (val table.Table, ok bool) {
func (is *infoSchema) AllocByID(id int64) (autoid.Allocators, bool) {
tbl, ok := is.TableByID(id)
if !ok {
- return nil, false
+ return autoid.Allocators{}, false
}
return tbl.Allocators(nil), true
}
@@ -393,6 +413,25 @@ func (is *infoSchema) PolicyByName(name model.CIStr) (*model.PolicyInfo, bool) {
return t, r
}
+// ResourceGroupByName is used to find the resource group.
+func (is *infoSchema) ResourceGroupByName(name model.CIStr) (*model.ResourceGroupInfo, bool) {
+ is.resourceGroupMutex.RLock()
+ defer is.resourceGroupMutex.RUnlock()
+ t, r := is.resourceGroupMap[name.L]
+ return t, r
+}
+
+// AllResourceGroups returns all resource groups.
+func (is *infoSchema) AllResourceGroups() []*model.ResourceGroupInfo {
+ is.resourceGroupMutex.RLock()
+ defer is.resourceGroupMutex.RUnlock()
+ groups := make([]*model.ResourceGroupInfo, 0, len(is.resourceGroupMap))
+ for _, group := range is.resourceGroupMap {
+ groups = append(groups, group)
+ }
+ return groups
+}
+
// AllPlacementPolicies returns all placement policies
func (is *infoSchema) AllPlacementPolicies() []*model.PolicyInfo {
is.policyMutex.RLock()
@@ -417,6 +456,18 @@ func (is *infoSchema) AllPlacementBundles() []*placement.Bundle {
return bundles
}
+func (is *infoSchema) setResourceGroup(resourceGroup *model.ResourceGroupInfo) {
+ is.resourceGroupMutex.Lock()
+ defer is.resourceGroupMutex.Unlock()
+ is.resourceGroupMap[resourceGroup.Name.L] = resourceGroup
+}
+
+func (is *infoSchema) deleteResourceGroup(name string) {
+ is.resourceGroupMutex.Lock()
+ defer is.resourceGroupMutex.Unlock()
+ delete(is.resourceGroupMap, name)
+}
+
func (is *infoSchema) setPolicy(policy *model.PolicyInfo) {
is.policyMutex.Lock()
defer is.policyMutex.Unlock()
@@ -674,6 +725,12 @@ func (ts *SessionExtendedInfoSchema) SchemaByTable(tableInfo *model.TableInfo) (
}
}
+ if ts.MdlTables != nil {
+ if tbl, ok := ts.MdlTables.SchemaByTable(tableInfo); ok {
+ return tbl, true
+ }
+ }
+
return ts.InfoSchema.SchemaByTable(tableInfo)
}
diff --git a/infoschema/infoschema_test.go b/infoschema/infoschema_test.go
index 66bd8cc33ef00..513dbfb6f7223 100644
--- a/infoschema/infoschema_test.go
+++ b/infoschema/infoschema_test.go
@@ -110,7 +110,7 @@ func TestBasic(t *testing.T) {
})
require.NoError(t, err)
- builder, err := infoschema.NewBuilder(dom.Store(), nil).InitWithDBInfos(dbInfos, nil, 1)
+ builder, err := infoschema.NewBuilder(dom.Store(), nil).InitWithDBInfos(dbInfos, nil, nil, 1)
require.NoError(t, err)
txn, err := store.Begin()
@@ -256,7 +256,7 @@ func TestInfoTables(t *testing.T) {
require.NoError(t, err)
}()
- builder, err := infoschema.NewBuilder(store, nil).InitWithDBInfos(nil, nil, 0)
+ builder, err := infoschema.NewBuilder(store, nil).InitWithDBInfos(nil, nil, nil, 0)
require.NoError(t, err)
is := builder.Build()
@@ -296,6 +296,7 @@ func TestInfoTables(t *testing.T) {
"DEADLOCKS",
"PLACEMENT_POLICIES",
"TRX_SUMMARY",
+ "RESOURCE_GROUPS",
}
for _, tbl := range infoTables {
tb, err1 := is.TableByName(util.InformationSchemaName, model.NewCIStr(tbl))
@@ -410,7 +411,7 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) {
// full load
newDB, ok := newIS.SchemaByName(model.NewCIStr("test"))
require.True(t, ok)
- builder, err := infoschema.NewBuilder(store, nil).InitWithDBInfos([]*model.DBInfo{newDB}, newIS.AllPlacementPolicies(), newIS.SchemaMetaVersion())
+ builder, err := infoschema.NewBuilder(store, nil).InitWithDBInfos([]*model.DBInfo{newDB}, newIS.AllPlacementPolicies(), newIS.AllResourceGroups(), newIS.SchemaMetaVersion())
require.NoError(t, err)
require.True(t, builder.Build().HasTemporaryTable())
@@ -535,7 +536,7 @@ func TestBuildBundle(t *testing.T) {
assertBundle(is, tbl2.Meta().ID, nil)
assertBundle(is, p1.ID, p1Bundle)
- builder, err := infoschema.NewBuilder(store, nil).InitWithDBInfos([]*model.DBInfo{db}, is.AllPlacementPolicies(), is.SchemaMetaVersion())
+ builder, err := infoschema.NewBuilder(store, nil).InitWithDBInfos([]*model.DBInfo{db}, is.AllPlacementPolicies(), is.AllResourceGroups(), is.SchemaMetaVersion())
require.NoError(t, err)
is2 := builder.Build()
assertBundle(is2, tbl1.Meta().ID, tb1Bundle)
diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go
index 9a5a36235cfa4..6889d61c3be85 100644
--- a/infoschema/perfschema/tables.go
+++ b/infoschema/perfschema/tables.go
@@ -207,6 +207,11 @@ func (vt *perfSchemaTable) Indices() []table.Index {
return vt.indices
}
+// GetPartitionedTable implements table.Table GetPartitionedTable interface.
+func (vt *perfSchemaTable) GetPartitionedTable() table.PartitionedTable {
+ return nil
+}
+
// initTableIndices initializes the indices of the perfSchemaTable.
func initTableIndices(t *perfSchemaTable) error {
tblInfo := t.meta
diff --git a/infoschema/tables.go b/infoschema/tables.go
index 656a99f52d838..3a2711eb57147 100644
--- a/infoschema/tables.go
+++ b/infoschema/tables.go
@@ -22,12 +22,17 @@ import (
"net/http"
"strconv"
"strings"
+ "sync"
+ "time"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
+ "github.com/pingcap/kvproto/pkg/diagnosticspb"
"github.com/pingcap/kvproto/pkg/metapb"
+ "github.com/pingcap/log"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/ddl/placement"
+ "github.com/pingcap/tidb/ddl/resourcegroup"
"github.com/pingcap/tidb/domain/infosync"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta/autoid"
@@ -47,9 +52,14 @@ import (
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/pdapi"
"github.com/pingcap/tidb/util/sem"
+ "github.com/pingcap/tidb/util/set"
"github.com/pingcap/tidb/util/stmtsummary"
"github.com/tikv/client-go/v2/tikv"
"go.uber.org/zap"
+ "golang.org/x/exp/slices"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
)
const (
@@ -190,6 +200,8 @@ const (
TableMemoryUsage = "MEMORY_USAGE"
// TableMemoryUsageOpsHistory is the memory control operators history.
TableMemoryUsageOpsHistory = "MEMORY_USAGE_OPS_HISTORY"
+ // TableResourceGroups is the metadata of resource groups.
+ TableResourceGroups = "RESOURCE_GROUPS"
)
const (
@@ -296,6 +308,7 @@ var tableIDMap = map[string]int64{
TableMemoryUsageOpsHistory: autoid.InformationSchemaDBID + 85,
ClusterTableMemoryUsage: autoid.InformationSchemaDBID + 86,
ClusterTableMemoryUsageOpsHistory: autoid.InformationSchemaDBID + 87,
+ TableResourceGroups: autoid.InformationSchemaDBID + 88,
}
// columnInfo represents the basic column information of all kinds of INFORMATION_SCHEMA tables
@@ -812,6 +825,7 @@ var tableProcesslistCols = []columnInfo{
{name: "MEM", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag},
{name: "DISK", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag},
{name: "TxnStart", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, deflt: ""},
+ {name: "RESOURCE_GROUP", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag, deflt: ""},
}
var tableTiDBIndexesCols = []columnInfo{
@@ -890,6 +904,7 @@ var slowQueryCols = []columnInfo{
{name: variable.SlowLogBackoffTotal, tp: mysql.TypeDouble, size: 22},
{name: variable.SlowLogWriteSQLRespTotal, tp: mysql.TypeDouble, size: 22},
{name: variable.SlowLogResultRows, tp: mysql.TypeLonglong, size: 22},
+ {name: variable.SlowLogWarnings, tp: mysql.TypeLongBlob, size: types.UnspecifiedLength},
{name: variable.SlowLogBackoffDetail, tp: mysql.TypeVarchar, size: 4096},
{name: variable.SlowLogPrepared, tp: mysql.TypeTiny, size: 1},
{name: variable.SlowLogSucc, tp: mysql.TypeTiny, size: 1},
@@ -1323,6 +1338,9 @@ var tableStatementsSummaryCols = []columnInfo{
{name: stmtsummary.PlanDigestStr, tp: mysql.TypeVarchar, size: 64, comment: "Digest of its execution plan"},
{name: stmtsummary.PlanStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "Sampled execution plan"},
{name: stmtsummary.BinaryPlan, tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "Sampled binary plan"},
+ {name: stmtsummary.Charset, tp: mysql.TypeVarchar, size: 64, comment: "Sampled charset"},
+ {name: stmtsummary.Collation, tp: mysql.TypeVarchar, size: 64, comment: "Sampled collation"},
+ {name: stmtsummary.PlanHint, tp: mysql.TypeVarchar, size: 64, comment: "Sampled plan hint"},
}
var tableStorageStatsCols = []columnInfo{
@@ -1574,6 +1592,12 @@ var tableMemoryUsageOpsHistoryCols = []columnInfo{
{name: "SQL_TEXT", tp: mysql.TypeVarchar, size: 256},
}
+var tableResourceGroupsCols = []columnInfo{
+ {name: "NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag},
+ {name: "RU_PER_SEC", tp: mysql.TypeLonglong, size: 21},
+ {name: "BURSTABLE", tp: mysql.TypeVarchar, size: 3},
+}
+
// GetShardingInfo returns a nil or description string for the sharding information of given TableInfo.
// The returned description string may be:
// - "NOT_SHARDED": for tables that SHARD_ROW_ID_BITS is not specified.
@@ -1729,8 +1753,8 @@ func FormatTiDBVersion(TiDBVersion string, isDefaultVersion bool) string {
// The user hasn't set the config 'ServerVersion'.
if isDefaultVersion {
- nodeVersion = TiDBVersion[strings.LastIndex(TiDBVersion, "TiDB-")+len("TiDB-"):]
- if nodeVersion[0] == 'v' {
+ nodeVersion = TiDBVersion[strings.Index(TiDBVersion, "TiDB-")+len("TiDB-"):]
+ if len(nodeVersion) > 0 && nodeVersion[0] == 'v' {
nodeVersion = nodeVersion[1:]
}
nodeVersions := strings.Split(nodeVersion, "-")
@@ -2036,6 +2060,7 @@ var tableNameToColumns = map[string][]columnInfo{
TableUserAttributes: tableUserAttributesCols,
TableMemoryUsage: tableMemoryUsageCols,
TableMemoryUsageOpsHistory: tableMemoryUsageOpsHistoryCols,
+ TableResourceGroups: tableResourceGroupsCols,
}
func createInfoSchemaTable(_ autoid.Allocators, meta *model.TableInfo) (table.Table, error) {
@@ -2123,7 +2148,7 @@ func (it *infoschemaTable) UpdateRecord(gctx context.Context, ctx sessionctx.Con
// Allocators implements table.Table Allocators interface.
func (it *infoschemaTable) Allocators(_ sessionctx.Context) autoid.Allocators {
- return nil
+ return autoid.Allocators{}
}
// Meta implements table.Table Meta interface.
@@ -2141,6 +2166,11 @@ func (it *infoschemaTable) Type() table.Type {
return it.tp
}
+// GetPartitionedTable implements table.Table GetPartitionedTable interface.
+func (it *infoschemaTable) GetPartitionedTable() table.PartitionedTable {
+ return nil
+}
+
// VirtualTable is a dummy table.Table implementation.
type VirtualTable struct{}
@@ -2206,7 +2236,7 @@ func (vt *VirtualTable) UpdateRecord(ctx context.Context, sctx sessionctx.Contex
// Allocators implements table.Table Allocators interface.
func (vt *VirtualTable) Allocators(_ sessionctx.Context) autoid.Allocators {
- return nil
+ return autoid.Allocators{}
}
// Meta implements table.Table Meta interface.
@@ -2223,3 +2253,142 @@ func (vt *VirtualTable) GetPhysicalID() int64 {
func (vt *VirtualTable) Type() table.Type {
return table.VirtualTable
}
+
+// GetTiFlashServerInfo returns all TiFlash server infos
+func GetTiFlashServerInfo(sctx sessionctx.Context) ([]ServerInfo, error) {
+ if config.GetGlobalConfig().DisaggregatedTiFlash {
+ return nil, table.ErrUnsupportedOp
+ }
+ serversInfo, err := GetStoreServerInfo(sctx)
+ if err != nil {
+ return nil, err
+ }
+ serversInfo = FilterClusterServerInfo(serversInfo, set.NewStringSet(kv.TiFlash.Name()), set.NewStringSet())
+ return serversInfo, nil
+}
+
+// FetchClusterServerInfoWithoutPrivilegeCheck fetches cluster server information
+func FetchClusterServerInfoWithoutPrivilegeCheck(ctx context.Context, sctx sessionctx.Context, serversInfo []ServerInfo, serverInfoType diagnosticspb.ServerInfoType, recordWarningInStmtCtx bool) ([][]types.Datum, error) {
+ type result struct {
+ idx int
+ rows [][]types.Datum
+ err error
+ }
+ wg := sync.WaitGroup{}
+ ch := make(chan result, len(serversInfo))
+ infoTp := serverInfoType
+ finalRows := make([][]types.Datum, 0, len(serversInfo)*10)
+ for i, srv := range serversInfo {
+ address := srv.Address
+ remote := address
+ if srv.ServerType == "tidb" {
+ remote = srv.StatusAddr
+ }
+ wg.Add(1)
+ go func(index int, remote, address, serverTP string) {
+ util.WithRecovery(func() {
+ defer wg.Done()
+ items, err := getServerInfoByGRPC(ctx, remote, infoTp)
+ if err != nil {
+ ch <- result{idx: index, err: err}
+ return
+ }
+ partRows := serverInfoItemToRows(items, serverTP, address)
+ ch <- result{idx: index, rows: partRows}
+ }, nil)
+ }(i, remote, address, srv.ServerType)
+ }
+ wg.Wait()
+ close(ch)
+ // Keep the original order to make the result more stable
+ var results []result //nolint: prealloc
+ for result := range ch {
+ if result.err != nil {
+ if recordWarningInStmtCtx {
+ sctx.GetSessionVars().StmtCtx.AppendWarning(result.err)
+ } else {
+ log.Warn(result.err.Error())
+ }
+ continue
+ }
+ results = append(results, result)
+ }
+ slices.SortFunc(results, func(i, j result) bool { return i.idx < j.idx })
+ for _, result := range results {
+ finalRows = append(finalRows, result.rows...)
+ }
+ return finalRows, nil
+}
+
+func serverInfoItemToRows(items []*diagnosticspb.ServerInfoItem, tp, addr string) [][]types.Datum {
+ rows := make([][]types.Datum, 0, len(items))
+ for _, v := range items {
+ for _, item := range v.Pairs {
+ row := types.MakeDatums(
+ tp,
+ addr,
+ v.Tp,
+ v.Name,
+ item.Key,
+ item.Value,
+ )
+ rows = append(rows, row)
+ }
+ }
+ return rows
+}
+
+func getServerInfoByGRPC(ctx context.Context, address string, tp diagnosticspb.ServerInfoType) ([]*diagnosticspb.ServerInfoItem, error) {
+ opt := grpc.WithTransportCredentials(insecure.NewCredentials())
+ security := config.GetGlobalConfig().Security
+ if len(security.ClusterSSLCA) != 0 {
+ clusterSecurity := security.ClusterSecurity()
+ tlsConfig, err := clusterSecurity.ToTLSConfig()
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ opt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
+ }
+ conn, err := grpc.Dial(address, opt)
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ err := conn.Close()
+ if err != nil {
+ log.Error("close grpc connection error", zap.Error(err))
+ }
+ }()
+
+ cli := diagnosticspb.NewDiagnosticsClient(conn)
+ ctx, cancel := context.WithTimeout(ctx, time.Second*10)
+ defer cancel()
+ r, err := cli.ServerInfo(ctx, &diagnosticspb.ServerInfoRequest{Tp: tp})
+ if err != nil {
+ return nil, err
+ }
+ return r.Items, nil
+}
+
+// FilterClusterServerInfo filters serversInfo by nodeTypes and addresses
+func FilterClusterServerInfo(serversInfo []ServerInfo, nodeTypes, addresses set.StringSet) []ServerInfo {
+ if len(nodeTypes) == 0 && len(addresses) == 0 {
+ return serversInfo
+ }
+
+ filterServers := make([]ServerInfo, 0, len(serversInfo))
+ for _, srv := range serversInfo {
+ // Skip some node type which has been filtered in WHERE clause
+ // e.g: SELECT * FROM cluster_config WHERE type='tikv'
+ if len(nodeTypes) > 0 && !nodeTypes.Exist(srv.ServerType) {
+ continue
+ }
+ // Skip some node address which has been filtered in WHERE clause
+ // e.g: SELECT * FROM cluster_config WHERE address='192.16.8.12:2379'
+ if len(addresses) > 0 && !addresses.Exist(srv.Address) {
+ continue
+ }
+ filterServers = append(filterServers, srv)
+ }
+ return filterServers
+}
diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go
index 3650f2833a1a6..d44721090b8b6 100644
--- a/infoschema/tables_test.go
+++ b/infoschema/tables_test.go
@@ -56,8 +56,7 @@ func newTestKitWithRoot(t *testing.T, store kv.Storage) *testkit.TestKit {
func newTestKitWithPlanCache(t *testing.T, store kv.Storage) *testkit.TestKit {
tk := testkit.NewTestKit(t, store)
- se, err := session.CreateSession4TestWithOpt(store, &session.Opt{PreparedPlanCache: plannercore.NewLRUPlanCache(100,
- 0.1, math.MaxUint64, plannercore.PickPlanFromBucket, tk.Session())})
+ se, err := session.CreateSession4TestWithOpt(store, &session.Opt{PreparedPlanCache: plannercore.NewLRUPlanCache(100, 0.1, math.MaxUint64, tk.Session())})
require.NoError(t, err)
tk.SetSession(se)
tk.RefreshConnectionID()
@@ -162,7 +161,8 @@ func TestInfoSchemaFieldValue(t *testing.T) {
" `DIGEST` varchar(64) DEFAULT '',\n" +
" `MEM` bigint(21) unsigned DEFAULT NULL,\n" +
" `DISK` bigint(21) unsigned DEFAULT NULL,\n" +
- " `TxnStart` varchar(64) NOT NULL DEFAULT ''\n" +
+ " `TxnStart` varchar(64) NOT NULL DEFAULT '',\n" +
+ " `RESOURCE_GROUP` varchar(32) NOT NULL DEFAULT ''\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
tk.MustQuery("show create table information_schema.cluster_log").Check(
testkit.Rows("" +
@@ -264,6 +264,8 @@ func TestCurrentTimestampAsDefault(t *testing.T) {
c_timestamp timestamp,
c_timestamp_default timestamp default current_timestamp,
c_timestamp_default_3 timestamp(3) default current_timestamp(3),
+ c_date_default date default current_date,
+ c_date_default_2 date default curdate(),
c_varchar_default varchar(20) default "current_timestamp",
c_varchar_default_3 varchar(20) default "current_timestamp(3)",
c_varchar_default_on_update datetime default current_timestamp on update current_timestamp,
@@ -276,6 +278,8 @@ func TestCurrentTimestampAsDefault(t *testing.T) {
WHERE table_schema = "default_time_test" AND table_name = "default_time_table"
ORDER BY column_name`,
).Check(testkit.Rows(
+ "c_date_default CURRENT_DATE ",
+ "c_date_default_2 CURRENT_DATE ",
"c_datetime ",
"c_datetime_default CURRENT_TIMESTAMP ",
"c_datetime_default_2 CURRENT_TIMESTAMP(2) ",
@@ -300,47 +304,50 @@ func TestSomeTables(t *testing.T) {
tk.SetSession(se)
sm := &testkit.MockSessionManager{PS: make([]*util.ProcessInfo, 0)}
sm.PS = append(sm.PS, &util.ProcessInfo{
- ID: 1,
- User: "user-1",
- Host: "localhost",
- Port: "",
- DB: "information_schema",
- Command: byte(1),
- Digest: "abc1",
- State: 1,
- Info: "do something",
- StmtCtx: tk.Session().GetSessionVars().StmtCtx,
+ ID: 1,
+ User: "user-1",
+ Host: "localhost",
+ Port: "",
+ DB: "information_schema",
+ Command: byte(1),
+ Digest: "abc1",
+ State: 1,
+ Info: "do something",
+ StmtCtx: tk.Session().GetSessionVars().StmtCtx,
+ ResourceGroupName: "rg1",
})
sm.PS = append(sm.PS, &util.ProcessInfo{
- ID: 2,
- User: "user-2",
- Host: "localhost",
- Port: "",
- DB: "test",
- Command: byte(2),
- Digest: "abc2",
- State: 2,
- Info: strings.Repeat("x", 101),
- StmtCtx: tk.Session().GetSessionVars().StmtCtx,
+ ID: 2,
+ User: "user-2",
+ Host: "localhost",
+ Port: "",
+ DB: "test",
+ Command: byte(2),
+ Digest: "abc2",
+ State: 2,
+ Info: strings.Repeat("x", 101),
+ StmtCtx: tk.Session().GetSessionVars().StmtCtx,
+ ResourceGroupName: "rg2",
})
sm.PS = append(sm.PS, &util.ProcessInfo{
- ID: 3,
- User: "user-3",
- Host: "127.0.0.1",
- Port: "12345",
- DB: "test",
- Command: byte(2),
- Digest: "abc3",
- State: 1,
- Info: "check port",
- StmtCtx: tk.Session().GetSessionVars().StmtCtx,
+ ID: 3,
+ User: "user-3",
+ Host: "127.0.0.1",
+ Port: "12345",
+ DB: "test",
+ Command: byte(2),
+ Digest: "abc3",
+ State: 1,
+ Info: "check port",
+ StmtCtx: tk.Session().GetSessionVars().StmtCtx,
+ ResourceGroupName: "rg3",
})
tk.Session().SetSessionManager(sm)
tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Sort().Check(
testkit.Rows(
- fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 ", "in transaction", "do something"),
- fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s abc2 0 0 ", "autocommit", strings.Repeat("x", 101)),
- fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s abc3 0 0 ", "in transaction", "check port"),
+ fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1", "in transaction", "do something"),
+ fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s abc2 0 0 rg2", "autocommit", strings.Repeat("x", 101)),
+ fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s abc3 0 0 rg3", "in transaction", "check port"),
))
tk.MustQuery("SHOW PROCESSLIST;").Sort().Check(
testkit.Rows(
@@ -357,30 +364,32 @@ func TestSomeTables(t *testing.T) {
sm = &testkit.MockSessionManager{PS: make([]*util.ProcessInfo, 0)}
sm.PS = append(sm.PS, &util.ProcessInfo{
- ID: 1,
- User: "user-1",
- Host: "localhost",
- DB: "information_schema",
- Command: byte(1),
- Digest: "abc1",
- State: 1,
+ ID: 1,
+ User: "user-1",
+ Host: "localhost",
+ DB: "information_schema",
+ Command: byte(1),
+ Digest: "abc1",
+ State: 1,
+ ResourceGroupName: "rg1",
})
sm.PS = append(sm.PS, &util.ProcessInfo{
- ID: 2,
- User: "user-2",
- Host: "localhost",
- Command: byte(2),
- Digest: "abc2",
- State: 2,
- Info: strings.Repeat("x", 101),
- CurTxnStartTS: 410090409861578752,
+ ID: 2,
+ User: "user-2",
+ Host: "localhost",
+ Command: byte(2),
+ Digest: "abc2",
+ State: 2,
+ Info: strings.Repeat("x", 101),
+ CurTxnStartTS: 410090409861578752,
+ ResourceGroupName: "rg2",
})
tk.Session().SetSessionManager(sm)
tk.Session().GetSessionVars().TimeZone = time.UTC
tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Check(
testkit.Rows(
- fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 ", "in transaction", ""),
- fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s abc2 0 0 07-29 03:26:05.158(410090409861578752)", "autocommit", strings.Repeat("x", 101)),
+ fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1", "in transaction", ""),
+ fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s abc2 0 0 07-29 03:26:05.158(410090409861578752) rg2", "autocommit", strings.Repeat("x", 101)),
))
tk.MustQuery("SHOW PROCESSLIST;").Sort().Check(
testkit.Rows(
@@ -394,11 +403,11 @@ func TestSomeTables(t *testing.T) {
))
tk.MustQuery("select * from information_schema.PROCESSLIST where db is null;").Check(
testkit.Rows(
- fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s abc2 0 0 07-29 03:26:05.158(410090409861578752)", "autocommit", strings.Repeat("x", 101)),
+ fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s abc2 0 0 07-29 03:26:05.158(410090409861578752) rg2", "autocommit", strings.Repeat("x", 101)),
))
tk.MustQuery("select * from information_schema.PROCESSLIST where Info is null;").Check(
testkit.Rows(
- fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 ", "in transaction", ""),
+ fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1", "in transaction", ""),
))
}
@@ -531,18 +540,168 @@ func TestSlowQuery(t *testing.T) {
slowLogFileName := "tidb_slow.log"
prepareSlowLogfile(t, slowLogFileName)
defer func() { require.NoError(t, os.Remove(slowLogFileName)) }()
+ expectedRes := [][]interface{}{
+ {"2019-02-12 19:33:56.571953",
+ "406315658548871171",
+ "root",
+ "localhost",
+ "6",
+ "57",
+ "0.12",
+ "4.895492",
+ "0.4",
+ "0.2",
+ "0.000000003",
+ "2",
+ "0.000000002",
+ "0.00000001",
+ "0.000000003",
+ "0.19",
+ "0.21",
+ "0.01",
+ "0",
+ "0.18",
+ "[txnLock]",
+ "0.03",
+ "0",
+ "15",
+ "480",
+ "1",
+ "8",
+ "0.3824278",
+ "0.161",
+ "0.101",
+ "0.092",
+ "1.71",
+ "1",
+ "100001",
+ "100000",
+ "100",
+ "10",
+ "10",
+ "10",
+ "100",
+ "test",
+ "",
+ "0",
+ "42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772",
+ "t1:1,t2:2",
+ "0.1",
+ "0.2",
+ "0.03",
+ "127.0.0.1:20160",
+ "0.05",
+ "0.6",
+ "0.8",
+ "0.0.0.0:20160",
+ "70724",
+ "65536",
+ "0",
+ "0",
+ "0",
+ "0",
+ "10",
+ "",
+ "",
+ "0",
+ "1",
+ "0",
+ "0",
+ "1",
+ "0",
+ "0",
+ "abcd",
+ "60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4",
+ "",
+ "update t set i = 2;",
+ "select * from t_slim;"},
+ {"2021-09-08 14:39:54.506967",
+ "427578666238083075",
+ "root",
+ "172.16.0.0",
+ "40507",
+ "0",
+ "0",
+ "25.571605962",
+ "0.002923536",
+ "0.006800973",
+ "0.002100764",
+ "0",
+ "0",
+ "0",
+ "0.000015801",
+ "25.542014572",
+ "0",
+ "0.002294647",
+ "0.000605473",
+ "12.483",
+ "[tikvRPC regionMiss tikvRPC regionMiss regionMiss]",
+ "0",
+ "0",
+ "624",
+ "172064",
+ "60",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "rtdb",
+ "",
+ "0",
+ "124acb3a0bec903176baca5f9da00b4e7512a41c93b417923f26502edeb324cc",
+ "",
+ "0",
+ "0",
+ "0",
+ "",
+ "0",
+ "0",
+ "0",
+ "",
+ "856544",
+ "0",
+ "86.635049185",
+ "0.015486658",
+ "100.054",
+ "0",
+ "0",
+ "",
+ "",
+ "0",
+ "1",
+ "0",
+ "0",
+ "0",
+ "0",
+ "0",
+ "",
+ "",
+ "",
+ "",
+ "INSERT INTO ...;",
+ },
+ }
tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", slowLogFileName))
tk.MustExec("set time_zone = '+08:00';")
re := tk.MustQuery("select * from information_schema.slow_query")
- re.Check(testkit.RowsWithSep("|", "2019-02-12 19:33:56.571953|406315658548871171|root|localhost|6|57|0.12|4.895492|0.4|0.2|0.000000003|2|0.000000002|0.00000001|0.000000003|0.19|0.21|0.01|0|0.18|[txnLock]|0.03|0|15|480|1|8|0.3824278|0.161|0.101|0.092|1.71|1|100001|100000|100|10|10|10|100|test||0|42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772|t1:1,t2:2|0.1|0.2|0.03|127.0.0.1:20160|0.05|0.6|0.8|0.0.0.0:20160|70724|65536|0|0|0|0|10||0|1|0|0|1|0|0|abcd|60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4||update t set i = 2;|select * from t_slim;",
- "2021-09-08|14:39:54.506967|427578666238083075|root|172.16.0.0|40507|0|0|25.571605962|0.002923536|0.006800973|0.002100764|0|0|0|0.000015801|25.542014572|0|0.002294647|0.000605473|12.483|[tikvRPC regionMiss tikvRPC regionMiss regionMiss]|0|0|624|172064|60|0|0|0|0|0|0|0|0|0|0|0|0|0|0|rtdb||0|124acb3a0bec903176baca5f9da00b4e7512a41c93b417923f26502edeb324cc||0|0|0||0|0|0||856544|0|86.635049185|0.015486658|100.054|0|0||0|1|0|0|0|0|0|||||INSERT INTO ...;",
- ))
+ re.Check(expectedRes)
+
tk.MustExec("set time_zone = '+00:00';")
re = tk.MustQuery("select * from information_schema.slow_query")
- re.Check(testkit.RowsWithSep("|", "2019-02-12 11:33:56.571953|406315658548871171|root|localhost|6|57|0.12|4.895492|0.4|0.2|0.000000003|2|0.000000002|0.00000001|0.000000003|0.19|0.21|0.01|0|0.18|[txnLock]|0.03|0|15|480|1|8|0.3824278|0.161|0.101|0.092|1.71|1|100001|100000|100|10|10|10|100|test||0|42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772|t1:1,t2:2|0.1|0.2|0.03|127.0.0.1:20160|0.05|0.6|0.8|0.0.0.0:20160|70724|65536|0|0|0|0|10||0|1|0|0|1|0|0|abcd|60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4||update t set i = 2;|select * from t_slim;",
- "2021-09-08|06:39:54.506967|427578666238083075|root|172.16.0.0|40507|0|0|25.571605962|0.002923536|0.006800973|0.002100764|0|0|0|0.000015801|25.542014572|0|0.002294647|0.000605473|12.483|[tikvRPC regionMiss tikvRPC regionMiss regionMiss]|0|0|624|172064|60|0|0|0|0|0|0|0|0|0|0|0|0|0|0|rtdb||0|124acb3a0bec903176baca5f9da00b4e7512a41c93b417923f26502edeb324cc||0|0|0||0|0|0||856544|0|86.635049185|0.015486658|100.054|0|0||0|1|0|0|0|0|0|||||INSERT INTO ...;",
- ))
+ expectedRes[0][0] = "2019-02-12 11:33:56.571953"
+ expectedRes[1][0] = "2021-09-08 06:39:54.506967"
+ re.Check(expectedRes)
// Test for long query.
f, err := os.OpenFile(slowLogFileName, os.O_CREATE|os.O_WRONLY, 0644)
@@ -588,12 +747,12 @@ INSERT INTO ...;
defer func() { require.NoError(t, os.Remove(slowLogFileName)) }()
tk := testkit.NewTestKit(t, store)
- //check schema
+ // check schema
tk.MustQuery(`select COUNT(*) from information_schema.columns
WHERE table_name = 'slow_query' and column_name = '` + columnName + `'`).
Check(testkit.Rows("1"))
- //check select
+ // check select
tk.MustQuery(`select ` + columnName +
` from information_schema.slow_query`).Check(testkit.Rows("1"))
}
@@ -676,8 +835,14 @@ func TestSelectHiddenColumn(t *testing.T) {
func TestFormatVersion(t *testing.T) {
// Test for defaultVersions.
- defaultVersions := []string{"5.7.25-TiDB-None", "5.7.25-TiDB-8.0.18", "5.7.25-TiDB-8.0.18-beta.1", "5.7.25-TiDB-v4.0.0-beta-446-g5268094af"}
- defaultRes := []string{"None", "8.0.18", "8.0.18-beta.1", "4.0.0-beta"}
+ defaultVersions := []string{
+ "5.7.25-TiDB-None",
+ "5.7.25-TiDB-8.0.18",
+ "5.7.25-TiDB-8.0.18-beta.1",
+ "5.7.25-TiDB-v4.0.0-beta-446-g5268094af",
+ "5.7.25-TiDB-",
+ "5.7.25-TiDB-v4.0.0-TiDB-446"}
+ defaultRes := []string{"None", "8.0.18", "8.0.18-beta.1", "4.0.0-beta", "", "4.0.0-TiDB"}
for i, v := range defaultVersions {
version := infoschema.FormatTiDBVersion(v, true)
require.Equal(t, defaultRes[i], version)
@@ -1052,7 +1217,7 @@ func TestStmtSummaryInternalQuery(t *testing.T) {
"where digest_text like \"select `original_sql` , `bind_sql` , `default_db` , status%\""
tk.MustQuery(sql).Check(testkit.Rows(
"select `original_sql` , `bind_sql` , `default_db` , status , `create_time` , `update_time` , charset , " +
- "collation , source from `mysql` . `bind_info` where `update_time` > ? order by `update_time` , `create_time`"))
+ "collation , source , `sql_digest` , `plan_digest` from `mysql` . `bind_info` where `update_time` > ? order by `update_time` , `create_time`"))
// Test for issue #21642.
tk.MustQuery(`select tidb_version()`)
@@ -1387,16 +1552,19 @@ func TestTiDBTrx(t *testing.T) {
tk.MustExec("update test_tidb_trx set i = i + 1")
_, digest := parser.NormalizeDigest("update test_tidb_trx set i = i + 1")
sm := &testkit.MockSessionManager{TxnInfo: make([]*txninfo.TxnInfo, 2)}
+ memDBTracker := memory.NewTracker(memory.LabelForMemDB, -1)
+ memDBTracker.Consume(19)
+ tk.Session().GetSessionVars().MemDBFootprint = memDBTracker
sm.TxnInfo[0] = &txninfo.TxnInfo{
StartTS: 424768545227014155,
CurrentSQLDigest: digest.String(),
State: txninfo.TxnIdle,
EntriesCount: 1,
- EntriesSize: 19,
ConnectionID: 2,
Username: "root",
CurrentDB: "test",
}
+
blockTime2 := time.Date(2021, 05, 20, 13, 18, 30, 123456000, time.Local)
sm.TxnInfo[1] = &txninfo.TxnInfo{
StartTS: 425070846483628033,
@@ -1413,7 +1581,7 @@ func TestTiDBTrx(t *testing.T) {
tk.MustQuery("select * from information_schema.TIDB_TRX;").Check(testkit.Rows(
"424768545227014155 2021-05-07 12:56:48.001000 "+digest.String()+" update `test_tidb_trx` set `i` = `i` + ? Idle 1 19 2 root test [] ",
- "425070846483628033 2021-05-20 21:16:35.778000 LockWaiting 2021-05-20 13:18:30.123456 0 0 10 user1 db1 [\"sql1\",\"sql2\",\""+digest.String()+"\"] "))
+ "425070846483628033 2021-05-20 21:16:35.778000 LockWaiting 2021-05-20 13:18:30.123456 0 19 10 user1 db1 [\"sql1\",\"sql2\",\""+digest.String()+"\"] "))
// Test the all_sql_digests column can be directly passed to the tidb_decode_sql_digests function.
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/expression/sqlDigestRetrieverSkipRetrieveGlobal", "return"))
@@ -1513,10 +1681,6 @@ func TestVariablesInfo(t *testing.T) {
tk := testkit.NewTestKit(t, store)
- if !variable.EnableConcurrentDDL.Load() {
- t.Skip("skip test when concurrent DDL is disabled")
- }
-
tk.MustExec("use information_schema")
tk.MustExec("SET GLOBAL innodb_compression_level = 8;")
@@ -1547,15 +1711,14 @@ func TestVariablesInfo(t *testing.T) {
// See session/bootstrap.go:doDMLWorks() for where the exceptions are defined.
stmt := tk.MustQuery(`SELECT variable_name, default_value, current_value FROM information_schema.variables_info WHERE current_value != default_value and default_value != '' ORDER BY variable_name`)
stmt.Check(testkit.Rows(
- "last_sql_use_alloc OFF ON", // for test stability
- "tidb_enable_auto_analyze ON OFF", // always changed for tests
- "tidb_enable_collect_execution_info ON OFF", // for test stability
- "tidb_enable_mutation_checker OFF ON", // for new installs
- "tidb_enable_plan_replayer_capture OFF false", // for enable plan replayer capture
- "tidb_mem_oom_action CANCEL LOG", // always changed for tests
- "tidb_row_format_version 1 2", // for new installs
- "tidb_txn_assertion_level OFF FAST", // for new installs
- "timestamp 0 123456789", // always dynamic
+ "last_sql_use_alloc OFF ON", // for test stability
+ "tidb_enable_auto_analyze ON OFF", // always changed for tests
+ "tidb_enable_collect_execution_info ON OFF", // for test stability
+ "tidb_enable_mutation_checker OFF ON", // for new installs
+ "tidb_mem_oom_action CANCEL LOG", // always changed for tests
+ "tidb_row_format_version 1 2", // for new installs
+ "tidb_txn_assertion_level OFF FAST", // for new installs
+ "timestamp 0 123456789", // always dynamic
))
}
@@ -1653,3 +1816,33 @@ func TestMemoryUsageAndOpsHistory(t *testing.T) {
require.Equal(t, row[10], "e3237ec256015a3566757e0c2742507cd30ae04e4cac2fbc14d269eafe7b067b") // SQL_DIGEST
require.Equal(t, row[11], "explain analyze select * from t t1 join t t2 join t t3 on t1.a=t2.a and t1.a=t3.a order by t1.a") // SQL_TEXT
}
+
+func TestAddFieldsForBinding(t *testing.T) {
+ s := new(clusterTablesSuite)
+ s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+ s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+ s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+ s.startTime = time.Now()
+ defer s.httpServer.Close()
+ defer s.rpcserver.Stop()
+ tk := s.newTestKitWithRoot(t)
+
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, key(a))")
+ tk.MustExec("select /*+ ignore_index(t, a)*/ * from t where a = 1")
+ planDigest := "4e3159169cc63c14b139a4e7d72eae1759875c9a9581f94bb2079aae961189cb"
+ rows := tk.MustQuery(fmt.Sprintf("select stmt_type, prepared, sample_user, schema_name, query_sample_text, charset, collation, plan_hint, digest_text "+
+ "from information_schema.cluster_statements_summary where plan_digest = '%s'", planDigest)).Rows()
+
+ require.Equal(t, rows[0][0], "Select")
+ require.Equal(t, rows[0][1], "0")
+ require.Equal(t, rows[0][2], "root")
+ require.Equal(t, rows[0][3], "test")
+ require.Equal(t, rows[0][4], "select /*+ ignore_index(t, a)*/ * from t where a = 1")
+ require.Equal(t, rows[0][5], "utf8mb4")
+ require.Equal(t, rows[0][6], "utf8mb4_bin")
+ require.Equal(t, rows[0][7], "use_index(@`sel_1` `test`.`t` ), ignore_index(`t` `a`)")
+ require.Equal(t, rows[0][8], "select * from `t` where `a` = ?")
+}
diff --git a/keyspace/BUILD.bazel b/keyspace/BUILD.bazel
new file mode 100644
index 0000000000000..a536722a018d7
--- /dev/null
+++ b/keyspace/BUILD.bazel
@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "keyspace",
+ srcs = ["keyspace.go"],
+ importpath = "github.com/pingcap/tidb/keyspace",
+ visibility = ["//visibility:public"],
+ deps = [
+ "@com_github_pingcap_kvproto//pkg/kvrpcpb",
+ "@com_github_tikv_client_go_v2//tikv",
+ ],
+)
diff --git a/keyspace/keyspace.go b/keyspace/keyspace.go
new file mode 100644
index 0000000000000..103d0f742cee7
--- /dev/null
+++ b/keyspace/keyspace.go
@@ -0,0 +1,38 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package keyspace
+
+import (
+ "fmt"
+
+ "github.com/pingcap/kvproto/pkg/kvrpcpb"
+ "github.com/tikv/client-go/v2/tikv"
+)
+
+const (
+ // tidbKeyspaceEtcdPathPrefix is the keyspace prefix for etcd namespace
+ tidbKeyspaceEtcdPathPrefix = "/keyspaces/tidb/"
+)
+
+// CodecV1 represents api v1 codec.
+var CodecV1 = tikv.NewCodecV1(tikv.ModeTxn)
+
+// MakeKeyspaceEtcdNamespace return the keyspace prefix path for etcd namespace
+func MakeKeyspaceEtcdNamespace(c tikv.Codec) string {
+ if c.GetAPIVersion() == kvrpcpb.APIVersion_V1 {
+ return ""
+ }
+ return fmt.Sprintf(tidbKeyspaceEtcdPathPrefix+"%d", c.GetKeyspaceID())
+}
diff --git a/kv/BUILD.bazel b/kv/BUILD.bazel
index 32dd9f1474179..309251993d7f0 100644
--- a/kv/BUILD.bazel
+++ b/kv/BUILD.bazel
@@ -30,6 +30,7 @@ go_library(
"//types",
"//util/codec",
"//util/dbterror",
+ "//util/intest",
"//util/logutil",
"//util/memory",
"//util/set",
@@ -48,6 +49,7 @@ go_library(
"@com_github_tikv_client_go_v2//tikvrpc",
"@com_github_tikv_client_go_v2//util",
"@com_github_tikv_pd_client//:client",
+ "@org_golang_x_exp//slices",
"@org_uber_go_zap//:zap",
],
)
@@ -82,6 +84,7 @@ go_test(
"@com_github_pingcap_kvproto//pkg/coprocessor",
"@com_github_pingcap_kvproto//pkg/deadlock",
"@com_github_pingcap_kvproto//pkg/kvrpcpb",
+ "@com_github_pingcap_tipb//go-tipb",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@com_github_tikv_client_go_v2//oracle",
diff --git a/kv/interface_mock_test.go b/kv/interface_mock_test.go
index 8090463c84223..283eb0858b716 100644
--- a/kv/interface_mock_test.go
+++ b/kv/interface_mock_test.go
@@ -52,6 +52,13 @@ func (t *mockTxn) LockKeys(_ context.Context, _ *LockCtx, _ ...Key) error {
return nil
}
+func (t *mockTxn) LockKeysFunc(_ context.Context, _ *LockCtx, fn func(), _ ...Key) error {
+ if fn != nil {
+ fn()
+ }
+ return nil
+}
+
func (t *mockTxn) SetOption(opt int, val interface{}) {
t.opts[opt] = val
}
@@ -161,6 +168,20 @@ func (t *mockTxn) UpdateMemBufferFlags(_ []byte, _ ...FlagsOp) {
}
+func (t *mockTxn) SetMemoryFootprintChangeHook(func(uint64)) {
+
+}
+
+func (t *mockTxn) Mem() uint64 {
+ return 0
+}
+
+func (t *mockTxn) StartAggressiveLocking() error { return nil }
+func (t *mockTxn) RetryAggressiveLocking(_ context.Context) error { return nil }
+func (t *mockTxn) CancelAggressiveLocking(_ context.Context) error { return nil }
+func (t *mockTxn) DoneAggressiveLocking(_ context.Context) error { return nil }
+func (t *mockTxn) IsInAggressiveLockingMode() bool { return false }
+
// newMockTxn new a mockTxn.
func newMockTxn() Transaction {
return &mockTxn{
@@ -172,6 +193,10 @@ func newMockTxn() Transaction {
// mockStorage is used to start a must commit-failed txn.
type mockStorage struct{}
+func (s *mockStorage) GetCodec() tikv.Codec {
+ return nil
+}
+
func (s *mockStorage) Begin(opts ...tikv.TxnOption) (Transaction, error) {
return newMockTxn(), nil
}
diff --git a/kv/key.go b/kv/key.go
index 6b5ca28400721..3c243a06ed2eb 100644
--- a/kv/key.go
+++ b/kv/key.go
@@ -149,6 +149,7 @@ type Handle interface {
// IntValue returns the int64 value if IsInt is true, it panics if IsInt returns false.
IntValue() int64
// Next returns the minimum handle that is greater than this handle.
+ // The returned handle is not guaranteed to be able to decode.
Next() Handle
// Equal returns if the handle equals to another handle, it panics if the types are different.
Equal(h Handle) bool
@@ -299,6 +300,7 @@ func (*CommonHandle) IntValue() int64 {
}
// Next implements the Handle interface.
+// Note that the returned encoded field is not guaranteed to be able to decode.
func (ch *CommonHandle) Next() Handle {
return &CommonHandle{
encoded: Key(ch.encoded).PrefixNext(),
diff --git a/kv/kv.go b/kv/kv.go
index 06e86f41659cb..057c22fb7312b 100644
--- a/kv/kv.go
+++ b/kv/kv.go
@@ -15,6 +15,7 @@
package kv
import (
+ "bytes"
"context"
"crypto/tls"
"time"
@@ -33,6 +34,7 @@ import (
"github.com/tikv/client-go/v2/tikvrpc"
"github.com/tikv/client-go/v2/util"
pd "github.com/tikv/pd/client"
+ "golang.org/x/exp/slices"
)
// UnCommitIndexKVFlag uses to indicate the index key/value is no need to commit.
@@ -201,8 +203,13 @@ type LockCtx = tikvstore.LockCtx
type Transaction interface {
RetrieverMutator
AssertionProto
+ AggressiveLockingController
// Size returns sum of keys and values length.
Size() int
+ // Mem returns the memory consumption of the transaction.
+ Mem() uint64
+ // SetMemoryFootprintChangeHook sets the hook that will be called when the memory footprint changes.
+ SetMemoryFootprintChangeHook(func(uint64))
// Len returns the number of entries in the DB.
Len() int
// Reset reset the Transaction to initial states.
@@ -216,6 +223,10 @@ type Transaction interface {
// LockKeys tries to lock the entries with the keys in KV store.
// Will block until all keys are locked successfully or an error occurs.
LockKeys(ctx context.Context, lockCtx *LockCtx, keys ...Key) error
+ // LockKeysFunc tries to lock the entries with the keys in KV store.
+ // Will block until all keys are locked successfully or an error occurs.
+ // fn is called before LockKeys unlocks the keys.
+ LockKeysFunc(ctx context.Context, lockCtx *LockCtx, fn func(), keys ...Key) error
// SetOption sets an option with a value, when val is nil, uses the default
// value of this option.
SetOption(opt int, val interface{})
@@ -271,6 +282,15 @@ type AssertionProto interface {
SetAssertion(key []byte, assertion ...FlagsOp) error
}
+// AggressiveLockingController is the interface that defines aggressive locking related operations.
+type AggressiveLockingController interface {
+ StartAggressiveLocking() error
+ RetryAggressiveLocking(ctx context.Context) error
+ CancelAggressiveLocking(ctx context.Context) error
+ DoneAggressiveLocking(ctx context.Context) error
+ IsInAggressiveLockingMode() bool
+}
+
// Client is used to send request to KV layer.
type Client interface {
// Send sends request to KV layer, returns a Response.
@@ -331,13 +351,168 @@ func (t StoreType) Name() string {
return "unspecified"
}
+// KeyRanges wrap the ranges for partitioned table cases.
+// We might send ranges from different in the one request.
+type KeyRanges struct {
+ ranges [][]KeyRange
+ rowCountHints [][]int
+
+ isPartitioned bool
+}
+
+// NewPartitionedKeyRanges constructs a new RequestRange for partitioned table.
+func NewPartitionedKeyRanges(ranges [][]KeyRange) *KeyRanges {
+ return NewPartitionedKeyRangesWithHints(ranges, nil)
+}
+
+// NewNonParitionedKeyRanges constructs a new RequestRange for a non partitioned table.
+func NewNonParitionedKeyRanges(ranges []KeyRange) *KeyRanges {
+ return NewNonParitionedKeyRangesWithHint(ranges, nil)
+}
+
+// NewPartitionedKeyRangesWithHints constructs a new RequestRange for partitioned table with row count hint.
+func NewPartitionedKeyRangesWithHints(ranges [][]KeyRange, hints [][]int) *KeyRanges {
+ return &KeyRanges{
+ ranges: ranges,
+ rowCountHints: hints,
+ isPartitioned: true,
+ }
+}
+
+// NewNonParitionedKeyRangesWithHint constructs a new RequestRange for a non partitioned table with rou count hint.
+func NewNonParitionedKeyRangesWithHint(ranges []KeyRange, hints []int) *KeyRanges {
+ rr := &KeyRanges{
+ ranges: [][]KeyRange{ranges},
+ isPartitioned: false,
+ }
+ if hints != nil {
+ rr.rowCountHints = [][]int{hints}
+ }
+ return rr
+}
+
+// FirstPartitionRange returns the the result of first range.
+// We may use some func to generate ranges for both partitioned table and non partitioned table.
+// This method provides a way to fallback to non-partitioned ranges.
+func (rr *KeyRanges) FirstPartitionRange() []KeyRange {
+ if len(rr.ranges) == 0 {
+ return []KeyRange{}
+ }
+ return rr.ranges[0]
+}
+
+// SetToNonPartitioned set the status to non-partitioned.
+func (rr *KeyRanges) SetToNonPartitioned() error {
+ if len(rr.ranges) > 1 {
+ return errors.Errorf("you want to change the partitioned ranges to non-partitioned ranges")
+ }
+ rr.isPartitioned = false
+ return nil
+}
+
+// AppendSelfTo appends itself to another slice.
+func (rr *KeyRanges) AppendSelfTo(ranges []KeyRange) []KeyRange {
+ for _, r := range rr.ranges {
+ ranges = append(ranges, r...)
+ }
+ return ranges
+}
+
+// SortByFunc sorts each partition's ranges.
+// Since the ranges are sorted in most cases, we check it first.
+func (rr *KeyRanges) SortByFunc(sortFunc func(i, j KeyRange) bool) {
+ if !slices.IsSortedFunc(rr.ranges, func(i, j []KeyRange) bool {
+ // A simple short-circuit since the empty range actually won't make anything wrong.
+ if len(i) == 0 || len(j) == 0 {
+ return true
+ }
+ return sortFunc(i[0], j[0])
+ }) {
+ slices.SortFunc(rr.ranges, func(i, j []KeyRange) bool {
+ if len(i) == 0 {
+ return true
+ }
+ if len(j) == 0 {
+ return false
+ }
+ return sortFunc(i[0], j[0])
+ })
+ }
+ for i := range rr.ranges {
+ if !slices.IsSortedFunc(rr.ranges[i], sortFunc) {
+ slices.SortFunc(rr.ranges[i], sortFunc)
+ }
+ }
+}
+
+// ForEachPartitionWithErr runs the func for each partition with an error check.
+func (rr *KeyRanges) ForEachPartitionWithErr(theFunc func([]KeyRange, []int) error) (err error) {
+ for i := range rr.ranges {
+ var hints []int
+ if len(rr.rowCountHints) > i {
+ hints = rr.rowCountHints[i]
+ }
+ err = theFunc(rr.ranges[i], hints)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// ForEachPartition runs the func for each partition without error check.
+func (rr *KeyRanges) ForEachPartition(theFunc func([]KeyRange)) {
+ for i := range rr.ranges {
+ theFunc(rr.ranges[i])
+ }
+}
+
+// PartitionNum returns how many partition is involved in the ranges.
+func (rr *KeyRanges) PartitionNum() int {
+ return len(rr.ranges)
+}
+
+// IsFullySorted checks whether the ranges are sorted inside partition and each partition is also sorated.
+func (rr *KeyRanges) IsFullySorted() bool {
+ sortedByPartition := slices.IsSortedFunc(rr.ranges, func(i, j []KeyRange) bool {
+ // A simple short-circuit since the empty range actually won't make anything wrong.
+ if len(i) == 0 || len(j) == 0 {
+ return true
+ }
+ return bytes.Compare(i[0].StartKey, j[0].StartKey) < 0
+ })
+ if !sortedByPartition {
+ return false
+ }
+ for _, ranges := range rr.ranges {
+ if !slices.IsSortedFunc(ranges, func(i, j KeyRange) bool {
+ return bytes.Compare(i.StartKey, j.StartKey) < 0
+ }) {
+ return false
+ }
+ }
+ return true
+}
+
+// TotalRangeNum returns how many ranges there are.
+func (rr *KeyRanges) TotalRangeNum() int {
+ ret := 0
+ for _, r := range rr.ranges {
+ ret += len(r)
+ }
+ return ret
+}
+
// Request represents a kv request.
type Request struct {
// Tp is the request type.
- Tp int64
- StartTs uint64
- Data []byte
- KeyRanges []KeyRange
+ Tp int64
+ StartTs uint64
+ Data []byte
+
+ // KeyRanges makes sure that the request is sent first by partition then by region.
+ // When the table is small, it's possible that multiple partitions are in the same region.
+ KeyRanges *KeyRanges
// For PartitionTableScan used by tiflash.
PartitionIDAndRanges []PartitionIDAndRanges
@@ -394,8 +569,12 @@ type Request struct {
}
// RequestSource indicates whether the request is an internal request.
RequestSource util.RequestSource
- // FixedRowCountHint is the optimization hint for copr request for task scheduling.
- FixedRowCountHint []int
+ // StoreBatchSize indicates the batch size of coprocessor in the same store.
+ StoreBatchSize int
+ // ResourceGroupName is the name of the bind resource group.
+ ResourceGroupName string
+ // LimitSize indicates whether the request is scan and limit
+ LimitSize uint64
}
// CoprRequestAdjuster is used to check and adjust a copr request according to specific rules.
@@ -504,6 +683,8 @@ type Storage interface {
GetMinSafeTS(txnScope string) uint64
// GetLockWaits return all lock wait information
GetLockWaits() ([]*deadlockpb.WaitForEntry, error)
+ // GetCodec gets the codec of the storage.
+ GetCodec() tikv.Codec
}
// EtcdBackend is used for judging a storage is a real TiKV.
diff --git a/kv/mpp.go b/kv/mpp.go
index b0752f8186deb..095352c02832a 100644
--- a/kv/mpp.go
+++ b/kv/mpp.go
@@ -16,33 +16,101 @@ package kv
import (
"context"
- "sync"
+ "strconv"
+ "strings"
"time"
"github.com/pingcap/kvproto/pkg/mpp"
+ "github.com/pingcap/tidb/util/memory"
+ "github.com/pingcap/tipb/go-tipb"
)
+// MppVersion indicates the mpp-version used to build mpp plan
+type MppVersion int64
+
+const (
+ // MppVersionV0 supports TiFlash version [~, ~]
+ MppVersionV0 MppVersion = iota
+
+ // MppVersionV1 supports TiFlash version [v6.6.x, ~]
+ MppVersionV1
+
+ // MppVersionV2
+ // MppVersionV3
+
+ mppVersionMax
+
+ newestMppVersion MppVersion = mppVersionMax - 1
+
+ // MppVersionUnspecified means the illegal or unspecified version, it only used in TiDB.
+ MppVersionUnspecified MppVersion = -1
+
+ // MppVersionUnspecifiedName denotes name of UNSPECIFIED mpp version
+ MppVersionUnspecifiedName string = "UNSPECIFIED"
+)
+
+// ToInt64 transforms MppVersion to int64
+func (v MppVersion) ToInt64() int64 {
+ return int64(v)
+}
+
+// ToMppVersion transforms string to MppVersion
+func ToMppVersion(name string) (MppVersion, bool) {
+ name = strings.ToUpper(name)
+ if name == MppVersionUnspecifiedName {
+ return MppVersionUnspecified, true
+ }
+ v, err := strconv.ParseInt(name, 10, 64)
+ if err != nil {
+ return MppVersionUnspecified, false
+ }
+ version := MppVersion(v)
+ if version >= MppVersionUnspecified && version <= newestMppVersion {
+ return version, true
+ }
+ return MppVersionUnspecified, false
+}
+
+// GetNewestMppVersion returns the mpp-version can be used in mpp plan
+func GetNewestMppVersion() MppVersion {
+ return newestMppVersion
+}
+
// MPPTaskMeta means the meta info such as location of a mpp task.
type MPPTaskMeta interface {
// GetAddress indicates which node this task should execute on.
GetAddress() string
}
+// MPPQueryID means the global unique id of a mpp query.
+type MPPQueryID struct {
+ QueryTs uint64 // timestamp of query execution, used for TiFlash minTSO schedule
+ LocalQueryID uint64 // unique mpp query id in local tidb memory.
+ ServerID uint64
+}
+
// MPPTask means the minimum execution unit of a mpp computation job.
type MPPTask struct {
- Meta MPPTaskMeta // on which store this task will execute
- ID int64 // mppTaskID
- StartTs uint64
- TableID int64 // physical table id
+ Meta MPPTaskMeta // on which store this task will execute
+ ID int64 // mppTaskID
+ StartTs uint64
+ MppQueryID MPPQueryID
+ TableID int64 // physical table id
+ MppVersion MppVersion // mpp version
- PartitionTableIDs []int64
+ PartitionTableIDs []int64
+ IsDisaggregatedTiFlashStaticPrune bool
}
// ToPB generates the pb structure.
func (t *MPPTask) ToPB() *mpp.TaskMeta {
meta := &mpp.TaskMeta{
- StartTs: t.StartTs,
- TaskId: t.ID,
+ StartTs: t.StartTs,
+ QueryTs: t.MppQueryID.QueryTs,
+ LocalQueryId: t.MppQueryID.LocalQueryID,
+ ServerId: t.MppQueryID.ServerID,
+ TaskId: t.ID,
+ MppVersion: t.MppVersion.ToInt64(),
}
if t.ID != -1 {
meta.Address = t.Meta.GetAddress()
@@ -71,20 +139,20 @@ type MPPDispatchRequest struct {
IsRoot bool // root task returns data to tidb directly.
Timeout uint64 // If task is assigned but doesn't receive a connect request during timeout, the task should be destroyed.
// SchemaVer is for any schema-ful storage (like tiflash) to validate schema correctness if necessary.
- SchemaVar int64
- StartTs uint64
- ID int64 // identify a single task
- State MppTaskStates
+ SchemaVar int64
+ StartTs uint64
+ MppQueryID MPPQueryID
+ ID int64 // identify a single task
+ State MppTaskStates
}
// MPPClient accepts and processes mpp requests.
type MPPClient interface {
// ConstructMPPTasks schedules task for a plan fragment.
// TODO:: This interface will be refined after we support more executors.
- ConstructMPPTasks(context.Context, *MPPBuildTasksRequest, *sync.Map, time.Duration) ([]MPPTaskMeta, error)
-
+ ConstructMPPTasks(context.Context, *MPPBuildTasksRequest, time.Duration) ([]MPPTaskMeta, error)
// DispatchMPPTasks dispatches ALL mpp requests at once, and returns an iterator that transfers the data.
- DispatchMPPTasks(ctx context.Context, vars interface{}, reqs []*MPPDispatchRequest, needTriggerFallback bool, startTs uint64) Response
+ DispatchMPPTasks(ctx context.Context, vars interface{}, reqs []*MPPDispatchRequest, needTriggerFallback bool, startTs uint64, mppQueryID MPPQueryID, mppVersion MppVersion, memTracker *memory.Tracker) Response
}
// MPPBuildTasksRequest request the stores allocation for a mpp plan fragment.
@@ -95,3 +163,56 @@ type MPPBuildTasksRequest struct {
PartitionIDAndRanges []PartitionIDAndRanges
}
+
+// ExchangeCompressionMode means the compress method used in exchange operator
+type ExchangeCompressionMode int
+
+const (
+ // ExchangeCompressionModeNONE indicates no compression
+ ExchangeCompressionModeNONE ExchangeCompressionMode = iota
+ // ExchangeCompressionModeFast indicates fast compression/decompression speed, compression ratio is lower than HC mode
+ ExchangeCompressionModeFast
+ // ExchangeCompressionModeHC indicates high compression (HC) ratio mode
+ ExchangeCompressionModeHC
+ // ExchangeCompressionModeUnspecified indicates unspecified compress method, let TiDB choose one
+ ExchangeCompressionModeUnspecified
+
+ // RecommendedExchangeCompressionMode indicates recommended compression mode
+ RecommendedExchangeCompressionMode ExchangeCompressionMode = ExchangeCompressionModeFast
+
+ exchangeCompressionModeUnspecifiedName string = "UNSPECIFIED"
+)
+
+// Name returns the name of ExchangeCompressionMode
+func (t ExchangeCompressionMode) Name() string {
+ if t == ExchangeCompressionModeUnspecified {
+ return exchangeCompressionModeUnspecifiedName
+ }
+ return t.ToTipbCompressionMode().String()
+}
+
+// ToExchangeCompressionMode returns the ExchangeCompressionMode from name
+func ToExchangeCompressionMode(name string) (ExchangeCompressionMode, bool) {
+ name = strings.ToUpper(name)
+ if name == exchangeCompressionModeUnspecifiedName {
+ return ExchangeCompressionModeUnspecified, true
+ }
+ value, ok := tipb.CompressionMode_value[name]
+ if ok {
+ return ExchangeCompressionMode(value), true
+ }
+ return ExchangeCompressionModeNONE, false
+}
+
+// ToTipbCompressionMode returns tipb.CompressionMode from kv.ExchangeCompressionMode
+func (t ExchangeCompressionMode) ToTipbCompressionMode() tipb.CompressionMode {
+ switch t {
+ case ExchangeCompressionModeNONE:
+ return tipb.CompressionMode_NONE
+ case ExchangeCompressionModeFast:
+ return tipb.CompressionMode_FAST
+ case ExchangeCompressionModeHC:
+ return tipb.CompressionMode_HIGH_COMPRESSION
+ }
+ return tipb.CompressionMode_NONE
+}
diff --git a/kv/option.go b/kv/option.go
index 888a1e24f0fa0..631e69da7613d 100644
--- a/kv/option.go
+++ b/kv/option.go
@@ -15,6 +15,8 @@
package kv
import (
+ "context"
+
"github.com/tikv/client-go/v2/util"
)
@@ -93,6 +95,10 @@ const (
ReplicaReadAdjuster
// ScanBatchSize set the iter scan batch size.
ScanBatchSize
+ // TxnSource set the source of this transaction.
+ TxnSource
+ // ResourceGroupName set the bind resource group name.
+ ResourceGroupName
)
// ReplicaReadType is the type of replica to read data from
@@ -109,6 +115,10 @@ const (
ReplicaReadClosest
// ReplicaReadClosestAdaptive stands for 'read from follower which locates in the same zone if the response size exceeds certain threshold'
ReplicaReadClosestAdaptive
+ // ReplicaReadLearner stands for 'read from learner'.
+ ReplicaReadLearner
+ // ReplicaReadPreferLeader stands for 'read from leader and auto-turn to followers if leader is abnormal'.
+ ReplicaReadPreferLeader
)
// IsFollowerRead checks if follower is going to be used to read data.
@@ -130,6 +140,15 @@ type RequestSource = util.RequestSource
// WithInternalSourceType create context with internal source.
var WithInternalSourceType = util.WithInternalSourceType
+// GetInternalSourceType get internal source
+func GetInternalSourceType(ctx context.Context) string {
+ v := ctx.Value(util.RequestSourceKey)
+ if v == nil {
+ return ""
+ }
+ return v.(util.RequestSource).RequestSourceType
+}
+
const (
// InternalTxnOthers is the type of requests that consume low resources.
// This reduces the size of metrics.
@@ -165,4 +184,6 @@ const (
InternalTxnBR = InternalTxnTools
// InternalTxnTrace handles the trace statement.
InternalTxnTrace = "Trace"
+ // InternalTxnTTL is the type of TTL usage
+ InternalTxnTTL = "TTL"
)
diff --git a/kv/txn.go b/kv/txn.go
index d7828c7fb3138..fbbe6bdf27137 100644
--- a/kv/txn.go
+++ b/kv/txn.go
@@ -17,7 +17,6 @@ package kv
import (
"context"
"errors"
- "flag"
"fmt"
"math"
"math/rand"
@@ -26,6 +25,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/parser/terror"
+ "github.com/pingcap/tidb/util/intest"
"github.com/pingcap/tidb/util/logutil"
"github.com/tikv/client-go/v2/oracle"
"go.uber.org/zap"
@@ -195,20 +195,22 @@ func BackOff(attempts uint) int {
func setRequestSourceForInnerTxn(ctx context.Context, txn Transaction) {
if source := ctx.Value(RequestSourceKey); source != nil {
requestSource := source.(RequestSource)
- if !requestSource.RequestSourceInternal {
- logutil.Logger(ctx).Warn("`RunInNewTxn` should be used by inner txn only")
+ if requestSource.RequestSourceType != "" {
+ if !requestSource.RequestSourceInternal {
+ logutil.Logger(ctx).Warn("`RunInNewTxn` should be used by inner txn only")
+ }
+ txn.SetOption(RequestSourceInternal, requestSource.RequestSourceInternal)
+ txn.SetOption(RequestSourceType, requestSource.RequestSourceType)
+ return
}
- txn.SetOption(RequestSourceInternal, requestSource.RequestSourceInternal)
- txn.SetOption(RequestSourceType, requestSource.RequestSourceType)
+ }
+ // panic in test mode in case there are requests without source in the future.
+ // log warnings in production mode.
+ if intest.InTest {
+ panic("unexpected no source type context, if you see this error, " +
+ "the `RequestSourceTypeKey` is missing in your context")
} else {
- // panic in test mode in case there are requests without source in the future.
- // log warnings in production mode.
- if flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil {
- panic("unexpected no source type context, if you see this error, " +
- "the `RequestSourceTypeKey` is missing in your context")
- } else {
- logutil.Logger(ctx).Warn("unexpected no source type context, if you see this warning, " +
- "the `RequestSourceTypeKey` is missing in the context")
- }
+ logutil.Logger(ctx).Warn("unexpected no source type context, if you see this warning, " +
+ "the `RequestSourceTypeKey` is missing in the context")
}
}
diff --git a/kv/version_test.go b/kv/version_test.go
index 1a2aa30143d10..d0b0a810e3f45 100644
--- a/kv/version_test.go
+++ b/kv/version_test.go
@@ -17,6 +17,7 @@ package kv
import (
"testing"
+ "github.com/pingcap/tipb/go-tipb"
"github.com/stretchr/testify/assert"
)
@@ -30,3 +31,57 @@ func TestVersion(t *testing.T) {
assert.True(t, eq == 0)
assert.True(t, MinVersion.Cmp(MaxVersion) < 0)
}
+
+func TestMppVersion(t *testing.T) {
+ assert.Equal(t, int64(1), GetNewestMppVersion().ToInt64())
+ {
+ v, ok := ToMppVersion("unspecified")
+ assert.True(t, ok)
+ assert.Equal(t, v, MppVersionUnspecified)
+ }
+ {
+ v, ok := ToMppVersion("-1")
+ assert.True(t, ok)
+ assert.Equal(t, v, MppVersionUnspecified)
+ }
+ {
+ v, ok := ToMppVersion("0")
+ assert.True(t, ok)
+ assert.Equal(t, v, MppVersionV0)
+ }
+ {
+ v, ok := ToMppVersion("1")
+ assert.True(t, ok)
+ assert.Equal(t, v, MppVersionV1)
+ }
+}
+
+func TestExchangeCompressionMode(t *testing.T) {
+ assert.Equal(t, "UNSPECIFIED", ExchangeCompressionModeUnspecified.Name())
+ {
+ a, ok := ToExchangeCompressionMode("UNSPECIFIED")
+ assert.Equal(t, a, ExchangeCompressionModeUnspecified)
+ assert.True(t, ok)
+ }
+ assert.Equal(t, "NONE", ExchangeCompressionModeNONE.Name())
+ {
+ a, ok := ToExchangeCompressionMode("NONE")
+ assert.Equal(t, a, ExchangeCompressionModeNONE)
+ assert.True(t, ok)
+ }
+ assert.Equal(t, "FAST", ExchangeCompressionModeFast.Name())
+ {
+ a, ok := ToExchangeCompressionMode("FAST")
+ assert.Equal(t, a, ExchangeCompressionModeFast)
+ assert.True(t, ok)
+ }
+ assert.Equal(t, "HIGH_COMPRESSION", ExchangeCompressionModeHC.Name())
+ {
+ a, ok := ToExchangeCompressionMode("HIGH_COMPRESSION")
+ assert.Equal(t, a, ExchangeCompressionModeHC)
+ assert.True(t, ok)
+ }
+ // default `FAST`
+ assert.Equal(t, ExchangeCompressionModeFast, RecommendedExchangeCompressionMode)
+ assert.Equal(t, tipb.CompressionMode_FAST, RecommendedExchangeCompressionMode.ToTipbCompressionMode())
+}
diff --git a/meta/BUILD.bazel b/meta/BUILD.bazel
index 791662be8c215..c6c796a9771c1 100644
--- a/meta/BUILD.bazel
+++ b/meta/BUILD.bazel
@@ -16,10 +16,8 @@ go_library(
"//parser/mysql",
"//structure",
"//util/dbterror",
- "//util/logutil",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_kvproto//pkg/kvrpcpb",
- "@org_uber_go_zap//:zap",
],
)
@@ -33,12 +31,10 @@ go_test(
embed = [":meta"],
flaky = True,
deps = [
- "//ddl",
"//kv",
"//parser/model",
"//store/mockstore",
"//testkit/testsetup",
- "//testkit/testutil",
"//util",
"@com_github_pingcap_errors//:errors",
"@com_github_stretchr_testify//require",
diff --git a/meta/autoid/BUILD.bazel b/meta/autoid/BUILD.bazel
index 7490d65691e4c..0eb6034820160 100644
--- a/meta/autoid/BUILD.bazel
+++ b/meta/autoid/BUILD.bazel
@@ -11,7 +11,7 @@ go_library(
importpath = "github.com/pingcap/tidb/meta/autoid",
visibility = ["//visibility:public"],
deps = [
- "//autoid_service",
+ "//config",
"//errno",
"//kv",
"//meta",
@@ -23,7 +23,7 @@ go_library(
"//util/execdetails",
"//util/logutil",
"//util/mathutil",
- "@com_github_opentracing_opentracing_go//:opentracing-go",
+ "//util/tracing",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/autoid",
@@ -31,6 +31,7 @@ go_library(
"@com_github_tikv_client_go_v2//util",
"@io_etcd_go_etcd_client_v3//:client",
"@org_golang_google_grpc//:grpc",
+ "@org_golang_google_grpc//credentials",
"@org_golang_google_grpc//credentials/insecure",
"@org_uber_go_zap//:zap",
],
diff --git a/meta/autoid/autoid.go b/meta/autoid/autoid.go
index def3245bb2da3..4cd03bea89219 100644
--- a/meta/autoid/autoid.go
+++ b/meta/autoid/autoid.go
@@ -23,10 +23,9 @@ import (
"sync"
"time"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
- autoid "github.com/pingcap/tidb/autoid_service"
+ "github.com/pingcap/kvproto/pkg/autoid"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/metrics"
@@ -37,6 +36,7 @@ import (
"github.com/pingcap/tidb/util/execdetails"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/mathutil"
+ "github.com/pingcap/tidb/util/tracing"
"github.com/tikv/client-go/v2/txnkv/txnsnapshot"
tikvutil "github.com/tikv/client-go/v2/util"
clientv3 "go.etcd.io/etcd/client/v3"
@@ -205,16 +205,36 @@ type Allocator interface {
}
// Allocators represents a set of `Allocator`s.
-type Allocators []Allocator
+type Allocators struct {
+ SepAutoInc bool
+ Allocs []Allocator
+}
// NewAllocators packs multiple `Allocator`s into Allocators.
-func NewAllocators(allocators ...Allocator) Allocators {
- return allocators
+func NewAllocators(sepAutoInc bool, allocators ...Allocator) Allocators {
+ return Allocators{
+ SepAutoInc: sepAutoInc,
+ Allocs: allocators,
+ }
+}
+
+// Append add an allocator to the allocators.
+func (all Allocators) Append(a Allocator) Allocators {
+ return Allocators{
+ SepAutoInc: all.SepAutoInc,
+ Allocs: append(all.Allocs, a),
+ }
}
// Get returns the Allocator according to the AllocatorType.
func (all Allocators) Get(allocType AllocatorType) Allocator {
- for _, a := range all {
+ if !all.SepAutoInc {
+ if allocType == AutoIncrementType {
+ allocType = RowIDAllocType
+ }
+ }
+
+ for _, a := range all.Allocs {
if a.GetType() == allocType {
return a
}
@@ -224,13 +244,16 @@ func (all Allocators) Get(allocType AllocatorType) Allocator {
// Filter filters all the allocators that match pred.
func (all Allocators) Filter(pred func(Allocator) bool) Allocators {
- var ret Allocators
- for _, a := range all {
+ var ret []Allocator
+ for _, a := range all.Allocs {
if pred(a) {
ret = append(ret, a)
}
}
- return ret
+ return Allocators{
+ SepAutoInc: all.SepAutoInc,
+ Allocs: ret,
+ }
}
type allocator struct {
@@ -262,11 +285,15 @@ func SetStep(s int64) {
// Base implements autoid.Allocator Base interface.
func (alloc *allocator) Base() int64 {
+ alloc.mu.Lock()
+ defer alloc.mu.Unlock()
return alloc.base
}
// End implements autoid.Allocator End interface.
func (alloc *allocator) End() int64 {
+ alloc.mu.Lock()
+ defer alloc.mu.Unlock()
return alloc.end
}
@@ -535,6 +562,11 @@ func NextStep(curStep int64, consumeDur time.Duration) int64 {
return res
}
+// MockForTest is exported for testing.
+// The actual implementation is in github.com/pingcap/tidb/autoid_service because of the
+// package circle depending issue.
+var MockForTest func(kv.Storage) autoid.AutoIDAllocClient
+
func newSinglePointAlloc(store kv.Storage, dbID, tblID int64, isUnsigned bool) *singlePointAlloc {
ebd, ok := store.(kv.EtcdBackend)
if !ok {
@@ -564,7 +596,7 @@ func newSinglePointAlloc(store kv.Storage, dbID, tblID int64, isUnsigned bool) *
spa.clientDiscover = clientDiscover{etcdCli: etcdCli}
} else {
spa.clientDiscover = clientDiscover{}
- spa.mu.AutoIDAllocClient = autoid.MockForTest(store)
+ spa.mu.AutoIDAllocClient = MockForTest(store)
}
// mockAutoIDChange failpoint is not implemented in this allocator, so fallback to use the default one.
@@ -593,10 +625,17 @@ func NewAllocator(store kv.Storage, dbID, tbID int64, isUnsigned bool,
}
// Use the MySQL compatible AUTO_INCREMENT mode.
- if allocType == RowIDAllocType && alloc.customStep && alloc.step == 1 {
- alloc1 := newSinglePointAlloc(store, dbID, tbID, isUnsigned)
- if alloc1 != nil {
- return alloc1
+ if alloc.customStep && alloc.step == 1 && alloc.tbVersion >= model.TableInfoVersion5 {
+ if allocType == AutoIncrementType {
+ alloc1 := newSinglePointAlloc(store, dbID, tbID, isUnsigned)
+ if alloc1 != nil {
+ return alloc1
+ }
+ } else if allocType == RowIDAllocType {
+ // Now that the autoid and rowid allocator are separated, the AUTO_ID_CACHE 1 setting should not make
+ // the rowid allocator do not use cache.
+ alloc.customStep = false
+ alloc.step = step
}
}
@@ -630,6 +669,10 @@ func NewAllocatorsFromTblInfo(store kv.Storage, schemaID int64, tblInfo *model.T
alloc := NewAllocator(store, dbID, tblInfo.ID, tblInfo.IsAutoIncColUnsigned(), RowIDAllocType, idCacheOpt, tblVer)
allocs = append(allocs, alloc)
}
+ if hasAutoIncID {
+ alloc := NewAllocator(store, dbID, tblInfo.ID, tblInfo.IsAutoIncColUnsigned(), AutoIncrementType, idCacheOpt, tblVer)
+ allocs = append(allocs, alloc)
+ }
hasAutoRandID := tblInfo.ContainsAutoRandomBits()
if hasAutoRandID {
alloc := NewAllocator(store, dbID, tblInfo.ID, tblInfo.IsAutoRandomBitColUnsigned(), AutoRandomType, idCacheOpt, tblVer)
@@ -638,7 +681,7 @@ func NewAllocatorsFromTblInfo(store kv.Storage, schemaID int64, tblInfo *model.T
if tblInfo.IsSequence() {
allocs = append(allocs, NewSequenceAllocator(store, dbID, tblInfo.ID, tblInfo.Sequence))
}
- return NewAllocators(allocs...)
+ return NewAllocators(tblInfo.SepAutoInc(), allocs...)
}
// Alloc implements autoid.Allocator Alloc interface.
@@ -839,7 +882,7 @@ func (alloc *allocator) alloc4Signed(ctx context.Context, n uint64, increment, o
var newBase, newEnd int64
startTime := time.Now()
nextStep := alloc.step
- if !alloc.customStep {
+ if !alloc.customStep && alloc.end > 0 {
// Although it may skip a segment here, we still think it is consumed.
consumeDur := startTime.Sub(alloc.lastAllocTime)
nextStep = NextStep(alloc.step, consumeDur)
@@ -857,11 +900,7 @@ func (alloc *allocator) alloc4Signed(ctx context.Context, n uint64, increment, o
ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
err := kv.RunInNewTxn(ctx, alloc.store, true, func(ctx context.Context, txn kv.Transaction) error {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("alloc.alloc4Signed", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- opentracing.ContextWithSpan(ctx, span1)
- }
+ defer tracing.StartRegion(ctx, "alloc.alloc4Signed").End()
if allocatorStats != nil {
txn.SetOption(kv.CollectRuntimeStats, allocatorStats.SnapshotRuntimeStats)
}
@@ -945,13 +984,14 @@ func (alloc *allocator) alloc4Unsigned(ctx context.Context, n uint64, increment,
}()
}
+ if codeRun := ctx.Value("testIssue39528"); codeRun != nil {
+ *(codeRun.(*bool)) = true
+ return 0, 0, errors.New("mock error for test")
+ }
+
ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
err := kv.RunInNewTxn(ctx, alloc.store, true, func(ctx context.Context, txn kv.Transaction) error {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("alloc.alloc4Unsigned", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- opentracing.ContextWithSpan(ctx, span1)
- }
+ defer tracing.StartRegion(ctx, "alloc.alloc4Unsigned").End()
if allocatorStats != nil {
txn.SetOption(kv.CollectRuntimeStats, allocatorStats.SnapshotRuntimeStats)
}
diff --git a/meta/autoid/autoid_service.go b/meta/autoid/autoid_service.go
index 6133dfdfc3cb2..8e359f6e523fb 100644
--- a/meta/autoid/autoid_service.go
+++ b/meta/autoid/autoid_service.go
@@ -20,14 +20,16 @@ import (
"sync"
"time"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/autoid"
+ "github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/util/logutil"
+ "github.com/pingcap/tidb/util/tracing"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"google.golang.org/grpc"
+ "google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
)
@@ -77,13 +79,23 @@ func (d *clientDiscover) GetClient(ctx context.Context) (autoid.AutoIDAllocClien
if err != nil {
return nil, errors.Trace(err)
}
-
if len(resp.Kvs) == 0 {
return nil, errors.New("autoid service leader not found")
}
addr := string(resp.Kvs[0].Value)
- grpcConn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ opt := grpc.WithTransportCredentials(insecure.NewCredentials())
+ security := config.GetGlobalConfig().Security
+ if len(security.ClusterSSLCA) != 0 {
+ clusterSecurity := security.ClusterSecurity()
+ tlsConfig, err := clusterSecurity.ToTLSConfig()
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ opt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
+ }
+ logutil.BgLogger().Info("[autoid client] connect to leader", zap.String("addr", addr))
+ grpcConn, err := grpc.Dial(addr, opt)
if err != nil {
return nil, errors.Trace(err)
}
@@ -100,11 +112,8 @@ func (d *clientDiscover) GetClient(ctx context.Context) (autoid.AutoIDAllocClien
// case increment=1 & offset=1: you can derive the ids like min+1, min+2... max.
// case increment=x & offset=y: you firstly need to seek to firstID by `SeekToFirstAutoIDXXX`, then derive the IDs like firstID, firstID + increment * 2... in the caller.
func (sp *singlePointAlloc) Alloc(ctx context.Context, n uint64, increment, offset int64) (min int64, max int64, _ error) {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("autoid.Alloc", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "autoid.Alloc")
+ defer r.End()
if !validIncrementAndOffset(increment, offset) {
return 0, 0, errInvalidIncrementAndOffset.GenWithStackByArgs(increment, offset)
@@ -128,11 +137,14 @@ retry:
if err != nil {
if strings.Contains(err.Error(), "rpc error") {
time.Sleep(backoffDuration)
- sp.resetConn()
+ sp.resetConn(err)
goto retry
}
return 0, 0, errors.Trace(err)
}
+ if len(resp.Errmsg) != 0 {
+ return 0, 0, errors.Trace(errors.New(string(resp.Errmsg)))
+ }
du := time.Since(start)
metrics.AutoIDReqDuration.Observe(du.Seconds())
@@ -142,7 +154,9 @@ retry:
const backoffDuration = 200 * time.Millisecond
-func (sp *singlePointAlloc) resetConn() {
+func (sp *singlePointAlloc) resetConn(reason error) {
+ logutil.BgLogger().Info("[autoid client] reset grpc connection",
+ zap.String("reason", reason.Error()))
var grpcConn *grpc.ClientConn
sp.mu.Lock()
grpcConn = sp.mu.ClientConn
@@ -152,7 +166,9 @@ func (sp *singlePointAlloc) resetConn() {
// Close grpc.ClientConn to release resource.
if grpcConn != nil {
err := grpcConn.Close()
- logutil.BgLogger().Info("[autoid client] AllocAutoID grpc error, reconnect", zap.Error(err))
+ if err != nil {
+ logutil.BgLogger().Warn("[autoid client] close grpc connection error", zap.Error(err))
+ }
}
}
@@ -167,11 +183,8 @@ func (*singlePointAlloc) AllocSeqCache() (a int64, b int64, c int64, err error)
// If allocIDs is true, it will allocate some IDs and save to the cache.
// If allocIDs is false, it will not allocate IDs.
func (sp *singlePointAlloc) Rebase(ctx context.Context, newBase int64, _ bool) error {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("autoid.Rebase", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
+ r, ctx := tracing.StartRegionEx(ctx, "autoid.Rebase")
+ defer r.End()
return sp.rebase(ctx, newBase, false)
}
@@ -182,7 +195,8 @@ retry:
if err != nil {
return errors.Trace(err)
}
- _, err = cli.Rebase(ctx, &autoid.RebaseRequest{
+ var resp *autoid.RebaseResponse
+ resp, err = cli.Rebase(ctx, &autoid.RebaseRequest{
DbID: sp.dbID,
TblID: sp.tblID,
Base: newBase,
@@ -192,13 +206,16 @@ retry:
if err != nil {
if strings.Contains(err.Error(), "rpc error") {
time.Sleep(backoffDuration)
- sp.resetConn()
+ sp.resetConn(err)
goto retry
}
return errors.Trace(err)
}
+ if len(resp.Errmsg) != 0 {
+ return errors.Trace(errors.New(string(resp.Errmsg)))
+ }
sp.lastAllocated = newBase
- return err
+ return nil
}
// ForceRebase set the next global auto ID to newBase.
@@ -232,5 +249,5 @@ func (sp *singlePointAlloc) NextGlobalAutoID() (int64, error) {
}
func (*singlePointAlloc) GetType() AllocatorType {
- return RowIDAllocType
+ return AutoIncrementType
}
diff --git a/meta/autoid/autoid_test.go b/meta/autoid/autoid_test.go
index 0b8cd60257cf4..912b1e173bc8c 100644
--- a/meta/autoid/autoid_test.go
+++ b/meta/autoid/autoid_test.go
@@ -20,6 +20,7 @@ import (
"math"
"math/rand"
"sync"
+ "sync/atomic"
"testing"
"time"
@@ -594,3 +595,57 @@ func TestAllocComputationIssue(t *testing.T) {
require.Equal(t, int64(7), min)
require.Equal(t, int64(13), max)
}
+
+func TestIssue40584(t *testing.T) {
+ store, err := mockstore.NewMockStore()
+ require.NoError(t, err)
+ defer func() {
+ err := store.Close()
+ require.NoError(t, err)
+ }()
+
+ ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnMeta)
+ err = kv.RunInNewTxn(ctx, store, false, func(ctx context.Context, txn kv.Transaction) error {
+ m := meta.NewMeta(txn)
+ err = m.CreateDatabase(&model.DBInfo{ID: 1, Name: model.NewCIStr("a")})
+ require.NoError(t, err)
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 1, Name: model.NewCIStr("t")})
+ require.NoError(t, err)
+ return nil
+ })
+ require.NoError(t, err)
+
+ alloc := autoid.NewAllocator(store, 1, 1, false, autoid.RowIDAllocType)
+ require.NotNil(t, alloc)
+
+ finishAlloc := make(chan bool)
+ finishBase := make(chan bool)
+ var done int32 = 0
+
+ // call allocator.Alloc and allocator.Base in parallel for 3 seconds to detect data race
+ go func() {
+ for {
+ alloc.Alloc(ctx, 1, 1, 1)
+ if atomic.LoadInt32(&done) > 0 {
+ break
+ }
+ }
+ finishAlloc <- true
+ }()
+
+ go func() {
+ for {
+ alloc.Base()
+ if atomic.LoadInt32(&done) > 0 {
+ break
+ }
+ }
+ finishBase <- true
+ }()
+
+ runTime := time.NewTimer(time.Second * 3)
+ <-runTime.C
+ atomic.AddInt32(&done, 1)
+ <-finishAlloc
+ <-finishBase
+}
diff --git a/meta/meta.go b/meta/meta.go
index ad2f5d02e0d05..af02ae7882ac3 100644
--- a/meta/meta.go
+++ b/meta/meta.go
@@ -19,7 +19,6 @@ import (
"encoding/binary"
"encoding/json"
"fmt"
- "math"
"strconv"
"strings"
"sync"
@@ -34,8 +33,6 @@ import (
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/structure"
"github.com/pingcap/tidb/util/dbterror"
- "github.com/pingcap/tidb/util/logutil"
- "go.uber.org/zap"
)
var (
@@ -59,25 +56,27 @@ var (
//
var (
- mMetaPrefix = []byte("m")
- mNextGlobalIDKey = []byte("NextGlobalID")
- mSchemaVersionKey = []byte("SchemaVersionKey")
- mDBs = []byte("DBs")
- mDBPrefix = "DB"
- mTablePrefix = "Table"
- mSequencePrefix = "SID"
- mSeqCyclePrefix = "SequenceCycle"
- mTableIDPrefix = "TID"
- mIncIDPrefix = "IID"
- mRandomIDPrefix = "TARID"
- mBootstrapKey = []byte("BootstrapKey")
- mSchemaDiffPrefix = "Diff"
- mPolicies = []byte("Policies")
- mPolicyPrefix = "Policy"
- mPolicyGlobalID = []byte("PolicyGlobalID")
- mPolicyMagicByte = CurrentMagicByteVer
- mDDLTableVersion = []byte("DDLTableVersion")
- mConcurrentDDL = []byte("concurrentDDL")
+ mMetaPrefix = []byte("m")
+ mNextGlobalIDKey = []byte("NextGlobalID")
+ mSchemaVersionKey = []byte("SchemaVersionKey")
+ mDBs = []byte("DBs")
+ mDBPrefix = "DB"
+ mTablePrefix = "Table"
+ mSequencePrefix = "SID"
+ mSeqCyclePrefix = "SequenceCycle"
+ mTableIDPrefix = "TID"
+ mIncIDPrefix = "IID"
+ mRandomIDPrefix = "TARID"
+ mBootstrapKey = []byte("BootstrapKey")
+ mSchemaDiffPrefix = "Diff"
+ mPolicies = []byte("Policies")
+ mPolicyPrefix = "Policy"
+ mResourceGroups = []byte("ResourceGroups")
+ mResourceGroupPrefix = "RG"
+ mPolicyGlobalID = []byte("PolicyGlobalID")
+ mPolicyMagicByte = CurrentMagicByteVer
+ mDDLTableVersion = []byte("DDLTableVersion")
+ mMetaDataLock = []byte("metadataLock")
)
const (
@@ -109,6 +108,10 @@ var (
ErrPolicyExists = dbterror.ClassMeta.NewStd(errno.ErrPlacementPolicyExists)
// ErrPolicyNotExists is the error for policy not exists.
ErrPolicyNotExists = dbterror.ClassMeta.NewStd(errno.ErrPlacementPolicyNotExists)
+ // ErrResourceGroupExists is the error for resource group exists.
+ ErrResourceGroupExists = dbterror.ClassMeta.NewStd(errno.ErrResourceGroupExists)
+ // ErrResourceGroupNotExists is the error for resource group not exists.
+ ErrResourceGroupNotExists = dbterror.ClassMeta.NewStd(errno.ErrResourceGroupNotExists)
// ErrTableExists is the error for table exists.
ErrTableExists = dbterror.ClassMeta.NewStd(mysql.ErrTableExists)
// ErrTableNotExists is the error for table not exists.
@@ -119,6 +122,25 @@ var (
ErrInvalidString = dbterror.ClassMeta.NewStd(errno.ErrInvalidCharacterString)
)
+// DDLTableVersion is to display ddl related table versions
+type DDLTableVersion int
+
+const (
+ // InitDDLTableVersion is the original version.
+ InitDDLTableVersion DDLTableVersion = 0
+ // BaseDDLTableVersion is for support concurrent DDL, it added tidb_ddl_job, tidb_ddl_reorg and tidb_ddl_history.
+ BaseDDLTableVersion DDLTableVersion = 1
+ // MDLTableVersion is for support MDL tables.
+ MDLTableVersion DDLTableVersion = 2
+ // BackfillTableVersion is for support distributed reorg stage, it added tidb_background_subtask, tidb_background_subtask_history.
+ BackfillTableVersion DDLTableVersion = 3
+)
+
+// Bytes returns the byte slice.
+func (ver DDLTableVersion) Bytes() []byte {
+ return []byte(strconv.Itoa(int(ver)))
+}
+
// Meta is for handling meta information in a transaction.
type Meta struct {
txn *structure.TxStructure
@@ -128,17 +150,13 @@ type Meta struct {
// NewMeta creates a Meta in transaction txn.
// If the current Meta needs to handle a job, jobListKey is the type of the job's list.
-func NewMeta(txn kv.Transaction, jobListKeys ...JobListKeyType) *Meta {
+func NewMeta(txn kv.Transaction) *Meta {
txn.SetOption(kv.Priority, kv.PriorityHigh)
txn.SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
t := structure.NewStructure(txn, txn, mMetaPrefix)
- listKey := DefaultJobListKey
- if len(jobListKeys) != 0 {
- listKey = jobListKeys[0]
- }
return &Meta{txn: t,
StartTS: txn.StartTS(),
- jobListKey: listKey,
+ jobListKey: DefaultJobListKey,
}
}
@@ -224,6 +242,10 @@ func (*Meta) policyKey(policyID int64) []byte {
return []byte(fmt.Sprintf("%s:%d", mPolicyPrefix, policyID))
}
+func (*Meta) resourceGroupKey(groupID int64) []byte {
+ return []byte(fmt.Sprintf("%s:%d", mResourceGroupPrefix, groupID))
+}
+
func (*Meta) dbKey(dbID int64) []byte {
return DBkey(dbID)
}
@@ -436,6 +458,22 @@ func (m *Meta) checkPolicyNotExists(policyKey []byte) error {
return errors.Trace(err)
}
+func (m *Meta) checkResourceGroupNotExists(groupKey []byte) error {
+ v, err := m.txn.HGet(mResourceGroups, groupKey)
+ if err == nil && v != nil {
+ err = ErrResourceGroupExists.GenWithStack("group already exists")
+ }
+ return errors.Trace(err)
+}
+
+func (m *Meta) checkResourceGroupExists(groupKey []byte) error {
+ v, err := m.txn.HGet(mResourceGroups, groupKey)
+ if err == nil && v == nil {
+ err = ErrResourceGroupNotExists.GenWithStack("group doesn't exist")
+ }
+ return errors.Trace(err)
+}
+
func (m *Meta) checkDBExists(dbKey []byte) error {
v, err := m.txn.HGet(mDBs, dbKey)
if err == nil && v == nil {
@@ -501,6 +539,47 @@ func (m *Meta) UpdatePolicy(policy *model.PolicyInfo) error {
return m.txn.HSet(mPolicies, policyKey, attachMagicByte(data))
}
+// AddResourceGroup creates a resource group.
+func (m *Meta) AddResourceGroup(group *model.ResourceGroupInfo) error {
+ if group.ID == 0 {
+ return errors.New("group.ID is invalid")
+ }
+ groupKey := m.resourceGroupKey(group.ID)
+ if err := m.checkResourceGroupNotExists(groupKey); err != nil {
+ return errors.Trace(err)
+ }
+
+ data, err := json.Marshal(group)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ return m.txn.HSet(mResourceGroups, groupKey, attachMagicByte(data))
+}
+
+// UpdateResourceGroup updates a resource group.
+func (m *Meta) UpdateResourceGroup(group *model.ResourceGroupInfo) error {
+ groupKey := m.resourceGroupKey(group.ID)
+ if err := m.checkResourceGroupExists(groupKey); err != nil {
+ return errors.Trace(err)
+ }
+
+ data, err := json.Marshal(group)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ return m.txn.HSet(mResourceGroups, groupKey, attachMagicByte(data))
+}
+
+// DropResourceGroup drops a resource group.
+func (m *Meta) DropResourceGroup(groupID int64) error {
+ // Check if group exists.
+ groupKey := m.resourceGroupKey(groupID)
+ if err := m.txn.HDel(mResourceGroups, groupKey); err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
// CreateDatabase creates a database with db info.
func (m *Meta) CreateDatabase(dbInfo *model.DBInfo) error {
dbKey := m.dbKey(dbInfo.ID)
@@ -556,15 +635,25 @@ func (m *Meta) CreateTableOrView(dbID int64, tableInfo *model.TableInfo) error {
}
// SetDDLTables write a key into storage.
-func (m *Meta) SetDDLTables() error {
- err := m.txn.Set(mDDLTableVersion, []byte("1"))
+func (m *Meta) SetDDLTables(ddlTableVersion DDLTableVersion) error {
+ err := m.txn.Set(mDDLTableVersion, ddlTableVersion.Bytes())
return errors.Trace(err)
}
-// SetMDLTables write a key into storage.
-func (m *Meta) SetMDLTables() error {
- err := m.txn.Set(mDDLTableVersion, []byte("2"))
- return errors.Trace(err)
+// CheckDDLTableVersion check if the tables related to concurrent DDL exists.
+func (m *Meta) CheckDDLTableVersion() (DDLTableVersion, error) {
+ v, err := m.txn.Get(mDDLTableVersion)
+ if err != nil {
+ return -1, errors.Trace(err)
+ }
+ if string(v) == "" {
+ return InitDDLTableVersion, nil
+ }
+ ver, err := strconv.Atoi(string(v))
+ if err != nil {
+ return -1, errors.Trace(err)
+ }
+ return DDLTableVersion(ver), nil
}
// CreateMySQLDatabaseIfNotExists creates mysql schema and return its DB ID.
@@ -603,43 +692,27 @@ func (m *Meta) GetSystemDBID() (int64, error) {
return 0, nil
}
-// CheckDDLTableExists check if the tables related to concurrent DDL exists.
-func (m *Meta) CheckDDLTableExists() (bool, error) {
- v, err := m.txn.Get(mDDLTableVersion)
- if err != nil {
- return false, errors.Trace(err)
- }
- return len(v) != 0, nil
-}
-
-// CheckMDLTableExists check if the tables related to concurrent DDL exists.
-func (m *Meta) CheckMDLTableExists() (bool, error) {
- v, err := m.txn.Get(mDDLTableVersion)
- if err != nil {
- return false, errors.Trace(err)
- }
- return bytes.Equal(v, []byte("2")), nil
-}
-
-// SetConcurrentDDL set the concurrent DDL flag.
-func (m *Meta) SetConcurrentDDL(b bool) error {
+// SetMetadataLock sets the metadata lock.
+func (m *Meta) SetMetadataLock(b bool) error {
var data []byte
if b {
data = []byte("1")
} else {
data = []byte("0")
}
- return errors.Trace(m.txn.Set(mConcurrentDDL, data))
+ return errors.Trace(m.txn.Set(mMetaDataLock, data))
}
-// IsConcurrentDDL returns true if the concurrent DDL flag is set.
-func (m *Meta) IsConcurrentDDL() (bool, error) {
- val, err := m.txn.Get(mConcurrentDDL)
+// GetMetadataLock gets the metadata lock.
+func (m *Meta) GetMetadataLock() (enable bool, isNull bool, err error) {
+ val, err := m.txn.Get(mMetaDataLock)
if err != nil {
- return false, errors.Trace(err)
+ return false, false, errors.Trace(err)
}
-
- return len(val) == 0 || bytes.Equal(val, []byte("1")), nil
+ if len(val) == 0 {
+ return false, true, nil
+ }
+ return bytes.Equal(val, []byte("1")), false, nil
}
// CreateTableAndSetAutoID creates a table with tableInfo in database,
@@ -884,6 +957,50 @@ func (m *Meta) GetPolicy(policyID int64) (*model.PolicyInfo, error) {
return policy, errors.Trace(err)
}
+// ListResourceGroups shows all resource groups.
+func (m *Meta) ListResourceGroups() ([]*model.ResourceGroupInfo, error) {
+ res, err := m.txn.HGetAll(mResourceGroups)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ groups := make([]*model.ResourceGroupInfo, 0, len(res))
+ for _, r := range res {
+ value, err := detachMagicByte(r.Value)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ group := &model.ResourceGroupInfo{}
+ err = json.Unmarshal(value, group)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ groups = append(groups, group)
+ }
+ return groups, nil
+}
+
+// GetResourceGroup gets the database value with ID.
+func (m *Meta) GetResourceGroup(groupID int64) (*model.ResourceGroupInfo, error) {
+ groupKey := m.resourceGroupKey(groupID)
+ value, err := m.txn.HGet(mResourceGroups, groupKey)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if value == nil {
+ return nil, ErrResourceGroupNotExists.GenWithStack("resource group id : %d doesn't exist", groupID)
+ }
+
+ value, err = detachMagicByte(value)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ group := &model.ResourceGroupInfo{}
+ err = json.Unmarshal(value, group)
+ return group, errors.Trace(err)
+}
+
func attachMagicByte(data []byte) []byte {
data = append(data, 0)
copy(data[1:], data)
@@ -930,6 +1047,27 @@ func (m *Meta) GetTable(dbID int64, tableID int64) (*model.TableInfo, error) {
return tableInfo, errors.Trace(err)
}
+// CheckTableExists checks if the table is existed with dbID and tableID.
+func (m *Meta) CheckTableExists(dbID int64, tableID int64) (bool, error) {
+ // Check if db exists.
+ dbKey := m.dbKey(dbID)
+ if err := m.checkDBExists(dbKey); err != nil {
+ return false, errors.Trace(err)
+ }
+
+ // Check if table exists.
+ tableKey := m.tableKey(tableID)
+ v, err := m.txn.HGet(dbKey, tableKey)
+ if err != nil {
+ return false, errors.Trace(err)
+ }
+ if v != nil {
+ return true, nil
+ }
+
+ return false, nil
+}
+
// DDL job structure
// DDLJobList: list jobs
// DDLJobHistory: hash
@@ -942,12 +1080,8 @@ var (
mDDLJobListKey = []byte("DDLJobList")
mDDLJobAddIdxList = []byte("DDLJobAddIdxList")
mDDLJobHistoryKey = []byte("DDLJobHistory")
- mDDLJobReorgKey = []byte("DDLJobReorg")
)
-// JobListKeyType is a key type of the DDL job queue.
-type JobListKeyType []byte
-
var (
// DefaultJobListKey keeps all actions of DDL jobs except "add index".
DefaultJobListKey JobListKeyType = mDDLJobListKey
@@ -973,31 +1107,8 @@ func (m *Meta) EnQueueDDLJob(job *model.Job, jobListKeys ...JobListKeyType) erro
return m.enQueueDDLJob(listKey, job, true)
}
-// EnQueueDDLJobNoUpdate adds a DDL job to the list without update raw args.
-func (m *Meta) EnQueueDDLJobNoUpdate(job *model.Job, jobListKeys ...JobListKeyType) error {
- listKey := m.jobListKey
- if len(jobListKeys) != 0 {
- listKey = jobListKeys[0]
- }
-
- return m.enQueueDDLJob(listKey, job, false)
-}
-
-func (m *Meta) deQueueDDLJob(key []byte) (*model.Job, error) {
- value, err := m.txn.LPop(key)
- if err != nil || value == nil {
- return nil, errors.Trace(err)
- }
-
- job := &model.Job{}
- err = job.Decode(value)
- return job, errors.Trace(err)
-}
-
-// DeQueueDDLJob pops a DDL job from the list.
-func (m *Meta) DeQueueDDLJob() (*model.Job, error) {
- return m.deQueueDDLJob(m.jobListKey)
-}
+// JobListKeyType is a key type of the DDL job queue.
+type JobListKeyType []byte
func (m *Meta) getDDLJob(key []byte, index int64) (*model.Job, error) {
value, err := m.txn.LIndex(key, index)
@@ -1018,61 +1129,6 @@ func (m *Meta) getDDLJob(key []byte, index int64) (*model.Job, error) {
return job, errors.Trace(err)
}
-// GetDDLJobByIdx returns the corresponding DDL job by the index.
-// The length of jobListKeys can only be 1 or 0.
-// If its length is 1, we need to replace m.jobListKey with jobListKeys[0].
-// Otherwise, we use m.jobListKey directly.
-func (m *Meta) GetDDLJobByIdx(index int64, jobListKeys ...JobListKeyType) (*model.Job, error) {
- listKey := m.jobListKey
- if len(jobListKeys) != 0 {
- listKey = jobListKeys[0]
- }
-
- startTime := time.Now()
- job, err := m.getDDLJob(listKey, index)
- metrics.MetaHistogram.WithLabelValues(metrics.GetDDLJobByIdx, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
- return job, errors.Trace(err)
-}
-
-// updateDDLJob updates the DDL job with index and key.
-// updateRawArgs is used to determine whether to update the raw args when encode the job.
-func (m *Meta) updateDDLJob(index int64, job *model.Job, key []byte, updateRawArgs bool) error {
- b, err := job.Encode(updateRawArgs)
- if err == nil {
- err = m.txn.LSet(key, index, b)
- }
- return errors.Trace(err)
-}
-
-// UpdateDDLJob updates the DDL job with index.
-// updateRawArgs is used to determine whether to update the raw args when encode the job.
-// The length of jobListKeys can only be 1 or 0.
-// If its length is 1, we need to replace m.jobListKey with jobListKeys[0].
-// Otherwise, we use m.jobListKey directly.
-func (m *Meta) UpdateDDLJob(index int64, job *model.Job, updateRawArgs bool, jobListKeys ...JobListKeyType) error {
- listKey := m.jobListKey
- if len(jobListKeys) != 0 {
- listKey = jobListKeys[0]
- }
-
- startTime := time.Now()
- err := m.updateDDLJob(index, job, listKey, updateRawArgs)
- metrics.MetaHistogram.WithLabelValues(metrics.UpdateDDLJob, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
- return errors.Trace(err)
-}
-
-// DDLJobQueueLen returns the DDL job queue length.
-// The length of jobListKeys can only be 1 or 0.
-// If its length is 1, we need to replace m.jobListKey with jobListKeys[0].
-// Otherwise, we use m.jobListKey directly.
-func (m *Meta) DDLJobQueueLen(jobListKeys ...JobListKeyType) (int64, error) {
- listKey := m.jobListKey
- if len(jobListKeys) != 0 {
- listKey = jobListKeys[0]
- }
- return m.txn.LLen(listKey)
-}
-
// GetAllDDLJobsInQueue gets all DDL Jobs in the current queue.
// The length of jobListKeys can only be 1 or 0.
// If its length is 1, we need to replace m.jobListKey with jobListKeys[0].
@@ -1107,45 +1163,6 @@ func (*Meta) jobIDKey(id int64) []byte {
return b
}
-func (m *Meta) reorgJobCurrentElement(id int64) []byte {
- b := make([]byte, 0, 12)
- b = append(b, m.jobIDKey(id)...)
- b = append(b, "_ele"...)
- return b
-}
-
-func (m *Meta) reorgJobStartHandle(id int64, element *Element) []byte {
- b := make([]byte, 0, 16+len(element.TypeKey))
- b = append(b, m.jobIDKey(id)...)
- b = append(b, element.TypeKey...)
- eID := make([]byte, 8)
- binary.BigEndian.PutUint64(eID, uint64(element.ID))
- b = append(b, eID...)
- return b
-}
-
-func (*Meta) reorgJobEndHandle(id int64, element *Element) []byte {
- b := make([]byte, 8, 25)
- binary.BigEndian.PutUint64(b, uint64(id))
- b = append(b, element.TypeKey...)
- eID := make([]byte, 8)
- binary.BigEndian.PutUint64(eID, uint64(element.ID))
- b = append(b, eID...)
- b = append(b, "_end"...)
- return b
-}
-
-func (*Meta) reorgJobPhysicalTableID(id int64, element *Element) []byte {
- b := make([]byte, 8, 25)
- binary.BigEndian.PutUint64(b, uint64(id))
- b = append(b, element.TypeKey...)
- eID := make([]byte, 8)
- binary.BigEndian.PutUint64(eID, uint64(element.ID))
- b = append(b, eID...)
- b = append(b, "_pid"...)
- return b
-}
-
func (m *Meta) addHistoryDDLJob(key []byte, job *model.Job, updateRawArgs bool) error {
b, err := job.Encode(updateRawArgs)
if err == nil {
@@ -1307,160 +1324,6 @@ func DecodeElement(b []byte) (*Element, error) {
return &Element{ID: int64(id), TypeKey: tp}, nil
}
-// UpdateDDLReorgStartHandle saves the job reorganization latest processed element and start handle for later resuming.
-func (m *Meta) UpdateDDLReorgStartHandle(job *model.Job, element *Element, startKey kv.Key) error {
- err := m.txn.HSet(mDDLJobReorgKey, m.reorgJobCurrentElement(job.ID), element.EncodeElement())
- if err != nil {
- return errors.Trace(err)
- }
- if startKey != nil {
- err = m.txn.HSet(mDDLJobReorgKey, m.reorgJobStartHandle(job.ID, element), startKey)
- if err != nil {
- return errors.Trace(err)
- }
- }
- return nil
-}
-
-// UpdateDDLReorgHandle saves the job reorganization latest processed information for later resuming.
-func (m *Meta) UpdateDDLReorgHandle(jobID int64, startKey, endKey kv.Key, physicalTableID int64, element *Element) error {
- err := m.txn.HSet(mDDLJobReorgKey, m.reorgJobCurrentElement(jobID), element.EncodeElement())
- if err != nil {
- return errors.Trace(err)
- }
- if startKey != nil {
- err = m.txn.HSet(mDDLJobReorgKey, m.reorgJobStartHandle(jobID, element), startKey)
- if err != nil {
- return errors.Trace(err)
- }
- }
- if endKey != nil {
- err = m.txn.HSet(mDDLJobReorgKey, m.reorgJobEndHandle(jobID, element), endKey)
- if err != nil {
- return errors.Trace(err)
- }
- }
- err = m.txn.HSet(mDDLJobReorgKey, m.reorgJobPhysicalTableID(jobID, element), []byte(strconv.FormatInt(physicalTableID, 10)))
- return errors.Trace(err)
-}
-
-// ClearAllDDLReorgHandle clears all reorganization related handles.
-func (m *Meta) ClearAllDDLReorgHandle() error {
- return m.txn.HClear(mDDLJobReorgKey)
-}
-
-// ClearALLDDLJob clears all DDL jobs.
-func (m *Meta) ClearALLDDLJob() error {
- if err := m.txn.LClear(mDDLJobAddIdxList); err != nil {
- return errors.Trace(err)
- }
- if err := m.txn.LClear(mDDLJobListKey); err != nil {
- return errors.Trace(err)
- }
- return nil
-}
-
-// ClearAllHistoryJob clears all history jobs. **IT IS VERY DANGEROUS**
-func (m *Meta) ClearAllHistoryJob() error {
- if err := m.txn.HClear(mDDLJobHistoryKey); err != nil {
- return errors.Trace(err)
- }
- return nil
-}
-
-// RemoveReorgElement removes the element of the reorganization information.
-func (m *Meta) RemoveReorgElement(job *model.Job) error {
- err := m.txn.HDel(mDDLJobReorgKey, m.reorgJobCurrentElement(job.ID))
- if err != nil {
- return errors.Trace(err)
- }
- return nil
-}
-
-// RemoveDDLReorgHandle removes the job reorganization related handles.
-func (m *Meta) RemoveDDLReorgHandle(job *model.Job, elements []*Element) error {
- if len(elements) == 0 {
- return nil
- }
-
- err := m.txn.HDel(mDDLJobReorgKey, m.reorgJobCurrentElement(job.ID))
- if err != nil {
- return errors.Trace(err)
- }
-
- for _, element := range elements {
- err = m.txn.HDel(mDDLJobReorgKey, m.reorgJobStartHandle(job.ID, element))
- if err != nil {
- return errors.Trace(err)
- }
- if err = m.txn.HDel(mDDLJobReorgKey, m.reorgJobEndHandle(job.ID, element)); err != nil {
- logutil.BgLogger().Warn("remove DDL reorg end handle", zap.Error(err))
- }
- if err = m.txn.HDel(mDDLJobReorgKey, m.reorgJobPhysicalTableID(job.ID, element)); err != nil {
- logutil.BgLogger().Warn("remove DDL reorg physical ID", zap.Error(err))
- }
- }
- return nil
-}
-
-// GetDDLReorgHandle gets the latest processed DDL reorganize position.
-func (m *Meta) GetDDLReorgHandle(job *model.Job) (element *Element, startKey, endKey kv.Key, physicalTableID int64, err error) {
- elementBytes, err := m.txn.HGet(mDDLJobReorgKey, m.reorgJobCurrentElement(job.ID))
- if err != nil {
- return nil, nil, nil, 0, errors.Trace(err)
- }
- if elementBytes == nil {
- return nil, nil, nil, 0, ErrDDLReorgElementNotExist
- }
- element, err = DecodeElement(elementBytes)
- if err != nil {
- return nil, nil, nil, 0, errors.Trace(err)
- }
-
- startKey, err = getReorgJobFieldHandle(m.txn, m.reorgJobStartHandle(job.ID, element))
- if err != nil {
- return nil, nil, nil, 0, errors.Trace(err)
- }
- endKey, err = getReorgJobFieldHandle(m.txn, m.reorgJobEndHandle(job.ID, element))
- if err != nil {
- return nil, nil, nil, 0, errors.Trace(err)
- }
-
- physicalTableID, err = m.txn.HGetInt64(mDDLJobReorgKey, m.reorgJobPhysicalTableID(job.ID, element))
- if err != nil {
- err = errors.Trace(err)
- return
- }
-
- // physicalTableID may be 0, because older version TiDB (without table partition) doesn't store them.
- // update them to table's in this case.
- if physicalTableID == 0 {
- if job.ReorgMeta != nil {
- endKey = kv.IntHandle(job.ReorgMeta.EndHandle).Encoded()
- } else {
- endKey = kv.IntHandle(math.MaxInt64).Encoded()
- }
- physicalTableID = job.TableID
- logutil.BgLogger().Warn("new TiDB binary running on old TiDB DDL reorg data",
- zap.Int64("partition ID", physicalTableID),
- zap.Stringer("startHandle", startKey),
- zap.Stringer("endHandle", endKey))
- }
- return
-}
-
-func getReorgJobFieldHandle(t *structure.TxStructure, reorgJobField []byte) (kv.Key, error) {
- bs, err := t.HGet(mDDLJobReorgKey, reorgJobField)
- if err != nil {
- return nil, errors.Trace(err)
- }
- keyNotFound := bs == nil
- if keyNotFound {
- return nil, nil
- }
- return bs, nil
-}
-
func (*Meta) schemaDiffKey(schemaVersion int64) []byte {
return []byte(fmt.Sprintf("%s:%d", mSchemaDiffPrefix, schemaVersion))
}
diff --git a/meta/meta_autoid.go b/meta/meta_autoid.go
index 18d384b2b25a7..5763aa268051a 100644
--- a/meta/meta_autoid.go
+++ b/meta/meta_autoid.go
@@ -102,7 +102,7 @@ type autoIDAccessors struct {
access autoIDAccessor
}
-const sepAutoIncVer = model.TableInfoVersion4 + 1
+const sepAutoIncVer = model.TableInfoVersion5
// Get implements the interface AutoIDAccessors.
func (a *autoIDAccessors) Get() (autoIDs AutoIDGroup, err error) {
diff --git a/meta/meta_test.go b/meta/meta_test.go
index 0529fdbcc5eda..4eda2126b3887 100644
--- a/meta/meta_test.go
+++ b/meta/meta_test.go
@@ -17,18 +17,15 @@ package meta_test
import (
"context"
"fmt"
- "math"
"strconv"
"testing"
"time"
"github.com/pingcap/errors"
- "github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/store/mockstore"
- "github.com/pingcap/tidb/testkit/testutil"
"github.com/pingcap/tidb/util"
"github.com/stretchr/testify/require"
)
@@ -104,6 +101,45 @@ func TestPlacementPolicy(t *testing.T) {
require.NoError(t, err)
}
+func TestResourceGroup(t *testing.T) {
+ store, err := mockstore.NewMockStore()
+ require.NoError(t, err)
+
+ defer func() {
+ require.NoError(t, store.Close())
+ }()
+
+ txn, err := store.Begin()
+ require.NoError(t, err)
+
+ // test the independent policy ID allocation.
+ m := meta.NewMeta(txn)
+
+ checkResourceGroup := func(ru uint64) {
+ rg, err := m.GetResourceGroup(1)
+ require.NoError(t, err)
+ require.Equal(t, rg.RURate, ru)
+ }
+
+ rg := &model.ResourceGroupInfo{
+ ID: 1,
+ Name: model.NewCIStr("aa"),
+ ResourceGroupSettings: &model.ResourceGroupSettings{
+ RURate: 100,
+ },
+ }
+ require.NoError(t, m.AddResourceGroup(rg))
+ checkResourceGroup(100)
+
+ rg.RURate = 200
+ require.NoError(t, m.UpdateResourceGroup(rg))
+ checkResourceGroup(200)
+
+ m.DropResourceGroup(1)
+ _, err = m.GetResourceGroup(1)
+ require.Error(t, err)
+}
+
func TestBackupAndRestoreAutoIDs(t *testing.T) {
store, err := mockstore.NewMockStore()
require.NoError(t, err)
@@ -252,10 +288,16 @@ func TestMeta(t *testing.T) {
table, err := m.GetTable(1, 1)
require.NoError(t, err)
require.Equal(t, tbInfo, table)
+ tblExist, err := m.CheckTableExists(1, 1)
+ require.NoError(t, err)
+ require.Equal(t, true, tblExist)
table, err = m.GetTable(1, 2)
require.NoError(t, err)
require.Nil(t, table)
+ tblExist, err = m.CheckTableExists(1, 2)
+ require.NoError(t, err)
+ require.Equal(t, false, tblExist)
tbInfo2 := &model.TableInfo{
ID: 2,
@@ -442,202 +484,6 @@ func TestElement(t *testing.T) {
require.EqualError(t, err, `invalid encoded element "_col_" length 5`)
}
-func TestDDL(t *testing.T) {
- testCases := []struct {
- desc string
- startHandle kv.Handle
- endHandle kv.Handle
- }{
- {
- "kv.IntHandle",
- kv.IntHandle(1),
- kv.IntHandle(2),
- },
- {
- "kv.CommonHandle",
- testutil.MustNewCommonHandle(t, "abc", 1222, "string"),
- testutil.MustNewCommonHandle(t, "dddd", 1222, "string"),
- },
- }
-
- for _, tc := range testCases {
- // copy iterator variable into a new variable, see issue #27779
- tc := tc
- t.Run(tc.desc, func(t *testing.T) {
- store, err := mockstore.NewMockStore()
- require.NoError(t, err)
- defer func() {
- err := store.Close()
- require.NoError(t, err)
- }()
-
- txn, err := store.Begin()
- require.NoError(t, err)
-
- m := meta.NewMeta(txn)
-
- job := &model.Job{ID: 1}
- err = m.EnQueueDDLJob(job)
- require.NoError(t, err)
- n, err := m.DDLJobQueueLen()
- require.NoError(t, err)
- require.Equal(t, int64(1), n)
-
- v, err := m.GetDDLJobByIdx(0)
- require.NoError(t, err)
- require.Equal(t, job, v)
- v, err = m.GetDDLJobByIdx(1)
- require.NoError(t, err)
- require.Nil(t, v)
-
- job.ID = 2
- err = m.UpdateDDLJob(0, job, true)
- require.NoError(t, err)
-
- element := &meta.Element{ID: 123, TypeKey: meta.IndexElementKey}
- // There are 3 meta key relate to index reorganization:
- // start_handle, end_handle and physical_table_id.
- // Only start_handle is initialized.
- err = m.UpdateDDLReorgStartHandle(job, element, kv.IntHandle(1).Encoded())
- require.NoError(t, err)
-
- // Since physical_table_id is uninitialized, we simulate older TiDB version that doesn't store them.
- // In this case GetDDLReorgHandle always return maxInt64 as end_handle.
- e, i, j, k, err := m.GetDDLReorgHandle(job)
- require.NoError(t, err)
- require.Equal(t, element, e)
- require.Equal(t, kv.Key(kv.IntHandle(1).Encoded()), i)
- require.Equal(t, kv.Key(kv.IntHandle(math.MaxInt64).Encoded()), j)
- require.Equal(t, int64(0), k)
-
- element = &meta.Element{ID: 222, TypeKey: meta.ColumnElementKey}
- err = m.UpdateDDLReorgHandle(job.ID, tc.startHandle.Encoded(), tc.endHandle.Encoded(), 3, element)
- require.NoError(t, err)
- element1 := &meta.Element{ID: 223, TypeKey: meta.IndexElementKey}
- err = m.UpdateDDLReorgHandle(job.ID, tc.startHandle.Encoded(), tc.endHandle.Encoded(), 3, element1)
- require.NoError(t, err)
-
- e, i, j, k, err = m.GetDDLReorgHandle(job)
- require.NoError(t, err)
- require.Equal(t, element1, e)
- require.Equal(t, kv.Key(tc.startHandle.Encoded()), i)
- require.Equal(t, kv.Key(tc.endHandle.Encoded()), j)
- require.Equal(t, int64(3), k)
-
- err = m.RemoveDDLReorgHandle(job, []*meta.Element{element, element1})
- require.NoError(t, err)
- e, i, j, k, err = m.GetDDLReorgHandle(job)
- require.True(t, meta.ErrDDLReorgElementNotExist.Equal(err))
- require.Nil(t, e)
- require.Nil(t, i)
- require.Nil(t, j)
- require.Equal(t, k, int64(0))
-
- // new TiDB binary running on old TiDB DDL reorg data.
- e, i, j, k, err = m.GetDDLReorgHandle(job)
- require.True(t, meta.ErrDDLReorgElementNotExist.Equal(err))
- require.Nil(t, e)
- require.Nil(t, i)
- require.Nil(t, j)
- require.Equal(t, k, int64(0))
-
- // Test GetDDLReorgHandle failed.
- _, _, _, _, err = m.GetDDLReorgHandle(job)
- require.True(t, meta.ErrDDLReorgElementNotExist.Equal(err))
-
- v, err = m.DeQueueDDLJob()
- require.NoError(t, err)
- require.Equal(t, job, v)
-
- err = m.AddHistoryDDLJob(job, true)
- require.NoError(t, err)
- v, err = m.GetHistoryDDLJob(2)
- require.NoError(t, err)
- require.Equal(t, job, v)
-
- // Add multiple history jobs.
- arg := "test arg"
- historyJob1 := &model.Job{ID: 1234}
- historyJob1.Args = append(job.Args, arg)
- err = m.AddHistoryDDLJob(historyJob1, true)
- require.NoError(t, err)
- historyJob2 := &model.Job{ID: 123}
- historyJob2.Args = append(job.Args, arg)
- err = m.AddHistoryDDLJob(historyJob2, false)
- require.NoError(t, err)
- all, err := ddl.GetAllHistoryDDLJobs(m)
- require.NoError(t, err)
- var lastID int64
- for _, job := range all {
- require.Greater(t, job.ID, lastID)
- lastID = job.ID
- arg1 := ""
- err := job.DecodeArgs(&arg1)
- require.NoError(t, err)
- if job.ID == historyJob1.ID {
- require.Equal(t, historyJob1.Args[0], *(job.Args[0].(*string)))
- } else {
- require.Len(t, job.Args, 0)
- }
- }
-
- // Test for get last N history ddl jobs.
- historyJobs, err := ddl.GetLastNHistoryDDLJobs(m, 2)
- require.NoError(t, err)
- require.Len(t, historyJobs, 2)
- require.Equal(t, int64(1234), historyJobs[0].ID)
- require.Equal(t, int64(123), historyJobs[1].ID)
-
- // Test GetAllDDLJobsInQueue.
- err = m.EnQueueDDLJob(job)
- require.NoError(t, err)
- job1 := &model.Job{ID: 2}
- err = m.EnQueueDDLJob(job1)
- require.NoError(t, err)
- jobs, err := m.GetAllDDLJobsInQueue()
- require.NoError(t, err)
- expectJobs := []*model.Job{job, job1}
- require.Equal(t, expectJobs, jobs)
-
- err = txn.Commit(context.Background())
- require.NoError(t, err)
- })
- }
-}
-
-func TestAddIndexJob(t *testing.T) {
- store, err := mockstore.NewMockStore()
- require.NoError(t, err)
- defer func() {
- err := store.Close()
- require.NoError(t, err)
- }()
-
- txn1, err := store.Begin()
- require.NoError(t, err)
-
- m := meta.NewMeta(txn1, meta.AddIndexJobListKey)
- job := &model.Job{ID: 1}
- err = m.EnQueueDDLJob(job)
- require.NoError(t, err)
- job.ID = 123
- err = m.UpdateDDLJob(0, job, true, meta.AddIndexJobListKey)
- require.NoError(t, err)
- v, err := m.GetDDLJobByIdx(0, meta.AddIndexJobListKey)
- require.NoError(t, err)
- require.Equal(t, job, v)
- l, err := m.DDLJobQueueLen(meta.AddIndexJobListKey)
- require.NoError(t, err)
- require.Equal(t, int64(1), l)
- jobs, err := m.GetAllDDLJobsInQueue(meta.AddIndexJobListKey)
- require.NoError(t, err)
- expectJobs := []*model.Job{job}
- require.Equal(t, expectJobs, jobs)
-
- err = txn1.Commit(context.Background())
- require.NoError(t, err)
-}
-
func BenchmarkGenGlobalIDs(b *testing.B) {
store, err := mockstore.NewMockStore()
require.NoError(b, err)
@@ -767,45 +613,6 @@ func TestSequenceKey(b *testing.T) {
require.Equal(b, tableID, id)
}
-func TestClearJob(t *testing.T) {
- store, err := mockstore.NewMockStore()
- require.NoError(t, err)
- defer func() {
- require.NoError(t, store.Close())
- }()
-
- txn, err := store.Begin()
- require.NoError(t, err)
-
- job1 := &model.Job{ID: 1, TableID: 1, Type: model.ActionAddColumn}
- job2 := &model.Job{ID: 2, TableID: 1, Type: model.ActionCreateTable}
- job3 := &model.Job{ID: 3, TableID: 2, Type: model.ActionDropColumn}
-
- m := meta.NewMeta(txn)
-
- require.NoError(t, m.EnQueueDDLJob(job1))
- require.NoError(t, m.EnQueueDDLJob(job2))
- require.NoError(t, m.EnQueueDDLJob(job3))
-
- require.NoError(t, m.AddHistoryDDLJob(job1, false))
- require.NoError(t, m.AddHistoryDDLJob(job2, false))
-
- jobs, err := m.GetAllDDLJobsInQueue()
- require.NoError(t, err)
- require.Len(t, jobs, 3)
- require.NoError(t, m.ClearALLDDLJob())
- jobs, err = m.GetAllDDLJobsInQueue()
- require.NoError(t, err)
- require.Len(t, jobs, 0)
-
- count, err := m.GetHistoryDDLCount()
- require.NoError(t, err)
- require.Equal(t, count, uint64(2))
-
- err = txn.Rollback()
- require.NoError(t, err)
-}
-
func TestCreateMySQLDatabase(t *testing.T) {
store, err := mockstore.NewMockStore()
require.NoError(t, err)
@@ -829,41 +636,3 @@ func TestCreateMySQLDatabase(t *testing.T) {
err = txn.Rollback()
require.NoError(t, err)
}
-
-func TestDDLTable(t *testing.T) {
- store, err := mockstore.NewMockStore()
- require.NoError(t, err)
- defer func() {
- require.NoError(t, store.Close())
- }()
-
- txn, err := store.Begin()
- require.NoError(t, err)
-
- m := meta.NewMeta(txn)
-
- exists, err := m.CheckDDLTableExists()
- require.NoError(t, err)
- require.False(t, exists)
-
- err = m.SetDDLTables()
- require.NoError(t, err)
-
- exists, err = m.CheckDDLTableExists()
- require.NoError(t, err)
- require.True(t, exists)
-
- err = m.SetConcurrentDDL(true)
- require.NoError(t, err)
- b, err := m.IsConcurrentDDL()
- require.NoError(t, err)
- require.True(t, b)
- err = m.SetConcurrentDDL(false)
- require.NoError(t, err)
- b, err = m.IsConcurrentDDL()
- require.NoError(t, err)
- require.False(t, b)
-
- err = txn.Rollback()
- require.NoError(t, err)
-}
diff --git a/metrics/BUILD.bazel b/metrics/BUILD.bazel
index ff37513801ede..c414e04d9c907 100644
--- a/metrics/BUILD.bazel
+++ b/metrics/BUILD.bazel
@@ -13,12 +13,14 @@ go_library(
"meta.go",
"metrics.go",
"owner.go",
+ "resourcemanager.go",
"server.go",
"session.go",
"sli.go",
"stats.go",
"telemetry.go",
"topsql.go",
+ "ttl.go",
],
importpath = "github.com/pingcap/tidb/metrics",
visibility = ["//visibility:public"],
diff --git a/metrics/ddl.go b/metrics/ddl.go
index 5efe636c8a038..c97077ed0aea0 100644
--- a/metrics/ddl.go
+++ b/metrics/ddl.go
@@ -147,6 +147,8 @@ const (
LblAddIndex = "add_index"
LblAddIndexMerge = "add_index_merge_tmp"
LblModifyColumn = "modify_column"
+
+ LblReorgPartition = "reorganize_partition"
)
// GenerateReorgLabel returns the label with schema name and table name.
diff --git a/metrics/grafana/overview.json b/metrics/grafana/overview.json
index 3ee88ac3ae4c8..88a3b8acf82dd 100644
--- a/metrics/grafana/overview.json
+++ b/metrics/grafana/overview.json
@@ -983,6 +983,214 @@
"transform": "timeseries_aggregations",
"type": "table-old"
},
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 9
+ },
+ "hiddenSeries": false,
+ "id": 200,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}[30s])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "CPU Usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 9
+ },
+ "hiddenSeries": false,
+ "id": 201,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "process-{{job}}-{{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "HeapSys-{{job}}-{{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Memory Usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
{
"aliasColors": {},
"bars": false,
diff --git a/metrics/grafana/performance_overview.json b/metrics/grafana/performance_overview.json
index 19f526e562280..5ee670584f99e 100644
--- a/metrics/grafana/performance_overview.json
+++ b/metrics/grafana/performance_overview.json
@@ -57,8 +57,8 @@
"editable": true,
"gnetId": null,
"graphTooltip": 1,
- "id": null,
- "iteration": 1577357354898,
+ "id": 31,
+ "iteration": 1669018858346,
"links": [],
"panels": [
{
@@ -479,6 +479,11 @@
"lines": true,
"linewidth": 2,
"stack": false
+ },
+ {
+ "$$hashKey": "object:321",
+ "alias": "tiflash_mpp",
+ "color": "#8F3BB8"
}
],
"spaceLength": 10,
@@ -508,6 +513,14 @@
"interval": "",
"legendFormat": "execute time",
"refId": "F"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tiflash_coprocessor_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) ",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "tiflash_mpp",
+ "refId": "A"
}
],
"thresholds": [],
@@ -1048,12 +1061,12 @@
},
{
"aliasColors": {},
- "bars": false,
+ "bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"decimals": null,
- "description": "TiDB current connection counts",
+ "description": "kv request time by command source",
"editable": true,
"error": false,
"fill": 1,
@@ -1066,7 +1079,7 @@
"y": 15
},
"hiddenSeries": false,
- "id": 188,
+ "id": 23763571995,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -1082,7 +1095,7 @@
"total": false,
"values": true
},
- "lines": true,
+ "lines": false,
"linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
@@ -1094,57 +1107,49 @@
"pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:243",
+ "alias": "kv request total time",
+ "bars": false,
+ "color": "#FADE2A",
+ "lines": true,
+ "stack": false
+ }
+ ],
"spaceLength": 10,
- "stack": false,
+ "stack": true,
"steppedLine": false,
"targets": [
{
"exemplar": true,
- "expr": "tidb_server_connections{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}",
+ "expr": "sum(rate(tidb_tikvclient_request_time_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, source)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
- "legendFormat": "{{instance}}",
+ "legendFormat": "{{type}}-{{source}}",
"refId": "A",
"step": 30
},
{
"exemplar": true,
- "expr": "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"})",
+ "expr": "sum(rate(tidb_tikvclient_request_time_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))",
"hide": false,
"interval": "",
- "intervalFactor": 2,
- "legendFormat": "total",
+ "legendFormat": "kv request total time",
"refId": "B"
- },
- {
- "exemplar": true,
- "expr": "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"})",
- "hide": false,
- "interval": "",
- "legendFormat": "active connections",
- "refId": "D"
- },
- {
- "exemplar": true,
- "expr": "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}) by (instance)",
- "hide": true,
- "interval": "",
- "legendFormat": "ac-{{instance}}",
- "refId": "E"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Connection Count",
+ "title": "KV Request Time By Source",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
- "value_type": "individual"
+ "value_type": "cumulative"
},
"type": "graph",
"xaxis": {
@@ -1156,9 +1161,9 @@
},
"yaxes": [
{
- "$$hashKey": "object:3472",
+ "$$hashKey": "object:178",
"decimals": 0,
- "format": "short",
+ "format": "s",
"label": null,
"logBase": 1,
"max": null,
@@ -1166,7 +1171,7 @@
"show": true
},
{
- "$$hashKey": "object:3473",
+ "$$hashKey": "object:179",
"format": "short",
"label": null,
"logBase": 1,
@@ -1480,8 +1485,8 @@
"fill": 1,
"fillGradient": 0,
"gridPos": {
- "h": 7,
- "w": 12,
+ "h": 6,
+ "w": 8,
"x": 0,
"y": 29
},
@@ -1601,9 +1606,9 @@
"fillGradient": 0,
"grid": {},
"gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
+ "h": 6,
+ "w": 8,
+ "x": 8,
"y": 29
},
"hiddenSeries": false,
@@ -1720,6 +1725,148 @@
"alignLevel": null
}
},
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": null,
+ "description": "TiDB current connection counts",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 188,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:113",
+ "alias": "disconnection/s",
+ "color": "#C4162A",
+ "linewidth": 2,
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "tidb_server_connections{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}}",
+ "refId": "A",
+ "step": 30
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"})",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"})",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "active connections",
+ "refId": "D"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", result=\"ok\"}[1m]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "disconnection/s",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Connection Count",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:3472",
+ "decimals": 0,
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:3473",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
{
"aliasColors": {},
"bars": false,
@@ -1737,7 +1884,7 @@
"h": 8,
"w": 8,
"x": 0,
- "y": 36
+ "y": 35
},
"hiddenSeries": false,
"id": 156,
@@ -1861,7 +2008,7 @@
"h": 8,
"w": 8,
"x": 8,
- "y": 36
+ "y": 35
},
"hiddenSeries": false,
"id": 170,
@@ -1972,7 +2119,7 @@
"h": 8,
"w": 8,
"x": 16,
- "y": 36
+ "y": 35
},
"hiddenSeries": false,
"id": 169,
@@ -2097,7 +2244,7 @@
"h": 8,
"w": 8,
"x": 0,
- "y": 44
+ "y": 43
},
"hiddenSeries": false,
"id": 172,
@@ -2205,7 +2352,7 @@
"h": 8,
"w": 8,
"x": 8,
- "y": 44
+ "y": 43
},
"hiddenSeries": false,
"id": 173,
@@ -2312,7 +2459,7 @@
"h": 8,
"w": 8,
"x": 16,
- "y": 44
+ "y": 43
},
"hiddenSeries": false,
"id": 77,
@@ -2459,7 +2606,7 @@
"h": 8,
"w": 8,
"x": 0,
- "y": 52
+ "y": 51
},
"hiddenSeries": false,
"id": 185,
@@ -2573,7 +2720,7 @@
"h": 8,
"w": 8,
"x": 8,
- "y": 52
+ "y": 51
},
"hiddenSeries": false,
"id": 183,
@@ -2689,7 +2836,7 @@
"h": 8,
"w": 8,
"x": 16,
- "y": 52
+ "y": 51
},
"hiddenSeries": false,
"id": 174,
@@ -2805,7 +2952,7 @@
"h": 9,
"w": 8,
"x": 0,
- "y": 60
+ "y": 59
},
"hiddenSeries": false,
"id": 176,
@@ -2921,7 +3068,7 @@
"h": 9,
"w": 8,
"x": 8,
- "y": 60
+ "y": 59
},
"hiddenSeries": false,
"id": 177,
@@ -3037,7 +3184,7 @@
"h": 9,
"w": 8,
"x": 16,
- "y": 60
+ "y": 59
},
"hiddenSeries": false,
"id": 186,
@@ -3135,27 +3282,3153 @@
"align": false,
"alignLevel": null
}
+ },
+ {
+ "collapsed": true,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 68
+ },
+ "id": 159,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": 1,
+ "description": "The CPU usage of each TiFlash instance",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 2
+ },
+ "hiddenSeries": false,
+ "id": 161,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": null,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}[1m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}}",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "CPU",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": 1,
+ "description": "The memory usage per TiFlash instance",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 2
+ },
+ "hiddenSeries": false,
+ "id": 1709,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": null,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "avg(tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}}",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": 1,
+ "description": "The I/O utilization per TiFlash instance",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 2
+ },
+ "hiddenSeries": false,
+ "id": 165,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": null,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"tiflash.*\"}[1m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}} - {{device}}",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "IO utilization",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The MPP query count in TiFlash",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 9
+ },
+ "hiddenSeries": false,
+ "id": 157,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "max(tiflash_mpp_task_manager{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance, type)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}-{{type}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "MPP Query count",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The number of coprocessor requests received by all TiFlash instances. batch is the number of batch requests. batch_cop is the number of coprocessor requests in the batch requests. cop is the number of coprocessor requests that are sent directly via the coprocessor interface. cop_dag is the number of dag requests in all coprocessor requests. super_batch is the number of requests to enable the Super Batch feature.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 9
+ },
+ "hiddenSeries": false,
+ "id": 9,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Request QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The number of each type of dag executors in the requests received by all TiFlash instances. table_scan is the table scan executor. selection is the selection executor. aggregation is the aggregation executor. top_n is the TopN executor. limit is the limit executor.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 9
+ },
+ "hiddenSeries": false,
+ "id": 2,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tiflash_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Executor QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The overview of the total duration of all TiFlash instances processing coprocessor requests.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 16
+ },
+ "hiddenSeries": false,
+ "id": 166,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": false,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tiflash_coprocessor_request_duration_seconds_sum[1m])) by (type)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{type}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Request Duration Overview",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": " The total duration of all TiFlash instances processing coprocessor requests.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 16
+ },
+ "hiddenSeries": false,
+ "id": 11,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))",
+ "format": "time_series",
+ "hide": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "999",
+ "refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,type))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "99-{{type}}",
+ "refId": "B"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tiflash_coprocessor_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type) /sum(rate(tiflash_coprocessor_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "avg-{{type}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Request Duration",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The duration of all TiFlash instances processing coprocessor requests. The processing time is from starting to execute the coprocessor request to completing the execution.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 16
+ },
+ "hiddenSeries": false,
+ "id": 13,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))",
+ "format": "time_series",
+ "hide": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "999",
+ "refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,type))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "99-{{type}}",
+ "refId": "B"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tiflash_coprocessor_request_handle_seconds_sum[1m])) by (type) /sum(rate(tiflash_coprocessor_request_handle_seconds_count[1m])) by (type)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "avg-{{type}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Request Handle Duration",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The time used by wait_index for all TiFlash instances, namely the time used to wait until local index >= read_index after the read_index request is received.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 12,
+ "x": 0,
+ "y": 22
+ },
+ "hiddenSeries": false,
+ "id": 37,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/timeout/",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))",
+ "format": "time_series",
+ "hide": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "max",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "99",
+ "refId": "B"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tiflash_raft_wait_index_duration_seconds_sum[1m]))/sum(rate(tiflash_raft_wait_index_duration_seconds_count[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "avg",
+ "refId": "D"
+ },
+ {
+ "expr": "sum(increase(tiflash_system_profile_event_RaftWaitIndexTimeout{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}-timeout",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Raft Wait Index Duration",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "decimals": 2,
+ "format": "opm",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The number of times that each TiFlash instance triggers the read_index request per second, which equals to the number of Regions triggered.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 12,
+ "x": 12,
+ "y": 22
+ },
+ "hiddenSeries": false,
+ "id": 23763571994,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/timeout/",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))",
+ "format": "time_series",
+ "hide": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "max",
+ "refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "99",
+ "refId": "B"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tiflash_raft_read_index_duration_seconds_sum[1m]))/sum(rate(tiflash_raft_read_index_duration_seconds_count[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "avg",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Raft Batch Read Index Duration",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "decimals": 2,
+ "format": "opm",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": 1,
+ "description": "The throughput of write by instance",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 24,
+ "x": 0,
+ "y": 28
+ },
+ "height": "",
+ "hiddenSeries": false,
+ "id": 89,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": 250,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeatedByRow": true,
+ "seriesOverrides": [
+ {
+ "alias": "/total/",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"write\"}[1m])) by (instance)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "write-{{instance}}",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"ingest\"}[1m])) by (instance)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "ingest-{{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Write Throughput By Instance",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "binBps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": 1,
+ "description": "The flow of different kinds of write operations",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 34
+ },
+ "height": "",
+ "hiddenSeries": false,
+ "id": 60,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": null,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeatedByRow": true,
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "File Descriptor",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "sum(rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Page",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(tiflash_system_profile_event_PSMBackgroundWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "PageBackGround",
+ "refId": "C"
+ },
+ {
+ "expr": "sum(rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))",
+ "format": "time_series",
+ "hide": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "AIO",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Write flow",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "binBps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": 1,
+ "description": "The flow of different kinds of read operations",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 34
+ },
+ "height": "",
+ "hiddenSeries": false,
+ "id": 59,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": null,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeatedByRow": true,
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "File Descriptor",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Page",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "PageBackGround",
+ "refId": "C"
+ },
+ {
+ "expr": "sum(rate(tiflash_system_profile_event_ReadBufferAIOReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))",
+ "format": "time_series",
+ "hide": true,
+ "intervalFactor": 1,
+ "legendFormat": "AIO",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Read flow",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "binBps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "TiFlash",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 69
+ },
+ "id": 515,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "CPU usage of TiCDC",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 71
+ },
+ "hiddenSeries": false,
+ "id": 24,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:284",
+ "alias": "/.*MaxProcs/",
+ "fill": 0,
+ "linewidth": 2,
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ },
+ {
+ "expr": "ticdc_server_go_max_procs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}-MaxProcs",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "CPU usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:295",
+ "format": "percentunit",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:296",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "Memory usage of TiCDC",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 71
+ },
+ "hiddenSeries": false,
+ "id": 23,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": true,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "process-{{instance}}",
+ "refId": "A"
+ },
+ {
+ "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "heap-{{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "Goroutine count of TiCDC",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 71
+ },
+ "hiddenSeries": false,
+ "id": 26,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": " go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ },
+ {
+ "expr": "go_threads{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
+ "format": "time_series",
+ "hide": true,
+ "intervalFactor": 1,
+ "legendFormat": "threads-{{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Goroutine count",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 78
+ },
+ "hiddenSeries": false,
+ "id": 3,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (changefeed)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{changefeed}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "max(ticdc_processor_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance,changefeed)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}-{{changefeed}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Changefeed checkpoint lag",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:79",
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:80",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 78
+ },
+ "hiddenSeries": false,
+ "id": 513,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (changefeed)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{changefeed}}",
+ "refId": "C"
+ },
+ {
+ "exemplar": true,
+ "expr": "max(ticdc_processor_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance,changefeed)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}-{{chanefeed}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Changefeed resolved ts lag",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The status of each changefeed.\n\n0: Normal\n\n1: Error\n\n2: Failed\n\n3: Stopped\n\n4: Finished\n\n-1: Unknown",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 78
+ },
+ "hiddenSeries": false,
+ "id": 163,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 1,
+ "points": true,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "max(ticdc_owner_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (changefeed)",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{changefeed}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "The status of changefeeds",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The number of events that puller outputs to sorter \n per second",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 85
+ },
+ "hiddenSeries": false,
+ "id": 218,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type!=\"resolved\"}[1m])) by (changefeed, instance, type)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{changefeed}}-{{instance}}-{{type}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Puller output events/s",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The number of events that sorter outputs to puller \n per second",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 85
+ },
+ "hiddenSeries": false,
+ "id": 228,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (changefeed, instance, type)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{changefeed}}-{{instance}}-{{type}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Sorter output events/s",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The number of events that mounter outputs to sink per second",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 92
+ },
+ "hiddenSeries": false,
+ "id": 219,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance,changefeed)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{changefeed}}-{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Mounter output events/s",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:196",
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:197",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The number of events that table sorter outputs to sink per second",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 92
+ },
+ "hiddenSeries": false,
+ "id": 108,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (changefeed, instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{changefeed}}-{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Table sink output events/s",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The number of rows that sink flushes to downstream per second.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 99
+ },
+ "hiddenSeries": false,
+ "id": 654,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(ticdc_sinkv2_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (changefeed, instance)",
+ "interval": "",
+ "legendFormat": "{{changefeed}}-{{instance}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "SinkV2 - Sink flush rows/s",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:258",
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:259",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "Full flush (backend flush + callback + conflict detector notify) duration",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 99
+ },
+ "hiddenSeries": false,
+ "id": 620,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.999, sum(rate(ticdc_sinkv2_txn_worker_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (le,changefeed,instance))",
+ "interval": "",
+ "legendFormat": "99.9-{{changefeed}}-{{instance}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(ticdc_sinkv2_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (changefeed,instance) / \nsum(rate(ticdc_sinkv2_txn_worker_flush_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (changefeed,instance)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{changefeed}}-{{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Transaction Sink Full Flush Duration",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:332",
+ "format": "s",
+ "label": null,
+ "logBase": 2,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:333",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "MQ worker send messages to Kafka, this metric record the time cost on send every message.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 106
+ },
+ "hiddenSeries": false,
+ "id": 653,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.999, sum(rate(ticdc_sinkv2_mq_worker_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (le,changefeed,instance))",
+ "interval": "",
+ "legendFormat": "{{changefeed}}-{{instance}}-P999",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(ticdc_sinkv2_mq_worker_send_message_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (changefeed,instance) / \nsum(rate(ticdc_sinkv2_mq_worker_send_message_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (changefeed,instance)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{changefeed}}-{{instance}}-avg",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "MQ Worker Send Message Duration Percentile",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:406",
+ "format": "s",
+ "label": null,
+ "logBase": 2,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:407",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "Bytes/second written off all brokers.\nvalue = one-minute moving average rate of Bytes per second",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 106
+ },
+ "hiddenSeries": false,
+ "id": 628,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.0.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(ticdc_sinkv2_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}) by (changefeed, instance, broker)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{changefeed}}-{{instance}}-{{broker}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Kafka Outgoing Bytes",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:480",
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:481",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "CDC",
+ "type": "row"
}
],
"refresh": "30s",
- "schemaVersion": 18,
+ "schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allValue": null,
- "current": {},
+ "current": {
+ "isNone": true,
+ "selected": false,
+ "text": "None",
+ "value": ""
+ },
"datasource": "${DS_TEST-CLUSTER}",
+ "definition": "",
+ "description": null,
+ "error": null,
"hide": 2,
"includeAll": false,
"label": "K8s-cluster",
"multi": false,
"name": "k8s_cluster",
"options": [],
- "query": "label_values(pd_cluster_status, k8s_cluster)",
+ "query": {
+ "query": "label_values(pd_cluster_status, k8s_cluster)",
+ "refId": "${DS_TEST-CLUSTER}-k8s_cluster-Variable-Query"
+ },
"refresh": 2,
"regex": "",
+ "skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
@@ -3165,17 +6438,29 @@
},
{
"allValue": null,
- "current": {},
+ "current": {
+ "isNone": true,
+ "selected": false,
+ "text": "None",
+ "value": ""
+ },
"datasource": "${DS_TEST-CLUSTER}",
+ "definition": "",
+ "description": null,
+ "error": null,
"hide": 2,
"includeAll": false,
"label": "tidb_cluster",
"multi": false,
"name": "tidb_cluster",
"options": [],
- "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)",
+ "query": {
+ "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)",
+ "refId": "${DS_TEST-CLUSTER}-tidb_cluster-Variable-Query"
+ },
"refresh": 2,
"regex": "",
+ "skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json
index bd686ffd05c92..8c19fe9a56bf9 100644
--- a/metrics/grafana/tidb.json
+++ b/metrics/grafana/tidb.json
@@ -3706,7 +3706,7 @@
"avg": false,
"current": false,
"max": false,
- "min": false,
+ "min": false,
"show": true,
"total": false,
"values": false
@@ -3845,12 +3845,21 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, txn_mode)",
+ "expr": "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[1m])) by (type, txn_mode)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{type}}-{{txn_mode}}",
"refId": "A",
"step": 10
+ },
+ {
+ "expr": "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"internal\"}[1m])) by (type, txn_mode)",
+ "format": "time_series",
+ "hide": true,
+ "intervalFactor": 2,
+ "legendFormat": "Internal-{{type}}-{{txn_mode}}",
+ "refId": "B",
+ "step": 10
}
],
"thresholds": [],
@@ -3947,24 +3956,17 @@
"steppedLine": false,
"targets": [
{
- "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, txn_mode))",
+ "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[1m])) by (le, txn_mode))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "99-{{txn_mode}}",
"refId": "A"
},
{
- "expr": "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, txn_mode))",
+ "expr": "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", scope=~\"general\"}[1m])) by (txn_mode) / sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", scope=~\"general\"}[1m])) by (txn_mode)",
"format": "time_series",
"intervalFactor": 2,
- "legendFormat": "95-{{txn_mode}}",
- "refId": "B"
- },
- {
- "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, txn_mode))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "80-{{txn_mode}}",
+ "legendFormat": "avg-{{txn_mode}}",
"refId": "C"
}
],
@@ -4045,7 +4047,7 @@
"reverseYBuckets": false,
"targets": [
{
- "expr": "sum(increase(tidb_session_transaction_statement_num_bucket{instance=~\"$instance\"}[1m])) by (le)",
+ "expr": "sum(increase(tidb_session_transaction_statement_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)",
"format": "heatmap",
"intervalFactor": 2,
"legendFormat": "{{le}}",
@@ -4114,7 +4116,7 @@
"reverseYBuckets": false,
"targets": [
{
- "expr": "sum(increase(tidb_session_retry_num_bucket{instance=~\"$instance\"}[1m])) by (le)",
+ "expr": "sum(increase(tidb_session_retry_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)",
"format": "heatmap",
"intervalFactor": 2,
"legendFormat": "{{le}}",
@@ -6799,6 +6801,204 @@
"align": false,
"alignLevel": null
}
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "Counters of transactions or statements in which aggressive locking is enabled / takes effect.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 74
+ },
+ "hiddenSeries": false,
+ "id": 295,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(irate(tidb_session_transaction_aggressive_locking_usage{instance=~\"$instance\"}[30s])) by (type)",
+ "interval": "",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Aggressive Locking Usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "Counters of keys involved in aggressive locking.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 74
+ },
+ "hiddenSeries": false,
+ "id": 296,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(irate(tidb_tikvclient_aggressive_locking_count{instance=~\"$instance\"}[30s])) by (type)",
+ "interval": "",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Aggressive Locking Keys",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
}
],
"repeat": null,
@@ -12883,7 +13083,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "tidb-cluster",
+ "datasource": "${DS_TEST-CLUSTER}",
"description": "Speed of add index",
"editable": true,
"error": false,
@@ -14337,57 +14537,46 @@
"align": false,
"alignLevel": null
}
- }
- ],
- "repeat": null,
- "title": "Statistics",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": null,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 12
- },
- "id": 161,
- "panels": [
+ },
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
- "description": "TiDB new session durations for new etcd sessions",
- "editable": true,
- "error": false,
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
"fill": 1,
- "grid": {},
+ "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
- "y": 150
+ "y": 184
},
- "id": 162,
+ "hiddenSeries": false,
+ "id": 236,
"legend": {
- "alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
- "rightSide": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
- "linewidth": 2,
+ "linewidth": 1,
"links": [],
- "nullPointMode": "null as zero",
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"percentage": false,
+ "pluginVersion": "7.5.11",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -14397,20 +14586,260 @@
"steppedLine": false,
"targets": [
{
- "expr": "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance, result))",
+ "exemplar": true,
+ "expr": "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[1m])) by (result)",
"format": "time_series",
+ "interval": "",
"intervalFactor": 2,
- "legendFormat": "{{instance}}-{{result}}",
+ "legendFormat": "dump-task-{{result}}",
"refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
- "title": "New ETCD Session Duration 95",
- "tooltip": {
+ "step": 30
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"capture\"}[1m])) by (result)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "capture-task-{{result}}",
+ "refId": "B",
+ "step": 30
+ },
+ {
+ "exemplar": true,
+ "expr": "avg(tidb_plan_replayer_register_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "register-task",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Plan Replayer Task OPM",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 184
+ },
+ "hiddenSeries": false,
+ "id": 237,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"generate\"}[1m])) by (result)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "generate-{{result}}",
+ "refId": "A",
+ "step": 30
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[1m])) by (result)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "dump-{{result}}",
+ "refId": "B",
+ "step": 30
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Historical Stats OPM",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "repeat": null,
+ "title": "Statistics",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": null,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 12
+ },
+ "id": 161,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "TiDB new session durations for new etcd sessions",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 150
+ },
+ "id": 162,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, instance, result))",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}}-{{result}}",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "New ETCD Session Duration 95",
+ "tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
@@ -16728,11 +17157,11 @@
"panels": [
{
"aliasColors": {},
- "bars": true,
+ "bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
- "description": "kv request count by instance and command source",
+ "description": "TiDB cpu usage calculated with process cpu running seconds",
"editable": true,
"error": false,
"fieldConfig": {
@@ -16743,28 +17172,26 @@
"fillGradient": 0,
"grid": {},
"gridPos": {
- "h": 7,
+ "h": 8,
"w": 12,
"x": 0,
- "y": 15
+ "y": 20
},
"hiddenSeries": false,
- "id": 259,
+ "id": 297,
"legend": {
"alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
+ "avg": false,
+ "current": true,
+ "max": false,
"min": false,
"rightSide": true,
"show": true,
- "sort": "max",
- "sortDesc": true,
"total": false,
"values": true
},
- "lines": false,
- "linewidth": 2,
+ "lines": true,
+ "linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
"options": {
@@ -16777,46 +17204,43 @@
"renderer": "flot",
"seriesOverrides": [
{
- "alias": "KV Requst Count",
- "bars": false,
- "color": "#FADE2A",
- "lines": true,
- "linewidth": 1,
- "stack": false
- }
- ],
+ "alias": "total",
+ "fill": 0,
+ "lines": false
+ },
+ {
+ "alias": "/limit/",
+ "color": "#C4162A",
+ "fill": 0,
+ "nullPointMode": "null"
+ }
+ ],
"spaceLength": 10,
- "stack": true,
+ "stack": false,
"steppedLine": false,
"targets": [
{
- "exemplar": true,
- "expr": "sum(rate(tidb_tikvclient_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type, source)",
+ "expr": "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tidb\"}[30s])",
"format": "time_series",
- "interval": "",
+ "hide": false,
"intervalFactor": 1,
- "legendFormat": "{{instance}}-{{type}}-{{source}}",
+ "legendFormat": "{{instance}}",
"refId": "A",
"step": 40
},
{
- "exemplar": true,
- "expr": "sum(rate(tidb_tikvclient_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "KV Requst Count",
- "refId": "B",
- "step": 40
+ "expr": "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tidb\"}",
+ "legendFormat": "limit-{{instance}}",
+ "refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "KV Request OPS by source",
+ "title": "TiDB CPU Usage",
"tooltip": {
- "msResolution": true,
+ "msResolution": false,
"shared": true,
"sort": 0,
"value_type": "individual"
@@ -16831,8 +17255,7 @@
},
"yaxes": [
{
- "$$hashKey": "object:62",
- "format": "short",
+ "format": "percentunit",
"label": null,
"logBase": 1,
"max": null,
@@ -16840,7 +17263,6 @@
"show": true
},
{
- "$$hashKey": "object:63",
"format": "short",
"label": null,
"logBase": 1,
@@ -16856,11 +17278,12 @@
},
{
"aliasColors": {},
- "bars": true,
+ "bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
- "description": "kv request time by instance and command source",
+ "decimals": null,
+ "description": "IO MBps: The total bytes of read and write in all TiKV instances",
"editable": true,
"error": false,
"fieldConfig": {
@@ -16871,29 +17294,27 @@
"fillGradient": 0,
"grid": {},
"gridPos": {
- "h": 7,
+ "h": 8,
"w": 12,
"x": 12,
- "y": 15
+ "y": 20
},
"hiddenSeries": false,
- "id": 260,
+ "id": 301,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"hideEmpty": true,
"hideZero": true,
- "max": true,
+ "max": false,
"min": false,
"rightSide": true,
"show": true,
- "sort": "max",
- "sortDesc": true,
"total": false,
"values": true
},
- "lines": false,
+ "lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
@@ -16905,46 +17326,155 @@
"pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "KV Requst Time",
- "bars": false,
- "color": "#FADE2A",
- "lines": true,
- "linewidth": 1,
- "stack": false
- }
- ],
+ "seriesOverrides": [],
"spaceLength": 10,
- "stack": true,
+ "stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
- "expr": "sum(rate(tidb_tikvclient_request_time_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type, source)",
- "format": "time_series",
+ "expr": "avg(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance))",
+ "hide": false,
+ "instant": false,
"interval": "",
- "legendFormat": "{{instance}}-{{type}}-{{source}}",
- "refId": "A",
- "step": 40
+ "legendFormat": "IO-Avg",
+ "refId": "D"
},
{
"exemplar": true,
- "expr": "sum(rate(tidb_tikvclient_request_time_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))",
- "format": "time_series",
+ "expr": "max(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance))",
+ "hide": false,
"interval": "",
- "legendFormat": "KV Requst Time",
- "refId": "B",
- "step": 40
+ "legendFormat": "IO-Max",
+ "refId": "E"
+ },
+ {
+ "exemplar": true,
+ "expr": "max(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance)) - min(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "IO-Delta",
+ "refId": "F"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "KV Request Time by source",
+ "title": "TiKV IO MBps",
"tooltip": {
- "msResolution": true,
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:208",
+ "decimals": null,
+ "format": "Bps",
+ "label": "MBps",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:209",
+ "decimals": null,
+ "format": "Bps",
+ "label": "MBps ",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": 1,
+ "description": "The CPU usage of each TiKV instance",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 28
+ },
+ "hiddenSeries": false,
+ "id": 299,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": null,
+ "sort": "max",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=~\".*tikv\"}[1m])) by (instance)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}}",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TiKV CPU",
+ "tooltip": {
+ "msResolution": false,
"shared": true,
"sort": 0,
"value_type": "individual"
@@ -16959,8 +17489,7 @@
},
"yaxes": [
{
- "$$hashKey": "object:62",
- "format": "s",
+ "format": "percentunit",
"label": null,
"logBase": 1,
"max": null,
@@ -16968,7 +17497,1432 @@
"show": true
},
{
- "$$hashKey": "object:63",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The query count per second for each type of query in TTL jobs",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 28
+ },
+ "hiddenSeries": false,
+ "id": 279,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "delete ok",
+ "color": "#73BF69"
+ },
+ {
+ "alias": "select ok",
+ "color": "#5794F2"
+ },
+ {
+ "alias": "delete error",
+ "color": "#F2495C"
+ },
+ {
+ "alias": "select error",
+ "color": "#FF7383"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_server_ttl_query_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (sql_type, result)",
+ "interval": "",
+ "legendFormat": "{{sql_type}} {{result}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TTL QPS By Type",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 36
+ },
+ "hiddenSeries": false,
+ "id": 302,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))",
+ "interval": "",
+ "legendFormat": "insert rows per second",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TTL Insert Rows Per Second",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:394",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:395",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The processed rows per second by TTL jobs",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 36
+ },
+ "hiddenSeries": false,
+ "id": 287,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (sql_type, result)",
+ "interval": "",
+ "legendFormat": "{{sql_type}} {{result}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TTL Processed Rows Per Second",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1185",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1186",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 296,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1h]))",
+ "interval": "1h",
+ "legendFormat": "insert rows per hour",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TTL Insert Rows Per Hour",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:394",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:395",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The rows deleted per hour by TTL jobs",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 303,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1h])) by (sql_type, result)",
+ "interval": "1h",
+ "legendFormat": "delete rows per hour",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TTL Delete Rows Per Hour",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1185",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1186",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The duration of the TTL scan queries",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 52
+ },
+ "hiddenSeries": false,
+ "id": 284,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[1m])) by (le))",
+ "interval": "",
+ "legendFormat": "50",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[1m])) by (le))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "80",
+ "refId": "B"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[1m])) by (le))\n",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "90",
+ "refId": "C"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[1m])) by (le))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "99",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TTL Scan Query Duration",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The duration of the TTL delete queries",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 52
+ },
+ "hiddenSeries": false,
+ "id": 285,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1m])) by (le))",
+ "interval": "",
+ "legendFormat": "50",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1m])) by (le))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "80",
+ "refId": "B"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1m])) by (le))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "90",
+ "refId": "C"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1m])) by (le))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "99",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TTL Delete Query Duration",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The time spent on each phase for scan workers",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 60
+ },
+ "hiddenSeries": false,
+ "id": 276,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "idle",
+ "color": "#73BF69"
+ },
+ {
+ "alias": "query",
+ "color": "#FADE2A"
+ },
+ {
+ "alias": "begin_txn",
+ "color": "#FFA6B0"
+ },
+ {
+ "alias": "commit_txn",
+ "color": "#FF7383"
+ },
+ {
+ "alias": "wait_retry",
+ "color": "#FF9830"
+ },
+ {
+ "alias": "check_ttl",
+ "color": "#C4162A"
+ },
+ {
+ "alias": "dispatch",
+ "color": "#8F3BB8"
+ },
+ {
+ "alias": "wait_token",
+ "color": "#8AB8FF"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan_worker\"}[1m])) by (phase)",
+ "interval": "",
+ "legendFormat": "{{phase}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Scan Worker Time By Phase",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The time spent on each phase for delete workers",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 60
+ },
+ "hiddenSeries": false,
+ "id": 282,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "idle",
+ "color": "#73BF69"
+ },
+ {
+ "alias": "query",
+ "color": "#FADE2A"
+ },
+ {
+ "alias": "begin_txn",
+ "color": "#FFA6B0"
+ },
+ {
+ "alias": "commit_txn",
+ "color": "#FF7383"
+ },
+ {
+ "alias": "wait_retry",
+ "color": "#FF9830"
+ },
+ {
+ "alias": "check_ttl",
+ "color": "#C4162A"
+ },
+ {
+ "alias": "dispatch",
+ "color": "#8F3BB8"
+ },
+ {
+ "alias": "wait_token",
+ "color": "#8AB8FF"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"delete_worker\"}[1m])) by (phase)\n",
+ "interval": "",
+ "legendFormat": "{{phase}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Delete Worker Time By Phase",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The TTL job statuses in each worker",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 68
+ },
+ "hiddenSeries": false,
+ "id": 281,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "running",
+ "color": "#5794F2"
+ },
+ {
+ "alias": "cancelling",
+ "color": "#F2495C"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, instance)",
+ "interval": "",
+ "legendFormat": "{{ instance }} {{ type }}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TTL Job Count By Status",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The TTL task statuses in each worker",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 68
+ },
+ "hiddenSeries": false,
+ "id": 294,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "running",
+ "color": "#5794F2"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, instance)",
+ "interval": "",
+ "legendFormat": "{{ instance }} {{ type }}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TTL Task Count By Status",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "TTL",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": null,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 19
+ },
+ "id": 291,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 20
+ },
+ "hiddenSeries": false,
+ "id": 289,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "tidb_server_gogc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "GOGC",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "exponential moving average of CPU Usage",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 20
+ },
+ "hiddenSeries": false,
+ "id": 293,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.11",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "tidb_rm_ema_cpu_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "EMA CPU Usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
"format": "short",
"label": null,
"logBase": 1,
@@ -16983,7 +18937,7 @@
}
}
],
- "title": "SourceSQL",
+ "title": "Resource Manager",
"type": "row"
}
],
diff --git a/metrics/grafana/tidb_resource_control.json b/metrics/grafana/tidb_resource_control.json
new file mode 100644
index 0000000000000..a34e2c92e95ab
--- /dev/null
+++ b/metrics/grafana/tidb_resource_control.json
@@ -0,0 +1,2072 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_TEST-CLUSTER",
+ "label": "test-cluster",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "7.5.11"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "id": null,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": null,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 11,
+ "panels": [],
+ "title": "Resource Unit",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The total request unit cost for all resource groups.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 12,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name) + sum(rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) + sum(rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "RU",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "short",
+ "label": null,
+ "logBase": 10,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 13,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "(sum(rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name) + sum(rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "(sum(rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) + sum(rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "RU Per Query",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "short",
+ "label": null,
+ "logBase": 10,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The read request unit cost for all resource groups.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "hiddenSeries": false,
+ "id": 14,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "RRU",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "short",
+ "label": null,
+ "logBase": 10,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 8
+ },
+ "hiddenSeries": false,
+ "id": 15,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "RRU Per Query",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "short",
+ "label": null,
+ "logBase": 10,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "The write request unit cost for all resource groups.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 15
+ },
+ "hiddenSeries": false,
+ "id": 16,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "WRU",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "short",
+ "label": null,
+ "logBase": 10,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 15
+ },
+ "hiddenSeries": false,
+ "id": 17,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "WRU Per Query",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "short",
+ "label": null,
+ "logBase": 10,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": true,
+ "datasource": null,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 23
+ },
+ "id": 18,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 24
+ },
+ "hiddenSeries": false,
+ "id": 19,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name, type)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}-{{type}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}-total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "KV Request Count",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "short",
+ "label": null,
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 24
+ },
+ "hiddenSeries": false,
+ "id": 20,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"read\"}[30s])) by (name) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}-read",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"write\"}[30s])) by (name) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}-write",
+ "refId": "B",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"read\"}[30s])) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total-read",
+ "refId": "C",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"write\"}[30s])) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total-write",
+ "refId": "D",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "KV Request Count Per Query",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "short",
+ "label": null,
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 21,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_read_byte_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_read_byte_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Bytes Read",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "decimals": null,
+ "format": "bits",
+ "label": null,
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 22,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_read_byte_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_read_byte_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Bytes Read Per Query",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "decimals": null,
+ "format": "bits",
+ "label": null,
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 23,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_write_byte_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_write_byte_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Bytes Written",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "bits",
+ "label": null,
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 24,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_write_byte_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_write_byte_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Bytes Written Per Query",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "bits",
+ "label": null,
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 25,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_kv_cpu_time_ms_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_kv_cpu_time_ms_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "KV CPU Time",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "decimals": null,
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 26,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_kv_cpu_time_ms_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_kv_cpu_time_ms_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "KV CPU Time Per Query",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "decimals": null,
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 55
+ },
+ "hiddenSeries": false,
+ "id": 27,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_sql_cpu_time_ms_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_sql_cpu_time_ms_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "SQL CPU Time",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "bits",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 55
+ },
+ "hiddenSeries": false,
+ "id": 28,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.10",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_sql_cpu_time_ms_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (name)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 40
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(resource_manager_resource_sql_cpu_time_ms_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) / sum(rate(tidb_session_resource_group_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "B",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "SQL CPU Time Per Query",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:869",
+ "format": "bits",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:870",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Resource",
+ "type": "row"
+ }
+ ],
+ "refresh": "30s",
+ "schemaVersion": 18,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "allValue": null,
+ "current": {},
+ "datasource": "${DS_TEST-CLUSTER}",
+ "hide": 2,
+ "includeAll": false,
+ "label": "K8s-cluster",
+ "multi": false,
+ "name": "k8s_cluster",
+ "options": [],
+ "query": "label_values(pd_cluster_status, k8s_cluster)",
+ "refresh": 2,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {},
+ "datasource": "${DS_TEST-CLUSTER}",
+ "hide": 2,
+ "includeAll": false,
+ "label": "tidb_cluster",
+ "multi": false,
+ "name": "tidb_cluster",
+ "options": [],
+ "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)",
+ "refresh": 2,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "Test-Cluster-TiDB-Resource-Control",
+ "uid": "000000201",
+ "version": 1
+}
diff --git a/metrics/grafana/tidb_runtime.json b/metrics/grafana/tidb_runtime.json
index 0324f6851de98..d17b1d90a104e 100644
--- a/metrics/grafana/tidb_runtime.json
+++ b/metrics/grafana/tidb_runtime.json
@@ -1384,6 +1384,131 @@
"align": false,
"alignLevel": null
}
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "description": "the time goroutines have spent in the scheduler",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 57
+ },
+ "id": 30,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.999, sum(rate(go_sched_latencies_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "999",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "histogram_quantile(0.9999, sum(rate(go_sched_latencies_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "9999",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "expr": "histogram_quantile(0.99999, sum(rate(go_sched_latencies_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99999",
+ "refId": "C",
+ "step": 10
+ },
+ {
+ "expr": "histogram_quantile(0.999999, sum(rate(go_sched_latencies_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "999999",
+ "refId": "D",
+ "step": 10
+ },
+ {
+ "expr": "histogram_quantile(1, sum(rate(go_sched_latencies_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "max",
+ "refId": "E",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "goroutine scheduler lattency",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
}
],
"repeat": "instance",
diff --git a/metrics/log_backup.go b/metrics/log_backup.go
index 6c706b8027a79..767fe2e251d8a 100644
--- a/metrics/log_backup.go
+++ b/metrics/log_backup.go
@@ -60,4 +60,10 @@ var (
Name: "region_request_failure",
Help: "The failure reasons of requesting region checkpoints.",
}, []string{"reason"})
+ RegionCheckpointSubscriptionEvent = prometheus.NewCounterVec(prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "log_backup",
+ Name: "region_checkpoint_event",
+ Help: "The region flush event count.",
+ }, []string{"store"})
)
diff --git a/metrics/meta.go b/metrics/meta.go
index 519ba6a0924a1..af967fe48a3bb 100644
--- a/metrics/meta.go
+++ b/metrics/meta.go
@@ -34,8 +34,6 @@ var (
GetSchemaDiff = "get_schema_diff"
SetSchemaDiff = "set_schema_diff"
- GetDDLJobByIdx = "get_ddl_job"
- UpdateDDLJob = "update_ddl_job"
GetHistoryDDLJob = "get_history_ddl_job"
MetaHistogram = prometheus.NewHistogramVec(
diff --git a/metrics/metrics.go b/metrics/metrics.go
index a843e794cff1b..f63af624a3a0c 100644
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@@ -51,6 +51,7 @@ const (
LabelDDLOwner = "ddl-owner"
LabelDDL = "ddl"
LabelDDLWorker = "ddl-worker"
+ LabelDistReorg = "dist-reorg"
LabelDDLSyncer = "ddl-syncer"
LabelGCWorker = "gcworker"
LabelAnalyze = "analyze"
@@ -124,7 +125,6 @@ func RegisterMetrics() {
prometheus.MustRegister(SyncLoadHistogram)
prometheus.MustRegister(ReadStatsHistogram)
prometheus.MustRegister(JobsGauge)
- prometheus.MustRegister(KeepAliveCounter)
prometheus.MustRegister(LoadPrivilegeCounter)
prometheus.MustRegister(InfoCacheCounters)
prometheus.MustRegister(LoadSchemaCounter)
@@ -182,6 +182,7 @@ func RegisterMetrics() {
prometheus.MustRegister(TokenGauge)
prometheus.MustRegister(ConfigStatus)
prometheus.MustRegister(TiFlashQueryTotalCounter)
+ prometheus.MustRegister(TiFlashFailedMPPStoreState)
prometheus.MustRegister(SmallTxnWriteDuration)
prometheus.MustRegister(TxnWriteThroughput)
prometheus.MustRegister(LoadSysVarCacheCounter)
@@ -194,6 +195,8 @@ func RegisterMetrics() {
prometheus.MustRegister(ReadFromTableCacheCounter)
prometheus.MustRegister(LoadTableCacheDurationHistogram)
prometheus.MustRegister(NonTransactionalDMLCount)
+ prometheus.MustRegister(PessimisticDMLDurationByAttempt)
+ prometheus.MustRegister(ResourceGroupQueryTotalCounter)
prometheus.MustRegister(MemoryUsage)
prometheus.MustRegister(StatsCacheLRUCounter)
prometheus.MustRegister(StatsCacheLRUGauge)
@@ -207,7 +210,23 @@ func RegisterMetrics() {
prometheus.MustRegister(RegionCheckpointRequest)
prometheus.MustRegister(RegionCheckpointFailure)
prometheus.MustRegister(AutoIDReqDuration)
+ prometheus.MustRegister(RegionCheckpointSubscriptionEvent)
prometheus.MustRegister(RCCheckTSWriteConfilictCounter)
+ prometheus.MustRegister(AggressiveLockingUsageCount)
+
+ prometheus.MustRegister(TTLQueryDuration)
+ prometheus.MustRegister(TTLProcessedExpiredRowsCounter)
+ prometheus.MustRegister(TTLJobStatus)
+ prometheus.MustRegister(TTLTaskStatus)
+ prometheus.MustRegister(TTLPhaseTime)
+ prometheus.MustRegister(TTLInsertRowsCount)
+
+ prometheus.MustRegister(EMACPUUsageGauge)
+ prometheus.MustRegister(PoolConcurrencyCounter)
+
+ prometheus.MustRegister(HistoricalStatsCounter)
+ prometheus.MustRegister(PlanReplayerTaskCounter)
+ prometheus.MustRegister(PlanReplayerRegisterTaskGauge)
tikvmetrics.InitMetrics(TiDB, TiKVClient)
tikvmetrics.RegisterMetrics()
@@ -230,6 +249,7 @@ func ToggleSimplifiedMode(simplified bool) {
InfoCacheCounters,
ReadFromTableCacheCounter,
TiFlashQueryTotalCounter,
+ TiFlashFailedMPPStoreState,
CampaignOwnerCounter,
NonTransactionalDMLCount,
MemoryUsage,
diff --git a/metrics/resourcemanager.go b/metrics/resourcemanager.go
new file mode 100644
index 0000000000000..c6715512500d4
--- /dev/null
+++ b/metrics/resourcemanager.go
@@ -0,0 +1,35 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+ // EMACPUUsageGauge means exponential moving average of CPU usage
+ EMACPUUsageGauge = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "tidb",
+ Subsystem: "rm",
+ Name: "ema_cpu_usage",
+ Help: "exponential moving average of CPU usage",
+ })
+ // PoolConcurrencyCounter means how much concurrency in the pool
+ PoolConcurrencyCounter = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: "tidb",
+ Subsystem: "rm",
+ Name: "pool_concurrency",
+ Help: "How many concurrency in the pool",
+ }, []string{LblType})
+)
diff --git a/metrics/server.go b/metrics/server.go
index 116b02eb122b6..830e03b28e986 100644
--- a/metrics/server.go
+++ b/metrics/server.go
@@ -120,14 +120,6 @@ var (
Help: "Counter of system time jumps backward.",
})
- KeepAliveCounter = prometheus.NewCounter(
- prometheus.CounterOpts{
- Namespace: "tidb",
- Subsystem: "monitor",
- Name: "keep_alive_total",
- Help: "Counter of TiDB keep alive.",
- })
-
PlanCacheCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "tidb",
@@ -279,6 +271,14 @@ var (
Help: "Counter of TiFlash queries.",
}, []string{LblType, LblResult})
+ TiFlashFailedMPPStoreState = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: "tidb",
+ Subsystem: "server",
+ Name: "tiflash_failed_store",
+ Help: "Statues of failed tiflash mpp store,-1 means detector heartbeat,0 means reachable,1 means abnormal.",
+ }, []string{LblAddress})
+
PDAPIExecutionHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "tidb",
diff --git a/metrics/session.go b/metrics/session.go
index 11c3bf7a143b8..208092c11b087 100644
--- a/metrics/session.go
+++ b/metrics/session.go
@@ -59,14 +59,14 @@ var (
Name: "schema_lease_error_total",
Help: "Counter of schema lease error",
}, []string{LblType})
- SessionRetry = prometheus.NewHistogram(
+ SessionRetry = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "tidb",
Subsystem: "session",
Name: "retry_num",
Help: "Bucketed histogram of session retry count.",
Buckets: prometheus.LinearBuckets(0, 1, 21), // 0 ~ 20
- })
+ }, []string{LblScope})
SessionRetryErrorCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "tidb",
@@ -90,7 +90,7 @@ var (
Name: "transaction_statement_num",
Help: "Bucketed histogram of statements count in each transaction.",
Buckets: prometheus.ExponentialBuckets(1, 2, 16), // 1 ~ 32768
- }, []string{LblTxnMode, LblType})
+ }, []string{LblTxnMode, LblType, LblScope})
TransactionDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
@@ -99,7 +99,7 @@ var (
Name: "transaction_duration_seconds",
Help: "Bucketed histogram of a transaction execution duration, including retry.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days
- }, []string{LblTxnMode, LblType})
+ }, []string{LblTxnMode, LblType, LblScope})
StatementDeadlockDetectDuration = prometheus.NewHistogram(
prometheus.HistogramOpts{
@@ -169,6 +169,31 @@ var (
Help: "Counter of setting tidb_constraint_check_in_place to false, note that it doesn't count the default value set by tidb config",
},
)
+
+ PessimisticDMLDurationByAttempt = prometheus.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Namespace: "tidb",
+ Subsystem: "session",
+ Name: "transaction_pessimistic_dml_duration_by_attempt",
+ Help: "Bucketed histogram of duration of pessimistic DMLs, distinguished by first attempt and retries",
+ Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days
+ }, []string{LblType, LblPhase})
+
+ ResourceGroupQueryTotalCounter = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "session",
+ Name: "resource_group_query_total",
+ Help: "Counter of the total number of queries for the resource group",
+ }, []string{LblName})
+
+ AggressiveLockingUsageCount = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "server",
+ Name: "transaction_aggressive_locking_usage",
+ Help: "The counter of statements and transactions in which aggressive locking is used or takes effect",
+ }, []string{LblType})
)
// Label constants.
@@ -210,4 +235,12 @@ const (
LblModule = "module"
LblRCReadCheckTS = "read_check"
LblRCWriteCheckTS = "write_check"
+
+ LblName = "name"
+
+ LblAggressiveLockingTxnUsed = "txn-used"
+ LblAggressiveLockingTxnEffective = "txn-effective"
+ LblAggressiveLockingStmtUsed = "stmt-used"
+ LblAggressiveLockingStmtEffective = "stmt-effective"
+ LblScope = "scope"
)
diff --git a/metrics/stats.go b/metrics/stats.go
index 76bd1ec7a936b..5d73753f5669c 100644
--- a/metrics/stats.go
+++ b/metrics/stats.go
@@ -150,4 +150,25 @@ var (
Name: "stats_healthy",
Help: "Gauge of stats healthy",
}, []string{LblType})
+
+ HistoricalStatsCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "statistics",
+ Name: "historical_stats",
+ Help: "counter of the historical stats operation",
+ }, []string{LblType, LblResult})
+
+ PlanReplayerTaskCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "plan_replayer",
+ Name: "task",
+ Help: "counter of plan replayer captured task",
+ }, []string{LblType, LblResult})
+
+ PlanReplayerRegisterTaskGauge = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "tidb",
+ Subsystem: "plan_replayer",
+ Name: "register_task",
+ Help: "gauge of plan replayer registered task",
+ })
)
diff --git a/metrics/telemetry.go b/metrics/telemetry.go
index e7629bcd76f6a..39838853e123d 100644
--- a/metrics/telemetry.go
+++ b/metrics/telemetry.go
@@ -155,6 +155,62 @@ var (
Name: "flashback_cluster_usage",
Help: "Counter of usage of flashback cluster",
})
+ TelemetryIndexMergeUsage = prometheus.NewCounter(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "telemetry",
+ Name: "index_merge_usage",
+ Help: "Counter of usage of index merge",
+ })
+ TelemetryCompactPartitionCnt = prometheus.NewCounter(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "telemetry",
+ Name: "compact_partition_usage",
+ Help: "Counter of compact table partition",
+ })
+ TelemetryReorganizePartitionCnt = prometheus.NewCounter(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "telemetry",
+ Name: "reorganize_partition_usage",
+ Help: "Counter of alter table reorganize partition",
+ })
+ TelemetryDistReorgCnt = prometheus.NewCounter(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "telemetry",
+ Name: "distributed_reorg_count",
+ Help: "Counter of usage of distributed reorg DDL tasks count",
+ })
+ TelemetryStoreBatchedQueryCnt = prometheus.NewCounter(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "telemetry",
+ Name: "store_batched_query",
+ Help: "Counter of queries which use store batched coprocessor tasks",
+ })
+ TelemetryBatchedQueryTaskCnt = prometheus.NewCounter(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "telemetry",
+ Name: "batched_query_task",
+ Help: "Counter of coprocessor tasks in batched queries",
+ })
+ TelemetryStoreBatchedCnt = prometheus.NewCounter(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "telemetry",
+ Name: "store_batched",
+ Help: "Counter of store batched coprocessor tasks",
+ })
+ TelemetryStoreBatchedFallbackCnt = prometheus.NewCounter(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "telemetry",
+ Name: "store_batched_fallback",
+ Help: "Counter of store batched fallback coprocessor tasks",
+ })
)
// readCounter reads the value of a prometheus.Counter.
@@ -254,6 +310,8 @@ type TablePartitionUsageCounter struct {
TablePartitionCreateIntervalPartitionsCnt int64 `json:"table_partition_create_interval_partitions_cnt"`
TablePartitionAddIntervalPartitionsCnt int64 `json:"table_partition_add_interval_partitions_cnt"`
TablePartitionDropIntervalPartitionsCnt int64 `json:"table_partition_drop_interval_partitions_cnt"`
+ TablePartitionComactCnt int64 `json:"table_TablePartitionComactCnt"`
+ TablePartitionReorganizePartitionCnt int64 `json:"table_reorganize_partition_cnt"`
}
// ExchangePartitionUsageCounter records the usages of exchange partition.
@@ -291,22 +349,25 @@ func (c TablePartitionUsageCounter) Cal(rhs TablePartitionUsageCounter) TablePar
TablePartitionCreateIntervalPartitionsCnt: c.TablePartitionCreateIntervalPartitionsCnt - rhs.TablePartitionCreateIntervalPartitionsCnt,
TablePartitionAddIntervalPartitionsCnt: c.TablePartitionAddIntervalPartitionsCnt - rhs.TablePartitionAddIntervalPartitionsCnt,
TablePartitionDropIntervalPartitionsCnt: c.TablePartitionDropIntervalPartitionsCnt - rhs.TablePartitionDropIntervalPartitionsCnt,
+ TablePartitionComactCnt: c.TablePartitionComactCnt - rhs.TablePartitionComactCnt,
+ TablePartitionReorganizePartitionCnt: c.TablePartitionReorganizePartitionCnt - rhs.TablePartitionReorganizePartitionCnt,
}
}
// ResetTablePartitionCounter gets the TxnCommitCounter.
func ResetTablePartitionCounter(pre TablePartitionUsageCounter) TablePartitionUsageCounter {
return TablePartitionUsageCounter{
- TablePartitionCnt: readCounter(TelemetryTablePartitionCnt),
- TablePartitionListCnt: readCounter(TelemetryTablePartitionListCnt),
- TablePartitionRangeCnt: readCounter(TelemetryTablePartitionRangeCnt),
- TablePartitionHashCnt: readCounter(TelemetryTablePartitionHashCnt),
- TablePartitionRangeColumnsCnt: readCounter(TelemetryTablePartitionRangeColumnsCnt),
- TablePartitionRangeColumnsGt1Cnt: readCounter(TelemetryTablePartitionRangeColumnsGt1Cnt),
- TablePartitionRangeColumnsGt2Cnt: readCounter(TelemetryTablePartitionRangeColumnsGt2Cnt),
- TablePartitionRangeColumnsGt3Cnt: readCounter(TelemetryTablePartitionRangeColumnsGt3Cnt),
- TablePartitionListColumnsCnt: readCounter(TelemetryTablePartitionListColumnsCnt),
- TablePartitionMaxPartitionsCnt: mathutil.Max(readCounter(TelemetryTablePartitionMaxPartitionsCnt)-pre.TablePartitionMaxPartitionsCnt, pre.TablePartitionMaxPartitionsCnt),
+ TablePartitionCnt: readCounter(TelemetryTablePartitionCnt),
+ TablePartitionListCnt: readCounter(TelemetryTablePartitionListCnt),
+ TablePartitionRangeCnt: readCounter(TelemetryTablePartitionRangeCnt),
+ TablePartitionHashCnt: readCounter(TelemetryTablePartitionHashCnt),
+ TablePartitionRangeColumnsCnt: readCounter(TelemetryTablePartitionRangeColumnsCnt),
+ TablePartitionRangeColumnsGt1Cnt: readCounter(TelemetryTablePartitionRangeColumnsGt1Cnt),
+ TablePartitionRangeColumnsGt2Cnt: readCounter(TelemetryTablePartitionRangeColumnsGt2Cnt),
+ TablePartitionRangeColumnsGt3Cnt: readCounter(TelemetryTablePartitionRangeColumnsGt3Cnt),
+ TablePartitionListColumnsCnt: readCounter(TelemetryTablePartitionListColumnsCnt),
+ TablePartitionMaxPartitionsCnt: mathutil.Max(readCounter(TelemetryTablePartitionMaxPartitionsCnt)-pre.TablePartitionMaxPartitionsCnt, pre.TablePartitionMaxPartitionsCnt),
+ TablePartitionReorganizePartitionCnt: readCounter(TelemetryReorganizePartitionCnt),
}
}
@@ -326,12 +387,15 @@ func GetTablePartitionCounter() TablePartitionUsageCounter {
TablePartitionCreateIntervalPartitionsCnt: readCounter(TelemetryTablePartitionCreateIntervalPartitionsCnt),
TablePartitionAddIntervalPartitionsCnt: readCounter(TelemetryTablePartitionAddIntervalPartitionsCnt),
TablePartitionDropIntervalPartitionsCnt: readCounter(TelemetryTablePartitionDropIntervalPartitionsCnt),
+ TablePartitionComactCnt: readCounter(TelemetryCompactPartitionCnt),
+ TablePartitionReorganizePartitionCnt: readCounter(TelemetryReorganizePartitionCnt),
}
}
// NonTransactionalStmtCounter records the usages of non-transactional statements.
type NonTransactionalStmtCounter struct {
DeleteCount int64 `json:"delete"`
+ UpdateCount int64 `json:"update"`
InsertCount int64 `json:"insert"`
}
@@ -339,6 +403,7 @@ type NonTransactionalStmtCounter struct {
func (n NonTransactionalStmtCounter) Sub(rhs NonTransactionalStmtCounter) NonTransactionalStmtCounter {
return NonTransactionalStmtCounter{
DeleteCount: n.DeleteCount - rhs.DeleteCount,
+ UpdateCount: n.UpdateCount - rhs.UpdateCount,
InsertCount: n.InsertCount - rhs.InsertCount,
}
}
@@ -347,6 +412,7 @@ func (n NonTransactionalStmtCounter) Sub(rhs NonTransactionalStmtCounter) NonTra
func GetNonTransactionalStmtCounter() NonTransactionalStmtCounter {
return NonTransactionalStmtCounter{
DeleteCount: readCounter(NonTransactionalDMLCount.With(prometheus.Labels{LblType: "delete"})),
+ UpdateCount: readCounter(NonTransactionalDMLCount.With(prometheus.Labels{LblType: "update"})),
InsertCount: readCounter(NonTransactionalDMLCount.With(prometheus.Labels{LblType: "insert"})),
}
}
@@ -366,6 +432,7 @@ type DDLUsageCounter struct {
AddIndexIngestUsed int64 `json:"add_index_ingest_used"`
MetadataLockUsed bool `json:"metadata_lock_used"`
FlashbackClusterUsed int64 `json:"flashback_cluster_used"`
+ DistReorgUsed int64 `json:"dist_reorg_used"`
}
// Sub returns the difference of two counters.
@@ -373,6 +440,7 @@ func (a DDLUsageCounter) Sub(rhs DDLUsageCounter) DDLUsageCounter {
return DDLUsageCounter{
AddIndexIngestUsed: a.AddIndexIngestUsed - rhs.AddIndexIngestUsed,
FlashbackClusterUsed: a.FlashbackClusterUsed - rhs.FlashbackClusterUsed,
+ DistReorgUsed: a.DistReorgUsed - rhs.DistReorgUsed,
}
}
@@ -381,5 +449,81 @@ func GetDDLUsageCounter() DDLUsageCounter {
return DDLUsageCounter{
AddIndexIngestUsed: readCounter(TelemetryAddIndexIngestCnt),
FlashbackClusterUsed: readCounter(TelemetryFlashbackClusterCnt),
+ DistReorgUsed: readCounter(TelemetryDistReorgCnt),
+ }
+}
+
+// IndexMergeUsageCounter records the usages of IndexMerge feature.
+type IndexMergeUsageCounter struct {
+ IndexMergeUsed int64 `json:"index_merge_used"`
+}
+
+// Sub returns the difference of two counters.
+func (i IndexMergeUsageCounter) Sub(rhs IndexMergeUsageCounter) IndexMergeUsageCounter {
+ return IndexMergeUsageCounter{
+ IndexMergeUsed: i.IndexMergeUsed - rhs.IndexMergeUsed,
+ }
+}
+
+// GetIndexMergeCounter gets the IndexMerge usage counter.
+func GetIndexMergeCounter() IndexMergeUsageCounter {
+ return IndexMergeUsageCounter{
+ IndexMergeUsed: readCounter(TelemetryIndexMergeUsage),
+ }
+}
+
+// StoreBatchCoprCounter records the usages of batch copr statements.
+type StoreBatchCoprCounter struct {
+ // BatchSize is the global value of `tidb_store_batch_size`
+ BatchSize int `json:"batch_size"`
+ // BatchedQuery is the counter of queries that use this feature.
+ BatchedQuery int64 `json:"query"`
+ // BatchedQueryTask is the counter of total tasks in queries above.
+ BatchedQueryTask int64 `json:"tasks"`
+ // BatchedCount is the counter of successfully batched tasks.
+ BatchedCount int64 `json:"batched"`
+ // BatchedFallbackCount is the counter of fallback batched tasks by region miss.
+ BatchedFallbackCount int64 `json:"batched_fallback"`
+}
+
+// Sub returns the difference of two counters.
+func (n StoreBatchCoprCounter) Sub(rhs StoreBatchCoprCounter) StoreBatchCoprCounter {
+ return StoreBatchCoprCounter{
+ BatchedQuery: n.BatchedQuery - rhs.BatchedQuery,
+ BatchedQueryTask: n.BatchedQueryTask - rhs.BatchedQueryTask,
+ BatchedCount: n.BatchedCount - rhs.BatchedCount,
+ BatchedFallbackCount: n.BatchedFallbackCount - rhs.BatchedFallbackCount,
+ }
+}
+
+// GetStoreBatchCoprCounter gets the IndexMerge usage counter.
+func GetStoreBatchCoprCounter() StoreBatchCoprCounter {
+ return StoreBatchCoprCounter{
+ BatchedQuery: readCounter(TelemetryStoreBatchedQueryCnt),
+ BatchedQueryTask: readCounter(TelemetryBatchedQueryTaskCnt),
+ BatchedCount: readCounter(TelemetryStoreBatchedCnt),
+ BatchedFallbackCount: readCounter(TelemetryStoreBatchedFallbackCnt),
+ }
+}
+
+// AggressiveLockingUsageCounter records the usage of Aggressive Locking feature of pessimistic transaction.
+type AggressiveLockingUsageCounter struct {
+ TxnAggressiveLockingUsed int64 `json:"txn_aggressive_locking_used"`
+ TxnAggressiveLockingEffective int64 `json:"txn_aggressive_locking_effective"`
+}
+
+// Sub returns the difference of two counters.
+func (i AggressiveLockingUsageCounter) Sub(rhs AggressiveLockingUsageCounter) AggressiveLockingUsageCounter {
+ return AggressiveLockingUsageCounter{
+ TxnAggressiveLockingUsed: i.TxnAggressiveLockingUsed - rhs.TxnAggressiveLockingUsed,
+ TxnAggressiveLockingEffective: i.TxnAggressiveLockingEffective - rhs.TxnAggressiveLockingEffective,
+ }
+}
+
+// GetAggressiveLockingUsageCounter returns the Aggressive Locking usage counter.
+func GetAggressiveLockingUsageCounter() AggressiveLockingUsageCounter {
+ return AggressiveLockingUsageCounter{
+ TxnAggressiveLockingUsed: readCounter(AggressiveLockingUsageCount.WithLabelValues(LblAggressiveLockingTxnUsed)),
+ TxnAggressiveLockingEffective: readCounter(AggressiveLockingUsageCount.WithLabelValues(LblAggressiveLockingTxnEffective)),
}
}
diff --git a/metrics/ttl.go b/metrics/ttl.go
new file mode 100644
index 0000000000000..52782b08dba76
--- /dev/null
+++ b/metrics/ttl.go
@@ -0,0 +1,69 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+// TTL metrics
+var (
+ TTLQueryDuration = prometheus.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Namespace: "tidb",
+ Subsystem: "server",
+ Name: "ttl_query_duration",
+ Help: "Bucketed histogram of processing time (s) of handled TTL queries.",
+ Buckets: prometheus.ExponentialBuckets(0.01, 2, 20), // 10ms ~ 1.45hour
+ }, []string{LblSQLType, LblResult})
+
+ TTLProcessedExpiredRowsCounter = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "server",
+ Name: "ttl_processed_expired_rows",
+ Help: "The count of expired rows processed in TTL jobs",
+ }, []string{LblSQLType, LblResult})
+
+ TTLJobStatus = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: "tidb",
+ Subsystem: "server",
+ Name: "ttl_job_status",
+ Help: "The jobs count in the specified status",
+ }, []string{LblType})
+
+ TTLTaskStatus = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: "tidb",
+ Subsystem: "server",
+ Name: "ttl_task_status",
+ Help: "The tasks count in the specified status",
+ }, []string{LblType})
+
+ TTLPhaseTime = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "server",
+ Name: "ttl_phase_time",
+ Help: "The time spent in each phase",
+ }, []string{LblType, LblPhase})
+
+ TTLInsertRowsCount = prometheus.NewCounter(
+ prometheus.CounterOpts{
+ Namespace: "tidb",
+ Subsystem: "server",
+ Name: "ttl_insert_rows",
+ Help: "The count of TTL rows inserted",
+ })
+)
diff --git a/parser/BUILD.bazel b/parser/BUILD.bazel
index f52b1fc9ac4f3..ec67a5c141849 100644
--- a/parser/BUILD.bazel
+++ b/parser/BUILD.bazel
@@ -17,6 +17,7 @@ go_library(
"//parser/ast",
"//parser/auth",
"//parser/charset",
+ "//parser/duration",
"//parser/model",
"//parser/mysql",
"//parser/opcode",
diff --git a/parser/ast/ddl.go b/parser/ast/ddl.go
index 9b7042e573cc1..97af9a3b95c6a 100644
--- a/parser/ast/ddl.go
+++ b/parser/ast/ddl.go
@@ -28,18 +28,21 @@ var (
_ DDLNode = &AlterTableStmt{}
_ DDLNode = &AlterSequenceStmt{}
_ DDLNode = &AlterPlacementPolicyStmt{}
+ _ DDLNode = &AlterResourceGroupStmt{}
_ DDLNode = &CreateDatabaseStmt{}
_ DDLNode = &CreateIndexStmt{}
_ DDLNode = &CreateTableStmt{}
_ DDLNode = &CreateViewStmt{}
_ DDLNode = &CreateSequenceStmt{}
_ DDLNode = &CreatePlacementPolicyStmt{}
+ _ DDLNode = &CreateResourceGroupStmt{}
_ DDLNode = &DropDatabaseStmt{}
_ DDLNode = &FlashBackDatabaseStmt{}
_ DDLNode = &DropIndexStmt{}
_ DDLNode = &DropTableStmt{}
_ DDLNode = &DropSequenceStmt{}
_ DDLNode = &DropPlacementPolicyStmt{}
+ _ DDLNode = &DropResourceGroupStmt{}
_ DDLNode = &RenameTableStmt{}
_ DDLNode = &TruncateTableStmt{}
_ DDLNode = &RepairTableStmt{}
@@ -1093,7 +1096,8 @@ func (n *CreateTableStmt) Restore(ctx *format.RestoreCtx) error {
ctx.WritePlain(")")
}
- for i, option := range n.Options {
+ options := tableOptionsWithRestoreTTLFlag(ctx.Flags, n.Options)
+ for i, option := range options {
ctx.WritePlain(" ")
if err := option.Restore(ctx); err != nil {
return errors.Annotatef(err, "An error occurred while splicing CreateTableStmt TableOption: [%v]", i)
@@ -1180,6 +1184,13 @@ func (n *CreateTableStmt) Accept(v Visitor) (Node, bool) {
}
n.Partition = node.(*PartitionOptions)
}
+ for i, option := range n.Options {
+ node, ok = option.Accept(v)
+ if !ok {
+ return n, false
+ }
+ n.Options[i] = node.(*TableOption)
+ }
return v.Leave(n)
}
@@ -1273,6 +1284,32 @@ func (n *DropPlacementPolicyStmt) Accept(v Visitor) (Node, bool) {
return v.Leave(n)
}
+type DropResourceGroupStmt struct {
+ ddlNode
+
+ IfExists bool
+ ResourceGroupName model.CIStr
+}
+
+// Restore implements Restore interface.
+func (n *DropResourceGroupStmt) Restore(ctx *format.RestoreCtx) error {
+ ctx.WriteKeyWord("DROP RESOURCE GROUP ")
+ if n.IfExists {
+ ctx.WriteKeyWord("IF EXISTS ")
+ }
+ ctx.WriteName(n.ResourceGroupName.O)
+ return nil
+}
+
+func (n *DropResourceGroupStmt) Accept(v Visitor) (Node, bool) {
+ newNode, skipChildren := v.Enter(n)
+ if skipChildren {
+ return v.Leave(newNode)
+ }
+ n = newNode.(*DropResourceGroupStmt)
+ return v.Leave(n)
+}
+
// DropSequenceStmt is a statement to drop a Sequence.
type DropSequenceStmt struct {
ddlNode
@@ -1533,6 +1570,43 @@ func (n *CreatePlacementPolicyStmt) Accept(v Visitor) (Node, bool) {
return v.Leave(n)
}
+// CreateResourceGroupStmt is a statement to create a policy.
+type CreateResourceGroupStmt struct {
+ ddlNode
+
+ IfNotExists bool
+ ResourceGroupName model.CIStr
+ ResourceGroupOptionList []*ResourceGroupOption
+}
+
+// Restore implements Node interface.
+func (n *CreateResourceGroupStmt) Restore(ctx *format.RestoreCtx) error {
+ ctx.WriteKeyWord("CREATE ")
+
+ ctx.WriteKeyWord("RESOURCE GROUP ")
+ if n.IfNotExists {
+ ctx.WriteKeyWord("IF NOT EXISTS ")
+ }
+ ctx.WriteName(n.ResourceGroupName.O)
+ for i, option := range n.ResourceGroupOptionList {
+ ctx.WritePlain(" ")
+ if err := option.Restore(ctx); err != nil {
+ return errors.Annotatef(err, "An error occurred while splicing CreatePlacementPolicy TableOption: [%v]", i)
+ }
+ }
+ return nil
+}
+
+// Accept implements Node Accept interface.
+func (n *CreateResourceGroupStmt) Accept(v Visitor) (Node, bool) {
+ newNode, skipChildren := v.Enter(n)
+ if skipChildren {
+ return v.Leave(newNode)
+ }
+ n = newNode.(*CreateResourceGroupStmt)
+ return v.Leave(n)
+}
+
// CreateSequenceStmt is a statement to create a Sequence.
type CreateSequenceStmt struct {
ddlNode
@@ -1951,6 +2025,7 @@ const (
PlacementOptionLearnerConstraints
PlacementOptionFollowerConstraints
PlacementOptionVoterConstraints
+ PlacementOptionSurvivalPreferences
PlacementOptionPolicy
)
@@ -2015,6 +2090,10 @@ func (n *PlacementOption) Restore(ctx *format.RestoreCtx) error {
ctx.WriteKeyWord("PLACEMENT POLICY ")
ctx.WritePlain("= ")
ctx.WriteName(n.StrValue)
+ case PlacementOptionSurvivalPreferences:
+ ctx.WriteKeyWord("SURVIVAL_PREFERENCES ")
+ ctx.WritePlain("= ")
+ ctx.WriteString(n.StrValue)
default:
return errors.Errorf("invalid PlacementOption: %d", n.Tp)
}
@@ -2024,6 +2103,61 @@ func (n *PlacementOption) Restore(ctx *format.RestoreCtx) error {
return ctx.WriteWithSpecialComments(tidb.FeatureIDPlacement, fn)
}
+// ResourceGroupOption is used for parsing resource group option.
+type ResourceGroupOption struct {
+ Tp ResourceUnitType
+ StrValue string
+ UintValue uint64
+ BoolValue bool
+}
+
+type ResourceUnitType int
+
+const (
+ // RU mode
+ ResourceRURate ResourceUnitType = iota
+ // Raw mode
+ ResourceUnitCPU
+ ResourceUnitIOReadBandwidth
+ ResourceUnitIOWriteBandwidth
+
+ // Options
+ ResourceBurstableOpiton
+)
+
+func (n *ResourceGroupOption) Restore(ctx *format.RestoreCtx) error {
+ if ctx.Flags.HasSkipPlacementRuleForRestoreFlag() {
+ return nil
+ }
+ fn := func() error {
+ switch n.Tp {
+ case ResourceRURate:
+ ctx.WriteKeyWord("RU_PER_SEC ")
+ ctx.WritePlain("= ")
+ ctx.WritePlainf("%d", n.UintValue)
+ case ResourceUnitCPU:
+ ctx.WriteKeyWord("CPU ")
+ ctx.WritePlain("= ")
+ ctx.WriteString(n.StrValue)
+ case ResourceUnitIOReadBandwidth:
+ ctx.WriteKeyWord("IO_READ_BANDWIDTH ")
+ ctx.WritePlain("= ")
+ ctx.WriteString(n.StrValue)
+ case ResourceUnitIOWriteBandwidth:
+ ctx.WriteKeyWord("IO_WRITE_BANDWIDTH ")
+ ctx.WritePlain("= ")
+ ctx.WriteString(n.StrValue)
+ case ResourceBurstableOpiton:
+ ctx.WriteKeyWord("BURSTABLE")
+ default:
+ return errors.Errorf("invalid PlacementOption: %d", n.Tp)
+ }
+ return nil
+ }
+ // WriteSpecialComment
+ return ctx.WriteWithSpecialComments(tidb.FeatureIDResourceGroup, fn)
+}
+
type StatsOptionType int
const (
@@ -2074,6 +2208,9 @@ const (
TableOptionTableCheckSum
TableOptionUnion
TableOptionEncryption
+ TableOptionTTL
+ TableOptionTTLEnable
+ TableOptionTTLJobInterval
TableOptionPlacementPolicy = TableOptionType(PlacementOptionPolicy)
TableOptionStatsBuckets = TableOptionType(StatsOptionBuckets)
TableOptionStatsTopN = TableOptionType(StatsOptionTopN)
@@ -2120,13 +2257,16 @@ const (
// TableOption is used for parsing table option from SQL.
type TableOption struct {
- Tp TableOptionType
- Default bool
- StrValue string
- UintValue uint64
- BoolValue bool
- Value ValueExpr
- TableNames []*TableName
+ node
+ Tp TableOptionType
+ Default bool
+ StrValue string
+ UintValue uint64
+ BoolValue bool
+ TimeUnitValue *TimeUnitExpr
+ Value ValueExpr
+ TableNames []*TableName
+ ColumnName *ColumnName
}
func (n *TableOption) Restore(ctx *format.RestoreCtx) error {
@@ -2405,12 +2545,67 @@ func (n *TableOption) Restore(ctx *format.RestoreCtx) error {
} else {
ctx.WriteString(n.StrValue)
}
+ case TableOptionTTL:
+ _ = ctx.WriteWithSpecialComments(tidb.FeatureIDTTL, func() error {
+ ctx.WriteKeyWord("TTL ")
+ ctx.WritePlain("= ")
+ ctx.WriteName(n.ColumnName.Name.String())
+ ctx.WritePlain(" + INTERVAL ")
+ err := n.Value.Restore(ctx)
+ ctx.WritePlain(" ")
+ if err != nil {
+ return err
+ }
+ return n.TimeUnitValue.Restore(ctx)
+ })
+ case TableOptionTTLEnable:
+ _ = ctx.WriteWithSpecialComments(tidb.FeatureIDTTL, func() error {
+ ctx.WriteKeyWord("TTL_ENABLE ")
+ ctx.WritePlain("= ")
+ if n.BoolValue {
+ ctx.WriteString("ON")
+ } else {
+ ctx.WriteString("OFF")
+ }
+ return nil
+ })
+ case TableOptionTTLJobInterval:
+ _ = ctx.WriteWithSpecialComments(tidb.FeatureIDTTL, func() error {
+ ctx.WriteKeyWord("TTL_JOB_INTERVAL ")
+ ctx.WritePlain("= ")
+ ctx.WriteString(n.StrValue)
+ return nil
+ })
default:
return errors.Errorf("invalid TableOption: %d", n.Tp)
}
return nil
}
+// Accept implements Node Accept interface.
+func (n *TableOption) Accept(v Visitor) (Node, bool) {
+ newNode, skipChildren := v.Enter(n)
+ if skipChildren {
+ return v.Leave(newNode)
+ }
+ n = newNode.(*TableOption)
+ if n.Value != nil {
+ node, ok := n.Value.Accept(v)
+ if !ok {
+ return n, false
+ }
+ n.Value = node.(ValueExpr)
+ }
+ if n.TimeUnitValue != nil {
+ node, ok := n.TimeUnitValue.Accept(v)
+ if !ok {
+ return n, false
+ }
+ n.TimeUnitValue = node.(*TimeUnitExpr)
+ }
+ return v.Leave(n)
+}
+
// SequenceOptionType is the type for SequenceOption
type SequenceOptionType int
@@ -2599,6 +2794,7 @@ const (
AlterTableAddLastPartition
AlterTableReorganizeLastPartition
AlterTableReorganizeFirstPartition
+ AlterTableRemoveTTL
)
// LockType is the type for AlterTableSpec.
@@ -3280,7 +3476,11 @@ func (n *AlterTableSpec) Restore(ctx *format.RestoreCtx) error {
if err := spec.Restore(ctx); err != nil {
return errors.Annotatef(err, "An error occurred while restore AlterTableSpec.StatsOptionsSpec")
}
-
+ case AlterTableRemoveTTL:
+ _ = ctx.WriteWithSpecialComments(tidb.FeatureIDTTL, func() error {
+ ctx.WriteKeyWord("REMOVE TTL")
+ return nil
+ })
default:
// TODO: not support
ctx.WritePlainf(" /* AlterTableType(%d) is not supported */ ", n.Tp)
@@ -3344,6 +3544,13 @@ func (n *AlterTableSpec) Accept(v Visitor) (Node, bool) {
}
n.Partition = node.(*PartitionOptions)
}
+ for i, option := range n.Options {
+ node, ok := option.Accept(v)
+ if !ok {
+ return n, false
+ }
+ n.Options[i] = node.(*TableOption)
+ }
for _, def := range n.PartDefinitions {
if !def.acceptInPlace(v) {
return n, false
@@ -3382,11 +3589,21 @@ func (n *AlterTableStmt) Restore(ctx *format.RestoreCtx) error {
if err := n.Table.Restore(ctx); err != nil {
return errors.Annotate(err, "An error occurred while restore AlterTableStmt.Table")
}
- var specs []*AlterTableSpec
+ specs := make([]*AlterTableSpec, 0, len(n.Specs))
for _, spec := range n.Specs {
- if !(spec.IsAllPlacementRule() && ctx.Flags.HasSkipPlacementRuleForRestoreFlag()) {
- specs = append(specs, spec)
+ if spec.IsAllPlacementRule() && ctx.Flags.HasSkipPlacementRuleForRestoreFlag() {
+ continue
+ }
+ if spec.Tp == AlterTableOption {
+ newOptions := tableOptionsWithRestoreTTLFlag(ctx.Flags, spec.Options)
+ if len(newOptions) == 0 {
+ continue
+ }
+ newSpec := *spec
+ newSpec.Options = newOptions
+ spec = &newSpec
}
+ specs = append(specs, spec)
}
for i, spec := range specs {
if i == 0 || spec.Tp == AlterTablePartition || spec.Tp == AlterTableRemovePartitioning || spec.Tp == AlterTableImportTablespace || spec.Tp == AlterTableDiscardTablespace {
@@ -4232,6 +4449,39 @@ func (n *AlterPlacementPolicyStmt) Accept(v Visitor) (Node, bool) {
return v.Leave(n)
}
+// AlterResourceGroupStmt is a statement to alter placement policy option.
+type AlterResourceGroupStmt struct {
+ ddlNode
+
+ ResourceGroupName model.CIStr
+ IfExists bool
+ ResourceGroupOptionList []*ResourceGroupOption
+}
+
+func (n *AlterResourceGroupStmt) Restore(ctx *format.RestoreCtx) error {
+ ctx.WriteKeyWord("ALTER RESOURCE GROUP ")
+ if n.IfExists {
+ ctx.WriteKeyWord("IF EXISTS ")
+ }
+ ctx.WriteName(n.ResourceGroupName.O)
+ for i, option := range n.ResourceGroupOptionList {
+ ctx.WritePlain(" ")
+ if err := option.Restore(ctx); err != nil {
+ return errors.Annotatef(err, "An error occurred while splicing AlterResourceStmt Options: [%v]", i)
+ }
+ }
+ return nil
+}
+
+func (n *AlterResourceGroupStmt) Accept(v Visitor) (Node, bool) {
+ newNode, skipChildren := v.Enter(n)
+ if skipChildren {
+ return v.Leave(newNode)
+ }
+ n = newNode.(*AlterResourceGroupStmt)
+ return v.Leave(n)
+}
+
// AlterSequenceStmt is a statement to alter sequence option.
type AlterSequenceStmt struct {
ddlNode
@@ -4286,3 +4536,25 @@ func restorePlacementStmtInSpecialComment(ctx *format.RestoreCtx, n DDLNode) err
return n.Restore(ctx)
})
}
+
+func tableOptionsWithRestoreTTLFlag(flags format.RestoreFlags, options []*TableOption) []*TableOption {
+ if !flags.HasRestoreWithTTLEnableOff() {
+ return options
+ }
+
+ newOptions := make([]*TableOption, 0, len(options))
+ for _, opt := range options {
+ if opt.Tp == TableOptionTTLEnable {
+ continue
+ }
+
+ newOptions = append(newOptions, opt)
+ if opt.Tp == TableOptionTTL {
+ newOptions = append(newOptions, &TableOption{
+ Tp: TableOptionTTLEnable,
+ BoolValue: false,
+ })
+ }
+ }
+ return newOptions
+}
diff --git a/parser/ast/ddl_test.go b/parser/ast/ddl_test.go
index 3a5b1087d0bd2..dbbb212037db8 100644
--- a/parser/ast/ddl_test.go
+++ b/parser/ast/ddl_test.go
@@ -248,6 +248,21 @@ func TestDDLColumnOptionRestore(t *testing.T) {
runNodeRestoreTest(t, testCases, "CREATE TABLE child (id INT %s)", extractNodeFunc)
}
+func TestGeneratedRestore(t *testing.T) {
+ testCases := []NodeRestoreTestCase{
+ {"generated always as(id + 1)", "GENERATED ALWAYS AS(`id`+1) VIRTUAL"},
+ {"generated always as(id + 1) virtual", "GENERATED ALWAYS AS(`id`+1) VIRTUAL"},
+ {"generated always as(id + 1) stored", "GENERATED ALWAYS AS(`id`+1) STORED"},
+ {"generated always as(lower(id)) stored", "GENERATED ALWAYS AS(LOWER(`id`)) STORED"},
+ {"generated always as(lower(child.id)) stored", "GENERATED ALWAYS AS(LOWER(`id`)) STORED"},
+ }
+ extractNodeFunc := func(node Node) Node {
+ return node.(*CreateTableStmt).Cols[0].Options[0]
+ }
+ runNodeRestoreTestWithFlagsStmtChange(t, testCases, "CREATE TABLE child (id INT %s)", extractNodeFunc,
+ format.DefaultRestoreFlags|format.RestoreWithoutSchemaName|format.RestoreWithoutTableName)
+}
+
func TestDDLColumnDefRestore(t *testing.T) {
testCases := []NodeRestoreTestCase{
// for type
@@ -840,3 +855,66 @@ func TestFlashBackDatabaseRestore(t *testing.T) {
}
runNodeRestoreTest(t, testCases, "%s", extractNodeFunc)
}
+
+func TestTableOptionTTLRestore(t *testing.T) {
+ sourceSQL1 := "create table t (created_at datetime) ttl = created_at + INTERVAL 1 YEAR"
+ sourceSQL2 := "alter table t ttl_enable = 'OFF'"
+ sourceSQL3 := "alter table t remove ttl"
+ cases := []struct {
+ sourceSQL string
+ flags format.RestoreFlags
+ expectSQL string
+ }{
+ {sourceSQL1, format.DefaultRestoreFlags, "CREATE TABLE `t` (`created_at` DATETIME) TTL = `created_at` + INTERVAL 1 YEAR"},
+ {sourceSQL1, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment, "CREATE TABLE `t` (`created_at` DATETIME) /*T![ttl] TTL = `created_at` + INTERVAL 1 YEAR */"},
+ {sourceSQL2, format.DefaultRestoreFlags, "ALTER TABLE `t` TTL_ENABLE = 'OFF'"},
+ {sourceSQL2, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment, "ALTER TABLE `t` /*T![ttl] TTL_ENABLE = 'OFF' */"},
+ {sourceSQL3, format.DefaultRestoreFlags, "ALTER TABLE `t` REMOVE TTL"},
+ {sourceSQL3, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment, "ALTER TABLE `t` /*T![ttl] REMOVE TTL */"},
+ }
+
+ extractNodeFunc := func(node Node) Node {
+ return node
+ }
+
+ for _, ca := range cases {
+ testCases := []NodeRestoreTestCase{
+ {ca.sourceSQL, ca.expectSQL},
+ }
+ runNodeRestoreTestWithFlags(t, testCases, "%s", extractNodeFunc, ca.flags)
+ }
+}
+
+func TestTableOptionTTLRestoreWithTTLEnableOffFlag(t *testing.T) {
+ sourceSQL1 := "create table t (created_at datetime) ttl = created_at + INTERVAL 1 YEAR"
+ sourceSQL2 := "alter table t ttl_enable = 'ON'"
+ sourceSQL3 := "alter table t remove ttl"
+ sourceSQL4 := "create table t (created_at datetime) ttl = created_at + INTERVAL 1 YEAR ttl_enable = 'ON'"
+ sourceSQL5 := "alter table t ttl_enable = 'ON' placement policy p1"
+ cases := []struct {
+ sourceSQL string
+ flags format.RestoreFlags
+ expectSQL string
+ }{
+ {sourceSQL1, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) TTL = `created_at` + INTERVAL 1 YEAR TTL_ENABLE = 'OFF'"},
+ {sourceSQL1, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) /*T![ttl] TTL = `created_at` + INTERVAL 1 YEAR */ /*T![ttl] TTL_ENABLE = 'OFF' */"},
+ {sourceSQL2, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "ALTER TABLE `t`"},
+ {sourceSQL2, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "ALTER TABLE `t`"},
+ {sourceSQL3, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "ALTER TABLE `t` REMOVE TTL"},
+ {sourceSQL3, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "ALTER TABLE `t` /*T![ttl] REMOVE TTL */"},
+ {sourceSQL4, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) TTL = `created_at` + INTERVAL 1 YEAR TTL_ENABLE = 'OFF'"},
+ {sourceSQL4, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) /*T![ttl] TTL = `created_at` + INTERVAL 1 YEAR */ /*T![ttl] TTL_ENABLE = 'OFF' */"},
+ {sourceSQL5, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "ALTER TABLE `t` /*T![placement] PLACEMENT POLICY = `p1` */"},
+ }
+
+ extractNodeFunc := func(node Node) Node {
+ return node
+ }
+
+ for _, ca := range cases {
+ testCases := []NodeRestoreTestCase{
+ {ca.sourceSQL, ca.expectSQL},
+ }
+ runNodeRestoreTestWithFlagsStmtChange(t, testCases, "%s", extractNodeFunc, ca.flags)
+ }
+}
diff --git a/parser/ast/dml.go b/parser/ast/dml.go
index 2712a8f7eba51..140331a3ba275 100644
--- a/parser/ast/dml.go
+++ b/parser/ast/dml.go
@@ -288,15 +288,17 @@ func (*TableName) resultSet() {}
// Restore implements Node interface.
func (n *TableName) restoreName(ctx *format.RestoreCtx) {
- // restore db name
- if n.Schema.String() != "" {
- ctx.WriteName(n.Schema.String())
- ctx.WritePlain(".")
- } else if ctx.DefaultDB != "" {
- // Try CTE, for a CTE table name, we shouldn't write the database name.
- if !ctx.IsCTETableName(n.Name.L) {
- ctx.WriteName(ctx.DefaultDB)
+ if !ctx.Flags.HasWithoutSchemaNameFlag() {
+ // restore db name
+ if n.Schema.String() != "" {
+ ctx.WriteName(n.Schema.String())
ctx.WritePlain(".")
+ } else if ctx.DefaultDB != "" {
+ // Try CTE, for a CTE table name, we shouldn't write the database name.
+ if !ctx.IsCTETableName(n.Name.L) {
+ ctx.WriteName(ctx.DefaultDB)
+ ctx.WritePlain(".")
+ }
}
}
// restore table name
@@ -356,6 +358,8 @@ const (
HintUse IndexHintType = iota + 1
HintIgnore
HintForce
+ HintOrderIndex
+ HintNoOrderIndex
)
// IndexHintScope is the type for index hint for join, order by or group by.
@@ -386,6 +390,10 @@ func (n *IndexHint) Restore(ctx *format.RestoreCtx) error {
indexHintType = "IGNORE INDEX"
case HintForce:
indexHintType = "FORCE INDEX"
+ case HintOrderIndex:
+ indexHintType = "ORDER INDEX"
+ case HintNoOrderIndex:
+ indexHintType = "NO ORDER INDEX"
default: // Prevent accidents
return errors.New("IndexHintType has an error while matching")
}
@@ -1793,12 +1801,24 @@ func (n *ColumnNameOrUserVar) Accept(v Visitor) (node Node, ok bool) {
return v.Leave(n)
}
+type FileLocRefTp int
+
+const (
+ // FileLocServerOrRemote is used when there's no keywords in SQL, which means the data file should be located on the
+ // tidb-server or on remote storage (S3 for example).
+ FileLocServerOrRemote FileLocRefTp = iota
+ // FileLocClient is used when there's LOCAL keyword in SQL, which means the data file should be located on the MySQL
+ // client.
+ FileLocClient
+)
+
// LoadDataStmt is a statement to load data from a specified file, then insert this rows into an existing table.
// See https://dev.mysql.com/doc/refman/5.7/en/load-data.html
+// in TiDB we extend the syntax to use LOAD DATA as a more general way to import data.
type LoadDataStmt struct {
dmlNode
- IsLocal bool
+ FileLocRef FileLocRefTp
Path string
OnDuplicate OnDuplicateKeyHandlingType
Table *TableName
@@ -1814,7 +1834,9 @@ type LoadDataStmt struct {
// Restore implements Node interface.
func (n *LoadDataStmt) Restore(ctx *format.RestoreCtx) error {
ctx.WriteKeyWord("LOAD DATA ")
- if n.IsLocal {
+ switch n.FileLocRef {
+ case FileLocServerOrRemote:
+ case FileLocClient:
ctx.WriteKeyWord("LOCAL ")
}
ctx.WriteKeyWord("INFILE ")
@@ -2221,8 +2243,8 @@ func (n *InsertStmt) SetWhereExpr(e ExprNode) {
s.Where = e
}
-// TableSource implements ShardableDMLStmt interface.
-func (n *InsertStmt) TableSource() (*TableSource, bool) {
+// TableRefsJoin implements ShardableDMLStmt interface.
+func (n *InsertStmt) TableRefsJoin() (*Join, bool) {
if n.Select == nil {
return nil, false
}
@@ -2230,8 +2252,7 @@ func (n *InsertStmt) TableSource() (*TableSource, bool) {
if !ok {
return nil, false
}
- table, ok := s.From.TableRefs.Left.(*TableSource)
- return table, ok
+ return s.From.TableRefs, true
}
// DeleteStmt is a statement to delete rows from table.
@@ -2410,10 +2431,9 @@ func (n *DeleteStmt) SetWhereExpr(e ExprNode) {
n.Where = e
}
-// TableSource implements ShardableDMLStmt interface.
-func (n *DeleteStmt) TableSource() (*TableSource, bool) {
- table, ok := n.TableRefs.TableRefs.Left.(*TableSource)
- return table, ok
+// TableRefsJoin implements ShardableDMLStmt interface.
+func (n *DeleteStmt) TableRefsJoin() (*Join, bool) {
+ return n.TableRefs.TableRefs, true
}
const (
@@ -2426,8 +2446,8 @@ type ShardableDMLStmt = interface {
StmtNode
WhereExpr() ExprNode
SetWhereExpr(ExprNode)
- // TableSource returns the *only* target table source in the statement.
- TableSource() (table *TableSource, ok bool)
+ // TableRefsJoin returns the table refs in the statement.
+ TableRefsJoin() (refs *Join, ok bool)
}
var _ ShardableDMLStmt = &DeleteStmt{}
@@ -2649,10 +2669,9 @@ func (n *UpdateStmt) SetWhereExpr(e ExprNode) {
n.Where = e
}
-// TableSource implements ShardableDMLStmt interface.
-func (n *UpdateStmt) TableSource() (*TableSource, bool) {
- table, ok := n.TableRefs.TableRefs.Left.(*TableSource)
- return table, ok
+// TableRefsJoin implements ShardableDMLStmt interface.
+func (n *UpdateStmt) TableRefsJoin() (*Join, bool) {
+ return n.TableRefs.TableRefs, true
}
// Limit is the limit clause.
@@ -2766,6 +2785,7 @@ const (
ShowPlacementForPartition
ShowPlacementLabels
ShowSessionStates
+ ShowCreateResourceGroup
)
const (
@@ -2786,19 +2806,20 @@ const (
type ShowStmt struct {
dmlNode
- Tp ShowStmtType // Databases/Tables/Columns/....
- DBName string
- Table *TableName // Used for showing columns.
- Partition model.CIStr // Used for showing partition.
- Column *ColumnName // Used for `desc table column`.
- IndexName model.CIStr
- Flag int // Some flag parsed from sql, such as FULL.
- Full bool
- User *auth.UserIdentity // Used for show grants/create user.
- Roles []*auth.RoleIdentity // Used for show grants .. using
- IfNotExists bool // Used for `show create database if not exists`
- Extended bool // Used for `show extended columns from ...`
- Limit *Limit // Used for partial Show STMTs to limit Result Set row numbers.
+ Tp ShowStmtType // Databases/Tables/Columns/....
+ DBName string
+ Table *TableName // Used for showing columns.
+ Partition model.CIStr // Used for showing partition.
+ Column *ColumnName // Used for `desc table column`.
+ IndexName model.CIStr
+ ResourceGroupName string // used for showing resource group
+ Flag int // Some flag parsed from sql, such as FULL.
+ Full bool
+ User *auth.UserIdentity // Used for show grants/create user.
+ Roles []*auth.RoleIdentity // Used for show grants .. using
+ IfNotExists bool // Used for `show create database if not exists`
+ Extended bool // Used for `show extended columns from ...`
+ Limit *Limit // Used for partial Show STMTs to limit Result Set row numbers.
CountWarningsOrErrors bool // Used for showing count(*) warnings | errors
@@ -2874,6 +2895,9 @@ func (n *ShowStmt) Restore(ctx *format.RestoreCtx) error {
case ShowCreatePlacementPolicy:
ctx.WriteKeyWord("CREATE PLACEMENT POLICY ")
ctx.WriteName(n.DBName)
+ case ShowCreateResourceGroup:
+ ctx.WriteKeyWord("CREATE RESOURCE GROUP ")
+ ctx.WriteName(n.ResourceGroupName)
case ShowCreateUser:
ctx.WriteKeyWord("CREATE USER ")
if err := n.User.Restore(ctx); err != nil {
diff --git a/parser/ast/expressions.go b/parser/ast/expressions.go
index 270c46218af61..6bca49f4d2a7d 100644
--- a/parser/ast/expressions.go
+++ b/parser/ast/expressions.go
@@ -512,11 +512,11 @@ type ColumnName struct {
// Restore implements Node interface.
func (n *ColumnName) Restore(ctx *format.RestoreCtx) error {
- if n.Schema.O != "" && !ctx.IsCTETableName(n.Table.L) {
+ if n.Schema.O != "" && !ctx.IsCTETableName(n.Table.L) && !ctx.Flags.HasWithoutSchemaNameFlag() {
ctx.WriteName(n.Schema.O)
ctx.WritePlain(".")
}
- if n.Table.O != "" {
+ if n.Table.O != "" && !ctx.Flags.HasWithoutTableNameFlag() {
ctx.WriteName(n.Table.O)
ctx.WritePlain(".")
}
diff --git a/parser/ast/functions.go b/parser/ast/functions.go
index af47b56515743..fdedf53b701cf 100644
--- a/parser/ast/functions.go
+++ b/parser/ast/functions.go
@@ -331,7 +331,9 @@ const (
JSONInsert = "json_insert"
JSONReplace = "json_replace"
JSONRemove = "json_remove"
+ JSONOverlaps = "json_overlaps"
JSONContains = "json_contains"
+ JSONMemberOf = "json_memberof"
JSONContainsPath = "json_contains_path"
JSONValid = "json_valid"
JSONArrayAppend = "json_array_append"
@@ -380,21 +382,9 @@ type FuncCallExpr struct {
// Restore implements Node interface.
func (n *FuncCallExpr) Restore(ctx *format.RestoreCtx) error {
- var specialLiteral string
- switch n.FnName.L {
- case DateLiteral:
- specialLiteral = "DATE "
- case TimeLiteral:
- specialLiteral = "TIME "
- case TimestampLiteral:
- specialLiteral = "TIMESTAMP "
- }
- if specialLiteral != "" {
- ctx.WritePlain(specialLiteral)
- if err := n.Args[0].Restore(ctx); err != nil {
- return errors.Annotatef(err, "An error occurred while restore FuncCastExpr.Expr")
- }
- return nil
+ done, err := n.customRestore(ctx)
+ if done {
+ return err
}
if len(n.Schema.String()) != 0 {
@@ -495,29 +485,70 @@ func (n *FuncCallExpr) Restore(ctx *format.RestoreCtx) error {
return nil
}
+func (n *FuncCallExpr) customRestore(ctx *format.RestoreCtx) (bool, error) {
+ var specialLiteral string
+ switch n.FnName.L {
+ case DateLiteral:
+ specialLiteral = "DATE "
+ case TimeLiteral:
+ specialLiteral = "TIME "
+ case TimestampLiteral:
+ specialLiteral = "TIMESTAMP "
+ }
+ if specialLiteral != "" {
+ ctx.WritePlain(specialLiteral)
+ if err := n.Args[0].Restore(ctx); err != nil {
+ return true, errors.Annotatef(err, "An error occurred while restore FuncCallExpr.Expr")
+ }
+ return true, nil
+ }
+ if n.FnName.L == JSONMemberOf {
+ if err := n.Args[0].Restore(ctx); err != nil {
+ return true, errors.Annotatef(err, "An error occurred while restore FuncCallExpr.(MEMBER OF).Args[0]")
+ }
+ ctx.WriteKeyWord(" MEMBER OF ")
+ ctx.WritePlain("(")
+ if err := n.Args[1].Restore(ctx); err != nil {
+ return true, errors.Annotatef(err, "An error occurred while restore FuncCallExpr.(MEMBER OF).Args[1]")
+ }
+ ctx.WritePlain(")")
+ return true, nil
+ }
+ return false, nil
+}
+
// Format the ExprNode into a Writer.
func (n *FuncCallExpr) Format(w io.Writer) {
- fmt.Fprintf(w, "%s(", n.FnName.L)
if !n.specialFormatArgs(w) {
+ fmt.Fprintf(w, "%s(", n.FnName.L)
for i, arg := range n.Args {
arg.Format(w)
if i != len(n.Args)-1 {
fmt.Fprint(w, ", ")
}
}
+ fmt.Fprint(w, ")")
}
- fmt.Fprint(w, ")")
}
// specialFormatArgs formats argument list for some special functions.
func (n *FuncCallExpr) specialFormatArgs(w io.Writer) bool {
switch n.FnName.L {
case DateAdd, DateSub, AddDate, SubDate:
+ fmt.Fprintf(w, "%s(", n.FnName.L)
n.Args[0].Format(w)
fmt.Fprint(w, ", INTERVAL ")
n.Args[1].Format(w)
fmt.Fprint(w, " ")
n.Args[2].Format(w)
+ fmt.Fprint(w, ")")
+ return true
+ case JSONMemberOf:
+ n.Args[0].Format(w)
+ fmt.Fprint(w, " MEMBER OF ")
+ fmt.Fprint(w, " (")
+ n.Args[1].Format(w)
+ fmt.Fprint(w, ")")
return true
}
return false
diff --git a/parser/ast/misc.go b/parser/ast/misc.go
index b0668c7a09a7e..719371ee89439 100644
--- a/parser/ast/misc.go
+++ b/parser/ast/misc.go
@@ -98,6 +98,7 @@ type AuthOption struct {
// ByAuthString set as true, if AuthString is used for authorization. Otherwise, authorization is done by HashString.
ByAuthString bool
AuthString string
+ ByHashString bool
HashString string
AuthPlugin string
}
@@ -112,7 +113,7 @@ func (n *AuthOption) Restore(ctx *format.RestoreCtx) error {
if n.ByAuthString {
ctx.WriteKeyWord(" BY ")
ctx.WriteString(n.AuthString)
- } else if n.HashString != "" {
+ } else if n.ByHashString {
ctx.WriteKeyWord(" AS ")
ctx.WriteString(n.HashString)
}
@@ -265,6 +266,15 @@ type PlanReplayerStmt struct {
Stmt StmtNode
Analyze bool
Load bool
+
+ // Capture indicates 'plan replayer capture '
+ Capture bool
+ // Remove indicates `plan replayer capture remove
+ Remove bool
+
+ SQLDigest string
+ PlanDigest string
+
// File is used to store 2 cases:
// 1. plan replayer load 'file';
// 2. plan replayer dump explain 'file'
@@ -284,6 +294,21 @@ func (n *PlanReplayerStmt) Restore(ctx *format.RestoreCtx) error {
ctx.WriteString(n.File)
return nil
}
+ if n.Capture {
+ ctx.WriteKeyWord("PLAN REPLAYER CAPTURE ")
+ ctx.WriteString(n.SQLDigest)
+ ctx.WriteKeyWord(" ")
+ ctx.WriteString(n.PlanDigest)
+ return nil
+ }
+ if n.Remove {
+ ctx.WriteKeyWord("PLAN REPLAYER CAPTURE REMOVE ")
+ ctx.WriteString(n.SQLDigest)
+ ctx.WriteKeyWord(" ")
+ ctx.WriteString(n.PlanDigest)
+ return nil
+ }
+
ctx.WriteKeyWord("PLAN REPLAYER DUMP EXPLAIN ")
if n.Analyze {
ctx.WriteKeyWord("ANALYZE ")
@@ -506,7 +531,6 @@ type Prepared struct {
StmtType string
Params []ParamMarkerExpr
SchemaVersion int64
- UseCache bool
CachedPlan interface{}
CachedNames interface{}
}
@@ -1493,11 +1517,19 @@ const (
PasswordExpireDefault
PasswordExpireNever
PasswordExpireInterval
+ PasswordHistory
+ PasswordHistoryDefault
+ PasswordReuseInterval
+ PasswordReuseDefault
Lock
Unlock
-
+ FailedLoginAttempts
+ PasswordLockTime
+ PasswordLockTimeUnbounded
UserCommentType
UserAttributeType
+
+ UserResourceGroupName
)
type PasswordOrLockOption struct {
@@ -1521,6 +1553,25 @@ func (p *PasswordOrLockOption) Restore(ctx *format.RestoreCtx) error {
ctx.WriteKeyWord("ACCOUNT LOCK")
case Unlock:
ctx.WriteKeyWord("ACCOUNT UNLOCK")
+ case FailedLoginAttempts:
+ ctx.WriteKeyWord("FAILED_LOGIN_ATTEMPTS")
+ ctx.WritePlainf(" %d", p.Count)
+ case PasswordLockTime:
+ ctx.WriteKeyWord("PASSWORD_LOCK_TIME")
+ ctx.WritePlainf(" %d", p.Count)
+ case PasswordLockTimeUnbounded:
+ ctx.WriteKeyWord("PASSWORD_LOCK_TIME UNBOUNDED")
+ case PasswordHistory:
+ ctx.WriteKeyWord("PASSWORD HISTORY")
+ ctx.WritePlainf(" %d", p.Count)
+ case PasswordHistoryDefault:
+ ctx.WriteKeyWord("PASSWORD HISTORY DEFAULT")
+ case PasswordReuseInterval:
+ ctx.WriteKeyWord("PASSWORD REUSE INTERVAL")
+ ctx.WritePlainf(" %d", p.Count)
+ ctx.WriteKeyWord(" DAY")
+ case PasswordReuseDefault:
+ ctx.WriteKeyWord("PASSWORD REUSE INTERVAL DEFAULT")
default:
return errors.Errorf("Unsupported PasswordOrLockOption.Type %d", p.Type)
}
@@ -1543,6 +1594,16 @@ func (c *CommentOrAttributeOption) Restore(ctx *format.RestoreCtx) error {
return nil
}
+type ResourceGroupNameOption struct {
+ Value string
+}
+
+func (c *ResourceGroupNameOption) Restore(ctx *format.RestoreCtx) error {
+ ctx.WriteKeyWord(" RESOURCE GROUP ")
+ ctx.WriteName(c.Value)
+ return nil
+}
+
// CreateUserStmt creates user account.
// See https://dev.mysql.com/doc/refman/8.0/en/create-user.html
type CreateUserStmt struct {
@@ -1555,6 +1616,7 @@ type CreateUserStmt struct {
ResourceOptions []*ResourceOption
PasswordOrLockOptions []*PasswordOrLockOption
CommentOrAttributeOption *CommentOrAttributeOption
+ ResourceGroupNameOption *ResourceGroupNameOption
}
// Restore implements Node interface.
@@ -1613,6 +1675,12 @@ func (n *CreateUserStmt) Restore(ctx *format.RestoreCtx) error {
}
}
+ if n.ResourceGroupNameOption != nil {
+ if err := n.ResourceGroupNameOption.Restore(ctx); err != nil {
+ return errors.Annotatef(err, "An error occurred while restore CreateUserStmt.ResourceGroupNameOption")
+ }
+ }
+
return nil
}
@@ -1649,6 +1717,7 @@ type AlterUserStmt struct {
ResourceOptions []*ResourceOption
PasswordOrLockOptions []*PasswordOrLockOption
CommentOrAttributeOption *CommentOrAttributeOption
+ ResourceGroupNameOption *ResourceGroupNameOption
}
// Restore implements Node interface.
@@ -1710,6 +1779,12 @@ func (n *AlterUserStmt) Restore(ctx *format.RestoreCtx) error {
}
}
+ if n.ResourceGroupNameOption != nil {
+ if err := n.ResourceGroupNameOption.Restore(ctx); err != nil {
+ return errors.Annotatef(err, "An error occurred while restore AlterUserStmt.ResourceGroupNameOption")
+ }
+ }
+
return nil
}
@@ -1813,6 +1888,7 @@ type CreateBindingStmt struct {
GlobalScope bool
OriginNode StmtNode
HintedNode StmtNode
+ PlanDigest string
}
func (n *CreateBindingStmt) Restore(ctx *format.RestoreCtx) error {
@@ -1822,13 +1898,18 @@ func (n *CreateBindingStmt) Restore(ctx *format.RestoreCtx) error {
} else {
ctx.WriteKeyWord("SESSION ")
}
- ctx.WriteKeyWord("BINDING FOR ")
- if err := n.OriginNode.Restore(ctx); err != nil {
- return errors.Trace(err)
- }
- ctx.WriteKeyWord(" USING ")
- if err := n.HintedNode.Restore(ctx); err != nil {
- return errors.Trace(err)
+ if n.OriginNode == nil {
+ ctx.WriteKeyWord("BINDING FROM HISTORY USING PLAN DIGEST ")
+ ctx.WriteString(n.PlanDigest)
+ } else {
+ ctx.WriteKeyWord("BINDING FOR ")
+ if err := n.OriginNode.Restore(ctx); err != nil {
+ return errors.Trace(err)
+ }
+ ctx.WriteKeyWord(" USING ")
+ if err := n.HintedNode.Restore(ctx); err != nil {
+ return errors.Trace(err)
+ }
}
return nil
}
@@ -1839,16 +1920,18 @@ func (n *CreateBindingStmt) Accept(v Visitor) (Node, bool) {
return v.Leave(newNode)
}
n = newNode.(*CreateBindingStmt)
- origNode, ok := n.OriginNode.Accept(v)
- if !ok {
- return n, false
- }
- n.OriginNode = origNode.(StmtNode)
- hintedNode, ok := n.HintedNode.Accept(v)
- if !ok {
- return n, false
+ if n.OriginNode != nil {
+ origNode, ok := n.OriginNode.Accept(v)
+ if !ok {
+ return n, false
+ }
+ n.OriginNode = origNode.(StmtNode)
+ hintedNode, ok := n.HintedNode.Accept(v)
+ if !ok {
+ return n, false
+ }
+ n.HintedNode = hintedNode.(StmtNode)
}
- n.HintedNode = hintedNode.(StmtNode)
return v.Leave(n)
}
@@ -1859,6 +1942,7 @@ type DropBindingStmt struct {
GlobalScope bool
OriginNode StmtNode
HintedNode StmtNode
+ SQLDigest string
}
func (n *DropBindingStmt) Restore(ctx *format.RestoreCtx) error {
@@ -1869,14 +1953,19 @@ func (n *DropBindingStmt) Restore(ctx *format.RestoreCtx) error {
ctx.WriteKeyWord("SESSION ")
}
ctx.WriteKeyWord("BINDING FOR ")
- if err := n.OriginNode.Restore(ctx); err != nil {
- return errors.Trace(err)
- }
- if n.HintedNode != nil {
- ctx.WriteKeyWord(" USING ")
- if err := n.HintedNode.Restore(ctx); err != nil {
+ if n.OriginNode == nil {
+ ctx.WriteKeyWord("SQL DIGEST ")
+ ctx.WriteString(n.SQLDigest)
+ } else {
+ if err := n.OriginNode.Restore(ctx); err != nil {
return errors.Trace(err)
}
+ if n.HintedNode != nil {
+ ctx.WriteKeyWord(" USING ")
+ if err := n.HintedNode.Restore(ctx); err != nil {
+ return errors.Trace(err)
+ }
+ }
}
return nil
}
@@ -1887,17 +1976,20 @@ func (n *DropBindingStmt) Accept(v Visitor) (Node, bool) {
return v.Leave(newNode)
}
n = newNode.(*DropBindingStmt)
- origNode, ok := n.OriginNode.Accept(v)
- if !ok {
- return n, false
- }
- n.OriginNode = origNode.(StmtNode)
- if n.HintedNode != nil {
- hintedNode, ok := n.HintedNode.Accept(v)
+ if n.OriginNode != nil {
+ // OriginNode is nil means we build drop binding by sql digest
+ origNode, ok := n.OriginNode.Accept(v)
if !ok {
return n, false
}
- n.HintedNode = hintedNode.(StmtNode)
+ n.OriginNode = origNode.(StmtNode)
+ if n.HintedNode != nil {
+ hintedNode, ok := n.HintedNode.Accept(v)
+ if !ok {
+ return n, false
+ }
+ n.HintedNode = hintedNode.(StmtNode)
+ }
}
return v.Leave(n)
}
@@ -1918,6 +2010,7 @@ type SetBindingStmt struct {
BindingStatusType BindingStatusType
OriginNode StmtNode
HintedNode StmtNode
+ SQLDigest string
}
func (n *SetBindingStmt) Restore(ctx *format.RestoreCtx) error {
@@ -1930,14 +2023,19 @@ func (n *SetBindingStmt) Restore(ctx *format.RestoreCtx) error {
ctx.WriteKeyWord("DISABLED ")
}
ctx.WriteKeyWord("FOR ")
- if err := n.OriginNode.Restore(ctx); err != nil {
- return errors.Trace(err)
- }
- if n.HintedNode != nil {
- ctx.WriteKeyWord(" USING ")
- if err := n.HintedNode.Restore(ctx); err != nil {
+ if n.OriginNode == nil {
+ ctx.WriteKeyWord("SQL DIGEST ")
+ ctx.WriteString(n.SQLDigest)
+ } else {
+ if err := n.OriginNode.Restore(ctx); err != nil {
return errors.Trace(err)
}
+ if n.HintedNode != nil {
+ ctx.WriteKeyWord(" USING ")
+ if err := n.HintedNode.Restore(ctx); err != nil {
+ return errors.Trace(err)
+ }
+ }
}
return nil
}
@@ -1948,17 +2046,20 @@ func (n *SetBindingStmt) Accept(v Visitor) (Node, bool) {
return v.Leave(newNode)
}
n = newNode.(*SetBindingStmt)
- origNode, ok := n.OriginNode.Accept(v)
- if !ok {
- return n, false
- }
- n.OriginNode = origNode.(StmtNode)
- if n.HintedNode != nil {
- hintedNode, ok := n.HintedNode.Accept(v)
+ if n.OriginNode != nil {
+ // OriginNode is nil means we set binding stmt by sql digest
+ origNode, ok := n.OriginNode.Accept(v)
if !ok {
return n, false
}
- n.HintedNode = hintedNode.(StmtNode)
+ n.OriginNode = origNode.(StmtNode)
+ if n.HintedNode != nil {
+ hintedNode, ok := n.HintedNode.Accept(v)
+ if !ok {
+ return n, false
+ }
+ n.HintedNode = hintedNode.(StmtNode)
+ }
}
return v.Leave(n)
}
@@ -3648,9 +3749,13 @@ func (n *TableOptimizerHint) Restore(ctx *format.RestoreCtx) error {
}
ctx.WriteName(n.QBName.String())
}
+ if n.HintName.L == "qb_name" && len(n.Tables) == 0 {
+ ctx.WritePlain(")")
+ return nil
+ }
// Hints without args except query block.
switch n.HintName.L {
- case "mpp_1phase_agg", "mpp_2phase_agg", "hash_agg", "stream_agg", "agg_to_cop", "read_consistent_replica", "no_index_merge", "qb_name", "ignore_plan_cache", "limit_to_cop", "straight_join", "merge", "no_decorrelate":
+ case "mpp_1phase_agg", "mpp_2phase_agg", "hash_agg", "stream_agg", "agg_to_cop", "read_consistent_replica", "no_index_merge", "ignore_plan_cache", "limit_to_cop", "straight_join", "merge", "no_decorrelate":
ctx.WritePlain(")")
return nil
}
@@ -3670,7 +3775,7 @@ func (n *TableOptimizerHint) Restore(ctx *format.RestoreCtx) error {
}
table.Restore(ctx)
}
- case "use_index", "ignore_index", "use_index_merge", "force_index":
+ case "use_index", "ignore_index", "use_index_merge", "force_index", "order_index", "no_order_index":
n.Tables[0].Restore(ctx)
ctx.WritePlain(" ")
for i, index := range n.Indexes {
@@ -3679,6 +3784,16 @@ func (n *TableOptimizerHint) Restore(ctx *format.RestoreCtx) error {
}
ctx.WriteName(index.String())
}
+ case "qb_name":
+ if len(n.Tables) > 0 {
+ ctx.WritePlain(", ")
+ for i, table := range n.Tables {
+ if i != 0 {
+ ctx.WritePlain(". ")
+ }
+ table.Restore(ctx)
+ }
+ }
case "use_toja", "use_cascades":
if n.HintData.(bool) {
ctx.WritePlain("TRUE")
@@ -3709,9 +3824,9 @@ func (n *TableOptimizerHint) Restore(ctx *format.RestoreCtx) error {
ctx.WriteString(hintData.To)
case "set_var":
hintData := n.HintData.(HintSetVar)
- ctx.WriteString(hintData.VarName)
- ctx.WritePlain(", ")
- ctx.WriteString(hintData.Value)
+ ctx.WritePlain(hintData.VarName)
+ ctx.WritePlain(" = ")
+ ctx.WritePlain(hintData.Value)
}
ctx.WritePlain(")")
return nil
diff --git a/parser/ast/misc_test.go b/parser/ast/misc_test.go
index 8042a171874d0..7379279d51567 100644
--- a/parser/ast/misc_test.go
+++ b/parser/ast/misc_test.go
@@ -228,6 +228,18 @@ func TestTableOptimizerHintRestore(t *testing.T) {
{"IGNORE_INDEX(@sel_1 t1 c1)", "IGNORE_INDEX(@`sel_1` `t1` `c1`)"},
{"IGNORE_INDEX(t1@sel_1 c1)", "IGNORE_INDEX(`t1`@`sel_1` `c1`)"},
{"IGNORE_INDEX(t1@sel_1 partition(p0, p1) c1)", "IGNORE_INDEX(`t1`@`sel_1` PARTITION(`p0`, `p1`) `c1`)"},
+ {"ORDER_INDEX(t1 c1)", "ORDER_INDEX(`t1` `c1`)"},
+ {"ORDER_INDEX(test.t1 c1)", "ORDER_INDEX(`test`.`t1` `c1`)"},
+ {"ORDER_INDEX(@sel_1 t1 c1)", "ORDER_INDEX(@`sel_1` `t1` `c1`)"},
+ {"ORDER_INDEX(t1@sel_1 c1)", "ORDER_INDEX(`t1`@`sel_1` `c1`)"},
+ {"ORDER_INDEX(test.t1@sel_1 c1)", "ORDER_INDEX(`test`.`t1`@`sel_1` `c1`)"},
+ {"ORDER_INDEX(test.t1@sel_1 partition(p0) c1)", "ORDER_INDEX(`test`.`t1`@`sel_1` PARTITION(`p0`) `c1`)"},
+ {"NO_ORDER_INDEX(t1 c1)", "NO_ORDER_INDEX(`t1` `c1`)"},
+ {"NO_ORDER_INDEX(test.t1 c1)", "NO_ORDER_INDEX(`test`.`t1` `c1`)"},
+ {"NO_ORDER_INDEX(@sel_1 t1 c1)", "NO_ORDER_INDEX(@`sel_1` `t1` `c1`)"},
+ {"NO_ORDER_INDEX(t1@sel_1 c1)", "NO_ORDER_INDEX(`t1`@`sel_1` `c1`)"},
+ {"NO_ORDER_INDEX(test.t1@sel_1 c1)", "NO_ORDER_INDEX(`test`.`t1`@`sel_1` `c1`)"},
+ {"NO_ORDER_INDEX(test.t1@sel_1 partition(p0) c1)", "NO_ORDER_INDEX(`test`.`t1`@`sel_1` PARTITION(`p0`) `c1`)"},
{"TIDB_SMJ(`t1`)", "TIDB_SMJ(`t1`)"},
{"TIDB_SMJ(t1)", "TIDB_SMJ(`t1`)"},
{"TIDB_SMJ(t1,t2)", "TIDB_SMJ(`t1`, `t2`)"},
diff --git a/parser/auth/auth.go b/parser/auth/auth.go
index 109fdda04b644..cb9abb883a33d 100644
--- a/parser/auth/auth.go
+++ b/parser/auth/auth.go
@@ -33,6 +33,7 @@ type UserIdentity struct {
CurrentUser bool
AuthUsername string // Username matched in privileges system
AuthHostname string // Match in privs system (i.e. could be a wildcard)
+ AuthPlugin string // The plugin specified in handshake, only used during authentication.
}
// Restore implements Node interface.
diff --git a/parser/consistent_test.go b/parser/consistent_test.go
index e78b7f31ddddd..1acc1a58bc850 100644
--- a/parser/consistent_test.go
+++ b/parser/consistent_test.go
@@ -14,7 +14,7 @@
package parser
import (
- "io/ioutil"
+ gio "io"
"os"
"sort"
"strings"
@@ -27,7 +27,7 @@ func TestKeywordConsistent(t *testing.T) {
parserFilename := "parser.y"
parserFile, err := os.Open(parserFilename)
requires.NoError(t, err)
- data, err := ioutil.ReadAll(parserFile)
+ data, err := gio.ReadAll(parserFile)
requires.NoError(t, err)
content := string(data)
diff --git a/parser/duration/BUILD.bazel b/parser/duration/BUILD.bazel
new file mode 100644
index 0000000000000..8015c5ce5df7d
--- /dev/null
+++ b/parser/duration/BUILD.bazel
@@ -0,0 +1,18 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "duration",
+ srcs = ["duration.go"],
+ importpath = "github.com/pingcap/tidb/parser/duration",
+ visibility = ["//visibility:public"],
+ deps = ["@com_github_pingcap_errors//:errors"],
+)
+
+go_test(
+ name = "duration_test",
+ timeout = "short",
+ srcs = ["duration_test.go"],
+ embed = [":duration"],
+ flaky = True,
+ deps = ["@com_github_stretchr_testify//require"],
+)
diff --git a/parser/duration/duration.go b/parser/duration/duration.go
new file mode 100644
index 0000000000000..6eb13111b0424
--- /dev/null
+++ b/parser/duration/duration.go
@@ -0,0 +1,73 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package duration provides a customized duration, which supports unit 'd', 'h' and 'm'
+package duration
+
+import (
+ "strconv"
+ "time"
+ "unicode"
+
+ "github.com/pingcap/errors"
+)
+
+func readFloat(s string) (float64, string, error) {
+ numbers := ""
+ for pos, ch := range s {
+ if !unicode.IsDigit(ch) && ch != '.' {
+ numbers = s[:pos]
+ break
+ }
+ }
+ if len(numbers) > 0 {
+ i, err := strconv.ParseFloat(numbers, 64)
+ if err != nil {
+ return 0, s, err
+ }
+ return i, s[len(numbers):], nil
+ }
+ return 0, s, errors.New("fail to read an integer")
+}
+
+// ParseDuration parses the duration which contains 'd', 'h' and 'm'
+func ParseDuration(s string) (time.Duration, error) {
+ duration := time.Duration(0)
+
+ if s == "0" {
+ return 0, nil
+ }
+
+ var err error
+ var i float64
+ for len(s) > 0 {
+ i, s, err = readFloat(s)
+ if err != nil {
+ return 0, err
+ }
+ switch s[0] {
+ case 'd':
+ duration += time.Duration(i * float64(time.Hour*24))
+ case 'h':
+ duration += time.Duration(i * float64(time.Hour))
+ case 'm':
+ duration += time.Duration(i * float64(time.Minute))
+ default:
+ return 0, errors.Errorf("unknown unit %c", s[0])
+ }
+
+ s = s[1:]
+ }
+
+ return duration, nil
+}
diff --git a/parser/duration/duration_test.go b/parser/duration/duration_test.go
new file mode 100644
index 0000000000000..4a1555db69e41
--- /dev/null
+++ b/parser/duration/duration_test.go
@@ -0,0 +1,65 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package duration
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestParseDuration(t *testing.T) {
+ cases := []struct {
+ str string
+ duration time.Duration
+ }{
+ {
+ "1h",
+ time.Hour,
+ },
+ {
+ "1h100m",
+ time.Hour + 100*time.Minute,
+ },
+ {
+ "1d10000m",
+ 24*time.Hour + 10000*time.Minute,
+ },
+ {
+ "1d100h",
+ 24*time.Hour + 100*time.Hour,
+ },
+ {
+ "1.5d",
+ 36 * time.Hour,
+ },
+ {
+ "1d1.5h",
+ 24*time.Hour + time.Hour + 30*time.Minute,
+ },
+ {
+ "1d3.555h",
+ 24*time.Hour + time.Duration(3.555*float64(time.Hour)),
+ },
+ }
+
+ for _, c := range cases {
+ t.Run(c.str, func(t *testing.T) {
+ d, err := ParseDuration(c.str)
+ require.NoError(t, err)
+ require.Equal(t, c.duration, d)
+ })
+ }
+}
diff --git a/parser/format/format.go b/parser/format/format.go
index adada122e255e..a60c8d7b6589d 100644
--- a/parser/format/format.go
+++ b/parser/format/format.go
@@ -235,6 +235,9 @@ const (
RestoreTiDBSpecialComment
SkipPlacementRuleForRestore
+ RestoreWithTTLEnableOff
+ RestoreWithoutSchemaName
+ RestoreWithoutTableName
)
const (
@@ -246,6 +249,16 @@ func (rf RestoreFlags) has(flag RestoreFlags) bool {
return rf&flag != 0
}
+// HasWithoutSchemaNameFlag returns a boolean indicating when `rf` has `RestoreWithoutSchemaName` flag.
+func (rf RestoreFlags) HasWithoutSchemaNameFlag() bool {
+ return rf.has(RestoreWithoutSchemaName)
+}
+
+// HasWithoutTableNameFlag returns a boolean indicating when `rf` has `RestoreWithoutTableName` flag.
+func (rf RestoreFlags) HasWithoutTableNameFlag() bool {
+ return rf.has(RestoreWithoutTableName)
+}
+
// HasStringSingleQuotesFlag returns a boolean indicating when `rf` has `RestoreStringSingleQuotes` flag.
func (rf RestoreFlags) HasStringSingleQuotesFlag() bool {
return rf.has(RestoreStringSingleQuotes)
@@ -321,6 +334,11 @@ func (rf RestoreFlags) HasSkipPlacementRuleForRestoreFlag() bool {
return rf.has(SkipPlacementRuleForRestore)
}
+// HasRestoreWithTTLEnableOff returns a boolean indicating whether to force set TTL_ENABLE='OFF' when restoring a TTL table
+func (rf RestoreFlags) HasRestoreWithTTLEnableOff() bool {
+ return rf.has(RestoreWithTTLEnableOff)
+}
+
// RestoreCtx is `Restore` context to hold flags and writer.
type RestoreCtx struct {
Flags RestoreFlags
diff --git a/parser/format/format_test.go b/parser/format/format_test.go
index 429c2c27c19d1..bc04033214cdd 100644
--- a/parser/format/format_test.go
+++ b/parser/format/format_test.go
@@ -15,7 +15,7 @@ package format
import (
"bytes"
- "io/ioutil"
+ "io"
"strings"
"testing"
@@ -26,7 +26,7 @@ import (
func checkFormat(t *testing.T, f Formatter, buf *bytes.Buffer, str, expect string) {
_, err := f.Format(str, 3)
require.NoError(t, err)
- b, err := ioutil.ReadAll(buf)
+ b, err := io.ReadAll(buf)
require.NoError(t, err)
require.Equal(t, expect, string(b))
}
diff --git a/parser/hintparser.go b/parser/hintparser.go
index 1b14e7d292e1e..998d9b3823d08 100644
--- a/parser/hintparser.go
+++ b/parser/hintparser.go
@@ -41,18 +41,18 @@ type yyhintXError struct {
}
const (
- yyhintDefault = 57424
+ yyhintDefault = 57426
yyhintEOFCode = 57344
yyhintErrCode = 57345
hintAggToCop = 57379
hintBCJoin = 57394
hintBKA = 57355
hintBNL = 57357
- hintDupsWeedOut = 57420
- hintFalse = 57416
- hintFirstMatch = 57421
- hintForceIndex = 57406
- hintGB = 57419
+ hintDupsWeedOut = 57422
+ hintFalse = 57418
+ hintFirstMatch = 57423
+ hintForceIndex = 57408
+ hintGB = 57421
hintHashAgg = 57381
hintHashJoin = 57359
hintHashJoinBuild = 57360
@@ -70,12 +70,12 @@ const (
hintJoinOrder = 57352
hintJoinPrefix = 57353
hintJoinSuffix = 57354
- hintLeading = 57408
- hintLimitToCop = 57405
- hintLooseScan = 57422
- hintMB = 57418
+ hintLeading = 57410
+ hintLimitToCop = 57407
+ hintLooseScan = 57424
+ hintMB = 57420
hintMRR = 57367
- hintMaterialization = 57423
+ hintMaterialization = 57425
hintMaxExecutionTime = 57375
hintMemoryQuota = 57388
hintMerge = 57363
@@ -83,175 +83,179 @@ const (
hintMpp2PhaseAgg = 57383
hintNoBKA = 57356
hintNoBNL = 57358
- hintNoDecorrelate = 57410
+ hintNoDecorrelate = 57412
hintNoHashJoin = 57362
hintNoICP = 57369
hintNoIndexMerge = 57366
hintNoMRR = 57368
hintNoMerge = 57364
+ hintNoOrderIndex = 57401
hintNoRangeOptimization = 57370
hintNoSemijoin = 57374
hintNoSkipScan = 57372
hintNoSwapJoinInputs = 57389
- hintNthPlan = 57404
- hintOLAP = 57411
- hintOLTP = 57412
- hintPartition = 57413
+ hintNthPlan = 57406
+ hintOLAP = 57413
+ hintOLTP = 57414
+ hintOrderIndex = 57400
+ hintPartition = 57415
hintQBName = 57378
hintQueryType = 57390
hintReadConsistentReplica = 57391
hintReadFromStorage = 57392
hintResourceGroup = 57377
hintSMJoin = 57393
- hintSemiJoinRewrite = 57409
+ hintSemiJoinRewrite = 57411
hintSemijoin = 57373
hintSetVar = 57376
hintShuffleJoin = 57395
hintSingleAtIdentifier = 57349
hintSkipScan = 57371
- hintStraightJoin = 57407
+ hintStraightJoin = 57409
hintStreamAgg = 57396
hintStringLit = 57350
hintSwapJoinInputs = 57397
- hintTiFlash = 57415
- hintTiKV = 57414
- hintTimeRange = 57402
- hintTrue = 57417
- hintUseCascades = 57403
+ hintTiFlash = 57417
+ hintTiKV = 57416
+ hintTimeRange = 57404
+ hintTrue = 57419
+ hintUseCascades = 57405
hintUseIndex = 57399
hintUseIndexMerge = 57398
- hintUsePlanCache = 57400
- hintUseToja = 57401
+ hintUsePlanCache = 57402
+ hintUseToja = 57403
yyhintMaxDepth = 200
- yyhintTabOfs = -193
+ yyhintTabOfs = -197
)
var (
yyhintXLAT = map[int]int{
- 41: 0, // ')' (146x)
- 57379: 1, // hintAggToCop (133x)
- 57394: 2, // hintBCJoin (133x)
- 57355: 3, // hintBKA (133x)
- 57357: 4, // hintBNL (133x)
- 57406: 5, // hintForceIndex (133x)
- 57381: 6, // hintHashAgg (133x)
- 57359: 7, // hintHashJoin (133x)
- 57360: 8, // hintHashJoinBuild (133x)
- 57361: 9, // hintHashJoinProbe (133x)
- 57384: 10, // hintIgnoreIndex (133x)
- 57380: 11, // hintIgnorePlanCache (133x)
- 57365: 12, // hintIndexMerge (133x)
- 57385: 13, // hintInlHashJoin (133x)
- 57386: 14, // hintInlJoin (133x)
- 57387: 15, // hintInlMergeJoin (133x)
- 57351: 16, // hintJoinFixedOrder (133x)
- 57352: 17, // hintJoinOrder (133x)
- 57353: 18, // hintJoinPrefix (133x)
- 57354: 19, // hintJoinSuffix (133x)
- 57408: 20, // hintLeading (133x)
- 57405: 21, // hintLimitToCop (133x)
- 57375: 22, // hintMaxExecutionTime (133x)
- 57388: 23, // hintMemoryQuota (133x)
- 57363: 24, // hintMerge (133x)
- 57382: 25, // hintMpp1PhaseAgg (133x)
- 57383: 26, // hintMpp2PhaseAgg (133x)
- 57367: 27, // hintMRR (133x)
- 57356: 28, // hintNoBKA (133x)
- 57358: 29, // hintNoBNL (133x)
- 57410: 30, // hintNoDecorrelate (133x)
- 57362: 31, // hintNoHashJoin (133x)
- 57369: 32, // hintNoICP (133x)
- 57366: 33, // hintNoIndexMerge (133x)
- 57364: 34, // hintNoMerge (133x)
- 57368: 35, // hintNoMRR (133x)
- 57370: 36, // hintNoRangeOptimization (133x)
- 57374: 37, // hintNoSemijoin (133x)
- 57372: 38, // hintNoSkipScan (133x)
- 57389: 39, // hintNoSwapJoinInputs (133x)
- 57404: 40, // hintNthPlan (133x)
- 57378: 41, // hintQBName (133x)
- 57390: 42, // hintQueryType (133x)
- 57391: 43, // hintReadConsistentReplica (133x)
- 57392: 44, // hintReadFromStorage (133x)
- 57377: 45, // hintResourceGroup (133x)
- 57373: 46, // hintSemijoin (133x)
- 57409: 47, // hintSemiJoinRewrite (133x)
- 57376: 48, // hintSetVar (133x)
- 57395: 49, // hintShuffleJoin (133x)
- 57371: 50, // hintSkipScan (133x)
- 57393: 51, // hintSMJoin (133x)
- 57407: 52, // hintStraightJoin (133x)
- 57396: 53, // hintStreamAgg (133x)
- 57397: 54, // hintSwapJoinInputs (133x)
- 57402: 55, // hintTimeRange (133x)
- 57403: 56, // hintUseCascades (133x)
- 57399: 57, // hintUseIndex (133x)
- 57398: 58, // hintUseIndexMerge (133x)
- 57400: 59, // hintUsePlanCache (133x)
- 57401: 60, // hintUseToja (133x)
- 44: 61, // ',' (130x)
- 57420: 62, // hintDupsWeedOut (110x)
- 57421: 63, // hintFirstMatch (110x)
- 57422: 64, // hintLooseScan (110x)
- 57423: 65, // hintMaterialization (110x)
- 57415: 66, // hintTiFlash (110x)
- 57414: 67, // hintTiKV (110x)
- 57416: 68, // hintFalse (109x)
- 57411: 69, // hintOLAP (109x)
- 57412: 70, // hintOLTP (109x)
- 57417: 71, // hintTrue (109x)
- 57419: 72, // hintGB (108x)
- 57418: 73, // hintMB (108x)
- 57347: 74, // hintIdentifier (107x)
- 57349: 75, // hintSingleAtIdentifier (93x)
- 93: 76, // ']' (84x)
- 46: 77, // '.' (83x)
- 57413: 78, // hintPartition (78x)
- 61: 79, // '=' (74x)
- 40: 80, // '(' (69x)
- 57344: 81, // $end (25x)
- 57444: 82, // QueryBlockOpt (20x)
- 57436: 83, // Identifier (15x)
- 57346: 84, // hintIntLit (8x)
- 57350: 85, // hintStringLit (5x)
- 57426: 86, // CommaOpt (4x)
- 57432: 87, // HintTable (4x)
- 57433: 88, // HintTableList (4x)
- 91: 89, // '[' (3x)
- 57425: 90, // BooleanHintName (2x)
- 57427: 91, // HintIndexList (2x)
- 57429: 92, // HintStorageType (2x)
- 57430: 93, // HintStorageTypeAndTable (2x)
- 57434: 94, // HintTableListOpt (2x)
- 57439: 95, // JoinOrderOptimizerHintName (2x)
- 57440: 96, // NullaryHintName (2x)
- 57443: 97, // PartitionListOpt (2x)
- 57446: 98, // StorageOptimizerHintOpt (2x)
- 57447: 99, // SubqueryOptimizerHintName (2x)
- 57450: 100, // SubqueryStrategy (2x)
- 57451: 101, // SupportedIndexLevelOptimizerHintName (2x)
- 57452: 102, // SupportedTableLevelOptimizerHintName (2x)
- 57453: 103, // TableOptimizerHintOpt (2x)
- 57455: 104, // UnsupportedIndexLevelOptimizerHintName (2x)
- 57456: 105, // UnsupportedTableLevelOptimizerHintName (2x)
- 57458: 106, // ViewName (2x)
- 57428: 107, // HintQueryType (1x)
- 57431: 108, // HintStorageTypeAndTableList (1x)
- 57435: 109, // HintTrueOrFalse (1x)
- 57437: 110, // IndexNameList (1x)
- 57438: 111, // IndexNameListOpt (1x)
- 57441: 112, // OptimizerHintList (1x)
- 57442: 113, // PartitionList (1x)
- 57445: 114, // Start (1x)
- 57448: 115, // SubqueryStrategies (1x)
- 57449: 116, // SubqueryStrategiesOpt (1x)
- 57454: 117, // UnitOfBytes (1x)
- 57457: 118, // Value (1x)
- 57459: 119, // ViewNameList (1x)
- 57424: 120, // $default (0x)
- 57345: 121, // error (0x)
- 57348: 122, // hintInvalid (0x)
+ 41: 0, // ')' (148x)
+ 57379: 1, // hintAggToCop (135x)
+ 57394: 2, // hintBCJoin (135x)
+ 57355: 3, // hintBKA (135x)
+ 57357: 4, // hintBNL (135x)
+ 57408: 5, // hintForceIndex (135x)
+ 57381: 6, // hintHashAgg (135x)
+ 57359: 7, // hintHashJoin (135x)
+ 57360: 8, // hintHashJoinBuild (135x)
+ 57361: 9, // hintHashJoinProbe (135x)
+ 57384: 10, // hintIgnoreIndex (135x)
+ 57380: 11, // hintIgnorePlanCache (135x)
+ 57365: 12, // hintIndexMerge (135x)
+ 57385: 13, // hintInlHashJoin (135x)
+ 57386: 14, // hintInlJoin (135x)
+ 57387: 15, // hintInlMergeJoin (135x)
+ 57351: 16, // hintJoinFixedOrder (135x)
+ 57352: 17, // hintJoinOrder (135x)
+ 57353: 18, // hintJoinPrefix (135x)
+ 57354: 19, // hintJoinSuffix (135x)
+ 57410: 20, // hintLeading (135x)
+ 57407: 21, // hintLimitToCop (135x)
+ 57375: 22, // hintMaxExecutionTime (135x)
+ 57388: 23, // hintMemoryQuota (135x)
+ 57363: 24, // hintMerge (135x)
+ 57382: 25, // hintMpp1PhaseAgg (135x)
+ 57383: 26, // hintMpp2PhaseAgg (135x)
+ 57367: 27, // hintMRR (135x)
+ 57356: 28, // hintNoBKA (135x)
+ 57358: 29, // hintNoBNL (135x)
+ 57412: 30, // hintNoDecorrelate (135x)
+ 57362: 31, // hintNoHashJoin (135x)
+ 57369: 32, // hintNoICP (135x)
+ 57366: 33, // hintNoIndexMerge (135x)
+ 57364: 34, // hintNoMerge (135x)
+ 57368: 35, // hintNoMRR (135x)
+ 57401: 36, // hintNoOrderIndex (135x)
+ 57370: 37, // hintNoRangeOptimization (135x)
+ 57374: 38, // hintNoSemijoin (135x)
+ 57372: 39, // hintNoSkipScan (135x)
+ 57389: 40, // hintNoSwapJoinInputs (135x)
+ 57406: 41, // hintNthPlan (135x)
+ 57400: 42, // hintOrderIndex (135x)
+ 57378: 43, // hintQBName (135x)
+ 57390: 44, // hintQueryType (135x)
+ 57391: 45, // hintReadConsistentReplica (135x)
+ 57392: 46, // hintReadFromStorage (135x)
+ 57377: 47, // hintResourceGroup (135x)
+ 57373: 48, // hintSemijoin (135x)
+ 57411: 49, // hintSemiJoinRewrite (135x)
+ 57376: 50, // hintSetVar (135x)
+ 57395: 51, // hintShuffleJoin (135x)
+ 57371: 52, // hintSkipScan (135x)
+ 57393: 53, // hintSMJoin (135x)
+ 57409: 54, // hintStraightJoin (135x)
+ 57396: 55, // hintStreamAgg (135x)
+ 57397: 56, // hintSwapJoinInputs (135x)
+ 57404: 57, // hintTimeRange (135x)
+ 57405: 58, // hintUseCascades (135x)
+ 57399: 59, // hintUseIndex (135x)
+ 57398: 60, // hintUseIndexMerge (135x)
+ 57402: 61, // hintUsePlanCache (135x)
+ 57403: 62, // hintUseToja (135x)
+ 44: 63, // ',' (132x)
+ 57422: 64, // hintDupsWeedOut (112x)
+ 57423: 65, // hintFirstMatch (112x)
+ 57424: 66, // hintLooseScan (112x)
+ 57425: 67, // hintMaterialization (112x)
+ 57417: 68, // hintTiFlash (112x)
+ 57416: 69, // hintTiKV (112x)
+ 57418: 70, // hintFalse (111x)
+ 57413: 71, // hintOLAP (111x)
+ 57414: 72, // hintOLTP (111x)
+ 57419: 73, // hintTrue (111x)
+ 57421: 74, // hintGB (110x)
+ 57420: 75, // hintMB (110x)
+ 57347: 76, // hintIdentifier (109x)
+ 57349: 77, // hintSingleAtIdentifier (95x)
+ 93: 78, // ']' (86x)
+ 46: 79, // '.' (85x)
+ 57415: 80, // hintPartition (80x)
+ 61: 81, // '=' (76x)
+ 40: 82, // '(' (71x)
+ 57344: 83, // $end (25x)
+ 57446: 84, // QueryBlockOpt (20x)
+ 57438: 85, // Identifier (15x)
+ 57346: 86, // hintIntLit (8x)
+ 57350: 87, // hintStringLit (5x)
+ 57428: 88, // CommaOpt (4x)
+ 57434: 89, // HintTable (4x)
+ 57435: 90, // HintTableList (4x)
+ 91: 91, // '[' (3x)
+ 57427: 92, // BooleanHintName (2x)
+ 57429: 93, // HintIndexList (2x)
+ 57431: 94, // HintStorageType (2x)
+ 57432: 95, // HintStorageTypeAndTable (2x)
+ 57436: 96, // HintTableListOpt (2x)
+ 57441: 97, // JoinOrderOptimizerHintName (2x)
+ 57442: 98, // NullaryHintName (2x)
+ 57445: 99, // PartitionListOpt (2x)
+ 57448: 100, // StorageOptimizerHintOpt (2x)
+ 57449: 101, // SubqueryOptimizerHintName (2x)
+ 57452: 102, // SubqueryStrategy (2x)
+ 57453: 103, // SupportedIndexLevelOptimizerHintName (2x)
+ 57454: 104, // SupportedTableLevelOptimizerHintName (2x)
+ 57455: 105, // TableOptimizerHintOpt (2x)
+ 57457: 106, // UnsupportedIndexLevelOptimizerHintName (2x)
+ 57458: 107, // UnsupportedTableLevelOptimizerHintName (2x)
+ 57460: 108, // ViewName (2x)
+ 57430: 109, // HintQueryType (1x)
+ 57433: 110, // HintStorageTypeAndTableList (1x)
+ 57437: 111, // HintTrueOrFalse (1x)
+ 57439: 112, // IndexNameList (1x)
+ 57440: 113, // IndexNameListOpt (1x)
+ 57443: 114, // OptimizerHintList (1x)
+ 57444: 115, // PartitionList (1x)
+ 57447: 116, // Start (1x)
+ 57450: 117, // SubqueryStrategies (1x)
+ 57451: 118, // SubqueryStrategiesOpt (1x)
+ 57456: 119, // UnitOfBytes (1x)
+ 57459: 120, // Value (1x)
+ 57461: 121, // ViewNameList (1x)
+ 57426: 122, // $default (0x)
+ 57345: 123, // error (0x)
+ 57348: 124, // hintInvalid (0x)
}
yyhintSymNames = []string{
@@ -291,11 +295,13 @@ var (
"hintNoIndexMerge",
"hintNoMerge",
"hintNoMRR",
+ "hintNoOrderIndex",
"hintNoRangeOptimization",
"hintNoSemijoin",
"hintNoSkipScan",
"hintNoSwapJoinInputs",
"hintNthPlan",
+ "hintOrderIndex",
"hintQBName",
"hintQueryType",
"hintReadConsistentReplica",
@@ -382,89 +388,76 @@ var (
yyhintReductions = []struct{ xsym, components int }{
{0, 1},
+ {116, 1},
{114, 1},
- {112, 1},
- {112, 3},
- {112, 1},
- {112, 3},
- {103, 4},
- {103, 4},
- {103, 4},
- {103, 4},
- {103, 4},
- {103, 4},
- {103, 5},
- {103, 5},
- {103, 5},
- {103, 6},
- {103, 4},
- {103, 4},
- {103, 6},
- {103, 6},
- {103, 6},
- {103, 5},
- {103, 4},
- {103, 5},
- {98, 5},
- {108, 1},
- {108, 3},
- {93, 4},
- {82, 0},
- {82, 1},
- {86, 0},
- {86, 1},
- {97, 0},
- {97, 4},
- {113, 1},
- {113, 3},
- {94, 1},
- {94, 1},
- {88, 2},
- {88, 3},
- {87, 3},
- {87, 5},
- {119, 3},
- {119, 1},
- {106, 2},
- {106, 1},
- {91, 4},
- {111, 0},
- {111, 1},
+ {114, 3},
+ {114, 1},
+ {114, 3},
+ {105, 4},
+ {105, 4},
+ {105, 4},
+ {105, 4},
+ {105, 4},
+ {105, 4},
+ {105, 5},
+ {105, 5},
+ {105, 5},
+ {105, 6},
+ {105, 4},
+ {105, 4},
+ {105, 6},
+ {105, 6},
+ {105, 6},
+ {105, 5},
+ {105, 4},
+ {105, 5},
+ {100, 5},
{110, 1},
{110, 3},
- {116, 0},
- {116, 1},
+ {95, 4},
+ {84, 0},
+ {84, 1},
+ {88, 0},
+ {88, 1},
+ {99, 0},
+ {99, 4},
{115, 1},
{115, 3},
+ {96, 1},
+ {96, 1},
+ {90, 2},
+ {90, 3},
+ {89, 3},
+ {89, 5},
+ {121, 3},
+ {121, 1},
+ {108, 2},
+ {108, 1},
+ {93, 4},
+ {113, 0},
+ {113, 1},
+ {112, 1},
+ {112, 3},
+ {118, 0},
{118, 1},
- {118, 1},
- {118, 1},
- {117, 1},
{117, 1},
- {109, 1},
- {109, 1},
- {95, 1},
- {95, 1},
- {95, 1},
- {105, 1},
- {105, 1},
- {105, 1},
- {105, 1},
- {105, 1},
- {105, 1},
- {102, 1},
- {102, 1},
- {102, 1},
- {102, 1},
- {102, 1},
- {102, 1},
- {102, 1},
- {102, 1},
- {102, 1},
- {102, 1},
- {102, 1},
- {102, 1},
- {102, 1},
+ {117, 3},
+ {120, 1},
+ {120, 1},
+ {120, 1},
+ {119, 1},
+ {119, 1},
+ {111, 1},
+ {111, 1},
+ {97, 1},
+ {97, 1},
+ {97, 1},
+ {107, 1},
+ {107, 1},
+ {107, 1},
+ {107, 1},
+ {107, 1},
+ {107, 1},
{104, 1},
{104, 1},
{104, 1},
@@ -472,449 +465,471 @@ var (
{104, 1},
{104, 1},
{104, 1},
+ {104, 1},
+ {104, 1},
+ {104, 1},
+ {104, 1},
+ {104, 1},
+ {104, 1},
+ {106, 1},
+ {106, 1},
+ {106, 1},
+ {106, 1},
+ {106, 1},
+ {106, 1},
+ {106, 1},
+ {103, 1},
+ {103, 1},
+ {103, 1},
+ {103, 1},
+ {103, 1},
+ {103, 1},
{101, 1},
{101, 1},
- {101, 1},
- {101, 1},
- {99, 1},
- {99, 1},
- {100, 1},
- {100, 1},
- {100, 1},
- {100, 1},
- {90, 1},
- {90, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {96, 1},
- {107, 1},
- {107, 1},
+ {102, 1},
+ {102, 1},
+ {102, 1},
+ {102, 1},
{92, 1},
{92, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
- {83, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {98, 1},
+ {109, 1},
+ {109, 1},
+ {94, 1},
+ {94, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
+ {85, 1},
}
yyhintXErrors = map[yyhintXError]string{}
- yyhintParseTab = [280][]uint16{
+ yyhintParseTab = [284][]uint16{
// 0
- {1: 258, 226, 219, 221, 248, 254, 234, 235, 236, 246, 262, 238, 230, 228, 233, 198, 216, 217, 218, 237, 259, 205, 210, 229, 255, 256, 239, 220, 222, 265, 223, 241, 260, 224, 240, 242, 250, 244, 232, 206, 209, 214, 261, 215, 208, 249, 264, 207, 227, 243, 225, 263, 257, 231, 211, 252, 245, 247, 253, 251, 90: 212, 95: 199, 213, 98: 197, 204, 101: 203, 201, 196, 202, 200, 112: 195, 114: 194},
- {81: 193},
- {1: 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 361, 81: 192, 86: 470},
- {1: 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 81: 191},
- {1: 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 81: 189},
+ {1: 264, 230, 223, 225, 252, 260, 238, 239, 240, 250, 268, 242, 234, 232, 237, 202, 220, 221, 222, 241, 265, 209, 214, 233, 261, 262, 243, 224, 226, 271, 227, 245, 266, 228, 244, 254, 246, 256, 248, 236, 210, 253, 213, 218, 267, 219, 212, 255, 270, 211, 231, 247, 229, 269, 263, 235, 215, 258, 249, 251, 259, 257, 92: 216, 97: 203, 217, 100: 201, 208, 103: 207, 205, 200, 206, 204, 114: 199, 116: 198},
+ {83: 197},
+ {1: 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 369, 83: 196, 88: 478},
+ {1: 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, 83: 195},
+ {1: 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, 83: 193},
// 5
- {80: 467},
- {80: 464},
- {80: 461},
- {80: 456},
- {80: 453},
+ {82: 475},
+ {82: 472},
+ {82: 469},
+ {82: 464},
+ {82: 461},
// 10
- {80: 442},
- {80: 430},
- {80: 426},
- {80: 422},
- {80: 414},
+ {82: 450},
+ {82: 438},
+ {82: 434},
+ {82: 430},
+ {82: 422},
// 15
- {80: 411},
- {80: 399},
- {80: 392},
- {80: 387},
- {80: 381},
+ {82: 419},
+ {82: 407},
+ {82: 400},
+ {82: 395},
+ {82: 389},
// 20
- {80: 378},
- {80: 372},
- {80: 266},
- {80: 131},
- {80: 130},
+ {82: 386},
+ {82: 380},
+ {82: 272},
+ {82: 135},
+ {82: 134},
// 25
- {80: 129},
- {80: 128},
- {80: 127},
- {80: 126},
- {80: 125},
+ {82: 133},
+ {82: 132},
+ {82: 131},
+ {82: 130},
+ {82: 129},
// 30
- {80: 124},
- {80: 123},
- {80: 122},
- {80: 121},
- {80: 120},
+ {82: 128},
+ {82: 127},
+ {82: 126},
+ {82: 125},
+ {82: 124},
// 35
- {80: 119},
- {80: 118},
- {80: 117},
- {80: 116},
- {80: 115},
+ {82: 123},
+ {82: 122},
+ {82: 121},
+ {82: 120},
+ {82: 119},
// 40
- {80: 114},
- {80: 113},
- {80: 112},
- {80: 111},
- {80: 110},
+ {82: 118},
+ {82: 117},
+ {82: 116},
+ {82: 115},
+ {82: 114},
// 45
- {80: 109},
- {80: 108},
- {80: 107},
- {80: 106},
- {80: 105},
+ {82: 113},
+ {82: 112},
+ {82: 111},
+ {82: 110},
+ {82: 109},
// 50
- {80: 104},
- {80: 103},
- {80: 102},
- {80: 101},
- {80: 100},
+ {82: 108},
+ {82: 107},
+ {82: 106},
+ {82: 105},
+ {82: 104},
// 55
- {80: 99},
- {80: 98},
- {80: 97},
- {80: 92},
- {80: 91},
+ {82: 103},
+ {82: 102},
+ {82: 101},
+ {82: 100},
+ {82: 99},
// 60
- {80: 90},
- {80: 89},
- {80: 88},
- {80: 87},
- {80: 86},
+ {82: 94},
+ {82: 93},
+ {82: 92},
+ {82: 91},
+ {82: 90},
// 65
- {80: 85},
- {80: 84},
- {80: 83},
- {80: 82},
- {80: 81},
+ {82: 89},
+ {82: 88},
+ {82: 87},
+ {82: 86},
+ {82: 85},
// 70
- {80: 80},
- {80: 79},
- {80: 78},
- {66: 165, 165, 75: 268, 82: 267},
- {66: 273, 272, 92: 271, 270, 108: 269},
+ {82: 84},
+ {82: 83},
+ {82: 82},
+ {82: 81},
+ {82: 80},
// 75
- {164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 76: 164, 164, 164, 84: 164},
- {369, 61: 370},
- {168, 61: 168},
- {89: 274},
- {89: 75},
+ {68: 169, 169, 77: 274, 84: 273},
+ {68: 279, 278, 94: 277, 276, 110: 275},
+ {168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, 78: 168, 168, 168, 86: 168},
+ {377, 63: 378},
+ {172, 63: 172},
// 80
- {89: 74},
- {1: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 276, 88: 275},
- {61: 367, 76: 366},
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 278, 87: 277},
- {155, 61: 155, 76: 155},
+ {91: 280},
+ {91: 77},
+ {91: 76},
+ {1: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 282, 90: 281},
+ {63: 375, 78: 374},
// 85
- {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 165, 353, 165, 82: 352},
- {73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73},
- {72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72},
- {71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71},
- {70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 284, 89: 283},
+ {159, 63: 159, 78: 159},
+ {169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 169, 361, 169, 84: 360},
+ {75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75},
+ {74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74},
// 90
- {69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69},
- {68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68},
- {67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67},
- {66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66},
- {65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65},
+ {73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73},
+ {72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72},
+ {71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71},
+ {70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70},
+ {69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69},
// 95
- {64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64},
- {63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63},
- {62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62},
- {61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61},
- {60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60},
+ {68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68},
+ {67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67},
+ {66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66},
+ {65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65},
+ {64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64},
// 100
- {59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59},
- {58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58},
- {57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57},
- {56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56},
- {55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55},
+ {63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63},
+ {62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62},
+ {61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61},
+ {60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60},
+ {59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59},
// 105
- {54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54},
- {53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53},
- {52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52},
- {51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51},
- {50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50},
+ {58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58},
+ {57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57},
+ {56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56},
+ {55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55},
+ {54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54},
// 110
- {49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49},
- {48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48},
- {47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47},
- {46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46},
- {45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45},
+ {53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53},
+ {52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52},
+ {51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51},
+ {50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50},
+ {49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49},
// 115
- {44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44},
- {43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43},
- {42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42},
- {41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41},
- {40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40},
+ {48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48},
+ {47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47},
+ {46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46},
+ {45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45},
+ {44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44},
// 120
- {39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39},
- {38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38},
- {37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37},
- {36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36},
- {35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35},
+ {43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43},
+ {42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42},
+ {41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41},
+ {40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40},
+ {39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39},
// 125
- {34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34},
- {33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33},
- {32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32},
- {31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31},
- {30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30},
+ {38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38},
+ {37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37},
+ {36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36},
+ {35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35},
+ {34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34},
// 130
- {29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29},
- {28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28},
- {27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27},
- {26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26},
- {25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25},
+ {33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33},
+ {32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32},
+ {31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31},
+ {30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30},
+ {29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29},
// 135
- {24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24},
- {23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23},
- {22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22},
- {21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21},
- {20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20},
+ {28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28},
+ {27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27},
+ {26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26},
+ {25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25},
+ {24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24},
// 140
- {19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19},
- {18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18},
- {17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17},
- {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16},
- {15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+ {23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23},
+ {22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22},
+ {21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21},
+ {20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20},
+ {19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19},
// 145
- {14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
- {13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13},
- {12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
- {11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11},
- {10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+ {18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18},
+ {17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17},
+ {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16},
+ {15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+ {14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
// 150
- {9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9},
- {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8},
- {7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7},
- {6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6},
- {5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5},
+ {13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13},
+ {12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+ {11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11},
+ {10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+ {9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9},
// 155
- {4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4},
- {3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3},
- {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2},
- {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
- {161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 76: 161, 78: 356, 97: 365},
+ {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8},
+ {7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7},
+ {6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6},
+ {5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5},
+ {4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4},
// 160
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 354},
- {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 165, 78: 165, 82: 355},
- {161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 76: 161, 78: 356, 97: 357},
- {80: 358},
- {152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 76: 152},
+ {3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3},
+ {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2},
+ {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+ {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 78: 165, 80: 364, 99: 373},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 362},
// 165
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 360, 113: 359},
- {362, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 361, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 86: 363},
- {159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159, 159},
- {162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 62: 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 85: 162},
- {160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 76: 160},
+ {169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 169, 80: 169, 84: 363},
+ {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 78: 165, 80: 364, 99: 365},
+ {82: 366},
+ {156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 78: 156},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 368, 115: 367},
// 170
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 364},
- {158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158},
- {153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 76: 153},
- {166, 61: 166},
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 278, 87: 368},
+ {370, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 369, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 88: 371},
+ {163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163},
+ {166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 64: 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 87: 166},
+ {164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 78: 164},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 372},
// 175
- {154, 61: 154, 76: 154},
- {1: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 81: 169},
- {66: 273, 272, 92: 271, 371},
- {167, 61: 167},
- {69: 165, 165, 75: 268, 82: 373},
+ {162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162},
+ {157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 78: 157},
+ {170, 63: 170},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 284, 89: 376},
+ {158, 63: 158, 78: 158},
// 180
- {69: 375, 376, 107: 374},
- {377},
- {77},
- {76},
- {1: 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, 81: 170},
+ {1: 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 83: 173},
+ {68: 279, 278, 94: 277, 379},
+ {171, 63: 171},
+ {71: 169, 169, 77: 274, 84: 381},
+ {71: 383, 384, 109: 382},
// 185
- {165, 75: 268, 82: 379},
- {380},
- {1: 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, 81: 171},
- {68: 165, 71: 165, 75: 268, 82: 382},
- {68: 385, 71: 384, 109: 383},
+ {385},
+ {79},
+ {78},
+ {1: 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 83: 174},
+ {169, 77: 274, 84: 387},
// 190
- {386},
- {133},
- {132},
- {1: 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 81: 172},
- {85: 388},
+ {388},
+ {1: 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 83: 175},
+ {70: 169, 73: 169, 77: 274, 84: 390},
+ {70: 393, 73: 392, 111: 391},
+ {394},
// 195
- {61: 361, 85: 163, 389},
- {85: 390},
- {391},
- {1: 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 173, 81: 173},
- {75: 268, 82: 393, 84: 165},
+ {137},
+ {136},
+ {1: 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 83: 176},
+ {87: 396},
+ {63: 369, 87: 167, 397},
// 200
- {84: 394},
- {72: 397, 396, 117: 395},
- {398},
- {135},
- {134},
+ {87: 398},
+ {399},
+ {1: 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 83: 177},
+ {77: 274, 84: 401, 86: 169},
+ {86: 402},
// 205
- {1: 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 81: 174},
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 400},
- {401, 61: 402},
- {1: 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 81: 176},
- {165, 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 268, 77: 165, 82: 406, 405, 106: 404, 119: 403},
+ {74: 405, 404, 119: 403},
+ {406},
+ {139},
+ {138},
+ {1: 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 83: 178},
// 210
- {408, 77: 409},
- {150, 77: 150},
- {165, 75: 268, 77: 165, 82: 407},
- {148, 77: 148},
- {149, 77: 149},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 408},
+ {409, 63: 410},
+ {1: 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 83: 180},
+ {169, 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 274, 79: 169, 84: 414, 413, 108: 412, 121: 411},
+ {416, 79: 417},
// 215
- {1: 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 81: 175},
- {165, 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 268, 77: 165, 82: 406, 405, 106: 410},
- {151, 77: 151},
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 412},
- {413},
+ {154, 79: 154},
+ {169, 77: 274, 79: 169, 84: 415},
+ {152, 79: 152},
+ {153, 79: 153},
+ {1: 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 83: 179},
// 220
- {1: 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 81: 177},
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 415},
- {79: 416},
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 419, 420, 418, 118: 417},
+ {169, 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 274, 79: 169, 84: 414, 413, 108: 418},
+ {155, 79: 155},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 420},
{421},
+ {1: 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 83: 181},
// 225
- {138},
- {137},
- {136},
- {1: 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 81: 178},
- {75: 268, 82: 423, 84: 165},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 423},
+ {81: 424},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 427, 428, 426, 120: 425},
+ {429},
+ {142},
// 230
- {84: 424},
- {425},
- {1: 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 81: 179},
- {75: 268, 82: 427, 84: 165},
- {84: 428},
+ {141},
+ {140},
+ {1: 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 83: 182},
+ {77: 274, 84: 431, 86: 169},
+ {86: 432},
// 235
- {429},
- {1: 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, 81: 180},
- {165, 62: 165, 165, 165, 165, 75: 268, 82: 431},
- {142, 62: 435, 436, 437, 438, 100: 434, 115: 433, 432},
- {441},
+ {433},
+ {1: 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 83: 183},
+ {77: 274, 84: 435, 86: 169},
+ {86: 436},
+ {437},
// 240
- {141, 61: 439},
- {140, 61: 140},
- {96, 61: 96},
- {95, 61: 95},
- {94, 61: 94},
+ {1: 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 83: 184},
+ {169, 64: 169, 169, 169, 169, 77: 274, 84: 439},
+ {146, 64: 443, 444, 445, 446, 102: 442, 117: 441, 440},
+ {449},
+ {145, 63: 447},
// 245
- {93, 61: 93},
- {62: 435, 436, 437, 438, 100: 440},
- {139, 61: 139},
- {1: 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, 81: 181},
- {1: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 444, 91: 443},
+ {144, 63: 144},
+ {98, 63: 98},
+ {97, 63: 97},
+ {96, 63: 96},
+ {95, 63: 95},
// 250
- {452},
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 278, 87: 445},
- {163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 361, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 86: 446},
- {146, 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 449, 110: 448, 447},
- {147},
+ {64: 443, 444, 445, 446, 102: 448},
+ {143, 63: 143},
+ {1: 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 83: 185},
+ {1: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 452, 93: 451},
+ {460},
// 255
- {145, 61: 450},
- {144, 61: 144},
- {1: 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 451},
- {143, 61: 143},
- {1: 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 81: 182},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 284, 89: 453},
+ {167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 369, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 88: 454},
+ {150, 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 457, 112: 456, 455},
+ {151},
+ {149, 63: 458},
// 260
- {1: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 444, 91: 454},
- {455},
- {1: 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 81: 183},
- {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 459, 88: 458, 94: 457},
- {460},
+ {148, 63: 148},
+ {1: 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 459},
+ {147, 63: 147},
+ {1: 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 83: 186},
+ {1: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 452, 93: 462},
// 265
- {157, 61: 367},
- {156, 308, 324, 284, 286, 335, 311, 288, 289, 290, 314, 310, 294, 315, 316, 317, 280, 281, 282, 283, 337, 309, 304, 318, 292, 312, 313, 296, 285, 287, 339, 291, 298, 295, 293, 297, 299, 303, 301, 319, 334, 307, 320, 321, 322, 306, 302, 338, 305, 325, 300, 323, 336, 326, 327, 332, 333, 329, 328, 330, 331, 62: 348, 349, 350, 351, 343, 342, 344, 340, 341, 345, 347, 346, 279, 83: 278, 87: 277},
- {1: 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, 81: 184},
- {165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 459, 88: 458, 94: 462},
{463},
+ {1: 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 83: 187},
+ {169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 467, 90: 466, 96: 465},
+ {468},
+ {161, 63: 375},
// 270
- {1: 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 81: 185},
- {1: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 62: 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 268, 82: 276, 88: 465},
- {466, 61: 367},
- {1: 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, 81: 186},
- {165, 75: 268, 82: 468},
+ {160, 314, 330, 290, 292, 343, 317, 294, 295, 296, 320, 316, 300, 321, 322, 323, 286, 287, 288, 289, 345, 315, 310, 324, 298, 318, 319, 302, 291, 293, 347, 297, 304, 301, 299, 303, 337, 305, 309, 307, 325, 342, 336, 313, 326, 327, 328, 312, 308, 346, 311, 331, 306, 329, 344, 332, 333, 340, 341, 335, 334, 338, 339, 64: 356, 357, 358, 359, 351, 350, 352, 348, 349, 353, 355, 354, 285, 85: 284, 89: 283},
+ {1: 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 83: 188},
+ {169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 467, 90: 466, 96: 470},
+ {471},
+ {1: 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, 83: 189},
// 275
- {469},
- {1: 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 81: 187},
- {1: 258, 226, 219, 221, 248, 254, 234, 235, 236, 246, 262, 238, 230, 228, 233, 198, 216, 217, 218, 237, 259, 205, 210, 229, 255, 256, 239, 220, 222, 265, 223, 241, 260, 224, 240, 242, 250, 244, 232, 206, 209, 214, 261, 215, 208, 249, 264, 207, 227, 243, 225, 263, 257, 231, 211, 252, 245, 247, 253, 251, 90: 212, 95: 199, 213, 98: 472, 204, 101: 203, 201, 471, 202, 200},
- {1: 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 81: 190},
- {1: 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 188, 81: 188},
+ {1: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 64: 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, 274, 84: 282, 90: 473},
+ {474, 63: 375},
+ {1: 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, 83: 190},
+ {169, 77: 274, 84: 476},
+ {477},
+ // 280
+ {1: 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, 83: 191},
+ {1: 264, 230, 223, 225, 252, 260, 238, 239, 240, 250, 268, 242, 234, 232, 237, 202, 220, 221, 222, 241, 265, 209, 214, 233, 261, 262, 243, 224, 226, 271, 227, 245, 266, 228, 244, 254, 246, 256, 248, 236, 210, 253, 213, 218, 267, 219, 212, 255, 270, 211, 231, 247, 229, 269, 263, 235, 215, 258, 249, 251, 259, 257, 92: 216, 97: 203, 217, 100: 480, 208, 103: 207, 205, 479, 206, 204},
+ {1: 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, 83: 194},
+ {1: 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 83: 192},
}
)
@@ -954,7 +969,7 @@ func yyhintlex1(yylex yyhintLexer, lval *yyhintSymType) (n int) {
}
func yyhintParse(yylex yyhintLexer, parser *hintParser) int {
- const yyError = 121
+ const yyError = 123
yyEx, _ := yylex.(yyhintLexerEx)
var yyn int
diff --git a/parser/hintparser.y b/parser/hintparser.y
index a2ccd21d2d401..0a29375be226d 100644
--- a/parser/hintparser.y
+++ b/parser/hintparser.y
@@ -103,6 +103,8 @@ import (
hintSwapJoinInputs "SWAP_JOIN_INPUTS"
hintUseIndexMerge "USE_INDEX_MERGE"
hintUseIndex "USE_INDEX"
+ hintOrderIndex "ORDER_INDEX"
+ hintNoOrderIndex "NO_ORDER_INDEX"
hintUsePlanCache "USE_PLAN_CACHE"
hintUseToja "USE_TOJA"
hintTimeRange "TIME_RANGE"
@@ -607,6 +609,8 @@ SupportedIndexLevelOptimizerHintName:
| "IGNORE_INDEX"
| "USE_INDEX_MERGE"
| "FORCE_INDEX"
+| "ORDER_INDEX"
+| "NO_ORDER_INDEX"
SubqueryOptimizerHintName:
"SEMIJOIN"
@@ -699,6 +703,8 @@ Identifier:
| "SWAP_JOIN_INPUTS"
| "USE_INDEX_MERGE"
| "USE_INDEX"
+| "ORDER_INDEX"
+| "NO_ORDER_INDEX"
| "USE_PLAN_CACHE"
| "USE_TOJA"
| "TIME_RANGE"
diff --git a/parser/lexer.go b/parser/lexer.go
index 0a172abfda027..01f1ef035914a 100644
--- a/parser/lexer.go
+++ b/parser/lexer.go
@@ -252,13 +252,17 @@ func (s *Scanner) Lex(v *yySymType) int {
if tok == not && s.sqlMode.HasHighNotPrecedenceMode() {
return not2
}
- if tok == as && s.getNextToken() == of {
+ if (tok == as || tok == member) && s.getNextToken() == of {
_, pos, lit = s.scan()
v.ident = fmt.Sprintf("%s %s", v.ident, lit)
- s.lastKeyword = asof
s.lastScanOffset = pos.Offset
v.offset = pos.Offset
- return asof
+ if tok == as {
+ s.lastKeyword = asof
+ return asof
+ }
+ s.lastKeyword = memberof
+ return memberof
}
if tok == to {
tok1, tok2 := s.getNextTwoTokens()
diff --git a/parser/misc.go b/parser/misc.go
index 372154e054867..87c0ef3dbde4c 100644
--- a/parser/misc.go
+++ b/parser/misc.go
@@ -153,6 +153,7 @@ var tokenMap = map[string]int{
"ANY": any,
"APPROX_COUNT_DISTINCT": approxCountDistinct,
"APPROX_PERCENTILE": approxPercentile,
+ "ARRAY": array,
"AS": as,
"ASC": asc,
"ASCII": ascii,
@@ -195,6 +196,7 @@ var tokenMap = map[string]int{
"BTREE": btree,
"BUCKETS": buckets,
"BUILTINS": builtins,
+ "BURSTABLE": burstable,
"BY": by,
"BYTE": byteType,
"CACHE": cache,
@@ -256,6 +258,7 @@ var tokenMap = map[string]int{
"CSV_NULL": csvNull,
"CSV_SEPARATOR": csvSeparator,
"CSV_TRIM_LAST_SEPARATORS": csvTrimLastSeparators,
+ "CURDATE": curDate,
"CURRENT_DATE": currentDate,
"CURRENT_ROLE": currentRole,
"CURRENT_TIME": currentTime,
@@ -289,6 +292,7 @@ var tokenMap = map[string]int{
"DEPTH": depth,
"DESC": desc,
"DESCRIBE": describe,
+ "DIGEST": digest,
"DIRECTORY": directory,
"DISABLE": disable,
"DISABLED": disabled,
@@ -408,6 +412,9 @@ var tokenMap = map[string]int{
"INVISIBLE": invisible,
"INVOKER": invoker,
"IO": io,
+ "RU_PER_SEC": ruRate,
+ "IO_READ_BANDWIDTH": ioReadBandwidth,
+ "IO_WRITE_BANDWIDTH": ioWriteBandwidth,
"IPC": ipc,
"IS": is,
"ISOLATION": isolation,
@@ -469,6 +476,7 @@ var tokenMap = map[string]int{
"MEDIUMINT": mediumIntType,
"MEDIUMTEXT": mediumtextType,
"MEMORY": memory,
+ "MEMBER": member,
"MERGE": merge,
"MICROSECOND": microsecond,
"MIN_ROWS": minRows,
@@ -593,6 +601,7 @@ var tokenMap = map[string]int{
"REQUIRE": require,
"REQUIRED": required,
"RESET": reset,
+ "RESOURCE": resource,
"RESPECT": respect,
"RESTART": restart,
"RESTORE": restore,
@@ -706,6 +715,7 @@ var tokenMap = map[string]int{
"SUBSTRING": substring,
"SUM": sum,
"SUPER": super,
+ "SURVIVAL_PREFERENCES": survivalPreferences,
"SWAPS": swaps,
"SWITCHES": switchesSym,
"SYSTEM": system,
@@ -726,6 +736,7 @@ var tokenMap = map[string]int{
"THEN": then,
"TIDB": tidb,
"TIDB_CURRENT_TSO": tidbCurrentTSO,
+ "TIDB_JSON": tidbJson,
"TIFLASH": tiFlash,
"TIKV_IMPORTER": tikvImporter,
"TIME": timeType,
@@ -759,6 +770,9 @@ var tokenMap = map[string]int{
"TRUE": trueKwd,
"TRUNCATE": truncate,
"TRUE_CARD_COST": trueCardCost,
+ "TTL": ttl,
+ "TTL_ENABLE": ttlEnable,
+ "TTL_JOB_INTERVAL": ttlJobInterval,
"TYPE": tp,
"UNBOUNDED": unbounded,
"UNCOMMITTED": uncommitted,
@@ -810,6 +824,9 @@ var tokenMap = map[string]int{
"YEAR": yearType,
"ZEROFILL": zerofill,
"WAIT": wait,
+ "FAILED_LOGIN_ATTEMPTS": failedLoginAttempts,
+ "PASSWORD_LOCK_TIME": passwordLockTime,
+ "REUSE": reuse,
}
// See https://dev.mysql.com/doc/refman/5.7/en/function-resolution.html for details.
@@ -942,6 +959,8 @@ var hintTokenMap = map[string]int{
"SWAP_JOIN_INPUTS": hintSwapJoinInputs,
"USE_INDEX_MERGE": hintUseIndexMerge,
"USE_INDEX": hintUseIndex,
+ "ORDER_INDEX": hintOrderIndex,
+ "NO_ORDER_INDEX": hintNoOrderIndex,
"USE_PLAN_CACHE": hintUsePlanCache,
"USE_TOJA": hintUseToja,
"TIME_RANGE": hintTimeRange,
diff --git a/parser/model/BUILD.bazel b/parser/model/BUILD.bazel
index f3f214ce29eeb..5387516deaf94 100644
--- a/parser/model/BUILD.bazel
+++ b/parser/model/BUILD.bazel
@@ -31,6 +31,7 @@ go_test(
deps = [
"//parser/charset",
"//parser/mysql",
+ "//parser/terror",
"//parser/types",
"@com_github_stretchr_testify//require",
],
diff --git a/parser/model/ddl.go b/parser/model/ddl.go
index 8a8acaa292666..36ada1ed9b3ea 100644
--- a/parser/model/ddl.go
+++ b/parser/model/ddl.go
@@ -98,6 +98,12 @@ const (
ActionMultiSchemaChange ActionType = 61
ActionFlashbackCluster ActionType = 62
ActionRecoverSchema ActionType = 63
+ ActionReorganizePartition ActionType = 64
+ ActionAlterTTLInfo ActionType = 65
+ ActionAlterTTLRemove ActionType = 67
+ ActionCreateResourceGroup ActionType = 68
+ ActionAlterResourceGroup ActionType = 69
+ ActionDropResourceGroup ActionType = 70
)
var actionMap = map[ActionType]string{
@@ -160,6 +166,12 @@ var actionMap = map[ActionType]string{
ActionMultiSchemaChange: "alter table multi-schema change",
ActionFlashbackCluster: "flashback cluster",
ActionRecoverSchema: "flashback schema",
+ ActionReorganizePartition: "alter table reorganize partition",
+ ActionAlterTTLInfo: "alter table ttl",
+ ActionAlterTTLRemove: "alter table no_ttl",
+ ActionCreateResourceGroup: "create resource group",
+ ActionAlterResourceGroup: "alter resource group",
+ ActionDropResourceGroup: "drop resource group",
// `ActionAlterTableAlterPartition` is removed and will never be used.
// Just left a tombstone here for compatibility.
@@ -225,6 +237,7 @@ type DDLReorgMeta struct {
WarningsCount map[errors.ErrorID]int64 `json:"warnings_count"`
Location *TimeZoneLocation `json:"location"`
ReorgTp ReorgType `json:"reorg_tp"`
+ IsDistReorg bool `json:"is_dist_reorg"`
}
// ReorgType indicates which process is used for the data reorganization.
@@ -310,12 +323,19 @@ type MultiSchemaInfo struct {
AddIndexes []CIStr `json:"-"`
DropIndexes []CIStr `json:"-"`
AlterIndexes []CIStr `json:"-"`
- ForeignKeys []CIStr `json:"-"`
+
+ AddForeignKeys []AddForeignKeyInfo `json:"-"`
RelativeColumns []CIStr `json:"-"`
PositionColumns []CIStr `json:"-"`
}
+// AddForeignKeyInfo contains foreign key information.
+type AddForeignKeyInfo struct {
+ Name CIStr
+ Cols []CIStr
+}
+
// NewMultiSchemaInfo new a MultiSchemaInfo.
func NewMultiSchemaInfo() *MultiSchemaInfo {
return &MultiSchemaInfo{
@@ -337,6 +357,7 @@ type SubJob struct {
Warning *terror.Error `json:"warning"`
CtxVars []interface{} `json:"-"`
SchemaVer int64 `json:"schema_version"`
+ ReorgTp ReorgType `json:"reorg_tp"`
}
// IsNormal returns true if the sub-job is normally running.
@@ -399,6 +420,49 @@ func (sub *SubJob) FromProxyJob(proxyJob *Job, ver int64) {
sub.Warning = proxyJob.Warning
sub.RowCount = proxyJob.RowCount
sub.SchemaVer = ver
+ sub.ReorgTp = proxyJob.ReorgMeta.ReorgTp
+}
+
+// JobMeta is meta info of Job.
+type JobMeta struct {
+ SchemaID int64 `json:"schema_id"`
+ TableID int64 `json:"table_id"`
+ // Type is the DDL job's type.
+ Type ActionType `json:"job_type"`
+ // Query is the DDL job's SQL string.
+ Query string `json:"query"`
+ // Priority is only used to set the operation priority of adding indices.
+ Priority int `json:"priority"`
+}
+
+// BackfillMeta is meta info of the backfill job.
+type BackfillMeta struct {
+ IsUnique bool `json:"is_unique"`
+ EndInclude bool `json:"end_include"`
+ Error *terror.Error `json:"err"`
+
+ SQLMode mysql.SQLMode `json:"sql_mode"`
+ Warnings map[errors.ErrorID]*terror.Error `json:"warnings"`
+ WarningsCount map[errors.ErrorID]int64 `json:"warnings_count"`
+ Location *TimeZoneLocation `json:"location"`
+ ReorgTp ReorgType `json:"reorg_tp"`
+ RowCount int64 `json:"row_count"`
+ StartKey []byte `json:"start_key"`
+ EndKey []byte `json:"end_key"`
+ CurrKey []byte `json:"curr_key"`
+ *JobMeta `json:"job_meta"`
+}
+
+// Encode encodes BackfillMeta with json format.
+func (bm *BackfillMeta) Encode() ([]byte, error) {
+ b, err := json.Marshal(bm)
+ return b, errors.Trace(err)
+}
+
+// Decode decodes BackfillMeta from the json buffer.
+func (bm *BackfillMeta) Decode(b []byte) error {
+ err := json.Unmarshal(b, bm)
+ return errors.Trace(err)
}
// Job is for a DDL operation.
@@ -536,13 +600,18 @@ func (job *Job) GetRowCount() int64 {
// SetWarnings sets the warnings of rows handled.
func (job *Job) SetWarnings(warnings map[errors.ErrorID]*terror.Error, warningsCount map[errors.ErrorID]int64) {
+ job.Mu.Lock()
job.ReorgMeta.Warnings = warnings
job.ReorgMeta.WarningsCount = warningsCount
+ job.Mu.Unlock()
}
// GetWarnings gets the warnings of the rows handled.
func (job *Job) GetWarnings() (map[errors.ErrorID]*terror.Error, map[errors.ErrorID]int64) {
- return job.ReorgMeta.Warnings, job.ReorgMeta.WarningsCount
+ job.Mu.Lock()
+ w, wc := job.ReorgMeta.Warnings, job.ReorgMeta.WarningsCount
+ job.Mu.Unlock()
+ return w, wc
}
// Encode encodes job with json format.
@@ -609,6 +678,10 @@ func (job *Job) String() string {
rowCount := job.GetRowCount()
ret := fmt.Sprintf("ID:%d, Type:%s, State:%s, SchemaState:%s, SchemaID:%d, TableID:%d, RowCount:%d, ArgLen:%d, start time: %v, Err:%v, ErrCount:%d, SnapshotVersion:%v",
job.ID, job.Type, job.State, job.SchemaState, job.SchemaID, job.TableID, rowCount, len(job.Args), TSConvert2Time(job.StartTS), job.Error, job.ErrorCount, job.SnapshotVer)
+ if job.ReorgMeta != nil {
+ warnings, _ := job.GetWarnings()
+ ret += fmt.Sprintf(", UniqueWarnings:%d", len(warnings))
+ }
if job.Type != ActionMultiSchemaChange && job.MultiSchemaInfo != nil {
ret += fmt.Sprintf(", Multi-Schema Change:true, Revertible:%v", job.MultiSchemaInfo.Revertible)
}
@@ -882,6 +955,30 @@ func (s JobState) String() string {
}
}
+// StrToJobState converts string to JobState.
+func StrToJobState(s string) JobState {
+ switch s {
+ case "running":
+ return JobStateRunning
+ case "rollingback":
+ return JobStateRollingback
+ case "rollback done":
+ return JobStateRollbackDone
+ case "done":
+ return JobStateDone
+ case "cancelled":
+ return JobStateCancelled
+ case "cancelling":
+ return JobStateCancelling
+ case "synced":
+ return JobStateSynced
+ case "queueing":
+ return JobStateQueueing
+ default:
+ return JobStateNone
+ }
+}
+
// SchemaDiff contains the schema modification at a particular schema version.
// It is used to reduce schema reload cost.
type SchemaDiff struct {
@@ -894,6 +991,8 @@ type SchemaDiff struct {
OldTableID int64 `json:"old_table_id"`
// OldSchemaID is the schema ID before rename table, only used by rename table DDL.
OldSchemaID int64 `json:"old_schema_id"`
+ // RegenerateSchemaMap means whether to rebuild the schema map when applying to the schema diff.
+ RegenerateSchemaMap bool `json:"regenerate_schema_map"`
AffectedOpts []*AffectedOption `json:"affected_options"`
}
diff --git a/parser/model/ddl_test.go b/parser/model/ddl_test.go
index 7fddcca54e351..7ea82606363ce 100644
--- a/parser/model/ddl_test.go
+++ b/parser/model/ddl_test.go
@@ -18,6 +18,7 @@ import (
"unsafe"
"github.com/pingcap/tidb/parser/model"
+ "github.com/pingcap/tidb/parser/terror"
"github.com/stretchr/testify/require"
)
@@ -51,3 +52,22 @@ func TestJobSize(t *testing.T) {
job := model.Job{}
require.Equal(t, 288, int(unsafe.Sizeof(job)), msg)
}
+
+func TestBackfillMetaCodec(t *testing.T) {
+ jm := &model.JobMeta{
+ SchemaID: 1,
+ TableID: 2,
+ Query: "alter table t add index idx(a)",
+ Priority: 1,
+ }
+ bm := &model.BackfillMeta{
+ EndInclude: true,
+ Error: terror.ErrResultUndetermined,
+ JobMeta: jm,
+ }
+ bmBytes, err := bm.Encode()
+ require.NoError(t, err)
+ bmRet := &model.BackfillMeta{}
+ bmRet.Decode(bmBytes)
+ require.Equal(t, bm, bmRet)
+}
diff --git a/parser/model/model.go b/parser/model/model.go
index add3d59d81ba3..ff710b05c7095 100644
--- a/parser/model/model.go
+++ b/parser/model/model.go
@@ -164,6 +164,9 @@ type ColumnInfo struct {
// Clone clones ColumnInfo.
func (c *ColumnInfo) Clone() *ColumnInfo {
+ if c == nil {
+ return nil
+ }
nc := *c
return &nc
}
@@ -378,8 +381,8 @@ func FindFKInfoByName(fks []*FKInfo, name string) *FKInfo {
}
// FindIndexByColumns find IndexInfo in indices which is cover the specified columns.
-func FindIndexByColumns(tbInfo *TableInfo, cols ...CIStr) *IndexInfo {
- for _, index := range tbInfo.Indices {
+func FindIndexByColumns(tbInfo *TableInfo, indices []*IndexInfo, cols ...CIStr) *IndexInfo {
+ for _, index := range indices {
if IsIndexPrefixCovered(tbInfo, index, cols...) {
return index
}
@@ -443,14 +446,16 @@ const (
// However, the convert is missed in some scenarios before v2.1.9, so for all those tables prior to TableInfoVersion3, their
// charsets / collations will be converted to lower-case while loading from the storage.
TableInfoVersion3 = uint16(3)
- // TableInfoVersion4 indicates that the auto_increment allocator in TiDB has been separated from
- // _tidb_rowid allocator. This version is introduced to preserve the compatibility of old tables:
- // the tables with version < TableInfoVersion4 still use a single allocator for auto_increment and _tidb_rowid.
- // Also see https://github.com/pingcap/tidb/issues/982.
+ // TableInfoVersion4 is not used.
TableInfoVersion4 = uint16(4)
+ // TableInfoVersion5 indicates that the auto_increment allocator in TiDB has been separated from
+ // _tidb_rowid allocator when AUTO_ID_CACHE is 1. This version is introduced to preserve the compatibility of old tables:
+ // the tables with version <= TableInfoVersion4 still use a single allocator for auto_increment and _tidb_rowid.
+ // Also see https://github.com/pingcap/tidb/issues/982.
+ TableInfoVersion5 = uint16(5)
// CurrLatestTableInfoVersion means the latest table info in the current TiDB.
- CurrLatestTableInfoVersion = TableInfoVersion4
+ CurrLatestTableInfoVersion = TableInfoVersion5
)
// ExtraHandleName is the name of ExtraHandle Column.
@@ -545,6 +550,13 @@ type TableInfo struct {
StatsOptions *StatsOptions `json:"stats_options"`
ExchangePartitionInfo *ExchangePartitionInfo `json:"exchange_partition_info"`
+
+ TTLInfo *TTLInfo `json:"ttl_info"`
+}
+
+// SepAutoInc decides whether _rowid and auto_increment id use separate allocator.
+func (t *TableInfo) SepAutoInc() bool {
+ return t.Version >= TableInfoVersion5 && t.AutoIdCache == 1
}
// TableCacheStatusType is the type of the table cache status
@@ -745,6 +757,13 @@ func (t *TableInfo) Clone() *TableInfo {
nt.ForeignKeys[i] = t.ForeignKeys[i].Clone()
}
+ if t.Partition != nil {
+ nt.Partition = t.Partition.Clone()
+ }
+ if t.TTLInfo != nil {
+ nt.TTLInfo = t.TTLInfo.Clone()
+ }
+
return &nt
}
@@ -1177,6 +1196,8 @@ type PartitionInfo struct {
DroppingDefinitions []PartitionDefinition `json:"dropping_definitions"`
States []PartitionState `json:"states"`
Num uint64 `json:"num"`
+ // Only used during ReorganizePartition so far
+ DDLState SchemaState `json:"ddl_state"`
}
// Clone clones itself.
@@ -1311,15 +1332,15 @@ func (ci *PartitionDefinition) MemoryUsage() (sum int64) {
}
// FindPartitionDefinitionByName finds PartitionDefinition by name.
-func (t *TableInfo) FindPartitionDefinitionByName(partitionDefinitionName string) *PartitionDefinition {
+func (pi *PartitionInfo) FindPartitionDefinitionByName(partitionDefinitionName string) int {
lowConstrName := strings.ToLower(partitionDefinitionName)
- definitions := t.Partition.Definitions
+ definitions := pi.Definitions
for i := range definitions {
if definitions[i].Name.L == lowConstrName {
- return &t.Partition.Definitions[i]
+ return i
}
}
- return nil
+ return -1
}
// IndexColumn provides index column info.
@@ -1403,10 +1424,14 @@ type IndexInfo struct {
Primary bool `json:"is_primary"` // Whether the index is primary key.
Invisible bool `json:"is_invisible"` // Whether the index is invisible.
Global bool `json:"is_global"` // Whether the index is global.
+ MVIndex bool `json:"mv_index"` // Whether the index is multivalued index.
}
// Clone clones IndexInfo.
func (index *IndexInfo) Clone() *IndexInfo {
+ if index == nil {
+ return nil
+ }
ni := *index
ni.Columns = make([]*IndexColumn, len(index.Columns))
for i := range index.Columns {
@@ -1716,6 +1741,7 @@ type PlacementSettings struct {
LearnerConstraints string `json:"learner_constraints"`
FollowerConstraints string `json:"follower_constraints"`
VoterConstraints string `json:"voter_constraints"`
+ SurvivalPreferences string `json:"survival_preferences"`
}
// PolicyInfo is the struct to store the placement policy.
@@ -1733,6 +1759,23 @@ func (p *PolicyInfo) Clone() *PolicyInfo {
return &cloned
}
+// TTLInfo records the TTL config
+type TTLInfo struct {
+ ColumnName CIStr `json:"column"`
+ IntervalExprStr string `json:"interval_expr"`
+ // `IntervalTimeUnit` is actually ast.TimeUnitType. Use `int` to avoid cycle dependency
+ IntervalTimeUnit int `json:"interval_time_unit"`
+ Enable bool `json:"enable"`
+ // JobInterval is the interval between two TTL scan jobs.
+ JobInterval string `json:"job_interval"`
+}
+
+// Clone clones TTLInfo
+func (t *TTLInfo) Clone() *TTLInfo {
+ cloned := *t
+ return &cloned
+}
+
func writeSettingItemToBuilder(sb *strings.Builder, item string) {
if sb.Len() != 0 {
sb.WriteString(" ")
@@ -1801,6 +1844,71 @@ func (p *PlacementSettings) Clone() *PlacementSettings {
return &cloned
}
+// ResourceGroupRefInfo is the struct to refer the resource group.
+type ResourceGroupRefInfo struct {
+ ID int64 `json:"id"`
+ Name CIStr `json:"name"`
+}
+
+// ResourceGroupSettings is the settings of the resource group
+type ResourceGroupSettings struct {
+ RURate uint64 `json:"ru_per_sec"`
+ CPULimiter string `json:"cpu_limit"`
+ IOReadBandwidth string `json:"io_read_bandwidth"`
+ IOWriteBandwidth string `json:"io_write_bandwidth"`
+ BurstLimit int64 `json:"burst_limit"`
+}
+
+func (p *ResourceGroupSettings) String() string {
+ sb := new(strings.Builder)
+ if p.RURate != 0 {
+ writeSettingIntegerToBuilder(sb, "RU_PER_SEC", p.RURate)
+ }
+ if len(p.CPULimiter) > 0 {
+ writeSettingStringToBuilder(sb, "CPU", p.CPULimiter)
+ }
+ if len(p.IOReadBandwidth) > 0 {
+ writeSettingStringToBuilder(sb, "IO_READ_BANDWIDTH", p.IOReadBandwidth)
+ }
+ if len(p.IOWriteBandwidth) > 0 {
+ writeSettingStringToBuilder(sb, "IO_WRITE_BANDWIDTH", p.IOWriteBandwidth)
+ }
+ // Once burst limit is negative, meaning allow burst with unlimit.
+ if p.BurstLimit < 0 {
+ writeSettingItemToBuilder(sb, "BURSTABLE")
+ }
+ return sb.String()
+}
+
+// Adjust adjusts the resource group settings.
+func (p *ResourceGroupSettings) Adjust() {
+ // Curretly we only support ru_per_sec sytanx, so BurstLimit(capicity) is always same as ru_per_sec.
+ if p.BurstLimit == 0 {
+ p.BurstLimit = int64(p.RURate)
+ }
+}
+
+// Clone clones the resource group settings.
+func (p *ResourceGroupSettings) Clone() *ResourceGroupSettings {
+ cloned := *p
+ return &cloned
+}
+
+// ResourceGroupInfo is the struct to store the resource group.
+type ResourceGroupInfo struct {
+ *ResourceGroupSettings
+ ID int64 `json:"id"`
+ Name CIStr `json:"name"`
+ State SchemaState `json:"state"`
+}
+
+// Clone clones the ResourceGroupInfo.
+func (p *ResourceGroupInfo) Clone() *ResourceGroupInfo {
+ cloned := *p
+ cloned.ResourceGroupSettings = p.ResourceGroupSettings.Clone()
+ return &cloned
+}
+
// StatsOptions is the struct to store the stats options.
type StatsOptions struct {
*StatsWindowSettings
diff --git a/parser/model/model_test.go b/parser/model/model_test.go
index 47a8ecad6e4a4..6062df58aabec 100644
--- a/parser/model/model_test.go
+++ b/parser/model/model_test.go
@@ -785,3 +785,23 @@ func TestIsIndexPrefixCovered(t *testing.T) {
require.Equal(t, true, IsIndexPrefixCovered(tbl, i1, NewCIStr("c_4"), NewCIStr("c_2")))
require.Equal(t, false, IsIndexPrefixCovered(tbl, i0, NewCIStr("c_2")))
}
+
+func TestTTLInfoClone(t *testing.T) {
+ ttlInfo := &TTLInfo{
+ ColumnName: NewCIStr("test"),
+ IntervalExprStr: "test_expr",
+ IntervalTimeUnit: 5,
+ Enable: true,
+ }
+
+ clonedTTLInfo := ttlInfo.Clone()
+ clonedTTLInfo.ColumnName = NewCIStr("test_2")
+ clonedTTLInfo.IntervalExprStr = "test_expr_2"
+ clonedTTLInfo.IntervalTimeUnit = 9
+ clonedTTLInfo.Enable = false
+
+ require.Equal(t, "test", ttlInfo.ColumnName.O)
+ require.Equal(t, "test_expr", ttlInfo.IntervalExprStr)
+ require.Equal(t, 5, ttlInfo.IntervalTimeUnit)
+ require.Equal(t, true, ttlInfo.Enable)
+}
diff --git a/parser/mysql/const.go b/parser/mysql/const.go
index 2d8da77f90e64..f836cc05e4286 100644
--- a/parser/mysql/const.go
+++ b/parser/mysql/const.go
@@ -209,6 +209,8 @@ const (
RoleEdgeTable = "role_edges"
// DefaultRoleTable is the table contain default active role info
DefaultRoleTable = "default_roles"
+ // PasswordHistoryTable is the table in system db contains password history.
+ PasswordHistoryTable = "password_history"
)
// MySQL type maximum length.
diff --git a/parser/mysql/errcode.go b/parser/mysql/errcode.go
index f82bbb1e0978f..0ea2134f88d75 100644
--- a/parser/mysql/errcode.go
+++ b/parser/mysql/errcode.go
@@ -884,6 +884,7 @@ const (
ErrErrorLast = 1863
ErrMaxExecTimeExceeded = 1907
ErrInvalidFieldSize = 3013
+ ErrPasswordExpireAnonymousUser = 3016
ErrIncorrectType = 3064
ErrInvalidJSONData = 3069
ErrGeneratedColumnFunctionIsNotAllowed = 3102
diff --git a/parser/mysql/errname.go b/parser/mysql/errname.go
index eef4defcb2465..5b1cff580f658 100644
--- a/parser/mysql/errname.go
+++ b/parser/mysql/errname.go
@@ -97,7 +97,7 @@ var MySQLErrName = map[uint16]*ErrMessage{
ErrMultiplePriKey: Message("Multiple primary key defined", nil),
ErrTooManyKeys: Message("Too many keys specified; max %d keys allowed", nil),
ErrTooManyKeyParts: Message("Too many key parts specified; max %d parts allowed", nil),
- ErrTooLongKey: Message("Specified key was too long; max key length is %d bytes", nil),
+ ErrTooLongKey: Message("Specified key was too long (%d bytes); max key length is %d bytes", nil),
ErrKeyColumnDoesNotExits: Message("Key column '%-.192s' doesn't exist in table", nil),
ErrBlobUsedAsKey: Message("BLOB column '%-.192s' can't be used in key specification with the used table type", nil),
ErrTooBigFieldlength: Message("Column length too big for column '%-.192s' (max = %d); use BLOB or TEXT instead", nil),
@@ -899,6 +899,7 @@ var MySQLErrName = map[uint16]*ErrMessage{
ErrDependentByGeneratedColumn: Message("Column '%s' has a generated column dependency.", nil),
ErrGeneratedColumnRefAutoInc: Message("Generated column '%s' cannot refer to auto-increment column.", nil),
ErrInvalidFieldSize: Message("Invalid size for column '%s'.", nil),
+ ErrPasswordExpireAnonymousUser: Message("The password for anonymous user cannot be expired.", nil),
ErrIncorrectType: Message("Incorrect type for argument %s in function %s.", nil),
ErrInvalidJSONData: Message("Invalid JSON data provided to function %s: %s", nil),
ErrInvalidJSONText: Message("Invalid JSON text: %-.192s", nil),
diff --git a/parser/mysql/type.go b/parser/mysql/type.go
index c54d0f8984b63..8a2531d870d3e 100644
--- a/parser/mysql/type.go
+++ b/parser/mysql/type.go
@@ -43,7 +43,7 @@ const (
TypeLongBlob byte = 0xfb
TypeBlob byte = 0xfc
TypeVarString byte = 0xfd
- TypeString byte = 0xfe
+ TypeString byte = 0xfe /* TypeString is char type */
TypeGeometry byte = 0xff
)
@@ -74,6 +74,7 @@ const (
PreventNullInsertFlag uint = 1 << 20 /* Prevent this Field from inserting NULL values */
EnumSetAsIntFlag uint = 1 << 21 /* Internal: Used for inferring enum eval type. */
DropColumnIndexFlag uint = 1 << 22 /* Internal: Used for indicate the column is being dropped with index */
+ GeneratedColumnFlag uint = 1 << 23 /* Internal: TiFlash will check this flag and add a placeholder for this column */
)
// TypeInt24 bounds.
diff --git a/parser/mysql/util.go b/parser/mysql/util.go
index 367839b2ce3af..c69e290369598 100644
--- a/parser/mysql/util.go
+++ b/parser/mysql/util.go
@@ -93,3 +93,10 @@ func GetDefaultFieldLengthAndDecimalForCast(tp byte) (flen int, decimal int) {
}
return -1, -1
}
+
+// IsAuthPluginClearText is used to indicated that the plugin need clear-text password.
+func IsAuthPluginClearText(authPlugin string) bool {
+ return authPlugin == AuthNativePassword ||
+ authPlugin == AuthTiDBSM3Password ||
+ authPlugin == AuthCachingSha2Password
+}
diff --git a/parser/parser.go b/parser/parser.go
index 4dc144d7a6b9d..ba2b05bfa1282 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -34,6 +34,7 @@ import (
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/auth"
"github.com/pingcap/tidb/parser/charset"
+ "github.com/pingcap/tidb/parser/duration"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/opcode"
@@ -54,2187 +55,2230 @@ type yyXError struct {
}
const (
- yyDefault = 58117
+ yyDefault = 58135
yyEOFCode = 57344
- account = 57575
- action = 57576
- add = 57360
- addDate = 57917
- admin = 58002
- advise = 57577
- after = 57578
- against = 57579
- ago = 57580
- algorithm = 57581
- all = 57361
- alter = 57362
- always = 57582
- analyze = 57363
- and = 57364
- andand = 57355
- andnot = 58078
- any = 57583
- approxCountDistinct = 57918
- approxPercentile = 57919
- as = 57365
- asc = 57366
- ascii = 57584
+ account = 57577
+ action = 57578
+ add = 57361
+ addDate = 57928
+ admin = 58020
+ advise = 57579
+ after = 57580
+ against = 57581
+ ago = 57582
+ algorithm = 57583
+ all = 57362
+ alter = 57363
+ always = 57584
+ analyze = 57364
+ and = 57365
+ andand = 57356
+ andnot = 58096
+ any = 57585
+ approxCountDistinct = 57929
+ approxPercentile = 57930
+ array = 57366
+ as = 57367
+ asc = 57368
+ ascii = 57586
asof = 57347
- assignmentEq = 58079
- attribute = 57585
- attributes = 57586
- autoIdCache = 57591
- autoIncrement = 57592
- autoRandom = 57593
- autoRandomBase = 57594
- avg = 57595
- avgRowLength = 57596
- backend = 57597
- backup = 57598
- backups = 57599
- batch = 58003
- begin = 57600
- bernoulli = 57601
- between = 57367
- bigIntType = 57368
- binaryType = 57369
- binding = 57602
- bindingCache = 57603
- bindings = 57604
- binlog = 57605
- bitAnd = 57920
- bitLit = 58077
- bitOr = 57921
- bitType = 57606
- bitXor = 57922
- blobType = 57370
- block = 57607
- boolType = 57609
- booleanType = 57608
- both = 57371
- bound = 57923
- briefType = 57924
- btree = 57610
- buckets = 58004
- builtinApproxCountDistinct = 58051
- builtinApproxPercentile = 58052
- builtinBitAnd = 58046
- builtinBitOr = 58047
- builtinBitXor = 58048
- builtinCast = 58049
- builtinCount = 58050
- builtinCurDate = 58053
- builtinCurTime = 58054
- builtinDateAdd = 58055
- builtinDateSub = 58056
- builtinExtract = 58057
- builtinGroupConcat = 58058
- builtinMax = 58059
- builtinMin = 58060
- builtinNow = 58061
- builtinPosition = 58062
- builtinStddevPop = 58066
- builtinStddevSamp = 58067
- builtinSubstring = 58063
- builtinSum = 58064
- builtinSysDate = 58065
- builtinTranslate = 58068
- builtinTrim = 58069
- builtinUser = 58070
- builtinVarPop = 58071
- builtinVarSamp = 58072
- builtins = 58005
- by = 57372
- byteType = 57611
- cache = 57612
- call = 57373
- cancel = 58006
- capture = 57613
- cardinality = 58007
- cascade = 57374
- cascaded = 57614
- caseKwd = 57375
- cast = 57925
- causal = 57615
- chain = 57616
- change = 57376
- charType = 57378
- character = 57377
- charsetKwd = 57617
- check = 57379
- checkpoint = 57618
- checksum = 57619
- cipher = 57620
- cleanup = 57621
- client = 57622
- clientErrorsSummary = 57623
- cluster = 57649
- clustered = 57650
- cmSketch = 58008
- coalesce = 57624
- collate = 57380
- collation = 57625
- column = 57381
- columnFormat = 57626
- columnStatsUsage = 58009
- columns = 57627
- comment = 57629
- commit = 57630
- committed = 57631
- compact = 57632
- compressed = 57633
- compression = 57634
- concurrency = 57635
- config = 57628
- connection = 57636
- consistency = 57637
- consistent = 57638
- constraint = 57382
- constraints = 57927
- context = 57639
- convert = 57383
- copyKwd = 57926
- correlation = 58010
- cpu = 57640
- create = 57384
- createTableSelect = 58101
- cross = 57385
- csvBackslashEscape = 57641
- csvDelimiter = 57642
- csvHeader = 57643
- csvNotNull = 57644
- csvNull = 57645
- csvSeparator = 57646
- csvTrimLastSeparators = 57647
- cumeDist = 57386
- curTime = 57928
- current = 57648
- currentDate = 57387
- currentRole = 57391
- currentTime = 57388
- currentTs = 57389
- currentUser = 57390
- cycle = 57651
- data = 57652
- database = 57392
- databases = 57393
- dateAdd = 57929
- dateSub = 57930
- dateType = 57654
- datetimeType = 57653
- day = 57655
- dayHour = 57394
- dayMicrosecond = 57395
- dayMinute = 57396
- daySecond = 57397
- ddl = 58011
- deallocate = 57656
- decLit = 58074
- decimalType = 57398
- defaultKwd = 57399
- definer = 57657
- delayKeyWrite = 57658
- delayed = 57400
- deleteKwd = 57401
- denseRank = 57402
- dependency = 58012
- depth = 58013
- desc = 57403
- describe = 57404
- directory = 57659
- disable = 57660
- disabled = 57661
- discard = 57662
- disk = 57663
- distinct = 57405
- distinctRow = 57406
- div = 57407
- do = 57664
- dotType = 57931
- doubleAtIdentifier = 57352
- doubleType = 57408
- drainer = 58014
- drop = 57409
- dry = 58015
- dual = 57410
- dump = 57932
- duplicate = 57665
- dynamic = 57666
- elseKwd = 57411
- empty = 58092
- enable = 57667
- enabled = 57668
- enclosed = 57412
- encryption = 57669
- end = 57670
- enforced = 57671
- engine = 57672
- engines = 57673
- enum = 57674
- eq = 58080
+ assignmentEq = 58097
+ attribute = 57587
+ attributes = 57588
+ autoIdCache = 57593
+ autoIncrement = 57594
+ autoRandom = 57595
+ autoRandomBase = 57596
+ avg = 57597
+ avgRowLength = 57598
+ backend = 57599
+ backup = 57600
+ backups = 57601
+ batch = 58021
+ begin = 57602
+ bernoulli = 57603
+ between = 57369
+ bigIntType = 57370
+ binaryType = 57371
+ binding = 57604
+ bindingCache = 57605
+ bindings = 57606
+ binlog = 57607
+ bitAnd = 57931
+ bitLit = 58095
+ bitOr = 57932
+ bitType = 57608
+ bitXor = 57933
+ blobType = 57372
+ block = 57609
+ boolType = 57611
+ booleanType = 57610
+ both = 57373
+ bound = 57934
+ briefType = 57935
+ btree = 57612
+ buckets = 58022
+ builtinApproxCountDistinct = 58069
+ builtinApproxPercentile = 58070
+ builtinBitAnd = 58064
+ builtinBitOr = 58065
+ builtinBitXor = 58066
+ builtinCast = 58067
+ builtinCount = 58068
+ builtinCurDate = 58071
+ builtinCurTime = 58072
+ builtinDateAdd = 58073
+ builtinDateSub = 58074
+ builtinExtract = 58075
+ builtinGroupConcat = 58076
+ builtinMax = 58077
+ builtinMin = 58078
+ builtinNow = 58079
+ builtinPosition = 58080
+ builtinStddevPop = 58084
+ builtinStddevSamp = 58085
+ builtinSubstring = 58081
+ builtinSum = 58082
+ builtinSysDate = 58083
+ builtinTranslate = 58086
+ builtinTrim = 58087
+ builtinUser = 58088
+ builtinVarPop = 58089
+ builtinVarSamp = 58090
+ builtins = 58023
+ burstable = 57936
+ by = 57374
+ byteType = 57613
+ cache = 57614
+ call = 57375
+ cancel = 58024
+ capture = 57615
+ cardinality = 58025
+ cascade = 57376
+ cascaded = 57616
+ caseKwd = 57377
+ cast = 57937
+ causal = 57617
+ chain = 57618
+ change = 57378
+ charType = 57380
+ character = 57379
+ charsetKwd = 57619
+ check = 57381
+ checkpoint = 57620
+ checksum = 57621
+ cipher = 57622
+ cleanup = 57623
+ client = 57624
+ clientErrorsSummary = 57625
+ cluster = 57651
+ clustered = 57652
+ cmSketch = 58026
+ coalesce = 57626
+ collate = 57382
+ collation = 57627
+ column = 57383
+ columnFormat = 57628
+ columnStatsUsage = 58027
+ columns = 57629
+ comment = 57631
+ commit = 57632
+ committed = 57633
+ compact = 57634
+ compressed = 57635
+ compression = 57636
+ concurrency = 57637
+ config = 57630
+ connection = 57638
+ consistency = 57639
+ consistent = 57640
+ constraint = 57384
+ constraints = 57939
+ context = 57641
+ convert = 57385
+ copyKwd = 57938
+ correlation = 58028
+ cpu = 57642
+ create = 57386
+ createTableSelect = 58119
+ cross = 57387
+ csvBackslashEscape = 57643
+ csvDelimiter = 57644
+ csvHeader = 57645
+ csvNotNull = 57646
+ csvNull = 57647
+ csvSeparator = 57648
+ csvTrimLastSeparators = 57649
+ cumeDist = 57388
+ curDate = 57941
+ curTime = 57940
+ current = 57650
+ currentDate = 57389
+ currentRole = 57393
+ currentTime = 57390
+ currentTs = 57391
+ currentUser = 57392
+ cycle = 57653
+ data = 57654
+ database = 57394
+ databases = 57395
+ dateAdd = 57942
+ dateSub = 57943
+ dateType = 57656
+ datetimeType = 57655
+ day = 57657
+ dayHour = 57396
+ dayMicrosecond = 57397
+ dayMinute = 57398
+ daySecond = 57399
+ ddl = 58029
+ deallocate = 57658
+ decLit = 58092
+ decimalType = 57400
+ defaultKwd = 57401
+ definer = 57659
+ delayKeyWrite = 57660
+ delayed = 57402
+ deleteKwd = 57403
+ denseRank = 57404
+ dependency = 58030
+ depth = 58031
+ desc = 57405
+ describe = 57406
+ digest = 57661
+ directory = 57662
+ disable = 57663
+ disabled = 57664
+ discard = 57665
+ disk = 57666
+ distinct = 57407
+ distinctRow = 57408
+ div = 57409
+ do = 57667
+ dotType = 57944
+ doubleAtIdentifier = 57353
+ doubleType = 57410
+ drainer = 58032
+ drop = 57411
+ dry = 58033
+ dual = 57412
+ dump = 57945
+ duplicate = 57668
+ dynamic = 57669
+ elseKwd = 57413
+ empty = 58110
+ enable = 57670
+ enabled = 57671
+ enclosed = 57414
+ encryption = 57672
+ end = 57673
+ enforced = 57674
+ engine = 57675
+ engines = 57676
+ enum = 57677
+ eq = 58098
yyErrCode = 57345
- errorKwd = 57675
- escape = 57676
- escaped = 57413
- event = 57677
- events = 57678
- evolve = 57679
- exact = 57933
- except = 57416
- exchange = 57680
- exclusive = 57681
- execute = 57682
- exists = 57414
- expansion = 57683
- expire = 57684
- explain = 57415
- exprPushdownBlacklist = 57934
- extended = 57685
- extract = 57935
- falseKwd = 57417
- faultsSym = 57686
- fetch = 57418
- fields = 57687
- file = 57688
- first = 57689
- firstValue = 57419
- fixed = 57690
- flashback = 57936
- floatLit = 58073
- floatType = 57420
- flush = 57691
- follower = 57937
- followerConstraints = 57938
- followers = 57939
- following = 57692
- forKwd = 57421
- force = 57422
- foreign = 57423
- format = 57693
- from = 57424
- full = 57694
- fulltext = 57425
- function = 57695
- ge = 58081
- general = 57696
- generated = 57426
- getFormat = 57940
- global = 57697
- grant = 57427
- grants = 57698
- group = 57428
- groupConcat = 57941
- groups = 57429
- hash = 57699
- having = 57430
- help = 57700
- hexLit = 58076
- highPriority = 57431
- higherThanComma = 58116
- higherThanParenthese = 58110
- hintComment = 57354
- histogram = 57701
- histogramsInFlight = 58035
- history = 57702
- hosts = 57703
- hour = 57704
- hourMicrosecond = 57432
- hourMinute = 57433
- hourSecond = 57434
- identSQLErrors = 57706
- identified = 57705
+ errorKwd = 57678
+ escape = 57679
+ escaped = 57415
+ event = 57680
+ events = 57681
+ evolve = 57682
+ exact = 57946
+ except = 57418
+ exchange = 57683
+ exclusive = 57684
+ execute = 57685
+ exists = 57416
+ expansion = 57686
+ expire = 57687
+ explain = 57417
+ exprPushdownBlacklist = 57947
+ extended = 57688
+ extract = 57948
+ failedLoginAttempts = 57926
+ falseKwd = 57419
+ faultsSym = 57689
+ fetch = 57420
+ fields = 57690
+ file = 57691
+ first = 57692
+ firstValue = 57421
+ fixed = 57693
+ flashback = 57949
+ floatLit = 58091
+ floatType = 57422
+ flush = 57694
+ follower = 57950
+ followerConstraints = 57951
+ followers = 57952
+ following = 57695
+ forKwd = 57423
+ force = 57424
+ foreign = 57425
+ format = 57696
+ from = 57426
+ full = 57697
+ fulltext = 57427
+ function = 57698
+ ge = 58099
+ general = 57699
+ generated = 57428
+ getFormat = 57953
+ global = 57700
+ grant = 57429
+ grants = 57701
+ group = 57430
+ groupConcat = 57954
+ groups = 57431
+ hash = 57702
+ having = 57432
+ help = 57703
+ hexLit = 58094
+ highPriority = 57433
+ higherThanComma = 58134
+ higherThanParenthese = 58128
+ hintComment = 57355
+ histogram = 57704
+ histogramsInFlight = 58053
+ history = 57705
+ hosts = 57706
+ hour = 57707
+ hourMicrosecond = 57434
+ hourMinute = 57435
+ hourSecond = 57436
+ identSQLErrors = 57709
+ identified = 57708
identifier = 57346
- ifKwd = 57435
- ignore = 57436
- importKwd = 57707
- imports = 57708
- in = 57437
- increment = 57709
- incremental = 57710
- index = 57438
- indexes = 57711
- infile = 57439
- inner = 57440
- inplace = 57943
- insert = 57447
- insertMethod = 57712
- insertValues = 58099
- instance = 57713
- instant = 57944
- int1Type = 57449
- int2Type = 57450
- int3Type = 57451
- int4Type = 57452
- int8Type = 57453
- intLit = 58075
- intType = 57448
- integerType = 57441
- internal = 57945
- intersect = 57442
- interval = 57443
- into = 57444
- invalid = 57353
- invisible = 57714
- invoker = 57715
- io = 57716
- ipc = 57717
- is = 57446
- isolation = 57718
- issuer = 57719
- job = 58017
- jobs = 58016
- join = 57454
- jsonArrayagg = 57946
- jsonObjectAgg = 57947
- jsonType = 57720
- jss = 58083
- juss = 58084
- key = 57455
- keyBlockSize = 57721
- keys = 57456
- kill = 57457
- labels = 57722
- lag = 57458
- language = 57723
- last = 57724
- lastBackup = 57725
- lastValue = 57459
- lastval = 57726
- le = 58082
- lead = 57460
- leader = 57948
- leaderConstraints = 57949
- leading = 57461
- learner = 57950
- learnerConstraints = 57951
- learners = 57952
- left = 57462
- less = 57727
- level = 57728
- like = 57463
- limit = 57464
- linear = 57466
- lines = 57465
- list = 57729
- load = 57467
- local = 57730
- localTime = 57468
- localTs = 57469
- location = 57732
- lock = 57470
- locked = 57731
- logs = 57733
- long = 57560
- longblobType = 57471
- longtextType = 57472
- lowPriority = 57473
- lowerThanCharsetKwd = 58102
- lowerThanComma = 58115
- lowerThanCreateTableSelect = 58100
- lowerThanEq = 58112
- lowerThanFunction = 58107
- lowerThanInsertValues = 58098
- lowerThanKey = 58103
- lowerThanLocal = 58104
- lowerThanNot = 58114
- lowerThanOn = 58111
- lowerThanParenthese = 58109
- lowerThanRemove = 58105
- lowerThanSelectOpt = 58093
- lowerThanSelectStmt = 58097
- lowerThanSetKeyword = 58096
- lowerThanStringLitToken = 58095
- lowerThanValueKeyword = 58094
- lowerThenOrder = 58106
- lsh = 58085
- master = 57734
- match = 57474
- max = 57954
- maxConnectionsPerHour = 57737
- maxQueriesPerHour = 57738
- maxRows = 57739
- maxUpdatesPerHour = 57740
- maxUserConnections = 57741
- maxValue = 57475
- max_idxnum = 57735
- max_minutes = 57736
- mb = 57742
- mediumIntType = 57477
- mediumblobType = 57476
- mediumtextType = 57478
- memory = 57743
- merge = 57744
- microsecond = 57745
- min = 57953
- minRows = 57746
- minValue = 57748
- minute = 57747
- minuteMicrosecond = 57479
- minuteSecond = 57480
- mod = 57481
- mode = 57749
- modify = 57750
- month = 57751
- names = 57752
- national = 57753
- natural = 57574
- ncharType = 57754
- neg = 58113
- neq = 58086
- neqSynonym = 58087
- never = 57755
- next = 57756
- next_row_id = 57942
- nextval = 57757
- no = 57758
- noWriteToBinLog = 57483
- nocache = 57759
- nocycle = 57760
- nodeID = 58018
- nodeState = 58019
- nodegroup = 57761
- nomaxvalue = 57762
- nominvalue = 57763
- nonclustered = 57764
- none = 57765
- not = 57482
- not2 = 58091
- now = 57955
- nowait = 57766
- nthValue = 57484
- ntile = 57485
- null = 57486
- nulleq = 58088
- nulls = 57768
- numericType = 57487
- nvarcharType = 57767
- odbcDateType = 57357
- odbcTimeType = 57358
- odbcTimestampType = 57359
- of = 57488
- off = 57769
- offset = 57770
- on = 57489
- onDuplicate = 57771
- online = 57772
- only = 57773
- open = 57774
- optRuleBlacklist = 57956
- optimistic = 58020
- optimize = 57490
- option = 57491
- optional = 57775
- optionally = 57492
- or = 57493
- order = 57494
- outer = 57495
- outfile = 57445
- over = 57496
- packKeys = 57776
- pageSym = 57777
- paramMarker = 58089
- parser = 57778
- partial = 57779
- partition = 57497
- partitioning = 57780
- partitions = 57781
- password = 57782
- per_db = 57784
- per_table = 57785
- percent = 57783
- percentRank = 57498
- pessimistic = 58021
- pipes = 57356
- pipesAsOr = 57786
- placement = 57957
- plan = 57958
- planCache = 57959
- plugins = 57787
- policy = 57788
- position = 57960
- preSplitRegions = 57789
- preceding = 57790
- precisionType = 57499
- predicate = 57961
- prepare = 57791
- preserve = 57792
- primary = 57500
- primaryRegion = 57962
- privileges = 57793
- procedure = 57501
- process = 57794
- processlist = 57795
- profile = 57796
- profiles = 57797
- proxy = 57798
- pump = 58022
- purge = 57799
- quarter = 57800
- queries = 57801
- query = 57802
- quick = 57803
- rangeKwd = 57502
- rank = 57503
- rateLimit = 57804
- read = 57504
- realType = 57505
- rebuild = 57805
- recent = 57963
- recover = 57806
- recursive = 57506
- redundant = 57807
- references = 57507
- regexpKwd = 57508
- region = 58045
- regions = 58044
- release = 57509
- reload = 57808
- remove = 57809
- rename = 57510
- reorganize = 57810
- repair = 57811
- repeat = 57511
- repeatable = 57812
- replace = 57512
- replayer = 57964
- replica = 57813
- replicas = 57814
- replication = 57815
- require = 57513
- required = 57816
- reset = 58043
- respect = 57817
- restart = 57818
- restore = 57819
- restores = 57820
- restrict = 57514
- resume = 57821
- reverse = 57822
- revoke = 57515
- right = 57516
- rlike = 57517
- role = 57823
- rollback = 57824
- routine = 57825
- row = 57518
- rowCount = 57826
- rowFormat = 57827
- rowNumber = 57520
- rows = 57519
- rsh = 58090
- rtree = 57828
- run = 58023
- running = 57965
- s3 = 57966
- sampleRate = 58025
- samples = 58024
- san = 57829
- savepoint = 57830
- schedule = 57967
- second = 57831
- secondMicrosecond = 57521
- secondaryEngine = 57832
- secondaryLoad = 57833
- secondaryUnload = 57834
- security = 57835
- selectKwd = 57522
- sendCredentialsToTiKV = 57836
- separator = 57837
- sequence = 57838
- serial = 57839
- serializable = 57840
- session = 57841
- sessionStates = 58026
- set = 57523
- setval = 57842
- shardRowIDBits = 57843
- share = 57844
- shared = 57845
- show = 57524
- shutdown = 57846
- signed = 57847
- simple = 57848
- singleAtIdentifier = 57351
- skip = 57849
- skipSchemaFiles = 57850
- slave = 57851
- slow = 57852
- smallIntType = 57525
- snapshot = 57853
- some = 57854
- source = 57855
- spatial = 57526
- split = 58041
- sql = 57527
- sqlBigResult = 57528
- sqlBufferResult = 57856
- sqlCache = 57857
- sqlCalcFoundRows = 57529
- sqlNoCache = 57858
- sqlSmallResult = 57530
- sqlTsiDay = 57859
- sqlTsiHour = 57860
- sqlTsiMinute = 57861
- sqlTsiMonth = 57862
- sqlTsiQuarter = 57863
- sqlTsiSecond = 57864
- sqlTsiWeek = 57865
- sqlTsiYear = 57866
- ssl = 57531
- staleness = 57968
- start = 57867
- starting = 57532
- statistics = 58027
- stats = 58028
- statsAutoRecalc = 57868
- statsBuckets = 58031
- statsColChoice = 57589
- statsColList = 57590
- statsExtended = 57533
- statsHealthy = 58032
- statsHistograms = 58030
- statsLocked = 58034
- statsMeta = 58029
- statsOptions = 57587
- statsPersistent = 57869
- statsSamplePages = 57870
- statsSampleRate = 57588
- statsTopN = 58033
- status = 57871
- std = 57969
- stddev = 57970
- stddevPop = 57971
- stddevSamp = 57972
- stop = 57973
- storage = 57872
- stored = 57538
- straightJoin = 57534
- strict = 57974
- strictFormat = 57873
- stringLit = 57350
- strong = 57975
- subDate = 57976
- subject = 57874
- subpartition = 57875
- subpartitions = 57876
- substring = 57978
- sum = 57977
- super = 57877
- swaps = 57878
- switchesSym = 57879
- system = 57880
- systemTime = 57881
- tableChecksum = 57882
- tableKwd = 57536
- tableRefPriority = 58108
- tableSample = 57537
- tables = 57883
- tablespace = 57884
- target = 57979
- telemetry = 58036
- telemetryID = 58037
- temporary = 57885
- temptable = 57886
- terminated = 57539
- textType = 57887
- than = 57888
- then = 57540
- tiFlash = 58039
- tidb = 58038
- tidbCurrentTSO = 57535
- tikvImporter = 57889
- timeType = 57891
- timestampAdd = 57980
- timestampDiff = 57981
- timestampType = 57890
- tinyIntType = 57542
- tinyblobType = 57541
- tinytextType = 57543
- tls = 57982
- to = 57544
+ ifKwd = 57437
+ ignore = 57438
+ importKwd = 57710
+ imports = 57711
+ in = 57439
+ increment = 57712
+ incremental = 57713
+ index = 57440
+ indexes = 57714
+ infile = 57441
+ inner = 57442
+ inplace = 57956
+ insert = 57449
+ insertMethod = 57715
+ insertValues = 58117
+ instance = 57716
+ instant = 57957
+ int1Type = 57451
+ int2Type = 57452
+ int3Type = 57453
+ int4Type = 57454
+ int8Type = 57455
+ intLit = 58093
+ intType = 57450
+ integerType = 57443
+ internal = 57958
+ intersect = 57444
+ interval = 57445
+ into = 57446
+ invalid = 57354
+ invisible = 57717
+ invoker = 57718
+ io = 57719
+ ioReadBandwidth = 58018
+ ioWriteBandwidth = 58019
+ ipc = 57720
+ is = 57448
+ isolation = 57721
+ issuer = 57722
+ job = 58035
+ jobs = 58034
+ join = 57456
+ jsonArrayagg = 57959
+ jsonObjectAgg = 57960
+ jsonType = 57723
+ jss = 58101
+ juss = 58102
+ key = 57457
+ keyBlockSize = 57724
+ keys = 57458
+ kill = 57459
+ labels = 57725
+ lag = 57460
+ language = 57726
+ last = 57727
+ lastBackup = 57728
+ lastValue = 57461
+ lastval = 57729
+ le = 58100
+ lead = 57462
+ leader = 57961
+ leaderConstraints = 57962
+ leading = 57463
+ learner = 57963
+ learnerConstraints = 57964
+ learners = 57965
+ left = 57464
+ less = 57730
+ level = 57731
+ like = 57465
+ limit = 57466
+ linear = 57468
+ lines = 57467
+ list = 57732
+ load = 57469
+ local = 57733
+ localTime = 57470
+ localTs = 57471
+ location = 57735
+ lock = 57472
+ locked = 57734
+ logs = 57736
+ long = 57562
+ longblobType = 57473
+ longtextType = 57474
+ lowPriority = 57475
+ lowerThanCharsetKwd = 58120
+ lowerThanComma = 58133
+ lowerThanCreateTableSelect = 58118
+ lowerThanEq = 58130
+ lowerThanFunction = 58125
+ lowerThanInsertValues = 58116
+ lowerThanKey = 58121
+ lowerThanLocal = 58122
+ lowerThanNot = 58132
+ lowerThanOn = 58129
+ lowerThanParenthese = 58127
+ lowerThanRemove = 58123
+ lowerThanSelectOpt = 58111
+ lowerThanSelectStmt = 58115
+ lowerThanSetKeyword = 58114
+ lowerThanStringLitToken = 58113
+ lowerThanValueKeyword = 58112
+ lowerThenOrder = 58124
+ lsh = 58103
+ master = 57737
+ match = 57476
+ max = 57967
+ maxConnectionsPerHour = 57740
+ maxQueriesPerHour = 57741
+ maxRows = 57742
+ maxUpdatesPerHour = 57743
+ maxUserConnections = 57744
+ maxValue = 57477
+ max_idxnum = 57738
+ max_minutes = 57739
+ mb = 57745
+ mediumIntType = 57479
+ mediumblobType = 57478
+ mediumtextType = 57480
+ member = 57746
+ memberof = 57349
+ memory = 57747
+ merge = 57748
+ microsecond = 57749
+ min = 57966
+ minRows = 57750
+ minValue = 57752
+ minute = 57751
+ minuteMicrosecond = 57481
+ minuteSecond = 57482
+ mod = 57483
+ mode = 57753
+ modify = 57754
+ month = 57755
+ names = 57756
+ national = 57757
+ natural = 57576
+ ncharType = 57758
+ neg = 58131
+ neq = 58104
+ neqSynonym = 58105
+ never = 57759
+ next = 57760
+ next_row_id = 57955
+ nextval = 57761
+ no = 57762
+ noWriteToBinLog = 57485
+ nocache = 57763
+ nocycle = 57764
+ nodeID = 58036
+ nodeState = 58037
+ nodegroup = 57765
+ nomaxvalue = 57766
+ nominvalue = 57767
+ nonclustered = 57768
+ none = 57769
+ not = 57484
+ not2 = 58109
+ now = 57968
+ nowait = 57770
+ nthValue = 57486
+ ntile = 57487
+ null = 57488
+ nulleq = 58106
+ nulls = 57772
+ numericType = 57489
+ nvarcharType = 57771
+ odbcDateType = 57358
+ odbcTimeType = 57359
+ odbcTimestampType = 57360
+ of = 57490
+ off = 57773
+ offset = 57774
+ on = 57491
+ onDuplicate = 57775
+ online = 57776
+ only = 57777
+ open = 57778
+ optRuleBlacklist = 57969
+ optimistic = 58038
+ optimize = 57492
+ option = 57493
+ optional = 57779
+ optionally = 57494
+ or = 57495
+ order = 57496
+ outer = 57497
+ outfile = 57447
+ over = 57498
+ packKeys = 57780
+ pageSym = 57781
+ paramMarker = 58107
+ parser = 57782
+ partial = 57783
+ partition = 57499
+ partitioning = 57784
+ partitions = 57785
+ password = 57786
+ passwordLockTime = 57927
+ per_db = 57788
+ per_table = 57789
+ percent = 57787
+ percentRank = 57500
+ pessimistic = 58039
+ pipes = 57357
+ pipesAsOr = 57790
+ placement = 57970
+ plan = 57971
+ planCache = 57972
+ plugins = 57791
+ policy = 57792
+ position = 57973
+ preSplitRegions = 57793
+ preceding = 57794
+ precisionType = 57501
+ predicate = 57974
+ prepare = 57795
+ preserve = 57796
+ primary = 57502
+ primaryRegion = 57975
+ privileges = 57797
+ procedure = 57503
+ process = 57798
+ processlist = 57799
+ profile = 57800
+ profiles = 57801
+ proxy = 57802
+ pump = 58040
+ purge = 57803
+ quarter = 57804
+ queries = 57805
+ query = 57806
+ quick = 57807
+ rangeKwd = 57504
+ rank = 57505
+ rateLimit = 57808
+ read = 57506
+ realType = 57507
+ rebuild = 57809
+ recent = 57976
+ recover = 57810
+ recursive = 57508
+ redundant = 57811
+ references = 57509
+ regexpKwd = 57510
+ region = 58063
+ regions = 58062
+ release = 57511
+ reload = 57812
+ remove = 57813
+ rename = 57512
+ reorganize = 57814
+ repair = 57815
+ repeat = 57513
+ repeatable = 57816
+ replace = 57514
+ replayer = 57977
+ replica = 57817
+ replicas = 57818
+ replication = 57819
+ require = 57515
+ required = 57820
+ reset = 58061
+ resource = 57821
+ respect = 57822
+ restart = 57823
+ restore = 57824
+ restores = 57825
+ restrict = 57516
+ resume = 57826
+ reuse = 57827
+ reverse = 57828
+ revoke = 57517
+ right = 57518
+ rlike = 57519
+ role = 57829
+ rollback = 57830
+ routine = 57831
+ row = 57520
+ rowCount = 57832
+ rowFormat = 57833
+ rowNumber = 57522
+ rows = 57521
+ rsh = 58108
+ rtree = 57834
+ ruRate = 58017
+ run = 58041
+ running = 57978
+ s3 = 57979
+ sampleRate = 58043
+ samples = 58042
+ san = 57835
+ savepoint = 57836
+ schedule = 57980
+ second = 57837
+ secondMicrosecond = 57523
+ secondaryEngine = 57838
+ secondaryLoad = 57839
+ secondaryUnload = 57840
+ security = 57841
+ selectKwd = 57524
+ sendCredentialsToTiKV = 57842
+ separator = 57843
+ sequence = 57844
+ serial = 57845
+ serializable = 57846
+ session = 57847
+ sessionStates = 58044
+ set = 57525
+ setval = 57848
+ shardRowIDBits = 57849
+ share = 57850
+ shared = 57851
+ show = 57526
+ shutdown = 57852
+ signed = 57853
+ simple = 57854
+ singleAtIdentifier = 57352
+ skip = 57855
+ skipSchemaFiles = 57856
+ slave = 57857
+ slow = 57858
+ smallIntType = 57527
+ snapshot = 57859
+ some = 57860
+ source = 57861
+ spatial = 57528
+ split = 58059
+ sql = 57529
+ sqlBigResult = 57530
+ sqlBufferResult = 57862
+ sqlCache = 57863
+ sqlCalcFoundRows = 57531
+ sqlNoCache = 57864
+ sqlSmallResult = 57532
+ sqlTsiDay = 57865
+ sqlTsiHour = 57866
+ sqlTsiMinute = 57867
+ sqlTsiMonth = 57868
+ sqlTsiQuarter = 57869
+ sqlTsiSecond = 57870
+ sqlTsiWeek = 57871
+ sqlTsiYear = 57872
+ ssl = 57533
+ staleness = 57981
+ start = 57873
+ starting = 57534
+ statistics = 58045
+ stats = 58046
+ statsAutoRecalc = 57874
+ statsBuckets = 58049
+ statsColChoice = 57591
+ statsColList = 57592
+ statsExtended = 57535
+ statsHealthy = 58050
+ statsHistograms = 58048
+ statsLocked = 58052
+ statsMeta = 58047
+ statsOptions = 57589
+ statsPersistent = 57875
+ statsSamplePages = 57876
+ statsSampleRate = 57590
+ statsTopN = 58051
+ status = 57877
+ std = 57982
+ stddev = 57983
+ stddevPop = 57984
+ stddevSamp = 57985
+ stop = 57986
+ storage = 57878
+ stored = 57540
+ straightJoin = 57536
+ strict = 57987
+ strictFormat = 57879
+ stringLit = 57351
+ strong = 57988
+ subDate = 57989
+ subject = 57880
+ subpartition = 57881
+ subpartitions = 57882
+ substring = 57991
+ sum = 57990
+ super = 57883
+ survivalPreferences = 57992
+ swaps = 57884
+ switchesSym = 57885
+ system = 57886
+ systemTime = 57887
+ tableChecksum = 57888
+ tableKwd = 57538
+ tableRefPriority = 58126
+ tableSample = 57539
+ tables = 57889
+ tablespace = 57890
+ target = 57993
+ telemetry = 58054
+ telemetryID = 58055
+ temporary = 57891
+ temptable = 57892
+ terminated = 57541
+ textType = 57893
+ than = 57894
+ then = 57542
+ tiFlash = 58057
+ tidb = 58056
+ tidbCurrentTSO = 57537
+ tidbJson = 57994
+ tikvImporter = 57895
+ timeType = 57897
+ timestampAdd = 57995
+ timestampDiff = 57996
+ timestampType = 57896
+ tinyIntType = 57544
+ tinyblobType = 57543
+ tinytextType = 57545
+ tls = 57997
+ to = 57546
toTimestamp = 57348
- tokenIssuer = 57892
- tokudbDefault = 57983
- tokudbFast = 57984
- tokudbLzma = 57985
- tokudbQuickLZ = 57986
- tokudbSmall = 57988
- tokudbSnappy = 57987
- tokudbUncompressed = 57989
- tokudbZlib = 57990
- tokudbZstd = 57991
- top = 57992
- topn = 58040
- tp = 57893
- trace = 57894
- traditional = 57895
- trailing = 57545
- transaction = 57896
- trigger = 57546
- triggers = 57897
- trim = 57993
- trueCardCost = 57998
- trueKwd = 57547
- truncate = 57898
- unbounded = 57899
- uncommitted = 57900
- undefined = 57901
- underscoreCS = 57349
- unicodeSym = 57902
- union = 57549
- unique = 57548
- unknown = 57903
- unlock = 57550
- unsigned = 57551
- update = 57552
- usage = 57553
- use = 57554
- user = 57904
- using = 57555
- utcDate = 57556
- utcTime = 57558
- utcTimestamp = 57557
- validation = 57905
- value = 57906
- values = 57559
- varPop = 57995
- varSamp = 57996
- varbinaryType = 57563
- varcharType = 57561
- varcharacter = 57562
- variables = 57907
- variance = 57994
- varying = 57564
- verboseType = 57997
- view = 57908
- virtual = 57565
- visible = 57909
- voter = 57999
- voterConstraints = 58000
- voters = 58001
- wait = 57916
- warnings = 57910
- week = 57911
- weightString = 57912
- when = 57566
- where = 57567
- width = 58042
- window = 57569
- with = 57570
- without = 57913
- write = 57568
- x509 = 57914
- xor = 57571
- yearMonth = 57572
- yearType = 57915
- zerofill = 57573
+ tokenIssuer = 57898
+ tokudbDefault = 57998
+ tokudbFast = 57999
+ tokudbLzma = 58000
+ tokudbQuickLZ = 58001
+ tokudbSmall = 58003
+ tokudbSnappy = 58002
+ tokudbUncompressed = 58004
+ tokudbZlib = 58005
+ tokudbZstd = 58006
+ top = 58007
+ topn = 58058
+ tp = 57899
+ trace = 57900
+ traditional = 57901
+ trailing = 57547
+ transaction = 57902
+ trigger = 57548
+ triggers = 57903
+ trim = 58008
+ trueCardCost = 58013
+ trueKwd = 57549
+ truncate = 57904
+ ttl = 57905
+ ttlEnable = 57906
+ ttlJobInterval = 57907
+ unbounded = 57908
+ uncommitted = 57909
+ undefined = 57910
+ underscoreCS = 57350
+ unicodeSym = 57911
+ union = 57551
+ unique = 57550
+ unknown = 57912
+ unlock = 57552
+ unsigned = 57553
+ update = 57554
+ usage = 57555
+ use = 57556
+ user = 57913
+ using = 57557
+ utcDate = 57558
+ utcTime = 57560
+ utcTimestamp = 57559
+ validation = 57914
+ value = 57915
+ values = 57561
+ varPop = 58010
+ varSamp = 58011
+ varbinaryType = 57565
+ varcharType = 57563
+ varcharacter = 57564
+ variables = 57916
+ variance = 58009
+ varying = 57566
+ verboseType = 58012
+ view = 57917
+ virtual = 57567
+ visible = 57918
+ voter = 58014
+ voterConstraints = 58015
+ voters = 58016
+ wait = 57925
+ warnings = 57919
+ week = 57920
+ weightString = 57921
+ when = 57568
+ where = 57569
+ width = 58060
+ window = 57571
+ with = 57572
+ without = 57922
+ write = 57570
+ x509 = 57923
+ xor = 57573
+ yearMonth = 57574
+ yearType = 57924
+ zerofill = 57575
yyMaxDepth = 200
- yyTabOfs = -2557
+ yyTabOfs = -2612
)
var (
yyXLAT = map[int]int{
- 57344: 0, // $end (2266x)
- 59: 1, // ';' (2265x)
- 58041: 2, // split (1878x)
- 57744: 3, // merge (1877x)
- 57809: 4, // remove (1876x)
- 57810: 5, // reorganize (1876x)
- 57629: 6, // comment (1866x)
- 57872: 7, // storage (1784x)
- 57592: 8, // autoIncrement (1773x)
- 44: 9, // ',' (1688x)
- 57689: 10, // first (1675x)
- 57578: 11, // after (1669x)
- 57839: 12, // serial (1665x)
- 57593: 13, // autoRandom (1664x)
- 57626: 14, // columnFormat (1664x)
- 57782: 15, // password (1633x)
- 57617: 16, // charsetKwd (1630x)
- 57619: 17, // checksum (1618x)
- 57957: 18, // placement (1616x)
- 57721: 19, // keyBlockSize (1600x)
- 57884: 20, // tablespace (1597x)
- 57669: 21, // encryption (1595x)
- 57672: 22, // engine (1592x)
- 57652: 23, // data (1590x)
- 57712: 24, // insertMethod (1588x)
- 57739: 25, // maxRows (1588x)
- 57746: 26, // minRows (1588x)
- 57761: 27, // nodegroup (1588x)
- 57636: 28, // connection (1580x)
- 57594: 29, // autoRandomBase (1577x)
- 58031: 30, // statsBuckets (1575x)
- 58033: 31, // statsTopN (1575x)
- 57591: 32, // autoIdCache (1574x)
- 57596: 33, // avgRowLength (1574x)
- 57634: 34, // compression (1574x)
- 57658: 35, // delayKeyWrite (1574x)
- 57776: 36, // packKeys (1574x)
- 57789: 37, // preSplitRegions (1574x)
- 57827: 38, // rowFormat (1574x)
- 57832: 39, // secondaryEngine (1574x)
- 57843: 40, // shardRowIDBits (1574x)
- 57868: 41, // statsAutoRecalc (1574x)
- 57589: 42, // statsColChoice (1574x)
- 57590: 43, // statsColList (1574x)
- 57869: 44, // statsPersistent (1574x)
- 57870: 45, // statsSamplePages (1574x)
- 57588: 46, // statsSampleRate (1574x)
- 57882: 47, // tableChecksum (1574x)
- 57585: 48, // attribute (1523x)
- 57575: 49, // account (1521x)
- 41: 50, // ')' (1514x)
- 57821: 51, // resume (1510x)
- 57847: 52, // signed (1510x)
- 57853: 53, // snapshot (1509x)
- 57597: 54, // backend (1508x)
- 57618: 55, // checkpoint (1508x)
- 57635: 56, // concurrency (1508x)
- 57641: 57, // csvBackslashEscape (1508x)
- 57642: 58, // csvDelimiter (1508x)
- 57643: 59, // csvHeader (1508x)
- 57644: 60, // csvNotNull (1508x)
- 57645: 61, // csvNull (1508x)
- 57646: 62, // csvSeparator (1508x)
- 57647: 63, // csvTrimLastSeparators (1508x)
- 57725: 64, // lastBackup (1508x)
- 57771: 65, // onDuplicate (1508x)
- 57772: 66, // online (1508x)
- 57804: 67, // rateLimit (1508x)
- 57836: 68, // sendCredentialsToTiKV (1508x)
- 57850: 69, // skipSchemaFiles (1508x)
- 57873: 70, // strictFormat (1508x)
- 57889: 71, // tikvImporter (1508x)
- 57898: 72, // truncate (1505x)
- 57758: 73, // no (1504x)
- 57867: 74, // start (1502x)
- 57612: 75, // cache (1499x)
- 57759: 76, // nocache (1498x)
- 57651: 77, // cycle (1497x)
- 57748: 78, // minValue (1497x)
- 57709: 79, // increment (1496x)
- 57760: 80, // nocycle (1496x)
- 57762: 81, // nomaxvalue (1496x)
- 57763: 82, // nominvalue (1496x)
- 57818: 83, // restart (1494x)
- 57581: 84, // algorithm (1493x)
- 57893: 85, // tp (1493x)
- 57650: 86, // clustered (1492x)
- 57714: 87, // invisible (1492x)
- 57764: 88, // nonclustered (1492x)
- 58044: 89, // regions (1492x)
- 57909: 90, // visible (1492x)
- 57875: 91, // subpartition (1489x)
- 57781: 92, // partitions (1488x)
- 57927: 93, // constraints (1485x)
- 57938: 94, // followerConstraints (1485x)
- 57939: 95, // followers (1485x)
- 57949: 96, // leaderConstraints (1485x)
- 57951: 97, // learnerConstraints (1485x)
- 57952: 98, // learners (1485x)
- 57962: 99, // primaryRegion (1485x)
- 57967: 100, // schedule (1485x)
- 58000: 101, // voterConstraints (1485x)
- 58001: 102, // voters (1485x)
- 57627: 103, // columns (1484x)
- 57908: 104, // view (1484x)
- 57915: 105, // yearType (1481x)
- 57655: 106, // day (1480x)
- 57584: 107, // ascii (1479x)
- 57611: 108, // byteType (1479x)
- 57831: 109, // second (1479x)
- 57866: 110, // sqlTsiYear (1479x)
- 57902: 111, // unicodeSym (1479x)
- 57687: 112, // fields (1478x)
- 57704: 113, // hour (1478x)
- 57745: 114, // microsecond (1478x)
- 57747: 115, // minute (1478x)
- 57751: 116, // month (1478x)
- 57800: 117, // quarter (1478x)
- 57859: 118, // sqlTsiDay (1478x)
- 57860: 119, // sqlTsiHour (1478x)
- 57861: 120, // sqlTsiMinute (1478x)
- 57862: 121, // sqlTsiMonth (1478x)
- 57863: 122, // sqlTsiQuarter (1478x)
- 57864: 123, // sqlTsiSecond (1478x)
- 57865: 124, // sqlTsiWeek (1478x)
- 57911: 125, // week (1478x)
- 57883: 126, // tables (1477x)
- 57871: 127, // status (1476x)
- 57837: 128, // separator (1475x)
- 57620: 129, // cipher (1474x)
- 57346: 130, // identifier (1474x)
- 57719: 131, // issuer (1474x)
- 57737: 132, // maxConnectionsPerHour (1474x)
- 57738: 133, // maxQueriesPerHour (1474x)
- 57740: 134, // maxUpdatesPerHour (1474x)
- 57741: 135, // maxUserConnections (1474x)
- 57790: 136, // preceding (1474x)
- 57829: 137, // san (1474x)
- 57874: 138, // subject (1474x)
- 57892: 139, // tokenIssuer (1474x)
- 57707: 140, // importKwd (1473x)
- 57730: 141, // local (1473x)
- 57802: 142, // query (1472x)
- 57849: 143, // skip (1472x)
- 57604: 144, // bindings (1471x)
- 57657: 145, // definer (1471x)
- 57699: 146, // hash (1471x)
- 57705: 147, // identified (1471x)
- 57733: 148, // logs (1471x)
- 57817: 149, // respect (1471x)
- 57630: 150, // commit (1470x)
- 57648: 151, // current (1470x)
- 57671: 152, // enforced (1470x)
- 57692: 153, // following (1470x)
- 57727: 154, // less (1470x)
- 57766: 155, // nowait (1470x)
- 57773: 156, // only (1470x)
- 57824: 157, // rollback (1470x)
- 57830: 158, // savepoint (1470x)
- 57888: 159, // than (1470x)
- 57906: 160, // value (1470x)
- 57600: 161, // begin (1469x)
- 57602: 162, // binding (1469x)
- 57670: 163, // end (1469x)
- 57697: 164, // global (1469x)
- 57942: 165, // next_row_id (1469x)
- 57770: 166, // offset (1469x)
- 57788: 167, // policy (1469x)
- 57961: 168, // predicate (1469x)
- 57885: 169, // temporary (1469x)
- 58039: 170, // tiFlash (1469x)
- 57899: 171, // unbounded (1469x)
- 57904: 172, // user (1469x)
- 57720: 173, // jsonType (1468x)
- 57959: 174, // planCache (1468x)
- 57791: 175, // prepare (1468x)
- 57823: 176, // role (1468x)
- 57903: 177, // unknown (1468x)
- 57916: 178, // wait (1468x)
- 57610: 179, // btree (1467x)
- 57653: 180, // datetimeType (1467x)
- 57654: 181, // dateType (1467x)
- 57690: 182, // fixed (1467x)
- 57706: 183, // identSQLErrors (1467x)
- 57718: 184, // isolation (1467x)
- 57724: 185, // last (1467x)
- 57732: 186, // location (1467x)
- 57735: 187, // max_idxnum (1467x)
- 57743: 188, // memory (1467x)
- 57769: 189, // off (1467x)
- 57775: 190, // optional (1467x)
- 57784: 191, // per_db (1467x)
- 57793: 192, // privileges (1467x)
- 57813: 193, // replica (1467x)
- 57816: 194, // required (1467x)
- 57828: 195, // rtree (1467x)
- 57965: 196, // running (1467x)
- 58025: 197, // sampleRate (1467x)
- 57838: 198, // sequence (1467x)
- 57841: 199, // session (1467x)
- 57852: 200, // slow (1467x)
- 58028: 201, // stats (1467x)
- 57891: 202, // timeType (1467x)
- 57905: 203, // validation (1467x)
- 57907: 204, // variables (1467x)
- 57586: 205, // attributes (1466x)
- 57632: 206, // compact (1466x)
- 57660: 207, // disable (1466x)
- 57665: 208, // duplicate (1466x)
- 57666: 209, // dynamic (1466x)
- 57667: 210, // enable (1466x)
- 57675: 211, // errorKwd (1466x)
- 57691: 212, // flush (1466x)
- 57694: 213, // full (1466x)
- 57742: 214, // mb (1466x)
- 57749: 215, // mode (1466x)
- 57755: 216, // never (1466x)
- 57958: 217, // plan (1466x)
- 57787: 218, // plugins (1466x)
- 57795: 219, // processlist (1466x)
- 57806: 220, // recover (1466x)
- 57811: 221, // repair (1466x)
- 57812: 222, // repeatable (1466x)
- 58027: 223, // statistics (1466x)
- 57876: 224, // subpartitions (1466x)
- 58038: 225, // tidb (1466x)
- 57890: 226, // timestampType (1466x)
- 57913: 227, // without (1466x)
- 58002: 228, // admin (1465x)
- 57598: 229, // backup (1465x)
- 58003: 230, // batch (1465x)
- 57605: 231, // binlog (1465x)
- 57607: 232, // block (1465x)
- 57608: 233, // booleanType (1465x)
- 57924: 234, // briefType (1465x)
- 58004: 235, // buckets (1465x)
- 58007: 236, // cardinality (1465x)
- 57616: 237, // chain (1465x)
- 57623: 238, // clientErrorsSummary (1465x)
- 58008: 239, // cmSketch (1465x)
- 57624: 240, // coalesce (1465x)
- 57633: 241, // compressed (1465x)
- 57639: 242, // context (1465x)
- 57926: 243, // copyKwd (1465x)
- 58010: 244, // correlation (1465x)
- 57640: 245, // cpu (1465x)
- 57656: 246, // deallocate (1465x)
- 58012: 247, // dependency (1465x)
- 57659: 248, // directory (1465x)
- 57662: 249, // discard (1465x)
- 57663: 250, // disk (1465x)
- 57664: 251, // do (1465x)
- 57931: 252, // dotType (1465x)
- 58014: 253, // drainer (1465x)
- 58015: 254, // dry (1465x)
- 57680: 255, // exchange (1465x)
- 57682: 256, // execute (1465x)
- 57683: 257, // expansion (1465x)
- 57936: 258, // flashback (1465x)
- 57693: 259, // format (1465x)
- 57696: 260, // general (1465x)
- 57700: 261, // help (1465x)
- 57701: 262, // histogram (1465x)
- 57703: 263, // hosts (1465x)
- 57943: 264, // inplace (1465x)
- 57713: 265, // instance (1465x)
- 57944: 266, // instant (1465x)
- 57717: 267, // ipc (1465x)
- 58017: 268, // job (1465x)
- 58016: 269, // jobs (1465x)
- 57722: 270, // labels (1465x)
- 57731: 271, // locked (1465x)
- 57750: 272, // modify (1465x)
- 57756: 273, // next (1465x)
- 58018: 274, // nodeID (1465x)
- 58019: 275, // nodeState (1465x)
- 57768: 276, // nulls (1465x)
- 57777: 277, // pageSym (1465x)
- 58022: 278, // pump (1465x)
- 57799: 279, // purge (1465x)
- 57805: 280, // rebuild (1465x)
- 57807: 281, // redundant (1465x)
- 57808: 282, // reload (1465x)
- 57819: 283, // restore (1465x)
- 57825: 284, // routine (1465x)
- 57966: 285, // s3 (1465x)
- 58024: 286, // samples (1465x)
- 57833: 287, // secondaryLoad (1465x)
- 57834: 288, // secondaryUnload (1465x)
- 57844: 289, // share (1465x)
- 57846: 290, // shutdown (1465x)
- 57855: 291, // source (1465x)
- 57587: 292, // statsOptions (1465x)
- 57973: 293, // stop (1465x)
- 57878: 294, // swaps (1465x)
- 57983: 295, // tokudbDefault (1465x)
- 57984: 296, // tokudbFast (1465x)
- 57985: 297, // tokudbLzma (1465x)
- 57986: 298, // tokudbQuickLZ (1465x)
- 57988: 299, // tokudbSmall (1465x)
- 57987: 300, // tokudbSnappy (1465x)
- 57989: 301, // tokudbUncompressed (1465x)
- 57990: 302, // tokudbZlib (1465x)
- 57991: 303, // tokudbZstd (1465x)
- 58040: 304, // topn (1465x)
- 57894: 305, // trace (1465x)
- 57895: 306, // traditional (1465x)
- 57998: 307, // trueCardCost (1465x)
- 57997: 308, // verboseType (1465x)
- 57910: 309, // warnings (1465x)
- 57576: 310, // action (1464x)
- 57577: 311, // advise (1464x)
- 57579: 312, // against (1464x)
- 57580: 313, // ago (1464x)
- 57582: 314, // always (1464x)
- 57599: 315, // backups (1464x)
- 57601: 316, // bernoulli (1464x)
- 57603: 317, // bindingCache (1464x)
- 57606: 318, // bitType (1464x)
- 57609: 319, // boolType (1464x)
- 58005: 320, // builtins (1464x)
- 58006: 321, // cancel (1464x)
- 57613: 322, // capture (1464x)
- 57614: 323, // cascaded (1464x)
- 57615: 324, // causal (1464x)
- 57621: 325, // cleanup (1464x)
- 57622: 326, // client (1464x)
- 57649: 327, // cluster (1464x)
- 57625: 328, // collation (1464x)
- 58009: 329, // columnStatsUsage (1464x)
- 57631: 330, // committed (1464x)
- 57628: 331, // config (1464x)
- 57637: 332, // consistency (1464x)
- 57638: 333, // consistent (1464x)
- 58011: 334, // ddl (1464x)
- 58013: 335, // depth (1464x)
- 57661: 336, // disabled (1464x)
- 57932: 337, // dump (1464x)
- 57668: 338, // enabled (1464x)
- 57673: 339, // engines (1464x)
- 57674: 340, // enum (1464x)
- 57678: 341, // events (1464x)
- 57679: 342, // evolve (1464x)
- 57684: 343, // expire (1464x)
- 57934: 344, // exprPushdownBlacklist (1464x)
- 57685: 345, // extended (1464x)
- 57686: 346, // faultsSym (1464x)
- 57695: 347, // function (1464x)
- 57698: 348, // grants (1464x)
- 58035: 349, // histogramsInFlight (1464x)
- 57702: 350, // history (1464x)
- 57708: 351, // imports (1464x)
- 57710: 352, // incremental (1464x)
- 57711: 353, // indexes (1464x)
- 57945: 354, // internal (1464x)
- 57715: 355, // invoker (1464x)
- 57716: 356, // io (1464x)
- 57723: 357, // language (1464x)
- 57728: 358, // level (1464x)
- 57729: 359, // list (1464x)
- 57734: 360, // master (1464x)
- 57736: 361, // max_minutes (1464x)
- 57753: 362, // national (1464x)
- 57754: 363, // ncharType (1464x)
- 57757: 364, // nextval (1464x)
- 57765: 365, // none (1464x)
- 57767: 366, // nvarcharType (1464x)
- 57774: 367, // open (1464x)
- 58020: 368, // optimistic (1464x)
- 57956: 369, // optRuleBlacklist (1464x)
- 57778: 370, // parser (1464x)
- 57779: 371, // partial (1464x)
- 57780: 372, // partitioning (1464x)
- 57785: 373, // per_table (1464x)
- 57783: 374, // percent (1464x)
- 58021: 375, // pessimistic (1464x)
- 57792: 376, // preserve (1464x)
- 57796: 377, // profile (1464x)
- 57797: 378, // profiles (1464x)
- 57801: 379, // queries (1464x)
- 57963: 380, // recent (1464x)
- 58045: 381, // region (1464x)
- 57964: 382, // replayer (1464x)
- 58043: 383, // reset (1464x)
- 57820: 384, // restores (1464x)
- 58023: 385, // run (1464x)
- 57835: 386, // security (1464x)
- 57840: 387, // serializable (1464x)
- 58026: 388, // sessionStates (1464x)
- 57848: 389, // simple (1464x)
- 57851: 390, // slave (1464x)
- 58032: 391, // statsHealthy (1464x)
- 58030: 392, // statsHistograms (1464x)
- 58034: 393, // statsLocked (1464x)
- 58029: 394, // statsMeta (1464x)
- 57974: 395, // strict (1464x)
- 57879: 396, // switchesSym (1464x)
- 57880: 397, // system (1464x)
- 57881: 398, // systemTime (1464x)
- 57979: 399, // target (1464x)
- 58037: 400, // telemetryID (1464x)
- 57886: 401, // temptable (1464x)
- 57887: 402, // textType (1464x)
- 57982: 403, // tls (1464x)
- 57992: 404, // top (1464x)
- 57896: 405, // transaction (1464x)
- 57897: 406, // triggers (1464x)
- 57900: 407, // uncommitted (1464x)
- 57901: 408, // undefined (1464x)
- 58042: 409, // width (1464x)
- 57914: 410, // x509 (1464x)
- 57917: 411, // addDate (1463x)
- 57583: 412, // any (1463x)
- 57918: 413, // approxCountDistinct (1463x)
- 57919: 414, // approxPercentile (1463x)
- 57595: 415, // avg (1463x)
- 57920: 416, // bitAnd (1463x)
- 57921: 417, // bitOr (1463x)
- 57922: 418, // bitXor (1463x)
- 57923: 419, // bound (1463x)
- 57925: 420, // cast (1463x)
- 57928: 421, // curTime (1463x)
- 57929: 422, // dateAdd (1463x)
- 57930: 423, // dateSub (1463x)
- 57676: 424, // escape (1463x)
- 57677: 425, // event (1463x)
- 57933: 426, // exact (1463x)
- 57681: 427, // exclusive (1463x)
- 57935: 428, // extract (1463x)
- 57688: 429, // file (1463x)
- 57937: 430, // follower (1463x)
- 57940: 431, // getFormat (1463x)
- 57941: 432, // groupConcat (1463x)
- 57946: 433, // jsonArrayagg (1463x)
- 57947: 434, // jsonObjectAgg (1463x)
- 57726: 435, // lastval (1463x)
- 57948: 436, // leader (1463x)
- 57950: 437, // learner (1463x)
- 57954: 438, // max (1463x)
- 57953: 439, // min (1463x)
- 57752: 440, // names (1463x)
- 57955: 441, // now (1463x)
- 57960: 442, // position (1463x)
- 57794: 443, // process (1463x)
- 57798: 444, // proxy (1463x)
- 57803: 445, // quick (1463x)
- 57814: 446, // replicas (1463x)
- 57815: 447, // replication (1463x)
- 57822: 448, // reverse (1463x)
- 57826: 449, // rowCount (1463x)
- 57842: 450, // setval (1463x)
- 57845: 451, // shared (1463x)
- 57854: 452, // some (1463x)
- 57856: 453, // sqlBufferResult (1463x)
- 57857: 454, // sqlCache (1463x)
- 57858: 455, // sqlNoCache (1463x)
- 57968: 456, // staleness (1463x)
- 57969: 457, // std (1463x)
- 57970: 458, // stddev (1463x)
- 57971: 459, // stddevPop (1463x)
- 57972: 460, // stddevSamp (1463x)
- 57975: 461, // strong (1463x)
- 57976: 462, // subDate (1463x)
- 57978: 463, // substring (1463x)
- 57977: 464, // sum (1463x)
- 57877: 465, // super (1463x)
- 58036: 466, // telemetry (1463x)
- 57980: 467, // timestampAdd (1463x)
- 57981: 468, // timestampDiff (1463x)
- 57993: 469, // trim (1463x)
- 57994: 470, // variance (1463x)
- 57995: 471, // varPop (1463x)
- 57996: 472, // varSamp (1463x)
- 57999: 473, // voter (1463x)
- 57912: 474, // weightString (1463x)
- 57489: 475, // on (1398x)
- 40: 476, // '(' (1331x)
- 57570: 477, // with (1215x)
- 57350: 478, // stringLit (1203x)
- 58091: 479, // not2 (1195x)
- 57482: 480, // not (1132x)
- 57365: 481, // as (1109x)
- 57399: 482, // defaultKwd (1103x)
- 57549: 483, // union (1061x)
- 57555: 484, // using (1054x)
- 57462: 485, // left (1049x)
- 57516: 486, // right (1049x)
- 57380: 487, // collate (1046x)
- 43: 488, // '+' (1026x)
- 45: 489, // '-' (1025x)
- 57481: 490, // mod (1005x)
- 57497: 491, // partition (965x)
- 57436: 492, // ignore (960x)
- 57416: 493, // except (953x)
- 57442: 494, // intersect (952x)
- 57486: 495, // null (950x)
- 57464: 496, // limit (933x)
- 57421: 497, // forKwd (930x)
- 57559: 498, // values (925x)
- 57444: 499, // into (923x)
- 57470: 500, // lock (919x)
- 57567: 501, // where (914x)
- 58080: 502, // eq (911x)
- 57424: 503, // from (911x)
- 57418: 504, // fetch (909x)
- 57494: 505, // order (905x)
- 57422: 506, // force (901x)
- 57512: 507, // replace (898x)
- 57378: 508, // charType (897x)
- 57523: 509, // set (892x)
- 57364: 510, // and (891x)
- 58075: 511, // intLit (887x)
- 57493: 512, // or (867x)
- 57355: 513, // andand (866x)
- 57786: 514, // pipesAsOr (866x)
- 57571: 515, // xor (866x)
- 57428: 516, // group (840x)
- 57430: 517, // having (840x)
- 57534: 518, // straightJoin (834x)
- 57569: 519, // window (826x)
- 57454: 520, // join (822x)
- 57463: 521, // like (815x)
- 57574: 522, // natural (812x)
- 42: 523, // '*' (811x)
- 57385: 524, // cross (811x)
- 57440: 525, // inner (811x)
- 125: 526, // '}' (808x)
- 57519: 527, // rows (796x)
- 57554: 528, // use (792x)
- 57537: 529, // tableSample (786x)
- 57502: 530, // rangeKwd (785x)
- 57429: 531, // groups (784x)
- 57403: 532, // desc (783x)
- 57369: 533, // binaryType (782x)
- 57366: 534, // asc (781x)
- 57394: 535, // dayHour (781x)
- 57395: 536, // dayMicrosecond (781x)
- 57396: 537, // dayMinute (781x)
- 57397: 538, // daySecond (781x)
- 57432: 539, // hourMicrosecond (781x)
- 57433: 540, // hourMinute (781x)
- 57434: 541, // hourSecond (781x)
- 57479: 542, // minuteMicrosecond (781x)
- 57480: 543, // minuteSecond (781x)
- 57521: 544, // secondMicrosecond (781x)
- 57572: 545, // yearMonth (781x)
- 57566: 546, // when (778x)
- 57437: 547, // in (776x)
- 57411: 548, // elseKwd (775x)
- 57540: 549, // then (772x)
- 47: 550, // '/' (769x)
- 37: 551, // '%' (768x)
- 38: 552, // '&' (768x)
- 94: 553, // '^' (768x)
- 124: 554, // '|' (768x)
- 57407: 555, // div (768x)
- 58085: 556, // lsh (768x)
- 58090: 557, // rsh (768x)
- 60: 558, // '<' (765x)
- 62: 559, // '>' (765x)
- 58081: 560, // ge (765x)
- 57446: 561, // is (765x)
- 58082: 562, // le (765x)
- 58086: 563, // neq (765x)
- 58087: 564, // neqSynonym (765x)
- 58088: 565, // nulleq (765x)
- 57367: 566, // between (763x)
- 57435: 567, // ifKwd (758x)
- 57508: 568, // regexpKwd (755x)
- 57517: 569, // rlike (755x)
- 57447: 570, // insert (749x)
- 57351: 571, // singleAtIdentifier (739x)
- 57536: 572, // tableKwd (739x)
- 57390: 573, // currentUser (735x)
- 57417: 574, // falseKwd (733x)
- 57547: 575, // trueKwd (733x)
- 58074: 576, // decLit (727x)
- 58073: 577, // floatLit (727x)
- 57518: 578, // row (727x)
- 58076: 579, // hexLit (725x)
- 58089: 580, // paramMarker (725x)
- 57443: 581, // interval (724x)
- 123: 582, // '{' (723x)
- 58077: 583, // bitLit (723x)
- 57455: 584, // key (723x)
- 57392: 585, // database (719x)
- 57414: 586, // exists (718x)
- 57383: 587, // convert (715x)
- 58061: 588, // builtinNow (714x)
- 57389: 589, // currentTs (714x)
- 57352: 590, // doubleAtIdentifier (714x)
- 57468: 591, // localTime (714x)
- 57469: 592, // localTs (714x)
- 57379: 593, // check (713x)
- 57500: 594, // primary (713x)
- 57349: 595, // underscoreCS (713x)
- 58050: 596, // builtinCount (712x)
- 57356: 597, // pipes (712x)
- 33: 598, // '!' (711x)
- 126: 599, // '~' (711x)
- 58051: 600, // builtinApproxCountDistinct (711x)
- 58052: 601, // builtinApproxPercentile (711x)
- 58046: 602, // builtinBitAnd (711x)
- 58047: 603, // builtinBitOr (711x)
- 58048: 604, // builtinBitXor (711x)
- 58049: 605, // builtinCast (711x)
- 58053: 606, // builtinCurDate (711x)
- 58054: 607, // builtinCurTime (711x)
- 58055: 608, // builtinDateAdd (711x)
- 58056: 609, // builtinDateSub (711x)
- 58057: 610, // builtinExtract (711x)
- 58058: 611, // builtinGroupConcat (711x)
- 58059: 612, // builtinMax (711x)
- 58060: 613, // builtinMin (711x)
- 58062: 614, // builtinPosition (711x)
- 58066: 615, // builtinStddevPop (711x)
- 58067: 616, // builtinStddevSamp (711x)
- 58063: 617, // builtinSubstring (711x)
- 58064: 618, // builtinSum (711x)
- 58065: 619, // builtinSysDate (711x)
- 58068: 620, // builtinTranslate (711x)
- 58069: 621, // builtinTrim (711x)
- 58070: 622, // builtinUser (711x)
- 58071: 623, // builtinVarPop (711x)
- 58072: 624, // builtinVarSamp (711x)
- 57375: 625, // caseKwd (711x)
- 57386: 626, // cumeDist (711x)
- 57387: 627, // currentDate (711x)
- 57391: 628, // currentRole (711x)
- 57388: 629, // currentTime (711x)
- 57402: 630, // denseRank (711x)
- 57419: 631, // firstValue (711x)
- 57458: 632, // lag (711x)
- 57459: 633, // lastValue (711x)
- 57460: 634, // lead (711x)
- 57484: 635, // nthValue (711x)
- 57485: 636, // ntile (711x)
- 57498: 637, // percentRank (711x)
- 57503: 638, // rank (711x)
- 57511: 639, // repeat (711x)
- 57520: 640, // rowNumber (711x)
- 57535: 641, // tidbCurrentTSO (711x)
- 57556: 642, // utcDate (711x)
- 57558: 643, // utcTime (711x)
- 57557: 644, // utcTimestamp (711x)
- 57548: 645, // unique (706x)
- 57382: 646, // constraint (704x)
- 57507: 647, // references (701x)
- 57426: 648, // generated (697x)
- 57522: 649, // selectKwd (696x)
- 57377: 650, // character (661x)
- 57474: 651, // match (653x)
- 57438: 652, // index (649x)
- 57544: 653, // to (572x)
- 57361: 654, // all (557x)
- 46: 655, // '.' (552x)
- 57363: 656, // analyze (536x)
- 57552: 657, // update (532x)
- 57475: 658, // maxValue (520x)
- 58083: 659, // jss (518x)
- 58084: 660, // juss (518x)
- 57465: 661, // lines (507x)
- 58079: 662, // assignmentEq (504x)
- 57372: 663, // by (504x)
- 58345: 664, // Identifier (502x)
- 58424: 665, // NotKeywordToken (502x)
- 58653: 666, // TiDBKeyword (502x)
- 58663: 667, // UnReservedKeyword (502x)
- 57362: 668, // alter (501x)
- 57513: 669, // require (499x)
- 64: 670, // '@' (494x)
- 57527: 671, // sql (491x)
- 57409: 672, // drop (488x)
- 57374: 673, // cascade (487x)
- 57504: 674, // read (487x)
- 57514: 675, // restrict (487x)
- 57347: 676, // asof (485x)
- 57384: 677, // create (483x)
- 57423: 678, // foreign (483x)
- 57425: 679, // fulltext (483x)
- 57348: 680, // toTimestamp (482x)
- 57562: 681, // varcharacter (481x)
- 57561: 682, // varcharType (481x)
- 57376: 683, // change (480x)
- 57398: 684, // decimalType (480x)
- 57408: 685, // doubleType (480x)
- 57420: 686, // floatType (480x)
- 57441: 687, // integerType (480x)
- 57448: 688, // intType (480x)
- 57505: 689, // realType (480x)
- 57510: 690, // rename (480x)
- 57568: 691, // write (480x)
- 57563: 692, // varbinaryType (479x)
- 57360: 693, // add (478x)
- 57368: 694, // bigIntType (478x)
- 57370: 695, // blobType (478x)
- 57449: 696, // int1Type (478x)
- 57450: 697, // int2Type (478x)
- 57451: 698, // int3Type (478x)
- 57452: 699, // int4Type (478x)
- 57453: 700, // int8Type (478x)
- 57560: 701, // long (478x)
- 57471: 702, // longblobType (478x)
- 57472: 703, // longtextType (478x)
- 57476: 704, // mediumblobType (478x)
- 57477: 705, // mediumIntType (478x)
- 57478: 706, // mediumtextType (478x)
- 57487: 707, // numericType (478x)
- 57490: 708, // optimize (478x)
- 57525: 709, // smallIntType (478x)
- 57541: 710, // tinyblobType (478x)
- 57542: 711, // tinyIntType (478x)
- 57543: 712, // tinytextType (478x)
- 58618: 713, // SubSelect (223x)
- 58673: 714, // UserVariable (181x)
- 58593: 715, // SimpleIdent (180x)
- 58398: 716, // Literal (178x)
- 58608: 717, // StringLiteral (178x)
- 58421: 718, // NextValueForSequence (177x)
- 58322: 719, // FunctionCallGeneric (176x)
- 58323: 720, // FunctionCallKeyword (176x)
- 58324: 721, // FunctionCallNonKeyword (176x)
- 58325: 722, // FunctionNameConflict (176x)
- 58326: 723, // FunctionNameDateArith (176x)
- 58327: 724, // FunctionNameDateArithMultiForms (176x)
- 58328: 725, // FunctionNameDatetimePrecision (176x)
- 58329: 726, // FunctionNameOptionalBraces (176x)
- 58330: 727, // FunctionNameSequence (176x)
- 58592: 728, // SimpleExpr (176x)
- 58619: 729, // SumExpr (176x)
- 58621: 730, // SystemVariable (176x)
- 58684: 731, // Variable (176x)
- 58707: 732, // WindowFuncCall (176x)
- 58168: 733, // BitExpr (163x)
- 58498: 734, // PredicateExpr (132x)
- 58171: 735, // BoolPri (129x)
- 58286: 736, // Expression (129x)
- 58419: 737, // NUM (103x)
- 58722: 738, // logAnd (97x)
- 58723: 739, // logOr (97x)
- 58631: 740, // TableName (77x)
- 58276: 741, // EqOpt (75x)
- 58609: 742, // StringName (56x)
- 57401: 743, // deleteKwd (52x)
- 57551: 744, // unsigned (47x)
- 58389: 745, // LengthNum (46x)
- 57496: 746, // over (45x)
- 57573: 747, // zerofill (45x)
- 58194: 748, // ColumnName (41x)
- 57405: 749, // distinct (36x)
- 57406: 750, // distinctRow (36x)
- 58712: 751, // WindowingClause (35x)
- 58546: 752, // SelectStmt (34x)
- 58547: 753, // SelectStmtBasic (34x)
- 58549: 754, // SelectStmtFromDualTable (34x)
- 58550: 755, // SelectStmtFromTable (34x)
- 58567: 756, // SetOprClause (34x)
- 57400: 757, // delayed (33x)
- 57431: 758, // highPriority (33x)
- 57473: 759, // lowPriority (33x)
- 58568: 760, // SetOprClauseList (33x)
- 58571: 761, // SetOprStmtWithLimitOrderBy (33x)
- 58572: 762, // SetOprStmtWoutLimitOrderBy (33x)
- 58713: 763, // WithClause (31x)
- 58559: 764, // SelectStmtWithClause (30x)
- 58570: 765, // SetOprStmt (30x)
- 57354: 766, // hintComment (27x)
- 58377: 767, // Int64Num (26x)
- 58297: 768, // FieldLen (25x)
- 58463: 769, // OptWindowingClause (24x)
- 58667: 770, // UpdateStmtNoWith (24x)
- 58251: 771, // DeleteWithoutUsingStmt (23x)
- 58469: 772, // OrderBy (23x)
- 58553: 773, // SelectStmtLimit (23x)
- 57528: 774, // sqlBigResult (23x)
- 57529: 775, // sqlCalcFoundRows (23x)
- 57530: 776, // sqlSmallResult (23x)
- 58374: 777, // InsertIntoStmt (21x)
- 58666: 778, // UpdateStmt (21x)
- 58182: 779, // CharsetKw (20x)
- 58520: 780, // ReplaceIntoStmt (20x)
- 58675: 781, // Username (20x)
- 58287: 782, // ExpressionList (18x)
- 58250: 783, // DeleteWithUsingStmt (17x)
- 58346: 784, // IfExists (17x)
- 58493: 785, // PlacementPolicyOption (17x)
- 58632: 786, // TableNameList (17x)
- 57539: 787, // terminated (16x)
- 58249: 788, // DeleteFromStmt (15x)
- 58253: 789, // DistinctKwd (15x)
- 58347: 790, // IfNotExists (15x)
- 58481: 791, // PartitionNameList (15x)
- 58254: 792, // DistinctOpt (14x)
- 57412: 793, // enclosed (14x)
- 58448: 794, // OptFieldLen (14x)
- 58697: 795, // WhereClause (14x)
- 58698: 796, // WhereClauseOptional (14x)
- 58246: 797, // DefaultKwdOpt (13x)
- 57413: 798, // escaped (13x)
- 57492: 799, // optionally (13x)
- 58655: 800, // TimestampUnit (13x)
- 58285: 801, // ExprOrDefault (12x)
- 58383: 802, // JoinTable (12x)
- 58442: 803, // OptBinary (12x)
- 57509: 804, // release (12x)
- 58536: 805, // RolenameComposed (12x)
- 58628: 806, // TableFactor (12x)
- 58641: 807, // TableRef (12x)
- 58141: 808, // AnalyzeOptionListOpt (11x)
- 58317: 809, // FromOrIn (11x)
- 58137: 810, // AlterTableStmt (10x)
- 58183: 811, // CharsetName (10x)
- 58195: 812, // ColumnNameList (10x)
- 58236: 813, // DBName (10x)
- 57467: 814, // load (10x)
- 58425: 815, // NotSym (10x)
- 57483: 816, // noWriteToBinLog (10x)
- 58470: 817, // OrderByOptional (10x)
- 58472: 818, // PartDefOption (10x)
- 58591: 819, // SignedNum (10x)
- 58654: 820, // TimeUnit (10x)
- 58174: 821, // BuggyDefaultFalseDistinctOpt (9x)
- 58245: 822, // DefaultFalseDistinctOpt (9x)
- 58384: 823, // JoinType (9x)
- 58432: 824, // NumLiteral (9x)
- 58535: 825, // Rolename (9x)
- 58530: 826, // RoleNameString (9x)
- 58235: 827, // CrossOpt (8x)
- 58277: 828, // EqOrAssignmentEq (8x)
- 58284: 829, // ExplainableStmt (8x)
- 58288: 830, // ExpressionListOpt (8x)
- 58368: 831, // IndexPartSpecification (8x)
- 58385: 832, // KeyOrIndex (8x)
- 58422: 833, // NoWriteToBinLogAliasOpt (8x)
- 58554: 834, // SelectStmtLimitOpt (8x)
- 58687: 835, // VariableName (8x)
- 58123: 836, // AllOrPartitionNameList (7x)
- 58219: 837, // ConstraintKeywordOpt (7x)
- 58241: 838, // DatabaseSym (7x)
- 58303: 839, // FieldsOrColumns (7x)
- 58315: 840, // ForceOpt (7x)
- 58369: 841, // IndexPartSpecificationList (7x)
- 58502: 842, // Priority (7x)
- 58540: 843, // RowFormat (7x)
- 58543: 844, // RowValue (7x)
- 58565: 845, // SetExpr (7x)
- 58577: 846, // ShowDatabaseNameOpt (7x)
- 58638: 847, // TableOption (7x)
- 57564: 848, // varying (7x)
- 58142: 849, // AnalyzeTableStmt (6x)
- 58163: 850, // BeginTransactionStmt (6x)
- 58165: 851, // BindableStmt (6x)
- 57381: 852, // column (6x)
- 58189: 853, // ColumnDef (6x)
- 58209: 854, // CommitStmt (6x)
- 58238: 855, // DatabaseOption (6x)
- 58279: 856, // EscapedTableRef (6x)
- 58301: 857, // FieldTerminator (6x)
- 57427: 858, // grant (6x)
- 58351: 859, // IgnoreOptional (6x)
- 58360: 860, // IndexInvisible (6x)
- 58365: 861, // IndexNameList (6x)
- 58371: 862, // IndexType (6x)
- 58402: 863, // LoadDataStmt (6x)
- 58482: 864, // PartitionNameListOpt (6x)
- 58515: 865, // ReleaseSavepointStmt (6x)
- 58537: 866, // RolenameList (6x)
- 58539: 867, // RollbackStmt (6x)
- 58544: 868, // SavepointStmt (6x)
- 58575: 869, // SetStmt (6x)
- 57524: 870, // show (6x)
- 58636: 871, // TableOptimizerHints (6x)
- 58676: 872, // UsernameList (6x)
- 58714: 873, // WithClustered (6x)
- 58121: 874, // AlgorithmClause (5x)
- 58176: 875, // ByItem (5x)
- 58188: 876, // CollationName (5x)
- 58192: 877, // ColumnKeywordOpt (5x)
- 58252: 878, // DirectPlacementOption (5x)
- 58299: 879, // FieldOpt (5x)
- 58300: 880, // FieldOpts (5x)
- 58343: 881, // IdentList (5x)
- 58363: 882, // IndexName (5x)
- 58366: 883, // IndexOption (5x)
- 58367: 884, // IndexOptionList (5x)
- 57439: 885, // infile (5x)
- 58394: 886, // LimitOption (5x)
- 58406: 887, // LockClause (5x)
- 58444: 888, // OptCharsetWithOptBinary (5x)
- 58455: 889, // OptNullTreatment (5x)
- 58496: 890, // PolicyName (5x)
- 58503: 891, // PriorityOpt (5x)
- 58545: 892, // SelectLockOpt (5x)
- 58552: 893, // SelectStmtIntoOption (5x)
- 58642: 894, // TableRefs (5x)
- 58669: 895, // UserSpec (5x)
- 58147: 896, // Assignment (4x)
- 58153: 897, // AuthString (4x)
- 58155: 898, // BRIEBooleanOptionName (4x)
- 58156: 899, // BRIEIntegerOptionName (4x)
- 58157: 900, // BRIEKeywordOptionName (4x)
- 58158: 901, // BRIEOption (4x)
- 58159: 902, // BRIEOptions (4x)
- 58161: 903, // BRIEStringOptionName (4x)
- 58175: 904, // BuiltinFunction (4x)
- 58177: 905, // ByList (4x)
- 58181: 906, // Char (4x)
- 58213: 907, // ConfigItemName (4x)
- 58217: 908, // Constraint (4x)
- 58311: 909, // FloatOpt (4x)
- 58372: 910, // IndexTypeName (4x)
- 57491: 911, // option (4x)
- 58460: 912, // OptWild (4x)
- 57495: 913, // outer (4x)
- 58497: 914, // Precision (4x)
- 58511: 915, // ReferDef (4x)
- 58526: 916, // RestrictOrCascadeOpt (4x)
- 58542: 917, // RowStmt (4x)
- 58560: 918, // SequenceOption (4x)
- 57533: 919, // statsExtended (4x)
- 58623: 920, // TableAsName (4x)
- 58624: 921, // TableAsNameOpt (4x)
- 58635: 922, // TableNameOptWild (4x)
- 58637: 923, // TableOptimizerHintsOpt (4x)
- 58639: 924, // TableOptionList (4x)
- 58657: 925, // TraceableStmt (4x)
- 58658: 926, // TransactionChar (4x)
- 58670: 927, // UserSpecList (4x)
- 58708: 928, // WindowName (4x)
- 58144: 929, // AsOfClause (3x)
- 58148: 930, // AssignmentList (3x)
- 58150: 931, // AttributesOpt (3x)
- 58172: 932, // Boolean (3x)
- 58201: 933, // ColumnOption (3x)
- 58204: 934, // ColumnPosition (3x)
- 58210: 935, // CommonTableExpr (3x)
- 58231: 936, // CreateTableStmt (3x)
- 58239: 937, // DatabaseOptionList (3x)
- 58247: 938, // DefaultTrueDistinctOpt (3x)
- 58273: 939, // EnforcedOrNot (3x)
- 57415: 940, // explain (3x)
- 58290: 941, // ExtendedPriv (3x)
- 58331: 942, // GeneratedAlways (3x)
- 58333: 943, // GlobalScope (3x)
- 58337: 944, // GroupByClause (3x)
- 58355: 945, // IndexHint (3x)
- 58359: 946, // IndexHintType (3x)
- 58364: 947, // IndexNameAndTypeOpt (3x)
- 57456: 948, // keys (3x)
- 58396: 949, // Lines (3x)
- 58416: 950, // MaxValueOrExpression (3x)
- 58426: 951, // NowSym (3x)
- 58427: 952, // NowSymFunc (3x)
- 58428: 953, // NowSymOptionFraction (3x)
- 58456: 954, // OptOrder (3x)
- 58459: 955, // OptTemporary (3x)
- 58473: 956, // PartDefOptionList (3x)
- 58475: 957, // PartitionDefinition (3x)
- 58485: 958, // PasswordExpire (3x)
- 58487: 959, // PasswordOrLockOption (3x)
- 58495: 960, // PluginNameList (3x)
- 58501: 961, // PrimaryOpt (3x)
- 58504: 962, // PrivElem (3x)
- 58506: 963, // PrivType (3x)
- 57501: 964, // procedure (3x)
- 58521: 965, // RequireClause (3x)
- 58522: 966, // RequireClauseOpt (3x)
- 58524: 967, // RequireListElement (3x)
- 58538: 968, // RolenameWithoutIdent (3x)
- 58531: 969, // RoleOrPrivElem (3x)
- 58551: 970, // SelectStmtGroup (3x)
- 58569: 971, // SetOprOpt (3x)
- 58622: 972, // TableAliasRefList (3x)
- 58625: 973, // TableElement (3x)
- 58634: 974, // TableNameListOpt2 (3x)
- 58650: 975, // TextString (3x)
- 58659: 976, // TransactionChars (3x)
- 57546: 977, // trigger (3x)
- 57550: 978, // unlock (3x)
- 57553: 979, // usage (3x)
- 58680: 980, // ValuesList (3x)
- 58682: 981, // ValuesStmtList (3x)
- 58678: 982, // ValueSym (3x)
- 58685: 983, // VariableAssignment (3x)
- 58705: 984, // WindowFrameStart (3x)
- 58119: 985, // AdminStmt (2x)
- 58122: 986, // AllColumnsOrPredicateColumnsOpt (2x)
- 58124: 987, // AlterDatabaseStmt (2x)
- 58125: 988, // AlterImportStmt (2x)
- 58126: 989, // AlterInstanceStmt (2x)
- 58127: 990, // AlterOrderItem (2x)
- 58129: 991, // AlterPolicyStmt (2x)
- 58130: 992, // AlterSequenceOption (2x)
- 58132: 993, // AlterSequenceStmt (2x)
- 58134: 994, // AlterTableSpec (2x)
- 58138: 995, // AlterUserStmt (2x)
- 58139: 996, // AnalyzeOption (2x)
- 58167: 997, // BinlogStmt (2x)
- 58160: 998, // BRIEStmt (2x)
- 58162: 999, // BRIETables (2x)
- 57373: 1000, // call (2x)
- 58178: 1001, // CallStmt (2x)
- 58179: 1002, // CastType (2x)
- 58180: 1003, // ChangeStmt (2x)
- 58186: 1004, // CheckConstraintKeyword (2x)
- 58196: 1005, // ColumnNameListOpt (2x)
- 58199: 1006, // ColumnNameOrUserVariable (2x)
- 58202: 1007, // ColumnOptionList (2x)
- 58203: 1008, // ColumnOptionListOpt (2x)
- 58205: 1009, // ColumnSetValue (2x)
- 58208: 1010, // CommentOrAttributeOption (2x)
- 58212: 1011, // CompletionTypeWithinTransaction (2x)
- 58214: 1012, // ConnectionOption (2x)
- 58216: 1013, // ConnectionOptions (2x)
- 58220: 1014, // CreateBindingStmt (2x)
- 58221: 1015, // CreateDatabaseStmt (2x)
- 58222: 1016, // CreateImportStmt (2x)
- 58223: 1017, // CreateIndexStmt (2x)
- 58224: 1018, // CreatePolicyStmt (2x)
- 58225: 1019, // CreateRoleStmt (2x)
- 58227: 1020, // CreateSequenceStmt (2x)
- 58228: 1021, // CreateStatisticsStmt (2x)
- 58229: 1022, // CreateTableOptionListOpt (2x)
- 58232: 1023, // CreateUserStmt (2x)
- 58234: 1024, // CreateViewStmt (2x)
- 57393: 1025, // databases (2x)
- 58243: 1026, // DeallocateStmt (2x)
- 58244: 1027, // DeallocateSym (2x)
- 57404: 1028, // describe (2x)
- 58255: 1029, // DoStmt (2x)
- 58256: 1030, // DropBindingStmt (2x)
- 58257: 1031, // DropDatabaseStmt (2x)
- 58258: 1032, // DropImportStmt (2x)
- 58259: 1033, // DropIndexStmt (2x)
- 58260: 1034, // DropPolicyStmt (2x)
- 58261: 1035, // DropRoleStmt (2x)
- 58262: 1036, // DropSequenceStmt (2x)
- 58263: 1037, // DropStatisticsStmt (2x)
- 58264: 1038, // DropStatsStmt (2x)
- 58265: 1039, // DropTableStmt (2x)
- 58266: 1040, // DropUserStmt (2x)
- 58267: 1041, // DropViewStmt (2x)
- 58269: 1042, // DuplicateOpt (2x)
- 58271: 1043, // EmptyStmt (2x)
- 58272: 1044, // EncryptionOpt (2x)
- 58274: 1045, // EnforcedOrNotOpt (2x)
- 58278: 1046, // ErrorHandling (2x)
- 58280: 1047, // ExecuteStmt (2x)
- 58281: 1048, // ExplainFormatType (2x)
- 58282: 1049, // ExplainStmt (2x)
- 58283: 1050, // ExplainSym (2x)
- 58292: 1051, // Field (2x)
- 58295: 1052, // FieldItem (2x)
- 58302: 1053, // Fields (2x)
- 58307: 1054, // FlashbackDatabaseStmt (2x)
- 58308: 1055, // FlashbackTableStmt (2x)
- 58309: 1056, // FlashbackToNewName (2x)
- 58310: 1057, // FlashbackToTimestampStmt (2x)
- 58314: 1058, // FlushStmt (2x)
- 58320: 1059, // FuncDatetimePrecList (2x)
- 58321: 1060, // FuncDatetimePrecListOpt (2x)
- 58334: 1061, // GrantProxyStmt (2x)
- 58335: 1062, // GrantRoleStmt (2x)
- 58336: 1063, // GrantStmt (2x)
- 58338: 1064, // HandleRange (2x)
- 58340: 1065, // HashString (2x)
- 58341: 1066, // HavingClause (2x)
- 58342: 1067, // HelpStmt (2x)
- 58354: 1068, // IndexAdviseStmt (2x)
- 58356: 1069, // IndexHintList (2x)
- 58357: 1070, // IndexHintListOpt (2x)
- 58362: 1071, // IndexLockAndAlgorithmOpt (2x)
- 58375: 1072, // InsertValues (2x)
- 58380: 1073, // IntoOpt (2x)
- 58386: 1074, // KeyOrIndexOpt (2x)
- 57457: 1075, // kill (2x)
- 58387: 1076, // KillOrKillTiDB (2x)
- 58388: 1077, // KillStmt (2x)
- 58393: 1078, // LimitClause (2x)
- 57466: 1079, // linear (2x)
- 58395: 1080, // LinearOpt (2x)
- 58399: 1081, // LoadDataSetItem (2x)
- 58403: 1082, // LoadStatsStmt (2x)
- 58404: 1083, // LocalOpt (2x)
- 58405: 1084, // LocationLabelList (2x)
- 58407: 1085, // LockStatsStmt (2x)
- 58408: 1086, // LockTablesStmt (2x)
- 58417: 1087, // MaxValueOrExpressionList (2x)
- 58423: 1088, // NonTransactionalDMLStmt (2x)
- 58429: 1089, // NowSymOptionFractionParentheses (2x)
- 58431: 1090, // NumList (2x)
- 58434: 1091, // ObjectType (2x)
- 57488: 1092, // of (2x)
- 58435: 1093, // OfTablesOpt (2x)
- 58436: 1094, // OnCommitOpt (2x)
- 58437: 1095, // OnDelete (2x)
- 58440: 1096, // OnUpdate (2x)
- 58445: 1097, // OptCollate (2x)
- 58450: 1098, // OptFull (2x)
- 58452: 1099, // OptInteger (2x)
- 58465: 1100, // OptionalBraces (2x)
- 58464: 1101, // OptionLevel (2x)
- 58454: 1102, // OptLeadLagInfo (2x)
- 58453: 1103, // OptLLDefault (2x)
- 58471: 1104, // OuterOpt (2x)
- 58476: 1105, // PartitionDefinitionList (2x)
- 58477: 1106, // PartitionDefinitionListOpt (2x)
- 58478: 1107, // PartitionIntervalOpt (2x)
- 58484: 1108, // PartitionOpt (2x)
- 58486: 1109, // PasswordOpt (2x)
- 58488: 1110, // PasswordOrLockOptionList (2x)
- 58489: 1111, // PasswordOrLockOptions (2x)
- 58492: 1112, // PlacementOptionList (2x)
- 58494: 1113, // PlanReplayerStmt (2x)
- 58500: 1114, // PreparedStmt (2x)
- 58505: 1115, // PrivLevel (2x)
- 58508: 1116, // PurgeImportStmt (2x)
- 58509: 1117, // QuickOptional (2x)
- 58510: 1118, // RecoverTableStmt (2x)
- 58512: 1119, // ReferOpt (2x)
- 58514: 1120, // RegexpSym (2x)
- 58516: 1121, // RenameTableStmt (2x)
- 58517: 1122, // RenameUserStmt (2x)
- 58519: 1123, // RepeatableOpt (2x)
- 58525: 1124, // RestartStmt (2x)
- 58527: 1125, // ResumeImportStmt (2x)
- 57515: 1126, // revoke (2x)
- 58528: 1127, // RevokeRoleStmt (2x)
- 58529: 1128, // RevokeStmt (2x)
- 58532: 1129, // RoleOrPrivElemList (2x)
- 58533: 1130, // RoleSpec (2x)
- 58555: 1131, // SelectStmtOpt (2x)
- 58558: 1132, // SelectStmtSQLCache (2x)
- 58562: 1133, // SetBindingStmt (2x)
- 58563: 1134, // SetDefaultRoleOpt (2x)
- 58564: 1135, // SetDefaultRoleStmt (2x)
- 58574: 1136, // SetRoleStmt (2x)
- 58578: 1137, // ShowImportStmt (2x)
- 58583: 1138, // ShowProfileType (2x)
- 58586: 1139, // ShowStmt (2x)
- 58587: 1140, // ShowTableAliasOpt (2x)
- 58589: 1141, // ShutdownStmt (2x)
- 58590: 1142, // SignedLiteral (2x)
- 58594: 1143, // SplitOption (2x)
- 58595: 1144, // SplitRegionStmt (2x)
- 58599: 1145, // Statement (2x)
- 58602: 1146, // StatsOptionsOpt (2x)
- 58603: 1147, // StatsPersistentVal (2x)
- 58604: 1148, // StatsType (2x)
- 58605: 1149, // StopImportStmt (2x)
- 58612: 1150, // SubPartDefinition (2x)
- 58615: 1151, // SubPartitionMethod (2x)
- 58620: 1152, // Symbol (2x)
- 58626: 1153, // TableElementList (2x)
- 58629: 1154, // TableLock (2x)
- 58633: 1155, // TableNameListOpt (2x)
- 58640: 1156, // TableOrTables (2x)
- 58649: 1157, // TablesTerminalSym (2x)
- 58647: 1158, // TableToTable (2x)
- 58651: 1159, // TextStringList (2x)
- 58656: 1160, // TraceStmt (2x)
- 58661: 1161, // TruncateTableStmt (2x)
- 58664: 1162, // UnlockStatsStmt (2x)
- 58665: 1163, // UnlockTablesStmt (2x)
- 58671: 1164, // UserToUser (2x)
- 58668: 1165, // UseStmt (2x)
- 58683: 1166, // Varchar (2x)
- 58686: 1167, // VariableAssignmentList (2x)
- 58695: 1168, // WhenClause (2x)
- 58700: 1169, // WindowDefinition (2x)
- 58703: 1170, // WindowFrameBound (2x)
- 58710: 1171, // WindowSpec (2x)
- 58715: 1172, // WithGrantOptionOpt (2x)
- 58716: 1173, // WithList (2x)
- 58720: 1174, // Writeable (2x)
- 58118: 1175, // AdminShowSlow (1x)
- 58120: 1176, // AdminStmtLimitOpt (1x)
- 58128: 1177, // AlterOrderList (1x)
- 58131: 1178, // AlterSequenceOptionList (1x)
- 58133: 1179, // AlterTablePartitionOpt (1x)
- 58135: 1180, // AlterTableSpecList (1x)
- 58136: 1181, // AlterTableSpecListOpt (1x)
- 58140: 1182, // AnalyzeOptionList (1x)
- 58143: 1183, // AnyOrAll (1x)
- 58145: 1184, // AsOfClauseOpt (1x)
- 58146: 1185, // AsOpt (1x)
- 58151: 1186, // AuthOption (1x)
- 58152: 1187, // AuthPlugin (1x)
- 58154: 1188, // AutoRandomOpt (1x)
- 58164: 1189, // BetweenOrNotOp (1x)
- 58166: 1190, // BindingStatusType (1x)
- 58169: 1191, // BitValueType (1x)
- 58170: 1192, // BlobType (1x)
- 58173: 1193, // BooleanType (1x)
- 57371: 1194, // both (1x)
- 58184: 1195, // CharsetNameOrDefault (1x)
- 58185: 1196, // CharsetOpt (1x)
- 58187: 1197, // ClearPasswordExpireOptions (1x)
- 58191: 1198, // ColumnFormat (1x)
- 58193: 1199, // ColumnList (1x)
- 58200: 1200, // ColumnNameOrUserVariableList (1x)
- 58197: 1201, // ColumnNameOrUserVarListOpt (1x)
- 58198: 1202, // ColumnNameOrUserVarListOptWithBrackets (1x)
- 58206: 1203, // ColumnSetValueList (1x)
- 58211: 1204, // CompareOp (1x)
- 58215: 1205, // ConnectionOptionList (1x)
- 58218: 1206, // ConstraintElem (1x)
- 58226: 1207, // CreateSequenceOptionListOpt (1x)
- 58230: 1208, // CreateTableSelectOpt (1x)
- 58233: 1209, // CreateViewSelectOpt (1x)
- 58240: 1210, // DatabaseOptionListOpt (1x)
- 58242: 1211, // DateAndTimeType (1x)
- 58237: 1212, // DBNameList (1x)
- 58248: 1213, // DefaultValueExpr (1x)
- 58268: 1214, // DryRunOptions (1x)
- 57410: 1215, // dual (1x)
- 58270: 1216, // ElseOpt (1x)
- 58275: 1217, // EnforcedOrNotOrNotNullOpt (1x)
- 58289: 1218, // ExpressionOpt (1x)
- 58291: 1219, // FetchFirstOpt (1x)
- 58293: 1220, // FieldAsName (1x)
- 58294: 1221, // FieldAsNameOpt (1x)
- 58296: 1222, // FieldItemList (1x)
- 58298: 1223, // FieldList (1x)
- 58304: 1224, // FirstAndLastPartOpt (1x)
- 58305: 1225, // FirstOrNext (1x)
- 58306: 1226, // FixedPointType (1x)
- 58312: 1227, // FloatingPointType (1x)
- 58313: 1228, // FlushOption (1x)
- 58316: 1229, // FromDual (1x)
- 58318: 1230, // FulltextSearchModifierOpt (1x)
- 58319: 1231, // FuncDatetimePrec (1x)
- 58332: 1232, // GetFormatSelector (1x)
- 58339: 1233, // HandleRangeList (1x)
- 58344: 1234, // IdentListWithParenOpt (1x)
- 58348: 1235, // IfNotRunning (1x)
- 58349: 1236, // IfRunning (1x)
- 58350: 1237, // IgnoreLines (1x)
- 58352: 1238, // ImportTruncate (1x)
- 58358: 1239, // IndexHintScope (1x)
- 58361: 1240, // IndexKeyTypeOpt (1x)
- 58370: 1241, // IndexPartSpecificationListOpt (1x)
- 58373: 1242, // IndexTypeOpt (1x)
- 58353: 1243, // InOrNotOp (1x)
- 58376: 1244, // InstanceOption (1x)
- 58378: 1245, // IntegerType (1x)
- 58379: 1246, // IntervalExpr (1x)
- 58382: 1247, // IsolationLevel (1x)
- 58381: 1248, // IsOrNotOp (1x)
- 57461: 1249, // leading (1x)
- 58390: 1250, // LikeEscapeOpt (1x)
- 58391: 1251, // LikeOrNotOp (1x)
- 58392: 1252, // LikeTableWithOrWithoutParen (1x)
- 58397: 1253, // LinesTerminated (1x)
- 58400: 1254, // LoadDataSetList (1x)
- 58401: 1255, // LoadDataSetSpecOpt (1x)
- 58409: 1256, // LockType (1x)
- 58410: 1257, // LogTypeOpt (1x)
- 58411: 1258, // Match (1x)
- 58412: 1259, // MatchOpt (1x)
- 58413: 1260, // MaxIndexNumOpt (1x)
- 58414: 1261, // MaxMinutesOpt (1x)
- 58415: 1262, // MaxValPartOpt (1x)
- 58418: 1263, // NChar (1x)
- 58430: 1264, // NullPartOpt (1x)
- 58433: 1265, // NumericType (1x)
- 58420: 1266, // NVarchar (1x)
- 58438: 1267, // OnDeleteUpdateOpt (1x)
- 58439: 1268, // OnDuplicateKeyUpdate (1x)
- 58441: 1269, // OptBinMod (1x)
- 58443: 1270, // OptCharset (1x)
- 58446: 1271, // OptErrors (1x)
- 58447: 1272, // OptExistingWindowName (1x)
- 58449: 1273, // OptFromFirstLast (1x)
- 58451: 1274, // OptGConcatSeparator (1x)
- 58466: 1275, // OptionalShardColumn (1x)
- 58457: 1276, // OptPartitionClause (1x)
- 58458: 1277, // OptTable (1x)
- 58461: 1278, // OptWindowFrameClause (1x)
- 58462: 1279, // OptWindowOrderByClause (1x)
- 58468: 1280, // Order (1x)
- 58467: 1281, // OrReplace (1x)
- 57445: 1282, // outfile (1x)
- 58474: 1283, // PartDefValuesOpt (1x)
- 58479: 1284, // PartitionKeyAlgorithmOpt (1x)
- 58480: 1285, // PartitionMethod (1x)
- 58483: 1286, // PartitionNumOpt (1x)
- 58490: 1287, // PerDB (1x)
- 58491: 1288, // PerTable (1x)
- 57499: 1289, // precisionType (1x)
- 58499: 1290, // PrepareSQL (1x)
- 58507: 1291, // ProcedureCall (1x)
- 57506: 1292, // recursive (1x)
- 58513: 1293, // RegexpOrNotOp (1x)
- 58518: 1294, // ReorganizePartitionRuleOpt (1x)
- 58523: 1295, // RequireList (1x)
- 58534: 1296, // RoleSpecList (1x)
- 58541: 1297, // RowOrRows (1x)
- 58548: 1298, // SelectStmtFieldList (1x)
- 58556: 1299, // SelectStmtOpts (1x)
- 58557: 1300, // SelectStmtOptsList (1x)
- 58561: 1301, // SequenceOptionList (1x)
- 58566: 1302, // SetOpr (1x)
- 58573: 1303, // SetRoleOpt (1x)
- 58576: 1304, // ShardableStmt (1x)
- 58579: 1305, // ShowIndexKwd (1x)
- 58580: 1306, // ShowLikeOrWhereOpt (1x)
- 58581: 1307, // ShowPlacementTarget (1x)
- 58582: 1308, // ShowProfileArgsOpt (1x)
- 58584: 1309, // ShowProfileTypes (1x)
- 58585: 1310, // ShowProfileTypesOpt (1x)
- 58588: 1311, // ShowTargetFilterable (1x)
- 57526: 1312, // spatial (1x)
- 58596: 1313, // SplitSyntaxOption (1x)
- 57531: 1314, // ssl (1x)
- 58597: 1315, // Start (1x)
- 58598: 1316, // Starting (1x)
- 57532: 1317, // starting (1x)
- 58600: 1318, // StatementList (1x)
- 58601: 1319, // StatementScope (1x)
- 58606: 1320, // StorageMedia (1x)
- 57538: 1321, // stored (1x)
- 58607: 1322, // StringList (1x)
- 58610: 1323, // StringNameOrBRIEOptionKeyword (1x)
- 58611: 1324, // StringType (1x)
- 58613: 1325, // SubPartDefinitionList (1x)
- 58614: 1326, // SubPartDefinitionListOpt (1x)
- 58616: 1327, // SubPartitionNumOpt (1x)
- 58617: 1328, // SubPartitionOpt (1x)
- 58627: 1329, // TableElementListOpt (1x)
- 58630: 1330, // TableLockList (1x)
- 58643: 1331, // TableRefsClause (1x)
- 58644: 1332, // TableSampleMethodOpt (1x)
- 58645: 1333, // TableSampleOpt (1x)
- 58646: 1334, // TableSampleUnitOpt (1x)
- 58648: 1335, // TableToTableList (1x)
- 58652: 1336, // TextType (1x)
- 57545: 1337, // trailing (1x)
- 58660: 1338, // TrimDirection (1x)
- 58662: 1339, // Type (1x)
- 58672: 1340, // UserToUserList (1x)
- 58674: 1341, // UserVariableList (1x)
- 58677: 1342, // UsingRoles (1x)
- 58679: 1343, // Values (1x)
- 58681: 1344, // ValuesOpt (1x)
- 58688: 1345, // ViewAlgorithm (1x)
- 58689: 1346, // ViewCheckOption (1x)
- 58690: 1347, // ViewDefiner (1x)
- 58691: 1348, // ViewFieldList (1x)
- 58692: 1349, // ViewName (1x)
- 58693: 1350, // ViewSQLSecurity (1x)
- 57565: 1351, // virtual (1x)
- 58694: 1352, // VirtualOrStored (1x)
- 58696: 1353, // WhenClauseList (1x)
- 58699: 1354, // WindowClauseOptional (1x)
- 58701: 1355, // WindowDefinitionList (1x)
- 58702: 1356, // WindowFrameBetween (1x)
- 58704: 1357, // WindowFrameExtent (1x)
- 58706: 1358, // WindowFrameUnits (1x)
- 58709: 1359, // WindowNameOrSpec (1x)
- 58711: 1360, // WindowSpecDetails (1x)
- 58717: 1361, // WithReadLockOpt (1x)
- 58718: 1362, // WithValidation (1x)
- 58719: 1363, // WithValidationOpt (1x)
- 58721: 1364, // Year (1x)
- 58117: 1365, // $default (0x)
- 58078: 1366, // andnot (0x)
- 58149: 1367, // AssignmentListOpt (0x)
- 58190: 1368, // ColumnDefList (0x)
- 58207: 1369, // CommaOpt (0x)
- 58101: 1370, // createTableSelect (0x)
- 58092: 1371, // empty (0x)
- 57345: 1372, // error (0x)
- 58116: 1373, // higherThanComma (0x)
- 58110: 1374, // higherThanParenthese (0x)
- 58099: 1375, // insertValues (0x)
- 57353: 1376, // invalid (0x)
- 58102: 1377, // lowerThanCharsetKwd (0x)
- 58115: 1378, // lowerThanComma (0x)
- 58100: 1379, // lowerThanCreateTableSelect (0x)
- 58112: 1380, // lowerThanEq (0x)
- 58107: 1381, // lowerThanFunction (0x)
- 58098: 1382, // lowerThanInsertValues (0x)
- 58103: 1383, // lowerThanKey (0x)
- 58104: 1384, // lowerThanLocal (0x)
- 58114: 1385, // lowerThanNot (0x)
- 58111: 1386, // lowerThanOn (0x)
- 58109: 1387, // lowerThanParenthese (0x)
- 58105: 1388, // lowerThanRemove (0x)
- 58093: 1389, // lowerThanSelectOpt (0x)
- 58097: 1390, // lowerThanSelectStmt (0x)
- 58096: 1391, // lowerThanSetKeyword (0x)
- 58095: 1392, // lowerThanStringLitToken (0x)
- 58094: 1393, // lowerThanValueKeyword (0x)
- 58106: 1394, // lowerThenOrder (0x)
- 58113: 1395, // neg (0x)
- 57357: 1396, // odbcDateType (0x)
- 57359: 1397, // odbcTimestampType (0x)
- 57358: 1398, // odbcTimeType (0x)
- 58108: 1399, // tableRefPriority (0x)
+ 57344: 0, // $end (2317x)
+ 59: 1, // ';' (2316x)
+ 58059: 2, // split (1911x)
+ 57748: 3, // merge (1910x)
+ 57813: 4, // remove (1910x)
+ 57814: 5, // reorganize (1909x)
+ 57631: 6, // comment (1904x)
+ 57878: 7, // storage (1817x)
+ 57594: 8, // autoIncrement (1806x)
+ 44: 9, // ',' (1718x)
+ 57692: 10, // first (1705x)
+ 57580: 11, // after (1699x)
+ 57845: 12, // serial (1695x)
+ 57595: 13, // autoRandom (1694x)
+ 57628: 14, // columnFormat (1694x)
+ 57786: 15, // password (1669x)
+ 57619: 16, // charsetKwd (1661x)
+ 57621: 17, // checksum (1649x)
+ 57970: 18, // placement (1647x)
+ 57724: 19, // keyBlockSize (1631x)
+ 57890: 20, // tablespace (1628x)
+ 57672: 21, // encryption (1626x)
+ 57675: 22, // engine (1623x)
+ 57654: 23, // data (1621x)
+ 57715: 24, // insertMethod (1619x)
+ 57742: 25, // maxRows (1619x)
+ 57750: 26, // minRows (1619x)
+ 57765: 27, // nodegroup (1619x)
+ 57638: 28, // connection (1611x)
+ 57596: 29, // autoRandomBase (1608x)
+ 58049: 30, // statsBuckets (1606x)
+ 58051: 31, // statsTopN (1606x)
+ 57905: 32, // ttl (1606x)
+ 57593: 33, // autoIdCache (1605x)
+ 57598: 34, // avgRowLength (1605x)
+ 57636: 35, // compression (1605x)
+ 57660: 36, // delayKeyWrite (1605x)
+ 57780: 37, // packKeys (1605x)
+ 57793: 38, // preSplitRegions (1605x)
+ 57833: 39, // rowFormat (1605x)
+ 57838: 40, // secondaryEngine (1605x)
+ 57849: 41, // shardRowIDBits (1605x)
+ 57874: 42, // statsAutoRecalc (1605x)
+ 57591: 43, // statsColChoice (1605x)
+ 57592: 44, // statsColList (1605x)
+ 57875: 45, // statsPersistent (1605x)
+ 57876: 46, // statsSamplePages (1605x)
+ 57590: 47, // statsSampleRate (1605x)
+ 57888: 48, // tableChecksum (1605x)
+ 57906: 49, // ttlEnable (1605x)
+ 57907: 50, // ttlJobInterval (1605x)
+ 57821: 51, // resource (1564x)
+ 57587: 52, // attribute (1556x)
+ 57577: 53, // account (1554x)
+ 57926: 54, // failedLoginAttempts (1554x)
+ 57927: 55, // passwordLockTime (1554x)
+ 57826: 56, // resume (1538x)
+ 57853: 57, // signed (1538x)
+ 41: 58, // ')' (1537x)
+ 57859: 59, // snapshot (1537x)
+ 57599: 60, // backend (1536x)
+ 57620: 61, // checkpoint (1536x)
+ 57637: 62, // concurrency (1536x)
+ 57643: 63, // csvBackslashEscape (1536x)
+ 57644: 64, // csvDelimiter (1536x)
+ 57645: 65, // csvHeader (1536x)
+ 57646: 66, // csvNotNull (1536x)
+ 57647: 67, // csvNull (1536x)
+ 57648: 68, // csvSeparator (1536x)
+ 57649: 69, // csvTrimLastSeparators (1536x)
+ 57728: 70, // lastBackup (1536x)
+ 57775: 71, // onDuplicate (1536x)
+ 57776: 72, // online (1536x)
+ 57808: 73, // rateLimit (1536x)
+ 57842: 74, // sendCredentialsToTiKV (1536x)
+ 57856: 75, // skipSchemaFiles (1536x)
+ 57879: 76, // strictFormat (1536x)
+ 57895: 77, // tikvImporter (1536x)
+ 57904: 78, // truncate (1533x)
+ 57762: 79, // no (1532x)
+ 57873: 80, // start (1530x)
+ 57614: 81, // cache (1527x)
+ 57763: 82, // nocache (1526x)
+ 57653: 83, // cycle (1525x)
+ 57752: 84, // minValue (1525x)
+ 57712: 85, // increment (1524x)
+ 57764: 86, // nocycle (1524x)
+ 57766: 87, // nomaxvalue (1524x)
+ 57767: 88, // nominvalue (1524x)
+ 57823: 89, // restart (1522x)
+ 57583: 90, // algorithm (1521x)
+ 58062: 91, // regions (1521x)
+ 57899: 92, // tp (1521x)
+ 57652: 93, // clustered (1520x)
+ 57717: 94, // invisible (1520x)
+ 57768: 95, // nonclustered (1520x)
+ 57918: 96, // visible (1520x)
+ 57881: 97, // subpartition (1517x)
+ 57785: 98, // partitions (1516x)
+ 57939: 99, // constraints (1514x)
+ 57951: 100, // followerConstraints (1514x)
+ 57952: 101, // followers (1514x)
+ 57962: 102, // leaderConstraints (1514x)
+ 57964: 103, // learnerConstraints (1514x)
+ 57965: 104, // learners (1514x)
+ 57975: 105, // primaryRegion (1514x)
+ 57980: 106, // schedule (1514x)
+ 57992: 107, // survivalPreferences (1514x)
+ 58015: 108, // voterConstraints (1514x)
+ 58016: 109, // voters (1514x)
+ 57629: 110, // columns (1512x)
+ 57917: 111, // view (1512x)
+ 57657: 112, // day (1510x)
+ 57924: 113, // yearType (1510x)
+ 57837: 114, // second (1508x)
+ 57872: 115, // sqlTsiYear (1508x)
+ 57586: 116, // ascii (1507x)
+ 57613: 117, // byteType (1507x)
+ 57707: 118, // hour (1507x)
+ 57749: 119, // microsecond (1507x)
+ 57751: 120, // minute (1507x)
+ 57755: 121, // month (1507x)
+ 57804: 122, // quarter (1507x)
+ 57865: 123, // sqlTsiDay (1507x)
+ 57866: 124, // sqlTsiHour (1507x)
+ 57867: 125, // sqlTsiMinute (1507x)
+ 57868: 126, // sqlTsiMonth (1507x)
+ 57869: 127, // sqlTsiQuarter (1507x)
+ 57870: 128, // sqlTsiSecond (1507x)
+ 57871: 129, // sqlTsiWeek (1507x)
+ 57911: 130, // unicodeSym (1507x)
+ 57920: 131, // week (1507x)
+ 57690: 132, // fields (1506x)
+ 57889: 133, // tables (1505x)
+ 57936: 134, // burstable (1504x)
+ 58017: 135, // ruRate (1504x)
+ 57877: 136, // status (1504x)
+ 57843: 137, // separator (1503x)
+ 57622: 138, // cipher (1502x)
+ 57346: 139, // identifier (1502x)
+ 57722: 140, // issuer (1502x)
+ 57740: 141, // maxConnectionsPerHour (1502x)
+ 57741: 142, // maxQueriesPerHour (1502x)
+ 57743: 143, // maxUpdatesPerHour (1502x)
+ 57744: 144, // maxUserConnections (1502x)
+ 57794: 145, // preceding (1502x)
+ 57835: 146, // san (1502x)
+ 57880: 147, // subject (1502x)
+ 57898: 148, // tokenIssuer (1502x)
+ 57710: 149, // importKwd (1501x)
+ 57733: 150, // local (1501x)
+ 57806: 151, // query (1500x)
+ 57855: 152, // skip (1500x)
+ 57606: 153, // bindings (1499x)
+ 57659: 154, // definer (1499x)
+ 57702: 155, // hash (1499x)
+ 57708: 156, // identified (1499x)
+ 57736: 157, // logs (1499x)
+ 57822: 158, // respect (1499x)
+ 57632: 159, // commit (1498x)
+ 57650: 160, // current (1498x)
+ 57674: 161, // enforced (1498x)
+ 57695: 162, // following (1498x)
+ 57730: 163, // less (1498x)
+ 57770: 164, // nowait (1498x)
+ 57777: 165, // only (1498x)
+ 57830: 166, // rollback (1498x)
+ 57836: 167, // savepoint (1498x)
+ 57894: 168, // than (1498x)
+ 57908: 169, // unbounded (1498x)
+ 57915: 170, // value (1498x)
+ 57602: 171, // begin (1497x)
+ 57604: 172, // binding (1497x)
+ 57673: 173, // end (1497x)
+ 57700: 174, // global (1497x)
+ 57955: 175, // next_row_id (1497x)
+ 57774: 176, // offset (1497x)
+ 57792: 177, // policy (1497x)
+ 57974: 178, // predicate (1497x)
+ 57891: 179, // temporary (1497x)
+ 58057: 180, // tiFlash (1497x)
+ 57913: 181, // user (1497x)
+ 57723: 182, // jsonType (1496x)
+ 57972: 183, // planCache (1496x)
+ 57795: 184, // prepare (1496x)
+ 57829: 185, // role (1496x)
+ 57912: 186, // unknown (1496x)
+ 57925: 187, // wait (1496x)
+ 57612: 188, // btree (1495x)
+ 57655: 189, // datetimeType (1495x)
+ 57656: 190, // dateType (1495x)
+ 57693: 191, // fixed (1495x)
+ 57709: 192, // identSQLErrors (1495x)
+ 57721: 193, // isolation (1495x)
+ 57727: 194, // last (1495x)
+ 57735: 195, // location (1495x)
+ 57738: 196, // max_idxnum (1495x)
+ 57747: 197, // memory (1495x)
+ 57773: 198, // off (1495x)
+ 57779: 199, // optional (1495x)
+ 57788: 200, // per_db (1495x)
+ 57971: 201, // plan (1495x)
+ 57797: 202, // privileges (1495x)
+ 57817: 203, // replica (1495x)
+ 57820: 204, // required (1495x)
+ 57834: 205, // rtree (1495x)
+ 57978: 206, // running (1495x)
+ 58043: 207, // sampleRate (1495x)
+ 57844: 208, // sequence (1495x)
+ 57847: 209, // session (1495x)
+ 57858: 210, // slow (1495x)
+ 58046: 211, // stats (1495x)
+ 57897: 212, // timeType (1495x)
+ 57914: 213, // validation (1495x)
+ 57916: 214, // variables (1495x)
+ 57588: 215, // attributes (1494x)
+ 57634: 216, // compact (1494x)
+ 57661: 217, // digest (1494x)
+ 57663: 218, // disable (1494x)
+ 57668: 219, // duplicate (1494x)
+ 57669: 220, // dynamic (1494x)
+ 57670: 221, // enable (1494x)
+ 57678: 222, // errorKwd (1494x)
+ 57694: 223, // flush (1494x)
+ 57697: 224, // full (1494x)
+ 57705: 225, // history (1494x)
+ 57745: 226, // mb (1494x)
+ 57753: 227, // mode (1494x)
+ 57791: 228, // plugins (1494x)
+ 57799: 229, // processlist (1494x)
+ 57810: 230, // recover (1494x)
+ 57815: 231, // repair (1494x)
+ 57816: 232, // repeatable (1494x)
+ 58045: 233, // statistics (1494x)
+ 57882: 234, // subpartitions (1494x)
+ 58056: 235, // tidb (1494x)
+ 57896: 236, // timestampType (1494x)
+ 57922: 237, // without (1494x)
+ 58020: 238, // admin (1493x)
+ 57600: 239, // backup (1493x)
+ 58021: 240, // batch (1493x)
+ 57607: 241, // binlog (1493x)
+ 57609: 242, // block (1493x)
+ 57610: 243, // booleanType (1493x)
+ 57935: 244, // briefType (1493x)
+ 58022: 245, // buckets (1493x)
+ 57615: 246, // capture (1493x)
+ 58025: 247, // cardinality (1493x)
+ 57618: 248, // chain (1493x)
+ 57625: 249, // clientErrorsSummary (1493x)
+ 58026: 250, // cmSketch (1493x)
+ 57626: 251, // coalesce (1493x)
+ 57635: 252, // compressed (1493x)
+ 57641: 253, // context (1493x)
+ 57938: 254, // copyKwd (1493x)
+ 58028: 255, // correlation (1493x)
+ 57642: 256, // cpu (1493x)
+ 57658: 257, // deallocate (1493x)
+ 58030: 258, // dependency (1493x)
+ 57662: 259, // directory (1493x)
+ 57665: 260, // discard (1493x)
+ 57666: 261, // disk (1493x)
+ 57667: 262, // do (1493x)
+ 57944: 263, // dotType (1493x)
+ 58032: 264, // drainer (1493x)
+ 58033: 265, // dry (1493x)
+ 57683: 266, // exchange (1493x)
+ 57685: 267, // execute (1493x)
+ 57686: 268, // expansion (1493x)
+ 57949: 269, // flashback (1493x)
+ 57696: 270, // format (1493x)
+ 57699: 271, // general (1493x)
+ 57703: 272, // help (1493x)
+ 57704: 273, // histogram (1493x)
+ 57706: 274, // hosts (1493x)
+ 57956: 275, // inplace (1493x)
+ 57716: 276, // instance (1493x)
+ 57957: 277, // instant (1493x)
+ 57720: 278, // ipc (1493x)
+ 58035: 279, // job (1493x)
+ 58034: 280, // jobs (1493x)
+ 57725: 281, // labels (1493x)
+ 57734: 282, // locked (1493x)
+ 57754: 283, // modify (1493x)
+ 57760: 284, // next (1493x)
+ 58036: 285, // nodeID (1493x)
+ 58037: 286, // nodeState (1493x)
+ 57772: 287, // nulls (1493x)
+ 57781: 288, // pageSym (1493x)
+ 58040: 289, // pump (1493x)
+ 57803: 290, // purge (1493x)
+ 57809: 291, // rebuild (1493x)
+ 57811: 292, // redundant (1493x)
+ 57812: 293, // reload (1493x)
+ 57824: 294, // restore (1493x)
+ 57831: 295, // routine (1493x)
+ 57979: 296, // s3 (1493x)
+ 58042: 297, // samples (1493x)
+ 57839: 298, // secondaryLoad (1493x)
+ 57840: 299, // secondaryUnload (1493x)
+ 57850: 300, // share (1493x)
+ 57852: 301, // shutdown (1493x)
+ 57861: 302, // source (1493x)
+ 57589: 303, // statsOptions (1493x)
+ 57986: 304, // stop (1493x)
+ 57884: 305, // swaps (1493x)
+ 57994: 306, // tidbJson (1493x)
+ 57998: 307, // tokudbDefault (1493x)
+ 57999: 308, // tokudbFast (1493x)
+ 58000: 309, // tokudbLzma (1493x)
+ 58001: 310, // tokudbQuickLZ (1493x)
+ 58003: 311, // tokudbSmall (1493x)
+ 58002: 312, // tokudbSnappy (1493x)
+ 58004: 313, // tokudbUncompressed (1493x)
+ 58005: 314, // tokudbZlib (1493x)
+ 58006: 315, // tokudbZstd (1493x)
+ 58058: 316, // topn (1493x)
+ 57900: 317, // trace (1493x)
+ 57901: 318, // traditional (1493x)
+ 58013: 319, // trueCardCost (1493x)
+ 58012: 320, // verboseType (1493x)
+ 57919: 321, // warnings (1493x)
+ 57578: 322, // action (1492x)
+ 57579: 323, // advise (1492x)
+ 57581: 324, // against (1492x)
+ 57582: 325, // ago (1492x)
+ 57584: 326, // always (1492x)
+ 57601: 327, // backups (1492x)
+ 57603: 328, // bernoulli (1492x)
+ 57605: 329, // bindingCache (1492x)
+ 57608: 330, // bitType (1492x)
+ 57611: 331, // boolType (1492x)
+ 58023: 332, // builtins (1492x)
+ 58024: 333, // cancel (1492x)
+ 57616: 334, // cascaded (1492x)
+ 57617: 335, // causal (1492x)
+ 57623: 336, // cleanup (1492x)
+ 57624: 337, // client (1492x)
+ 57651: 338, // cluster (1492x)
+ 57627: 339, // collation (1492x)
+ 58027: 340, // columnStatsUsage (1492x)
+ 57633: 341, // committed (1492x)
+ 57630: 342, // config (1492x)
+ 57639: 343, // consistency (1492x)
+ 57640: 344, // consistent (1492x)
+ 58029: 345, // ddl (1492x)
+ 58031: 346, // depth (1492x)
+ 57664: 347, // disabled (1492x)
+ 57945: 348, // dump (1492x)
+ 57671: 349, // enabled (1492x)
+ 57676: 350, // engines (1492x)
+ 57677: 351, // enum (1492x)
+ 57681: 352, // events (1492x)
+ 57682: 353, // evolve (1492x)
+ 57687: 354, // expire (1492x)
+ 57947: 355, // exprPushdownBlacklist (1492x)
+ 57688: 356, // extended (1492x)
+ 57689: 357, // faultsSym (1492x)
+ 57698: 358, // function (1492x)
+ 57701: 359, // grants (1492x)
+ 58053: 360, // histogramsInFlight (1492x)
+ 57711: 361, // imports (1492x)
+ 57713: 362, // incremental (1492x)
+ 57714: 363, // indexes (1492x)
+ 57958: 364, // internal (1492x)
+ 57718: 365, // invoker (1492x)
+ 57719: 366, // io (1492x)
+ 57726: 367, // language (1492x)
+ 57731: 368, // level (1492x)
+ 57732: 369, // list (1492x)
+ 57737: 370, // master (1492x)
+ 57739: 371, // max_minutes (1492x)
+ 57757: 372, // national (1492x)
+ 57758: 373, // ncharType (1492x)
+ 57759: 374, // never (1492x)
+ 57761: 375, // nextval (1492x)
+ 57769: 376, // none (1492x)
+ 57771: 377, // nvarcharType (1492x)
+ 57778: 378, // open (1492x)
+ 58038: 379, // optimistic (1492x)
+ 57969: 380, // optRuleBlacklist (1492x)
+ 57782: 381, // parser (1492x)
+ 57783: 382, // partial (1492x)
+ 57784: 383, // partitioning (1492x)
+ 57789: 384, // per_table (1492x)
+ 57787: 385, // percent (1492x)
+ 58039: 386, // pessimistic (1492x)
+ 57796: 387, // preserve (1492x)
+ 57800: 388, // profile (1492x)
+ 57801: 389, // profiles (1492x)
+ 57805: 390, // queries (1492x)
+ 57976: 391, // recent (1492x)
+ 58063: 392, // region (1492x)
+ 57977: 393, // replayer (1492x)
+ 58061: 394, // reset (1492x)
+ 57825: 395, // restores (1492x)
+ 57827: 396, // reuse (1492x)
+ 58041: 397, // run (1492x)
+ 57841: 398, // security (1492x)
+ 57846: 399, // serializable (1492x)
+ 58044: 400, // sessionStates (1492x)
+ 57854: 401, // simple (1492x)
+ 57857: 402, // slave (1492x)
+ 58050: 403, // statsHealthy (1492x)
+ 58048: 404, // statsHistograms (1492x)
+ 58052: 405, // statsLocked (1492x)
+ 58047: 406, // statsMeta (1492x)
+ 57987: 407, // strict (1492x)
+ 57885: 408, // switchesSym (1492x)
+ 57886: 409, // system (1492x)
+ 57887: 410, // systemTime (1492x)
+ 57993: 411, // target (1492x)
+ 58055: 412, // telemetryID (1492x)
+ 57892: 413, // temptable (1492x)
+ 57893: 414, // textType (1492x)
+ 57997: 415, // tls (1492x)
+ 58007: 416, // top (1492x)
+ 57902: 417, // transaction (1492x)
+ 57903: 418, // triggers (1492x)
+ 57909: 419, // uncommitted (1492x)
+ 57910: 420, // undefined (1492x)
+ 58060: 421, // width (1492x)
+ 57923: 422, // x509 (1492x)
+ 57928: 423, // addDate (1491x)
+ 57585: 424, // any (1491x)
+ 57929: 425, // approxCountDistinct (1491x)
+ 57930: 426, // approxPercentile (1491x)
+ 57597: 427, // avg (1491x)
+ 57931: 428, // bitAnd (1491x)
+ 57932: 429, // bitOr (1491x)
+ 57933: 430, // bitXor (1491x)
+ 57934: 431, // bound (1491x)
+ 57937: 432, // cast (1491x)
+ 57941: 433, // curDate (1491x)
+ 57940: 434, // curTime (1491x)
+ 57942: 435, // dateAdd (1491x)
+ 57943: 436, // dateSub (1491x)
+ 57679: 437, // escape (1491x)
+ 57680: 438, // event (1491x)
+ 57946: 439, // exact (1491x)
+ 57684: 440, // exclusive (1491x)
+ 57948: 441, // extract (1491x)
+ 57691: 442, // file (1491x)
+ 57950: 443, // follower (1491x)
+ 57953: 444, // getFormat (1491x)
+ 57954: 445, // groupConcat (1491x)
+ 58018: 446, // ioReadBandwidth (1491x)
+ 58019: 447, // ioWriteBandwidth (1491x)
+ 57959: 448, // jsonArrayagg (1491x)
+ 57960: 449, // jsonObjectAgg (1491x)
+ 57729: 450, // lastval (1491x)
+ 57961: 451, // leader (1491x)
+ 57963: 452, // learner (1491x)
+ 57967: 453, // max (1491x)
+ 57746: 454, // member (1491x)
+ 57966: 455, // min (1491x)
+ 57756: 456, // names (1491x)
+ 57968: 457, // now (1491x)
+ 57973: 458, // position (1491x)
+ 57798: 459, // process (1491x)
+ 57802: 460, // proxy (1491x)
+ 57807: 461, // quick (1491x)
+ 57818: 462, // replicas (1491x)
+ 57819: 463, // replication (1491x)
+ 57828: 464, // reverse (1491x)
+ 57832: 465, // rowCount (1491x)
+ 57848: 466, // setval (1491x)
+ 57851: 467, // shared (1491x)
+ 57860: 468, // some (1491x)
+ 57862: 469, // sqlBufferResult (1491x)
+ 57863: 470, // sqlCache (1491x)
+ 57864: 471, // sqlNoCache (1491x)
+ 57981: 472, // staleness (1491x)
+ 57982: 473, // std (1491x)
+ 57983: 474, // stddev (1491x)
+ 57984: 475, // stddevPop (1491x)
+ 57985: 476, // stddevSamp (1491x)
+ 57988: 477, // strong (1491x)
+ 57989: 478, // subDate (1491x)
+ 57991: 479, // substring (1491x)
+ 57990: 480, // sum (1491x)
+ 57883: 481, // super (1491x)
+ 58054: 482, // telemetry (1491x)
+ 57995: 483, // timestampAdd (1491x)
+ 57996: 484, // timestampDiff (1491x)
+ 58008: 485, // trim (1491x)
+ 58009: 486, // variance (1491x)
+ 58010: 487, // varPop (1491x)
+ 58011: 488, // varSamp (1491x)
+ 58014: 489, // voter (1491x)
+ 57921: 490, // weightString (1491x)
+ 57491: 491, // on (1420x)
+ 40: 492, // '(' (1356x)
+ 57572: 493, // with (1236x)
+ 57351: 494, // stringLit (1235x)
+ 58109: 495, // not2 (1214x)
+ 57401: 496, // defaultKwd (1154x)
+ 57484: 497, // not (1150x)
+ 57367: 498, // as (1131x)
+ 57382: 499, // collate (1097x)
+ 57551: 500, // union (1081x)
+ 57557: 501, // using (1072x)
+ 57464: 502, // left (1067x)
+ 57518: 503, // right (1067x)
+ 43: 504, // '+' (1044x)
+ 45: 505, // '-' (1042x)
+ 57483: 506, // mod (1022x)
+ 57499: 507, // partition (1015x)
+ 57438: 508, // ignore (980x)
+ 57418: 509, // except (970x)
+ 57488: 510, // null (970x)
+ 57444: 511, // intersect (969x)
+ 57466: 512, // limit (950x)
+ 57423: 513, // forKwd (948x)
+ 57380: 514, // charType (946x)
+ 57561: 515, // values (946x)
+ 57446: 516, // into (940x)
+ 57472: 517, // lock (936x)
+ 58098: 518, // eq (933x)
+ 57569: 519, // where (931x)
+ 57426: 520, // from (929x)
+ 57420: 521, // fetch (926x)
+ 57514: 522, // replace (923x)
+ 57496: 523, // order (922x)
+ 57424: 524, // force (921x)
+ 58093: 525, // intLit (911x)
+ 57525: 526, // set (909x)
+ 57365: 527, // and (908x)
+ 57495: 528, // or (884x)
+ 57356: 529, // andand (883x)
+ 57790: 530, // pipesAsOr (883x)
+ 57573: 531, // xor (883x)
+ 57430: 532, // group (862x)
+ 57432: 533, // having (857x)
+ 57536: 534, // straightJoin (851x)
+ 57571: 535, // window (843x)
+ 57456: 536, // join (839x)
+ 57465: 537, // like (831x)
+ 57576: 538, // natural (829x)
+ 57387: 539, // cross (828x)
+ 57442: 540, // inner (828x)
+ 42: 541, // '*' (827x)
+ 125: 542, // '}' (825x)
+ 57521: 543, // rows (813x)
+ 57556: 544, // use (809x)
+ 57539: 545, // tableSample (803x)
+ 57504: 546, // rangeKwd (802x)
+ 57431: 547, // groups (801x)
+ 57405: 548, // desc (800x)
+ 57371: 549, // binaryType (799x)
+ 57396: 550, // dayHour (799x)
+ 57397: 551, // dayMicrosecond (799x)
+ 57398: 552, // dayMinute (799x)
+ 57399: 553, // daySecond (799x)
+ 57434: 554, // hourMicrosecond (799x)
+ 57435: 555, // hourMinute (799x)
+ 57436: 556, // hourSecond (799x)
+ 57481: 557, // minuteMicrosecond (799x)
+ 57482: 558, // minuteSecond (799x)
+ 57523: 559, // secondMicrosecond (799x)
+ 57574: 560, // yearMonth (799x)
+ 57368: 561, // asc (798x)
+ 57568: 562, // when (795x)
+ 57413: 563, // elseKwd (792x)
+ 57439: 564, // in (792x)
+ 57542: 565, // then (789x)
+ 47: 566, // '/' (785x)
+ 37: 567, // '%' (784x)
+ 38: 568, // '&' (784x)
+ 94: 569, // '^' (784x)
+ 124: 570, // '|' (784x)
+ 57409: 571, // div (784x)
+ 58103: 572, // lsh (784x)
+ 58108: 573, // rsh (784x)
+ 60: 574, // '<