From c3a1204887595eb6bce3b8dcc6032e6043032d72 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Wed, 10 Aug 2022 19:10:48 +0800 Subject: [PATCH] pdutil(ticdc): split `tidb_ddl_job` table (#6673) close pingcap/tiflow#4756 --- cdc/sink/mq/codec/craft/codec_test.go | 15 ++++++-- pkg/pdutil/pd_api_client.go | 25 +++++++++++- pkg/pdutil/pd_api_client_test.go | 55 +++++++++++++++++++++++++++ pkg/regionspan/span.go | 5 +-- 4 files changed, 91 insertions(+), 9 deletions(-) diff --git a/cdc/sink/mq/codec/craft/codec_test.go b/cdc/sink/mq/codec/craft/codec_test.go index 3d87b47723d..bb0aa73a342 100644 --- a/cdc/sink/mq/codec/craft/codec_test.go +++ b/cdc/sink/mq/codec/craft/codec_test.go @@ -18,16 +18,21 @@ import ( "testing" "time" + "github.com/pingcap/tiflow/pkg/leakutil" "github.com/stretchr/testify/require" - "go.uber.org/goleak" ) func init() { rand.Seed(time.Now().UnixNano()) } +func TestMain(m *testing.M) { + leakutil.SetUpLeakTest(m) +} + func TestSizeTable(t *testing.T) { - defer goleak.VerifyNone(t) + t.Parallel() + tables := [][]int64{ { 1, 3, 5, 7, 9, @@ -47,7 +52,8 @@ func TestSizeTable(t *testing.T) { } func TestUvarintReverse(t *testing.T) { - defer goleak.VerifyNone(t) + t.Parallel() + var i uint64 = 0 for i < 0x8000000000000000 { @@ -72,7 +78,8 @@ func newNullableString(a string) *string { } func TestEncodeChunk(t *testing.T) { - defer goleak.VerifyNone(t) + t.Parallel() + stringChunk := []string{"a", "b", "c"} nullableStringChunk := []*string{newNullableString("a"), newNullableString("b"), newNullableString("c")} int64Chunk := []int64{1, 2, 3} diff --git a/pkg/pdutil/pd_api_client.go b/pkg/pdutil/pd_api_client.go index c1248b3b855..ac8e82c8730 100644 --- a/pkg/pdutil/pd_api_client.go +++ b/pkg/pdutil/pd_api_client.go @@ -33,8 +33,13 @@ const ( regionLabelPrefix = "/pd/api/v1/config/region-label/rules" gcServiceSafePointURL = "/pd/api/v1/gc/safepoint" - // Split the default rule by `6e000000000000000000f8` to keep metadata region - // isolated from the normal data area. + // Split the default rule by following keys to keep metadata region isolated + // from the normal data area. + // + // * `6e000000000000000000f8`, keys starts with "m". + // * `748000fffffffffffffe00000000000000f8`, the table prefix of + // `tidb_ddl_job` table, which has the table ID 281474976710654, + // see "github.com/pingcap/tidb/ddl.JobTableID" addMetaJSON = `{ "sets": [ { @@ -52,6 +57,22 @@ const ( "end_key": "6e00000000000000f8" } ] + }, + { + "id": "ticdc/meta_tidb_ddl_job", + "labels": [ + { + "key": "data-type", + "value": "meta" + } + ], + "rule_type": "key-range", + "data": [ + { + "start_key": "748000fffffffffffffe00000000000000f8", + "end_key": "748000ffffffffffffff00000000000000f8" + } + ] } ] }` diff --git a/pkg/pdutil/pd_api_client_test.go b/pkg/pdutil/pd_api_client_test.go index bf3506c5aff..1b53f6a2ae9 100644 --- a/pkg/pdutil/pd_api_client_test.go +++ b/pkg/pdutil/pd_api_client_test.go @@ -15,11 +15,16 @@ package pdutil import ( "context" + "encoding/hex" + "encoding/json" "net/http" "net/http/httptest" "testing" + "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/util/codec" cerror "github.com/pingcap/tiflow/pkg/errors" + "github.com/pingcap/tiflow/pkg/regionspan" "github.com/stretchr/testify/require" pd "github.com/tikv/pd/client" ) @@ -95,3 +100,53 @@ func TestListGcServiceSafePoint(t *testing.T) { require.Nil(t, err) mockClient.testServer.Close() } + +// LabelRulePatch is the patch to update the label rules. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +// Copied from github.com/tikv/pd/server/schedule/labeler +type LabelRulePatch struct { + SetRules []*LabelRule `json:"sets"` + DeleteRules []string `json:"deletes"` +} + +// LabelRule is the rule to assign labels to a region. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +// Copied from github.com/tikv/pd/server/schedule/labeler +type LabelRule struct { + ID string `json:"id"` + Index int `json:"index"` + Labels []RegionLabel `json:"labels"` + RuleType string `json:"rule_type"` + Data interface{} `json:"data"` +} + +// RegionLabel is the label of a region. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +// Copied from github.com/tikv/pd/server/schedule/labeler +type RegionLabel struct { + Key string `json:"key"` + Value string `json:"value"` +} + +func TestMetaLabelDecodeJSON(t *testing.T) { + t.Parallel() + + meta := LabelRulePatch{} + require.Nil(t, json.Unmarshal([]byte(addMetaJSON), &meta)) + require.Len(t, meta.SetRules, 2) + keys := meta.SetRules[1].Data.([]interface{})[0].(map[string]interface{}) + startKey, err := hex.DecodeString(keys["start_key"].(string)) + require.Nil(t, err) + endKey, err := hex.DecodeString(keys["end_key"].(string)) + require.Nil(t, err) + + _, startKey, err = codec.DecodeBytes(startKey, nil) + require.Nil(t, err) + require.EqualValues( + t, regionspan.JobTableID, tablecodec.DecodeTableID(startKey), keys["start_key"].(string)) + + _, endKey, err = codec.DecodeBytes(endKey, nil) + require.Nil(t, err) + require.EqualValues( + t, regionspan.JobTableID+1, tablecodec.DecodeTableID(endKey), keys["end_key"].(string)) +} diff --git a/pkg/regionspan/span.go b/pkg/regionspan/span.go index 30765b6408a..bd7b79fcdc2 100644 --- a/pkg/regionspan/span.go +++ b/pkg/regionspan/span.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/pingcap/log" + "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/codec" @@ -27,10 +28,8 @@ import ( ) const ( - // MaxInt48 is the max value of int48. - MaxInt48 = 0x0000FFFFFFFFFFFF // JobTableID is the id of `tidb_ddl_job`. - JobTableID = MaxInt48 - 1 + JobTableID = ddl.JobTableID ) // UpperBoundKey represents the maximum value.