diff --git a/br/pkg/storage/gcs.go b/br/pkg/storage/gcs.go index 07ce5c8a862b9..e4835e0eb6111 100644 --- a/br/pkg/storage/gcs.go +++ b/br/pkg/storage/gcs.go @@ -180,11 +180,6 @@ func (s *gcsStorage) WalkDir(ctx context.Context, opt *WalkOption, fn func(strin opt = &WalkOption{} } - maxKeys := int64(1000) - if opt.ListCount > 0 { - maxKeys = opt.ListCount - } - prefix := path.Join(s.gcs.Prefix, opt.SubDir) if len(prefix) > 0 && !strings.HasSuffix(prefix, "/") { prefix += "/" @@ -194,7 +189,7 @@ func (s *gcsStorage) WalkDir(ctx context.Context, opt *WalkOption, fn func(strin // only need each object's name and size query.SetAttrSelection([]string{"Name", "Size"}) iter := s.bucket.Objects(ctx, query) - for i := int64(0); i != maxKeys; i++ { + for { attrs, err := iter.Next() if err == iterator.Done { break diff --git a/br/pkg/storage/gcs_test.go b/br/pkg/storage/gcs_test.go index c3e63d6d410a2..ccf3927497bea 100644 --- a/br/pkg/storage/gcs_test.go +++ b/br/pkg/storage/gcs_test.go @@ -4,6 +4,7 @@ package storage import ( "context" + "fmt" "io" "os" @@ -95,6 +96,31 @@ func (r *testStorageSuite) TestGCS(c *C) { c.Assert(list, Equals, "keykey1key2") c.Assert(totalSize, Equals, int64(42)) + // test 1003 files + totalSize = 0 + for i := 0; i < 1000; i += 1 { + err = stg.WriteFile(ctx, fmt.Sprintf("f%d", i), []byte("data")) + c.Assert(err, IsNil) + } + filesSet := make(map[string]struct{}, 1003) + err = stg.WalkDir(ctx, nil, func(name string, size int64) error { + filesSet[name] = struct{}{} + totalSize += size + return nil + }) + c.Assert(err, IsNil) + c.Assert(totalSize, Equals, int64(42+4000)) + _, ok := filesSet["key"] + c.Assert(ok, IsTrue) + _, ok = filesSet["key1"] + c.Assert(ok, IsTrue) + _, ok = filesSet["key2"] + c.Assert(ok, IsTrue) + for i := 0; i < 1000; i += 1 { + _, ok = filesSet[fmt.Sprintf("f%d", i)] + c.Assert(ok, IsTrue) + } + efr, err := stg.Open(ctx, "key2") c.Assert(err, IsNil)